diff --git a/.vscode/terminals.json b/.vscode/terminals.json index 6f82f2758c..8d8f1c391d 100644 --- a/.vscode/terminals.json +++ b/.vscode/terminals.json @@ -51,6 +51,13 @@ "onlySingle": true, "cwd": "[workspaceFolder]/packages/desktop-gui", "command": "npm run cypress:open" + }, + { + "name": "packages/desktop-gui watch", + "focus": true, + "onlySingle": true, + "cwd": "[workspaceFolder]/packages/desktop-gui", + "command": "npm run watch" } ] } diff --git a/circle.yml b/circle.yml index baaa3c8842..cdea4541ad 100644 --- a/circle.yml +++ b/circle.yml @@ -220,6 +220,7 @@ jobs: - run: npm run all test -- --package extension - run: npm run all test -- --package https-proxy - run: npm run all test -- --package launcher + - run: npm run all test -- --package network # how to pass Mocha reporter through zunder? - run: npm run all test -- --package reporter - run: npm run all test -- --package runner @@ -264,6 +265,15 @@ jobs: - store_test_results: path: /tmp/cypress + "server-performance-tests": + <<: *defaults + steps: + - attach_workspace: + at: ~/ + - run: npm run all test-performance -- --package server + - store_test_results: + path: /tmp/cypress + "server-e2e-tests-1": <<: *defaults steps: @@ -709,6 +719,9 @@ linux-workflow: &linux-workflow - server-integration-tests: requires: - build + - server-performance-tests: + requires: + - build - server-e2e-tests-1: requires: - build diff --git a/cli/lib/exec/spawn.js b/cli/lib/exec/spawn.js index b818407e68..d0142989ca 100644 --- a/cli/lib/exec/spawn.js +++ b/cli/lib/exec/spawn.js @@ -75,6 +75,12 @@ module.exports = { const overrides = util.getEnvOverrides() const node11WindowsFix = isPlatform('win32') + const proxySource = util.loadSystemProxySettings() + + if (proxySource) { + args.push(`--proxy-source="${proxySource}"`) + } + debug('spawning Cypress with executable: %s', executable) debug('spawn forcing env overrides %o', overrides) debug('spawn args %o %o', args, _.omit(options, 'env')) diff --git a/cli/lib/tasks/download.js b/cli/lib/tasks/download.js index 1664d0c703..a7773cebfe 100644 --- a/cli/lib/tasks/download.js +++ b/cli/lib/tasks/download.js @@ -150,6 +150,8 @@ const start = ({ version, downloadDestination, progress }) => { } } } + util.loadSystemProxySettings() + const url = getUrl(version) progress.throttle = 100 diff --git a/cli/lib/util.js b/cli/lib/util.js index 488e7b1204..ecbfa7e678 100644 --- a/cli/lib/util.js +++ b/cli/lib/util.js @@ -9,6 +9,7 @@ const getos = require('getos') const chalk = require('chalk') const Promise = require('bluebird') const cachedir = require('cachedir') +const getWindowsProxy = require('@cypress/get-windows-proxy') const executable = require('executable') const supportsColor = require('supports-color') const isInstalledGlobally = require('is-installed-globally') @@ -88,6 +89,30 @@ const util = { } }, + _getWindowsProxy () { + return getWindowsProxy() + }, + + loadSystemProxySettings () { + // load user's OS-specific proxy settings in to environment vars + if (!_.isUndefined(process.env.HTTP_PROXY)) { + // user has set proxy explicitly in environment vars, don't mess with it + return + } + + if (os.platform() === 'win32') { + const proxy = this._getWindowsProxy() + + if (proxy) { + // environment variables are the only way to make request lib use NO_PROXY + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = proxy.httpProxy + process.env.NO_PROXY = process.env.NO_PROXY || proxy.noProxy + } + + return 'win32' + } + }, + isTty (fd) { return tty.isatty(fd) }, diff --git a/cli/package.json b/cli/package.json index 5e58762b00..9f62378191 100644 --- a/cli/package.json +++ b/cli/package.json @@ -39,6 +39,7 @@ }, "types": "types", "dependencies": { + "@cypress/get-windows-proxy": "1.5.1", "@cypress/listr-verbose-renderer": "0.4.1", "@cypress/xvfb": "1.2.4", "bluebird": "3.5.0", diff --git a/cli/test/lib/exec/spawn_spec.js b/cli/test/lib/exec/spawn_spec.js index c0cb8d6e1d..90d3556a4d 100644 --- a/cli/test/lib/exec/spawn_spec.js +++ b/cli/test/lib/exec/spawn_spec.js @@ -133,6 +133,41 @@ describe('lib/exec/spawn', function () { }) }) + context('proxy', function () { + beforeEach(function () { + this.oldEnv = Object.assign({}, process.env) + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = process.env.NO_PROXY = undefined + }) + + it('loads proxy settings from Windows registry', function () { + this.spawnedProcess.on.withArgs('close').yieldsAsync(0) + os.platform.returns('win32') + sinon.stub(util, '_getWindowsProxy').returns({ + httpProxy: 'http://foo-bar.baz', + noProxy: 'a,b,c', + }) + + return spawn.start([], {}) + .then(() => { + return expect(cp.spawn).to.be.calledWithMatch('/path/to/cypress', [ + '--cwd', + cwd, + '--proxy-source="win32"', + ], { + env: { + 'HTTP_PROXY': 'http://foo-bar.baz', + 'HTTPS_PROXY': 'http://foo-bar.baz', + 'NO_PROXY': 'a,b,c', + }, + }) + }) + }) + + afterEach(function () { + Object.assign(process.env, this.oldEnv) + }) + }) + it('unrefs if options.detached is true', function () { this.spawnedProcess.on.withArgs('close').yieldsAsync(0) diff --git a/cli/test/lib/tasks/download_spec.js b/cli/test/lib/tasks/download_spec.js index 4c41257452..a21fbd554d 100644 --- a/cli/test/lib/tasks/download_spec.js +++ b/cli/test/lib/tasks/download_spec.js @@ -203,4 +203,15 @@ describe('lib/tasks/download', function () { return snapshot('download status errors 1', normalize(ctx.stdout.toString())) }) }) + + it('calls loadSystemProxySettings before downloading', function () { + sinon.stub(fs, 'ensureDirAsync').rejects({ type: 'FAKE_ERR' }) + sinon.spy(util, 'loadSystemProxySettings') + + return download.start(this.options) + .catch(() => {}) + .finally(() => { + expect(util.loadSystemProxySettings).to.be.calledOnce + }) + }) }) diff --git a/package.json b/package.json index 6cb50be247..48a91bdbd2 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,17 @@ "@cypress/env-or-json-file": "2.0.0", "@cypress/npm-run-all": "4.0.5", "@cypress/questions-remain": "1.0.1", + "@types/bluebird": "3.5.21", + "@types/chai": "3.5.2", + "@types/debug": "0.0.31", + "@types/execa": "0.7.2", + "@types/fs-extra": "3.0.0", + "@types/lodash": "4.14.122", + "@types/mocha": "2.2.48", + "@types/node": "11.12.0", + "@types/ramda": "0.25.47", + "@types/request-promise": "4.1.42", + "@types/sinon-chai": "3.2.2", "ansi-styles": "3.2.1", "ascii-table": "0.0.9", "babel-eslint": "10.0.1", diff --git a/packages/desktop-gui/cypress/integration/settings_spec.coffee b/packages/desktop-gui/cypress/integration/settings_spec.coffee index b6ceb975a1..1ce22af0ad 100644 --- a/packages/desktop-gui/cypress/integration/settings_spec.coffee +++ b/packages/desktop-gui/cypress/integration/settings_spec.coffee @@ -113,7 +113,7 @@ describe "Settings", -> it "displays project id section", -> cy.contains(@config.projectId) - describe "when record key panels is opened", -> + describe "when record key panel is opened", -> beforeEach -> cy.contains("Record Key").click() @@ -180,6 +180,45 @@ describe "Settings", -> cy.get(".settings-record-key") .contains("cypress run --record --key #{@keys[0].id}") + describe "when proxy settings panel is opened", -> + beforeEach -> + cy.contains("Proxy Settings").click() + + it "with no proxy config set informs the user no proxy configuration is active", -> + cy.get(".settings-proxy").should("contain", "There is no active proxy configuration.") + + it "opens help link on click", -> + cy.get(".settings-proxy .learn-more").click().then -> + expect(@ipc.externalOpen).to.be.calledWith("https://on.cypress.io/proxy-configuration") + + it "with Windows proxy settings indicates proxy and the source", -> + cy.setAppStore({ + projectRoot: "/foo/bar", + proxySource: "win32", + proxyServer: "http://foo-bar.baz", + proxyBypassList: "a,b,c,d" + }) + cy.get(".settings-proxy").should("contain", "from Windows system settings") + cy.get(".settings-proxy tr:nth-child(1) > td > code").should("contain", "http://foo-bar.baz") + cy.get(".settings-proxy tr:nth-child(2) > td > code").should("contain", "a, b, c, d") + + it "with environment proxy settings indicates proxy and the source", -> + cy.setAppStore({ + projectRoot: "/foo/bar", + proxyServer: "http://foo-bar.baz", + proxyBypassList: "a,b,c,d" + }) + cy.get(".settings-proxy").should("contain", "from environment variables") + cy.get(".settings-proxy tr:nth-child(1) > td > code").should("contain", "http://foo-bar.baz") + cy.get(".settings-proxy tr:nth-child(2) > td > code").should("contain", "a, b, c, d") + + it "with no bypass list but a proxy set shows 'none' in bypass list", -> + cy.setAppStore({ + projectRoot: "/foo/bar", + proxyServer: "http://foo-bar.baz", + }) + cy.get(".settings-proxy tr:nth-child(2) > td").should("contain", "none") + context "on:focus:tests clicked", -> beforeEach -> @ipc.onFocusTests.yield() diff --git a/packages/desktop-gui/cypress/support/index.coffee b/packages/desktop-gui/cypress/support/index.coffee index e1766d0a64..883f8394c7 100644 --- a/packages/desktop-gui/cypress/support/index.coffee +++ b/packages/desktop-gui/cypress/support/index.coffee @@ -29,3 +29,8 @@ Cypress.Commands.add "logOut", -> Cypress.Commands.add "shouldBeLoggedOut", -> cy.contains(".main-nav a", "Log In") + +Cypress.Commands.add "setAppStore", (options = {}) -> + cy.window() + .then (win) -> + win.AppStore.set(options) diff --git a/packages/desktop-gui/src/app/app.jsx b/packages/desktop-gui/src/app/app.jsx index 1e46d6b8ee..729f1ee02b 100644 --- a/packages/desktop-gui/src/app/app.jsx +++ b/packages/desktop-gui/src/app/app.jsx @@ -20,7 +20,7 @@ class App extends Component { appApi.listenForMenuClicks() ipc.getOptions().then((options = {}) => { - appStore.set(_.pick(options, 'cypressEnv', 'os', 'projectRoot', 'version')) + appStore.set(_.pick(options, 'cypressEnv', 'os', 'projectRoot', 'version', 'proxySource', 'proxyServer', 'proxyBypassList')) viewStore.showApp() }) @@ -40,7 +40,7 @@ class App extends Component { default: return ( - + ) } diff --git a/packages/desktop-gui/src/lib/app-store.js b/packages/desktop-gui/src/lib/app-store.js index 547067b562..4487c1320c 100644 --- a/packages/desktop-gui/src/lib/app-store.js +++ b/packages/desktop-gui/src/lib/app-store.js @@ -9,6 +9,15 @@ class AppStore { @observable version @observable localInstallNoticeDismissed = localData.get('local-install-notice-dimissed') @observable error + @observable proxyServer + @observable proxyBypassList + @observable proxySource + + constructor () { + if (window.Cypress) { + window.AppStore = this // for testing + } + } @computed get displayVersion () { return this.isDev ? `${this.version} (dev)` : this.version @@ -34,6 +43,10 @@ class AppStore { if (props.projectRoot != null) this.projectRoot = props.projectRoot if (props.version != null) this.version = this.newVersion = props.version + + this.proxyServer = props.proxyServer || this.proxyServer + this.proxyBypassList = props.proxyBypassList || this.proxyBypassList + this.proxySource = props.proxySource || this.proxySource } @action setNewVersion (newVersion) { diff --git a/packages/desktop-gui/src/project/project.jsx b/packages/desktop-gui/src/project/project.jsx index ad7ab26cb1..24b2d8a42a 100644 --- a/packages/desktop-gui/src/project/project.jsx +++ b/packages/desktop-gui/src/project/project.jsx @@ -66,7 +66,7 @@ class Project extends Component { case C.PROJECT_RUNS: return case C.PROJECT_SETTINGS: - return + return default: return } diff --git a/packages/desktop-gui/src/settings/proxy-settings.jsx b/packages/desktop-gui/src/settings/proxy-settings.jsx new file mode 100644 index 0000000000..f9f5fdb3fd --- /dev/null +++ b/packages/desktop-gui/src/settings/proxy-settings.jsx @@ -0,0 +1,80 @@ +import { observer } from 'mobx-react' +import Tooltip from '@cypress/react-tooltip' +import { trim } from 'lodash' +import React from 'react' + +import ipc from '../lib/ipc' + +const trimQuotes = (input) => + trim(input, '"') + +const getProxySourceName = (proxySource) => { + if (proxySource === 'win32') { + return 'Windows system settings' + } + + return 'environment variables' +} + +const openHelp = (e) => { + e.preventDefault() + ipc.externalOpen('https://on.cypress.io/proxy-configuration') +} + +const renderLearnMore = () => { + return ( + + Learn more + + ) +} + +const ProxySettings = observer(({ app }) => { + if (!app.proxyServer) { + return ( +
+ {renderLearnMore()} +

+ There is no active proxy configuration. +

+
+ ) + } + + const proxyBypassList = trimQuotes(app.proxyBypassList) + const proxySource = getProxySourceName(trimQuotes(app.proxySource)) + + return ( +
+ {renderLearnMore()} +

Cypress auto-detected the following proxy settings from {proxySource}:

+ + + + + + + + + + + +
Proxy Server + + {trimQuotes(app.proxyServer)} + +
+ Proxy Bypass List{' '} + + + + + {proxyBypassList ? {proxyBypassList.split(',').join(', ')} : none} +
+
+ ) +}) + +export default ProxySettings diff --git a/packages/desktop-gui/src/settings/settings.jsx b/packages/desktop-gui/src/settings/settings.jsx index 38ba06137d..c793e100c5 100644 --- a/packages/desktop-gui/src/settings/settings.jsx +++ b/packages/desktop-gui/src/settings/settings.jsx @@ -6,8 +6,9 @@ import Collapse, { Panel } from 'rc-collapse' import Configuration from './configuration' import ProjectId from './project-id' import RecordKey from './record-key' +import ProxySettings from './proxy-settings' -const Settings = observer(({ project }) => ( +const Settings = observer(({ project, app }) => (
( + + +
diff --git a/packages/desktop-gui/src/settings/settings.scss b/packages/desktop-gui/src/settings/settings.scss index d350ddcf6b..3c41c96d22 100644 --- a/packages/desktop-gui/src/settings/settings.scss +++ b/packages/desktop-gui/src/settings/settings.scss @@ -207,3 +207,21 @@ color: #999; margin: 1em; } + +.proxy-settings { + .proxy-table { + width: 100%; + + th:first-child { + width: 150px; + } + + .no-bypass { + opacity: .8; + } + + td, th { + padding: 3px; + } + } +} diff --git a/packages/electron/lib/electron.coffee b/packages/electron/lib/electron.coffee index 0b93e1081b..535c4f5c61 100644 --- a/packages/electron/lib/electron.coffee +++ b/packages/electron/lib/electron.coffee @@ -59,8 +59,6 @@ module.exports = { .then -> execPath = paths.getPathToExec() - debug("spawning %s", execPath) - ## we have an active debugger session if inspector.url() dp = process.debugPort + 1 @@ -72,6 +70,8 @@ module.exports = { if opts.inspectBrk argv.unshift("--inspect-brk=5566") + debug("spawning %s with args", execPath, argv) + cp.spawn(execPath, argv, {stdio: "inherit"}) .on "close", (code) -> debug("electron closing with code", code) diff --git a/packages/https-proxy/index.js b/packages/https-proxy/index.js index 65bd49e8f7..4c553f7e2c 100644 --- a/packages/https-proxy/index.js +++ b/packages/https-proxy/index.js @@ -1,3 +1,6 @@ require('@packages/coffee/register') +require('@packages/ts/register') module.exports = require('./lib/proxy') + +module.exports.CA = require('./lib/ca') diff --git a/packages/https-proxy/lib/ca.coffee b/packages/https-proxy/lib/ca.coffee index de7720ba2b..aa4dcf8f41 100644 --- a/packages/https-proxy/lib/ca.coffee +++ b/packages/https-proxy/lib/ca.coffee @@ -1,5 +1,6 @@ _ = require("lodash") fs = require("fs-extra") +os = require("os") path = require("path") Forge = require("node-forge") Promise = require("bluebird") @@ -217,6 +218,9 @@ class CA @create = (caFolder) -> ca = new CA + if not caFolder + caFolder = path.join(os.tmpdir(), 'cy-ca') + ca.baseCAFolder = caFolder ca.certsFolder = path.join(ca.baseCAFolder, "certs") ca.keysFolder = path.join(ca.baseCAFolder, "keys") @@ -233,4 +237,4 @@ class CA .catch(ca.generateCA) .return(ca) -module.exports = CA \ No newline at end of file +module.exports = CA diff --git a/packages/https-proxy/lib/server.coffee b/packages/https-proxy/lib/server.coffee index 2cf8979f7f..e69ca5747b 100644 --- a/packages/https-proxy/lib/server.coffee +++ b/packages/https-proxy/lib/server.coffee @@ -1,26 +1,38 @@ _ = require("lodash") +agent = require("@packages/network").agent +allowDestroy = require("server-destroy-vvo") +debug = require("debug")("cypress:https-proxy") fs = require("fs-extra") -net = require("net") -url = require("url") +getProxyForUrl = require("proxy-from-env").getProxyForUrl https = require("https") +net = require("net") +parse = require("./util/parse") Promise = require("bluebird") semaphore = require("semaphore") -allowDestroy = require("server-destroy-vvo") -log = require("debug")("cypress:https-proxy") -parse = require("./util/parse") +url = require("url") fs = Promise.promisifyAll(fs) sslServers = {} sslSemaphores = {} +## https://en.wikipedia.org/wiki/Transport_Layer_Security#TLS_record +SSL_RECORD_TYPES = [ + 22 ## Handshake + 128, 0 ## TODO: what do these unknown types mean? +] + class Server constructor: (@_ca, @_port) -> @_onError = null connect: (req, socket, head, options = {}) -> + ## don't buffer writes - thanks a lot, Nagle + ## https://github.com/cypress-io/cypress/issues/3192 + socket.setNoDelay(true) + if not head or head.length is 0 - log("Writing socket connection headers for URL:", req.url) + debug("Writing socket connection headers for URL:", req.url) socket.once "data", (data) => @connect(req, socket, data, options) @@ -38,10 +50,9 @@ class Server ## if onDirectConnection return true ## then dont proxy, just pass this through if odc.call(@, req, socket, head) is true - log("Making direct connection to #{req.url}") return @_makeDirectConnection(req, socket, head) else - log("Not making direct connection to #{req.url}") + debug("Not making direct connection to #{req.url}") socket.pause() @@ -69,42 +80,84 @@ class Server res.end() .pipe(res) + _upstreamProxyForHostPort: (hostname, port) -> + getProxyForUrl("https://#{hostname}:#{port}") + _makeDirectConnection: (req, socket, head) -> { port, hostname } = url.parse("http://#{req.url}") + if upstreamProxy = @_upstreamProxyForHostPort(hostname, port) + return @_makeUpstreamProxyConnection(upstreamProxy, socket, head, port, hostname) + + debug("Making direct connection to #{hostname}:#{port}") @_makeConnection(socket, head, port, hostname) _makeConnection: (socket, head, port, hostname) -> - cb = -> + onConnect = -> socket.pipe(conn) conn.pipe(socket) - socket.emit("data", head) + conn.write(head) socket.resume() - ## compact out hostname when undefined - args = _.compact([port, hostname, cb]) - - conn = net.connect.apply(net, args) + conn = new net.Socket() + conn.setNoDelay(true) conn.on "error", (err) => if @_onError @_onError(err, socket, head, port) + ## compact out hostname when undefined + args = _.compact([port, hostname, onConnect]) + conn.connect.apply(conn, args) + + # todo: as soon as all requests are intercepted, this can go away since this is just for pass-through + _makeUpstreamProxyConnection: (upstreamProxy, socket, head, toPort, toHostname) -> + debug("making proxied connection to #{toHostname}:#{toPort} with upstream #{upstreamProxy}") + + onUpstreamSock = (err, upstreamSock) -> + if @_onError + if err + return @_onError(err, socket, head, port) + upstreamSock.on "error", (err) => + @_onError(err, socket, head, port) + + if not upstreamSock + ## couldn't establish a proxy connection, fail gracefully + socket.resume() + return socket.destroy() + + upstreamSock.setNoDelay(true) + upstreamSock.pipe(socket) + socket.pipe(upstreamSock) + upstreamSock.write(head) + + socket.resume() + + agent.httpsAgent.createProxiedConnection { + proxy: upstreamProxy + href: "https://#{toHostname}:#{toPort}" + uri: { + port: toPort + hostname: toHostname + } + }, onUpstreamSock.bind(@) + _onServerConnectData: (req, socket, head) -> firstBytes = head[0] makeConnection = (port) => - log("Making intercepted connection to %s", port) + debug("Making intercepted connection to %s", port) @_makeConnection(socket, head, port) - if firstBytes is 0x16 or firstBytes is 0x80 or firstBytes is 0x00 + if firstBytes in SSL_RECORD_TYPES {hostname} = url.parse("http://#{req.url}") if sslServer = sslServers[hostname] return makeConnection(sslServer.port) + ## only be creating one SSL server per hostname at once if not sem = sslSemaphores[hostname] sem = sslSemaphores[hostname] = semaphore(1) @@ -168,7 +221,7 @@ class Server ## store the port of our current sniServer @_sniPort = @_sniServer.address().port - log("Created SNI HTTPS Proxy on port %s", @_sniPort) + debug("Created SNI HTTPS Proxy on port %s", @_sniPort) resolve() diff --git a/packages/https-proxy/package.json b/packages/https-proxy/package.json index f97d5141da..4f21cf52ba 100644 --- a/packages/https-proxy/package.json +++ b/packages/https-proxy/package.json @@ -11,6 +11,7 @@ "clean-deps": "rm -rf node_modules", "pretest": "npm run check-deps-pre", "test": "cross-env NODE_ENV=test bin-up mocha --reporter mocha-multi-reporters --reporter-options configFile=../../mocha-reporter-config.json", + "test-debug": "cross-env NODE_ENV=test bin-up mocha --inspect-brk --reporter mocha-multi-reporters --reporter-options configFile=../../mocha-reporter-config.json", "pretest-watch": "npm run check-deps-pre", "test-watch": "cross-env NODE_ENV=test bin-up mocha --watch", "https": "node https.js" @@ -22,7 +23,7 @@ "bin-up": "1.1.0", "chai": "3.5.0", "cross-env": "5.2.0", - "http-mitm-proxy": "0.5.3", + "debugging-proxy": "1.6.0", "request": "2.88.0", "request-promise": "4.2.4", "sinon": "1.17.7", @@ -37,6 +38,7 @@ "fs-extra": "0.30.0", "lodash": "4.17.11", "node-forge": "0.6.49", + "proxy-from-env": "1.0.0", "semaphore": "1.1.0", "server-destroy-vvo": "1.0.1", "ssl-root-cas": "1.3.1" diff --git a/packages/https-proxy/test/helpers/mitm.coffee b/packages/https-proxy/test/helpers/mitm.coffee deleted file mode 100644 index f66ebcc659..0000000000 --- a/packages/https-proxy/test/helpers/mitm.coffee +++ /dev/null @@ -1,17 +0,0 @@ -Promise = require("bluebird") - -Proxy = require("http-mitm-proxy") -proxy = Proxy() - -proxy.onRequest (ctx, cb) -> - cb() - -module.exports = { - start: -> - new Promise (resolve) -> - proxy.listen({port: 8081, forceSNI: true}, resolve) - - stop: -> - proxy.close() - -} \ No newline at end of file diff --git a/packages/https-proxy/test/helpers/proxy_bak.coffee b/packages/https-proxy/test/helpers/proxy_bak.coffee deleted file mode 100644 index 6a3d387cb0..0000000000 --- a/packages/https-proxy/test/helpers/proxy_bak.coffee +++ /dev/null @@ -1,284 +0,0 @@ -fs = require("fs-extra") -net = require("net") -url = require("url") -path = require("path") -http = require("http") -https = require("https") -request = require("request") -Promise = require("bluebird") -sempahore = require("semaphore") -CA = require("../../lib/ca") - -Promise.promisifyAll(fs) - -ca = null -httpsSrv = null -httpsPort = null - -sslServers = {} -sslSemaphores = {} - -onClientError = (err) -> - console.log "CLIENT ERROR", err - -onError = (err) -> - console.log "ERROR", err - -onRequest = (req, res) -> - console.log "onRequest!!!!!!!!!", req.url, req.headers, req.method - - hostPort = parseHostAndPort(req) - - # req.pause() - - opts = { - url: req.url - baseUrl: "https://" + hostPort.host + ":" + hostPort.port - method: req.method - headers: req.headers - } - - req.pipe(request(opts)) - .on "error", -> - console.log "**ERROR", req.url - res.statusCode = 500 - res.end() - .pipe(res) - -parseHostAndPort = (req, defaultPort) -> - host = req.headers.host - - return null if not host - - hostPort = parseHost(host, defaultPort) - - ## this handles paths which include the full url. This could happen if it's a proxy - if m = req.url.match(/^http:\/\/([^\/]*)\/?(.*)$/) - parsedUrl = url.parse(req.url) - hostPort.host = parsedUrl.hostname - hostPort.port = parsedUrl.port - req.url = parsedUrl.path - - hostPort - -parseHost = (hostString, defaultPort) -> - if m = hostString.match(/^http:\/\/(.*)/) - parsedUrl = url.parse(hostString) - - return { - host: parsedUrl.hostname - port: parsedUrl.port - } - - hostPort = hostString.split(':') - host = hostPort[0] - port = if hostPort.length is 2 then +hostPort[1] else defaultPort - - return { - host: host - port: port - } - -onConnect = (req, socket, head) -> - console.log "ON CONNECT!!!!!!!!!!!!!!!" - ## tell the client that the connection is established - # socket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'UTF-8', function() { - # // creating pipes in both ends - # conn.pipe(socket); - # socket.pipe(conn); - # }); - - console.log "URL", req.url - console.log "HEADERS", req.headers - console.log "HEAD IS", head - console.log "HEAD LENGTH", head.length - - # srvUrl = url.parse("http://#{req.url}") - - # conn = null - - # cb = -> - # socket.write('HTTP/1.1 200 Connection Established\r\n' + - # 'Proxy-agent: Cypress\r\n' + - # '\r\n') - # conn.write(head) - # conn.pipe(socket) - # socket.pipe(conn) - - # conn = net.connect(srvUrl.port, srvUrl.hostname, cb) - - # conn.on "error", (err) -> - # ## TODO: attach error handling here - # console.log "*******ERROR CONNECTING", err, err.stack - - # # conn.on "close", -> - # # console.log "CONNECTION CLOSED", arguments - - # return - - # URL www.cypress.io:443 - # HEADERS { host: 'www.cypress.io:443', - # 'proxy-connection': 'keep-alive', - # 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2609.0 Safari/537.36' } - # HEAD IS - # HEAD LENGTH 0 - - getHttpsServer = (hostname) -> - onCertificateRequired(hostname) - .then (certPaths) -> - Promise.props({ - keyFileExists: fs.statAsync(certPaths.keyFile) - certFileExists: fs.statAsync(certPaths.certFile) - }) - .catch (err) -> - onCertificateMissing(certPaths) - .then (data = {}) -> - return { - key: data.keyFileData - cert: data.certFileData - hosts: data.hosts - } - .then (data = {}) -> - hosts = [hostname] - delete data.hosts - - hosts.forEach (host) -> - console.log "ADD CONTEXT", host, data - httpsSrv.addContext(host, data) - # sslServers[host] = { port: httpsPort } - - # return cb(null, self.httpsPort) - - return httpsPort - - onCertificateMissing = (certPaths) -> - hosts = certPaths.hosts #or [ctx.hostname] - - ca.generateServerCertificateKeys(hosts) - .spread (certPEM, privateKeyPEM) -> - return { - hosts: hosts - keyFileData: privateKeyPEM - certFileData: certPEM - } - - onCertificateRequired = (hostname) -> - Promise.resolve({ - keyFile: "" - certFile: "" - hosts: [hostname] - }) - - makeConnection = (port) -> - console.log "makeConnection", port - conn = net.connect port, -> - console.log "connected to", port#, socket, conn, head - socket.pipe(conn) - conn.pipe(socket) - socket.emit("data", head) - - return socket.resume() - - conn.on "error", onError - - onServerConnectData = (head) -> - firstBytes = head[0] - - if firstBytes is 0x16 or firstBytes is 0x80 or firstBytes is 0x00 - {hostname} = url.parse("http://#{req.url}") - - if sslServer = sslServers[hostname] - return makeConnection(sslServer.port) - - wildcardhost = hostname.replace(/[^\.]+\./, "*.") - - sem = sslSemaphores[wildcardhost] - - if not sem - sem = sslSemaphores[wildcardhost] = sempahore(1) - - sem.take -> - leave = -> - process.nextTick -> - console.log "leaving sem" - sem.leave() - - if sslServer = sslServers[hostname] - leave() - return makeConnection(sslServer.port) - - if sslServer = sslServers[wildcardhost] - leave() - sslServers[hostname] = { - port: sslServer - } - - return makeConnection(sslServers[hostname].port) - - getHttpsServer(hostname) - .then (port) -> - leave() - - makeConnection(port) - - else - throw new Error("@httpPort") - makeConnection(@httpPort) - - if not head or head.length is 0 - socket.once "data", onConnect.bind(@, req, socket) - - socket.write "HTTP/1.1 200 OK\r\n" - - if req.headers["proxy-connection"] is "keep-alive" - socket.write("Proxy-Connection: keep-alive\r\n") - socket.write("Connection: keep-alive\r\n") - - return socket.write("\r\n") - - else - socket.pause() - - onServerConnectData(head) - -prx = http.createServer() - -prx.on("connect", onConnect) -prx.on("request", onRequest) -prx.on("clientError", onClientError) -prx.on("error", onError) - -module.exports = { - prx: prx - - startHttpsSrv: -> - new Promise (resolve) -> - httpsSrv = https.createServer({}) - # httpsSrv.timeout = 0 - httpsSrv.on("connect", onConnect) - httpsSrv.on("request", onRequest) - httpsSrv.on("clientError", onClientError) - httpsSrv.on("error", onError) - httpsSrv.listen -> - resolve([httpsSrv.address().port, httpsSrv]) - - start: -> - dir = path.join(process.cwd(), "ca") - - CA.create(dir) - .then (c) => - ca = c - - @startHttpsSrv() - .spread (port, httpsSrv) -> - httpsPort = port - - new Promise (resolve) -> - prx.listen 3333, -> - console.log "server listening on port: 3333" - resolve(prx) - - stop: -> - new Promise (resolve) -> - prx.close(resolve) -} \ No newline at end of file diff --git a/packages/https-proxy/test/integration/proxy_spec.coffee b/packages/https-proxy/test/integration/proxy_spec.coffee index b49ef466f3..3fd6cf981c 100644 --- a/packages/https-proxy/test/integration/proxy_spec.coffee +++ b/packages/https-proxy/test/integration/proxy_spec.coffee @@ -2,10 +2,12 @@ require("../spec_helper") process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" +_ = require("lodash") +DebugProxy = require("debugging-proxy") +net = require("net") path = require("path") Promise = require("bluebird") proxy = require("../helpers/proxy") -mitmProxy = require("../helpers/mitm") httpServer = require("../helpers/http_server") httpsServer = require("../helpers/https_server") @@ -14,8 +16,6 @@ describe "Proxy", -> Promise.join( httpServer.start() - # mitmProxy.start() - httpsServer.start(8443) httpsServer.start(8444) @@ -91,6 +91,25 @@ describe "Proxy", -> .then (html) -> expect(html).to.include("https server") + it "closes outgoing connections when client disconnects", -> + @sandbox.spy(net.Socket.prototype, 'connect') + + request({ + strictSSL: false + url: "https://localhost:8444/replace" + proxy: "http://localhost:3333" + resolveWithFullResponse: true + }) + .then (res) => + ## ensure client has disconnected + expect(res.socket.destroyed).to.be.true + ## ensure the outgoing socket created for this connection was destroyed + socket = net.Socket.prototype.connect.getCalls() + .find (call) => + _.isEqual(call.args.slice(0,2), ["8444", "localhost"]) + .thisValue + expect(socket.destroyed).to.be.true + it "can boot the httpServer", -> request({ strictSSL: false @@ -139,3 +158,78 @@ describe "Proxy", -> url: "https://localhost:8443/" proxy: "http://localhost:3333" }) + + context "with an upstream proxy", -> + beforeEach -> + @oldEnv = Object.assign({}, process.env) + process.env.NO_PROXY = "" + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = "http://localhost:9001" + + @upstream = new DebugProxy({ + keepRequests: true + }) + + @upstream.start(9001) + + it "passes a request to an https server through the upstream", -> + request({ + strictSSL: false + url: "https://localhost:8444/" + proxy: "http://localhost:3333" + }).then (res) => + expect(@upstream.getRequests()[0]).to.include({ + url: 'localhost:8444' + https: true + }) + expect(res).to.contain("https server") + + it "uses HTTP basic auth when provided", -> + @upstream.setAuth({ + username: 'foo' + password: 'bar' + }) + + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = "http://foo:bar@localhost:9001" + + request({ + strictSSL: false + url: "https://localhost:8444/" + proxy: "http://localhost:3333" + }).then (res) => + expect(@upstream.getRequests()[0]).to.include({ + url: 'localhost:8444' + https: true + }) + expect(res).to.contain("https server") + + it "closes outgoing connections when client disconnects", -> + @sandbox.spy(net.Socket.prototype, 'connect') + + request({ + strictSSL: false + url: "https://localhost:8444/replace" + proxy: "http://localhost:3333" + resolveWithFullResponse: true + forever: false + }) + .then (res) => + ## ensure client has disconnected + expect(res.socket.destroyed).to.be.true + + ## ensure the outgoing socket created for this connection was destroyed + socket = net.Socket.prototype.connect.getCalls() + .find (call) => + _.isEqual(call.args[0][0], { + host: 'localhost' + port: 9001 + }) + .thisValue + + new Promise (resolve) -> + socket.on 'close', => + expect(socket.destroyed).to.be.true + resolve() + + afterEach -> + @upstream.stop() + Object.assign(process.env, @oldEnv) diff --git a/packages/https-proxy/test/mocha.opts b/packages/https-proxy/test/mocha.opts index 6cf813fc8a..c88a378ae8 100644 --- a/packages/https-proxy/test/mocha.opts +++ b/packages/https-proxy/test/mocha.opts @@ -1,5 +1,5 @@ test/unit test/integration --reporter spec ---compilers coffee:@packages/coffee/register +--compilers ts:@packages/ts/register,coffee:@packages/coffee/register --recursive diff --git a/packages/https-proxy/test/unit/server_spec.coffee b/packages/https-proxy/test/unit/server_spec.coffee index b9dd77bbc9..49b9b89680 100644 --- a/packages/https-proxy/test/unit/server_spec.coffee +++ b/packages/https-proxy/test/unit/server_spec.coffee @@ -1,5 +1,6 @@ require("../spec_helper") +EE = require("events") Promise = require("bluebird") proxy = require("../helpers/proxy") Server = require("../../lib/server") @@ -35,7 +36,7 @@ describe "lib/server", -> it "calls options.onError with err and port", (done) -> onError = @sandbox.stub() - socket = {} + socket = new EE() head = {} @setup({onError: onError}) @@ -46,7 +47,9 @@ describe "lib/server", -> err = onError.getCall(0).args[0] expect(err.message).to.eq("connect ECONNREFUSED 127.0.0.1:8444") - expect(onError).to.be.calledWithMatch(err, socket, head, 8444) + expect(onError.getCall(0).args[1]).to.eq(socket) + expect(onError.getCall(0).args[2]).to.eq(head) + expect(onError.getCall(0).args[3]).to.eq("8444") done() return diff --git a/packages/launcher/package.json b/packages/launcher/package.json index 740d1ce6b3..20e48874ec 100644 --- a/packages/launcher/package.json +++ b/packages/launcher/package.json @@ -26,15 +26,6 @@ "lib" ], "devDependencies": { - "@types/bluebird": "3.5.21", - "@types/chai": "3.5.2", - "@types/debug": "0.0.31", - "@types/execa": "0.7.2", - "@types/fs-extra": "3.0.0", - "@types/lodash": "4.14.122", - "@types/mocha": "2.2.48", - "@types/node": "7.10.3", - "@types/ramda": "0.25.47", "bin-up": "1.1.0", "chai": "3.5.0", "prettier": "1.16.4", diff --git a/packages/network/index.js b/packages/network/index.js new file mode 100644 index 0000000000..261c24c807 --- /dev/null +++ b/packages/network/index.js @@ -0,0 +1,10 @@ +// @ts-check + +if (process.env.CYPRESS_ENV !== 'production') { + require('@packages/ts/register') +} + +module.exports = { + agent: require('./lib/agent').default, + connect: require('./lib/connect'), +} diff --git a/packages/network/lib/agent.ts b/packages/network/lib/agent.ts new file mode 100644 index 0000000000..5f7ae73817 --- /dev/null +++ b/packages/network/lib/agent.ts @@ -0,0 +1,285 @@ +import * as http from 'http' +import * as https from 'https' +import * as net from 'net' +import * as tls from 'tls' +import * as url from 'url' +import * as _ from 'lodash' +import * as debugModule from 'debug' +import { getProxyForUrl } from 'proxy-from-env' +import * as Promise from 'bluebird' +import { getAddress } from './connect' + +const debug = debugModule('cypress:network:agent') +const CRLF = '\r\n' +const statusCodeRe = /^HTTP\/1.[01] (\d*)/ + +interface RequestOptionsWithProxy extends http.RequestOptions { + proxy: string +} + +export function _buildConnectReqHead(hostname: string, port: string, proxy: url.Url) { + const connectReq = [`CONNECT ${hostname}:${port} HTTP/1.1`] + + connectReq.push(`Host: ${hostname}:${port}`) + + if (proxy.auth) { + connectReq.push(`Proxy-Authorization: basic ${Buffer.from(proxy.auth).toString('base64')}`) + } + + return connectReq.join(CRLF) + _.repeat(CRLF, 2) +} + +export function _createProxySock (proxy: url.Url) { + if (proxy.protocol === 'http:') { + return net.connect(Number(proxy.port || 80), proxy.hostname) + } + + if (proxy.protocol === 'https:') { + // if the upstream is https, we need to wrap the socket with tls + return tls.connect(Number(proxy.port || 443), proxy.hostname) + } + + // socksv5, etc... + throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`) +} + +export function isRequestHttps(options: http.RequestOptions) { + // WSS connections will not have an href, but you can tell protocol from the defaultAgent + return _.get(options, '_defaultAgent.protocol') === 'https:' || (options.href || '').slice(0, 6) === 'https' +} + +export function isResponseStatusCode200(head: string) { + // read status code from proxy's response + const matches = head.match(statusCodeRe) + return _.get(matches, 1) === '200' +} + +export function _regenerateRequestHead(req: http.ClientRequest) { + delete req._header + req._implicitHeader() + if (req.output && req.output.length > 0) { + // the _header has already been queued to be written to the socket + const first = req.output[0] + const endOfHeaders = first.indexOf(_.repeat(CRLF, 2)) + 4 + req.output[0] = req._header + first.substring(endOfHeaders) + } +} + +export class CombinedAgent { + httpAgent: HttpAgent + httpsAgent: HttpsAgent + familyCache: { [host: string] : 4 | 6 } = {} + + constructor(httpOpts: http.AgentOptions = {}, httpsOpts: https.AgentOptions = {}) { + this.httpAgent = new HttpAgent(httpOpts) + this.httpsAgent = new HttpsAgent(httpsOpts) + this._getFirstWorkingFamily = Promise.method(this._getFirstWorkingFamily) + } + + // called by Node.js whenever a new request is made internally + addRequest(req: http.ClientRequest, options: http.RequestOptions) { + const isHttps = isRequestHttps(options) + + if (!options.href) { + // options.path can contain query parameters, which url.format will not-so-kindly urlencode for us... + // so just append it to the resultant URL string + options.href = url.format({ + protocol: isHttps ? 'https:' : 'http:', + slashes: true, + hostname: options.host, + port: options.port, + }) + options.path + + if (!options.uri) { + options.uri = url.parse(options.href) + } + } + + debug(`addRequest called for ${options.href}`) + + this._getFirstWorkingFamily(options) + .then((family: number) => { + options.family = family + + if (isHttps) { + return this.httpsAgent.addRequest(req, options) + } + + this.httpAgent.addRequest(req, options) + }) + } + + _getFirstWorkingFamily({ port, host }: http.RequestOptions) { + // this is a workaround for localhost (and potentially others) having invalid + // A records but valid AAAA records. here, we just cache the family of the first + // returned A/AAAA record for a host that we can establish a connection to. + // https://github.com/cypress-io/cypress/issues/112 + + const isIP = net.isIP(host) + if (isIP) { + // isIP conveniently returns the family of the address + return isIP + } + + if (process.env.HTTP_PROXY) { + // can't make direct connections through the proxy, this won't work + return + } + + if (this.familyCache[host]) { + return this.familyCache[host] + } + + return getAddress(port, host) + .then((firstWorkingAddress: net.Address) => { + this.familyCache[host] = firstWorkingAddress.family + return firstWorkingAddress.family + }) + .catchReturn() + } +} + +class HttpAgent extends http.Agent { + httpsAgent: https.Agent + + constructor (opts: http.AgentOptions = {}) { + opts.keepAlive = true + super(opts) + // we will need this if they wish to make http requests over an https proxy + this.httpsAgent = new https.Agent({ keepAlive: true }) + } + + createSocket (req: http.ClientRequest, options: http.RequestOptions, cb: http.SocketCallback) { + if (process.env.HTTP_PROXY) { + const proxy = getProxyForUrl(options.href) + + if (proxy) { + options.proxy = proxy + + return this._createProxiedSocket(req, options, cb) + } + } + + super.createSocket(req, options, cb) + } + + _createProxiedSocket (req: http.ClientRequest, options: RequestOptionsWithProxy, cb: http.SocketCallback) { + debug(`Creating proxied socket for ${options.href} through ${options.proxy}`) + + const proxy = url.parse(options.proxy) + + // set req.path to the full path so the proxy can resolve it + // @ts-ignore: Cannot assign to 'path' because it is a constant or a read-only property. + req.path = options.href + + delete req._header // so we can set headers again + + req.setHeader('host', `${options.host}:${options.port}`) + if (proxy.auth) { + req.setHeader('proxy-authorization', `basic ${Buffer.from(proxy.auth).toString('base64')}`) + } + + // node has queued an HTTP message to be sent already, so we need to regenerate the + // queued message with the new path and headers + // https://github.com/TooTallNate/node-http-proxy-agent/blob/master/index.js#L93 + _regenerateRequestHead(req) + + options.port = Number(proxy.port || 80) + options.host = proxy.hostname || 'localhost' + delete options.path // so the underlying net.connect doesn't default to IPC + + if (proxy.protocol === 'https:') { + // gonna have to use the https module to reach the proxy, even though this is an http req + req.agent = this.httpsAgent + + return this.httpsAgent.addRequest(req, options) + } + + super.createSocket(req, options, cb) + } +} + +class HttpsAgent extends https.Agent { + constructor (opts: https.AgentOptions = {}) { + opts.keepAlive = true + super(opts) + } + + createConnection (options: http.RequestOptions, cb: http.SocketCallback) { + if (process.env.HTTPS_PROXY) { + const proxy = getProxyForUrl(options.href) + + if (typeof proxy === "string") { + options.proxy = proxy + + return this.createProxiedConnection(options, cb) + } + } + + // @ts-ignore + cb(null, super.createConnection(options)) + } + + createProxiedConnection (options: RequestOptionsWithProxy, cb: http.SocketCallback) { + // heavily inspired by + // https://github.com/mknj/node-keepalive-proxy-agent/blob/master/index.js + debug(`Creating proxied socket for ${options.href} through ${options.proxy}`) + + const proxy = url.parse(options.proxy) + const port = options.uri.port || '443' + const hostname = options.uri.hostname || 'localhost' + + const proxySocket = _createProxySock(proxy) + + const onClose = () => { + onError(new Error("Connection closed while sending request to upstream proxy")) + } + + const onError = (err: Error) => { + proxySocket.destroy() + cb(err, undefined) + } + + let buffer = '' + + const onData = (data: Buffer) => { + debug(`Proxy socket for ${options.href} established`) + + buffer += data.toString() + + if (!_.includes(buffer, _.repeat(CRLF, 2))) { + // haven't received end of headers yet, keep buffering + proxySocket.once('data', onData) + return + } + + proxySocket.removeListener('error', onError) + proxySocket.removeListener('close', onClose) + + if (!isResponseStatusCode200(buffer)) { + return onError(new Error(`Error establishing proxy connection. Response from server was: ${buffer}`)) + } + + if (options._agentKey) { + // https.Agent will upgrade and reuse this socket now + options.socket = proxySocket + options.servername = hostname + return cb(undefined, super.createConnection(options, undefined)) + } + + cb(undefined, proxySocket) + } + + proxySocket.once('error', onError) + proxySocket.once('close', onClose) + proxySocket.once('data', onData) + + const connectReq = _buildConnectReqHead(hostname, port, proxy) + + proxySocket.write(connectReq) + } +} + +const agent = new CombinedAgent() + +export default agent diff --git a/packages/network/lib/connect.ts b/packages/network/lib/connect.ts new file mode 100644 index 0000000000..31957b4cdb --- /dev/null +++ b/packages/network/lib/connect.ts @@ -0,0 +1,37 @@ +import * as net from 'net' +import * as dns from 'dns' +import * as Promise from 'bluebird' + +export function byPortAndAddress (port: number, address: net.Address) { + // https://nodejs.org/api/net.html#net_net_connect_port_host_connectlistener + return new Promise((resolve, reject) => { + const onConnect = () => { + client.end() + resolve(address) + } + + const client = net.connect(port, address.address, onConnect) + + client.on('error', reject) + }) +} + +export function getAddress (port: number, hostname: string) { + const fn = byPortAndAddress.bind({}, port) + + // promisify at the very last second which enables us to + // modify dns lookup function (via hosts overrides) + const lookupAsync = Promise.promisify(dns.lookup, { context: dns }) + + // this does not go out to the network to figure + // out the addresess. in fact it respects the /etc/hosts file + // https://github.com/nodejs/node/blob/dbdbdd4998e163deecefbb1d34cda84f749844a4/lib/dns.js#L108 + // https://nodejs.org/api/dns.html#dns_dns_lookup_hostname_options_callback + // @ts-ignore + return lookupAsync(hostname, { all: true }) + .then((addresses: net.Address[]) => { + // convert to an array if string + return Array.prototype.concat.call(addresses).map(fn) + }) + .any() +} diff --git a/packages/network/package.json b/packages/network/package.json new file mode 100644 index 0000000000..19e49b8cca --- /dev/null +++ b/packages/network/package.json @@ -0,0 +1,24 @@ +{ + "name": "@packages/network", + "version": "0.0.0", + "private": true, + "main": "index.js", + "files": [ + "lib" + ], + "scripts": { + "test": "bin-up mocha --reporter mocha-multi-reporters --reporter-options configFile=../../mocha-reporter-config.json" + }, + "devDependencies": { + "bin-up": "1.1.0", + "debugging-proxy": "1.6.0", + "express": "4.16.4", + "request": "2.88.0", + "request-promise": "4.2.4", + "sinon": "7.3.1", + "sinon-chai": "3.3.0" + }, + "dependencies": { + "proxy-from-env": "1.0.0" + } +} diff --git a/packages/network/test/mocha.opts b/packages/network/test/mocha.opts new file mode 100644 index 0000000000..3622a28170 --- /dev/null +++ b/packages/network/test/mocha.opts @@ -0,0 +1,4 @@ +test/unit +--compilers ts:@packages/ts/register +--timeout 10000 +--recursive diff --git a/packages/network/test/support/servers.ts b/packages/network/test/support/servers.ts new file mode 100644 index 0000000000..b737d67db6 --- /dev/null +++ b/packages/network/test/support/servers.ts @@ -0,0 +1,104 @@ +import * as http from 'http' +import * as https from 'https' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import * as express from 'express' +import * as Promise from 'bluebird' +import { CA } from '@packages/https-proxy' +import * as Io from '@packages/socket' + +export interface AsyncServer { + closeAsync: () => Promise + destroyAsync: () => Promise + listenAsync: (port) => Promise +} + +function addDestroy(server: http.Server | https.Server) { + let connections = [] + + server.on('connection', function(conn) { + connections.push(conn) + + conn.on('close', () => { + connections = connections.filter(connection => connection !== conn) + }) + }) + + // @ts-ignore Property 'destroy' does not exist on type 'Server'. + server.destroy = function(cb) { + server.close(cb) + connections.map(connection => connection.destroy()) + } + + return server +} + +function createExpressApp() { + const app: express.Application = express() + + app.get('/get', (req, res) => { + res.send('It worked!') + }) + + app.get('/empty-response', (req, res) => { + // ERR_EMPTY_RESPONSE in Chrome + setTimeout(() => res.connection.destroy(), 100) + }) + + return app +} + +function getLocalhostCertKeys() { + return CA.create() + .then(ca => ca.generateServerCertificateKeys('localhost')) +} + +function onWsConnection(socket) { + socket.send('It worked!') +} + +export class Servers { + https: { cert: string, key: string } + httpServer: http.Server & AsyncServer + httpsServer: https.Server & AsyncServer + wsServer: any + wssServer: any + + start(httpPort: number, httpsPort: number) { + return Promise.join( + createExpressApp(), + getLocalhostCertKeys(), + ) + .spread((app: Express.Application, [cert, key]: string[]) => { + this.httpServer = Promise.promisifyAll( + addDestroy(http.createServer(app)) + ) as http.Server & AsyncServer + this.wsServer = Io.server(this.httpServer) + + this.https = { cert, key } + this.httpsServer = Promise.promisifyAll( + addDestroy(https.createServer(this.https, app)) + ) as https.Server & AsyncServer + this.wssServer = Io.server(this.httpsServer) + + ;[this.wsServer, this.wssServer].map(ws => { + ws.on('connection', onWsConnection) + }) + + // @ts-skip + return Promise.join( + this.httpServer.listenAsync(httpPort), + this.httpsServer.listenAsync(httpsPort) + ) + .return() + }) + } + + stop() { + return Promise.join( + this.httpServer.destroyAsync(), + this.httpsServer.destroyAsync() + ) + } +} diff --git a/packages/network/test/unit/agent_spec.ts b/packages/network/test/unit/agent_spec.ts new file mode 100644 index 0000000000..d5d7821666 --- /dev/null +++ b/packages/network/test/unit/agent_spec.ts @@ -0,0 +1,420 @@ +import * as http from 'http' +import * as https from 'https' +import * as net from 'net' +import * as tls from 'tls' +import * as url from 'url' +import * as chai from 'chai' +import DebuggingProxy = require('debugging-proxy') +import * as Promise from 'bluebird' +import * as request from 'request-promise' +import * as sinon from 'sinon' +import * as sinonChai from 'sinon-chai' +import { + CombinedAgent, + _buildConnectReqHead, + _createProxySock, + isRequestHttps, + isResponseStatusCode200, + _regenerateRequestHead +} from '../../lib/agent' +import * as Io from '@packages/socket' +import { Servers, AsyncServer } from '../support/servers' + +const expect = chai.expect +chai.use(sinonChai) + +const PROXY_PORT = 31000 +const HTTP_PORT = 31080 +const HTTPS_PORT = 31443 + +describe('lib/agent', function() { + beforeEach(function() { + this.oldEnv = Object.assign({}, process.env) + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' + }) + + afterEach(function() { + process.env = this.oldEnv + sinon.restore() + }) + + context('CombinedAgent', function() { + before(function() { + this.servers = new Servers() + return this.servers.start(HTTP_PORT, HTTPS_PORT) + }) + + after(function() { + return this.servers.stop() + }) + + ;[ + { + name: 'with no upstream', + }, + { + name: 'with an HTTP upstream', + proxyUrl: `http://localhost:${PROXY_PORT}`, + }, + { + name: 'with an HTTPS upstream', + proxyUrl: `https://localhost:${PROXY_PORT}`, + httpsProxy: true, + }, + { + name: 'with an HTTP upstream requiring auth', + proxyUrl: `http://foo:bar@localhost:${PROXY_PORT}`, + proxyAuth: true, + }, + { + name: 'with an HTTPS upstream requiring auth', + proxyUrl: `https://foo:bar@localhost:${PROXY_PORT}`, + httpsProxy: true, + proxyAuth: true + } + ].slice().map((testCase) => { + context(testCase.name, function() { + beforeEach(function() { + if (testCase.proxyUrl) { + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = testCase.proxyUrl + process.env.NO_PROXY = '' + } + + this.agent = new CombinedAgent() + + this.request = request.defaults({ + proxy: null, + agent: this.agent + }) + + if (testCase.proxyUrl) { + let options: any = { + keepRequests: true, + https: false, + auth: false + } + + if (testCase.httpsProxy) { + options.https = this.servers.https + } + + if (testCase.proxyAuth) { + options.auth = { + username: 'foo', + password: 'bar' + } + } + + this.debugProxy = new DebuggingProxy(options) + return this.debugProxy.start(PROXY_PORT) + } + }) + + afterEach(function() { + if (testCase.proxyUrl) { + this.debugProxy.stop() + } + }) + + it('HTTP pages can be loaded', function() { + return this.request({ + url: `http://localhost:${HTTP_PORT}/get`, + }).then(body => { + expect(body).to.eq('It worked!') + if (this.debugProxy) { + expect(this.debugProxy.requests[0]).to.include({ + url: `http://localhost:${HTTP_PORT}/get` + }) + } + }) + }) + + it('HTTPS pages can be loaded', function() { + return this.request({ + url: `https://localhost:${HTTPS_PORT}/get` + }).then(body => { + expect(body).to.eq('It worked!') + if (this.debugProxy) { + expect(this.debugProxy.requests[0]).to.include({ + https: true, + url: `localhost:${HTTPS_PORT}` + }) + } + }) + }) + + it('HTTP errors are catchable', function() { + return this.request({ + url: `http://localhost:${HTTP_PORT}/empty-response`, + }) + .then(() => { + throw new Error("Shouldn't reach this") + }) + .catch(err => { + if (this.debugProxy) { + expect(this.debugProxy.requests[0]).to.include({ + url: `http://localhost:${HTTP_PORT}/empty-response` + }) + expect(err.statusCode).to.eq(502) + } else { + expect(err.message).to.eq('Error: socket hang up') + } + }) + }) + + it('HTTPS errors are catchable', function() { + return this.request({ + url: `https://localhost:${HTTPS_PORT}/empty-response`, + }) + .then(() => { + throw new Error("Shouldn't reach this") + }) + .catch(err => { + expect(err.message).to.eq('Error: socket hang up') + }) + }) + + it('HTTP websocket connections can be established and used', function() { + return new Promise((resolve) => { + Io.client(`http://localhost:${HTTP_PORT}`, { + agent: this.agent, + transports: ['websocket'] + }).on('message', resolve) + }) + .then(msg => { + expect(msg).to.eq('It worked!') + if (this.debugProxy) { + expect(this.debugProxy.requests[0].ws).to.be.true + expect(this.debugProxy.requests[0].url).to.include('http://localhost:31080') + } + }) + }) + + it('HTTPS websocket connections can be established and used', function() { + return new Promise((resolve) => { + Io.client(`https://localhost:${HTTPS_PORT}`, { + agent: this.agent, + transports: ['websocket'] + }).on('message', resolve) + }) + .then(msg => { + expect(msg).to.eq('It worked!') + if (this.debugProxy) { + expect(this.debugProxy.requests[0]).to.include({ + url: 'localhost:31443' + }) + } + }) + }) + }) + }) + + context('HttpsAgent', function() { + it("#createProxiedConnection calls to super for caching, TLS-ifying", function() { + const combinedAgent = new CombinedAgent() + const spy = sinon.spy(https.Agent.prototype, 'createConnection') + + const proxy = new DebuggingProxy() + const proxyPort = PROXY_PORT + 1 + + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = `http://localhost:${proxyPort}` + process.env.NO_PROXY = '' + + return proxy.start(proxyPort) + .then(() => { + return request({ + url: `https://localhost:${HTTPS_PORT}/get`, + agent: combinedAgent, + proxy: null + }) + }) + .then(() => { + const options = spy.getCall(0).args[0] + const session = combinedAgent.httpsAgent._sessionCache.map[options._agentKey] + expect(spy).to.be.calledOnce + expect(combinedAgent.httpsAgent._sessionCache.list).to.have.length(1) + expect(session).to.not.be.undefined + + return proxy.stop() + }) + }) + + it("#createProxiedConnection throws when connection is accepted then closed", function() { + const combinedAgent = new CombinedAgent() + + const proxy = Promise.promisifyAll( + net.createServer((socket) => { + socket.end() + }) + ) as net.Server & AsyncServer + + const proxyPort = PROXY_PORT + 2 + + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = `http://localhost:${proxyPort}` + process.env.NO_PROXY = '' + + return proxy.listenAsync(proxyPort) + .then(() => { + return request({ + url: `https://localhost:${HTTPS_PORT}/get`, + agent: combinedAgent, + proxy: null + }) + }) + .then(() => { + throw new Error('should not succeed') + }) + .catch((e) => { + expect(e.message).to.eq('Error: Connection closed while sending request to upstream proxy') + + return proxy.closeAsync() + }) + }) + }) + }) + + context("._buildConnectReqHead", function() { + it('builds the correct request', function() { + const head = _buildConnectReqHead('foo.bar', '1234', {}) + expect(head).to.eq([ + 'CONNECT foo.bar:1234 HTTP/1.1', + 'Host: foo.bar:1234', + '', '' + ].join('\r\n')) + }) + + it('can do Proxy-Authorization', function() { + const head = _buildConnectReqHead('foo.bar', '1234', { + auth: 'baz:quux' + }) + expect(head).to.eq([ + 'CONNECT foo.bar:1234 HTTP/1.1', + 'Host: foo.bar:1234', + 'Proxy-Authorization: basic YmF6OnF1dXg=', + '', '' + ].join('\r\n')) + }) + }) + + context("._createProxySock", function() { + it("creates a `net` socket for an http url", function() { + sinon.stub(net, 'connect') + const proxy = url.parse('http://foo.bar:1234') + _createProxySock(proxy) + expect(net.connect).to.be.calledWith(1234, 'foo.bar') + }) + + it("creates a `tls` socket for an https url", function() { + sinon.stub(tls, 'connect') + const proxy = url.parse('https://foo.bar:1234') + _createProxySock(proxy) + expect(tls.connect).to.be.calledWith(1234, 'foo.bar') + }) + + it("throws on unsupported proxy protocol", function() { + const proxy = url.parse('socksv5://foo.bar:1234') + try { + _createProxySock(proxy) + throw new Error("Shouldn't be reached") + } catch (e) { + expect(e.message).to.eq("Unsupported proxy protocol: socksv5:") + } + }) + }) + + context(".isRequestHttps", function() { + [ + { + protocol: 'http', + agent: http.globalAgent, + expect: false + }, + { + protocol: 'https', + agent: https.globalAgent, + expect: true + } + ].map((testCase) => { + it(`detects correctly from ${testCase.protocol} requests`, () => { + const spy = sinon.spy(testCase.agent, 'addRequest') + + return request({ + url: `${testCase.protocol}://foo.bar.baz.invalid`, + agent: testCase.agent + }) + .then(() => { + throw new Error('Shouldn\'t succeed') + }) + .catch((e) => { + const requestOptions = spy.getCall(0).args[1] + expect(isRequestHttps(requestOptions)).to.equal(testCase.expect) + }) + }) + + it(`detects correctly from ${testCase.protocol} websocket requests`, () => { + const spy = sinon.spy(testCase.agent, 'addRequest') + + return new Promise((resolve, reject) => { + Io.client(`${testCase.protocol}://foo.bar.baz.invalid`, { + agent: testCase.agent, + transports: ['websocket'], + timeout: 1 + }) + .on('message', reject) + .on('connect_error', resolve) + }) + .then(() => { + const requestOptions = spy.getCall(0).args[1] + expect(isRequestHttps(requestOptions)).to.equal(testCase.expect) + }) + }) + }) + }) + + context(".isResponseStatusCode200", function() { + it("matches a 200 OK response correctly", function() { + const result = isResponseStatusCode200("HTTP/1.1 200 Connection established") + expect(result).to.be.true + }) + + it("matches a 500 error response correctly", function() { + const result = isResponseStatusCode200("HTTP/1.1 500 Internal Server Error") + expect(result).to.be.false + }) + }) + + context("._regenerateRequestHead", function() { + it("regenerates changed request head", () => { + const spy = sinon.spy(http.globalAgent, 'createSocket') + return request({ + url: 'http://foo.bar.baz.invalid', + agent: http.globalAgent + }) + .then(() => { + throw new Error('this should fail') + }) + .catch(() => { + const req = spy.getCall(0).args[0] + expect(req._header).to.equal([ + 'GET / HTTP/1.1', + 'host: foo.bar.baz.invalid', + 'Connection: close', + '', '' + ].join('\r\n')) + // now change some stuff, regen, and expect it to work + delete req._header + req.path = 'http://quuz.quux.invalid/abc?def=123' + req.setHeader('Host', 'foo.fleem.invalid') + req.setHeader('bing', 'bang') + _regenerateRequestHead(req) + expect(req._header).to.equal([ + 'GET http://quuz.quux.invalid/abc?def=123 HTTP/1.1', + 'Host: foo.fleem.invalid', + 'bing: bang', + 'Connection: close', + '', '' + ].join('\r\n')) + }) + }) + }) +}) diff --git a/packages/network/tsconfig.json b/packages/network/tsconfig.json new file mode 100644 index 0000000000..ffda540c36 --- /dev/null +++ b/packages/network/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./../ts/tsconfig.json", + "include": [ + "lib/*.ts", + "lib/**/*.ts" + ], + "files": [ + "./../ts/index.d.ts" + ] +} diff --git a/packages/runner/scripts/set-zunder-config.js b/packages/runner/scripts/set-zunder-config.js index 16e4a50211..326ca914a1 100644 --- a/packages/runner/scripts/set-zunder-config.js +++ b/packages/runner/scripts/set-zunder-config.js @@ -9,6 +9,8 @@ module.exports = function setZunderConfig (zunder) { coffeeCompiler: require('@packages/coffee'), }, ]) + browserifyOptions.extensions.push('.ts') + browserifyOptions.plugin.push(zunder.defaults.browserify.pluginTsify.module) // ensure no duplicates of common dependencies between runner, reporter, & driver browserifyOptions.transform.push([ zunder.defaults.browserify.transformAliasify.module, diff --git a/packages/server/__snapshots__/4_request_spec.coffee.js b/packages/server/__snapshots__/4_request_spec.coffee.js index f56e8a37cc..62723ff4db 100644 --- a/packages/server/__snapshots__/4_request_spec.coffee.js +++ b/packages/server/__snapshots__/4_request_spec.coffee.js @@ -246,6 +246,7 @@ The request we sent was: Method: GET URL: http://localhost:2294/statusCode?code=503 Headers: { + "Connection": "keep-alive", "user-agent": "foo", "accept": "*/*", "accept-encoding": "gzip, deflate" @@ -262,7 +263,7 @@ Headers: { "content-length": "19", "etag": "W/13-52060a5f", "date": "Fri, 18 Aug 2017 XX:XX GMT", - "connection": "close" + "connection": "keep-alive" } Body: Service Unavailable diff --git a/packages/server/index.js b/packages/server/index.js index ed19bcf678..283e80622b 100644 --- a/packages/server/index.js +++ b/packages/server/index.js @@ -7,6 +7,12 @@ if (process.versions.electron) { require('./timers/parent').fix() } +if (process.env.CY_NET_PROFILE && process.env.CYPRESS_ENV) { + const netProfiler = require('./lib/util/net_profiler')() + + process.stdout.write(`Network profiler writing to ${netProfiler.logPath}\n`) +} + process.env.UV_THREADPOOL_SIZE = 128 require('graceful-fs').gracefulify(require('fs')) // if running in production mode (CYPRESS_ENV) diff --git a/packages/server/lib/api.coffee b/packages/server/lib/api.coffee index 3fa0623ee1..9a4fe890fe 100644 --- a/packages/server/lib/api.coffee +++ b/packages/server/lib/api.coffee @@ -6,6 +6,7 @@ request = require("request-promise") errors = require("request-promise/errors") Promise = require("bluebird") humanInterval = require("human-interval") +agent = require("@packages/network").agent pkg = require("@packages/root") routes = require("./util/routes") system = require("./util/system") @@ -34,6 +35,8 @@ if intervals = process.env.API_RETRY_INTERVALS rp = request.defaults (params = {}, callback) -> _.defaults(params, { + agent: agent + proxy: null gzip: true }) @@ -79,7 +82,7 @@ machineId = -> ## retry on timeouts, 5xx errors, or any error without a status code isRetriableError = (err) -> - (err instanceof Promise.TimeoutError) or + (err instanceof Promise.TimeoutError) or (500 <= err.statusCode < 600) or not err.statusCode? diff --git a/packages/server/lib/environment.coffee b/packages/server/lib/environment.coffee index bebd91a0d6..fd99167a0a 100644 --- a/packages/server/lib/environment.coffee +++ b/packages/server/lib/environment.coffee @@ -1,4 +1,3 @@ -require("./util/http_overrides") require("./util/fs") os = require("os") @@ -19,7 +18,7 @@ try app = require("electron").app app.commandLine.appendSwitch("disable-renderer-backgrounding", true) app.commandLine.appendSwitch("ignore-certificate-errors", true) - + ## These flags are for webcam/WebRTC testing ## https://github.com/cypress-io/cypress/issues/2704 app.commandLine.appendSwitch("use-fake-ui-for-media-stream") diff --git a/packages/server/lib/gui/events.coffee b/packages/server/lib/gui/events.coffee index 9076275d50..6e51d66e2b 100644 --- a/packages/server/lib/gui/events.coffee +++ b/packages/server/lib/gui/events.coffee @@ -13,7 +13,7 @@ errors = require("../errors") Updater = require("../updater") Project = require("../project") openProject = require("../open_project") -connect = require("../util/connect") +ensureUrl = require("../util/ensure-url") browsers = require("../browsers") konfig = require("../konfig") @@ -271,7 +271,7 @@ handleEvent = (options, bus, event, id, type, arg) -> when "ping:api:server" apiUrl = konfig("api_url") - connect.ensureUrl(apiUrl) + ensureUrl.isListening(apiUrl) .then(send) .catch (err) -> ## if it's an aggegrate error, just send the first one diff --git a/packages/server/lib/gui/windows.coffee b/packages/server/lib/gui/windows.coffee index fb83f1fab6..6206c67f69 100644 --- a/packages/server/lib/gui/windows.coffee +++ b/packages/server/lib/gui/windows.coffee @@ -32,6 +32,16 @@ firstOrNull = (cookies) -> ## normalize into null when empty array cookies[0] ? null +setWindowProxy = (win) -> + if not process.env.HTTP_PROXY + return + + return new Promise (resolve) -> + win.webContents.session.setProxy({ + proxyRules: process.env.HTTP_PROXY + proxyBypassRules: process.env.NO_PROXY + }, resolve) + module.exports = { reset: -> windows = {} @@ -280,9 +290,11 @@ module.exports = { ## enable our url to be a promise ## and wait for this to be resolved - Promise - .resolve(options.url) - .then (url) -> + Promise.join( + options.url, + setWindowProxy(win) + ) + .spread (url) -> if options.type is "GITHUB_LOGIN" ## remove the GitHub warning banner about an outdated browser ## TODO: remove this once we have upgraded Electron or added native browser auth diff --git a/packages/server/lib/request.coffee b/packages/server/lib/request.coffee index ef6a5e254a..92ee6e4760 100644 --- a/packages/server/lib/request.coffee +++ b/packages/server/lib/request.coffee @@ -6,6 +6,7 @@ tough = require("tough-cookie") debug = require("debug")("cypress:server:request") moment = require("moment") Promise = require("bluebird") +agent = require("@packages/network").agent statusCode = require("./util/status_code") Cookies = require("./automation/cookies") @@ -132,7 +133,13 @@ createCookieString = (c) -> module.exports = (options = {}) -> defaults = { timeout: options.timeout ? 20000 - # forever: true + agent: agent + ## send keep-alive with requests since Chrome won't send it in proxy mode + ## https://github.com/cypress-io/cypress/pull/3531#issuecomment-476269041 + headers: { + "Connection": "keep-alive" + } + proxy: null ## upstream proxying is handled by CombinedAgent } r = r.defaults(defaults) diff --git a/packages/server/lib/server.coffee b/packages/server/lib/server.coffee index 4899c9ef91..500468adf2 100644 --- a/packages/server/lib/server.coffee +++ b/packages/server/lib/server.coffee @@ -13,10 +13,11 @@ check = require("check-more-types") httpsProxy = require("@packages/https-proxy") compression = require("compression") debug = require("debug")("cypress:server:server") +agent = require("@packages/network").agent cors = require("./util/cors") uri = require("./util/uri") origin = require("./util/origin") -connect = require("./util/connect") +ensureUrl = require("./util/ensure-url") appData = require("./util/app_data") buffers = require("./util/buffers") blacklist = require("./util/blacklist") @@ -232,7 +233,7 @@ class Server if baseUrl @_baseUrl = baseUrl - connect.ensureUrl(baseUrl) + ensureUrl.isListening(baseUrl) .return(null) .catch (err) => if config.isTextTerminal @@ -596,6 +597,7 @@ class Server port: port protocol: protocol } + agent: agent }, onProxyErr) else ## we can't do anything with this socket diff --git a/packages/server/lib/socket.coffee b/packages/server/lib/socket.coffee index 3402c56e32..ee662ade0b 100644 --- a/packages/server/lib/socket.coffee +++ b/packages/server/lib/socket.coffee @@ -2,7 +2,6 @@ _ = require("lodash") path = require("path") debug = require('debug')('cypress:server:socket') Promise = require("bluebird") -shell = require("electron").shell socketIo = require("@packages/socket") fs = require("./util/fs") open = require("./util/open") @@ -333,7 +332,7 @@ class Socket cb() if cb socket.on "external:open", (url) -> - shell.openExternal(url) + require("electron").shell.openExternal(url) reporterEvents.forEach (event) => socket.on event, (data) => diff --git a/packages/server/lib/updater.coffee b/packages/server/lib/updater.coffee index 5c9f5a2cbf..01cbfb9c97 100644 --- a/packages/server/lib/updater.coffee +++ b/packages/server/lib/updater.coffee @@ -5,6 +5,7 @@ semver = require("semver") request = require("request") NwUpdater = require("node-webkit-updater") pkg = require("@packages/root") +agent = require("@packages/network").agent cwd = require("./cwd") konfig = require("./konfig") @@ -41,7 +42,9 @@ NwUpdater.prototype.checkNewVersion = (cb) -> headers: { "x-cypress-version": pkg.version "x-machine-id": id - } + }, + agent: agent + proxy: null }, gotManifest.bind(@)) ## return hashed value because we dont care nor want diff --git a/packages/server/lib/util/args.js b/packages/server/lib/util/args.js index a4e2d21661..b269ba7f11 100644 --- a/packages/server/lib/util/args.js +++ b/packages/server/lib/util/args.js @@ -18,13 +18,13 @@ const debug = require('debug')('cypress:server:args') const minimist = require('minimist') const coerce = require('./coerce') const config = require('../config') -const cwd = require('../cwd') +const proxy = require('./proxy') const nestedObjectsInCurlyBracesRe = /\{(.+?)\}/g const nestedArraysInSquareBracketsRe = /\[(.+?)\]/g const everythingAfterFirstEqualRe = /=(.*)/ -const whitelist = 'cwd appPath execPath apiKey smokeTest getKey generateKey runProject project spec reporter reporterOptions port env ci record updating ping key logs clearLogs returnPkg version mode headed config exit exitWithCode browser runMode outputPath parallel ciBuildId group inspectBrk'.split(' ') +const whitelist = 'cwd appPath execPath apiKey smokeTest getKey generateKey runProject project spec reporter reporterOptions port env ci record updating ping key logs clearLogs returnPkg version mode headed config exit exitWithCode browser runMode outputPath parallel ciBuildId group inspectBrk proxySource'.split(' ') // returns true if the given string has double quote character " // only at the last position. @@ -171,6 +171,7 @@ module.exports = { 'reporter-options': 'reporterOptions', 'output-path': 'outputPath', 'inspect-brk': 'inspectBrk', + 'proxy-source': 'proxySource', } //# takes an array of args and converts @@ -224,6 +225,17 @@ module.exports = { options.env = sanitizeAndConvertNestedArgs(envs) } + const proxySource = proxy.loadSystemProxySettings() + + if (process.env.HTTP_PROXY) { + if (proxySource) { + options.proxySource = proxySource + } + + options.proxyServer = process.env.HTTP_PROXY + options.proxyBypassList = process.env.NO_PROXY + } + if (ro = options.reporterOptions) { options.reporterOptions = sanitizeAndConvertNestedArgs(ro) } diff --git a/packages/server/lib/util/connect.js b/packages/server/lib/util/connect.js deleted file mode 100644 index e0c201c6a2..0000000000 --- a/packages/server/lib/util/connect.js +++ /dev/null @@ -1,51 +0,0 @@ -const net = require('net') -const dns = require('dns') -const url = require('url') -const Promise = require('bluebird') - -module.exports = { - byPortAndAddress (port, address) { - // https://nodejs.org/api/net.html#net_net_connect_port_host_connectlistener - return new Promise((resolve, reject) => { - const client = net.connect(port, address.address) - - client.on('connect', () => { - client.end() - - resolve(address) - }) - - client.on('error', reject) - }) - }, - - getAddress (port, hostname) { - const fn = this.byPortAndAddress.bind(this, port) - - // promisify at the very last second which enables us to - // modify dns lookup function (via hosts overrides) - const lookupAsync = Promise.promisify(dns.lookup, { context: dns }) - - // this does not go out to the network to figure - // out the addresess. in fact it respects the /etc/hosts file - // https://github.com/nodejs/node/blob/dbdbdd4998e163deecefbb1d34cda84f749844a4/lib/dns.js#L108 - // https://nodejs.org/api/dns.html#dns_dns_lookup_hostname_options_callback - return lookupAsync(hostname, { all: true }) - .then((addresses) => { - // convert to an array if string - return [].concat(addresses).map(fn) - }) - .any() - }, - - ensureUrl (urlStr) { - // takes a urlStr and verifies the hostname + port - let { hostname, protocol, port } = url.parse(urlStr) - - if (port == null) { - port = protocol === 'https:' ? '443' : '80' - } - - return this.getAddress(port, hostname) - }, -} diff --git a/packages/server/lib/util/ensure-url.ts b/packages/server/lib/util/ensure-url.ts new file mode 100644 index 0000000000..a4f871e827 --- /dev/null +++ b/packages/server/lib/util/ensure-url.ts @@ -0,0 +1,24 @@ +import * as url from 'url' +import * as rp from 'request-promise' +import { agent, connect } from '@packages/network' + +export function isListening (urlStr: string) { + // takes a urlStr and verifies the hostname + port is listening + let { hostname, protocol, port } = url.parse(urlStr) + + if (port == null) { + port = protocol === 'https:' ? '443' : '80' + } + + if (process.env.HTTP_PROXY) { + // cannot make arbitrary connections behind a proxy, attempt HTTP/HTTPS + return rp({ + url: urlStr, + agent, + proxy: null, + }) + .catch({ name: 'StatusCodeError' }, () => {}) // we just care if it can connect, not if it's a valid resource + } + + return connect.getAddress(Number(port), String(hostname)) +} diff --git a/packages/server/lib/util/http_overrides.js b/packages/server/lib/util/http_overrides.js deleted file mode 100644 index 0b6e110fe9..0000000000 --- a/packages/server/lib/util/http_overrides.js +++ /dev/null @@ -1,47 +0,0 @@ -const http = require('http') -const connect = require('./connect') - -const cache = {} -const { addRequest } = http.Agent.prototype - -const setCache = (options, family) => { - return cache[`${options.host}:${options.port}`] = family -} - -const getCachedFamily = (options) => { - return cache[`${options.host}:${options.port}`] -} - -// https://github.com/nodejs/node/blob/v5.1.1/lib/_http_agent.js#L110 -http.Agent.prototype.addRequest = function (req, options) { - const agent = this - - const makeRequest = () => { - return addRequest.call(agent, req, options) - } - - const family = getCachedFamily(options) - - // if we have a cached family for this - // specific host + port then just - // set that as the family and make - // the request - if (family) { - options.family = family - - return makeRequest() - } - - // else lets go ahead and make a dns lookup - return connect - .getAddress(options.port, options.host) - .tap((address) => { - // the first net.connect which resolves - // should be cached and the request should - // be immediately made - setCache(options, address.family) - options.family = address.family - }) - .then(makeRequest) - .catch(makeRequest) -} diff --git a/packages/server/lib/util/net_profiler.js b/packages/server/lib/util/net_profiler.js new file mode 100644 index 0000000000..6080ad8ab4 --- /dev/null +++ b/packages/server/lib/util/net_profiler.js @@ -0,0 +1,264 @@ +const fs = require('fs') +const debug = require('debug')('net-profiler') + +function getCaller (level = 5) { + try { + return new Error().stack.split('\n')[level].slice(7) + } catch (e) { + return 'unknown' + } +} + +function getLogPath (logPath) { + if (!logPath) { + const os = require('os') + const dirName = fs.mkdtempSync(`${os.tmpdir()}/net-profiler-`) + + logPath = `${dirName}/timeline.txt` + } + + return logPath +} + +function Connection (host, port, type = 'connection', toHost, toPort) { + this.type = type + this.host = host || 'localhost' + this.port = port + this.toHost = toHost || 'localhost' + this.toPort = toPort +} + +Connection.prototype.beginning = function () { + switch (this.type) { + case 'server': + return `O server began listening on ${this.host}:${this.port} at ${getCaller()}` + case 'client': + return `C client connected from ${this.host}:${this.port} to server on ${this.toHost}:${this.toPort}` + default: + return `X connection opened to ${this.host}:${this.port} by ${getCaller()}` + } +} + +Connection.prototype.ending = function () { + switch (this.type) { + case 'server': + return 'O server closed' + case 'client': + return 'C client disconnected' + default: + return 'X connection closed' + } +} + +/** + * Tracks all incoming and outgoing network connections and logs a timeline of network traffic to a file. + * + * @param options.net the `net` object to stub, default: nodejs net object + * @param options.tickMs the number of milliseconds between ticks in the profile, default: 1000 + * @param options.tickWhenNoneActive should ticks be recorded when no connections are active, default: false + * @param options.logPath path to the file to append to, default: new file in your temp directory + */ +function NetProfiler (options = {}) { + if (!(this instanceof NetProfiler)) return new NetProfiler(options) + + if (!options.net) { + options.net = require('net') + } + + this.net = options.net + this.proxies = {} + this.activeConnections = [] + this.startTs = new Date() / 1000 + this.tickMs = options.tickMs || 1000 + this.tickWhenNoneActive = options.tickWhenNoneActive || false + + this.logPath = getLogPath(options.logPath) + debug('logging to ', this.logPath) + + this.startProfiling() +} + +NetProfiler.prototype.install = function () { + const net = this.net + const self = this + + function netSocketPrototypeConnectApply (target, thisArg, args) { + const client = target.bind(thisArg)(...args) + + let options = self.net._normalizeArgs(args)[0] + + if (Array.isArray(options)) { + options = options[0] + } + + options.host = options.host || 'localhost' + + const connection = new Connection(options.host, options.port) + + client.on('close', () => { + self.removeActiveConnection(connection) + }) + + self.addActiveConnection(connection) + + return client + } + + function netServerPrototypeListenApply (target, thisArg, args) { + const server = thisArg + + server.on('listening', () => { + const { host, port } = server.address() + + const connection = new Connection(host, port, 'server') + + self.addActiveConnection(connection) + server.on('close', () => { + self.removeActiveConnection(connection) + }) + + server.on('connection', (client) => { + const clientConn = new Connection(client.remoteAddress, client.remotePort, 'client', host, port) + + self.addActiveConnection(clientConn) + client.on('close', () => { + self.removeActiveConnection(clientConn) + }) + }) + }) + + const listener = target.bind(thisArg)(...args) + + return listener + } + + this.proxies['net.Socket.prototype.connect'] = Proxy.revocable(net.Socket.prototype.connect, { + apply: netSocketPrototypeConnectApply, + }) + + this.proxies['net.Server.prototype.listen'] = Proxy.revocable(net.Server.prototype.listen, { + apply: netServerPrototypeListenApply, + }) + + net.Socket.prototype.connect = this.proxies['net.Socket.prototype.connect'].proxy + net.Server.prototype.listen = this.proxies['net.Server.prototype.listen'].proxy +} + +NetProfiler.prototype.uninstall = function () { + const net = this.net + + net.Socket.prototype.connect = this.proxies['net.Socket.prototype.connect'].proxy['[[Target]]'] + net.Server.prototype.listen = this.proxies['net.Server.prototype.listen'].proxy['[[Target]]'] + + this.proxies.forEach((proxy) => { + proxy.revoke() + }) +} + +NetProfiler.prototype.startProfiling = function () { + this.install() + debug('profiling started') + this.logStream = fs.openSync(this.logPath, 'a') + this.writeTimeline('Profiling started!') + this.startTimer() +} + +NetProfiler.prototype.startTimer = function () { + if (!this.tickMs) { + return + } + + this.timer = setInterval(() => { + const tick = this.tickWhenNoneActive || this.activeConnections.find((x) => { + return !!x + }) + + if (tick) { + this.writeTimeline() + } + }, this.tickMs) +} + +NetProfiler.prototype.stopTimer = function () { + clearInterval(this.timer) +} + +NetProfiler.prototype.stopProfiling = function () { + this.writeTimeline('Profiling stopped!') + this.stopTimer() + fs.closeSync(this.logStream) + debug('profiling ended') + this.uninstall() +} + +NetProfiler.prototype.addActiveConnection = function (connection) { + let index = this.activeConnections.findIndex((x) => { + return typeof x === 'undefined' + }) + + if (index === -1) { + index = this.activeConnections.length + this.activeConnections.push(connection) + } else { + this.activeConnections[index] = connection + } + + this.writeTimeline(index, connection.beginning()) +} + +NetProfiler.prototype.removeActiveConnection = function (connection) { + let index = this.activeConnections.findIndex((x) => { + return x === connection + }) + + this.writeTimeline(index, connection.ending()) + this.activeConnections[index] = undefined +} + +NetProfiler.prototype.getTimestamp = function () { + let elapsed = (new Date() / 1000 - this.startTs).toString() + const parts = elapsed.split('.', 2) + + if (!parts[1]) { + parts[1] = '000' + } + + while (parts[1].length < 3) { + parts[1] += '0' + } + + elapsed = `${parts[0]}.${parts[1] ? parts[1].slice(0, 3) : '000'}` + + while (elapsed.length < 11) { + elapsed = ` ${elapsed}` + } + + return elapsed +} + +NetProfiler.prototype.writeTimeline = function (index, message) { + if (!message) { + message = index || '' + index = this.activeConnections.length + } + + let row = ` ${this.activeConnections.map((conn, i) => { + if (conn) { + return ['|', '1', 'l', ':'][i % 4] + } + + return ' ' + }).join(' ')}` + + if (message) { + const column = 3 + index * 4 + + row = `${row.substring(0, column - 2)}[ ${message} ]${row.substring(2 + column + message.length)}` + } + + row = `${this.getTimestamp()}${row.replace(/\s+$/, '')}\n` + + fs.writeSync(this.logStream, row) +} + +module.exports = NetProfiler diff --git a/packages/server/lib/util/proxy.ts b/packages/server/lib/util/proxy.ts new file mode 100644 index 0000000000..76629d9672 --- /dev/null +++ b/packages/server/lib/util/proxy.ts @@ -0,0 +1,43 @@ +import * as os from 'os' +import getWindowsProxy = require('@cypress/get-windows-proxy') + +export = { + _getWindowsProxy: function () { + return getWindowsProxy() + }, + + _normalizeEnvironmentProxy: function () { + if (!process.env.HTTPS_PROXY) { + // request library will use HTTP_PROXY as a fallback for HTTPS urls, but + // proxy-from-env will not, so let's just force it to fall back like this + process.env.HTTPS_PROXY = process.env.HTTP_PROXY + } + + if (!process.env.NO_PROXY) { + // don't proxy localhost, to match Chrome's default behavior and user expectation + process.env.NO_PROXY = 'localhost' + } + }, + + // @ts-ignore: Not all code paths return a value + loadSystemProxySettings: function () { + if (process.env.HTTP_PROXY !== undefined) { + this._normalizeEnvironmentProxy() + + return + } + + if (os.platform() === 'win32') { + const windowsProxy = this._getWindowsProxy() + + if (windowsProxy) { + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = windowsProxy.httpProxy + process.env.NO_PROXY = process.env.NO_PROXY || windowsProxy.noProxy + } + + this._normalizeEnvironmentProxy() + + return 'win32' + } + } +} diff --git a/packages/server/package.json b/packages/server/package.json index f0e6b0cc74..10a47d43d0 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -17,6 +17,7 @@ "test-unit-watch": "./test/support/watch test/unit", "test-integration": "node ./test/scripts/run.js test/integration", "test-integration-watch": "./test/support/watch test-integration", + "test-performance": "node ./test/scripts/run.js test/performance", "test-e2e": "node ./test/scripts/e2e.js", "test-e2e-chrome": "node ./test/scripts/run.js test/e2e chrome", "test-cov": "NODE_COVERAGE=true NODE_ENV=test CYPRESS_ENV=test BLUEBIRD_DEBUG=1 xvfb-maybe istanbul cover node_modules/.bin/_mocha -- --opts ./test/support/mocha.opts", @@ -61,10 +62,13 @@ "body-parser": "1.12.4", "chai-uuid": "1.0.6", "chokidar-cli": "1.2.1", + "chrome-har-capturer": "0.13.4", "codecov": "1.0.1", "coffee-coverage": "1.0.1", + "console-table-printer": "1.0.0-beta12", "cors": "2.8.5", "coveralls": "2.13.3", + "debugging-proxy": "1.6.0", "electron-osx-sign": "0.4.11", "eol": "0.9.1", "eventsource": "1.0.7", @@ -97,6 +101,7 @@ "dependencies": { "@cypress/browserify-preprocessor": "1.1.2", "@cypress/commit-info": "2.1.2", + "@cypress/get-windows-proxy": "1.5.1", "@cypress/icons": "0.7.0", "@cypress/mocha-teamcity-reporter": "1.0.0", "@ffmpeg-installer/ffmpeg": "1.0.17", diff --git a/packages/server/test/integration/cypress_spec.coffee b/packages/server/test/integration/cypress_spec.coffee index e8db73d573..07199d9581 100644 --- a/packages/server/test/integration/cypress_spec.coffee +++ b/packages/server/test/integration/cypress_spec.coffee @@ -17,7 +17,6 @@ pkg = require("@packages/root") launcher = require("@packages/launcher") extension = require("@packages/extension") fs = require("#{root}lib/util/fs") -connect = require("#{root}lib/util/connect") ciProvider = require("#{root}lib/util/ci_provider") settings = require("#{root}lib/util/settings") Events = require("#{root}lib/gui/events") diff --git a/packages/server/test/performance/proxy_performance.js b/packages/server/test/performance/proxy_performance.js new file mode 100644 index 0000000000..f84c5e0248 --- /dev/null +++ b/packages/server/test/performance/proxy_performance.js @@ -0,0 +1,310 @@ +const cp = require('child_process') +const fs = require('fs') +const os = require('os') +const path = require('path') +const { it, after, before, beforeEach, describe } = require('mocha') +const { expect } = require('chai') +const debug = require('debug')('test:proxy-performance') +const DebuggingProxy = require('debugging-proxy') +const HarCapturer = require('chrome-har-capturer') +const Promise = require('bluebird') +const Table = require('console-table-printer').Table + +process.env.CYPRESS_ENV = 'development' + +const CA = require('@packages/https-proxy').CA +const Config = require('../../lib/config') +const Server = require('../../lib/server') +const { _getArgs } = require('../../lib/browsers/chrome') + +const CHROME_PATH = 'google-chrome' +const URL_UNDER_TEST = 'https://flotwig.github.io/cypress-fetch-page/index1000.html' + +const start = (new Date()) / 1000 + +const PROXY_PORT = process.env.PROXY_PORT || 45678 +const HTTPS_PROXY_PORT = process.env.HTTPS_PROXY_PORT || 45681 +const CDP_PORT = 45679 /** port range starts here, not the actual port */ +const CY_PROXY_PORT = 45680 + +const TEST_CASES = [ + // these 5 test cases cover Chrome, useful only for comparison + // { + // name: 'Chrome', + // }, + // { + // name: 'Chrome w/o HTTP/2', + // disableHttp2: true, + // }, + // { + // name: 'With proxy', + // upstreamProxy: true, + // }, + // { + // name: 'With HTTPS proxy', + // httpsUpstreamProxy: true, + // }, + // { + // name: 'With proxy w/o HTTP/2', + // disableHttp2: true, + // upstreamProxy: true, + // }, + { + name: 'With Cypress proxy, Not Intercepted', + cyProxy: true, + }, + { + name: 'With Cypress proxy w/o HTTP/2, Not Intercepted', + cyProxy: true, + disableHttp2: true, + }, + { + name: 'With Cypress proxy, Intercepted', + cyProxy: true, + cyIntercept: true, + }, + { + name: 'With Cypress proxy and upstream, Intercepted', + cyProxy: true, + upstreamProxy: true, + cyIntercept: true, + }, + { + name: 'With Cypress proxy and HTTPS upstream, Intercepted', + cyProxy: true, + httpsUpstreamProxy: true, + cyIntercept: true, + }, + { + name: 'With Cypress proxy and upstream, Not Intercepted', + cyProxy: true, + upstreamProxy: true, + }, + { + name: 'With Cypress proxy and HTTPS upstream, Not Intercepted', + cyProxy: true, + httpsUpstreamProxy: true, + }, +] + +let defaultArgs = _getArgs() + +// additionally... +defaultArgs = defaultArgs.concat([ + '--headless', + '--disable-background-networking', + '--no-sandbox', // allows us to run as root, for CI +]) + +const getMaxExpectedRunTime = (testCase) => { + // cy interceptor doesn't do http2, it'll always be slower + if (testCase.disableHttp2 || (testCase.cyProxy && testCase.cyIntercept)) { + // circle has faster Internet than the Cypress offices :( + return process.env.CI ? 3000 : 6000 + } + + return 1000 +} + +const getResultsFromHar = (har, testCase) => { + // HAR 1.2 Spec: + // http://www.softwareishard.com/blog/har-12-spec/ + const { entries } = har.log + + const first = entries[0] + const last = entries[entries.length - 1] + const elapsed = Number(new Date(last.startedDateTime)) + last.time - Number(new Date(first.startedDateTime)) + + testCase['Total'] = `${Math.round(elapsed)}ms` + + let mins = {} + let maxes = {} + + const aggTimings = entries.reduce((prev, cur) => { + cur = cur.timings + Object.keys(cur).forEach((timingKey) => { + if (cur[timingKey] === -1) return + + const ms = Math.round(cur[timingKey]) + + if (!mins[timingKey] || ms < mins[timingKey]) mins[timingKey] = ms + + if (!maxes[timingKey] || ms > maxes[timingKey]) maxes[timingKey] = ms + + if (!prev[timingKey]) prev[timingKey] = ms + else prev[timingKey] += ms + }) + + return prev + }, {}) + + Object.keys(aggTimings).forEach((timingKey) => { + if (!['receive', 'wait', 'send'].find((x) => { + return x === timingKey + })) return + + testCase[`Avg ${timingKey}`] = `${Math.round(aggTimings[timingKey] / entries.length)}ms` + }) +} + +let cyServer + +describe('Proxy Performance', () => { + beforeEach(() => { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' + }) + + before(() => { + return CA.create() + .then((ca) => { + return ca.generateServerCertificateKeys('localhost') + }) + .spread((cert, key) => { + return Promise.join( + new DebuggingProxy().start(PROXY_PORT), + + new DebuggingProxy({ + https: { cert, key }, + }).start(HTTPS_PROXY_PORT), + + Config.set({ + projectRoot: '/tmp/a', + }).then((config) => { + config.port = CY_PROXY_PORT + cyServer = Server() + + return cyServer.open(config) + }) + ) + }) + }) + + after(() => { + debug(`Done in ${Math.round((new Date() / 1000) - start)}s`) + // console.table has forsaken us :( + const t = new Table() + + t.addRows(TEST_CASES) + t.printTable() + }) + + TEST_CASES.map((testCase) => { + it(`${testCase.name} loads 1000 images in less than ${getMaxExpectedRunTime(testCase)}ms`, function () { + debug('Current test: ', testCase.name) + + // configure command line args + const cdpPort = CDP_PORT + Math.round(Math.random() * 10000) + + let args = defaultArgs.concat([ + `--remote-debugging-port=${cdpPort}`, + `--user-data-dir=${fs.mkdtempSync(path.join(os.tmpdir(), 'cy-perf-'))}`, + ]) + + if (testCase.disableHttp2) { + args.push('--disable-http2') + } + + if (testCase.cyProxy) { + args.push(`--proxy-server=http://localhost:${CY_PROXY_PORT}`) + } + + if (testCase.upstreamProxy && !testCase.cyProxy) { + args.push(`--proxy-server=http://localhost:${PROXY_PORT}`) + } else if (testCase.httpsUpstreamProxy && !testCase.cyProxy) { + args.push(`--proxy-server=https://localhost:${HTTPS_PROXY_PORT}`) + } + + if (testCase.upstreamProxy && testCase.cyProxy) { + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = `http://localhost:${PROXY_PORT}` + } else if (testCase.httpsUpstreamProxy && testCase.cyProxy) { + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = `https://localhost:${HTTPS_PROXY_PORT}` + } else { + delete process.env.HTTPS_PROXY + delete process.env.HTTP_PROXY + } + + if (testCase.cyIntercept) { + cyServer._onDomainSet(URL_UNDER_TEST) + } else { + cyServer._onDomainSet('') + } + + let cmd = CHROME_PATH + + debug('Launching Chrome: ', cmd, args.join(' ')) + + const proc = cp.spawn(cmd, args, { + stdio: 'ignore', + }) + + const runHar = () => { + // wait for Chrome to open, then start capturing + return Promise.delay(500).then(() => { + debug('Trying to connect to Chrome...') + + const harCapturer = HarCapturer.run([ + URL_UNDER_TEST, + ], { + port: cdpPort, + // disable SSL verification on older Chrome versions, copied from the HAR CLI + // https://github.com/cyrus-and/chrome-har-capturer/blob/587550508bddc23b7f4b4328c158322be4749298/bin/cli.js#L60 + preHook: (_, cdp) => { + const { Security } = cdp + + return Security.enable().then(() => { + return Security.setOverrideCertificateErrors({ override: true }) + }) + .then(() => { + return Security.certificateError(({ eventId }) => { + debug('EVENT ID', eventId) + + return Security.handleCertificateError({ eventId, action: 'continue' }) + }) + }) + }, + // wait til all data is done before finishing + // https://github.com/cyrus-and/chrome-har-capturer/issues/59 + postHook: (_, cdp) => { + let timeout + + return new Promise((resolve) => { + cdp.on('event', (message) => { + if (message.method === 'Network.dataReceived') { + // reset timer + clearTimeout(timeout) + timeout = setTimeout(resolve, 1000) + } + }) + }) + }, + }) + + return new Promise((resolve, reject) => { + harCapturer.on('fail', (_, err) => { + return reject(err) + }) + + harCapturer.on('har', resolve) + }) + .catch((err) => { + // sometimes chrome takes surprisingly long, just reconn + debug('Chrome connection failed: ', err) + + return runHar() + }) + .then((har) => { + proc.kill(9) + debug('Received HAR from Chrome') + getResultsFromHar(har, testCase) + + const runTime = Number(testCase['Total'].replace('ms', '')) + + expect(runTime).to.be.lessThan(getMaxExpectedRunTime(testCase)) + }) + }) + } + + return runHar() + }) + }) +}) diff --git a/packages/server/test/scripts/run.js b/packages/server/test/scripts/run.js index c09ec45f24..c804473db0 100644 --- a/packages/server/test/scripts/run.js +++ b/packages/server/test/scripts/run.js @@ -60,7 +60,7 @@ if (isWindows()) { if (options['inspect-brk']) { commandAndArguments.args.push( '--inspect', - `--inspect-brk=${options['inspect-brk']}` + `--inspect-brk${options['inspect-brk'] === true ? '' : `=${options['inspect-brk']}`}` ) } diff --git a/packages/server/test/unit/api_spec.coffee b/packages/server/test/unit/api_spec.coffee index 28949283eb..dae6ac3efb 100644 --- a/packages/server/test/unit/api_spec.coffee +++ b/packages/server/test/unit/api_spec.coffee @@ -3,6 +3,7 @@ require("../spec_helper") _ = require("lodash") os = require("os") nmi = require("node-machine-id") +agent = require("@packages/network").agent pkg = require("@packages/root") api = require("#{root}lib/api") browsers = require("#{root}lib/browsers") @@ -15,6 +16,39 @@ describe "lib/api", -> beforeEach -> sinon.stub(os, "platform").returns("linux") + context ".rp", -> + beforeEach -> + sinon.spy(agent, 'addRequest') + nock.enableNetConnect() ## nock will prevent requests from reaching the agent + + it "makes calls using the correct agent", -> + api.ping() + .thenThrow() + .catch => + expect(agent.addRequest).to.be.calledOnce + expect(agent.addRequest).to.be.calledWithMatch(sinon.match.any, { + href: 'http://localhost:1234/ping' + }) + + context "with a proxy defined", -> + beforeEach -> + @oldEnv = Object.assign({}, process.env) + + it "makes calls using the correct agent", -> + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = 'http://foo.invalid:1234' + process.env.NO_PROXY = '' + + api.ping() + .thenThrow() + .catch => + expect(agent.addRequest).to.be.calledOnce + expect(agent.addRequest).to.be.calledWithMatch(sinon.match.any, { + href: 'http://localhost:1234/ping' + }) + + afterEach -> + process.env = @oldEnv + context ".getOrgs", -> it "GET /orgs + returns orgs", -> orgs = [] diff --git a/packages/server/test/unit/args_spec.coffee b/packages/server/test/unit/args_spec.coffee index f9631c14cc..6fb4e03d59 100644 --- a/packages/server/test/unit/args_spec.coffee +++ b/packages/server/test/unit/args_spec.coffee @@ -2,7 +2,9 @@ require("../spec_helper") _ = require("lodash") path = require("path") +os = require("os") argsUtil = require("#{root}lib/util/args") +proxyUtil = require("#{root}lib/util/proxy") cwd = process.cwd() @@ -49,7 +51,6 @@ describe "lib/util/args", -> options = @setup("--run-project", "foo", "--spec", "'cypress/integration/foo_spec.js'") expect(options.spec[0]).to.eq("#{cwd}/cypress/integration/foo_spec.js") - context "--port", -> it "converts to Number", -> options = @setup("--port", "8080") @@ -299,3 +300,45 @@ describe "lib/util/args", -> execPath: "e" updating: true }) + + context "with proxy", -> + beforeEach -> + @beforeEnv = Object.assign({}, process.env) + delete process.env.HTTP_PROXY + delete process.env.HTTPS_PROXY + delete process.env.NO_PROXY + + it "sets options from environment", -> + process.env.HTTP_PROXY = "http://foo-bar.baz:123" + process.env.NO_PROXY = "a,b,c" + options = @setup() + expect(options.proxySource).to.be.undefined + expect(options.proxyServer).to.eq process.env.HTTP_PROXY + expect(options.proxyServer).to.eq "http://foo-bar.baz:123" + expect(options.proxyBypassList).to.eq "a,b,c" + expect(process.env.HTTPS_PROXY).to.eq process.env.HTTP_PROXY + + it "loads from Windows registry if not defined", -> + sinon.stub(proxyUtil, "_getWindowsProxy").returns({ + httpProxy: "http://quux.quuz", + noProxy: "d,e,f" + }) + sinon.stub(os, "platform").returns("win32") + options = @setup() + expect(options.proxySource).to.eq "win32" + expect(options.proxyServer).to.eq "http://quux.quuz" + expect(options.proxyServer).to.eq process.env.HTTP_PROXY + expect(options.proxyServer).to.eq process.env.HTTPS_PROXY + expect(options.proxyBypassList).to.eq "d,e,f" + expect(options.proxyBypassList).to.eq process.env.NO_PROXY + + it "sets a default NO_PROXY", -> + process.env.HTTP_PROXY = "http://foo-bar.baz:123" + options = @setup() + expect(options.proxySource).to.be.undefined + expect(options.proxyServer).to.eq process.env.HTTP_PROXY + expect(options.proxyBypassList).to.eq "localhost" + expect(options.proxyBypassList).to.eq process.env.NO_PROXY + + afterEach -> + Object.assign(process.env, @beforeEnv) diff --git a/packages/server/test/unit/ensure_url_spec.ts b/packages/server/test/unit/ensure_url_spec.ts new file mode 100644 index 0000000000..0a36404940 --- /dev/null +++ b/packages/server/test/unit/ensure_url_spec.ts @@ -0,0 +1,62 @@ +require('../spec_helper') + +import { connect, agent } from '@packages/network' +import { isListening } from '../../lib/util/ensure-url' + +describe('lib/util/ensure-url', function() { + context(".isListening", function() { + it("resolves if a URL connects", function() { + const stub = sinon.stub(connect, 'getAddress').withArgs(80, 'foo.bar.invalid').resolves() + + return isListening('http://foo.bar.invalid') + .then(() => { + expect(stub).to.be.calledOnce + }) + }) + + it("rejects if a URL doesn't connect", function() { + const stub = sinon.stub(connect, 'getAddress').withArgs(80, 'foo.bar.invalid').rejects() + + return isListening('http://foo.bar.invalid') + .then(() => { + const err: any = new Error('should not reach this') + err.fromTest = true + }) + .catch((e) => { + if (e.fromTest) { + throw e + } + expect(stub).to.be.calledOnce + }) + }) + }) + + context('with a proxy', function() { + beforeEach(function() { + this.oldEnv = Object.assign({}, process.env) + }) + + afterEach(function() { + process.env = this.oldEnv + }) + + it('calls into the agent to check availability', function() { + process.env.HTTP_PROXY = process.env.HTTPS_PROXY = 'http://localhost:12345' + process.env.NO_PROXY = '' + + const stub = sinon.stub(agent, 'addRequest').throws() + nock.enableNetConnect() + + return isListening('http://foo.bar.invalid') + .then(() => { + throw new Error('should not succeed') + }) + .catch((e) => { + expect(agent.addRequest).to.be.calledOnce + expect(agent.addRequest).to.be.calledWithMatch(sinon.match.any, { + href: 'http://foo.bar.invalid/' + }) + }) + }) + }) +}) diff --git a/packages/server/test/unit/gui/events_spec.coffee b/packages/server/test/unit/gui/events_spec.coffee index 47467a57bb..eb9f1b7062 100644 --- a/packages/server/test/unit/gui/events_spec.coffee +++ b/packages/server/test/unit/gui/events_spec.coffee @@ -18,7 +18,7 @@ logs = require("#{root}../lib/gui/logs") events = require("#{root}../lib/gui/events") dialog = require("#{root}../lib/gui/dialog") Windows = require("#{root}../lib/gui/windows") -connect = require("#{root}../lib/util/connect") +ensureUrl = require("#{root}../lib/util/ensure-url") konfig = require("#{root}../lib/konfig") describe "lib/gui/events", -> @@ -663,15 +663,15 @@ describe "lib/gui/events", -> describe "ping:api:server", -> it "returns ensures url", -> - sinon.stub(connect, "ensureUrl").resolves() + sinon.stub(ensureUrl, "isListening").resolves() @handleEvent("ping:api:server").then (assert) => - expect(connect.ensureUrl).to.be.calledWith(konfig("api_url")) + expect(ensureUrl.isListening).to.be.calledWith(konfig("api_url")) assert.sendCalledWith() it "catches errors", -> err = new Error("foo") - sinon.stub(connect, "ensureUrl").rejects(err) + sinon.stub(ensureUrl, "isListening").rejects(err) @handleEvent("ping:api:server").then (assert) => assert.sendErrCalledWith(err) @@ -686,7 +686,7 @@ describe "lib/gui/events", -> address: "127.0.0.1" } err.length = 1 - sinon.stub(connect, "ensureUrl").rejects(err) + sinon.stub(ensureUrl, "isListening").rejects(err) @handleEvent("ping:api:server").then (assert) => assert.sendErrCalledWith(err) diff --git a/packages/server/test/unit/request_spec.coffee b/packages/server/test/unit/request_spec.coffee index c1e11bee14..c7f88bba5a 100644 --- a/packages/server/test/unit/request_spec.coffee +++ b/packages/server/test/unit/request_spec.coffee @@ -128,13 +128,14 @@ describe "lib/request", -> expect(resp.requestHeaders).to.deep.eq({ "accept": "*/*" "accept-encoding": "gzip, deflate" + "connection": "keep-alive" "content-length": 9 "host": "www.github.com" }) expect(resp.allRequestResponses).to.deep.eq([ { "Request Body": "foobarbaz" - "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "content-length": 9, "host": "www.github.com"} + "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "connection": "keep-alive", "content-length": 9, "host": "www.github.com"} "Request URL": "http://www.github.com/foo" "Response Body": "hello" "Response Headers": {"content-type": "text/html"} @@ -176,28 +177,29 @@ describe "lib/request", -> ]) expect(resp.requestHeaders).to.deep.eq({ "accept": "*/*" - "accept-encoding": "gzip, deflate", + "accept-encoding": "gzip, deflate" + "connection": "keep-alive" "referer": "http://www.github.com/auth" "host": "www.github.com" }) expect(resp.allRequestResponses).to.deep.eq([ { "Request Body": null - "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "host": "www.github.com"} + "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "connection": "keep-alive", "host": "www.github.com"} "Request URL": "http://www.github.com/dashboard" "Response Body": null "Response Headers": {"location": "/auth"} "Response Status": 301 }, { "Request Body": null - "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "host": "www.github.com", "referer": "http://www.github.com/dashboard"} + "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "connection": "keep-alive", "host": "www.github.com", "referer": "http://www.github.com/dashboard"} "Request URL": "http://www.github.com/auth" "Response Body": null "Response Headers": {"location": "/login"} "Response Status": 302 }, { "Request Body": null - "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "host": "www.github.com", "referer": "http://www.github.com/auth"} + "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "connection": "keep-alive", "host": "www.github.com", "referer": "http://www.github.com/auth"} "Request URL": "http://www.github.com/login" "Response Body": "log in" "Response Headers": {"content-type": "text/html"} @@ -300,6 +302,19 @@ describe "lib/request", -> .then (resp) -> expect(resp.body).to.eq("derp") + it "sends connection: keep-alive by default", -> + nock("http://localhost:8080") + .matchHeader("connection", "keep-alive") + .get("/foo") + .reply(200, "it worked") + + request.send({}, @fn, { + url: "http://localhost:8080/foo" + cookies: false + }) + .then (resp) -> + expect(resp.body).to.eq("it worked") + context "accept header", -> it "sets to */* by default", -> nock("http://localhost:8080") diff --git a/packages/server/test/unit/server_spec.coffee b/packages/server/test/unit/server_spec.coffee index a80f135fea..5bad0ce26e 100644 --- a/packages/server/test/unit/server_spec.coffee +++ b/packages/server/test/unit/server_spec.coffee @@ -5,13 +5,14 @@ os = require("os") http = require("http") express = require("express") Promise = require("bluebird") +connect = require("@packages/network").connect routes = require("#{root}lib/routes") config = require("#{root}lib/config") logger = require("#{root}lib/logger") Server = require("#{root}lib/server") Socket = require("#{root}lib/socket") fileServer = require("#{root}lib/file_server") -connect = require("#{root}lib/util/connect") +ensureUrl = require("#{root}lib/util/ensure-url") buffers = require("#{root}lib/util/buffers") morganFn = -> @@ -147,7 +148,7 @@ describe "lib/server", -> ) it "resolves with warning if cannot connect to baseUrl", -> - sinon.stub(connect, "ensureUrl").rejects() + sinon.stub(ensureUrl, "isListening").rejects() @server.createServer(@app, {port: @port, baseUrl: "http://localhost:#{@port}"}) .spread (port, warning) => expect(warning.type).to.eq("CANNOT_CONNECT_BASE_URL_WARNING") @@ -265,7 +266,7 @@ describe "lib/server", -> @server.proxyWebsockets(@proxy, "/foo", req, @socket, @head) - expect(@proxy.ws).to.be.calledWith(req, @socket, @head, { + expect(@proxy.ws).to.be.calledWithMatch(req, @socket, @head, { secure: false target: { host: "www.google.com" diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json new file mode 100644 index 0000000000..ffda540c36 --- /dev/null +++ b/packages/server/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./../ts/tsconfig.json", + "include": [ + "lib/*.ts", + "lib/**/*.ts" + ], + "files": [ + "./../ts/index.d.ts" + ] +} diff --git a/packages/ts/index.d.ts b/packages/ts/index.d.ts index b64051b2a7..d1cee3c184 100644 --- a/packages/ts/index.d.ts +++ b/packages/ts/index.d.ts @@ -1,7 +1,68 @@ -// missing type definitions for 3rd party libraries +// missing type definitions for libraries // https://glebbahmutov.com/blog/trying-typescript/#manual-types-for-3rd-party-libraries -// for execa module use @types/execa +declare module '@cypress/get-windows-proxy' { + type ProxyConfig = { + httpProxy: string, + noProxy: string + } + function getWindowsProxy(): Optional + export = getWindowsProxy +} + +declare module 'http' { + import { Socket } from 'net' + import { Url } from 'url' + + type SocketCallback = (err: Optional, sock: Optional) => void + + interface Agent { + addRequest(req: ClientRequest, options: RequestOptions): void + createSocket(req: ClientRequest, options: RequestOptions, cb: SocketCallback): void + createConnection(options: RequestOptions, cb: Optional): void + protocol: 'http:' | 'https:' | string + } + + interface ClientRequest { + _header: { [key: string]:string } + _implicitHeader: () => void + output: string[] + agent: Agent + } + + interface RequestOptions extends ClientRequestArgs { + _agentKey: Optional + host: string + href: string + port: number + proxy: Optional + servername: Optional + socket: Optional + uri: Url + } + + export const CRLF: string +} + +declare module 'https' { + interface Agent { + _sessionCache: { [_agentKey: string]: Buffer } + } +} + +declare module 'net' { + interface Address { + address: string + family: 4 | 6 + } +} + +declare interface Object { + assign(...obj: any[]): any +} + +declare type Optional = T | void + declare module 'plist' { interface Plist { @@ -10,3 +71,11 @@ declare module 'plist' { const plist: Plist export = plist } + +declare module 'proxy-from-env' { + const getProxyForUrl: (url: string) => string +} + +declare interface SymbolConstructor { + for(str: string): SymbolConstructor +} diff --git a/packages/ts/tsconfig.json b/packages/ts/tsconfig.json index f41a73893b..b6741d9fc2 100644 --- a/packages/ts/tsconfig.json +++ b/packages/ts/tsconfig.json @@ -36,8 +36,8 @@ // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ - // "typeRoots": [], /* List of folders to include type definitions from. */ - "types": ["mocha"] /* Type declaration files to be included in compilation. */ + //"typeRoots": [], /* List of folders to include type definitions from. */ + "types": ["mocha", "node"] /* Type declaration files to be included in compilation. */ // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ /* Source Map Options */