Improve Error logging around Cypress verify (#1986)

close #1984 improve error logging for `cypress verify`
close #1985 remove "skipping install" message to local users, keep in CI

cli/__snapshots__/errors_spec.js

@@ -4,6 +4,7 @@ exports['errors individual has the following errors 1'] = [
   "missingApp",
   "missingDependency",
   "versionMismatch",
+  "binaryNotExecutable",
   "unexpected",
   "failedDownload",
   "failedUnzip",

cli/__snapshots__/install_spec.js

@@ -1,13 +1,3 @@
-exports['version already installed 1'] = `
-Cypress 1.2.3 is already installed in /cache/Cypress/1.2.3
-
-Skipping installation:
-
-  Pass the --force option if you'd like to reinstall anyway.
-
-
-`
-
 exports['skip installation 1'] = `
 Note: Skipping binary installation: Environment variable CYPRESS_INSTALL_BINARY = 0.
 
@@ -65,6 +55,7 @@ https://on.cypress.io/installing-cypress
 `
 
 exports['installed version does not match needed version 1'] = `
+
 Cypress x.x.x is already installed in /cache/Cypress/1.2.3
 
 Installing Cypress (version: 1.2.3)
@@ -81,6 +72,7 @@ https://on.cypress.io/installing-cypress
 `
 
 exports['forcing true always installs 1'] = `
+
 Cypress 1.2.3 is already installed in /cache/Cypress/1.2.3
 
 Installing Cypress (version: 1.2.3)
@@ -97,6 +89,7 @@ https://on.cypress.io/installing-cypress
 `
 
 exports['warning installing as global 1'] = `
+
 Cypress x.x.x is already installed in /cache/Cypress/1.2.3
 
 Installing Cypress (version: 1.2.3)
@@ -119,6 +112,7 @@ Installing Cypress (version: 1.2.3)
 `
 
 exports['installing in ci 1'] = `
+
 Cypress x.x.x is already installed in /cache/Cypress/1.2.3
 
 Installing Cypress (version: 1.2.3)
@@ -165,3 +159,20 @@ Cypress Version: 1.2.3
 exports['silent install 1'] = `
 [no output]
 `
+
+exports['version already installed - cypress install 1'] = `
+
+Cypress 1.2.3 is already installed in /cache/Cypress/1.2.3
+
+Skipping installation:
+
+  Pass the --force option if you'd like to reinstall anyway.
+
+`
+
+exports['version already installed - postInstall 1'] = `
+
+Cypress 1.2.3 is already installed in /cache/Cypress/1.2.3
+
+
+`

cli/__snapshots__/verify_spec.js

@@ -13,7 +13,7 @@ Error: No version of Cypress is installed in: /cache/Cypress/1.2.3/Cypress.app
 Please reinstall Cypress by running: cypress install
 ----------
 
-Cypress executable not found at: /cache/Cypress/1.2.3/Cypress.app/executable
+Cypress executable not found at: /cache/Cypress/1.2.3/Cypress.app/Contents/MacOS/Cypress
 ----------
 
 Platform: darwin (Foo-OsVersion)
@@ -37,7 +37,7 @@ Error: No version of Cypress is installed in: /cache/Cypress/1.2.3/Cypress.app
 Please reinstall Cypress by running: cypress install
 ----------
 
-Cypress executable not found at: /cache/Cypress/1.2.3/Cypress.app/executable
+Cypress executable not found at: /cache/Cypress/1.2.3/Cypress.app/Contents/MacOS/Cypress
 ----------
 
 Platform: darwin (Foo-OsVersion)
@@ -78,7 +78,7 @@ Cypress Version: 1.2.3
 
 `
 
-exports['no existing version verified 1'] = `
+exports['current version has not been verified 1'] = `
 It looks like this is your first time using Cypress: 1.2.3
 
  ✔  Verified Cypress! /cache/Cypress/1.2.3/Cypress.app
@@ -87,40 +87,10 @@ Opening Cypress...
 
 `
 
-exports['current version has not been verified 1'] = `
-Found binary version different version installed in: /cache/Cypress/1.2.3/Cypress.app
-
-⚠ Warning: Binary version different version does not match the expected package version 1.2.3
-
-  These versions may not work properly together.
-
-It looks like this is your first time using Cypress: different version
-
- ✔  Verified Cypress! /cache/Cypress/1.2.3/Cypress.app
-
-Opening Cypress...
-
-`
-
-exports['current version has not been verified 2'] = `
-Found binary version 9.8.7 installed in: /cache/Cypress/1.2.3/Cypress.app
-
-⚠ Warning: Binary version 9.8.7 does not match the expected package version 1.2.3
-
-  These versions may not work properly together.
-
-It looks like this is your first time using Cypress: 9.8.7
-
- ✔  Verified Cypress! /cache/Cypress/1.2.3/Cypress.app
-
-Opening Cypress...
-
-`
-
 exports['no welcome message 1'] = `
-Found binary version different version installed in: /cache/Cypress/1.2.3/Cypress.app
+Found binary version 7.8.9 installed in: /cache/Cypress/1.2.3/Cypress.app
 
-⚠ Warning: Binary version different version does not match the expected package version 1.2.3
+⚠ Warning: Binary version 7.8.9 does not match the expected package version 1.2.3
 
   These versions may not work properly together.
 
@@ -175,45 +145,54 @@ Opening Cypress...
 `
 
 exports['darwin: error when invalid CYPRESS_RUN_BINARY 1'] = `
-Note: You have set the environment variable: CYPRESS_RUN_BINARY=/custom:
+Note: You have set the environment variable: CYPRESS_RUN_BINARY=/custom/:
 
       This overrides the default Cypress binary path used.
 
-Error: Could not run binary set by environment variable CYPRESS_RUN_BINARY=/custom
+Error: Could not run binary set by environment variable CYPRESS_RUN_BINARY=/custom/
 
 Ensure the environment variable is a path to the Cypress binary, matching **/Contents/MacOS/Cypress
 ----------
 
+ENOENT: no such file or directory, stat '/custom/'
+----------
+
 Platform: darwin (Foo-OsVersion)
 Cypress Version: 1.2.3
 
 `
 
 exports['linux: error when invalid CYPRESS_RUN_BINARY 1'] = `
-Note: You have set the environment variable: CYPRESS_RUN_BINARY=/custom:
+Note: You have set the environment variable: CYPRESS_RUN_BINARY=/custom/:
 
       This overrides the default Cypress binary path used.
 
-Error: Could not run binary set by environment variable CYPRESS_RUN_BINARY=/custom
+Error: Could not run binary set by environment variable CYPRESS_RUN_BINARY=/custom/
 
 Ensure the environment variable is a path to the Cypress binary, matching **/Cypress
 ----------
 
+ENOENT: no such file or directory, stat '/custom/'
+----------
+
 Platform: linux (Foo-OsVersion)
 Cypress Version: 1.2.3
 
 `
 
 exports['win32: error when invalid CYPRESS_RUN_BINARY 1'] = `
-Note: You have set the environment variable: CYPRESS_RUN_BINARY=/custom:
+Note: You have set the environment variable: CYPRESS_RUN_BINARY=/custom/:
 
       This overrides the default Cypress binary path used.
 
-Error: Could not run binary set by environment variable CYPRESS_RUN_BINARY=/custom
+Error: Could not run binary set by environment variable CYPRESS_RUN_BINARY=/custom/
 
 Ensure the environment variable is a path to the Cypress binary, matching **/Cypress.exe
 ----------
 
+ENOENT: no such file or directory, stat '/custom/'
+----------
+
 Platform: win32 (Foo-OsVersion)
 Cypress Version: 1.2.3
 
@@ -222,3 +201,68 @@ Cypress Version: 1.2.3
 exports['silent verify 1'] = `
 [no output]
 `
+
+exports['no Cypress executable 1'] = `
+Error: No version of Cypress is installed in: /cache/Cypress/1.2.3/Cypress.app
+
+Please reinstall Cypress by running: cypress install
+----------
+
+Cypress executable not found at: /cache/Cypress/1.2.3/Cypress.app/Contents/MacOS/Cypress
+----------
+
+Platform: darwin (Foo-OsVersion)
+Cypress Version: 1.2.3
+
+`
+
+exports['Cypress non-executable permissions 1'] = `
+Error: Cypress cannot run because the binary does not have executable permissions: /cache/Cypress/1.2.3/Cypress.app/Contents/MacOS/Cypress
+
+Reasons this may happen:
+
+- node was installed as 'root' or with 'sudo'
+- the cypress npm package as 'root' or with 'sudo'
+
+Please check that you have the appropriate user permissions.
+----------
+
+Platform: darwin (Foo-OsVersion)
+Cypress Version: 1.2.3
+
+`
+
+exports['different version installed 1'] = `
+Found binary version 7.8.9 installed in: /cache/Cypress/1.2.3/Cypress.app
+
+⚠ Warning: Binary version 7.8.9 does not match the expected package version 1.2.3
+
+  These versions may not work properly together.
+
+It looks like this is your first time using Cypress: 7.8.9
+
+ ✔  Verified Cypress! /cache/Cypress/1.2.3/Cypress.app
+
+Opening Cypress...
+
+`
+
+exports['fails with no stderr 1'] = `
+Error: Cypress failed to start.
+
+This is usually caused by a missing library or dependency.
+
+The error below should indicate which dependency is missing.
+
+https://on.cypress.io/required-dependencies
+
+If you are using Docker, we provide containers with all required dependencies installed.
+----------
+
+Error: EPERM NOT PERMITTED
+----------
+
+Platform: darwin (Foo-OsVersion)
+Cypress Version: 1.2.3
+
+`

cli/lib/errors.js

@@ -32,6 +32,19 @@ const missingApp = (binaryDir) => ({
   `,
 })
 
+const binaryNotExecutable = (executable) => ({
+  description: `Cypress cannot run because the binary does not have executable permissions: ${executable}`,
+  solution: stripIndent`\n
+    Reasons this may happen:
+      
+    - node was installed as 'root' or with 'sudo'
+    - the cypress npm package as 'root' or with 'sudo'
+    
+    Please check that you have the appropriate user permissions.
+  `,
+})
+
+
 const nonZeroExitCodeXvfb = {
   description: 'XVFB exited with a non zero exit code.',
   solution: stripIndent`
@@ -205,6 +218,7 @@ module.exports = {
     missingApp,
     missingDependency,
     versionMismatch,
+    binaryNotExecutable,
     unexpected,
     failedDownload,
     failedUnzip,

cli/lib/tasks/install.js

@@ -17,12 +17,13 @@ const logger = require('../logger')
 const { throwFormErrorText, errors } = require('../errors')
 
 const alreadyInstalledMsg = () => {
-  logger.log(stripIndent`    
-    Skipping installation:
-
-      Pass the ${chalk.yellow('--force')} option if you'd like to reinstall anyway.
-  `)
-  logger.log()
+  if (!util.isPostInstall()) {
+    logger.log(stripIndent`    
+      Skipping installation:
+  
+        Pass the ${chalk.yellow('--force')} option if you'd like to reinstall anyway.
+    `)
+  }
 }
 
 const displayCompletionMsg = () => {
@@ -201,10 +202,12 @@ const start = (options = {}) => {
       return true
     }
 
-    // debug('installed version is', binaryVersion, 'version needed is', needVersion)
+    debug('installed version is', binaryVersion, 'version needed is', needVersion)
+
+    logger.log()
     logger.log(stripIndent`
-    Cypress ${chalk.green(binaryVersion)} is already installed in ${chalk.cyan(installDir)}
-    `)
+      Cypress ${chalk.green(binaryVersion)} is already installed in ${chalk.cyan(installDir)}
+      `)
     logger.log()
 
     if (options.force) {

cli/lib/tasks/state.js

@@ -2,7 +2,6 @@ const _ = require('lodash')
 const os = require('os')
 const path = require('path')
 const debug = require('debug')('cypress:cli')
-const cachedir = require('cachedir')
 
 const fs = require('../fs')
 const util = require('../util')
@@ -52,7 +51,7 @@ const getVersionDir = (version = util.pkgVersion()) => {
 }
 
 const getCacheDir = () => {
-  let cache_directory = cachedir('Cypress')
+  let cache_directory = util.getCacheDir()
   if (util.getEnv('CYPRESS_CACHE_FOLDER')) {
     const envVarCacheDir = util.getEnv('CYPRESS_CACHE_FOLDER')
     debug('using environment variable CYPRESS_CACHE_FOLDER %s', envVarCacheDir)
@@ -120,6 +119,7 @@ const getPathToExecutable = (binaryDir) => {
 
 const getBinaryPkgVersionAsync = (binaryDir) => {
   const pathToPackageJson = getBinaryPkgPath(binaryDir)
+  debug('Reading binary package.json from:', pathToPackageJson)
   return fs.pathExistsAsync(pathToPackageJson)
   .then((exists) => {
     if (!exists) {

cli/lib/tasks/verify.js

@@ -1,7 +1,5 @@
 const _ = require('lodash')
-const cp = require('child_process')
 const chalk = require('chalk')
-const path = require('path')
 const Listr = require('listr')
 const debug = require('debug')('cypress:cli')
 const verbose = require('@cypress/listr-verbose-renderer')
@@ -11,90 +9,71 @@ const logSymbols = require('log-symbols')
 
 
 const { throwFormErrorText, errors } = require('../errors')
-const fs = require('../fs')
 const util = require('../util')
 const logger = require('../logger')
 const xvfb = require('../exec/xvfb')
 const state = require('./state')
 
-const verificationError = (message) => {
-  return _.extend(new Error(''), { name: '', message, isVerificationError: true })
-}
-
-const xvfbError = (message) => {
-  return _.extend(new Error(''), { name: '', message, isXvfbError: true })
-}
-
-const isMissingExecutable = (binaryDir) => {
+const checkExecutable = (binaryDir) => {
   const executable = state.getPathToExecutable(binaryDir)
   debug('checking if executable exists', executable)
-  return fs.pathExistsAsync(executable)
-  .then((exists) => {
-    if (!exists) {
-      return throwFormErrorText(errors.missingApp(binaryDir))(stripIndent`
+  return util.isExecutableAsync(executable)
+  .then((isExecutable) => {
+    debug('Binary is executable? :', isExecutable)
+    if (!isExecutable) {
+      return throwFormErrorText(errors.binaryNotExecutable(executable))()
+    }
+  })
+  .catch({ code: 'ENOENT' }, () => {
+    return throwFormErrorText(errors.missingApp(binaryDir))(stripIndent`
       Cypress executable not found at: ${chalk.cyan(executable)}
     `)
-    }
   })
 }
 
 const runSmokeTest = (binaryDir) => {
   debug('running smoke test')
-  let stderr = ''
-  let stdout = ''
   const cypressExecPath = state.getPathToExecutable(binaryDir)
   debug('using Cypress executable %s', cypressExecPath)
 
-  // TODO switch to execa for this?
-  const spawn = () => {
-    return new Promise((resolve, reject) => {
-      const random = _.random(0, 1000)
-      const args = ['--smoke-test', `--ping=${random}`]
-      const smokeTestCommand = `${cypressExecPath} ${args.join(' ')}`
-      debug('smoke test command:', smokeTestCommand)
-      const child = cp.spawn(cypressExecPath, args)
-
-      child.stderr.on('data', (data) => {
-        stderr += data.toString()
-      })
-
-      child.stdout.on('data', (data) => {
-        stdout += data.toString()
-      })
-
-      child.on('error', reject)
-
-      child.on('close', (code) => {
-        if (code === 0) {
-          const smokeTestReturned = stdout.trim()
-          debug('smoke test output "%s"', smokeTestReturned)
-
-          if (!util.stdoutLineMatches(String(random), smokeTestReturned)) {
-            return reject(new Error(stripIndent`
-              Smoke test returned wrong code.
-
-              Command was: ${smokeTestCommand}
-
-              Returned: ${smokeTestReturned}
-            `))
-          }
-
-          return resolve()
-        }
-
-        reject(verificationError(stderr))
-      })
-    })
-  }
-
   const onXvfbError = (err) => {
     debug('caught xvfb error %s', err.message)
-    throw xvfbError(`Caught error trying to run XVFB: "${err.message}"`)
+    return throwFormErrorText(errors.missingXvfb)(`Caught error trying to run XVFB: "${err.message}"`)
+  }
+
+  const onSmokeTestError = (err) => {
+    debug('Smoke test failed:', err)
+    return throwFormErrorText(errors.missingDependency)(err.stderr || err.message)
   }
 
   const needsXvfb = xvfb.isNeeded()
   debug('needs XVFB?', needsXvfb)
 
+  const spawn = () => {
+    const random = _.random(0, 1000)
+    const args = ['--smoke-test', `--ping=${random}`]
+    const smokeTestCommand = `${cypressExecPath} ${args.join(' ')}`
+    debug('smoke test command:', smokeTestCommand)
+
+    return Promise.resolve(util.exec(cypressExecPath, args))
+    .catch(onSmokeTestError)
+    .then((result) => {
+      const smokeTestReturned = result.stdout
+      debug('smoke test stdout "%s"', smokeTestReturned)
+
+      if (!util.stdoutLineMatches(String(random), smokeTestReturned)) {
+        debug('Smoke test failed:', result)
+        throw new Error(stripIndent`
+          Smoke failed.
+
+          Command was: ${smokeTestCommand}
+
+          Returned: ${smokeTestReturned}
+        `)
+      }
+    })
+  }
+
   if (needsXvfb) {
     return xvfb.start()
     .catch(onXvfbError)
@@ -155,8 +134,6 @@ function testBinary (version, binaryDir) {
             rendererOptions.renderer
           )
         })
-        .catch({ isXvfbError: true }, throwFormErrorText(errors.missingXvfb))
-        .catch({ isVerificationError: true }, throwFormErrorText(errors.missingDependency))
       },
     },
   ], rendererOptions)
@@ -200,46 +177,47 @@ const start = (options = {}) => {
     welcomeMessage: true,
   })
 
-  const checkEnvVar = () => {
-    debug('checking environment variables')
-    if (util.getEnv('CYPRESS_RUN_BINARY')) {
-      const envBinaryPath = path.resolve(util.getEnv('CYPRESS_RUN_BINARY'))
-      debug('CYPRESS_RUN_BINARY exists, =', envBinaryPath)
-      logger.log(stripIndent`
+  const parseBinaryEnvVar = () => {
+    const envBinaryPath = util.getEnv('CYPRESS_RUN_BINARY')
+    debug('CYPRESS_RUN_BINARY exists, =', envBinaryPath)
+    logger.log(stripIndent`
         ${chalk.yellow('Note:')} You have set the environment variable: ${chalk.white('CYPRESS_RUN_BINARY=')}${chalk.cyan(envBinaryPath)}:
         
               This overrides the default Cypress binary path used.
         `)
-      logger.log()
+    logger.log()
 
-      return util.isExecutableAsync(envBinaryPath)
-      .then((isExecutable) => {
-        debug('CYPRESS_RUN_BINARY is executable? :', isExecutable)
-        if (!isExecutable) {
-          return throwFormErrorText(errors.CYPRESS_RUN_BINARY.notValid(envBinaryPath))(stripIndent`
+    return util.isExecutableAsync(envBinaryPath)
+    .then((isExecutable) => {
+      debug('CYPRESS_RUN_BINARY is executable? :', isExecutable)
+      if (!isExecutable) {
+        return throwFormErrorText(errors.CYPRESS_RUN_BINARY.notValid(envBinaryPath))(stripIndent`
           The supplied binary path is not executable
           `)
-        }
-      })
-      .then(() => state.parseRealPlatformBinaryFolderAsync(envBinaryPath))
-      .then((envBinaryDir) => {
-        if (!envBinaryDir) {
-          return throwFormErrorText(errors.CYPRESS_RUN_BINARY.notValid(envBinaryPath))()
-        }
-        debug('CYPRESS_RUN_BINARY has binaryDir:', envBinaryDir)
+      }
+    })
+    .then(() => state.parseRealPlatformBinaryFolderAsync(envBinaryPath))
+    .then((envBinaryDir) => {
+      if (!envBinaryDir) {
+        return throwFormErrorText(errors.CYPRESS_RUN_BINARY.notValid(envBinaryPath))()
+      }
+      debug('CYPRESS_RUN_BINARY has binaryDir:', envBinaryDir)
 
-        binaryDir = envBinaryDir
-      })
-      .catch({ code: 'ENOENT' }, (err) => {
-        return throwFormErrorText(errors.CYPRESS_RUN_BINARY.notValid(envBinaryPath))(err.message)
-      })
-    }
-    return Promise.resolve()
+      binaryDir = envBinaryDir
+    })
+    .catch({ code: 'ENOENT' }, (err) => {
+      return throwFormErrorText(errors.CYPRESS_RUN_BINARY.notValid(envBinaryPath))(err.message)
+    })
   }
 
 
-  return checkEnvVar()
-  .then(() => isMissingExecutable(binaryDir))
+  return Promise.try(() => {
+    debug('checking environment variables')
+    if (util.getEnv('CYPRESS_RUN_BINARY')) {
+      return parseBinaryEnvVar()
+    }
+  })
+  .then(() => checkExecutable(binaryDir))
   .tap(() => debug('binaryDir is ', binaryDir))
   .then(() => state.getBinaryPkgVersionAsync(binaryDir))
   .then((binaryVersion) => {

cli/lib/util.js

@@ -4,9 +4,11 @@ const os = require('os')
 const tty = require('tty')
 const path = require('path')
 const isCi = require('is-ci')
+const execa = require('execa')
 const getos = require('getos')
 const chalk = require('chalk')
 const Promise = require('bluebird')
+const cachedir = require('cachedir')
 const executable = require('executable')
 const supportsColor = require('supports-color')
 const isInstalledGlobally = require('is-installed-globally')
@@ -229,6 +231,16 @@ const util = {
 
   },
 
+  getCacheDir () {
+    return cachedir('Cypress')
+  },
+
+  isPostInstall () {
+    return process.env.npm_lifecycle_event === 'postinstall'
+  },
+
+  exec: execa,
+
   stdoutLineMatches,
 }
 

cli/package.json

@@ -52,6 +52,7 @@
     "commander": "2.11.0",
     "common-tags": "1.4.0",
     "debug": "3.1.0",
+    "execa": "0.10.0",
     "executable": "4.1.1",
     "extract-zip": "1.6.6",
     "fs-extra": "4.0.1",

cli/test/lib/tasks/install_spec.js

@@ -45,6 +45,7 @@ describe('/lib/tasks/install', function () {
 
       // sinon.stub(os, 'tmpdir').returns('/tmp')
       sinon.stub(util, 'isCi').returns(false)
+      sinon.stub(util, 'isPostInstall').returns(false)
       sinon.stub(util, 'pkgVersion').returns(packageVersion)
       sinon.stub(download, 'start').resolves(packageVersion)
       sinon.stub(unzip, 'start').resolves()
@@ -134,18 +135,37 @@ describe('/lib/tasks/install', function () {
         beforeEach(function () {
           state.getBinaryPkgVersionAsync.resolves(packageVersion)
 
-          return install.start()
         })
 
-        it('logs noop message', function () {
-          expect(state.getBinaryPkgVersionAsync).to.be.calledWith('/cache/Cypress/1.2.3/Cypress.app')
+        it('doesn\'t attempt to download', function () {
+          return install.start()
+          .then(() => {
+            expect(download.start).not.to.be.called
+            expect(state.getBinaryPkgVersionAsync).to.be.calledWith('/cache/Cypress/1.2.3/Cypress.app')
+          })
 
-          expect(download.start).not.to.be.called
+        })
 
-          snapshot(
-            'version already installed',
-            normalize(this.stdout.toString())
-          )
+        it('logs \'skipping install\' when explicit cypress install', function () {
+          return install.start()
+          .then(() => {
+            return snapshot(
+              'version already installed - cypress install',
+              normalize(this.stdout.toString())
+            )
+          })
+
+        })
+
+        it('logs when already installed when run from postInstall', function () {
+          util.isPostInstall.returns(true)
+          return install.start()
+          .then(() => {
+            snapshot(
+              'version already installed - postInstall',
+              normalize(this.stdout.toString())
+            )
+          })
         })
       })
 

cli/test/lib/tasks/state_spec.js

@@ -2,25 +2,22 @@ require('../../spec_helper')
 
 const os = require('os')
 const path = require('path')
-const proxyquire = require('proxyquire')
 const Promise = require('bluebird')
+const proxyquire = require('proxyquire')
 
 const fs = require(`${lib}/fs`)
 const logger = require(`${lib}/logger`)
 const util = require(`${lib}/util`)
+const state = require(`${lib}/tasks/state`)
 
-const fakeCachedir = (cacheName) => path.join('.cache', cacheName)
-
-const cacheDir = path.join(fakeCachedir('Cypress'))
+const cacheDir = path.join('.cache/Cypress')
 const versionDir = path.join(cacheDir, '1.2.3')
 const binaryDir = path.join(versionDir, 'Cypress.app')
 const binaryPkgPath = path.join(binaryDir, 'Contents', 'Resources', 'app', 'package.json')
 
-let state
-
 describe('lib/tasks/state', function () {
   beforeEach(function () {
-    state = proxyquire(`${lib}/tasks/state`, { cachedir: fakeCachedir })
+    sinon.stub(util, 'getCacheDir').returns(cacheDir)
     logger.reset()
     sinon.stub(process, 'exit')
     sinon.stub(util, 'pkgVersion').returns('1.2.3')
@@ -87,7 +84,7 @@ describe('lib/tasks/state', function () {
     })
 
     it('resolves path on windows', function () {
-      const state = proxyquire(`${lib}/tasks/state`, { path: path.win32, cachedir: fakeCachedir })
+      const state = proxyquire(`${lib}/tasks/state`, { path: path.win32 })
       os.platform.returns('win32')
       const pathToExec = state.getBinaryDir()
       expect(pathToExec).to.be.equal(path.win32.join(versionDir, 'Cypress'))

cli/test/lib/tasks/verify_spec.js

@@ -2,28 +2,478 @@ require('../../spec_helper')
 
 const _ = require('lodash')
 const os = require('os')
-const cp = require('child_process')
-const EE = require('events').EventEmitter
+const mockfs = require('mock-fs')
+const _snapshot = require('snap-shot-it')
 const Promise = require('bluebird')
-const snapshot = require('snap-shot-it')
 const { stripIndent } = require('common-tags')
 
 const fs = require(`${lib}/fs`)
 const util = require(`${lib}/util`)
 const logger = require(`${lib}/logger`)
 const xvfb = require(`${lib}/exec/xvfb`)
-const state = require(`${lib}/tasks/state`)
 const verify = require(`${lib}/tasks/verify`)
 
-const stdout = require('../../support/stdout')
+const Stdout = require('../../support/stdout')
 const normalize = require('../../support/normalize')
 
 const packageVersion = '1.2.3'
-const executablePath = '/cache/Cypress/1.2.3/Cypress.app/executable'
-const binaryDir = '/cache/Cypress/1.2.3/Cypress.app'
+const cacheDir = '/cache/Cypress'
+const executablePath = '/cache/Cypress/1.2.3/Cypress.app/Contents/MacOS/Cypress'
+const binaryStatePath = '/cache/Cypress/1.2.3/Cypress.app/binary_state.json'
+
+let stdout
+let spawnedProcess
+
+/* eslint-disable no-octal */
+
+const snapshot = (...args) => {
+  mockfs.restore()
+  _snapshot(...args)
+}
+
+context('lib/tasks/verify', () => {
+  require('mocha-banner').register()
+
+  beforeEach(() => {
+
+    stdout = Stdout.capture()
+    spawnedProcess = {
+      code: 0,
+      stderr: sinon.stub(),
+      stdout: '222',
+    }
+
+    os.platform.returns('darwin')
+
+    sinon.stub(util, 'getCacheDir').returns(cacheDir)
+    sinon.stub(util, 'isCi').returns(false)
+    sinon.stub(util, 'pkgVersion').returns(packageVersion)
+    sinon.stub(util, 'exec')
 
 
-const slice = (str) => {
+    sinon.stub(xvfb, 'start').resolves()
+    sinon.stub(xvfb, 'stop').resolves()
+    sinon.stub(xvfb, 'isNeeded').returns(false)
+    sinon.stub(Promise.prototype, 'delay').resolves()
+
+    sinon.stub(_, 'random').returns('222')
+
+    util.exec.withArgs(executablePath, [
+      '--smoke-test',
+      '--ping=222',
+    ]).resolves(spawnedProcess)
+  })
+
+  afterEach(() => {
+    Stdout.restore()
+  })
+
+  it('logs error and exits when no version of Cypress is installed', () => {
+    mockfs({})
+    return verify.start()
+    .then(() => {
+      throw new Error('should have caught error')
+    })
+    .catch((err) => {
+      logger.error(err)
+
+      snapshot(
+        'no version of Cypress installed',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  it('is noop when binary is already verified', () => {
+    // make it think the executable exists and is verified
+    createfs({
+      alreadyVerified: true,
+      executable: mockfs.file({ mode: 0777 }),
+      packageVersion,
+    })
+    return verify.start()
+    .then(() => {
+      // nothing should have been logged to stdout
+      // since no verification took place
+      expect(stdout.toString()).to.be.empty
+
+      expect(util.exec).not.to.be.called
+    })
+  })
+
+  it('logs warning when installed version does not match verified version', () => {
+
+    createfs({
+      alreadyVerified: true,
+      executable: mockfs.file({ mode: 0777 }),
+      packageVersion: 'bloop',
+    })
+
+    return verify.start()
+    .then(() => {
+      throw new Error('should have caught error')
+    })
+    .catch(() => {
+      return snapshot(
+        'warning installed version does not match verified version',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  it('logs error and exits when executable cannot be found', () => {
+
+    return verify.start()
+    .then(() => {
+      throw new Error('should have caught error')
+    })
+    .catch((err) => {
+      logger.error(err)
+
+      snapshot(
+        'executable cannot be found',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  describe('with force: true', () => {
+    beforeEach(() => {
+      createfs({
+        alreadyVerified: true,
+        executable: mockfs.file({ mode: 0777 }),
+        packageVersion,
+      })
+    })
+
+    it('shows full path to executable when verifying', () => {
+
+      return verify.start({ force: true })
+      .then(() => {
+        snapshot(
+          'verification with executable',
+          normalize(stdout.toString())
+        )
+      })
+    })
+
+    it('clears verified version from state if verification fails', () => {
+
+      util.exec.restore()
+      sinon.stub(util, 'exec').withArgs(executablePath).rejects({
+        code: 1,
+        stderr: 'an error about dependencies',
+      })
+
+      return verify.start({ force: true })
+      .then(() => { throw new Error('Should have thrown') })
+      .catch((err) => {
+        logger.error(err)
+      })
+      .then(() => fs.pathExistsAsync(binaryStatePath))
+      .then((exists) => expect(exists).to.eq(false))
+      .then(() => {
+        return snapshot(
+          'fails verifying Cypress',
+          normalize(slice(stdout.toString()))
+        )
+      })
+    })
+  })
+
+  describe('smoke test with DEBUG output', () => {
+    beforeEach(() => {
+
+      const stdoutWithDebugOutput = stripIndent`
+        some debug output
+        date: more debug output
+        222
+        after that more text
+      `
+      util.exec.withArgs(executablePath).resolves({
+        stdout: stdoutWithDebugOutput,
+      })
+
+      createfs({
+        alreadyVerified: false,
+        executable: mockfs.file({ mode: 0777 }),
+        packageVersion,
+      })
+    })
+
+    it('finds ping value in the verbose output', () => {
+
+      return verify.start()
+      .then(() => {
+        snapshot(
+          'verbose stdout output',
+          normalize(stdout.toString())
+        )
+      })
+    })
+  })
+
+  it('logs an error if Cypress executable does not exist', () => {
+    createfs({
+      alreadyVerified: false,
+      executable: false,
+      packageVersion,
+    })
+    return verify.start()
+    .then(() => { throw new Error('Should have thrown') })
+    .catch((err) => {
+      stdout = Stdout.capture()
+      logger.error(err)
+
+      return snapshot(
+        'no Cypress executable',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  it('logs an error if Cypress executable does not have permissions', () => {
+    mockfs.restore()
+    createfs({
+      alreadyVerified: false,
+      executable: mockfs.file({ mode: 0666 }),
+      packageVersion,
+    })
+    return verify.start()
+    .then(() => { throw new Error('Should have thrown') })
+    .catch((err) => {
+      stdout = Stdout.capture()
+      logger.error(err)
+
+      return snapshot(
+        'Cypress non-executable permissions',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  it('logs and runs when current version has not been verified', () => {
+    createfs({
+      alreadyVerified: false,
+      executable: mockfs.file({ mode: 0777 }),
+      packageVersion,
+    })
+    return verify.start()
+    .then(() => {
+      return snapshot(
+        'current version has not been verified',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  it('logs and runs when installed version is different than package version', () => {
+    createfs({
+      alreadyVerified: false,
+      executable: mockfs.file({ mode: 0777 }),
+      packageVersion: '7.8.9',
+    })
+    return verify.start()
+    .then(() => {
+      return snapshot(
+        'different version installed',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  it('is silent when logLevel is silent', () => {
+    createfs({
+      alreadyVerified: false,
+      executable: mockfs.file({ mode: 0777 }),
+      packageVersion,
+    })
+
+    process.env.npm_config_loglevel = 'silent'
+
+    return verify.start()
+    .then(() => {
+      return snapshot(
+        'silent verify',
+        normalize(`[no output]${stdout.toString()}`)
+      )
+    })
+  })
+
+  it('turns off Opening Cypress...', () => {
+    createfs({
+      alreadyVerified: true,
+      executable: mockfs.file({ mode: 0777 }),
+      packageVersion: '7.8.9',
+    })
+
+    return verify.start({
+      welcomeMessage: false,
+    })
+    .then(() => {
+      return snapshot(
+        'no welcome message',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  it('logs error when fails smoke test unexpectedly without stderr', () => {
+    createfs({
+      alreadyVerified: false,
+      executable: mockfs.file({ mode: 0777 }),
+      packageVersion,
+    })
+
+    util.exec.restore()
+    sinon.stub(util, 'exec').rejects({
+      stderr: '',
+      stdout: '',
+      message: 'Error: EPERM NOT PERMITTED',
+    })
+
+    return verify.start()
+    .then(() => { throw new Error('Should have thrown') })
+    .catch((err) => {
+      stdout = Stdout.capture()
+      logger.error(err)
+      return snapshot(
+        'fails with no stderr',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  describe('on linux', () => {
+    beforeEach(() => {
+      xvfb.isNeeded.returns(true)
+      createfs({
+        alreadyVerified: false,
+        executable: mockfs.file({ mode: 0777 }),
+        packageVersion,
+      })
+    })
+
+    it('starts xvfb', () => {
+      return verify.start()
+      .then(() => {
+        expect(xvfb.start).to.be.called
+      })
+    })
+
+    it('stops xvfb on spawned process close', () => {
+      return verify.start()
+      .then(() => {
+        expect(xvfb.stop).to.be.called
+      })
+    })
+
+    it('logs error and exits when starting xvfb fails', () => {
+      const err = new Error('test without xvfb')
+      err.stack = 'xvfb? no dice'
+      xvfb.start.rejects(err)
+      return verify.start()
+      .catch((err) => {
+        expect(xvfb.stop).to.be.calledOnce
+
+        logger.error(err)
+
+        snapshot(
+          'xvfb fails',
+          normalize(slice(stdout.toString()))
+        )
+      })
+    })
+  })
+
+  describe('when running in CI', () => {
+    beforeEach(() => {
+      createfs({
+        alreadyVerified: false,
+        executable: mockfs.file({ mode: 0777 }),
+        packageVersion,
+      })
+      util.isCi.returns(true)
+
+      return verify.start({ force: true })
+    })
+
+    it('uses verbose renderer', () => {
+      snapshot(
+        'verifying in ci',
+        normalize(stdout.toString())
+      )
+    })
+  })
+
+  describe('when env var CYPRESS_RUN_BINARY', () => {
+
+    it('can validate and use executable', () => {
+      const envBinaryPath = '/custom/Contents/MacOS/Cypress'
+      const realEnvBinaryPath = `/real${envBinaryPath}`
+
+      process.env.CYPRESS_RUN_BINARY = envBinaryPath
+      createfs({
+        alreadyVerified: false,
+        executable: mockfs.file({ mode: 0777 }),
+        packageVersion,
+        customDir: '/real/custom',
+      })
+      util.exec.withArgs(realEnvBinaryPath, [
+        '--smoke-test',
+        '--ping=222',
+      ]).resolves(spawnedProcess)
+      return verify.start()
+      .then(() => {
+        expect(util.exec.firstCall.args[0]).to.equal(realEnvBinaryPath)
+        snapshot('valid CYPRESS_RUN_BINARY', normalize(stdout.toString()))
+      })
+    })
+
+
+    ;['darwin', 'linux', 'win32'].forEach((platform) => it('can log error to user', () => {
+      process.env.CYPRESS_RUN_BINARY = '/custom/'
+      os.platform.returns(platform)
+      return verify.start()
+      .then(() => { throw new Error('Should have thrown') })
+      .catch((err) => {
+        logger.error(err)
+        snapshot(
+          `${platform}: error when invalid CYPRESS_RUN_BINARY`,
+          normalize(stdout.toString())
+        )
+      })
+    }))
+  })
+})
+
+function createfs ({ alreadyVerified, executable, packageVersion, customDir }) {
+  let mockFiles = {
+    [customDir ? customDir : '/cache/Cypress/1.2.3/Cypress.app']: {
+      'binary_state.json': `{"verified": ${alreadyVerified}}`,
+      Contents: {
+        MacOS: executable ? {
+          Cypress: executable,
+        } : {},
+        Resources: {
+          app: {
+            'package.json': `{"version": "${packageVersion}"}`,
+          },
+        },
+      },
+    },
+  }
+
+  if (customDir) {
+    mockFiles['/custom/Contents/MacOS/Cypress'] = mockfs.symlink({
+      path: '/real/custom/Contents/MacOS/Cypress',
+      mode: 0777,
+    })
+  }
+  return mockfs(mockFiles)
+}
+
+function slice (str) {
   // strip answer and split by new lines
   str = str.split('\n')
 
@@ -39,396 +489,3 @@ const slice = (str) => {
 
   return str.join('\n')
 }
-
-context('lib/tasks/verify', function () {
-  require('mocha-banner').register()
-
-  beforeEach(function () {
-    this.stdout = stdout.capture()
-    this.cpstderr = sinon.stub(new EE())
-    this.cpstderr.on.returns(undefined)
-    this.cpstdout = sinon.stub(new EE())
-    this.cpstdout.on.returns(undefined)
-
-    sinon.stub(util, 'isCi').returns(false)
-    sinon.stub(util, 'pkgVersion').returns(packageVersion)
-    os.platform.returns('darwin')
-    this.spawnedProcess = _.extend(new EE(), {
-      on: sinon.stub(),
-      unref: sinon.stub(),
-      stderr: this.cpstderr,
-      stdout: this.cpstdout,
-    })
-    this.spawnedProcess.on.withArgs('error')
-    this.spawnedProcess.on.withArgs('close').yieldsAsync(0)
-
-    sinon.stub(cp, 'spawn').returns(this.spawnedProcess)
-    sinon.stub(state, 'getPathToExecutable').returns(executablePath)
-    sinon.stub(state, 'getBinaryDir').returns(binaryDir)
-    sinon.stub(xvfb, 'start').resolves()
-    sinon.stub(xvfb, 'stop').resolves()
-    sinon.stub(xvfb, 'isNeeded').returns(false)
-    sinon.stub(Promise.prototype, 'delay').resolves()
-    sinon.stub(state, 'writeBinaryVerifiedAsync').resolves()
-    sinon.stub(state, 'clearBinaryStateAsync').resolves()
-    sinon.stub(fs, 'realpathAsync')
-  })
-
-  afterEach(function () {
-    stdout.restore()
-  })
-
-  it('logs error and exits when no version of Cypress is installed', function () {
-    const ctx = this
-    sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(false)
-    return verify.start()
-    .then(() => {
-      throw new Error('should have caught error')
-    })
-    .catch((err) => {
-      logger.error(err)
-
-      snapshot(
-        'no version of Cypress installed',
-        normalize(ctx.stdout.toString())
-      )
-    })
-  })
-
-  it('is noop when binary is already verified', function () {
-    const ctx = this
-
-    // make it think the executable exists and is verified
-    sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-    sinon.stub(state, 'getBinaryPkgVersionAsync').resolves(packageVersion)
-    sinon.stub(state, 'getBinaryVerifiedAsync').resolves(true)
-    return verify.start()
-    .then(() => {
-      // nothing should have been logged to stdout
-      // since no verification took place
-      expect(ctx.stdout.toString()).to.be.empty
-
-      expect(cp.spawn).not.to.be.called
-    })
-  })
-
-  it('logs warning when installed version does not match verified version', function () {
-    const ctx = this
-    sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-    sinon.stub(state, 'getBinaryPkgVersionAsync').resolves('bloop')
-    // force this to throw to short circuit actually running smoke test
-    sinon.stub(state, 'getBinaryVerifiedAsync').rejects(new Error())
-
-    return verify.start()
-    .then(() => {
-      throw new Error('should have caught error')
-    })
-    .catch(() => {
-      snapshot(
-        'warning installed version does not match verified version',
-        normalize(ctx.stdout.toString())
-      )
-    })
-  })
-
-  it('logs error and exits when executable cannot be found', function () {
-    const ctx = this
-    sinon.stub(state, 'getBinaryPkgVersionAsync').resolves(packageVersion)
-
-    return verify.start()
-    .then(() => {
-      throw new Error('should have caught error')
-    })
-    .catch((err) => {
-      logger.error(err)
-
-      snapshot(
-        'executable cannot be found',
-        normalize(ctx.stdout.toString())
-      )
-    })
-  })
-
-  describe('with force: true', function () {
-    beforeEach(function () {
-      sinon.stub(_, 'random').returns('222')
-      this.cpstdout.on.yieldsAsync('222')
-    })
-
-    it('shows full path to executable when verifying', function () {
-      const ctx = this
-
-      sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-      sinon.stub(state, 'getBinaryPkgVersionAsync').resolves(packageVersion)
-      sinon.stub(state, 'getBinaryVerifiedAsync').resolves(false)
-
-      return verify.start({ force: true })
-      .then(() => {
-        expect(cp.spawn).to.be.calledWith(executablePath, [
-          '--smoke-test',
-          '--ping=222',
-        ])
-      })
-      .then(() => {
-        snapshot(
-          'verification with executable',
-          normalize(ctx.stdout.toString())
-        )
-      })
-    })
-
-    it('clears verified version from state if verification fails', function () {
-      const ctx = this
-      const stderr = 'an error about dependencies'
-
-      sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-      sinon.stub(state, 'getBinaryPkgVersionAsync').resolves(packageVersion)
-      sinon.stub(state, 'getBinaryVerifiedAsync').resolves(true)
-
-      this.cpstderr.on.withArgs('data').yields(stderr)
-      this.spawnedProcess.on.withArgs('close').yieldsAsync(1)
-
-      return verify.start({ force: true })
-      .catch((err) => {
-        logger.error(err)
-
-        snapshot(
-          'fails verifying Cypress',
-          normalize(slice(ctx.stdout.toString()))
-        )
-      })
-      .then(() => {
-        expect(state.clearBinaryStateAsync).to.be.called
-        expect(state.writeBinaryVerifiedAsync).to.not.be.called
-      })
-    })
-  })
-
-  describe('smoke test with DEBUG output', function () {
-    beforeEach(function () {
-      sinon.stub(fs, 'statAsync').resolves()
-      sinon.stub(_, 'random').returns('222')
-      const stdoutWithDebugOutput = stripIndent`
-        some debug output
-        date: more debug output
-        222
-        after that more text
-      `
-      this.cpstdout.on.yieldsAsync(stdoutWithDebugOutput)
-    })
-
-    it('finds ping value in the verbose output', function () {
-      const ctx = this
-      sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-      sinon.stub(state, 'getBinaryPkgVersionAsync').resolves(packageVersion)
-      sinon.stub(state, 'getBinaryVerifiedAsync').resolves(false)
-
-      return verify.start()
-      .then(() => {
-        snapshot(
-          'verbose stdout output',
-          normalize(ctx.stdout.toString())
-        )
-      })
-    })
-  })
-
-  describe('smoke test', function () {
-    beforeEach(function () {
-      sinon.stub(fs, 'statAsync').resolves()
-      sinon.stub(_, 'random').returns('222')
-      this.cpstdout.on.yieldsAsync('222')
-    })
-
-    it('logs and runs when no version has been verified', function () {
-      const ctx = this
-      sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-      sinon.stub(state, 'getBinaryPkgVersionAsync').resolves(packageVersion)
-      sinon.stub(state, 'getBinaryVerifiedAsync').resolves(false)
-
-      return verify.start()
-      .then(() => {
-        snapshot(
-          'no existing version verified',
-          normalize(ctx.stdout.toString())
-        )
-      })
-    })
-
-    it('logs and runs when current version has not been verified', function () {
-      const ctx = this
-      sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-      sinon.stub(state, 'getBinaryPkgVersionAsync').resolves('different version')
-      sinon.stub(state, 'getBinaryVerifiedAsync').resolves(false)
-      return verify.start()
-      .then(() => {
-        snapshot(
-          'current version has not been verified',
-          normalize(ctx.stdout.toString())
-        )
-      })
-    })
-
-    it('logs and runs when installed version is different than verified version', function () {
-      const ctx = this
-      sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-      sinon.stub(state, 'getBinaryPkgVersionAsync').resolves('9.8.7')
-      sinon.stub(state, 'getBinaryVerifiedAsync').resolves(false)
-
-      return verify.start()
-      .then(() => {
-        snapshot(
-          'current version has not been verified',
-          normalize(ctx.stdout.toString())
-        )
-      })
-    })
-
-    it('is silent when logLevel is silent', function () {
-      const ctx = this
-      sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-      sinon.stub(state, 'getBinaryPkgVersionAsync').resolves('9.8.7')
-      sinon.stub(state, 'getBinaryVerifiedAsync').resolves(false)
-      process.env.npm_config_loglevel = 'silent'
-
-      return verify.start()
-      .then(() => {
-        snapshot(
-          'silent verify',
-          normalize(`[no output]${ctx.stdout.toString()}`)
-        )
-      })
-    })
-
-    it('turns off Opening Cypress...', function () {
-      const ctx = this
-      sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-      sinon.stub(state, 'getBinaryPkgVersionAsync').resolves('different version')
-      sinon.stub(state, 'getBinaryVerifiedAsync').resolves(true)
-
-      return verify.start({
-        welcomeMessage: false,
-      })
-      .then(() => {
-        snapshot(
-          'no welcome message',
-          normalize(ctx.stdout.toString())
-        )
-      })
-    })
-
-    describe('on linux', function () {
-      beforeEach(function () {
-        xvfb.isNeeded.returns(true)
-        sinon.stub(fs, 'pathExistsAsync').withArgs(executablePath).resolves(true)
-        sinon.stub(state, 'getBinaryPkgVersionAsync').resolves(packageVersion)
-        sinon.stub(state, 'getBinaryVerifiedAsync').resolves(false)
-      })
-
-      it('starts xvfb', function () {
-        return verify.start()
-        .then(() => {
-          expect(xvfb.start).to.be.called
-        })
-      })
-
-      it('stops xvfb on spawned process close', function () {
-        this.spawnedProcess.on.withArgs('close').yieldsAsync(0)
-        return verify.start()
-        .then(() => {
-          expect(xvfb.stop).to.be.called
-        })
-      })
-
-      it('logs error and exits when starting xvfb fails', function () {
-        const ctx = this
-
-        const err = new Error('test without xvfb')
-        err.stack = 'xvfb? no dice'
-        xvfb.start.rejects(err)
-        return verify.start()
-        .catch((err) => {
-          expect(xvfb.stop).to.be.calledOnce
-
-          logger.error(err)
-
-          snapshot(
-            'xvfb fails',
-            normalize(slice(ctx.stdout.toString()))
-          )
-        })
-      })
-    })
-
-    describe('when running in CI', function () {
-      beforeEach(function () {
-        sinon.stub(fs, 'pathExistsAsync').resolves(true)
-        sinon.stub(state, 'getBinaryPkgVersionAsync').resolves(packageVersion)
-        sinon.stub(state, 'getBinaryVerifiedAsync').resolves(false)
-        util.isCi.returns(true)
-
-        return verify.start({ force: true })
-      })
-
-      it('uses verbose renderer', function () {
-        snapshot(
-          'verifying in ci',
-          normalize(this.stdout.toString())
-        )
-      })
-    })
-
-    describe('when env var CYPRESS_RUN_BINARY', function () {
-      beforeEach(function () {
-        xvfb.isNeeded.returns(true)
-        sinon.stub(state, 'getBinaryPkgVersionAsync').resolves(packageVersion)
-        sinon.stub(state, 'getBinaryVerifiedAsync').resolves(false)
-        sinon.stub(util, 'isExecutableAsync').resolves(true)
-      })
-
-      it('can validate and use executable', function () {
-        const envBinaryPath = '/custom/Contents/MacOS/Cypress'
-        const realEnvBinaryPath = `/real${envBinaryPath}`
-
-        fs.realpathAsync.resolves(realEnvBinaryPath)
-        state.getPathToExecutable.restore()
-
-        sinon.stub(state, 'getPathToExecutable')
-        .withArgs('/real/custom')
-        .returns(realEnvBinaryPath)
-
-        sinon.stub(fs, 'pathExistsAsync')
-        .withArgs(realEnvBinaryPath)
-        .resolves(true)
-
-        process.env.CYPRESS_RUN_BINARY = envBinaryPath
-        return verify.start()
-        .then(() => {
-          expect(cp.spawn.firstCall.args[0]).to.equal(realEnvBinaryPath)
-          snapshot('valid CYPRESS_RUN_BINARY', normalize(this.stdout.toString()))
-        })
-      })
-
-
-      ;['darwin', 'linux', 'win32'].forEach((platform) => it('can log error to user', function () {
-        os.platform.returns(platform)
-        const envBinaryPath = '/custom/'
-        const realEnvBinaryPath = `/real${envBinaryPath}`
-
-        fs.realpathAsync.resolves(realEnvBinaryPath)
-        state.getPathToExecutable.restore()
-
-        process.env.CYPRESS_RUN_BINARY = envBinaryPath
-        return verify.start()
-        .then(() => { throw new Error('Should have thrown') })
-        .catch((err) => {
-          logger.error(err)
-          snapshot(
-            `${platform}: error when invalid CYPRESS_RUN_BINARY`,
-            normalize(this.stdout.toString())
-          )
-        })
-      }))
-    })
-  })
-})