const _ = require('lodash') const fse = require('fs-extra') const cp = require('child_process') const execa = require('execa') const path = require('path') const Promise = require('bluebird') const os = require('os') const verify = require('../../cli/lib/tasks/verify') const Fixtures = require('@tooling/system-tests') const { scaffoldCommonNodeModules } = require('@tooling/system-tests/lib/dep-installer') const fs = Promise.promisifyAll(fse) const canRecordVideo = () => { return os.platform() !== 'win32' } const shouldSkipProjectTest = () => { return os.platform() === 'win32' } const runSmokeTest = function (buildAppExecutable, timeoutSeconds = 30) { const rand = String(_.random(0, 1000)) console.log(`executable path ${buildAppExecutable}`) console.log(`timeout ${timeoutSeconds} seconds`) const hasRightResponse = function (stdout) { // there could be more debug lines in the output, so find 1 line with // expected random value const lines = stdout.split('\n').map((s) => { return s.trim() }) return lines.includes(rand) } const args = [] if (verify.needsSandbox()) { args.push('--no-sandbox') } // separate any Electron command line arguments from Cypress args args.push('--') args.push('--smoke-test') args.push(`--ping=${rand}`) const options = { timeout: timeoutSeconds * 1000, } return execa(`${buildAppExecutable}`, args, options) .catch((err) => { console.error('smoke test failed with error %s', err.message) throw err }).then(({ stdout }) => { stdout = stdout.replace(/\s/, '') if (!hasRightResponse(stdout)) { throw new Error(`Stdout: '${stdout}' did not match the random number: '${rand}'`) } console.log('smoke test response', stdout) return console.log('smokeTest passes') }) } const runProjectTest = function (buildAppExecutable, e2e) { if (shouldSkipProjectTest()) { console.log('skipping project test') return Promise.resolve() } return new Promise((resolve, reject) => { const env = _.omit(process.env, 'CYPRESS_INTERNAL_ENV') if (!canRecordVideo()) { console.log('cannot record video on this platform yet, disabling') env.CYPRESS_VIDEO_RECORDING = 'false' } const args = [ `--run-project=${e2e}`, `--spec=${e2e}/cypress/e2e/simple_passing.cy.js`, ] if (verify.needsSandbox()) { args.push('--no-sandbox') } const options = { stdio: 'inherit', env, } console.log('running project test') console.log(buildAppExecutable, args.join(' ')) return cp.spawn(buildAppExecutable, args, options) .on('exit', (code) => { if (code === 0) { return resolve() } return reject(new Error(`running project tests failed with: '${code}' errors.`)) }) }) } const runFailingProjectTest = function (buildAppExecutable, e2e) { if (shouldSkipProjectTest()) { console.log('skipping failing project test') return Promise.resolve() } console.log('running failing project test') const verifyScreenshots = function () { const screenshot1 = path.join(e2e, 'cypress', 'screenshots', 'simple_failing.cy.js', 'simple failing spec -- fails1 (failed).png') const screenshot2 = path.join(e2e, 'cypress', 'screenshots', 'simple_failing.cy.js', 'simple failing spec -- fails2 (failed).png') return Promise.all([ fs.statAsync(screenshot1), fs.statAsync(screenshot2), ]) } const spawn = () => { return new Promise((resolve, reject) => { const env = _.omit(process.env, 'CYPRESS_INTERNAL_ENV') const args = [ `--run-project=${e2e}`, `--spec=${e2e}/cypress/e2e/simple_failing.cy.js`, ] if (verify.needsSandbox()) { args.push('--no-sandbox') } const options = { stdio: 'inherit', env, } return cp.spawn(buildAppExecutable, args, options) .on('exit', (code) => { if (code === 2) { return resolve() } return reject(new Error(`running project tests failed with: '${code}' errors.`)) }) }) } return spawn() .then(verifyScreenshots) } const runV8SnapshotProjectTest = function (buildAppExecutable, e2e) { if (shouldSkipProjectTest()) { console.log('skipping failing project test') return Promise.resolve() } console.log('running v8 snapshot project test') const spawn = () => { return new Promise((resolve, reject) => { const env = _.omit(process.env, 'CYPRESS_INTERNAL_ENV') const args = [ `--run-project=${e2e}`, `--spec=${e2e}/cypress/e2e/simple_v8_snapshot.cy.js`, ] if (verify.needsSandbox()) { args.push('--no-sandbox') } const options = { stdio: 'inherit', env, } return cp.spawn(buildAppExecutable, args, options) .on('exit', (code) => { if (code === 0) { return resolve() } return reject(new Error(`running project tests failed with: '${code}' errors.`)) }) }) } return spawn() } const runErroringProjectTest = function (buildAppExecutable, e2e, testName, errorMessage) { return new Promise((resolve, reject) => { const env = _.omit(process.env, 'CYPRESS_INTERNAL_ENV') if (!canRecordVideo()) { console.log('cannot record video on this platform yet, disabling') env.CYPRESS_VIDEO_RECORDING = 'false' } const args = [ `--run-project=${e2e}`, `--spec=${e2e}/cypress/e2e/simple_passing.cy.js`, ] if (verify.needsSandbox()) { args.push('--no-sandbox') } const options = { stdio: ['inherit', 'inherit', 'pipe'], env, } console.log('running project test') console.log(buildAppExecutable, args.join(' ')) const childProcess = cp.spawn(buildAppExecutable, args, options) let errorOutput = '' childProcess.stderr.on('data', (data) => { errorOutput += data.toString() }) childProcess.on('exit', (code) => { if (code === 0) { return reject(new Error(`running project tests should have failed for test: ${testName}`)) } if (!errorOutput.includes(errorMessage)) { return reject(new Error(`running project tests failed with errors: ${errorOutput} but did not include the expected error message: '${errorMessage}'`)) } return resolve() }) }) } const runIntegrityTest = async function (buildAppExecutable, buildAppDir, e2e) { const testCorruptingFile = async (file, errorMessage) => { const contents = await fs.readFile(file) // Backup state await fs.move(file, `${file}.bak`) // Modify app await fs.writeFile(file, Buffer.concat([contents, Buffer.from(`\nconsole.log('modified code')`)])) await runErroringProjectTest(buildAppExecutable, e2e, `corrupting ${file}`, errorMessage) // Restore original state await fs.move(`${file}.bak`, file, { overwrite: true }) } await testCorruptingFile(path.join(buildAppDir, 'index.js'), 'Integrity check failed for main index.js file') await testCorruptingFile(path.join(buildAppDir, 'packages', 'server', 'index.jsc'), 'Integrity check failed for main server index.jsc file') const testAlteringEntryPoint = async (additionalCode, errorMessage) => { const packageJsonContents = await fs.readJSON(path.join(buildAppDir, 'package.json')) // Backup state await fs.move(path.join(buildAppDir, 'package.json'), path.join(buildAppDir, 'package.json.bak')) // Modify app await fs.writeJSON(path.join(buildAppDir, 'package.json'), { ...packageJsonContents, main: 'index2.js', }) await fs.writeFile(path.join(buildAppDir, 'index2.js'), `${additionalCode}\nrequire("./index.js")`) await runErroringProjectTest(buildAppExecutable, e2e, 'altering entry point', errorMessage) // Restore original state await fs.move(path.join(buildAppDir, 'package.json.bak'), path.join(buildAppDir, 'package.json'), { overwrite: true }) await fs.remove(path.join(buildAppDir, 'index2.js')) } await testAlteringEntryPoint('console.log("simple alteration")', 'Integrity check failed with expected stack length 9 but got 10') await testAlteringEntryPoint('console.log("accessing " + global.getSnapshotResult())', 'getSnapshotResult can only be called once') function compareGlobals () { const childProcess = require('child_process') const nodeGlobalKeys = JSON.parse(childProcess.spawnSync('node -p "const x = Object.getOwnPropertyNames(global);JSON.stringify(x)"', { shell: true, encoding: 'utf8' }).stdout) const extraKeys = Object.getOwnPropertyNames(global).filter((key) => { return !nodeGlobalKeys.includes(key) }) console.error(`extra keys in electron process: ${extraKeys}`) } const allowList = ['regeneratorRuntime', '__core-js_shared__', 'getSnapshotResult', 'supportTypeScript'] await testAlteringEntryPoint(`(${compareGlobals.toString()})()`, `extra keys in electron process: ${allowList}\nIntegrity check failed with expected stack length 9 but got 10`) const testTemporarilyRewritingEntryPoint = async () => { const file = path.join(buildAppDir, 'index.js') const backupFile = path.join(buildAppDir, 'index.js.bak') const contents = await fs.readFile(file) // Backup state await fs.move(file, backupFile) // Modify app await fs.writeFile(file, `console.log("rewritten code");const fs=require('fs');const { join } = require('path');fs.writeFileSync(join(__dirname,'index.js'),fs.readFileSync(join(__dirname,'index.js.bak')));${contents}`) await runErroringProjectTest(buildAppExecutable, e2e, 'temporarily rewriting index.js', 'Integrity check failed with expected column number 2573 but got') // Restore original state await fs.move(backupFile, file, { overwrite: true }) } await testTemporarilyRewritingEntryPoint() } const test = async function (buildAppExecutable, buildAppDir) { await scaffoldCommonNodeModules() await Fixtures.scaffoldProject('e2e') const e2e = Fixtures.projectPath('e2e') await runSmokeTest(buildAppExecutable) await runProjectTest(buildAppExecutable, e2e) await runFailingProjectTest(buildAppExecutable, e2e) if (!['1', 'true'].includes(process.env.DISABLE_SNAPSHOT_REQUIRE)) { await runIntegrityTest(buildAppExecutable, buildAppDir, e2e) await runV8SnapshotProjectTest(buildAppExecutable, e2e) } Fixtures.remove() } module.exports = { test, } if (require.main === module) { const buildAppExecutable = path.join(__dirname, `../../build/${os.platform()}-unpacked/Cypress`) console.log('Script invoked directly, running smoke tests.') test(buildAppExecutable) }