diff --git a/cli/lib/tasks/download.js b/cli/lib/tasks/download.js index 03ca27b259..22c324370c 100644 --- a/cli/lib/tasks/download.js +++ b/cli/lib/tasks/download.js @@ -79,6 +79,95 @@ const prettyDownloadErr = (err, version) => { return throwFormErrorText(errors.failedDownload)(msg) } +/** + * Checks checksum and file size for the given file. Allows both + * values or just one of them to be checked. + */ +const verifyDownloadedFile = (filename, expectedSize, expectedChecksum) => { + if (expectedSize && expectedChecksum) { + debug('verifying checksum and file size') + + return Promise.join( + util.getFileChecksum(filename), + util.getFileSize(filename), + (checksum, filesize) => { + if (checksum === expectedChecksum && filesize === expectedSize) { + debug('downloaded file has the expected checksum and size ✅') + + return + } + + debug('raising error: checksum or file size mismatch') + const text = stripIndent` + Corrupted download + + Expected downloaded file to have checksum: ${expectedChecksum} + Computed checksum: ${checksum} + + Expected downloaded file to have size: ${expectedSize} + Computed size: ${filesize} + ` + + debug(text) + + throw new Error(text) + }) + } + + if (expectedChecksum) { + debug('only checking expected file checksum %d', expectedChecksum) + + return util.getFileChecksum(filename) + .then((checksum) => { + if (checksum === expectedChecksum) { + debug('downloaded file has the expected checksum ✅') + + return + } + + debug('raising error: file checksum mismatch') + const text = stripIndent` + Corrupted download + + Expected downloaded file to have checksum: ${expectedChecksum} + Computed checksum: ${checksum} + ` + + throw new Error(text) + }) + } + + if (expectedSize) { + // maybe we don't have a checksum, but at least CDN returns content length + // which we can check against the file size + debug('only checking expected file size %d', expectedSize) + + return util.getFileSize(filename) + .then((filesize) => { + if (filesize === expectedSize) { + debug('downloaded file has the expected size ✅') + + return + } + + debug('raising error: file size mismatch') + const text = stripIndent` + Corrupted download + + Expected downloaded file to have size: ${expectedSize} + Computed size: ${filesize} + ` + + throw new Error(text) + }) + } + + debug('downloaded file lacks checksum or size to verify') + + return Promise.resolve() + +} + // downloads from given url // return an object with // {filename: ..., downloaded: true} @@ -109,11 +198,31 @@ const downloadFromUrl = ({ url, downloadDestination, progress }) => { // closure let started = null + let expectedSize + let expectedChecksum requestProgress(req, { throttle: progress.throttle, }) .on('response', (response) => { + // we have computed checksum and filesize during test runner binary build + // and have set it on the S3 object as user meta data, available via + // these custom headers "x-amz-meta-..." + // see https://github.com/cypress-io/cypress/pull/4092 + expectedSize = response.headers['x-amz-meta-size'] || + response.headers['content-length'] + expectedChecksum = response.headers['x-amz-meta-checksum'] + + if (expectedChecksum) { + debug('expected checksum %s', expectedChecksum) + } + + if (expectedSize) { + // convert from string (all Amazon custom headers are strings) + expectedSize = Number(expectedSize) + debug('expected file size %d', expectedSize) + } + // start counting now once we've gotten // response headers started = new Date() @@ -152,11 +261,19 @@ const downloadFromUrl = ({ url, downloadDestination, progress }) => { .on('finish', () => { debug('downloading finished') - resolve(redirectVersion) + verifyDownloadedFile(downloadDestination, expectedSize, expectedChecksum) + .then(() => { + return resolve(redirectVersion) + }, reject) }) }) } +/** + * Download Cypress.zip from external url to local file. + * @param [string] version Could be "3.3.0" or full URL + * @param [string] downloadDestination Local filename to save as + */ const start = ({ version, downloadDestination, progress }) => { if (!downloadDestination) { la(is.unemptyString(downloadDestination), 'missing download dir', arguments) @@ -173,6 +290,7 @@ const start = ({ version, downloadDestination, progress }) => { progress.throttle = 100 debug('needed Cypress version: %s', version) + debug('source url %s', url) debug(`downloading cypress.zip to "${downloadDestination}"`) // ensure download dir exists diff --git a/cli/lib/util.js b/cli/lib/util.js index 9db9677c8a..cc2ed3c03c 100644 --- a/cli/lib/util.js +++ b/cli/lib/util.js @@ -1,6 +1,7 @@ const _ = require('lodash') const R = require('ramda') const os = require('os') +const crypto = require('crypto') const la = require('lazy-ass') const is = require('check-more-types') const tty = require('tty') @@ -19,11 +20,47 @@ const isInstalledGlobally = require('is-installed-globally') const pkg = require(path.join(__dirname, '..', 'package.json')) const logger = require('./logger') const debug = require('debug')('cypress:cli') +const fs = require('./fs') const issuesUrl = 'https://github.com/cypress-io/cypress/issues' const getosAsync = Promise.promisify(getos) +/** + * Returns SHA512 of a file + * + * Implementation lifted from https://github.com/sindresorhus/hasha + * but without bringing that dependency (since hasha is Node v8+) + */ +const getFileChecksum = (filename) => { + la(is.unemptyString(filename), 'expected filename', filename) + + const hashStream = () => { + const s = crypto.createHash('sha512') + + s.setEncoding('hex') + + return s + } + + return new Promise((resolve, reject) => { + const stream = fs.createReadStream(filename) + + stream.on('error', reject) + .pipe(hashStream()) + .on('error', reject) + .on('finish', function () { + resolve(this.read()) + }) + }) +} + +const getFileSize = (filename) => { + la(is.unemptyString(filename), 'expected filename', filename) + + return fs.statAsync(filename).get('size') +} + const isBrokenGtkDisplayRe = /Gtk: cannot open display/ const stringify = (val) => { @@ -347,6 +384,9 @@ const util = { return `${issuesUrl}/${number}` }, + getFileChecksum, + + getFileSize, } module.exports = util diff --git a/cli/package.json b/cli/package.json index 7a9751e862..7921bc18c6 100644 --- a/cli/package.json +++ b/cli/package.json @@ -85,6 +85,7 @@ "dependency-check": "3.3.0", "dtslint": "0.7.6", "execa-wrap": "1.4.0", + "hasha": "5.0.0", "mock-fs": "4.9.0", "mocked-env": "1.2.4", "nock": "9.6.1", diff --git a/cli/test/lib/tasks/download_spec.js b/cli/test/lib/tasks/download_spec.js index 214d12a5fd..2d5e107bcc 100644 --- a/cli/test/lib/tasks/download_spec.js +++ b/cli/test/lib/tasks/download_spec.js @@ -5,6 +5,8 @@ const la = require('lazy-ass') const is = require('check-more-types') const path = require('path') const nock = require('nock') +const hasha = require('hasha') +const debug = require('debug')('test') const snapshot = require('../../support/snapshot') const fs = require(`${lib}/fs`) @@ -17,6 +19,7 @@ const normalize = require('../../support/normalize') const downloadDestination = path.join(os.tmpdir(), 'Cypress', 'download', 'cypress.zip') const version = '1.2.3' +const examplePath = 'test/fixture/example.zip' describe('lib/tasks/download', function () { require('mocha-banner').register() @@ -110,7 +113,7 @@ describe('lib/tasks/download', function () { nock('https://aws.amazon.com') .get('/some.zip') .reply(200, () => { - return fs.createReadStream('test/fixture/example.zip') + return fs.createReadStream(examplePath) }) nock('https://download.cypress.io') @@ -135,11 +138,115 @@ describe('lib/tasks/download', function () { }) }) + describe('verify downloaded file', function () { + before(function () { + this.expectedChecksum = hasha.fromFileSync(examplePath) + this.expectedFileSize = fs.statSync(examplePath).size + this.onProgress = sinon.stub().returns(undefined) + debug('example file %s should have checksum %s and file size %d', + examplePath, this.expectedChecksum, this.expectedFileSize) + }) + + it('throws if file size is different from expected', function () { + nock('https://download.cypress.io') + .get('/desktop/1.2.3') + .query(true) + .reply(200, () => { + return fs.createReadStream(examplePath) + }, { + // definitely incorrect file size + 'content-length': '10', + }) + + return expect(download.start({ + downloadDestination: this.options.downloadDestination, + version: this.options.version, + progress: { onProgress: this.onProgress }, + })).to.be.rejected + }) + + it('throws if file size is different from expected x-amz-meta-size', function () { + nock('https://download.cypress.io') + .get('/desktop/1.2.3') + .query(true) + .reply(200, () => { + return fs.createReadStream(examplePath) + }, { + // definitely incorrect file size + 'x-amz-meta-size': '10', + }) + + return expect(download.start({ + downloadDestination: this.options.downloadDestination, + version: this.options.version, + progress: { onProgress: this.onProgress }, + })).to.be.rejected + }) + + it('throws if checksum is different from expected', function () { + nock('https://download.cypress.io') + .get('/desktop/1.2.3') + .query(true) + .reply(200, () => { + return fs.createReadStream(examplePath) + }, { + 'x-amz-meta-checksum': 'incorrect-checksum', + }) + + return expect(download.start({ + downloadDestination: this.options.downloadDestination, + version: this.options.version, + progress: { onProgress: this.onProgress }, + })).to.be.rejected + }) + + it('throws if checksum and file size are different from expected', function () { + nock('https://download.cypress.io') + .get('/desktop/1.2.3') + .query(true) + .reply(200, () => { + return fs.createReadStream(examplePath) + }, { + 'x-amz-meta-checksum': 'incorrect-checksum', + 'x-amz-meta-size': '10', + }) + + return expect(download.start({ + downloadDestination: this.options.downloadDestination, + version: this.options.version, + progress: { onProgress: this.onProgress }, + })).to.be.rejected + }) + + it('passes when checksum and file size match', function () { + nock('https://download.cypress.io') + .get('/desktop/1.2.3') + .query(true) + .reply(200, () => { + debug('creating read stream for %s', examplePath) + + return fs.createReadStream(examplePath) + }, { + 'x-amz-meta-checksum': this.expectedChecksum, + 'x-amz-meta-size': String(this.expectedFileSize), + }) + + debug('downloading %s to %s for test version %s', + examplePath, this.options.downloadDestination, this.options.version) + + return download.start({ + downloadDestination: this.options.downloadDestination, + version: this.options.version, + progress: { onProgress: this.onProgress }, + }) + }) + }) + it('resolves with response x-version if present', function () { nock('https://aws.amazon.com') .get('/some.zip') .reply(200, () => { - return fs.createReadStream('test/fixture/example.zip') + return fs.createReadStream(examplePath) }) nock('https://download.cypress.io') @@ -161,7 +268,7 @@ describe('lib/tasks/download', function () { nock('https://aws.amazon.com') .get('/some.zip') .reply(200, () => { - return fs.createReadStream('test/fixture/example.zip') + return fs.createReadStream(examplePath) }) nock('https://download.cypress.io') diff --git a/cli/test/lib/util_spec.js b/cli/test/lib/util_spec.js index ce3bfe5fc3..50020b8dc0 100644 --- a/cli/test/lib/util_spec.js +++ b/cli/test/lib/util_spec.js @@ -5,6 +5,8 @@ const tty = require('tty') const snapshot = require('../support/snapshot') const supportsColor = require('supports-color') const proxyquire = require('proxyquire') +const hasha = require('hasha') +const la = require('lazy-ass') const util = require(`${lib}/util`) const logger = require(`${lib}/logger`) @@ -471,4 +473,16 @@ describe('util', () => { }) }) }) + + context('.getFileChecksum', () => { + it('computes same hash as Hasha SHA512', () => { + return Promise.all([ + util.getFileChecksum(__filename), + hasha.fromFile(__filename, { algorithm: 'sha512' }), + ]).then(([checksum, expectedChecksum]) => { + la(checksum === expectedChecksum, 'our computed checksum', checksum, + 'is different from expected', expectedChecksum) + }) + }) + }) }) diff --git a/scripts/binary/upload-unique-binary.coffee b/scripts/binary/upload-unique-binary.coffee index 39361cddfd..12cbec09f5 100644 --- a/scripts/binary/upload-unique-binary.coffee +++ b/scripts/binary/upload-unique-binary.coffee @@ -96,7 +96,7 @@ setChecksum = (filename, key) => la(check.unemptyString(filename), 'expected filename', filename) la(check.unemptyString(key), 'expected uploaded S3 key', key) - checksum = hasha.fromFileSync(filename) + checksum = hasha.fromFileSync(filename, { algorithm: 'sha512' }) size = fs.statSync(filename).size console.log('SHA256 checksum %s', checksum) console.log('size', size)