refactor: update binary script files to be more procedural

This commit is contained in:
Tim Griesser
2021-06-30 18:49:40 -04:00
parent 00eca92afa
commit 1e005644ae
3 changed files with 388 additions and 660 deletions

View File

@@ -1,387 +1,253 @@
const _ = require('lodash')
const fse = require('fs-extra')
const os = require('os')
const del = require('del')
const path = require('path')
const cp = require('child_process')
const chalk = require('chalk')
const Promise = require('bluebird')
const pluralize = require('pluralize')
const execa = require('execa')
const electron = require('@packages/electron')
const debug = require('debug')('cypress:binary')
const R = require('ramda')
const la = require('lazy-ass')
const check = require('check-more-types')
import os from 'os'
import fs from 'fs-extra'
import path from 'path'
import _ from 'lodash'
import del from 'del'
import chalk from 'chalk'
import electron from '@packages/electron'
import la from 'lazy-ass'
const meta = require('./meta')
const smoke = require('./smoke')
const packages = require('./util/packages')
const xvfb = require('../../cli/lib/exec/xvfb')
const { transformRequires } = require('./util/transform-requires')
const { testStaticAssets } = require('./util/testStaticAssets')
const performanceTracking = require('../../packages/server/test/support/helpers/performance.js')
import * as packages from './util/packages'
import * as meta from './meta'
import xvfb from '../../cli/lib/exec/xvfb'
import smoke from './smoke'
import { spawn, execSync } from 'child_process'
import { transformRequires } from './util/transform-requires'
import execa from 'execa'
import { testStaticAssets } from './util/testStaticAssets'
import performanceTracking from '../../packages/server/test/support/helpers/performance.js'
const rootPackage = require('@packages/root')
const CY_ROOT_DIR = path.join(__dirname, '..', '..')
const fs = Promise.promisifyAll(fse)
const logger = function (msg, platform) {
const log = function (msg) {
const time = new Date()
const timeStamp = time.toLocaleTimeString()
return console.log(timeStamp, chalk.yellow(msg), chalk.blue(platform))
console.log(timeStamp, chalk.yellow(msg), chalk.blue(meta.PLATFORM))
}
const logBuiltAllPackages = () => {
return console.log('built all packages')
interface BuildCypressAppOpts {
platform: meta.PlatformName
version: string
skipSigning?: boolean
keepBuild?: boolean
}
// can pass options to better control the build
// for example
// skipClean - do not delete "dist" folder before build
const buildCypressApp = function (platform, version, options = {}) {
la(check.unemptyString(version), 'missing version to build', version)
// For debugging the flow without rebuilding each time
const distDir = _.partial(meta.distDir, platform)
const buildDir = _.partial(meta.buildDir, platform)
const buildAppDir = _.partial(meta.buildAppDir, platform)
export async function buildCypressApp (options: BuildCypressAppOpts) {
const { platform, version, skipSigning = false, keepBuild = false } = options
const log = _.partialRight(logger, platform)
log('#checkPlatform')
if (platform !== os.platform()) {
throw new Error('Platform mismatch')
}
const testVersion = (folderNameFn) => {
return (function () {
log('#testVersion')
const dir = folderNameFn()
const DIST_DIR = meta.distDir()
la(check.unemptyString(dir), 'missing folder for platform', platform)
console.log('testing dist package version')
console.log('by calling: node index.js --version')
console.log('in the folder %s', dir)
log('#cleanupPlatform')
fs.rmSync(meta.TMP_BUILD_DIR, { force: true, recursive: true })
fs.rmSync(path.resolve('build'), { force: true, recursive: true })
fs.rmSync(path.resolve('packages', 'electron', 'dist'), { force: true, recursive: true })
return execa('node', ['index.js', '--version'], {
cwd: dir,
}).then((result) => {
la(check.unemptyString(result.stdout),
'missing output when getting built version', result)
log(`symlinking ${meta.TMP_BUILD_DIR} -> ${path.resolve('build')}`)
fs.symlinkSync(
meta.TMP_BUILD_DIR,
path.resolve('build'),
'dir',
)
console.log('app in %s', dir)
console.log('built app version', result.stdout)
la(result.stdout === version, 'different version reported',
result.stdout, 'from input version to build', version)
return console.log('✅ using node --version works')
})
if (!keepBuild) {
log('#buildPackages')
await execa('yarn', ['lerna', 'run', 'build-prod', '--stream', '--ignore', 'cli'], {
stdio: 'inherit',
cwd: CY_ROOT_DIR,
})
}
const testBuiltStaticAssets = function () {
log('#testBuiltStaticAssets')
// Copy Packages: We want to copy the package.json, files, and output
log('#copyAllToDist')
await packages.copyAllToDist(DIST_DIR)
return testStaticAssets(distDir())
}
const jsonRoot = fs.readJSONSync(path.join(CY_ROOT_DIR, 'package.json'))
const checkPlatform = function () {
log('#checkPlatform')
if (platform === os.platform()) {
return
}
fs.writeJsonSync(meta.distDir('package.json'), _.omit(jsonRoot, [
'scripts',
'devDependencies',
'lint-staged',
'engines',
]), { spaces: 2 })
return Promise.reject(new Error('Build platform mismatch'))
}
// Copy the yarn.lock file so we have a consistent install
fs.copySync(path.join(CY_ROOT_DIR, 'yarn.lock'), meta.distDir('yarn.lock'))
const cleanupPlatform = function () {
log('#cleanupPlatform')
// replaceLocalNpmVersions
const dirsSeen = await packages.replaceLocalNpmVersions(DIST_DIR)
if (options.skipClean) {
log('skipClean')
// remove local npm dirs that aren't needed
await packages.removeLocalNpmDirs(DIST_DIR, dirsSeen)
return
}
execSync('yarn --production', {
cwd: DIST_DIR,
stdio: 'inherit',
})
const cleanup = function () {
const dir = distDir()
// TODO: Validate no-hoists / single copies of libs
la(check.unemptyString(dir), 'empty dist dir', dir, 'for platform', platform)
// Remove extra directories that are large/unneeded
log('#remove extra dirs')
await del([
meta.distDir('**', 'image-q', 'demo'),
meta.distDir('**', 'gifwrap', 'test'),
meta.distDir('**', 'pixelmatch', 'test'),
meta.distDir('**', '@jimp', 'tiff', 'test'),
meta.distDir('**', '@cypress', 'icons', '**/*.{ai,eps}'),
meta.distDir('**', 'esprima', 'test'),
meta.distDir('**', 'bmp-js', 'test'),
meta.distDir('**', 'exif-parser', 'test'),
], { force: true })
return fs.removeAsync(distDir())
}
console.log('Deleted excess directories')
return cleanup()
.catch(cleanup)
}
log('#createRootPackage')
const electronVersion = electron.getElectronVersion()
const electronNodeVersion = await electron.getElectronNodeVersion()
const buildPackages = function () {
log('#buildPackages')
fs.writeJSONSync(meta.distDir('package.json'), {
name: 'cypress',
productName: 'Cypress',
description: jsonRoot.description,
version, // Cypress version
electronVersion,
electronNodeVersion,
main: 'index.js',
scripts: {},
env: 'production',
}, { spaces: 2 })
return packages.runAllBuild()
// Promise.resolve()
.then(R.tap(logBuiltAllPackages))
}
const copyPackages = function () {
log('#copyPackages')
return packages.copyAllToDist(distDir())
}
const replaceLocalNpmVersions = function () {
log('#replaceLocalNpmVersions')
return packages.replaceLocalNpmVersions(distDir())
}
const npmInstallPackages = function () {
log('#npmInstallPackages')
const pathToPackages = distDir('packages', '*')
return packages.npmInstallAll(pathToPackages)
}
const cleanLocalNpmPackages = function () {
log('#cleanLocalNpmPackages')
return fs.removeAsync(distDir('npm'))
}
/**
* Creates the package.json file that sits in the root of the output app
*/
const createRootPackage = function () {
log(`#createRootPackage ${platform} ${version}`)
const electronVersion = electron.getElectronVersion()
la(electronVersion, 'missing Electron version', electronVersion)
return electron.getElectronNodeVersion()
.then((electronNodeVersion) => {
la(electronNodeVersion, 'missing Electron Node version', electronNodeVersion)
const json = {
name: 'cypress',
productName: 'Cypress',
description: rootPackage.description,
version, // Cypress version
electronVersion,
electronNodeVersion,
main: 'index.js',
scripts: {},
env: 'production',
}
const outputFilename = distDir('package.json')
debug('writing to %s json %o', outputFilename, json)
return fs.outputJsonAsync(outputFilename, json)
.then(() => {
const str = `\
fs.writeFileSync(meta.distDir('index.js'), `\
process.env.CYPRESS_INTERNAL_ENV = process.env.CYPRESS_INTERNAL_ENV || 'production'
require('./packages/server')\
`
`)
return fs.outputFileAsync(distDir('index.js'), str)
})
})
// removeTypeScript
await del([
// include ts files of packages
meta.distDir('**', '*.ts'),
// except those in node_modules
`!${meta.distDir('**', 'node_modules', '**', '*.ts')}`,
], { force: true })
// cleanJs
if (!keepBuild) {
await packages.runAllCleanJs()
}
const removeTypeScript = function () {
// remove the .ts files in our packages
log('#removeTypeScript')
// transformSymlinkRequires
log('#transformSymlinkRequires')
return del([
// include ts files of packages
distDir('**', '*.ts'),
await transformRequires(meta.distDir())
// except those in node_modules
`!${distDir('**', 'node_modules', '**', '*.ts')}`,
])
.then((paths) => {
console.log(
'deleted %d TS %s',
paths.length,
pluralize('file', paths.length),
)
log(`#testVersion ${meta.distDir()}`)
await testVersion(meta.distDir(), version)
return console.log(paths)
})
}
// testBuiltStaticAssets
await testStaticAssets(meta.distDir())
const cleanJs = function () {
log('#cleanJs')
log('#removeCyAndBinFolders')
await del([
meta.distDir('node_modules', '.bin'),
meta.distDir('packages', '*', 'node_modules', '.bin'),
meta.distDir('packages', 'server', '.cy'),
], { force: true })
return packages.runAllCleanJs()
}
// when we copy packages/electron, we get the "dist" folder with
// empty Electron app, symlinked to our server folder
// in production build, we do not need this link, and it
// would not work anyway with code signing
const transformSymlinkRequires = function () {
log('#transformSymlinkRequires')
// hint: you can see all symlinks in the build folder
// using "find build/darwin/Cypress.app/ -type l -ls"
log('#removeDevElectronApp')
fs.removeSync(meta.distDir('packages', 'electron', 'dist'))
return transformRequires(distDir())
.then((replaceCount) => {
return la(replaceCount > 5, 'expected to replace more than 5 symlink requires, but only replaced', replaceCount)
})
}
// electronPackAndSign
log('#electronPackAndSign')
// See the internal wiki document "Signing Test Runner on MacOS"
// to learn how to get the right Mac certificate for signing and notarizing
// the built Test Runner application
// we also don't need ".bin" links inside Electron application
// thus we can go through dist/packages/*/node_modules and remove all ".bin" folders
const removeBinFolders = function () {
log('#removeBinFolders')
const appFolder = meta.distDir()
const outputFolder = meta.buildRootDir()
const searchMask = distDir('packages', '*', 'node_modules', '.bin')
const iconFilename = getIconFilename()
console.log('searching for', searchMask)
console.log(`output folder: ${outputFolder}`)
return del([searchMask])
.then((paths) => {
console.log(
'deleted %d .bin %s',
paths.length,
pluralize('folder', paths.length),
)
const args = [
'--publish=never',
`--c.electronVersion=${electronVersion}`,
`--c.directories.app=${appFolder}`,
`--c.directories.output=${outputFolder}`,
`--c.icon=${iconFilename}`,
// for now we cannot pack source files in asar file
// because electron-builder does not copy nested folders
// from packages/*/node_modules
// see https://github.com/electron-userland/electron-builder/issues/3185
// so we will copy those folders later ourselves
'--c.asar=false',
]
return console.log(paths)
})
}
console.log('electron-builder arguments:')
console.log(args.join(' '))
const removeCyFolders = function () {
log('#removeCyFolders')
const searchMask = distDir('packages', 'server', '.cy')
console.log('searching', searchMask)
return del([searchMask])
.then((paths) => {
console.log(
'deleted %d .cy %s',
paths.length,
pluralize('file', paths.length),
)
return console.log(paths)
})
}
const getIconFilename = function (platform) {
const filenames = {
darwin: 'cypress.icns',
win32: 'cypress.ico',
linux: 'icon_512x512.png',
}
const iconFilename = electron.icons().getPathToIcon(filenames[platform])
console.log(`For platform ${platform} using icon ${iconFilename}`)
return iconFilename
}
const removeDevElectronApp = function () {
log('#removeDevElectronApp')
// when we copy packages/electron, we get the "dist" folder with
// empty Electron app, symlinked to our server folder
// in production build, we do not need this link, and it
// would not work anyway with code signing
// hint: you can see all symlinks in the build folder
// using "find build/darwin/Cypress.app/ -type l -ls"
console.log('platform', platform)
const electronDistFolder = distDir('packages', 'electron', 'dist')
la(check.unemptyString(electronDistFolder),
'empty electron dist folder for platform', platform)
console.log(`Removing unnecessary folder '${electronDistFolder}'`)
return fs.removeAsync(electronDistFolder) // .catch(_.noop) why are we ignoring an error here?!
}
const electronPackAndSign = function () {
log('#electronPackAndSign')
// See the internal wiki document "Signing Test Runner on MacOS"
// to learn how to get the right Mac certificate for signing and notarizing
// the built Test Runner application
const appFolder = distDir()
const outputFolder = meta.buildRootDir(platform)
const electronVersion = electron.getElectronVersion()
la(check.unemptyString(electronVersion), 'missing Electron version to pack', electronVersion)
const iconFilename = getIconFilename(platform)
console.log(`output folder: ${outputFolder}`)
const args = [
'--publish=never',
`--c.electronVersion=${electronVersion}`,
`--c.directories.app=${appFolder}`,
`--c.directories.output=${outputFolder}`,
`--c.icon=${iconFilename}`,
// for now we cannot pack source files in asar file
// because electron-builder does not copy nested folders
// from packages/*/node_modules
// see https://github.com/electron-userland/electron-builder/issues/3185
// so we will copy those folders later ourselves
'--c.asar=false',
]
const opts = {
try {
await execa('electron-builder', args, {
stdio: 'inherit',
})
} catch (e) {
if (!skipSigning) {
throw e
}
console.log('electron-builder arguments:')
console.log(args.join(' '))
return execa('electron-builder', args, opts)
}
const lsDistFolder = function () {
log('#lsDistFolder')
const buildFolder = buildDir()
// lsDistFolder
console.log('in build folder %s', meta.buildDir())
console.log('in build folder %s', buildFolder)
const { stdout } = await execa('ls', ['-la', meta.buildDir()])
return execa('ls', ['-la', buildFolder])
.then(R.prop('stdout'))
.then(console.log)
console.log(stdout)
// testVersion(buildAppDir)
await testVersion(meta.buildAppDir(), version)
// runSmokeTests
let usingXvfb = xvfb.isNeeded()
try {
if (usingXvfb) {
await xvfb.start()
}
const executablePath = meta.buildAppExecutable()
await smoke.test(executablePath)
} finally {
if (usingXvfb) {
await xvfb.stop()
}
}
const runSmokeTests = function () {
log('#runSmokeTests')
// verifyAppCanOpen
if (platform === 'darwin' && !skipSigning) {
const appFolder = meta.zipDir()
const run = function () {
// make sure to use a longer timeout - on Mac the first
// launch of a built application invokes gatekeeper check
// which takes a couple of seconds
const executablePath = meta.buildAppExecutable(platform)
return smoke.test(executablePath)
}
if (xvfb.isNeeded()) {
return xvfb.start()
.then(run)
.finally(xvfb.stop)
}
return run()
}
const verifyAppCanOpen = function () {
if (platform !== 'darwin') {
return Promise.resolve()
}
const appFolder = meta.zipDir(platform)
log(`#verifyAppCanOpen ${appFolder}`)
return new Promise((resolve, reject) => {
await new Promise<void>((resolve, reject) => {
const args = ['-a', '-vvvv', appFolder]
debug(`cmd: spctl ${args.join(' ')}`)
const sp = cp.spawn('spctl', args, { stdio: 'inherit' })
console.log(`cmd: spctl ${args.join(' ')}`)
const sp = spawn('spctl', args, { stdio: 'inherit' })
return sp.on('exit', (code) => {
if (code === 0) {
@@ -393,82 +259,73 @@ require('./packages/server')\
})
}
const printPackageSizes = function () {
const appFolder = meta.buildAppDir(platform, 'packages')
log(`#printPackageSizes ${appFolder}`)
if (platform === 'win32') {
return Promise.resolve()
}
// "du" - disk usage utility
// -d -1 depth of 1
// -h human readable sizes (K and M)
const args = ['-d', '1', appFolder]
const parseDiskUsage = function (result) {
const lines = result.stdout.split(os.EOL)
// will store {package name: package size}
const data = {}
lines.forEach((line) => {
const parts = line.split('\t')
const packageSize = parseFloat(parts[0])
const folder = parts[1]
const packageName = path.basename(folder)
if (packageName === 'packages') {
return // root "packages" information
}
data[packageName] = packageSize
})
return data
}
const printDiskUsage = function (sizes) {
const bySize = R.sortBy(R.prop('1'))
return console.log(bySize(R.toPairs(sizes)))
}
return execa('du', args)
.then(parseDiskUsage)
.then(R.tap(printDiskUsage))
.then((sizes) => {
return performanceTracking.track('test runner size', sizes)
})
if (platform === 'win32') {
return
}
return Promise.resolve()
.then(checkPlatform)
.then(cleanupPlatform)
.then(buildPackages)
.then(copyPackages)
.then(replaceLocalNpmVersions)
.then(npmInstallPackages)
.then(cleanLocalNpmPackages)
.then(createRootPackage)
.then(removeTypeScript)
.then(cleanJs)
.then(transformSymlinkRequires)
.then(testVersion(distDir))
.then(testBuiltStaticAssets)
.then(removeBinFolders)
.then(removeCyFolders)
.then(removeDevElectronApp)
.then(electronPackAndSign)
.then(lsDistFolder)
.then(testVersion(buildAppDir))
.then(runSmokeTests)
.then(verifyAppCanOpen)
.then(printPackageSizes)
.return({
buildDir: buildDir(),
log(`#printPackageSizes ${appFolder}`)
// "du" - disk usage utility
// -d -1 depth of 1
// -h human readable sizes (K and M)
const diskUsageResult = await execa('du', ['-d', '1', appFolder])
const lines = diskUsageResult.stdout.split(os.EOL)
// will store {package name: package size}
const data = {}
lines.forEach((line) => {
const parts = line.split('\t')
const packageSize = parseFloat(parts[0])
const folder = parts[1]
const packageName = path.basename(folder)
if (packageName === 'packages') {
return // root "packages" information
}
data[packageName] = packageSize
})
const sizes = _.fromPairs(_.sortBy(_.toPairs(data), 1))
console.log(sizes)
performanceTracking.track('test runner size', sizes)
}
module.exports = buildCypressApp
function getIconFilename () {
const filenames = {
darwin: 'cypress.icns',
win32: 'cypress.ico',
linux: 'icon_512x512.png',
}
const iconFilename = electron.icons().getPathToIcon(filenames[meta.PLATFORM])
console.log(`For platform ${meta.PLATFORM} using icon ${iconFilename}`)
return iconFilename
}
async function testVersion (dir: string, version: string) {
log('#testVersion')
console.log('testing dist package version')
console.log('by calling: node index.js --version')
console.log('in the folder %s', dir)
const result = await execa('node', ['index.js', '--version'], {
cwd: dir,
})
la(result.stdout, 'missing output when getting built version', result)
console.log('app in %s', dir)
console.log('built app version', result.stdout)
la(result.stdout === version, 'different version reported',
result.stdout, 'from input version to build', version)
console.log('✅ using node --version works')
}

View File

@@ -1,24 +1,26 @@
const path = require('path')
const la = require('lazy-ass')
const check = require('check-more-types')
const R = require('ramda')
const os = require('os')
import path from 'path'
import os from 'os'
// canonical platform names
const platforms = {
export const platforms = {
darwin: 'darwin',
linux: 'linux',
windows: 'win32',
} as const
export const PLATFORM = os.platform() as any
if (!Object.values(platforms).includes(PLATFORM)) {
throw new Error(`Invalid build platform ${PLATFORM}`)
}
const isValidPlatform = check.oneOf(R.values(platforms))
export type PlatformName = {[K in keyof typeof platforms]: typeof platforms[K]}[keyof typeof platforms]
const checkPlatform = (platform) => {
return la(isValidPlatform(platform),
'invalid build platform', platform, 'valid choices', R.values(platforms))
export const buildRootDir = () => {
return path.join(TMP_BUILD_DIR, 'build')
}
const buildRootDir = () => {
export const buildLinkDir = () => {
return path.resolve('build')
}
@@ -27,11 +29,10 @@ const buildRootDir = () => {
// build/
// <platform>/ = linux or darwin
// ... platform-specific files
const buildDir = function (platform, ...args) {
checkPlatform(platform)
export const buildDir = function (...args: string[]) {
const root = buildRootDir()
switch (platform) {
switch (PLATFORM) {
case 'darwin':
// the new electron-builder for some reason adds its own platform
// subfolder and it is NOT "darwin" but "mac"
@@ -50,23 +51,21 @@ const buildDir = function (platform, ...args) {
}
}
// returns a path into the /dist directory
const distDir = function (platform, ...args) {
checkPlatform(platform)
export const TMP_BUILD_DIR = path.join(os.tmpdir(), 'cypress-build', PLATFORM)
return path.resolve('dist', platform, ...args)
// returns a path into the /dist directory
export const distDir = function (...args: string[]) {
return path.resolve(TMP_BUILD_DIR, 'dist', ...args)
}
// returns folder to zip before uploading
const zipDir = function (platform) {
checkPlatform(platform)
switch (platform) {
export const zipDir = function () {
switch (PLATFORM) {
case 'darwin':
return buildDir(platform, 'Cypress.app')
return buildDir('Cypress.app')
case 'linux':
return buildDir(platform)
case 'win32':
return buildDir(platform)
return buildDir()
default:
throw new Error('unexpected platform')
}
@@ -74,42 +73,28 @@ const zipDir = function (platform) {
// returns a path into the /build/*/app directory
// specific to each platform
const buildAppDir = function (platform, ...args) {
checkPlatform(platform)
switch (platform) {
export const buildAppDir = function (...args: string[]) {
switch (PLATFORM) {
case 'darwin':
return buildDir(platform, 'Cypress.app', 'Contents', 'resources', 'app', ...args)
return buildDir('Cypress.app', 'Contents', 'resources', 'app', ...args)
case 'linux':
return buildDir(platform, 'resources', 'app', ...args)
case 'win32':
return buildDir(platform, 'resources', 'app', ...args)
return buildDir('resources', 'app', ...args)
default:
throw new Error('unexpected platform')
}
}
const buildAppExecutable = function (platform) {
checkPlatform(platform)
switch (platform) {
export const buildAppExecutable = function () {
switch (PLATFORM) {
case 'darwin':
return buildDir(platform, 'Cypress.app', 'Contents', 'MacOS', 'Cypress')
return buildDir('Cypress.app', 'Contents', 'MacOS', 'Cypress')
case 'linux':
return buildDir(platform, 'Cypress')
case 'win32':
return buildDir(platform, 'Cypress')
return buildDir('Cypress')
default:
throw new Error('unexpected platform')
}
}
module.exports = {
isValidPlatform,
buildRootDir,
buildDir,
distDir,
zipDir,
buildAppDir,
buildAppExecutable,
cacheDir: path.join(process.cwd(), 'cache'),
platforms,
}
export const cacheDir = path.join(process.cwd(), 'cache')

View File

@@ -1,23 +1,17 @@
const _ = require('lodash')
let fs = require('fs-extra')
const path = require('path')
import _ from 'lodash'
import fs from 'fs-extra'
import path from 'path'
// we wrap glob to handle EMFILE error
let glob = require('glob')
const Promise = require('bluebird')
const retry = require('bluebird-retry')
const la = require('lazy-ass')
const check = require('check-more-types')
const execa = require('execa')
const R = require('ramda')
const prettyMs = require('pretty-ms')
const pluralize = require('pluralize')
const debug = require('debug')('cypress:binary')
const externalUtils = require('./3rd-party')
import Promise from 'bluebird'
import la from 'lazy-ass'
import check from 'check-more-types'
import execa from 'execa'
import R from 'ramda'
import debugLib from 'debug'
fs = Promise.promisifyAll(fs)
glob = Promise.promisify(glob)
import externalUtils, { globby } from './3rd-party'
const DEFAULT_PATHS = 'package.json'.split(' ')
const debug = debugLib('cypress:binary')
const pathToPackageJson = function (packageFolder) {
la(check.unemptyString(packageFolder), 'expected package path', packageFolder)
@@ -26,7 +20,7 @@ const pathToPackageJson = function (packageFolder) {
}
const createCLIExecutable = (command) => {
return (function (args, cwd, env = {}) {
return function (args, cwd = undefined, env = {}) {
const commandToExecute = `${command} ${args.join(' ')}`
console.log(commandToExecute)
@@ -34,9 +28,9 @@ const createCLIExecutable = (command) => {
console.log('in folder:', cwd)
}
la(check.maybe.string(cwd), 'invalid CWD string', cwd)
// la(check.maybe.string(cwd), 'invalid CWD string', cwd)
return execa(command, args, { stdio: 'inherit', cwd, env })
return execa(command, args, { stdio: 'inherit', cwd })
// if everything is ok, resolve with nothing
.then(R.always(undefined))
.catch((result) => {
@@ -44,240 +38,132 @@ const createCLIExecutable = (command) => {
throw new Error(msg)
})
})
}
}
const yarn = createCLIExecutable('yarn')
const npx = createCLIExecutable('npx')
const runAllBuild = _.partial(npx, ['lerna', 'run', 'build-prod', '--ignore', 'cli'])
export const runAllBuild = _.partial(yarn, ['lerna', 'run', 'build-prod', '--ignore', 'cli'])
// removes transpiled JS files in the original package folders
const runAllCleanJs = _.partial(npx, ['lerna', 'run', 'clean-js', '--ignore', 'cli'])
export const runAllCleanJs = _.partial(yarn, ['lerna', 'run', 'clean-js', '--ignore', 'cli'])
const copyAllToDist = function (distDir) {
const copyRelativePathToDist = function (relative) {
const dest = path.join(distDir, relative)
export async function copyAllToDist (distDir: string) {
await fs.ensureDir(distDir)
return retry(() => {
console.log(relative, '->', dest)
return fs.copyAsync(relative, dest)
})
}
const copyPackage = function (pkg) {
console.log('** copy package: %s **', pkg)
const started = new Date().valueOf()
const globbed = await externalUtils.globby(['./packages/*', './npm/*'], {
onlyFiles: false,
})
for (const pkg of globbed) {
// copies the package to dist
// including the default paths
// and any specified in package.json files
return Promise.resolve(fs.readJsonAsync(pathToPackageJson(pkg)))
.then((json) => {
// grab all the files that match "files" wildcards
// but without all negated files ("!src/**/*.spec.js" for example)
// and default included paths
// and convert to relative paths
return DEFAULT_PATHS
.concat(json.files || [])
.concat(json.main || [])
}).then((pkgFileMasks) => {
debug('for pkg %s have the following file masks %o', pkg, pkgFileMasks)
const globOptions = {
cwd: pkg, // search in the package folder
absolute: false, // and return relative file paths
followSymbolicLinks: false, // do not follow symlinks
}
const json = await fs.readJSON(pathToPackageJson(pkg))
return externalUtils.globby(pkgFileMasks, globOptions)
}).map((foundFileRelativeToPackageFolder) => {
return path.join(pkg, foundFileRelativeToPackageFolder)
// grab all the files that match "files" wildcards
// but without all negated files ("!src/**/*.spec.js" for example)
// and default included paths
// and convert to relative paths
const pkgFileMasks = [].concat(json.files || []).concat(json.main || [])
debug('for pkg %s have the following file masks %o', pkg, pkgFileMasks)
const foundFileRelativeToPackageFolder = await externalUtils.globby(pkgFileMasks, {
cwd: pkg, // search in the package folder
absolute: false, // and return relative file paths
followSymbolicLinks: false, // do not follow symlinks
})
.tap(debug)
.map(copyRelativePathToDist, { concurrency: 1 })
}
// fs-extra concurrency tests (copyPackage / copyRelativePathToDist)
// 1/1 41688
// 1/5 42218
// 1/10 42566
// 2/1 45041
// 2/2 43589
// 3/3 51399
console.log(`Copying ${pkg} to ${path.join(distDir, pkg)}`)
// cp -R concurrency tests
// 1/1 65811
// fs-extra concurrency tests (copyPackage / copyRelativePathToDist)
// 1/1 41688
// 1/5 42218
// 1/10 42566
// 2/1 45041
// 2/2 43589
// 3/3 51399
const started = new Date()
// cp -R concurrency tests
// 1/1 65811
for (const relativeFile of foundFileRelativeToPackageFolder) {
const dest = path.join(distDir, pkg, relativeFile)
return fs.ensureDirAsync(distDir)
.then(() => {
const globs = ['./packages/*', './npm/*']
const globOptions = {
onlyFiles: false,
await fs.copy(path.join(pkg, relativeFile), dest, { recursive: true })
}
return Promise.resolve(externalUtils.globby(globs, globOptions))
.map(copyPackage, { concurrency: 1 })
}).then(() => {
console.log('Finished Copying %dms', new Date() - started)
try {
// Strip out dev-dependencies & scripts for everything in /packages so we can yarn install in there
await fs.writeJson(path.join(distDir, pkg, 'package.json'), _.omit(json, [
'scripts',
'devDependencies',
'lint-staged',
'engines',
]), { spaces: 2 })
} catch (e) {
if (!e.message.includes('ENOENT')) {
throw e
}
}
}
return console.log('')
})
console.log('Finished Copying %dms', new Date().valueOf() - started)
}
// replaces local npm version 0.0.0-development
// with the path to the package
// we need to do this instead of just changing the symlink (like we do for require('@packages/...'))
// so the packages actually get installed to node_modules and work with peer dependencies
const replaceLocalNpmVersions = function (basePath) {
const visited = []
export const replaceLocalNpmVersions = async function (basePath: string) {
const visited = new Set<string>()
const updateNpmPackage = function (pkg) {
if (!visited.includes(pkg)) {
visited.push(pkg)
const pkgPaths = await globby('./packages/*/package.json', { cwd: basePath })
return updatePackageJson(`./npm/${pkg}/package.json`)
}
async function updatePackageJson (pkg: string) {
const pkgJsonPath = path.join(basePath, pkg)
return Promise.resolve()
}
visited.add(pkgJsonPath)
const json = await fs.readJson(pkgJsonPath)
const updatePackageJson = function (pattern) {
return Promise.resolve(glob(pattern, { cwd: basePath }))
.map((pkgPath) => {
const pkgJsonPath = path.join(basePath, pkgPath)
const { dependencies } = json
return fs.readJsonAsync(pkgJsonPath)
.then((json) => {
const { dependencies } = json
let shouldWriteFile = false
if (dependencies) {
let shouldWriteFile = false
if (dependencies) {
return Promise.all(_.map(dependencies, (version, pkg) => {
const parsedPkg = /(@cypress\/)(.*)/g.exec(pkg)
if (parsedPkg && parsedPkg.length === 3 && version === '0.0.0-development') {
const pkgName = parsedPkg[2]
json.dependencies[`@cypress/${pkgName}`] = `file:${path.join(basePath, 'npm', pkgName)}`
shouldWriteFile = true
return updateNpmPackage(pkgName)
}
}))
.then(() => {
if (shouldWriteFile) {
return fs.writeJsonAsync(pkgJsonPath, json, { spaces: 2 })
}
})
for (const [depName, version] of Object.entries(dependencies)) {
if (!depName.startsWith('@cypress/') || version !== '0.0.0-development') {
continue
}
return Promise.resolve()
})
})
const [, localPkg] = depName.split('/')
const localPkgJsonPath = path.join(basePath, 'npm', localPkg)
dependencies[`@cypress/${localPkg}`] = `file:${localPkgJsonPath}`
if (!visited.has(localPkgJsonPath)) {
await updatePackageJson(`./npm/${localPkg}/package.json`)
}
shouldWriteFile = true
}
if (shouldWriteFile) {
await fs.writeJson(pkgJsonPath, json, { spaces: 2 })
}
}
}
return updatePackageJson('./packages/*/package.json')
await Promise.all(pkgPaths.map(updatePackageJson))
return Array.from(visited)
}
const forceNpmInstall = function (packagePath, packageToInstall) {
console.log('Force installing %s', packageToInstall)
console.log('in %s', packagePath)
la(check.unemptyString(packageToInstall), 'missing package to install')
return yarn(['install', '--force', packageToInstall], packagePath)
}
const removeDevDependencies = function (packageFolder) {
const packagePath = pathToPackageJson(packageFolder)
console.log('removing devDependencies from %s', packagePath)
return fs.readJsonAsync(packagePath)
.then((json) => {
delete json.devDependencies
return fs.writeJsonAsync(packagePath, json, { spaces: 2 })
export async function removeLocalNpmDirs (distPath: string, except: string[]) {
const toRemove = await globby(`${distPath}/npm/*`, {
ignore: except.map((e) => e.replace('/package.json', '')),
onlyDirectories: true,
})
}
const retryGlobbing = function (pathToPackages, delay = 1000) {
const retryGlob = () => {
return glob(pathToPackages)
.catch({ code: 'EMFILE' }, () => {
// wait, then retry
return Promise
.delay(delay)
.then(retryGlob)
})
for (const dir of toRemove) {
await fs.remove(dir)
}
return retryGlob()
}
// installs all packages given a wildcard
// pathToPackages would be something like "C:\projects\cypress\dist\win32\packages\*"
const npmInstallAll = function (pathToPackages) {
console.log(`npmInstallAll packages in ${pathToPackages}`)
const started = new Date()
const retryNpmInstall = function (pkg) {
console.log('installing %s', pkg)
console.log('NODE_ENV is %s', process.env.NODE_ENV)
// force installing only PRODUCTION dependencies
// https://docs.npmjs.com/cli/install
const npmInstall = _.partial(yarn, ['install', '--production'])
return npmInstall(pkg, { NODE_ENV: 'production' })
.catch({ code: 'EMFILE' }, () => {
return Promise
.delay(1000)
.then(() => {
return retryNpmInstall(pkg)
})
}).catch((err) => {
console.log(err, err.code)
throw err
})
}
const printFolders = (folders) => {
return console.log('found %s', pluralize('folder', folders.length, true))
}
// only installs production dependencies
return retryGlobbing(pathToPackages)
.tap(printFolders)
.mapSeries((packageFolder) => {
return removeDevDependencies(packageFolder)
.then(() => {
return retryNpmInstall(packageFolder)
})
}).then(() => {
const end = new Date()
return console.log('Finished NPM Installing', prettyMs(end - started))
})
}
module.exports = {
runAllBuild,
copyAllToDist,
npmInstallAll,
runAllCleanJs,
forceNpmInstall,
replaceLocalNpmVersions,
}
if (!module.parent) {
console.log('demo force install')
forceNpmInstall('packages/server', '@ffmpeg-installer/win32-x64')
}