cypress/scripts/binary/util/packages.js

const _ = require('lodash')
let fs = require('fs-extra')
const path = require('path')
// we wrap glob to handle EMFILE error
let glob = require('glob')
const Promise = require('bluebird')
const retry = require('bluebird-retry')
const la = require('lazy-ass')
const check = require('check-more-types')
const execa = require('execa')
const R = require('ramda')
const os = require('os')
const prettyMs = require('pretty-ms')
const pluralize = require('pluralize')
const debug = require('debug')('cypress:binary')
const externalUtils = require('./3rd-party')

fs = Promise.promisifyAll(fs)
glob = Promise.promisify(glob)

const DEFAULT_PATHS = 'package.json'.split(' ')

const pathToPackageJson = function (packageFolder) {
  la(check.unemptyString(packageFolder), 'expected package path', packageFolder)

  return path.join(packageFolder, 'package.json')
}

const createCLIExecutable = (command) => {
  return (function (args, cwd, env = {}) {
    const commandToExecute = `${command} ${args.join(' ')}`

    console.log(commandToExecute)
    if (cwd) {
      console.log('in folder:', cwd)
    }

    la(check.maybe.string(cwd), 'invalid CWD string', cwd)

    return execa(command, args, { stdio: 'inherit', cwd, env })
    // if everything is ok, resolve with nothing
    .then(R.always(undefined))
    .catch((result) => {
      const msg = `${commandToExecute} failed with exit code: ${result.code}`

      throw new Error(msg)
    })
  })
}

const yarn = createCLIExecutable('yarn')
const npx = createCLIExecutable('npx')

const runAllBuild = _.partial(npx, ['lerna', 'run', 'build-prod', '--ignore', 'cli'])

// removes transpiled JS files in the original package folders
const runAllCleanJs = _.partial(npx, ['lerna', 'run', 'clean-js', '--ignore', 'cli'])

// @returns string[] with names of packages, e.g. ['runner', 'driver', 'server']
const getPackagesWithScript = (scriptName) => {
  return Promise.resolve(glob('./packages/*/package.json'))
  .map((pkgPath) => {
    return fs.readJsonAsync(pkgPath)
    .then((json) => {
      if (json.scripts != null ? json.scripts.build : undefined) {
        return path.basename(path.dirname(pkgPath))
      }
    })
  }).filter(Boolean)
}

const copyAllToDist = function (distDir) {
  const copyRelativePathToDist = function (relative) {
    const dest = path.join(distDir, relative)

    return retry(() => {
      console.log(relative, '->', dest)

      return fs.copyAsync(relative, dest)
    })
  }

  const copyPackage = function (pkg) {
    console.log('** copy package: %s **', pkg)

    // copies the package to dist
    // including the default paths
    // and any specified in package.json files
    return Promise.resolve(fs.readJsonAsync(pathToPackageJson(pkg)))
    .then((json) => {
      // grab all the files that match "files" wildcards
      // but without all negated files ("!src/**/*.spec.js" for example)
      // and default included paths
      // and convert to relative paths
      return DEFAULT_PATHS
      .concat(json.files || [])
      .concat(json.main || [])
    }).then((pkgFileMasks) => {
      debug('for pkg %s have the following file masks %o', pkg, pkgFileMasks)
      const globOptions = {
        cwd: pkg, // search in the package folder
        absolute: false, // and return relative file paths
        followSymbolicLinks: false, // do not follow symlinks
      }

      return externalUtils.globby(pkgFileMasks, globOptions)
    }).map((foundFileRelativeToPackageFolder) => {
      return path.join(pkg, foundFileRelativeToPackageFolder)
    })
    .tap(debug)
    .map(copyRelativePathToDist, { concurrency: 1 })
  }

  // fs-extra concurrency tests (copyPackage / copyRelativePathToDist)
  // 1/1  41688
  // 1/5  42218
  // 1/10 42566
  // 2/1  45041
  // 2/2  43589
  // 3/3  51399

  // cp -R concurrency tests
  // 1/1 65811

  const started = new Date()

  return fs.ensureDirAsync(distDir)
  .then(() => {
    return glob('./packages/*')
    .map(copyPackage, { concurrency: 1 })
  }).then(() => {
    console.log('Finished Copying %dms', new Date() - started)

    return console.log('')
  })
}

const forceNpmInstall = function (packagePath, packageToInstall) {
  console.log('Force installing %s', packageToInstall)
  console.log('in %s', packagePath)
  la(check.unemptyString(packageToInstall), 'missing package to install')

  return yarn(['install', '--force', packageToInstall], packagePath)
}

const removeDevDependencies = function (packageFolder) {
  const packagePath = pathToPackageJson(packageFolder)

  console.log('removing devDependencies from %s', packagePath)

  return fs.readJsonAsync(packagePath)
  .then((json) => {
    delete json.devDependencies

    return fs.writeJsonAsync(packagePath, json, { spaces: 2 })
  })
}

const retryGlobbing = function (pathToPackages, delay = 1000) {
  const retryGlob = () => {
    return glob(pathToPackages)
    .catch({ code: 'EMFILE' }, () => {
      // wait, then retry
      return Promise
      .delay(delay)
      .then(retryGlob)
    })
  }

  return retryGlob()
}

// installs all packages given a wildcard
// pathToPackages would be something like "C:\projects\cypress\dist\win32\packages\*"
const npmInstallAll = function (pathToPackages) {
  console.log(`npmInstallAll packages in ${pathToPackages}`)

  const started = new Date()

  const retryNpmInstall = function (pkg) {
    console.log('installing %s', pkg)
    console.log('NODE_ENV is %s', process.env.NODE_ENV)

    // force installing only PRODUCTION dependencies
    // https://docs.npmjs.com/cli/install
    const npmInstall = _.partial(yarn, ['install', '--production'])

    return npmInstall(pkg, { NODE_ENV: 'production' })
    .catch({ code: 'EMFILE' }, () => {
      return Promise
      .delay(1000)
      .then(() => {
        return retryNpmInstall(pkg)
      })
    }).catch((err) => {
      console.log(err, err.code)
      throw err
    })
  }

  const printFolders = (folders) => {
    return console.log('found %s', pluralize('folder', folders.length, true))
  }

  // only installs production dependencies
  return retryGlobbing(pathToPackages)
  .tap(printFolders)
  .mapSeries((packageFolder) => {
    return removeDevDependencies(packageFolder)
    .then(() => {
      return retryNpmInstall(packageFolder)
    })
  }).then(() => {
    const end = new Date()

    return console.log('Finished NPM Installing', prettyMs(end - started))
  })
}

const removePackageJson = function (filename) {
  if (filename.endsWith('/package.json')) {
    return path.dirname(filename)
  }

  return filename
}

const ensureFoundSomething = function (files) {
  if (files.length === 0) {
    throw new Error('Could not find any files')
  }

  return files
}

const symlinkType = function () {
  if (os.platform() === 'win32') {
    return 'junction'
  }

  return 'dir'
}

const symlinkAll = function (pathToDistPackages, pathTo) {
  console.log('symlink these packages', pathToDistPackages)
  la(check.unemptyString(pathToDistPackages),
    'missing paths to dist packages', pathToDistPackages)

  const symlink = function (pkg) {
    // console.log(pkg, dist)
    // strip off the initial './'
    // ./packages/foo -> node_modules/@packages/foo
    pkg = removePackageJson(pkg)
    const dest = pathTo('node_modules', '@packages', path.basename(pkg))
    const relativeDest = path.relative(`${dest}/..`, pkg)

    const type = symlinkType()

    console.log(relativeDest, 'link ->', dest, 'type', type)

    return fs.ensureSymlinkAsync(relativeDest, dest, symlinkType)
    .catch((err) => {
      if (!err.message.includes('EEXIST')) {
        throw err
      }
    })
  }

  return glob(pathToDistPackages)
  .then(ensureFoundSomething)
  .map(symlink)
}

module.exports = {
  runAllBuild,

  copyAllToDist,

  npmInstallAll,

  symlinkAll,

  runAllCleanJs,

  forceNpmInstall,

  getPackagesWithScript,
}

if (!module.parent) {
  console.log('demo force install')
  forceNpmInstall('packages/server', '@ffmpeg-installer/win32-x64')
}