build: properly use local npm packages for development and binary build (#14182)

scripts/binary/build.js

@@ -130,13 +130,10 @@ const buildCypressApp = function (platform, version, options = {}) {
     return packages.copyAllToDist(distDir())
   }
 
-  const transformSymlinkRequires = function () {
-    log('#transformSymlinkRequires')
+  const replaceLocalNpmVersions = function () {
+    log('#replaceLocalNpmVersions')
 
-    return transformRequires(distDir())
-    .then((replaceCount) => {
-      return la(replaceCount > 5, 'expected to replace more than 5 symlink requires, but only replaced', replaceCount)
-    })
+    return packages.replaceLocalNpmVersions(distDir())
   }
 
   const npmInstallPackages = function () {
@@ -147,6 +144,12 @@ const buildCypressApp = function (platform, version, options = {}) {
     return packages.npmInstallAll(pathToPackages)
   }
 
+  const cleanLocalNpmPackages = function () {
+    log('#cleanLocalNpmPackages')
+
+    return fs.removeAsync(distDir('npm'))
+  }
+
   /**
    * Creates the package.json file that sits in the root of the output app
    */
@@ -211,6 +214,21 @@ require('./packages/server')\
     })
   }
 
+  const cleanJs = function () {
+    log('#cleanJs')
+
+    return packages.runAllCleanJs()
+  }
+
+  const transformSymlinkRequires = function () {
+    log('#transformSymlinkRequires')
+
+    return transformRequires(distDir())
+    .then((replaceCount) => {
+      return la(replaceCount > 5, 'expected to replace more than 5 symlink requires, but only replaced', replaceCount)
+    })
+  }
+
   // we also don't need ".bin" links inside Electron application
   // thus we can go through dist/packages/*/node_modules and remove all ".bin" folders
   const removeBinFolders = function () {
@@ -251,12 +269,6 @@ require('./packages/server')\
     })
   }
 
-  const cleanJs = function () {
-    log('#cleanJs')
-
-    return packages.runAllCleanJs()
-  }
-
   const getIconFilename = function (platform) {
     const filenames = {
       darwin: 'cypress.icns',
@@ -270,6 +282,26 @@ require('./packages/server')\
     return iconFilename
   }
 
+  const removeDevElectronApp = function () {
+    log('#removeDevElectronApp')
+    // when we copy packages/electron, we get the "dist" folder with
+    // empty Electron app, symlinked to our server folder
+    // in production build, we do not need this link, and it
+    // would not work anyway with code signing
+
+    // hint: you can see all symlinks in the build folder
+    // using "find build/darwin/Cypress.app/ -type l -ls"
+    console.log('platform', platform)
+    const electronDistFolder = distDir('packages', 'electron', 'dist')
+
+    la(check.unemptyString(electronDistFolder),
+      'empty electron dist folder for platform', platform)
+
+    console.log(`Removing unnecessary folder '${electronDistFolder}'`)
+
+    return fs.removeAsync(electronDistFolder) // .catch(_.noop) why are we ignoring an error here?!
+  }
+
   const electronPackAndSign = function () {
     log('#electronPackAndSign')
 
@@ -309,26 +341,6 @@ require('./packages/server')\
     return execa('electron-builder', args, opts)
   }
 
-  const removeDevElectronApp = function () {
-    log('#removeDevElectronApp')
-    // when we copy packages/electron, we get the "dist" folder with
-    // empty Electron app, symlinked to our server folder
-    // in production build, we do not need this link, and it
-    // would not work anyway with code signing
-
-    // hint: you can see all symlinks in the build folder
-    // using "find build/darwin/Cypress.app/ -type l -ls"
-    console.log('platform', platform)
-    const electronDistFolder = distDir('packages', 'electron', 'dist')
-
-    la(check.unemptyString(electronDistFolder),
-      'empty electron dist folder for platform', platform)
-
-    console.log(`Removing unnecessary folder '${electronDistFolder}'`)
-
-    return fs.removeAsync(electronDistFolder) // .catch(_.noop) why are we ignoring an error here?!
-  }
-
   const lsDistFolder = function () {
     log('#lsDistFolder')
     const buildFolder = buildDir()
@@ -441,7 +453,9 @@ require('./packages/server')\
   .then(cleanupPlatform)
   .then(buildPackages)
   .then(copyPackages)
+  .then(replaceLocalNpmVersions)
   .then(npmInstallPackages)
+  .then(cleanLocalNpmPackages)
   .then(createRootPackage)
   .then(removeTypeScript)
   .then(cleanJs)

scripts/binary/util/packages.js

@@ -9,7 +9,6 @@ const la = require('lazy-ass')
 const check = require('check-more-types')
 const execa = require('execa')
 const R = require('ramda')
-const os = require('os')
 const prettyMs = require('pretty-ms')
 const pluralize = require('pluralize')
 const debug = require('debug')('cypress:binary')
@@ -56,19 +55,6 @@ const runAllBuild = _.partial(npx, ['lerna', 'run', 'build-prod', '--ignore', 'c
 // removes transpiled JS files in the original package folders
 const runAllCleanJs = _.partial(npx, ['lerna', 'run', 'clean-js', '--ignore', 'cli'])
 
-// @returns string[] with names of packages, e.g. ['runner', 'driver', 'server']
-const getPackagesWithScript = (scriptName) => {
-  return Promise.resolve(glob('./packages/*/package.json'))
-  .map((pkgPath) => {
-    return fs.readJsonAsync(pkgPath)
-    .then((json) => {
-      if (json.scripts != null ? json.scripts.build : undefined) {
-        return path.basename(path.dirname(pkgPath))
-      }
-    })
-  }).filter(Boolean)
-}
-
 const copyAllToDist = function (distDir) {
   const copyRelativePathToDist = function (relative) {
     const dest = path.join(distDir, relative)
@@ -126,7 +112,12 @@ const copyAllToDist = function (distDir) {
 
   return fs.ensureDirAsync(distDir)
   .then(() => {
-    return glob('./packages/*')
+    const globs = ['./packages/*', './npm/*']
+    const globOptions = {
+      onlyFiles: false,
+    }
+
+    return Promise.resolve(externalUtils.globby(globs, globOptions))
     .map(copyPackage, { concurrency: 1 })
   }).then(() => {
     console.log('Finished Copying %dms', new Date() - started)
@@ -135,6 +126,61 @@ const copyAllToDist = function (distDir) {
   })
 }
 
+// replaces local npm version 0.0.0-development
+// with the path to the package
+// we need to do this instead of just changing the symlink (like we do for require('@packages/...'))
+// so the packages actually get installed to node_modules and work with peer dependencies
+const replaceLocalNpmVersions = function (basePath) {
+  const visited = []
+
+  const updateNpmPackage = function (pkg) {
+    if (!visited.includes(pkg)) {
+      visited.push(pkg)
+
+      return updatePackageJson(`./npm/${pkg}/package.json`)
+    }
+
+    return Promise.resolve()
+  }
+
+  const updatePackageJson = function (pattern) {
+    return Promise.resolve(glob(pattern, { cwd: basePath }))
+    .map((pkgPath) => {
+      const pkgJsonPath = path.join(basePath, pkgPath)
+
+      return fs.readJsonAsync(pkgJsonPath)
+      .then((json) => {
+        const { dependencies } = json
+        let shouldWriteFile = false
+
+        if (dependencies) {
+          return Promise.all(_.map(dependencies, (version, pkg) => {
+            const parsedPkg = /(@cypress\/)(.*)/g.exec(pkg)
+
+            if (parsedPkg && parsedPkg.length === 3 && version === '0.0.0-development') {
+              const pkgName = parsedPkg[2]
+
+              json.dependencies[`@cypress/${pkgName}`] = `file:${path.join(basePath, 'npm', pkgName)}`
+              shouldWriteFile = true
+
+              return updateNpmPackage(pkgName)
+            }
+          }))
+          .then(() => {
+            if (shouldWriteFile) {
+              return fs.writeJsonAsync(pkgJsonPath, json, { spaces: 2 })
+            }
+          })
+        }
+
+        return Promise.resolve()
+      })
+    })
+  }
+
+  return updatePackageJson('./packages/*/package.json')
+}
+
 const forceNpmInstall = function (packagePath, packageToInstall) {
   console.log('Force installing %s', packageToInstall)
   console.log('in %s', packagePath)
@@ -217,60 +263,6 @@ const npmInstallAll = function (pathToPackages) {
   })
 }
 
-const removePackageJson = function (filename) {
-  if (filename.endsWith('/package.json')) {
-    return path.dirname(filename)
-  }
-
-  return filename
-}
-
-const ensureFoundSomething = function (files) {
-  if (files.length === 0) {
-    throw new Error('Could not find any files')
-  }
-
-  return files
-}
-
-const symlinkType = function () {
-  if (os.platform() === 'win32') {
-    return 'junction'
-  }
-
-  return 'dir'
-}
-
-const symlinkAll = function (pathToDistPackages, pathTo) {
-  console.log('symlink these packages', pathToDistPackages)
-  la(check.unemptyString(pathToDistPackages),
-    'missing paths to dist packages', pathToDistPackages)
-
-  const symlink = function (pkg) {
-    // console.log(pkg, dist)
-    // strip off the initial './'
-    // ./packages/foo -> node_modules/@packages/foo
-    pkg = removePackageJson(pkg)
-    const dest = pathTo('node_modules', '@packages', path.basename(pkg))
-    const relativeDest = path.relative(`${dest}/..`, pkg)
-
-    const type = symlinkType()
-
-    console.log(relativeDest, 'link ->', dest, 'type', type)
-
-    return fs.ensureSymlinkAsync(relativeDest, dest, symlinkType)
-    .catch((err) => {
-      if (!err.message.includes('EEXIST')) {
-        throw err
-      }
-    })
-  }
-
-  return glob(pathToDistPackages)
-  .then(ensureFoundSomething)
-  .map(symlink)
-}
-
 module.exports = {
   runAllBuild,
 
@@ -278,13 +270,11 @@ module.exports = {
 
   npmInstallAll,
 
-  symlinkAll,
-
   runAllCleanJs,
 
   forceNpmInstall,
 
-  getPackagesWithScript,
+  replaceLocalNpmVersions,
 }
 
 if (!module.parent) {