add printing package folder sizes to the build script (#5978)

- and work around `parse-domain` Jest inclusion to minimize bundle size

.vscode/settings.json

@@ -37,5 +37,13 @@
     "typescript",
     "typescriptreact",
     "json"
-  ]
+  ],
+  "editor.codeActionsOnSave": {
+    "source.fixAll.eslint": true
+  },
+  "[coffeescript]": {
+    "editor.codeActionsOnSave": {
+      "source.fixAll.eslint": false
+    }
+  }
 }

circle.yml

@@ -970,54 +970,71 @@ linux-workflow: &linux-workflow
         requires:
           - build
     - server-performance-tests:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-chrome-1:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-chrome-2:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-chrome-3:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-chrome-4:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-chrome-5:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-chrome-6:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-chrome-7:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-chrome-8:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-electron-1:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-electron-2:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-electron-3:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-electron-4:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-electron-5:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-electron-6:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-electron-7:
+        context: test-runner:performance-tracking
         requires:
           - build
     - server-e2e-tests-electron-8:
+        context: test-runner:performance-tracking
         requires:
           - build
     - driver-integration-tests-chrome:
@@ -1069,6 +1086,7 @@ linux-workflow: &linux-workflow
         requires:
           - build-npm-package
     - build-binary:
+        context: test-runner:performance-tracking
         requires:
           - build
     - upload-binary:

packages/driver/package.json

@@ -3,6 +3,8 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
+    "build": "cd node_modules/parse-domain && npm run postinstall",
+    "build-prod": "npm run build",
     "check-deps": "node ../../scripts/check-deps.js --verbose",
     "check-deps-pre": "npm run check-deps -- --prescript",
     "clean-deps": "rm -rf node_modules",
@@ -52,7 +54,7 @@
     "moment": "2.24.0",
     "morgan": "1.9.1",
     "npm-install-version": "6.0.2",
-    "parse-domain": "2.3.4",
+    "parse-domain": "bahmutov/parse-domain#fb60bd4",
     "setimmediate": "1.0.5",
     "sinon": "3.3.0",
     "text-mask-addons": "3.8.0",

packages/network/package.json

@@ -6,6 +6,7 @@
   "scripts": {
     "build-js": "bin-up tsc --project .",
     "clean-deps": "rm -rf node_modules",
+    "postinstall": "cd node_modules/parse-domain && npm run postinstall",
     "test": "bin-up mocha --reporter mocha-multi-reporters --reporter-options configFile=../../mocha-reporter-config.json"
   },
   "dependencies": {
@@ -13,7 +14,7 @@
     "concat-stream": "1.6.2",
     "debug": "4.1.1",
     "lodash": "4.17.15",
-    "parse-domain": "2.3.4",
+    "parse-domain": "bahmutov/parse-domain#fb60bd4",
     "proxy-from-env": "1.0.0"
   },
   "devDependencies": {

packages/server/test/support/helpers/performance.js

@@ -3,6 +3,8 @@ const { commitInfo } = require('@cypress/commit-info')
 const pkg = require('../../../../../package.json')
 const Promise = require('bluebird')
 const rp = require('request-promise')
+const debug = require('debug')('cypress:performance')
+const R = require('ramda')
 
 const API_URL = process.env.PERF_API_URL || 'http://localhost:2999/track'
 const API_KEY = process.env.PERF_API_KEY
@@ -10,14 +12,19 @@ const API_KEY = process.env.PERF_API_KEY
 // Store this performance record permanently.
 function track (type, data) {
   if (!API_KEY) {
+    debug('skip tracking "%s", API key is not set', type)
+
     return Promise.resolve()
   }
 
-  return commitInfo()
-  .then(({ message, timestamp }) => {
-    const { sha, branch, author } = ciProvider.commitParams()
+  debug('getting commit information')
 
-    timestamp = new Date(timestamp * 1000).toISOString()
+  return commitInfo()
+  .then((commitInformation) => {
+    const ciInformation = ciProvider.commitParams() || {}
+    const merged = R.mergeWith(R.or, commitInformation, ciInformation)
+    const { sha, branch, author, message, timestamp } = merged
+    const timestampISO = new Date(timestamp * 1000).toISOString()
 
     const body = {
       type,
@@ -28,7 +35,7 @@ function track (type, data) {
         'Commit Branch': branch,
         'Commit Author': author,
         'Commit Message': message,
-        'Commit Timestamp': timestamp,
+        'Commit Timestamp': timestampISO,
         'Build URL': process.env.CIRCLE_BUILD_URL,
         'Build Platform': process.platform,
         'Build Arch': process.arch,
@@ -36,6 +43,9 @@ function track (type, data) {
       },
     }
 
+    debug('sending performance numbers "%s"', type)
+    debug('%o', body.data)
+
     return rp.post({
       url: API_URL,
       json: true,
@@ -46,7 +56,10 @@ function track (type, data) {
       timeout: 5000,
     })
   })
-  .catchReturn()
+  .catch((err) => {
+    /* eslint-disable no-console */
+    console.error('Track error for type %s %s', type, err.message)
+  })
 }
 
 module.exports = {

scripts/binary/build.coffee

@@ -27,6 +27,7 @@ xvfb = require("../../cli/lib/exec/xvfb")
 linkPackages = require('../link-packages')
 { transformRequires } = require('./util/transform-requires')
 { testStaticAssets } = require('./util/testStaticAssets')
+performanceTracking = require('../../packages/server/test/support/helpers/performance.js')
 
 rootPackage = require("@packages/root")
 
@@ -301,6 +302,46 @@ buildCypressApp = (platform, version, options = {}) ->
         else
           reject new Error("Verifying App via GateKeeper failed")
 
+  printPackageSizes = ->
+    appFolder = meta.buildAppDir(platform, "packages")
+    log("#printPackageSizes #{appFolder}")
+
+    if (platform == "win32") then return Promise.resolve()
+
+    # "du" - disk usage utility
+    # -d -1 depth of 1
+    # -h human readable sizes (K and M)
+    args = ["-d", "1", appFolder]
+
+    parseDiskUsage = (result) ->
+      lines = result.stdout.split(os.EOL)
+      # will store {package name: package size}
+      data = {}
+
+      lines.forEach (line) ->
+        parts = line.split('\t')
+        packageSize = parseFloat(parts[0])
+        folder = parts[1]
+
+        packageName = path.basename(folder)
+        if packageName is "packages"
+          return # root "packages" information
+
+        data[packageName] = packageSize
+
+      return data
+
+    printDiskUsage = (sizes) ->
+      bySize = R.sortBy(R.prop('1'))
+      console.log(bySize(R.toPairs(sizes)))
+
+    execa("du", args)
+    .then(parseDiskUsage)
+    .then(R.tap(printDiskUsage))
+    .then((sizes) ->
+      performanceTracking.track('test runner size', sizes)
+    )
+
   Promise.resolve()
   .then(checkPlatform)
   .then(cleanupPlatform)
@@ -320,6 +361,7 @@ buildCypressApp = (platform, version, options = {}) ->
   .then(runSmokeTests)
   .then(codeSign) ## codesign after running smoke tests due to changing .cy
   .then(verifyAppCanOpen)
+  .then(printPackageSizes)
   .return({
     buildDir: buildDir()
   })

scripts/binary/upload-npm-package.coffee

@@ -10,8 +10,6 @@ gulpDebug = require('gulp-debug')
 gulp = require("gulp")
 human = require("human-interval")
 R = require("ramda")
-
-konfig = require('../binary/get-config')()
 uploadUtils = require("./util/upload")
 
 npmPackageExtension = ".tgz"
@@ -30,7 +28,9 @@ getCDN = ({version, hash, filename}) ->
   la(check.unemptyString(hash), 'missing hash', hash)
   la(check.unemptyString(filename), 'missing filename', filename)
   la(isNpmPackageFile(filename), 'wrong extension for file', filename)
-  [konfig("cdn_url"), rootFolder, npmFolder, version, hash, filename].join("/")
+  url = uploadUtils.getUploadUrl()
+  la(check.url(url), "could not get upload url", url)
+  [url, rootFolder, npmFolder, version, hash, filename].join("/")
 
 getUploadDirName = (options) ->
   la(check.unemptyString(options.version), 'missing version', options)

scripts/binary/upload-unique-binary.coffee

@@ -12,7 +12,6 @@ human = require("human-interval")
 R = require("ramda")
 hasha = require('hasha')
 
-konfig = require('../binary/get-config')()
 uploadUtils = require("./util/upload")
 s3helpers = require("./s3-api").s3helpers
 
@@ -35,7 +34,8 @@ getCDN = ({version, hash, filename, platform}) ->
   la(isBinaryFile(filename), 'wrong extension for file', filename)
   la(check.unemptyString(platform), 'missing platform', platform)
 
-  cdnUrl = konfig("cdn_url")
+  cdnUrl = uploadUtils.getUploadUrl()
+  la(check.url(cdnUrl), "could not get cdn url", cdnUrl)
   [cdnUrl, rootFolder, folder, version, platform, hash, filename].join("/")
 
 # returns folder that contains beta (unreleased) binaries for given version

scripts/binary/upload.coffee

@@ -6,7 +6,6 @@ cp      = require("child_process")
 path    = require("path")
 gulp    = require("gulp")
 human   = require("human-interval")
-konfig  = require('../binary/get-config')()
 Promise = require("bluebird")
 meta    = require("./meta")
 la      = require("lazy-ass")
@@ -59,8 +58,10 @@ module.exports = {
     dirName
 
   getManifestUrl: (folder, version, uploadOsName) ->
+    url = uploadUtils.getUploadUrl()
+    la(check.url(url), "could not get upload url", url)
     {
-      url: [konfig('cdn_url'), folder, version, uploadOsName, zipName].join("/")
+      url: [url, folder, version, uploadOsName, zipName].join("/")
     }
 
   getRemoteManifest: (folder, version) ->

scripts/binary/util/upload.coffee

@@ -9,9 +9,15 @@ fse = require("fs-extra")
 os = require("os")
 Promise = require("bluebird")
 {configFromEnvOrJsonFile, filenameToShellVariable} = require('@cypress/env-or-json-file')
-konfig = require('../../binary/get-config')()
+konfig = require('../get-config')()
 { purgeCloudflareCache } = require('./purge-cloudflare-cache')
 
+getUploadUrl = () ->
+  url = konfig("cdn_url")
+  la(check.url(url), "could not get CDN url", url)
+  console.log("upload url", url)
+  url
+
 formHashFromEnvironment = () ->
   env = process.env
   if env.BUILDKITE
@@ -59,7 +65,8 @@ getPublisher = (getAwsObj = getS3Credentials) ->
   }
 
 getDesktopUrl = (version, osName, zipName) ->
-  [konfig("cdn_url"), "desktop", version, osName, zipName].join("/")
+  url = getUploadUrl()
+  [url, "desktop", version, osName, zipName].join("/")
 
 # purges desktop application url from Cloudflare cache
 purgeDesktopAppFromCache = ({version, platform, zipName}) ->
@@ -135,5 +142,6 @@ module.exports = {
   getValidPlatformArchs,
   isValidPlatformArch,
   saveUrl,
-  formHashFromEnvironment
+  formHashFromEnvironment,
+  getUploadUrl
 }

scripts/unit/konfig-spec.js

@@ -8,7 +8,7 @@ describe('konfig check', () => {
     script tests should NOT suddenly change the current working directory to
     packages/server - otherwise the local path filenames might be all wrong
     and unexpected. The current working directory changes when we
-    require `packages/server/lib/konfig` which in tern requires
+    require `packages/server/lib/konfig` which in turn requires
     `lib/cwd` which changes CWD.
 
     From the scripts unit tests we should not use `lib/konfig` directly,