Validate mocha test results in a new stage (#6496)

* wip: validate mocha test results in a new stage

* clean up lerna run commands

* better logging of parse failures

* update expectedCount, integration test command

* revert package.json

* expect a variable amount of tests in e2e

* account for skipped tests

* static has no test script

* fix comparison

* really skip 4_window_open_spec

* fix expectedResultCount

* Update scripts/verify_mocha_results.js

Co-Authored-By: Zach Bloomquist <github@chary.us>

* assert at least 1 test ran by default, always set skipped

* summarize results at end

* verify more stages

* Revert "verify more stages"

This reverts commit dda1339b28.

Co-authored-by: Gleb Bahmutov <gleb.bahmutov@gmail.com>

circle.yml

@@ -98,6 +98,7 @@ commands:
           at: ~/
       - run:
           command: yarn lerna exec --scope @packages/server "yarn test-e2e --chunk << parameters.chunk >> --browser << parameters.browser >>"
+      - verify-mocha-results
       - store_test_results:
           path: /tmp/cypress
       - store-npm-logs
@@ -123,6 +124,17 @@ commands:
               --npm npm-package-url.json \
               --binary binary-url.json
 
+  verify-mocha-results:
+    description: Double-check that Mocha tests ran as expected.
+    parameters:
+      expectedResultCount:
+        description: The number of result files to expect, ie, the number of Mocha test suites that ran.
+        type: integer
+        ## by default, assert that at least 1 test ran
+        default: 0
+    steps:
+      - run: yarn verify:mocha:results << parameters.expectedResultCount >>
+
   test-binary-against-repo:
     description: |
       Takes the built binary and NPM package, clones given example repo
@@ -261,6 +273,8 @@ jobs:
       - run: yarn lerna run build-js --stream
       # run unit tests from each individual package
       - run: yarn test
+      - verify-mocha-results:
+          expectedResultCount: 6
       - store_test_results:
           path: /tmp/cypress
       - store-npm-logs
@@ -288,6 +302,8 @@ jobs:
       - attach_workspace:
           at: ~/
       - run: yarn test-unit --scope @packages/server
+      - verify-mocha-results:
+          expectedResultCount: 1
       - store_test_results:
           path: /tmp/cypress
       - store-npm-logs
@@ -299,6 +315,8 @@ jobs:
       - attach_workspace:
           at: ~/
       - run: yarn test-integration --scope @packages/server
+      - verify-mocha-results:
+          expectedResultCount: 1
       - store_test_results:
           path: /tmp/cypress
       - store-npm-logs
@@ -310,6 +328,8 @@ jobs:
           at: ~/
       - run:
           command: yarn lerna exec --scope @packages/server "yarn test-performance"
+      - verify-mocha-results:
+          expectedResultCount: 1
       - store_test_results:
           path: /tmp/cypress
       - store_artifacts:

package.json

@@ -51,7 +51,7 @@
     "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,src --exclude e2e.coffee,e2e.js",
     "stop-only-all": "yarn stop-only --folder packages",
     "pretest": "yarn ensure-deps",
-    "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{electron,extension,https-proxy,launcher,network,reporter,runner,socket,static}'\"",
+    "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{electron,extension,https-proxy,launcher,network,reporter,runner,socket}'\"",
     "test-debug": "lerna exec yarn test-debug --ignore \"'@packages/{coffee,desktop-gui,driver,root,static,web-config}'\"",
     "pretest-e2e": "yarn ensure-deps",
     "test-e2e": "lerna exec yarn test-e2e --ignore \"'@packages/{coffee,desktop-gui,driver,root,static,web-config}'\"",
@@ -66,6 +66,7 @@
     "test-unit": "lerna exec yarn test-unit --ignore \"'@packages/{coffee,desktop-gui,driver,root,static,web-config}'\"",
     "pretest-watch": "yarn ensure-deps",
     "test-watch": "lerna exec yarn test-watch --ignore \"'@packages/{coffee,desktop-gui,driver,root,static,web-config}'\"",
+    "verify:mocha:results": "node ./scripts/verify_mocha_results",
     "prewatch": "yarn ensure-deps",
     "watch": "lerna exec yarn watch --parallel --stream"
   },

packages/server/test/e2e/4_window_open_spec.coffee

@@ -1,13 +1,13 @@
 e2e = require("../support/helpers/e2e")
 
-describe.skip "e2e window.open", ->
+describe "e2e window.open", ->
   e2e.setup()
 
-  ## skipping this for now due to
-  ## snap-shot-it monkey patching
-  ## causing test failures
-  # it "passes", ->
-  #   e2e.exec(@, {
-  #     spec: "window_open_spec.coffee"
-  #     snapshot: true
-  #   })
+  # skipping this for now due to
+  # snap-shot-it monkey patching
+  # causing test failures
+  it.skip "passes", ->
+    e2e.exec(@, {
+      spec: "window_open_spec.coffee"
+      snapshot: true
+    })

packages/static/package.json

@@ -7,8 +7,7 @@
     "build": "gulp build",
     "build-prod": "yarn build",
     "clean-deps": "rm -rf node_modules",
-    "postinstall": "echo '@packages/static needs: yarn build'",
-    "test": "echo 'Nothing to test yet'"
+    "postinstall": "echo '@packages/static needs: yarn build'"
   },
   "devDependencies": {
     "@cypress/icons": "0.7.0",

scripts/verify_mocha_results.js

@@ -0,0 +1,80 @@
+/* eslint-disable no-console */
+
+// this is a safety script to ensure that Mocha tests ran, by checking:
+// 1. that there are N test results in the reports dir (or at least 1, if N is not set)
+// 2. each of them contains 0 failures and >0 tests
+// usage: yarn verify:mocha:results <N>
+
+const Bluebird = require('bluebird')
+const fse = Bluebird.promisifyAll(require('fs-extra'))
+const la = require('lazy-ass')
+const path = require('path')
+
+const RESULT_REGEX = /<testsuites name="([^"]+)" time="([^"]+)" tests="([^"]+)" failures="([^"]+)"(?: skipped="([^"]+)"|)>/
+const REPORTS_PATH = '/tmp/cypress/junit'
+
+const expectedResultCount = Number(process.argv[process.argv.length - 1])
+
+const parseResult = (xml) => {
+  const [name, time, tests, failures, skipped] = RESULT_REGEX.exec(xml).slice(1)
+
+  return {
+    name, time, tests: Number(tests), failures: Number(failures), skipped: Number(skipped || 0),
+  }
+}
+
+const total = { tests: 0, failures: 0, skipped: 0 }
+
+console.log(`Looking for reports in ${REPORTS_PATH}`)
+
+fse.readdir(REPORTS_PATH)
+.catch((err) => {
+  throw new Error(`Problem reading from ${REPORTS_PATH}: ${err.message}`)
+})
+.then((files) => {
+  const resultCount = files.length
+
+  console.log(`Found ${resultCount} files in ${REPORTS_PATH}:`, files)
+
+  if (!expectedResultCount) {
+    console.log('Expecting at least 1 report...')
+    la(resultCount > 0, 'Expected at least 1 report, but found', resultCount, '. Verify that all tests ran as expected.')
+  } else {
+    console.log(`Expecting exactly ${expectedResultCount} reports...`)
+    la(expectedResultCount === resultCount, 'Expected', expectedResultCount, 'reports, but found', resultCount, '. Verify that all tests ran as expected.')
+  }
+
+  return Bluebird.mapSeries(files, (file) => {
+    console.log(`Checking that ${file} contains a valid report...`)
+
+    return fse.readFile(path.join(REPORTS_PATH, file))
+    .catch((err) => {
+      throw new Error(`Unable to read the report in ${file}: ${err.message}`)
+    })
+    .then((xml) => {
+      try {
+        return parseResult(xml)
+      } catch (err) {
+        throw new Error(`Error parsing result: ${err.message}. File contents:\n\n${xml}`)
+      }
+    })
+    .then(({ name, time, tests, failures, skipped }) => {
+      console.log(`Report parsed successfully. Name: ${name}\tTests ran: ${tests}\tFailing: ${failures}\tSkipped: ${skipped}\tTotal time: ${time}`)
+
+      la(tests > 0, 'Expected the total number of tests to be >0, but it was', tests, 'instead.')
+      la(failures === 0, 'Expected the number of failures to be equal to 0, but it was', failures, '. This stage should not have been reached. Check why the failed test stage did not cause this entire build to fail.')
+
+      total.tests += tests
+      total.failures += failures
+      total.skipped += skipped
+    })
+  })
+})
+.then(() => {
+  console.log('All reports are valid.')
+  console.log(`Total tests ran: ${total.tests}\tTotal failing: ${total.failures}\tTotal skipped: ${total.skipped}`)
+})
+.catch((err) => {
+  console.error(err)
+  process.exit(1)
+})