Fix flaky fast_visit_spec (#4494)

* lint plugins/index.js

* Fix flaky fast_visit_spec by using percentiles

packages/server/__snapshots__/6_visit_spec.coffee.js

@@ -845,8 +845,40 @@ You have set the browser to: 'chrome'
 A video will not be recorded when using this browser.
 
 
-  ✓ always finishes in less than XX:XX on localhost with connection: close
-  ✓ always finishes in less than XX:XX on localhost with connection: keep-alive
+  on localhost 100% of visits are faster than XX:XX, 90% are faster than XX:XX
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+    ✓ with connection: close
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+    ✓ with connection: keep-alive
+
 
   2 passing
 
@@ -899,8 +931,40 @@ exports['e2e visit resolves visits quickly in electron (headless) 1'] = `
   Running: fast_visit_spec.coffee...                                                       (1 of 1) 
 
 
-  ✓ always finishes in less than XX:XX on localhost with connection: close
-  ✓ always finishes in less than XX:XX on localhost with connection: keep-alive
+  on localhost 100% of visits are faster than XX:XX, 90% are faster than XX:XX
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+    ✓ with connection: close
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+histogram line
+    ✓ with connection: keep-alive
+
 
   2 passing
 

packages/server/test/e2e/6_visit_spec.coffee

@@ -171,12 +171,17 @@ describe "e2e visit", ->
       }
     })
 
+    onStdout = (stdout) ->
+      stdout
+      .replace(/^\d+%\s+of visits to [^\s]+ finished in less than.*$/gm, 'histogram line')
+
     it "in chrome (headed)", ->
       e2e.exec(@, {
         spec: "fast_visit_spec.coffee"
         snapshot: true
         expectedExitCode: 0
         browser: 'chrome'
+        onStdout
       })
 
     it "in electron (headless)", ->
@@ -185,4 +190,5 @@ describe "e2e visit", ->
         snapshot: true
         expectedExitCode: 0
         browser: 'electron'
+        onStdout
       })

packages/server/test/support/fixtures/projects/e2e/cypress/integration/fast_visit_spec.coffee

@@ -5,22 +5,40 @@ beforeEach ->
 
   return null
 
-it "always finishes in less than 150ms on localhost with connection: close", ->
-  cy.visit('/close')
+fastVisitSpec = (url) ->
+  cy.visit(url)
+
+  times = []
 
   Cypress._.times 100, ->
-    cy.visit('/close')
+    cy.visit(url)
     .then ->
-      expect(@lastLog.get("totalTime")).to.be.lte(150)
+      time = @lastLog.get("totalTime")
+      times.push(time)
 
-  return undefined
+  cy.then ->
+    times.sort (a, b) ->
+      a - b
 
-it "always finishes in less than 150ms on localhost with connection: keep-alive", ->
-  cy.visit('/close')
+    percentile = (p) ->
+      i = Math.floor(p / 100 * times.length) - 1
+      times[i]
 
-  Cypress._.times 100, ->
-    cy.visit('/keepalive')
-    .then ->
-      expect(@lastLog.get("totalTime")).to.be.lte(150)
+    message = [1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 97, 99, 100].map (p) ->
+      "#{p}%\t of visits to #{url} finished in less than #{percentile(p)}ms"
+    .join("\n")
 
-  return undefined
+    cy.task 'console:log', message
+
+    expect(percentile(90)).to.be.lte(100)
+
+    expect(percentile(100)).to.be.lte(250)
+
+    return undefined
+
+context "on localhost 100% of visits are faster than 250ms, 90% are faster than 100ms", ->
+  it "with connection: close", ->
+    fastVisitSpec '/close'
+
+  it "with connection: keep-alive", ->
+    fastVisitSpec '/keepalive'

packages/server/test/support/fixtures/projects/e2e/cypress/plugins/index.js

@@ -6,13 +6,17 @@ const Promise = require('bluebird')
 module.exports = (on) => {
   // save some time by only reading the originals once
   let cache = {}
+
   function getCachedImage (name) {
     const cachedImage = cache[name]
+
     if (cachedImage) return Promise.resolve(cachedImage)
 
     const imagePath = path.join(__dirname, '..', 'screenshots', `${name}.png`)
+
     return Jimp.read(imagePath).then((image) => {
       cache[name] = image
+
       return image
     })
   }
@@ -52,6 +56,7 @@ module.exports = (on) => {
       }
 
       const comparePath = path.join(__dirname, '..', 'screenshots', `${b}.png`)
+
       return Promise.all([
         getCachedImage(a),
         Jimp.read(comparePath),
@@ -91,5 +96,11 @@ module.exports = (on) => {
         return null
       })
     },
+
+    'console:log' (obj) {
+      console.log(obj) // eslint-disable-line no-console
+
+      return null
+    },
   })
 }