Retry certain requests on failure (#4015)

* retry requests, basic e2e test

* basic e2e test for chrome behavior

* don't use timeouts to test

* some minor cleanup

* validate google chrome's retry behavior w/ a proxy

* get retries on network errors workin

* cleanup

* final changes before switching approach

* Reverting previous approach in request.coffee, server.coffee

* add retryOnNetworkFailure

* now works with retryOnStatusCodeFailure

* retry 4 times in addition to the 1 initial attempt

* add tests for subresources

* much improved error handling

* have the e2e test really work

* e2e baseurl check

* retry baseurl check

* remove special handling for node 8.2.1 bug

* WIP: continue making progress building out request retry

- swap out passthrough’s for pumpify / duplexify / pump
- clean up error handling / messaging / retry logic

* pipe clientrequest events

* buffer req body to disk, restore error handling/retrying

* don't bubble up errors from failed attempts

* actually pipe reqstream, oops

* add some e2e tests for request body

* revert lib/request.coffee to 7be1051

* add almost-working lazy-stream

* manually fire the 'pipe' event on the reqStream to copy headers to the outgoing message

- restore the ‘error’ propagation so that all tests pass for now

* cleanup leaking 'undefined' into stdout, causing failing e2e tests

- do not set onWarning as a property of project, just pass as an
argument

* add new options to request_spec, deduplicate default opts

* use stream buffer in request.coffee

* revert request.coffee

* last stream_buffer changes before removing fs stuff

* remove fs stuff from stream_buffer, add stream piping tests

* it works! :tada::tada:🎉 using duplexify to separate delayStream and reqBodyBuffer

* retry for .5s max on ECONNREFUSED

* add error codes

* don't timeout proxied requests

* restore baseurl check

* update new e2e tests

* make delay work with rp

* propagate clientresponse events

* removing tests that don't do anything now that we don't ESOCKETTIMEOUT on proxied requests

* add new visit, req options to index.d.ts

* don't fail on server-performance-test

* make retries with status code work again

* account for different stack trace in ci

* fix test

* retry https requests

* add tests for https passthru retries working

* clean up error handling for https-proxy

* fix failing https-proxy tests, tweak agent error handling to prevent multiple callbacks

* make expectedExitCode actual vs. expected in the correct order

* bump up e2e test timeout so it can retry and still work

* update tests

* retry up to 500ms on proxied requests

* add tests for incompatible options errors

* remove .only

* maybe this will help it act more consistently

* help e2e test work in ci

* don't reconnect on already made connections

* clarify naming

* wip: testing https proxy

* better debug calls

* WIP: getting proxy passthrough retry tests going

* handle retrying upstream proxy errors

- add tests for successfully retrying proxy errors and for unsuccessful
retries
- fix onClose errors when proxy connection is closed before enough data
is received
- fix not returning setTimeout correctly

* group related code accordingly

* do not build typescript by default, drop extension from main

* more TODO notes

* don't set a default NO_PROXY if NO_PROXY = ''

* debugging-proxy@2.0.0

* null out reqBodyBuffer when req finishes

* wip: retry in agent, not https-proxy [skip-ci]

* update https-proxy to use @packages/network retries

* retry after connection but before proxy tunnel established

* appease my linty overlords

* update https-proxy tests

* update agent specs, decided to still use tls.connect

it's easier to test and has less complexity

* test retrying HTTPS passthru

* debugging-proxy@2.0.1

* increase defaultCommandTimeout 100 -> 200 to prevent flake in CI

* auto formatting

* fix test to be dynamic and not rely on magic constants

* copy types field when linking proxy images, update packages/network types field

* linting

* add network index.js file

* linting

* improve error messaging experience when verifying base url

* only insert 1 new line

* fix failing test not binding to localhost

* removed test that's covered by e2e specs

* remove dash in 're-try'

* some cleanup for readability

* use allocUnsafe per perf

* unset NO_PROXY with an empty string

* move retry ensuring the baseUrl into url, cleanup some imperative code

* if the head is already present during connect, make connection, else wait for first head bytes

* minor formatting, clarity around conditions, naming

* rename retryInterval -> retryIntervals

* set defaults for requests during creation for clarity

* rename send -> sendPromise to pair better with sendStream

* use retryIntervals instead of juggling MAX_REQUEST_RETRIES

- ensure debug messages are consistent between request streams +
promises
- set static constants

* DRY up status check + network failure + retry logic

- keeps the debug logic identical between promises + streams
- ensures all logic paths are also consistent
- consolidates the pop’ing of intervals in a single place

* find / replace fail

* derp

* make the logic actually correct, set intervals as cloned defaults for recursive lookup

* pass arg correctly

* reduce debugging noise, formatting

* rename intervals -> delaysRemaining for clarity

* added unit tests around getDelayForRetry

* set retryIntervals as default options correctly, add unit tests


Co-authored-by: Brian Mann <brian.mann86@gmail.com>
Co-authored-by: Jennifer Shehane <jennifer@cypress.io>
Co-authored-by: Gleb Bahmutov <gleb.bahmutov@gmail.com>

scripts/link-packages.js

@@ -30,7 +30,7 @@ function deleteOutputFolder () {
   .catch(_.noop)
 }
 
-function proxyModule (name, pathToMain, pathToBrowser) {
+function proxyModule (name, pathToMain, pathToBrowser, pathToTypes) {
   la(is.unemptyString(name), 'missing name')
   la(is.unemptyString(pathToMain), 'missing path to main', pathToMain)
   la(isRelative(pathToMain), 'path to main should be relative', pathToMain)
@@ -43,11 +43,23 @@ function proxyModule (name, pathToMain, pathToBrowser) {
   }
 
   if (pathToBrowser) {
-    la(isRelative(pathToBrowser),
-      'path to browser module should be relative', pathToBrowser)
+    la(
+      isRelative(pathToBrowser),
+      'path to browser module should be relative',
+      pathToBrowser
+    )
     pkg.browser = pathToBrowser
   }
 
+  if (pathToTypes) {
+    la(
+      isRelative(pathToTypes),
+      'path to types file should be relative',
+      pathToTypes
+    )
+    pkg.types = pathToTypes
+  }
+
   return pkg
 }
 
@@ -62,12 +74,10 @@ function needsRegister (name) {
 function makeProxies () {
   return glob('./packages/*/package.json')
   .map((filename) => {
-    return fs.readJsonAsync(filename)
-    .then((json) => {
+    return fs.readJsonAsync(filename).then((json) => {
       return { filename, json }
     })
-  }
-  )
+  })
   .map(({ filename, json }) => {
     if (!json.main) {
       throw new Error(`Package ${json.name} is missing main`)
@@ -93,22 +103,45 @@ function makeProxies () {
 
     if (is.unemptyString(json.browser)) {
       debug('package has browser field %s', json.browser)
-      relativePathToBrowser = path.relative(destinationFolder,
+      relativePathToBrowser = path.relative(
+        destinationFolder,
         path.resolve(dirname, json.browser)
       )
       debug('relative path to browser', relativePathToBrowser)
     }
 
-    const proxy = proxyModule(json.name, relativePathToMain, relativePathToBrowser)
+    // if the package has types field, compute new path to it
+    let relativePathTypes
+
+    if (is.unemptyString(json.types)) {
+      debug('package has types field %s', json.types)
+      relativePathTypes = path.relative(
+        destinationFolder,
+        path.resolve(dirname, json.types)
+      )
+      debug('relative path to types', relativePathTypes)
+    }
+
+    const proxy = proxyModule(
+      json.name,
+      relativePathToMain,
+      relativePathToBrowser,
+      relativePathTypes
+    )
 
     console.log(path.dirname(destPackageFilename), '->', relativePathToMain)
 
-    return fs.outputJsonAsync(destPackageFilename, proxy)
+    return fs
+    .outputJsonAsync(destPackageFilename, proxy, { spaces: 2 })
     .then(() => {
       if (needsRegister(json.name)) {
         console.log('adding register file', registerPath)
 
-        return fs.outputFileAsync(registerPath, proxyRegister(bareName), 'utf8')
+        return fs.outputFileAsync(
+          registerPath,
+          proxyRegister(bareName),
+          'utf8'
+        )
       }
     })
   })