Merge branch 'develop' into feature-multidomain

packages/proxy/lib/http/error-middleware.ts

@@ -3,6 +3,7 @@ import { HttpMiddleware } from '.'
 import { InterceptError } from '@packages/net-stubbing'
 import { Readable } from 'stream'
 import { Request } from '@cypress/request'
+import errors from '@packages/server/lib/errors'
 
 const debug = debugModule('cypress:proxy:http:error-middleware')
 
@@ -22,6 +23,17 @@ const LogError: ErrorMiddleware = function () {
   this.next()
 }
 
+const SendToDriver: ErrorMiddleware = function () {
+  if (this.req.browserPreRequest) {
+    this.socket.toDriver('request:event', 'request:error', {
+      requestId: this.req.browserPreRequest.requestId,
+      error: errors.clone(this.error),
+    })
+  }
+
+  this.next()
+}
+
 export const AbortRequest: ErrorMiddleware = function () {
   if (this.outgoingReq) {
     debug('aborting outgoingReq')
@@ -47,6 +59,7 @@ export const DestroyResponse: ErrorMiddleware = function () {
 
 export default {
   LogError,
+  SendToDriver,
   InterceptError,
   AbortRequest,
   UnpipeResponse,

packages/proxy/lib/http/index.ts

@@ -56,6 +56,7 @@ export type ServerCtx = Readonly<{
   shouldCorrelatePreRequests?: () => boolean
   getFileServerToken: () => string
   getRemoteState: CyServer.getRemoteState
+  getRenderedHTMLOrigins: Http['getRenderedHTMLOrigins']
   netStubbingState: NetStubbingState
   middleware: HttpMiddlewareStacks
   socket: CyServer.Socket
@@ -176,6 +177,16 @@ export function _runStage (type: HttpStages, ctx: any, onError) {
   return runMiddlewareStack()
 }
 
+function getUniqueRequestId (requestId: string) {
+  const match = /^(.*)-retry-([\d]+)$/.exec(requestId)
+
+  if (match) {
+    return `${match[1]}-retry-${Number(match[2]) + 1}`
+  }
+
+  return `${requestId}-retry-1`
+}
+
 export class Http {
   buffers: HttpBuffers
   config: CyServer.Config
@@ -188,6 +199,7 @@ export class Http {
   preRequests: PreRequests = new PreRequests()
   request: any
   socket: CyServer.Socket
+  renderedHTMLOrigins: {[key: string]: boolean} = {}
 
   constructor (opts: ServerCtx & { middleware?: HttpMiddlewareStacks }) {
     this.buffers = new HttpBuffers()
@@ -229,6 +241,7 @@ export class Http {
           ...opts,
         })
       },
+      getRenderedHTMLOrigins: this.getRenderedHTMLOrigins,
       getPreRequest: (cb) => {
         this.preRequests.get(ctx.req, ctx.debug, cb)
       },
@@ -237,9 +250,14 @@ export class Http {
     const onError = () => {
       if (ctx.req.browserPreRequest) {
         // browsers will retry requests in the event of network errors, but they will not send pre-requests,
-        // so try to re-use the current browserPreRequest for the next retry
-        ctx.debug('Re-using pre-request data %o', ctx.req.browserPreRequest)
-        this.addPendingBrowserPreRequest(ctx.req.browserPreRequest)
+        // so try to re-use the current browserPreRequest for the next retry after incrementing the ID.
+        const preRequest = {
+          ...ctx.req.browserPreRequest,
+          requestId: getUniqueRequestId(ctx.req.browserPreRequest.requestId),
+        }
+
+        ctx.debug('Re-using pre-request data %o', preRequest)
+        this.addPendingBrowserPreRequest(preRequest)
       }
     }
 
@@ -253,6 +271,10 @@ export class Http {
     })
   }
 
+  getRenderedHTMLOrigins = () => {
+    return this.renderedHTMLOrigins
+  }
+
   async handleSourceMapRequest (req: Request, res: Response) {
     try {
       const sm = await this.deferredSourceMapCache.resolve(req.params.id, req.headers)
@@ -269,7 +291,6 @@ export class Http {
 
   reset () {
     this.buffers.reset()
-    this.preRequests = new PreRequests()
   }
 
   setBuffer (buffer) {

packages/proxy/lib/http/request-middleware.ts

@@ -27,6 +27,25 @@ const CorrelateBrowserPreRequest: RequestMiddleware = async function () {
   if (this.req.headers['x-cypress-resolving-url']) {
     this.debug('skipping prerequest for resolve:url')
     delete this.req.headers['x-cypress-resolving-url']
+    const requestId = `cy.visit-${Date.now()}`
+
+    this.req.browserPreRequest = {
+      requestId,
+      method: this.req.method,
+      url: this.req.proxiedUrl,
+      // @ts-ignore
+      headers: this.req.headers,
+      resourceType: 'document',
+      originalResourceType: 'document',
+    }
+
+    this.res.on('close', () => {
+      this.socket.toDriver('request:event', 'response:received', {
+        requestId,
+        headers: this.res.getHeaders(),
+        status: this.res.statusCode,
+      })
+    })
 
     return this.next()
   }
@@ -42,7 +61,7 @@ const SendToDriver: RequestMiddleware = function () {
   const { browserPreRequest } = this.req
 
   if (browserPreRequest) {
-    this.socket.toDriver('proxy:incoming:request', browserPreRequest)
+    this.socket.toDriver('request:event', 'incoming:request', browserPreRequest)
   }
 
   this.next()
@@ -167,9 +186,9 @@ const SendRequestOutgoing: RequestMiddleware = function () {
 
 export default {
   LogRequest,
+  MaybeEndRequestWithBufferedResponse,
   CorrelateBrowserPreRequest,
   SendToDriver,
-  MaybeEndRequestWithBufferedResponse,
   InterceptRequest,
   RedirectToClientRouteIfUnloaded,
   EndRequestsToBlockedHosts,

packages/proxy/lib/http/response-middleware.ts

@@ -230,6 +230,14 @@ const PatchExpressSetHeader: ResponseMiddleware = function () {
 const SetInjectionLevel: ResponseMiddleware = function () {
   this.res.isInitial = this.req.cookies['__cypress.initial'] === 'true'
 
+  const isRenderedHTML = reqWillRenderHtml(this.req)
+
+  if (isRenderedHTML) {
+    const origin = new URL(this.req.proxiedUrl).origin
+
+    this.getRenderedHTMLOrigins()[origin] = true
+  }
+
   const isReqMatchOriginPolicy = reqMatchesOriginPolicy(this.req, this.getRemoteState())
   const getInjectionLevel = () => {
     if (this.incomingRes.headers['x-cypress-file-server-error'] && !this.res.isInitial) {
@@ -248,7 +256,7 @@ const SetInjectionLevel: ResponseMiddleware = function () {
       return 'full'
     }
 
-    if (!reqWillRenderHtml(this.req)) {
+    if (!isRenderedHTML) {
       return false
     }
 

packages/proxy/lib/types.ts

@@ -32,7 +32,7 @@ export { RequestMiddleware } from './http/request-middleware'
 
 export { ResponseMiddleware } from './http/response-middleware'
 
-export type ResourceType = 'fetch' | 'xhr' | 'websocket' | 'stylesheet' | 'script' | 'image' | 'font' | 'cspviolationreport' | 'ping' | 'manifest' | 'other'
+export type ResourceType = 'document' | 'fetch' | 'xhr' | 'websocket' | 'stylesheet' | 'script' | 'image' | 'font' | 'cspviolationreport' | 'ping' | 'manifest' | 'other'
 
 /**
  * Metadata about an HTTP request, according to the browser's pre-request event.
@@ -41,6 +41,21 @@ export type BrowserPreRequest = {
   requestId: string
   method: string
   url: string
+  headers: { [key: string]: string | string[] }
   resourceType: ResourceType
   originalResourceType: string | undefined
 }
+
+/**
+ * Notification that the browser has received a response for a request for which a pre-request may have been emitted.
+ */
+export type BrowserResponseReceived = {
+  requestId: string
+  status: number
+  headers: { [key: string]: string | string[] }
+}
+
+export type RequestError = {
+  requestId: string
+  error: any
+}

packages/proxy/test/integration/net-stubbing.spec.ts

@@ -2,7 +2,7 @@ import { NetworkProxy } from '../../'
 import {
   netStubbingState as _netStubbingState,
   NetStubbingState,
-  onNetEvent,
+  onNetStubbingEvent,
 } from '@packages/net-stubbing'
 import { defaultMiddleware } from '../../lib/http'
 import express from 'express'
@@ -166,7 +166,7 @@ context('network stubbing', () => {
 
     socket.toDriver.callsFake((_, event, data) => {
       if (event === 'before:request') {
-        onNetEvent({
+        onNetStubbingEvent({
           eventName: 'send:static:response',
           // @ts-ignore
           frame: {
@@ -234,7 +234,7 @@ context('network stubbing', () => {
     socket.toDriver.callsFake((_, event, data) => {
       if (event === 'before:request') {
         sendContentLength = data.data.headers['content-length']
-        onNetEvent({
+        onNetStubbingEvent({
           eventName: 'send:static:response',
           // @ts-ignore
           frame: {

packages/proxy/test/unit/http/error-middleware.spec.ts

@@ -14,6 +14,7 @@ describe('http/error-middleware', function () {
   it('exports the members in the correct order', function () {
     expect(_.keys(ErrorMiddleware)).to.have.ordered.members([
       'LogError',
+      'SendToDriver',
       'InterceptError',
       'AbortRequest',
       'UnpipeResponse',

packages/proxy/test/unit/http/request-middleware.spec.ts

@@ -6,9 +6,9 @@ describe('http/request-middleware', function () {
   it('exports the members in the correct order', function () {
     expect(_.keys(RequestMiddleware)).to.have.ordered.members([
       'LogRequest',
+      'MaybeEndRequestWithBufferedResponse',
       'CorrelateBrowserPreRequest',
       'SendToDriver',
-      'MaybeEndRequestWithBufferedResponse',
       'InterceptRequest',
       'RedirectToClientRouteIfUnloaded',
       'EndRequestsToBlockedHosts',