cypress/packages/server/lib/request.js

/*
 * decaffeinate suggestions:
 * DS102: Remove unnecessary code created because of implicit returns
 * DS103: Rewrite code to no longer use __guard__
 * DS207: Consider shorter variations of null checks
 * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
 */
const _          = require("lodash");
let r          = require("@cypress/request");
let rp         = require("@cypress/request-promise");
const url        = require("url");
const tough      = require("tough-cookie");
const debug      = require("debug")("cypress:server:request");
const Promise    = require("bluebird");
const stream     = require("stream");
const duplexify  = require("duplexify");
const {
  agent
} = require("@packages/network");
const statusCode = require("./util/status_code");
const {
  streamBuffer
} = require("./util/stream_buffer");

const SERIALIZABLE_COOKIE_PROPS = ['name', 'value', 'domain', 'expiry', 'path', 'secure', 'hostOnly', 'httpOnly', 'sameSite'];
const NETWORK_ERRORS = "ECONNREFUSED ECONNRESET EPIPE EHOSTUNREACH EAI_AGAIN ENOTFOUND".split(" ");
const VERBOSE_REQUEST_OPTS = "followRedirect strictSSL".split(" ");
const HTTP_CLIENT_REQUEST_EVENTS = "abort connect continue information socket timeout upgrade".split(" ");
const TLS_VERSION_ERROR_RE =  /TLSV1_ALERT_PROTOCOL_VERSION|UNSUPPORTED_PROTOCOL/;
const SAMESITE_NONE_RE = /; +samesite=(?:'none'|"none"|none)/i;

process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";

const convertSameSiteToughToExtension = (sameSite, setCookie) => {
  //# tough-cookie@4.0.0 uses 'none' as a default, so run this regex to detect if
  //# SameSite=None was not explicitly set
  //# @see https://github.com/salesforce/tough-cookie/issues/191
  const isUnspecified = (sameSite === "none") && !SAMESITE_NONE_RE.test(setCookie);

  if (isUnspecified) {
    //# not explicitly set, so fall back to the browser's default
    return undefined;
  }

  if (sameSite === 'none') {
    return 'no_restriction';
  }

  return sameSite;
};

const getOriginalHeaders = (req = {}) => //# the request instance holds an instance
//# of the original ClientRequest
//# as the 'req' property which holds the
//# original headers else fall back to
//# the normal req.headers
_.get(req, 'req.headers', req.headers);

const getDelayForRetry = function(options = {}) {
  const { err, opts, delaysRemaining, retryIntervals, onNext, onElse } = options;

  let delay = delaysRemaining.shift();

  if (!_.isNumber(delay)) {
    //# no more delays, bailing
    debug("exhausted all attempts retrying request %o", merge(opts, { err }));

    return onElse();
  }

  //# figure out which attempt we're on...
  const attempt = retryIntervals.length - delaysRemaining.length;

  //# if this ECONNREFUSED and we are
  //# retrying greater than 1 second
  //# then divide the delay interval
  //# by 10 so it doesn't wait as long to retry
  //# TODO: do we really want to do this?
  if ((delay >= 1000) && (_.get(err, "code") === "ECONNREFUSED")) {
    delay = delay / 10;
  }

  debug("retrying request %o", merge(opts, {
    delay,
    attempt,
  }));

  return onNext(delay, attempt);
};

const hasRetriableStatusCodeFailure = (res, retryOnStatusCodeFailure) => //# everything must be true in order to
//# retry a status code failure
_.every([
  retryOnStatusCodeFailure,
  !statusCode.isOk(res.statusCode)
]);

const isRetriableError = (err = {}, retryOnNetworkFailure) => _.every([
  retryOnNetworkFailure,
  _.includes(NETWORK_ERRORS, err.code)
]);

const maybeRetryOnNetworkFailure = function(err, options = {}) {
  const {
    opts,
    retryIntervals,
    delaysRemaining,
    retryOnNetworkFailure,
    onNext,
    onElse,
  } = options;

  debug("received an error making http request %o", merge(opts, { err }));

  const isTlsVersionError = TLS_VERSION_ERROR_RE.test(err.message);

  if (isTlsVersionError) {
    //# because doing every connection via TLSv1 can lead to slowdowns, we set it only on failure
    //# https://github.com/cypress-io/cypress/pull/6705
    debug('detected TLS version error, setting min version to TLSv1');
    opts.minVersion = 'TLSv1';
  }

  if (!isTlsVersionError && !isRetriableError(err, retryOnNetworkFailure)) {
    return onElse();
  }

  //# else see if we have more delays left...
  return getDelayForRetry({
    err,
    opts,
    retryIntervals,
    delaysRemaining,
    onNext,
    onElse,
  });
};

const maybeRetryOnStatusCodeFailure = function(res, options = {}) {
  const {
    err,
    opts,
    requestId,
    retryIntervals,
    delaysRemaining,
    retryOnStatusCodeFailure,
    onNext,
    onElse,
  } = options;

  debug("received status code & headers on request %o", {
    requestId,
    statusCode: res.statusCode,
    headers: _.pick(res.headers, 'content-type', 'set-cookie', 'location')
  });

  //# is this a retryable status code failure?
  if (!hasRetriableStatusCodeFailure(res, retryOnStatusCodeFailure)) {
    //# if not then we're done here
    return onElse();
  }

  //# else see if we have more delays left...
  return getDelayForRetry({
    err,
    opts,
    retryIntervals,
    delaysRemaining,
    onNext,
    onElse,
  });
};

var merge = (...args) => _.chain({})
.extend(...args)
.omit(VERBOSE_REQUEST_OPTS)
.value();

const pick = function(resp = {}) {
  const req = resp.request != null ? resp.request : {};

  const headers = getOriginalHeaders(req);

  return {
    "Request Body":     req.body != null ? req.body : null,
    "Request Headers":  headers,
    "Request URL":      req.href,
    "Response Body":    resp.body != null ? resp.body : null,
    "Response Headers": resp.headers,
    "Response Status":  resp.statusCode
  };
};

var createRetryingRequestPromise = function(opts) {
  const {
    requestId,
    retryIntervals,
    delaysRemaining,
    retryOnNetworkFailure,
    retryOnStatusCodeFailure
  } = opts;

  const retry = delay => Promise.delay(delay)
  .then(() => createRetryingRequestPromise(opts));

  return rp(opts)
  .catch(err => //# rp wraps network errors in a RequestError, so might need to unwrap it to check
  maybeRetryOnNetworkFailure(err.error || err, {
    opts,
    retryIntervals,
    delaysRemaining,
    retryOnNetworkFailure,
    onNext: retry,
    onElse() {
      throw err;
    }
  })).then(res => //# ok, no net error, but what about a bad status code?
  maybeRetryOnStatusCodeFailure(res, {
    opts,
    requestId,
    retryIntervals,
    delaysRemaining,
    retryOnStatusCodeFailure,
    onNext: retry,
    onElse: _.constant(res)
  }));
};

const pipeEvent = (source, destination, event) => source.on(event, (...args) => destination.emit(event, ...args));

const createRetryingRequestStream = function(opts = {}) {
  const {
    requestId,
    retryIntervals,
    delaysRemaining,
    retryOnNetworkFailure,
    retryOnStatusCodeFailure
  } = opts;

  let req = null;

  const delayStream = stream.PassThrough();
  let reqBodyBuffer = streamBuffer();
  const retryStream = duplexify(reqBodyBuffer, delayStream);

  const cleanup = function() {
    if (reqBodyBuffer) {
      //# null req body out to free memory
      reqBodyBuffer.unpipeAll();
      return reqBodyBuffer = null;
    }
  };

  const emitError = function(err) {
    retryStream.emit("error", err);

    return cleanup();
  };

  var tryStartStream = function() {
    //# if our request has been aborted
    //# in the time that we were waiting to retry
    //# then immediately bail
    if (retryStream.aborted) {
      return;
    }

    const reqStream = r(opts);
    let didReceiveResponse = false;

    const retry = function(delay, attempt) {
      retryStream.emit("retry", { attempt, delay });

      return setTimeout(tryStartStream, delay);
    };

    //# if we're retrying and we previous piped
    //# into the reqStream, then reapply this now
    if (req) {
      reqStream.emit('pipe', req);
      reqBodyBuffer.createReadStream().pipe(reqStream);
    }

    //# forward the abort call to the underlying request
    retryStream.abort = function() {
      debug('aborting', { requestId });
      retryStream.aborted = true;

      return reqStream.abort();
    };

    const onPiped = function(src) {
      //# store this IncomingMessage so we can reapply it
      //# if we need to retry
      req = src;

      //# https://github.com/request/request/blob/b3a218dc7b5689ce25be171e047f0d4f0eef8919/request.js#L493
      //# the request lib expects this 'pipe' event in
      //# order to copy the request headers onto the
      //# outgoing message - so we manually pipe it here
      return src.pipe(reqStream);
    };

    //# when this passthrough stream is being piped into
    //# then make sure we properly "forward" and connect
    //# forward it to the real reqStream which enables
    //# request to read off the IncomingMessage readable stream
    retryStream.once("pipe", onPiped);

    reqStream.on("error", function(err) {
      if (didReceiveResponse) {
        //# if we've already begun processing the requests
        //# response, then that means we failed during transit
        //# and its no longer safe to retry. all we can do now
        //# is propogate the error upwards
        debug("received an error on request after response started %o", merge(opts, { err }));

        return emitError(err);
      }

      //# otherwise, see if we can retry another request under the hood...
      return maybeRetryOnNetworkFailure(err, {
        opts,
        retryIntervals,
        delaysRemaining,
        retryOnNetworkFailure,
        onNext: retry,
        onElse() {
          return emitError(err);
        }
      });
    });

    reqStream.once("request", req => //# remove the pipe listener since once the request has
    //# been made, we cannot pipe into the reqStream anymore
    retryStream.removeListener("pipe", onPiped));

    return reqStream.once("response", function(incomingRes) {
      didReceiveResponse = true;

      //# ok, no net error, but what about a bad status code?
      return maybeRetryOnStatusCodeFailure(incomingRes, {
        opts,
        requestId,
        delaysRemaining,
        retryIntervals,
        retryOnStatusCodeFailure,
        onNext: retry,
        onElse() {
          debug("successful response received", { requestId });

          cleanup();

          //# forward the response event upwards which should happen
          //# prior to the pipe event, same as what request does
          //# https://github.com/request/request/blob/master/request.js#L1059
          retryStream.emit("response", incomingRes);

          reqStream.pipe(delayStream);

          //# `http.ClientRequest` events
          return _.map(HTTP_CLIENT_REQUEST_EVENTS, _.partial(pipeEvent, reqStream, retryStream));
        }
      });
    });
  };

  tryStartStream();

  return retryStream;
};

const caseInsensitiveGet = function(obj, property) {
  const lowercaseProperty = property.toLowerCase();

  for (let key of Object.keys(obj)) {
    if (key.toLowerCase() === lowercaseProperty) {
      return obj[key];
    }
  }
};

//# first, attempt to set on an existing property with differing case
//# if that fails, set the lowercase `property`
const caseInsensitiveSet = function(obj, property, val) {
  const lowercaseProperty = property.toLowerCase();

  for (let key of Object.keys(obj)) {
    if (key.toLowerCase() === lowercaseProperty) {
      return obj[key] = val;
    }
  }

  return obj[lowercaseProperty] = val;
};

const setDefaults = opts => _
.chain(opts)
.defaults({
  requestId: _.uniqueId('request'),
  retryIntervals: [0, 1000, 2000, 2000],
  retryOnNetworkFailure: true,
  retryOnStatusCodeFailure: false
})
.thru(opts => _.defaults(opts, {
  delaysRemaining: _.clone(opts.retryIntervals)
})).value();

module.exports = function(options = {}) {
  const defaults = {
    timeout: options.timeout,
    agent,
    //# send keep-alive with requests since Chrome won't send it in proxy mode
    //# https://github.com/cypress-io/cypress/pull/3531#issuecomment-476269041
    headers: {
      "Connection": "keep-alive"
    },
    proxy: null //# upstream proxying is handled by CombinedAgent
  };

  r  = r.defaults(defaults);
  rp = rp.defaults(defaults);

  return {
    r: require("@cypress/request"),

    rp: require("@cypress/request-promise"),

    getDelayForRetry,

    setDefaults,

    create(strOrOpts, promise) {
      let opts;
      switch (false) {
        case !_.isString(strOrOpts):
          opts = {
            url: strOrOpts
          };
          break;
        default:
          opts = strOrOpts;
      }

      opts = setDefaults(opts);

      if (promise) {
        return createRetryingRequestPromise(opts);
      } else {
        return createRetryingRequestStream(opts);
      }
    },

    contentTypeIsJson(response) {
      //# TODO: use https://github.com/jshttp/type-is for this
      //# https://github.com/cypress-io/cypress/pull/5166
      return __guard__(__guard__(response != null ? response.headers : undefined, x1 => x1["content-type"]), x => x.split(';', 2)[0].endsWith("json"));
    },

    parseJsonBody(body) {
      try {
        return JSON.parse(body);
      } catch (e) {
        return body;
      }
    },

    normalizeResponse(push, response) {
      const req = response.request != null ? response.request : {};

      push(response);

      response = _.pick(response, "statusCode", "body", "headers");

      //# normalize status
      response.status = response.statusCode;
      delete response.statusCode;

      _.extend(response, {
        //# normalize what is an ok status code
        statusText:     statusCode.getText(response.status),
        isOkStatusCode: statusCode.isOk(response.status),
        requestHeaders: getOriginalHeaders(req),
        requestBody:    req.body
      });

      //# if body is a string and content type is json
      //# try to convert the body to JSON
      if (_.isString(response.body) && this.contentTypeIsJson(response)) {
        response.body = this.parseJsonBody(response.body);
      }

      return response;
    },

    setRequestCookieHeader(req, reqUrl, automationFn, existingHeader) {
      return automationFn('get:cookies', { url: reqUrl })
      .then(function(cookies) {
        debug('got cookies from browser %o', { reqUrl, cookies });
        let header = cookies.map(cookie => `${cookie.name}=${cookie.value}`).join("; ") || undefined;

        if (header) {
          if (existingHeader) {
            //# existingHeader = whatever Cookie header the user is already trying to set
            debug('there is an existing cookie header, merging %o', { header, existingHeader });
            //# order does not not matter here
            //# @see https://tools.ietf.org/html/rfc6265#section-4.2.2
            header = [existingHeader, header].join(';');
          }

          return caseInsensitiveSet(req.headers, 'Cookie', header);
        }
      });
    },

    setCookiesOnBrowser(res, resUrl, automationFn) {
      let cookies = res.headers['set-cookie'];
      if (!cookies) {
        return Promise.resolve();
      }

      if (!(cookies instanceof Array)) {
        cookies = [cookies];
      }

      const parsedUrl = url.parse(resUrl);
      const defaultDomain = parsedUrl.hostname;

      debug('setting cookies on browser %o', { url: parsedUrl.href, defaultDomain, cookies });

      return Promise.map(cookies, function(cyCookie) {
        let cookie = tough.Cookie.parse(cyCookie, { loose: true });

        debug('parsing cookie %o', { cyCookie, toughCookie: cookie });

        if (!cookie) {
          //# ignore invalid cookies (same as browser behavior)
          //# https://github.com/cypress-io/cypress/issues/6890
          debug('tough-cookie failed to parse, ignoring');
          return;
        }

        cookie.name = cookie.key;

        if (!cookie.domain) {
          //# take the domain from the URL
          cookie.domain = defaultDomain;
          cookie.hostOnly = true;
        }

        if (!tough.domainMatch(defaultDomain, cookie.domain)) {
          debug('domain match failed:', { defaultDomain });
          return;
        }

        const expiry = cookie.expiryTime();
        if (isFinite(expiry)) {
          cookie.expiry = expiry / 1000;
        }

        cookie.sameSite = convertSameSiteToughToExtension(cookie.sameSite, cyCookie);

        cookie = _.pick(cookie, SERIALIZABLE_COOKIE_PROPS);

        let automationCmd = 'set:cookie';

        if (expiry <= 0) {
          automationCmd = 'clear:cookie';
        }

        return automationFn(automationCmd, cookie)
        .catch(err => debug('automation threw an error during cookie change %o', { automationCmd, cyCookie, cookie, err }));
      });
    },

    sendStream(headers, automationFn, options = {}) {
      let ua;
      _.defaults(options, {
        headers: {},
        onBeforeReqInit(fn) { return fn(); }
      });

      if (!caseInsensitiveGet(options.headers, "user-agent") && (ua = headers["user-agent"])) {
        options.headers["user-agent"] = ua;
      }

      _.extend(options, {
        strictSSL: false
      });

      const self = this;

      const {
        followRedirect
      } = options;

      let currentUrl = options.url;

      options.followRedirect = function(incomingRes) {
        if (followRedirect && !followRedirect(incomingRes)) {
          return false;
        }

        const newUrl = url.resolve(currentUrl, incomingRes.headers.location);

        //# and when we know we should follow the redirect
        //# we need to override the init method and
        //# first set the received cookies on the browser
        //# and then grab the cookies for the new url
        return self.setCookiesOnBrowser(incomingRes, currentUrl, automationFn)
        .then(cookies => {
          return self.setRequestCookieHeader(this, newUrl, automationFn);
      }).then(() => {
          currentUrl = newUrl;
          return true;
        });
      };

      return this.setRequestCookieHeader(options, options.url, automationFn, caseInsensitiveGet(options.headers, 'cookie'))
      .then(() => {
        return () => {
          debug("sending request as stream %o", merge(options));

          return this.create(options);
        };
      });
    },

    sendPromise(headers, automationFn, options = {}) {
      let a, c, ua;
      _.defaults(options, {
        headers: {},
        gzip: true,
        cookies: true,
        followRedirect: true
      });

      if (!caseInsensitiveGet(options.headers, "user-agent") && (ua = headers["user-agent"])) {
        options.headers["user-agent"] = ua;
      }

      //# normalize case sensitivity
      //# to be lowercase
      if (a = options.headers.Accept) {
        delete options.headers.Accept;
        options.headers.accept = a;
      }

      //# https://github.com/cypress-io/cypress/issues/338
      _.defaults(options.headers, {
        accept: "*/*"
      });

      _.extend(options, {
        strictSSL: false,
        simple: false,
        resolveWithFullResponse: true
      });

      //# https://github.com/cypress-io/cypress/issues/322
      //# either turn these both on or off
      options.followAllRedirects = options.followRedirect;

      if (options.form === true) {
        //# reset form to whatever body is
        //# and nuke body
        options.form = options.body;
        delete options.json;
        delete options.body;
      }

      const self             = this;

      const send = () => {
        const ms = Date.now();

        const redirects        = [];
        const requestResponses = [];

        const push = response => requestResponses.push(pick(response));

        let currentUrl = options.url;

        if (options.followRedirect) {
          options.followRedirect = function(incomingRes) {
            const newUrl = url.resolve(currentUrl, incomingRes.headers.location);

            //# normalize the url
            redirects.push([incomingRes.statusCode, newUrl].join(": "));

            push(incomingRes);

            //# and when we know we should follow the redirect
            //# we need to override the init method and
            //# first set the new cookies on the browser
            //# and then grab the cookies for the new url
            return self.setCookiesOnBrowser(incomingRes, currentUrl, automationFn)
            .then(() => {
              return self.setRequestCookieHeader(this, newUrl, automationFn);
          }).then(() => {
              currentUrl = newUrl;
              return true;
            });
          };
        }

        return this.create(options, true)
        .then(this.normalizeResponse.bind(this, push))
        .then(resp => {
          //# TODO: move duration somewhere...?
          //# does node store this somewhere?
          //# we could probably calculate this ourselves
          //# by using the date headers
          let loc;
          resp.duration            = Date.now() - ms;
          resp.allRequestResponses = requestResponses;

          if (redirects.length) {
            resp.redirects = redirects;
          }

          if ((options.followRedirect === false) && (loc = resp.headers.location)) {
            //# resolve the new location head against
            //# the current url
            resp.redirectedToUrl = url.resolve(options.url, loc);
          }

          return this.setCookiesOnBrowser(resp, currentUrl, automationFn)
          .return(resp);
        });
      };

      if ((c = options.cookies)) {
        return self.setRequestCookieHeader(options, options.url, automationFn, caseInsensitiveGet(options.headers, 'cookie'))
        .then(send);
      } else {
        return send();
      }
    }

  };
};

function __guard__(value, transform) {
  return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
}