fix: Standardize request filtering between cloud / self-hosted (#9914)

* fix: Add request-filtering-agent to self-hosted environment * refactor * Debug logging * self-review * Remove unused AbortController * test * test * Address feedback
2026-05-03 08:00:15 -05:00 · 2025-08-15 07:16:29 -04:00
parent 99655c65d4
commit 8fcb629bdf
6 changed files with 181 additions and 31 deletions
@@ -134,7 +134,6 @@
    "es6-error": "^4.1.1",
    "fast-deep-equal": "^3.1.3",
    "fetch-retry": "^5.0.6",
-    "fetch-with-proxy": "^3.0.1",
    "form-data": "^4.0.4",
    "fractional-index": "^1.0.0",
    "framer-motion": "^4.1.17",
@@ -205,6 +204,7 @@
    "prosemirror-tables": "^1.7.1",
    "prosemirror-transform": "1.10.0",
    "prosemirror-view": "^1.40.1",
+    "proxy-from-env": "^1.1.0",
    "query-string": "^7.1.3",
    "rate-limiter-flexible": "^2.4.2",
    "react": "^17.0.2",
@@ -255,6 +255,7 @@
    "throng": "^5.0.0",
    "tiny-cookie": "^2.5.1",
    "tmp": "^0.2.4",
+    "tunnel-agent": "^0.6.0",
    "turndown": "^7.2.0",
    "ukkonen": "^2.1.0",
    "umzug": "^3.8.2",
@@ -316,6 +317,7 @@
    "@types/passport-oauth2": "^1.4.17",
    "@types/pluralize": "^0.0.33",
    "@types/png-chunks-extract": "^1.0.2",
+    "@types/proxy-from-env": "^1.0.4",
    "@types/quoted-printable": "^1.0.2",
    "@types/react": "^17.0.34",
    "@types/react-avatar-editor": "^13.0.4",
@@ -2,6 +2,9 @@ import "reflect-metadata";
 import sharedEnv from "@shared/env";
 import env from "@server/env";

+require("jest-fetch-mock").enableMocks();
+fetchMock.dontMock();
+
 require("@server/storage/database");

 // Enable mocks for Redis-related modules
@@ -1,6 +0,0 @@
-declare module "fetch-with-proxy" {
-  // oxlint-disable-next-line no-restricted-imports
-  import nodeFetch from "node-fetch";
-
-  export = nodeFetch;
-}
@@ -0,0 +1,35 @@
+declare module "tunnel-agent" {
+  import { Agent as HttpAgent } from "http";
+  import { Agent as HttpsAgent } from "https";
+
+  interface TunnelOptions {
+    proxy: {
+      host: string | null;
+      port: string | number | null;
+      proxyAuth?: string | null;
+      headers?: { [key: string]: string };
+    };
+    ca?: Buffer[] | Buffer;
+    cert?: Buffer | string;
+    key?: Buffer | string;
+    passphrase?: string;
+    rejectUnauthorized?: boolean;
+    secureOptions?: number;
+    secureProtocol?: string;
+    ciphers?: string;
+    localAddress?: string;
+    maxSockets?: number;
+    keepAlive?: boolean;
+    keepAliveMsecs?: number;
+  }
+
+  export interface TunnelAgent {
+    httpOverHttp: (options: TunnelOptions) => HttpAgent;
+    httpsOverHttp: (options: TunnelOptions) => HttpsAgent;
+    httpOverHttps: (options: TunnelOptions) => HttpAgent;
+    httpsOverHttps: (options: TunnelOptions) => HttpsAgent;
+  }
+
+  const tunnel: TunnelAgent;
+  export = tunnel;
+}
@@ -1,9 +1,26 @@
 /* oxlint-disable no-restricted-imports */
-import fetchWithProxy from "fetch-with-proxy";
-import nodeFetch, { RequestInit, Response } from "node-fetch";
-import { useAgent } from "request-filtering-agent";
+import http from "http";
+import https from "https";
+import nodeFetch, { type RequestInit, type Response } from "node-fetch";
+import { getProxyForUrl } from "proxy-from-env";
+import tunnelAgent, { type TunnelAgent } from "tunnel-agent";
+import { useAgent as useFilteringAgent } from "request-filtering-agent";
 import env from "@server/env";
 import Logger from "@server/logging/Logger";
+import { capitalize, defaults } from "lodash";
+
+interface UrlWithTunnel extends URL {
+  tunnelMethod?: string;
+}
+
+const DefaultOptions = {
+  keepAlive: true,
+  timeout: 1000,
+  keepAliveMsecs: 500,
+  maxSockets: 200,
+  maxFreeSockets: 5,
+  maxCachedSessions: 500,
+};

 export type { RequestInit } from "node-fetch";

@@ -34,19 +51,15 @@ export default async function fetch(
  url: string,
  init?: RequestInit
 ): Promise<Response> {
-  // In self-hosted, webhooks support proxying and are also allowed to connect
-  // to internal services, so use fetchWithProxy without the filtering agent.
-  const fetchMethod = env.isCloudHosted ? nodeFetch : fetchWithProxy;
-
  Logger.silly("http", `Network request to ${url}`, init);

-  const response = await fetchMethod(url, {
+  const response = await nodeFetch(url, {
    ...init,
    headers: {
      "User-Agent": outlineUserAgent,
      ...init?.headers,
    },
-    agent: env.isCloudHosted ? useAgent(url) : undefined,
+    agent: buildAgent(url),
  });

  if (!response.ok) {
@@ -60,3 +73,114 @@ export default async function fetch(

  return response;
 }
+
+/**
+ * Parses the proxy URL and returns an object with the properties
+ *
+ * @param url The URL to be fetched
+ * @param proxyURL The proxy URL to be used
+ * @returns An object containing the parsed proxy URL and tunnel method
+ */
+const parseProxy = (url: URL, proxyURL: string) => {
+  const proxyObject = new URL(proxyURL) as UrlWithTunnel;
+  const proxyProtocol = proxyObject.protocol?.replace(":", "");
+  const proxyPort =
+    proxyObject.port || (proxyProtocol === "https" ? "443" : "80");
+  proxyObject.port = proxyPort;
+  proxyObject.tunnelMethod = url.protocol
+    ?.replace(":", "")
+    .concat("Over")
+    .concat(capitalize(proxyProtocol));
+  return proxyObject;
+};
+
+/**
+ * Builds a tunnel agent for the given proxy URL and options. Note that tunnel
+ * agents do not perform request filtering.
+ *
+ * @param proxy The parsed proxy URL
+ * @param options The request options
+ * @returns A tunnel agent for the proxy
+ */
+const buildTunnel = (proxy: UrlWithTunnel, options: RequestInit) => {
+  if (!proxy.tunnelMethod) {
+    Logger.warn("Proxy tunnel method not defined");
+    return;
+  }
+  if (!(proxy.tunnelMethod in tunnelAgent)) {
+    Logger.warn(`Proxy tunnel method not supported: ${proxy.tunnelMethod}`);
+    return;
+  }
+
+  const proxyAuth =
+    proxy.username || proxy.password
+      ? `${proxy.username}:${proxy.password}`
+      : undefined;
+
+  return tunnelAgent[proxy.tunnelMethod as keyof TunnelAgent]({
+    ...options,
+    proxy: {
+      port: proxy.port,
+      host: proxy.hostname,
+      proxyAuth,
+    },
+  });
+};
+
+/**
+ * Creates a http or https agent for the given URL, applying request filtering
+ * if necessary. If a proxy is detected in the environment, it will use that
+ * proxy agent to tunnel the request.
+ *
+ * @param url The URL to fetch
+ * @param options The fetch options
+ * @returns An http or https agent configured for the URL
+ */
+function buildAgent(url: string, options: RequestInit = {}) {
+  const agentOptions = defaults(options, DefaultOptions);
+  const parsedURL = new URL(url);
+  const proxyURL = getProxyForUrl(parsedURL.href);
+  let agent: https.Agent | http.Agent | undefined;
+
+  if (proxyURL) {
+    const parsedProxyURL = parseProxy(parsedURL, proxyURL);
+
+    Logger.silly("http", `Using proxy for request`, {
+      url: parsedURL.toString(),
+      proxy: parsedProxyURL.href,
+      tunnelMethod: parsedProxyURL.tunnelMethod,
+      username: parsedProxyURL.username || undefined,
+      password: parsedProxyURL.password ? "******" : undefined,
+    });
+
+    if (parsedProxyURL.tunnelMethod?.startsWith("httpOver")) {
+      const proxyURL = new URL(parsedProxyURL.href);
+      proxyURL.pathname = (parsedURL.protocol ?? "")
+        .concat("//")
+        .concat(parsedURL.host ?? "")
+        .concat(parsedURL.pathname + parsedURL.search);
+      if (parsedProxyURL.username || parsedProxyURL.password) {
+        proxyURL.username = parsedProxyURL.username;
+        proxyURL.password = parsedProxyURL.password;
+      }
+      agent = useFilteringAgent(proxyURL.toString(), agentOptions);
+    } else {
+      // Note request filtering agent does not support https tunneling via a proxy
+      agent =
+        buildTunnel(parsedProxyURL, agentOptions) ||
+        useFilteringAgent(parsedURL.toString(), agentOptions);
+    }
+  } else {
+    agent = useFilteringAgent(parsedURL.toString(), agentOptions);
+  }
+
+  if (options.signal) {
+    options.signal.addEventListener("abort", () => {
+      if (agent && "destroy" in agent) {
+        agent.destroy();
+      }
+    });
+  }
+
+  return agent;
+}
@@ -5240,6 +5240,13 @@
  resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.4.tgz#fcf7205c25dff795ee79af1e30da2c9790808f11"
  integrity "sha1-/PcgXCXf95Xuea8eMNosl5CAjxE= sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ=="

+"@types/proxy-from-env@^1.0.4":
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/@types/proxy-from-env/-/proxy-from-env-1.0.4.tgz#0a0545768f2d6c16b81a84ffefb53b423807907c"
+  integrity sha512-TPR9/bCZAr3V1eHN4G3LD3OLicdJjqX1QRXWuNcCYgE66f/K8jO2ZRtHxI2D9MbnuUP6+qiKSS8eUHp6TFHGCw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/qs@*":
  version "6.9.7"
  resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"
@@ -8245,16 +8252,6 @@ fetch-retry@^5.0.6:
  resolved "https://registry.yarnpkg.com/fetch-retry/-/fetch-retry-5.0.6.tgz#17d0bc90423405b7a88b74355bf364acd2a7fa56"
  integrity "sha1-F9C8kEI0Bbeoi3Q1W/NkrNKn+lY= sha512-3yurQZ2hD9VISAhJJP9bpYFNQrHHBXE2JxxjY5aLEcDi46RmAzJE2OC9FAde0yis5ElW0jTTzs0zfg/Cca4XqQ=="

-fetch-with-proxy@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/fetch-with-proxy/-/fetch-with-proxy-3.0.1.tgz#29ed6d0e2550ef999d40b18de2ba476af4b7dee4"
-  integrity "sha1-Ke1tDiVQ75mdQLGN4rpHavS33uQ= sha512-8C5JZ+Ea2eTOkFuQhB252QPgEc68LS7+8uNrFbYFs7t114Bgdj7hiYmtwkHhmN8TvafGVRbspMMD/Rg/tw0RwA=="
-  dependencies:
-    node-abort-controller "^1.1.0"
-    node-fetch "^2.6.1"
-    proxy-from-env "^1.1.0"
-    tunnel-agent "^0.6.0"
-
 file-selector@^0.4.0:
  version "0.4.0"
  resolved "https://registry.yarnpkg.com/file-selector/-/file-selector-0.4.0.tgz#59ec4f27aa5baf0841e9c6385c8386bef4d18b17"
@@ -11313,11 +11310,6 @@ negotiator@0.6.3:
  resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd"
  integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==

-node-abort-controller@^1.1.0:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-1.2.1.tgz#1eddb57eb8fea734198b11b28857596dc6165708"
-  integrity "sha1-Ht21frj+pzQZixGyiFdZbcYWVwg= sha512-79PYeJuj6S9+yOHirR0JBLFOgjB6sQCir10uN6xRx25iD+ZD4ULqgRn3MwWBRaQGB0vEgReJzWwJo42T1R6YbQ=="
-
 node-abort-controller@^3.0.1:
  version "3.1.1"
  resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548"