http,https: add built-in proxy support in http/https.request and Agent

This patch implements proxy support for HTTP and HTTPS clients and
agents in the `http` and `https` built-ins`. When NODE_USE_ENV_PROXY
is set to 1, the default global agent would parse the
HTTP_PROXY/http_proxy, HTTPS_PROXY/https_proxy, NO_PROXY/no_proxy
settings from the environment variables, and proxy the requests
sent through the built-in http/https client accordingly.

To support this, `http.Agent` and `https.Agent` now accept a few new
options:

- `proxyEnv`: when it's an object, the agent would read and parse
  the HTTP_PROXY/http_proxy, HTTPS_PROXY/https_proxy, NO_PROXY/no_proxy
  properties from it, and apply them based on the protocol it uses
  to send requests. This option allows custom agents to
  reuse built-in proxy support by composing options. Global agents
  set this to `process.env` when NODE_USE_ENV_PROXY is 1.
- `defaultPort` and `protocol`: these allow setting of the default port
  and protocol of the agents. We also need these when configuring
  proxy settings and deciding whether a request should be proxied.

Implementation-wise, this adds a `ProxyConfig` internal class to handle
parsing and application of proxy configurations. The configuration
is parsed during agent construction. When requests are made,
the `createConnection()` methods on the agents would check whether
the request should be proxied. If yes, they either connect to the
proxy server (in the case of HTTP reqeusts) or establish a tunnel
(in the case of HTTPS requests) through either a TCP socket (if the
proxy uses HTTP) or a TLS socket (if the proxy uses HTTPS).

When proxying HTTPS requests through a tunnel, the connection listener
is invoked after the tunnel is established. Tunnel establishment uses
the timeout of the request options, if there is one. Otherwise it uses
the timeout of the agent.

If an error is encountered during tunnel establishment, an
ERR_PROXY_TUNNEL would be emitted on the returned socket. If the proxy
server sends a errored status code, the error would contain an
`statusCode` property. If the error is caused by timeout, the error
would contain a `proxyTunnelTimeout` property.

This implementation honors the built-in socket pool and socket limits.
Pooled sockets are still keyed by request endpoints, they are just
connected to the proxy server instead, and the persistence of the
connection can be maintained as long as the proxy server respects
connection/proxy-connection or persist by default (HTTP/1.1)

PR-URL: https://github.com/nodejs/node/pull/58980
Refs: https://github.com/nodejs/node/issues/57872
Refs: https://github.com/nodejs/node/issues/8381
Refs: https://github.com/nodejs/node/issues/15620
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>

lib/https.js

@@ -29,6 +29,7 @@ const {
   ArrayPrototypeUnshift,
   FunctionPrototypeCall,
   JSONStringify,
+  NumberParseInt,
   ObjectAssign,
   ObjectSetPrototypeOf,
   ReflectApply,
@@ -40,10 +41,18 @@ const {
   assertCrypto,
   kEmptyObject,
   promisify,
+  once,
 } = require('internal/util');
+const { ERR_PROXY_TUNNEL } = require('internal/errors').codes;
 assertCrypto();
 
 const tls = require('tls');
+const {
+  kProxyConfig,
+  checkShouldUseProxy,
+  filterEnvForProxies,
+  kWaitForProxyTunnel,
+} = require('internal/http');
 const { Agent: HttpAgent } = require('_http_agent');
 const {
   httpServerPreClose,
@@ -56,8 +65,11 @@ const { ClientRequest } = require('_http_client');
 let debug = require('internal/util/debuglog').debuglog('https', (fn) => {
   debug = fn;
 });
+const net = require('net');
 const { URL, urlToHttpOptions, isURL } = require('internal/url');
 const { validateObject } = require('internal/validators');
+const { isIP, isIPv6 } = require('internal/net');
+const assert = require('internal/assert');
 
 function Server(opts, requestListener) {
   if (!(this instanceof Server)) return new Server(opts, requestListener);
@@ -135,26 +147,198 @@ function createServer(opts, requestListener) {
   return new Server(opts, requestListener);
 }
 
+// When proxying a HTTPS request, the following needs to be done:
+// https://datatracker.ietf.org/doc/html/rfc9110#CONNECT
+// 1. Send a CONNECT request to the proxy server.
+// 2. Wait for 200 connection established response to establish the tunnel.
+// 3. Perform TLS handshake with the endpoint over the socket.
+// 4. Tunnel the request using the established connection.
+//
+// This function computes the tunnel configuration for HTTPS requests.
+// The handling of the tunnel connection is done in createConnection.
+function getTunnelConfigForProxiedHttps(agent, reqOptions) {
+  if (!agent[kProxyConfig]) {
+    return null;
+  }
+  if ((reqOptions.protocol || agent.protocol) !== 'https:') {
+    return null;
+  }
+  const shouldUseProxy = checkShouldUseProxy(agent[kProxyConfig], reqOptions);
+  debug(`getTunnelConfigForProxiedHttps should use proxy for ${reqOptions.host}:${reqOptions.port}:`, shouldUseProxy);
+  if (!shouldUseProxy) {
+    return null;
+  }
+  const { auth, href } = agent[kProxyConfig];
+  // The request is a HTTPS request, assemble the payload for establishing the tunnel.
+  const requestHost = isIPv6(reqOptions.host) ? `[${reqOptions.host}]` : reqOptions.host;
+  const requestPort = reqOptions.port || agent.defaultPort;
+  const endpoint = `${requestHost}:${requestPort}`;
+  // The ClientRequest constructor should already have validated the host and the port.
+  // When the request options come from a string invalid characters would be stripped away,
+  // when it's an object ERR_INVALID_CHAR would be thrown. Here we just assert in case
+  // agent.createConnection() is called with invalid options.
+  assert(!endpoint.includes('\r'));
+  assert(!endpoint.includes('\n'));
+
+  let payload = `CONNECT ${endpoint} HTTP/1.1\r\n`;
+  // The parseProxyConfigFromEnv() method should have already validated the authorization header
+  // value.
+  if (auth) {
+    payload += `proxy-authorization: ${auth}\r\n`;
+  }
+  if (agent.keepAlive || agent.maxSockets !== Infinity) {
+    payload += 'proxy-connection: keep-alive\r\n';
+  }
+  payload += `host: ${endpoint}`;
+  payload += '\r\n\r\n';
+
+  const result = {
+    __proto__: null,
+    proxyTunnelPayload: payload,
+    requestOptions: {  // Options used for the request sent after the tunnel is established.
+      __proto__: null,
+      servername: reqOptions.servername || (isIP(reqOptions.host) ? undefined : reqOptions.host),
+      ...reqOptions,
+    },
+  };
+  debug(`updated request for HTTPS proxy ${href} with`, result);
+  return result;
+};
+
+function establishTunnel(agent, socket, options, tunnelConfig, afterSocket) {
+  const { proxyTunnelPayload } = tunnelConfig;
+  // By default, the socket is in paused mode. Read to look for the 200
+  // connection established response.
+  function read() {
+    let chunk;
+    while ((chunk = socket.read()) !== null) {
+      if (onProxyData(chunk) !== -1) {
+        break;
+      }
+    }
+    socket.on('readable', read);
+  }
+
+  function cleanup() {
+    socket.removeListener('end', onProxyEnd);
+    socket.removeListener('error', onProxyError);
+    socket.removeListener('readable', read);
+    socket.setTimeout(0);  // Clear the timeout for the tunnel establishment.
+  }
+
+  function onProxyError(err) {
+    debug('onProxyError', err);
+    cleanup();
+    afterSocket(err, socket);
+  }
+
+  // Read the headers from the chunks and check for the status code. If it fails we
+  // clean up the socket and return an error. Otherwise we establish the tunnel.
+  let buffer = '';
+  function onProxyData(chunk) {
+    const str = chunk.toString();
+    debug('onProxyData', str);
+    buffer += str;
+    const headerEndIndex = buffer.indexOf('\r\n\r\n');
+    if (headerEndIndex === -1) return headerEndIndex;
+    const statusLine = buffer.substring(0, buffer.indexOf('\r\n'));
+    const statusCode = statusLine.split(' ')[1];
+    if (statusCode !== '200') {
+      debug(`onProxyData receives ${statusCode}, cleaning up`);
+      cleanup();
+      const targetHost = proxyTunnelPayload.split('\r')[0].split(' ')[1];
+      const message = `Failed to establish tunnel to ${targetHost} via ${agent[kProxyConfig].href}: ${statusLine}`;
+      const err = new ERR_PROXY_TUNNEL(message);
+      err.statusCode = NumberParseInt(statusCode);
+      afterSocket(err, socket);
+    } else {
+      // https://datatracker.ietf.org/doc/html/rfc9110#CONNECT
+      // RFC 9110 says that it can be 2xx but in the real world, proxy clients generally only
+      // accepts 200.
+      // Proxy servers are not supposed to send anything after the headers - the payload must be
+      // be empty. So after this point we will proceed with the tunnel e.g. starting TLS handshake.
+      debug('onProxyData receives 200, establishing tunnel');
+      cleanup();
+
+      // Reuse the tunneled socket to perform the TLS handshake with the endpoint,
+      // then send the request.
+      const { requestOptions } = tunnelConfig;
+      tunnelConfig.requestOptions = null;
+      requestOptions.socket = socket;
+      let tunneldSocket;
+      const onTLSHandshakeError = (err) => {
+        debug('Propagate error event from tunneled socket to tunnel socket');
+        afterSocket(err, tunneldSocket);
+      };
+      tunneldSocket = tls.connect(requestOptions, () => {
+        debug('TLS handshake over tunnel succeeded');
+        tunneldSocket.removeListener('error', onTLSHandshakeError);
+        afterSocket(null, tunneldSocket);
+      });
+      tunneldSocket.on('free', () => {
+        debug('Propagate free event from tunneled socket to tunnel socket');
+        socket.emit('free');
+      });
+      tunneldSocket.on('error', onTLSHandshakeError);
+    }
+    return headerEndIndex;
+  }
+
+  function onProxyEnd() {
+    cleanup();
+    const err = new ERR_PROXY_TUNNEL('Connection to establish proxy tunnel ended unexpectedly');
+    afterSocket(err, socket);
+  }
+
+  const proxyTunnelTimeout = tunnelConfig.requestOptions.timeout;
+  debug('proxyTunnelTimeout', proxyTunnelTimeout, options.timeout);
+  // It may be worth a separate timeout error/event.
+  // But it also makes sense to treat the tunnel establishment timeout as
+  // a normal timeout for the request.
+  function onProxyTimeout() {
+    debug('onProxyTimeout', proxyTunnelTimeout);
+    cleanup();
+    const err = new ERR_PROXY_TUNNEL(`Connection to establish proxy tunnel timed out after ${proxyTunnelTimeout}ms`);
+    err.proxyTunnelTimeout = proxyTunnelTimeout;
+    afterSocket(err, socket);
+  }
+
+  if (proxyTunnelTimeout && proxyTunnelTimeout > 0) {
+    debug('proxy tunnel setTimeout', proxyTunnelTimeout);
+    socket.setTimeout(proxyTunnelTimeout, onProxyTimeout);
+  }
+
+  socket.on('error', onProxyError);
+  socket.on('end', onProxyEnd);
+  socket.write(proxyTunnelPayload);
+
+  read();
+}
 
 // HTTPS agents.
-
-function createConnection(port, host, options) {
-  if (port !== null && typeof port === 'object') {
-    options = port;
-  } else if (host !== null && typeof host === 'object') {
-    options = { ...host };
-  } else if (options === null || typeof options !== 'object') {
+// See ProxyConfig in internal/http.js for how the connection should be handled
+// when the agent is configured to use a proxy server.
+function createConnection(...args) {
+  // XXX: This signature (port, host, options) is different from all the other
+  // createConnection() methods.
+  let options, cb;
+  if (args[0] !== null && typeof args[0] === 'object') {
+    options = args[0];
+  } else if (args[1] !== null && typeof args[1] === 'object') {
+    options = { ...args[1] };
+  } else if (args[2] === null || typeof args[2] !== 'object') {
     options = {};
   } else {
-    options = { ...options };
+    options = { ...args[2] };
   }
-
-  if (typeof port === 'number') {
-    options.port = port;
+  if (typeof args[0] === 'number') {
+    options.port = args[0];
   }
-
-  if (typeof host === 'string') {
-    options.host = host;
+  if (typeof args[1] === 'string') {
+    options.host = args[1];
+  }
+  if (typeof args[args.length - 1] === 'function') {
+    cb = args[args.length - 1];
   }
 
   debug('createConnection', options);
@@ -170,7 +354,61 @@ function createConnection(port, host, options) {
     }
   }
 
-  const socket = tls.connect(options);
+  let socket;
+  const tunnelConfig = getTunnelConfigForProxiedHttps(this, options);
+  debug(`https createConnection should use proxy for ${options.host}:${options.port}:`, tunnelConfig);
+
+  if (!tunnelConfig) {
+    socket = tls.connect(options);
+  } else {
+    const connectOptions = {
+      ...this[kProxyConfig].proxyConnectionOptions,
+    };
+    debug('Create proxy socket', connectOptions);
+    const onError = (err) => {
+      cleanupAndPropagate(err, socket);
+    };
+    const proxyTunnelTimeout = tunnelConfig.requestOptions.timeout;
+    const onTimeout = () => {
+      const err = new ERR_PROXY_TUNNEL(`Connection to establish proxy tunnel timed out after ${proxyTunnelTimeout}ms`);
+      err.proxyTunnelTimeout = proxyTunnelTimeout;
+      cleanupAndPropagate(err, socket);
+    };
+    const cleanupAndPropagate = once((err, currentSocket) => {
+      debug('cleanupAndPropagate', err);
+      socket.removeListener('error', onError);
+      socket.removeListener('timeout', onTimeout);
+      // An error occurred during tunnel establishment, in that case just destroy the socket.
+      // and propagate the error to the callback.
+
+      // When the error comes from unexpected status code, the stream is still in good shape,
+      // in that case let req.onSocket handle the destruction instead.
+      if (err && err.code === 'ERR_PROXY_TUNNEL' && !err.statusCode) {
+        socket.destroy();
+      }
+      // This error should go to:
+      // -> oncreate in Agent.prototype.createSocket
+      // -> closure in Agent.prototype.addRequest or Agent.prototype.removeSocket
+      if (cb) {
+        cb(err, currentSocket);
+      }
+    });
+    const onProxyConnection = () => {
+      socket.removeListener('error', onError);
+      establishTunnel(this, socket, options, tunnelConfig, cleanupAndPropagate);
+    };
+    if (this[kProxyConfig].protocol === 'http:') {
+      socket = net.connect(connectOptions, onProxyConnection);
+    } else {
+      socket = tls.connect(connectOptions, onProxyConnection);
+    }
+
+    socket.on('error', onError);
+    if (proxyTunnelTimeout) {
+      socket.setTimeout(proxyTunnelTimeout, onTimeout);
+    }
+    socket[kWaitForProxyTunnel] = true;
+  }
 
   if (options._agentKey) {
     // Cache new session for reuse
@@ -200,6 +438,9 @@ function createConnection(port, host, options) {
  *   timeout?: number;
  *   maxCachedSessions?: number;
  *   servername?: string;
+ *   defaultPort?: number;
+ *   protocol?: string;
+ *   proxyEnv?: object;
  *   }} [options]
  * @constructor
  */
@@ -207,9 +448,11 @@ function Agent(options) {
   if (!(this instanceof Agent))
     return new Agent(options);
 
+  options = { __proto__: null, ...options };
+  options.defaultPort ??= 443;
+  options.protocol ??= 'https:';
   FunctionPrototypeCall(HttpAgent, this, options);
-  this.defaultPort = 443;
-  this.protocol = 'https:';
+
   this.maxCachedSessions = this.options.maxCachedSessions;
   if (this.maxCachedSessions === undefined)
     this.maxCachedSessions = 100;
@@ -354,7 +597,10 @@ Agent.prototype._evictSession = function _evictSession(key) {
   delete this._sessionCache.map[key];
 };
 
-const globalAgent = new Agent({ keepAlive: true, scheduling: 'lifo', timeout: 5000 });
+const globalAgent = new Agent({
+  keepAlive: true, scheduling: 'lifo', timeout: 5000,
+  proxyEnv: process.env.NODE_USE_ENV_PROXY ? filterEnvForProxies(process.env) : undefined,
+});
 
 /**
  * Makes a request to a secure web server.