Add system certificates to https proxy requests

.devcontainer/https-proxy-test/.gitignore

@@ -0,0 +1 @@
+mitmproxy-config

.devcontainer/https-proxy-test/.vscode/tasks.json

@@ -0,0 +1,13 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "mitmweb",
+            "type": "shell",
+            "command": "mitmweb --web-host '*'",
+            "runOptions": {
+                "runOn": "folderOpen"
+            }
+        }
+    ]
+}

.devcontainer/https-proxy-test/README.md

@@ -0,0 +1,9 @@
+## HTTPS Proxy Test
+
+- `Dev Containers: Reopen in Container` > `HTTPS Proxy Test`.
+- First time: Install the certificate from `.devcontainer/https-proxy-test/mitmproxy-config` in the OS and restart VS Code. See https://docs.mitmproxy.org/stable/concepts-certificates/#installing-the-mitmproxy-ca-certificate-manually.
+- Add the user setting `"http.proxy": "https://localhost:8080"`.
+- Install GitHub Copilot Chat and use `Developer: GitHub Copilot Chat Diagnostics` to test connections with a HTTPS proxy. Use a second window to test connections from a local extension host.
+- Verify in the log terminal of the dev container that the proxy is being used.
+
+Note: Due to an issue in mitmproxy, Electron's `fetch` currently doesn't work. Add the user setting `"github.copilot.advanced.debug.useElectronFetcher": false` as a workaround.

.devcontainer/https-proxy-test/devcontainer.json

@@ -0,0 +1,6 @@
+{
+	"name": "HTTPS Proxy Test",
+	"dockerComposeFile": "docker-compose.yml",
+	"service": "devcontainer",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}/.devcontainer/https-proxy-test"
+}

.devcontainer/https-proxy-test/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '3.4'
+
+services:
+    devcontainer:
+        image: mitmproxy/mitmproxy
+        volumes:
+          - ../../..:/workspaces
+          - ./mitmproxy-config:/root/.mitmproxy
+        ports:
+            - "127.0.0.1:8080:8080"
+            - "127.0.0.1:8081:8081"
+        command: sleep inf

.gitignore

@@ -1,4 +1,3 @@
-.vscode
 node_modules
 *.pem
 out

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Change Log
 Notable changes will be documented here.
 
+## [0.27.0]
+- Add system certificates to https proxy requests ([microsoft/vscode#235410](https://github.com/microsoft/vscode/issues/235410))
+
 ## [0.26.0]
 - Move fetch patching to agent package ([microsoft/vscode#228697](https://github.com/microsoft/vscode/issues/228697))
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vscode/proxy-agent",
-  "version": "0.26.0",
+  "version": "0.27.0",
   "description": "NodeJS http(s) agent implementation for VS Code",
   "main": "out/index.js",
   "types": "out/index.d.ts",

src/agent.ts

@@ -34,14 +34,17 @@ type FindProxyForURL = (req: http.ClientRequest, opts: http.RequestOptions, url:
 export class PacProxyAgent extends Agent {
 	resolver: FindProxyForURL;
 	opts: PacProxyAgentOptions;
+	addCAs: (opts: PacProxyAgentOptions) => Promise<void>;
+	casAdded = false;
 	cache?: Readable;
 
-	constructor(resolver: FindProxyForURL, opts: PacProxyAgentOptions = {}) {
+	constructor(resolver: FindProxyForURL, opts: PacProxyAgentOptions = {}, addCAs: (opts: PacProxyAgentOptions) => Promise<void> = async () => {}) {
 		super(opts);
 		debug('Creating PacProxyAgent with options %o', opts);
 
 		this.resolver = resolver;
 		this.opts = { ...opts };
+		this.addCAs = addCAs;
 		this.cache = undefined;
 	}
 
@@ -96,6 +99,11 @@ export class PacProxyAgent extends Agent {
 		} else if (proxyURL.startsWith('http')) {
 			// Use an HTTP or HTTPS proxy
 			// http://dev.chromium.org/developers/design-documents/secure-web-proxy
+			if (!this.casAdded && proxyURL.startsWith('https')) {
+				debug('Adding CAs to proxy options');
+				this.casAdded = true;
+				await this.addCAs(this.opts);
+			}
 			if (secureEndpoint) {
 				agent = new HttpsProxyAgent2(proxyURL, this.opts);
 			} else {
@@ -235,7 +243,8 @@ class HttpsProxyAgent2<Uri extends string> extends HttpsProxyAgent<Uri> {
 
 export function createPacProxyAgent(
 	resolver: FindProxyForURL,
-	opts?: PacProxyAgentOptions
+	opts?: PacProxyAgentOptions,
+	addCAs?: (opts: PacProxyAgentOptions) => Promise<void>,
 ): PacProxyAgent {
 	if (!opts) {
 		opts = {};
@@ -245,12 +254,13 @@ export function createPacProxyAgent(
 		throw new TypeError('a resolve function must be specified!');
 	}
 
-	return new PacProxyAgent(resolver, opts);
+	return new PacProxyAgent(resolver, opts, addCAs);
 }
 type PacProxyAgentOptions =
 		HttpProxyAgentOptions<''> &
 		HttpsProxyAgentOptions2<''> &
 		SocksProxyAgentOptions & {
 	fallbackToDirect?: boolean;
 	originalAgent?: false | http.Agent;
+	_vscodeTestReplaceCaCerts?: boolean;
 }

src/index.ts

@@ -138,7 +138,7 @@ export function createProxyResolver(params: ProxyAgentParams) {
 
 		const stackText = ''; // getLogLevel() === LogLevel.Trace ? '\n' + new Error('Error for stack trace').stack : '';
 
-		addCertificatesV1(params, flags.addCertificatesV1, opts, () => {
+		addCertificatesToOptionsV1(params, flags.addCertificatesV1, opts, () => {
 			if (!flags.useProxySettings) {
 				callback('DIRECT');
 				return;
@@ -395,7 +395,8 @@ export function createHttpPatch(params: ProxyAgentParams, originals: typeof http
 					originalAgent: (!useProxySettings || isLocalhost || config === 'fallback') ? originalAgent : undefined,
 					lookupProxyAuthorization: params.lookupProxyAuthorization,
 					// keepAlive: ((originalAgent || originals.globalAgent) as { keepAlive?: boolean }).keepAlive, // Skipping due to https://github.com/microsoft/vscode/issues/228872.
-				});
+					_vscodeTestReplaceCaCerts: (options as SecureContextOptionsPatch)._vscodeTestReplaceCaCerts,
+				}, opts => new Promise<void>(resolve => addCertificatesToOptionsV1(params, params.addCertificatesV1(), opts, resolve)));
 				agent.protocol = isHttps ? 'https:' : 'http:';
 				options.agent = agent
 				if (isHttps) {
@@ -597,8 +598,8 @@ export function createFetchPatch(params: ProxyAgentParams, originalFetch: typeof
 				uri: proxyURL,
 				allowH2,
 				headers: proxyAuthorization ? { 'Proxy-Authorization': proxyAuthorization } : undefined,
-				...(requestCA ? { requestTls: { ca: requestCA } } : {}),
-				...(proxyCA ? { proxyTls: { ca: proxyCA } } : {}),
+				requestTls: requestCA ? { allowH2, ca: requestCA } : { allowH2 },
+				proxyTls: proxyCA ? { allowH2, ca: proxyCA } : { allowH2 },
 				clientFactory: (origin: URL, opts: object): undici.Dispatcher => (new undici.Pool(origin, opts) as any).compose((dispatch: undici.Dispatcher['dispatch']) => {
 					class ProxyAuthHandler extends undici.DecoratorHandler {
 						private abort: ((err?: Error) => void) | undefined;
@@ -727,7 +728,7 @@ function getAgentOptions(systemCA: string[] | undefined, requestInit: RequestIni
 	return { allowH2, requestCA, proxyCA };
 }
 
-function addCertificatesV1(params: ProxyAgentParams, addCertificatesV1: boolean, opts: http.RequestOptions, callback: () => void) {
+function addCertificatesToOptionsV1(params: ProxyAgentParams, addCertificatesV1: boolean, opts: http.RequestOptions | tls.ConnectionOptions, callback: () => void) {
 	if (addCertificatesV1) {
 		getOrLoadAdditionalCertificates(params)
 			.then(caCertificates => {

tests/docker-compose.yml

@@ -19,12 +19,16 @@ services:
         build: test-proxy-client
         volumes:
             - ..:/repo
+            - ./test-https-proxy/mitmproxy-config:/root/.mitmproxy
         networks:
             - test-proxies
         working_dir: /repo/tests/test-client
         environment:
             - MOCHA_TESTS=src/proxy.test.ts
         command: /bin/sh -c '
+            while [ ! -f /root/.mitmproxy/mitmproxy-ca-cert.pem ]; do sleep 1; done &&
+            cp /root/.mitmproxy/mitmproxy-ca-cert.pem /usr/local/share/ca-certificates/mitmproxy.crt &&
+            update-ca-certificates &&
             /usr/local/bin/configure-kerberos-client.sh &&
             rm -rf /root/.npm &&
             npm run test:watch'
@@ -35,6 +39,8 @@ services:
                 condition: service_started
             test-http-kerberos-proxy:
                 condition: service_started
+            test-https-proxy:
+                condition: service_started
     test-http-proxy:
         image: ubuntu/squid:latest
         networks:
@@ -68,6 +74,21 @@ services:
         depends_on:
             test-https-server:
                 condition: service_healthy
+    test-https-proxy:
+        image: mitmproxy/mitmproxy:latest
+        # https://stackoverflow.com/q/61453754
+        command: /bin/sh -c 'update-ca-certificates && mitmdump --set ssl_insecure=true'
+        volumes:
+            - ./test-https-proxy/mitmproxy-config:/root/.mitmproxy
+            - ./test-https-server/ssl_cert.pem:/usr/local/share/ca-certificates/test-https-server.crt
+        networks:
+            - test-proxies
+            - test-proxies-and-servers
+        ports:
+            - 8080
+        depends_on:
+            test-https-server:
+                condition: service_healthy
     test-https-server:
         image: test-https-server:latest
         build: test-https-server

tests/test-client/package.json

@@ -13,10 +13,11 @@
   "devDependencies": {
     "@types/kerberos": "^1.1.2",
     "@types/mocha": "5.2.5",
-    "@types/node": "^16.17.1",
+    "@types/node": "^20.8.4",
     "kerberos": "^2.0.1",
     "mocha": "10.2.0",
     "ts-node": "9.1.1",
-    "typescript": "^4.2.2"
+    "typescript": "^5.2.2",
+    "undici": "^6.20.1"
   }
 }

tests/test-client/src/proxy.test.ts

@@ -1,8 +1,11 @@
 import * as https from 'https';
+import * as tls from 'tls';
 import * as assert from 'assert';
+import * as fs from 'fs';
+import * as path from 'path';
 import * as vpa from '../../..';
 import { createPacProxyAgent } from '../../../src/agent';
-import { testRequest, ca, unusedCa, proxiedProxyAgentParamsV1 } from './utils';
+import { testRequest, ca, unusedCa, proxiedProxyAgentParamsV1, tlsProxiedProxyAgentParamsV1 } from './utils';
 
 describe('Proxied client', function () {
 	it('should use HTTP proxy for HTTPS connection', function () {
@@ -14,6 +17,27 @@ describe('Proxied client', function () {
 		});
 	});
 
+	it('should use HTTPS proxy for HTTPS connection', function () {
+		const { resolveProxyWithRequest: resolveProxy } = vpa.createProxyResolver(tlsProxiedProxyAgentParamsV1);
+		const patchedHttps: typeof https = {
+			...https,
+			...vpa.createHttpPatch(tlsProxiedProxyAgentParamsV1, https, resolveProxy),
+		} as any;
+		return testRequest(patchedHttps, {
+			hostname: 'test-https-server',
+			path: '/test-path',
+			_vscodeTestReplaceCaCerts: true,
+		});
+	});
+
+	it('should use HTTPS proxy for HTTPS connection (fetch)', async function () {
+		const { resolveProxyURL } = vpa.createProxyResolver(tlsProxiedProxyAgentParamsV1);
+		const patchedFetch = vpa.createFetchPatch(tlsProxiedProxyAgentParamsV1, globalThis.fetch, resolveProxyURL);
+		const res = await patchedFetch('https://test-https-server/test-path');
+		assert.strictEqual(res.status, 200);
+		assert.strictEqual((await res.json()).status, 'OK!');
+	});
+
 	it('should support basic auth', function () {
 		return testRequest(https, {
 			hostname: 'test-https-server',