OpenCloud
/
MatrixSynapse
mirror of https://github.com/matrix-org/synapse
1
0
Fork 0
Code Issues Releases Wiki Activity

allow specifying https:// proxy (#10411)

pull/10490/head
Dirk Klimpel 2021-07-27 18:31:06 +02:00 committed by GitHub
parent e16eab29d6
commit 076deade02
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 455 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/10411.feature Normal file
View File

@ -0,0 +1 @@
Add support for https connections to a proxy server. Contributed by @Bubu and @dklimpel.

194
synapse/http/proxyagent.py
View File

@ -14,21 +14,32 @@
import base64
import logging
import re
from typing import Optional, Tuple
from urllib.request import getproxies_environment, proxy_bypass_environment
from typing import Any, Dict, Optional, Tuple
from urllib.parse import urlparse
from urllib.request import ( # type: ignore[attr-defined]
getproxies_environment,
proxy_bypass_environment,
)
import attr
from zope.interface import implementer
from twisted.internet import defer
from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
from twisted.internet.interfaces import IReactorCore, IStreamClientEndpoint
from twisted.python.failure import Failure
from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _AgentBase
from twisted.web.client import (
URI,
BrowserLikePolicyForHTTPS,
HTTPConnectionPool,
_AgentBase,
)
from twisted.web.error import SchemeNotSupported
from twisted.web.http_headers import Headers
from twisted.web.iweb import IAgent, IPolicyForHTTPS
from twisted.web.iweb import IAgent, IBodyProducer, IPolicyForHTTPS
from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint
from synapse.types import ISynapseReactor
logger = logging.getLogger(__name__)
@ -63,35 +74,38 @@ class ProxyAgent(_AgentBase):
reactor might have some blacklisting applied (i.e. for DNS queries),
but we need unblocked access to the proxy.
contextFactory (IPolicyForHTTPS): A factory for TLS contexts, to control the
contextFactory: A factory for TLS contexts, to control the
verification parameters of OpenSSL. The default is to use a
`BrowserLikePolicyForHTTPS`, so unless you have special
requirements you can leave this as-is.
connectTimeout (Optional[float]): The amount of time that this Agent will wait
connectTimeout: The amount of time that this Agent will wait
for the peer to accept a connection, in seconds. If 'None',
HostnameEndpoint's default (30s) will be used.
This is used for connections to both proxies and destination servers.
bindAddress (bytes): The local address for client sockets to bind to.
bindAddress: The local address for client sockets to bind to.
pool (HTTPConnectionPool|None): connection pool to be used. If None, a
pool: connection pool to be used. If None, a
non-persistent pool instance will be created.
use_proxy (bool): Whether proxy settings should be discovered and used
use_proxy: Whether proxy settings should be discovered and used
from conventional environment variables.
Raises:
ValueError if use_proxy is set and the environment variables
contain an invalid proxy specification.
"""
def __init__(
self,
reactor,
proxy_reactor=None,
reactor: IReactorCore,
proxy_reactor: Optional[ISynapseReactor] = None,
contextFactory: Optional[IPolicyForHTTPS] = None,
connectTimeout=None,
bindAddress=None,
pool=None,
use_proxy=False,
connectTimeout: Optional[float] = None,
bindAddress: Optional[bytes] = None,
pool: Optional[HTTPConnectionPool] = None,
use_proxy: bool = False,
):
contextFactory = contextFactory or BrowserLikePolicyForHTTPS()
@ -102,7 +116,7 @@ class ProxyAgent(_AgentBase):
else:
self.proxy_reactor = proxy_reactor
self._endpoint_kwargs = {}
self._endpoint_kwargs: Dict[str, Any] = {}
if connectTimeout is not None:
self._endpoint_kwargs["timeout"] = connectTimeout
if bindAddress is not None:
@ -117,16 +131,12 @@ class ProxyAgent(_AgentBase):
https_proxy = proxies["https"].encode() if "https" in proxies else None
no_proxy = proxies["no"] if "no" in proxies else None
# Parse credentials from http and https proxy connection string if present
self.http_proxy_creds, http_proxy = parse_username_password(http_proxy)
self.https_proxy_creds, https_proxy = parse_username_password(https_proxy)
self.http_proxy_endpoint = _http_proxy_endpoint(
http_proxy, self.proxy_reactor, **self._endpoint_kwargs
self.http_proxy_endpoint, self.http_proxy_creds = _http_proxy_endpoint(
http_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs
)
self.https_proxy_endpoint = _http_proxy_endpoint(
https_proxy, self.proxy_reactor, **self._endpoint_kwargs
self.https_proxy_endpoint, self.https_proxy_creds = _http_proxy_endpoint(
https_proxy, self.proxy_reactor, contextFactory, **self._endpoint_kwargs
)
self.no_proxy = no_proxy
@ -134,7 +144,13 @@ class ProxyAgent(_AgentBase):
self._policy_for_https = contextFactory
self._reactor = reactor
def request(self, method, uri, headers=None, bodyProducer=None):
def request(
self,
method: bytes,
uri: bytes,
headers: Optional[Headers] = None,
bodyProducer: Optional[IBodyProducer] = None,
) -> defer.Deferred:
"""
Issue a request to the server indicated by the given uri.
@ -146,16 +162,15 @@ class ProxyAgent(_AgentBase):
See also: twisted.web.iweb.IAgent.request
Args:
method (bytes): The request method to use, such as `GET`, `POST`, etc
method: The request method to use, such as `GET`, `POST`, etc
uri (bytes): The location of the resource to request.
uri: The location of the resource to request.
headers (Headers|None): Extra headers to send with the request
headers: Extra headers to send with the request
bodyProducer (IBodyProducer|None): An object which can generate bytes to
make up the body of this request (for example, the properly encoded
contents of a file for a file upload). Or, None if the request is to
have no body.
bodyProducer: An object which can generate bytes to make up the body of
this request (for example, the properly encoded contents of a file for
a file upload). Or, None if the request is to have no body.
Returns:
Deferred[IResponse]: completes when the header of the response has
@ -253,70 +268,89 @@ class ProxyAgent(_AgentBase):
)
def _http_proxy_endpoint(proxy: Optional[bytes], reactor, **kwargs):
def _http_proxy_endpoint(
proxy: Optional[bytes],
reactor: IReactorCore,
tls_options_factory: IPolicyForHTTPS,
**kwargs,
) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]:
"""Parses an http proxy setting and returns an endpoint for the proxy
Args:
proxy: the proxy setting in the form: [<username>:<password>@]<host>[:<port>]
Note that compared to other apps, this function currently lacks support
for specifying a protocol schema (i.e. protocol://...).
proxy: the proxy setting in the form: [scheme://][<username>:<password>@]<host>[:<port>]
This currently supports http:// and https:// proxies.
A hostname without scheme is assumed to be http.
reactor: reactor to be used to connect to the proxy
tls_options_factory: the TLS options to use when connecting through a https proxy
kwargs: other args to be passed to HostnameEndpoint
Returns:
interfaces.IStreamClientEndpoint|None: endpoint to use to connect to the proxy,
or None
a tuple of
endpoint to use to connect to the proxy, or None
ProxyCredentials or if no credentials were found, or None
Raise:
ValueError if proxy has no hostname or unsupported scheme.
"""
if proxy is None:
return None
return None, None
# Parse the connection string
host, port = parse_host_port(proxy, default_port=1080)
return HostnameEndpoint(reactor, host, port, **kwargs)
# Note: urlsplit/urlparse cannot be used here as that does not work (for Python
# 3.9+) on scheme-less proxies, e.g. host:port.
scheme, host, port, credentials = parse_proxy(proxy)
proxy_endpoint = HostnameEndpoint(reactor, host, port, **kwargs)
if scheme == b"https":
tls_options = tls_options_factory.creatorForNetloc(host, port)
proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint)
return proxy_endpoint, credentials
def parse_username_password(proxy: bytes) -> Tuple[Optional[ProxyCredentials], bytes]:
def parse_proxy(
proxy: bytes, default_scheme: bytes = b"http", default_port: int = 1080
) -> Tuple[bytes, bytes, int, Optional[ProxyCredentials]]:
"""
Parses the username and password from a proxy declaration e.g
username:password@hostname:port.
Parse a proxy connection string.
Given a HTTP proxy URL, breaks it down into components and checks that it
has a hostname (otherwise it is not useful to us when trying to find a
proxy) and asserts that the URL has a scheme we support.
Args:
proxy: The proxy connection string.
Returns
An instance of ProxyCredentials and the proxy connection string with any credentials
stripped, i.e u:p@host:port -> host:port. If no credentials were found, the
ProxyCredentials instance is replaced with None.
"""
if proxy and b"@" in proxy:
# We use rsplit here as the password could contain an @ character
credentials, proxy_without_credentials = proxy.rsplit(b"@", 1)
return ProxyCredentials(credentials), proxy_without_credentials
return None, proxy
def parse_host_port(hostport: bytes, default_port: int = None) -> Tuple[bytes, int]:
"""
Parse the hostname and port from a proxy connection byte string.
Args:
hostport: The proxy connection string. Must be in the form 'host[:port]'.
default_port: The default port to return if one is not found in `hostport`.
proxy: The proxy connection string. Must be in the form '[scheme://][<username>:<password>@]host[:port]'.
default_scheme: The default scheme to return if one is not found in `proxy`. Defaults to http
default_port: The default port to return if one is not found in `proxy`. Defaults to 1080
Returns:
A tuple containing the hostname and port. Uses `default_port` if one was not found.
"""
if b":" in hostport:
host, port = hostport.rsplit(b":", 1)
try:
port = int(port)
return host, port
except ValueError:
# the thing after the : wasn't a valid port; presumably this is an
# IPv6 address.
pass
A tuple containing the scheme, hostname, port and ProxyCredentials.
If no credentials were found, the ProxyCredentials instance is replaced with None.
return hostport, default_port
Raise:
ValueError if proxy has no hostname or unsupported scheme.
"""
# First check if we have a scheme present
# Note: urlsplit/urlparse cannot be used (for Python # 3.9+) on scheme-less proxies, e.g. host:port.
if b"://" not in proxy:
proxy = b"".join([default_scheme, b"://", proxy])
url = urlparse(proxy)
if not url.hostname:
raise ValueError("Proxy URL did not contain a hostname! Please specify one.")
if url.scheme not in (b"http", b"https"):
raise ValueError(
f"Unknown proxy scheme {url.scheme!s}; only 'http' and 'https' is supported."
)
credentials = None
if url.username and url.password:
credentials = ProxyCredentials(b"".join([url.username, b":", url.password]))
return url.scheme, url.hostname, url.port or default_port, credentials

398
tests/http/test_proxyagent.py
View File

@ -14,19 +14,22 @@
import base64
import logging
import os
from typing import Optional
from typing import Iterable, Optional
from unittest.mock import patch
import treq
from netaddr import IPSet
from parameterized import parameterized
from twisted.internet import interfaces # noqa: F401
from twisted.internet.endpoints import HostnameEndpoint, _WrapperEndpoint
from twisted.internet.interfaces import IProtocol, IProtocolFactory
from twisted.internet.protocol import Factory
from twisted.protocols.tls import TLSMemoryBIOFactory
from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
from twisted.web.http import HTTPChannel
from synapse.http.client import BlacklistingReactorWrapper
from synapse.http.proxyagent import ProxyAgent
from synapse.http.proxyagent import ProxyAgent, ProxyCredentials, parse_proxy
from tests.http import TestServerTLSConnectionFactory, get_test_https_policy
from tests.server import FakeTransport, ThreadedMemoryReactorClock
@ -37,33 +40,208 @@ logger = logging.getLogger(__name__)
HTTPFactory = Factory.forProtocol(HTTPChannel)
class ProxyParserTests(TestCase):
"""
Values for test
[
proxy_string,
expected_scheme,
expected_hostname,
expected_port,
expected_credentials,
]
"""
@parameterized.expand(
[
# host
[b"localhost", b"http", b"localhost", 1080, None],
[b"localhost:9988", b"http", b"localhost", 9988, None],
# host+scheme
[b"https://localhost", b"https", b"localhost", 1080, None],
[b"https://localhost:1234", b"https", b"localhost", 1234, None],
# ipv4
[b"1.2.3.4", b"http", b"1.2.3.4", 1080, None],
[b"1.2.3.4:9988", b"http", b"1.2.3.4", 9988, None],
# ipv4+scheme
[b"https://1.2.3.4", b"https", b"1.2.3.4", 1080, None],
[b"https://1.2.3.4:9988", b"https", b"1.2.3.4", 9988, None],
# ipv6 - without brackets is broken
# [
# b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
# b"http",
# b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
# 1080,
# None,
# ],
# [
# b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
# b"http",
# b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
# 1080,
# None,
# ],
# [b"::1", b"http", b"::1", 1080, None],
# [b"::ffff:0.0.0.0", b"http", b"::ffff:0.0.0.0", 1080, None],
# ipv6 - with brackets
[
b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
b"http",
b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
1080,
None,
],
[
b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
b"http",
b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
1080,
None,
],
[b"[::1]", b"http", b"::1", 1080, None],
[b"[::ffff:0.0.0.0]", b"http", b"::ffff:0.0.0.0", 1080, None],
# ipv6+port
[
b"[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
b"http",
b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
9988,
None,
],
[
b"[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
b"http",
b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
9988,
None,
],
[b"[::1]:9988", b"http", b"::1", 9988, None],
[b"[::ffff:0.0.0.0]:9988", b"http", b"::ffff:0.0.0.0", 9988, None],
# ipv6+scheme
[
b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]",
b"https",
b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
1080,
None,
],
[
b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]",
b"https",
b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
1080,
None,
],
[b"https://[::1]", b"https", b"::1", 1080, None],
[b"https://[::ffff:0.0.0.0]", b"https", b"::ffff:0.0.0.0", 1080, None],
# ipv6+scheme+port
[
b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:effe]:9988",
b"https",
b"2001:0db8:85a3:0000:0000:8a2e:0370:effe",
9988,
None,
],
[
b"https://[2001:0db8:85a3:0000:0000:8a2e:0370:1234]:9988",
b"https",
b"2001:0db8:85a3:0000:0000:8a2e:0370:1234",
9988,
None,
],
[b"https://[::1]:9988", b"https", b"::1", 9988, None],
# with credentials
[
b"https://user:pass@1.2.3.4:9988",
b"https",
b"1.2.3.4",
9988,
b"user:pass",
],
[b"user:pass@1.2.3.4:9988", b"http", b"1.2.3.4", 9988, b"user:pass"],
[
b"https://user:pass@proxy.local:9988",
b"https",
b"proxy.local",
9988,
b"user:pass",
],
[
b"user:pass@proxy.local:9988",
b"http",
b"proxy.local",
9988,
b"user:pass",
],
]
)
def test_parse_proxy(
self,
proxy_string: bytes,
expected_scheme: bytes,
expected_hostname: bytes,
expected_port: int,
expected_credentials: Optional[bytes],
):
"""
Tests that a given proxy URL will be broken into the components.
Args:
proxy_string: The proxy connection string.
expected_scheme: Expected value of proxy scheme.
expected_hostname: Expected value of proxy hostname.
expected_port: Expected value of proxy port.
expected_credentials: Expected value of credentials.
Must be in form '<username>:<password>' or None
"""
proxy_cred = None
if expected_credentials:
proxy_cred = ProxyCredentials(expected_credentials)
self.assertEqual(
(
expected_scheme,
expected_hostname,
expected_port,
proxy_cred,
),
parse_proxy(proxy_string),
)
class MatrixFederationAgentTests(TestCase):
def setUp(self):
self.reactor = ThreadedMemoryReactorClock()
def _make_connection(
self, client_factory, server_factory, ssl=False, expected_sni=None
):
self,
client_factory: IProtocolFactory,
server_factory: IProtocolFactory,
ssl: bool = False,
expected_sni: Optional[bytes] = None,
tls_sanlist: Optional[Iterable[bytes]] = None,
) -> IProtocol:
"""Builds a test server, and completes the outgoing client connection
Args:
client_factory (interfaces.IProtocolFactory): the the factory that the
client_factory: the the factory that the
application is trying to use to make the outbound connection. We will
invoke it to build the client Protocol
server_factory (interfaces.IProtocolFactory): a factory to build the
server_factory: a factory to build the
server-side protocol
ssl (bool): If true, we will expect an ssl connection and wrap
ssl: If true, we will expect an ssl connection and wrap
server_factory with a TLSMemoryBIOFactory
expected_sni (bytes|None): the expected SNI value
expected_sni: the expected SNI value
tls_sanlist: list of SAN entries for the TLS cert presented by the server.
Defaults to [b'DNS:test.com']
Returns:
IProtocol: the server Protocol returned by server_factory
the server Protocol returned by server_factory
"""
if ssl:
server_factory = _wrap_server_factory_for_tls(server_factory)
server_factory = _wrap_server_factory_for_tls(server_factory, tls_sanlist)
server_protocol = server_factory.buildProtocol(None)
@ -98,22 +276,28 @@ class MatrixFederationAgentTests(TestCase):
self.assertEqual(
server_name,
expected_sni,
"Expected SNI %s but got %s" % (expected_sni, server_name),
f"Expected SNI {expected_sni!s} but got {server_name!s}",
)
return http_protocol
def _test_request_direct_connection(self, agent, scheme, hostname, path):
def _test_request_direct_connection(
self,
agent: ProxyAgent,
scheme: bytes,
hostname: bytes,
path: bytes,
):
"""Runs a test case for a direct connection not going through a proxy.
Args:
agent (ProxyAgent): the proxy agent being tested
agent: the proxy agent being tested
scheme (bytes): expected to be either "http" or "https"
scheme: expected to be either "http" or "https"
hostname (bytes): the hostname to connect to in the test
hostname: the hostname to connect to in the test
path (bytes): the path to connect to in the test
path: the path to connect to in the test
"""
is_https = scheme == b"https"
@ -208,7 +392,7 @@ class MatrixFederationAgentTests(TestCase):
"""
Tests that requests can be made through a proxy.
"""
self._do_http_request_via_proxy(auth_credentials=None)
self._do_http_request_via_proxy(ssl=False, auth_credentials=None)
@patch.dict(
os.environ,
@ -218,12 +402,28 @@ class MatrixFederationAgentTests(TestCase):
"""
Tests that authenticated requests can be made through a proxy.
"""
self._do_http_request_via_proxy(auth_credentials="bob:pinkponies")
self._do_http_request_via_proxy(ssl=False, auth_credentials=b"bob:pinkponies")
@patch.dict(
os.environ, {"http_proxy": "https://proxy.com:8888", "no_proxy": "unused.com"}
)
def test_http_request_via_https_proxy(self):
self._do_http_request_via_proxy(ssl=True, auth_credentials=None)
@patch.dict(
os.environ,
{
"http_proxy": "https://bob:pinkponies@proxy.com:8888",
"no_proxy": "unused.com",
},
)
def test_http_request_via_https_proxy_with_auth(self):
self._do_http_request_via_proxy(ssl=True, auth_credentials=b"bob:pinkponies")
@patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"})
def test_https_request_via_proxy(self):
"""Tests that TLS-encrypted requests can be made through a proxy"""
self._do_https_request_via_proxy(auth_credentials=None)
self._do_https_request_via_proxy(ssl=False, auth_credentials=None)
@patch.dict(
os.environ,
@ -231,16 +431,40 @@ class MatrixFederationAgentTests(TestCase):
)
def test_https_request_via_proxy_with_auth(self):
"""Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
self._do_https_request_via_proxy(auth_credentials="bob:pinkponies")
self._do_https_request_via_proxy(ssl=False, auth_credentials=b"bob:pinkponies")
@patch.dict(
os.environ, {"https_proxy": "https://proxy.com", "no_proxy": "unused.com"}
)
def test_https_request_via_https_proxy(self):
"""Tests that TLS-encrypted requests can be made through a proxy"""
self._do_https_request_via_proxy(ssl=True, auth_credentials=None)
@patch.dict(
os.environ,
{"https_proxy": "https://bob:pinkponies@proxy.com", "no_proxy": "unused.com"},
)
def test_https_request_via_https_proxy_with_auth(self):
"""Tests that authenticated, TLS-encrypted requests can be made through a proxy"""
self._do_https_request_via_proxy(ssl=True, auth_credentials=b"bob:pinkponies")
def _do_http_request_via_proxy(
self,
auth_credentials: Optional[str] = None,
ssl: bool = False,
auth_credentials: Optional[bytes] = None,
):
"""Send a http request via an agent and check that it is correctly received at
the proxy. The proxy can use either http or https.
Args:
ssl: True if we expect the request to connect via https to proxy
auth_credentials: credentials to authenticate at proxy
"""
Tests that requests can be made through a proxy.
"""
agent = ProxyAgent(self.reactor, use_proxy=True)
if ssl:
agent = ProxyAgent(
self.reactor, use_proxy=True, contextFactory=get_test_https_policy()
)
else:
agent = ProxyAgent(self.reactor, use_proxy=True)
self.reactor.lookups["proxy.com"] = "1.2.3.5"
d = agent.request(b"GET", b"http://test.com")
@ -254,7 +478,11 @@ class MatrixFederationAgentTests(TestCase):
# make a test server, and wire up the client
http_server = self._make_connection(
client_factory, _get_test_protocol_factory()
client_factory,
_get_test_protocol_factory(),
ssl=ssl,
tls_sanlist=[b"DNS:proxy.com"] if ssl else None,
expected_sni=b"proxy.com" if ssl else None,
)
# the FakeTransport is async, so we need to pump the reactor
@ -272,7 +500,7 @@ class MatrixFederationAgentTests(TestCase):
if auth_credentials is not None:
# Compute the correct header value for Proxy-Authorization
encoded_credentials = base64.b64encode(b"bob:pinkponies")
encoded_credentials = base64.b64encode(auth_credentials)
expected_header_value = b"Basic " + encoded_credentials
# Validate the header's value
@ -295,8 +523,15 @@ class MatrixFederationAgentTests(TestCase):
def _do_https_request_via_proxy(
self,
auth_credentials: Optional[str] = None,
ssl: bool = False,
auth_credentials: Optional[bytes] = None,
):
"""Send a https request via an agent and check that it is correctly received at
the proxy and client. The proxy can use either http or https.
Args:
ssl: True if we expect the request to connect via https to proxy
auth_credentials: credentials to authenticate at proxy
"""
agent = ProxyAgent(
self.reactor,
contextFactory=get_test_https_policy(),
@ -313,18 +548,15 @@ class MatrixFederationAgentTests(TestCase):
self.assertEqual(host, "1.2.3.5")
self.assertEqual(port, 1080)
# make a test HTTP server, and wire up the client
# make a test server to act as the proxy, and wire up the client
proxy_server = self._make_connection(
client_factory, _get_test_protocol_factory()
client_factory,
_get_test_protocol_factory(),
ssl=ssl,
tls_sanlist=[b"DNS:proxy.com"] if ssl else None,
expected_sni=b"proxy.com" if ssl else None,
)
# fish the transports back out so that we can do the old switcheroo
s2c_transport = proxy_server.transport
client_protocol = s2c_transport.other
c2s_transport = client_protocol.transport
# the FakeTransport is async, so we need to pump the reactor
self.reactor.advance(0)
assert isinstance(proxy_server, HTTPChannel)
# now there should be a pending CONNECT request
self.assertEqual(len(proxy_server.requests), 1)
@ -340,7 +572,7 @@ class MatrixFederationAgentTests(TestCase):
if auth_credentials is not None:
# Compute the correct header value for Proxy-Authorization
encoded_credentials = base64.b64encode(b"bob:pinkponies")
encoded_credentials = base64.b64encode(auth_credentials)
expected_header_value = b"Basic " + encoded_credentials
# Validate the header's value
@ -352,31 +584,49 @@ class MatrixFederationAgentTests(TestCase):
# tell the proxy server not to close the connection
proxy_server.persistent = True
# this just stops the http Request trying to do a chunked response
# request.setHeader(b"Content-Length", b"0")
request.finish()
# now we can replace the proxy channel with a new, SSL-wrapped HTTP channel
ssl_factory = _wrap_server_factory_for_tls(_get_test_protocol_factory())
ssl_protocol = ssl_factory.buildProtocol(None)
http_server = ssl_protocol.wrappedProtocol
# now we make another test server to act as the upstream HTTP server.
server_ssl_protocol = _wrap_server_factory_for_tls(
_get_test_protocol_factory()
).buildProtocol(None)
ssl_protocol.makeConnection(
FakeTransport(client_protocol, self.reactor, ssl_protocol)
)
c2s_transport.other = ssl_protocol
# Tell the HTTP server to send outgoing traffic back via the proxy's transport.
proxy_server_transport = proxy_server.transport
server_ssl_protocol.makeConnection(proxy_server_transport)
# ... and replace the protocol on the proxy's transport with the
# TLSMemoryBIOProtocol for the test server, so that incoming traffic
# to the proxy gets sent over to the HTTP(s) server.
#
# This needs a bit of gut-wrenching, which is different depending on whether
# the proxy is using TLS or not.
#
# (an alternative, possibly more elegant, approach would be to use a custom
# Protocol to implement the proxy, which starts out by forwarding to an
# HTTPChannel (to implement the CONNECT command) and can then be switched
# into a mode where it forwards its traffic to another Protocol.)
if ssl:
assert isinstance(proxy_server_transport, TLSMemoryBIOProtocol)
proxy_server_transport.wrappedProtocol = server_ssl_protocol
else:
assert isinstance(proxy_server_transport, FakeTransport)
client_protocol = proxy_server_transport.other
c2s_transport = client_protocol.transport
c2s_transport.other = server_ssl_protocol
self.reactor.advance(0)
server_name = ssl_protocol._tlsConnection.get_servername()
server_name = server_ssl_protocol._tlsConnection.get_servername()
expected_sni = b"test.com"
self.assertEqual(
server_name,
expected_sni,
"Expected SNI %s but got %s" % (expected_sni, server_name),
f"Expected SNI {expected_sni!s} but got {server_name!s}",
)
# now there should be a pending request
http_server = server_ssl_protocol.wrappedProtocol
self.assertEqual(len(http_server.requests), 1)
request = http_server.requests[0]
@ -510,7 +760,7 @@ class MatrixFederationAgentTests(TestCase):
self.assertEqual(
server_name,
expected_sni,
"Expected SNI %s but got %s" % (expected_sni, server_name),
f"Expected SNI {expected_sni!s} but got {server_name!s}",
)
# now there should be a pending request
@ -529,16 +779,48 @@ class MatrixFederationAgentTests(TestCase):
body = self.successResultOf(treq.content(resp))
self.assertEqual(body, b"result")
@patch.dict(os.environ, {"http_proxy": "proxy.com:8888"})
def test_proxy_with_no_scheme(self):
http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
self.assertIsInstance(http_proxy_agent.http_proxy_endpoint, HostnameEndpoint)
self.assertEqual(http_proxy_agent.http_proxy_endpoint._hostStr, "proxy.com")
self.assertEqual(http_proxy_agent.http_proxy_endpoint._port, 8888)
def _wrap_server_factory_for_tls(factory, sanlist=None):
@patch.dict(os.environ, {"http_proxy": "socks://proxy.com:8888"})
def test_proxy_with_unsupported_scheme(self):
with self.assertRaises(ValueError):
ProxyAgent(self.reactor, use_proxy=True)
@patch.dict(os.environ, {"http_proxy": "http://proxy.com:8888"})
def test_proxy_with_http_scheme(self):
http_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
self.assertIsInstance(http_proxy_agent.http_proxy_endpoint, HostnameEndpoint)
self.assertEqual(http_proxy_agent.http_proxy_endpoint._hostStr, "proxy.com")
self.assertEqual(http_proxy_agent.http_proxy_endpoint._port, 8888)
@patch.dict(os.environ, {"http_proxy": "https://proxy.com:8888"})
def test_proxy_with_https_scheme(self):
https_proxy_agent = ProxyAgent(self.reactor, use_proxy=True)
self.assertIsInstance(https_proxy_agent.http_proxy_endpoint, _WrapperEndpoint)
self.assertEqual(
https_proxy_agent.http_proxy_endpoint._wrappedEndpoint._hostStr, "proxy.com"
)
self.assertEqual(
https_proxy_agent.http_proxy_endpoint._wrappedEndpoint._port, 8888
)
def _wrap_server_factory_for_tls(
factory: IProtocolFactory, sanlist: Iterable[bytes] = None
) -> IProtocolFactory:
"""Wrap an existing Protocol Factory with a test TLSMemoryBIOFactory
The resultant factory will create a TLS server which presents a certificate
signed by our test CA, valid for the domains in `sanlist`
Args:
factory (interfaces.IProtocolFactory): protocol factory to wrap
sanlist (iterable[bytes]): list of domains the cert should be valid for
factory: protocol factory to wrap
sanlist: list of domains the cert should be valid for
Returns:
interfaces.IProtocolFactory
@ -552,7 +834,7 @@ def _wrap_server_factory_for_tls(factory, sanlist=None):
)
def _get_test_protocol_factory():
def _get_test_protocol_factory() -> IProtocolFactory:
"""Get a protocol Factory which will build an HTTPChannel
Returns:
@ -566,6 +848,6 @@ def _get_test_protocol_factory():
return server_factory
def _log_request(request):
def _log_request(request: str):
"""Implements Factory.log, which is expected by Request.finish"""
logger.info("Completed request %s", request)
logger.info(f"Completed request {request}")