connection.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423
  1. from __future__ import absolute_import
  2. import re
  3. import datetime
  4. import logging
  5. import os
  6. import socket
  7. from socket import error as SocketError, timeout as SocketTimeout
  8. import warnings
  9. from .packages import six
  10. from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection
  11. from .packages.six.moves.http_client import HTTPException # noqa: F401
  12. try: # Compiled with SSL?
  13. import ssl
  14. BaseSSLError = ssl.SSLError
  15. except (ImportError, AttributeError): # Platform-specific: No SSL.
  16. ssl = None
  17. class BaseSSLError(BaseException):
  18. pass
  19. try:
  20. # Python 3: not a no-op, we're adding this to the namespace so it can be imported.
  21. ConnectionError = ConnectionError
  22. except NameError:
  23. # Python 2
  24. class ConnectionError(Exception):
  25. pass
  26. from .exceptions import (
  27. NewConnectionError,
  28. ConnectTimeoutError,
  29. SubjectAltNameWarning,
  30. SystemTimeWarning,
  31. )
  32. from .packages.ssl_match_hostname import match_hostname, CertificateError
  33. from .util.ssl_ import (
  34. resolve_cert_reqs,
  35. resolve_ssl_version,
  36. assert_fingerprint,
  37. create_urllib3_context,
  38. ssl_wrap_socket,
  39. )
  40. from .util import connection
  41. from ._collections import HTTPHeaderDict
  42. log = logging.getLogger(__name__)
  43. port_by_scheme = {"http": 80, "https": 443}
  44. # When it comes time to update this value as a part of regular maintenance
  45. # (ie test_recent_date is failing) update it to ~6 months before the current date.
  46. RECENT_DATE = datetime.date(2019, 1, 1)
  47. _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]")
  48. class DummyConnection(object):
  49. """Used to detect a failed ConnectionCls import."""
  50. pass
  51. class HTTPConnection(_HTTPConnection, object):
  52. """
  53. Based on httplib.HTTPConnection but provides an extra constructor
  54. backwards-compatibility layer between older and newer Pythons.
  55. Additional keyword parameters are used to configure attributes of the connection.
  56. Accepted parameters include:
  57. - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool`
  58. - ``source_address``: Set the source address for the current connection.
  59. - ``socket_options``: Set specific options on the underlying socket. If not specified, then
  60. defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
  61. Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
  62. For example, if you wish to enable TCP Keep Alive in addition to the defaults,
  63. you might pass::
  64. HTTPConnection.default_socket_options + [
  65. (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
  66. ]
  67. Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
  68. """
  69. default_port = port_by_scheme["http"]
  70. #: Disable Nagle's algorithm by default.
  71. #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
  72. default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
  73. #: Whether this connection verifies the host's certificate.
  74. is_verified = False
  75. def __init__(self, *args, **kw):
  76. if not six.PY2:
  77. kw.pop("strict", None)
  78. # Pre-set source_address.
  79. self.source_address = kw.get("source_address")
  80. #: The socket options provided by the user. If no options are
  81. #: provided, we use the default options.
  82. self.socket_options = kw.pop("socket_options", self.default_socket_options)
  83. _HTTPConnection.__init__(self, *args, **kw)
  84. @property
  85. def host(self):
  86. """
  87. Getter method to remove any trailing dots that indicate the hostname is an FQDN.
  88. In general, SSL certificates don't include the trailing dot indicating a
  89. fully-qualified domain name, and thus, they don't validate properly when
  90. checked against a domain name that includes the dot. In addition, some
  91. servers may not expect to receive the trailing dot when provided.
  92. However, the hostname with trailing dot is critical to DNS resolution; doing a
  93. lookup with the trailing dot will properly only resolve the appropriate FQDN,
  94. whereas a lookup without a trailing dot will search the system's search domain
  95. list. Thus, it's important to keep the original host around for use only in
  96. those cases where it's appropriate (i.e., when doing DNS lookup to establish the
  97. actual TCP connection across which we're going to send HTTP requests).
  98. """
  99. return self._dns_host.rstrip(".")
  100. @host.setter
  101. def host(self, value):
  102. """
  103. Setter for the `host` property.
  104. We assume that only urllib3 uses the _dns_host attribute; httplib itself
  105. only uses `host`, and it seems reasonable that other libraries follow suit.
  106. """
  107. self._dns_host = value
  108. def _new_conn(self):
  109. """ Establish a socket connection and set nodelay settings on it.
  110. :return: New socket connection.
  111. """
  112. extra_kw = {}
  113. if self.source_address:
  114. extra_kw["source_address"] = self.source_address
  115. if self.socket_options:
  116. extra_kw["socket_options"] = self.socket_options
  117. try:
  118. conn = connection.create_connection(
  119. (self._dns_host, self.port), self.timeout, **extra_kw
  120. )
  121. except SocketTimeout:
  122. raise ConnectTimeoutError(
  123. self,
  124. "Connection to %s timed out. (connect timeout=%s)"
  125. % (self.host, self.timeout),
  126. )
  127. except SocketError as e:
  128. raise NewConnectionError(
  129. self, "Failed to establish a new connection: %s" % e
  130. )
  131. return conn
  132. def _prepare_conn(self, conn):
  133. self.sock = conn
  134. # Google App Engine's httplib does not define _tunnel_host
  135. if getattr(self, "_tunnel_host", None):
  136. # TODO: Fix tunnel so it doesn't depend on self.sock state.
  137. self._tunnel()
  138. # Mark this connection as not reusable
  139. self.auto_open = 0
  140. def connect(self):
  141. conn = self._new_conn()
  142. self._prepare_conn(conn)
  143. def putrequest(self, method, url, *args, **kwargs):
  144. """Send a request to the server"""
  145. match = _CONTAINS_CONTROL_CHAR_RE.search(method)
  146. if match:
  147. raise ValueError(
  148. "Method cannot contain non-token characters %r (found at least %r)"
  149. % (method, match.group())
  150. )
  151. return _HTTPConnection.putrequest(self, method, url, *args, **kwargs)
  152. def request_chunked(self, method, url, body=None, headers=None):
  153. """
  154. Alternative to the common request method, which sends the
  155. body with chunked encoding and not as one block
  156. """
  157. headers = HTTPHeaderDict(headers if headers is not None else {})
  158. skip_accept_encoding = "accept-encoding" in headers
  159. skip_host = "host" in headers
  160. self.putrequest(
  161. method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
  162. )
  163. for header, value in headers.items():
  164. self.putheader(header, value)
  165. if "transfer-encoding" not in headers:
  166. self.putheader("Transfer-Encoding", "chunked")
  167. self.endheaders()
  168. if body is not None:
  169. stringish_types = six.string_types + (bytes,)
  170. if isinstance(body, stringish_types):
  171. body = (body,)
  172. for chunk in body:
  173. if not chunk:
  174. continue
  175. if not isinstance(chunk, bytes):
  176. chunk = chunk.encode("utf8")
  177. len_str = hex(len(chunk))[2:]
  178. self.send(len_str.encode("utf-8"))
  179. self.send(b"\r\n")
  180. self.send(chunk)
  181. self.send(b"\r\n")
  182. # After the if clause, to always have a closed body
  183. self.send(b"0\r\n\r\n")
  184. class HTTPSConnection(HTTPConnection):
  185. default_port = port_by_scheme["https"]
  186. cert_reqs = None
  187. ca_certs = None
  188. ca_cert_dir = None
  189. ca_cert_data = None
  190. ssl_version = None
  191. assert_fingerprint = None
  192. def __init__(
  193. self,
  194. host,
  195. port=None,
  196. key_file=None,
  197. cert_file=None,
  198. key_password=None,
  199. strict=None,
  200. timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
  201. ssl_context=None,
  202. server_hostname=None,
  203. **kw
  204. ):
  205. HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw)
  206. self.key_file = key_file
  207. self.cert_file = cert_file
  208. self.key_password = key_password
  209. self.ssl_context = ssl_context
  210. self.server_hostname = server_hostname
  211. # Required property for Google AppEngine 1.9.0 which otherwise causes
  212. # HTTPS requests to go out as HTTP. (See Issue #356)
  213. self._protocol = "https"
  214. def set_cert(
  215. self,
  216. key_file=None,
  217. cert_file=None,
  218. cert_reqs=None,
  219. key_password=None,
  220. ca_certs=None,
  221. assert_hostname=None,
  222. assert_fingerprint=None,
  223. ca_cert_dir=None,
  224. ca_cert_data=None,
  225. ):
  226. """
  227. This method should only be called once, before the connection is used.
  228. """
  229. # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
  230. # have an SSLContext object in which case we'll use its verify_mode.
  231. if cert_reqs is None:
  232. if self.ssl_context is not None:
  233. cert_reqs = self.ssl_context.verify_mode
  234. else:
  235. cert_reqs = resolve_cert_reqs(None)
  236. self.key_file = key_file
  237. self.cert_file = cert_file
  238. self.cert_reqs = cert_reqs
  239. self.key_password = key_password
  240. self.assert_hostname = assert_hostname
  241. self.assert_fingerprint = assert_fingerprint
  242. self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
  243. self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
  244. self.ca_cert_data = ca_cert_data
  245. def connect(self):
  246. # Add certificate verification
  247. conn = self._new_conn()
  248. hostname = self.host
  249. # Google App Engine's httplib does not define _tunnel_host
  250. if getattr(self, "_tunnel_host", None):
  251. self.sock = conn
  252. # Calls self._set_hostport(), so self.host is
  253. # self._tunnel_host below.
  254. self._tunnel()
  255. # Mark this connection as not reusable
  256. self.auto_open = 0
  257. # Override the host with the one we're requesting data from.
  258. hostname = self._tunnel_host
  259. server_hostname = hostname
  260. if self.server_hostname is not None:
  261. server_hostname = self.server_hostname
  262. is_time_off = datetime.date.today() < RECENT_DATE
  263. if is_time_off:
  264. warnings.warn(
  265. (
  266. "System time is way off (before {0}). This will probably "
  267. "lead to SSL verification errors"
  268. ).format(RECENT_DATE),
  269. SystemTimeWarning,
  270. )
  271. # Wrap socket using verification with the root certs in
  272. # trusted_root_certs
  273. default_ssl_context = False
  274. if self.ssl_context is None:
  275. default_ssl_context = True
  276. self.ssl_context = create_urllib3_context(
  277. ssl_version=resolve_ssl_version(self.ssl_version),
  278. cert_reqs=resolve_cert_reqs(self.cert_reqs),
  279. )
  280. context = self.ssl_context
  281. context.verify_mode = resolve_cert_reqs(self.cert_reqs)
  282. # Try to load OS default certs if none are given.
  283. # Works well on Windows (requires Python3.4+)
  284. if (
  285. not self.ca_certs
  286. and not self.ca_cert_dir
  287. and not self.ca_cert_data
  288. and default_ssl_context
  289. and hasattr(context, "load_default_certs")
  290. ):
  291. context.load_default_certs()
  292. self.sock = ssl_wrap_socket(
  293. sock=conn,
  294. keyfile=self.key_file,
  295. certfile=self.cert_file,
  296. key_password=self.key_password,
  297. ca_certs=self.ca_certs,
  298. ca_cert_dir=self.ca_cert_dir,
  299. ca_cert_data=self.ca_cert_data,
  300. server_hostname=server_hostname,
  301. ssl_context=context,
  302. )
  303. if self.assert_fingerprint:
  304. assert_fingerprint(
  305. self.sock.getpeercert(binary_form=True), self.assert_fingerprint
  306. )
  307. elif (
  308. context.verify_mode != ssl.CERT_NONE
  309. and not getattr(context, "check_hostname", False)
  310. and self.assert_hostname is not False
  311. ):
  312. # While urllib3 attempts to always turn off hostname matching from
  313. # the TLS library, this cannot always be done. So we check whether
  314. # the TLS Library still thinks it's matching hostnames.
  315. cert = self.sock.getpeercert()
  316. if not cert.get("subjectAltName", ()):
  317. warnings.warn(
  318. (
  319. "Certificate for {0} has no `subjectAltName`, falling back to check for a "
  320. "`commonName` for now. This feature is being removed by major browsers and "
  321. "deprecated by RFC 2818. (See https://github.com/urllib3/urllib3/issues/497 "
  322. "for details.)".format(hostname)
  323. ),
  324. SubjectAltNameWarning,
  325. )
  326. _match_hostname(cert, self.assert_hostname or server_hostname)
  327. self.is_verified = (
  328. context.verify_mode == ssl.CERT_REQUIRED
  329. or self.assert_fingerprint is not None
  330. )
  331. def _match_hostname(cert, asserted_hostname):
  332. try:
  333. match_hostname(cert, asserted_hostname)
  334. except CertificateError as e:
  335. log.warning(
  336. "Certificate did not match expected hostname: %s. Certificate: %s",
  337. asserted_hostname,
  338. cert,
  339. )
  340. # Add cert to exception and reraise so client code can inspect
  341. # the cert when catching the exception, if they want to
  342. e._peer_cert = cert
  343. raise
  344. if not ssl:
  345. HTTPSConnection = DummyConnection # noqa: F811
  346. VerifiedHTTPSConnection = HTTPSConnection