utils.py 4.1 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697
  1. from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
  2. from pip._internal.exceptions import NetworkConnectionError
  3. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  4. if MYPY_CHECK_RUNNING:
  5. from typing import Dict, Iterator
  6. # The following comments and HTTP headers were originally added by
  7. # Donald Stufft in git commit 22c562429a61bb77172039e480873fb239dd8c03.
  8. #
  9. # We use Accept-Encoding: identity here because requests defaults to
  10. # accepting compressed responses. This breaks in a variety of ways
  11. # depending on how the server is configured.
  12. # - Some servers will notice that the file isn't a compressible file
  13. # and will leave the file alone and with an empty Content-Encoding
  14. # - Some servers will notice that the file is already compressed and
  15. # will leave the file alone, adding a Content-Encoding: gzip header
  16. # - Some servers won't notice anything at all and will take a file
  17. # that's already been compressed and compress it again, and set
  18. # the Content-Encoding: gzip header
  19. # By setting this to request only the identity encoding we're hoping
  20. # to eliminate the third case. Hopefully there does not exist a server
  21. # which when given a file will notice it is already compressed and that
  22. # you're not asking for a compressed file and will then decompress it
  23. # before sending because if that's the case I don't think it'll ever be
  24. # possible to make this work.
  25. HEADERS = {'Accept-Encoding': 'identity'} # type: Dict[str, str]
  26. def raise_for_status(resp):
  27. # type: (Response) -> None
  28. http_error_msg = u''
  29. if isinstance(resp.reason, bytes):
  30. # We attempt to decode utf-8 first because some servers
  31. # choose to localize their reason strings. If the string
  32. # isn't utf-8, we fall back to iso-8859-1 for all other
  33. # encodings.
  34. try:
  35. reason = resp.reason.decode('utf-8')
  36. except UnicodeDecodeError:
  37. reason = resp.reason.decode('iso-8859-1')
  38. else:
  39. reason = resp.reason
  40. if 400 <= resp.status_code < 500:
  41. http_error_msg = u'%s Client Error: %s for url: %s' % (
  42. resp.status_code, reason, resp.url)
  43. elif 500 <= resp.status_code < 600:
  44. http_error_msg = u'%s Server Error: %s for url: %s' % (
  45. resp.status_code, reason, resp.url)
  46. if http_error_msg:
  47. raise NetworkConnectionError(http_error_msg, response=resp)
  48. def response_chunks(response, chunk_size=CONTENT_CHUNK_SIZE):
  49. # type: (Response, int) -> Iterator[bytes]
  50. """Given a requests Response, provide the data chunks.
  51. """
  52. try:
  53. # Special case for urllib3.
  54. for chunk in response.raw.stream(
  55. chunk_size,
  56. # We use decode_content=False here because we don't
  57. # want urllib3 to mess with the raw bytes we get
  58. # from the server. If we decompress inside of
  59. # urllib3 then we cannot verify the checksum
  60. # because the checksum will be of the compressed
  61. # file. This breakage will only occur if the
  62. # server adds a Content-Encoding header, which
  63. # depends on how the server was configured:
  64. # - Some servers will notice that the file isn't a
  65. # compressible file and will leave the file alone
  66. # and with an empty Content-Encoding
  67. # - Some servers will notice that the file is
  68. # already compressed and will leave the file
  69. # alone and will add a Content-Encoding: gzip
  70. # header
  71. # - Some servers won't notice anything at all and
  72. # will take a file that's already been compressed
  73. # and compress it again and set the
  74. # Content-Encoding: gzip header
  75. #
  76. # By setting this not to decode automatically we
  77. # hope to eliminate problems with the second case.
  78. decode_content=False,
  79. ):
  80. yield chunk
  81. except AttributeError:
  82. # Standard file-like object.
  83. while True:
  84. chunk = response.raw.read(chunk_size)
  85. if not chunk:
  86. break
  87. yield chunk