Add urllib3 AuthorizedHttp

docs/conf.py

@@ -363,7 +363,10 @@
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/3.5': None}
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3.5', None),
+    'urllib3': ('https://urllib3.readthedocs.io/en/latest', None),
+}
 
 # Autodoc config
 autoclass_content = 'both'

google/auth/transport/__init__.py

@@ -27,6 +27,15 @@
 import abc
 
 import six
+from six.moves import http_client
+
+DEFAULT_REFRESH_STATUS_CODES = (http_client.UNAUTHORIZED,)
+"""Sequence[int]:  Which HTTP status code indicate that credentials should be
+refreshed and a request should be retried.
+"""
+
+DEFAULT_MAX_REFRESH_ATTEMPTS = 2
+"""int: How many times to refresh the credentials and retry a request."""
 
 
 @six.add_metaclass(abc.ABCMeta)

google/auth/transport/urllib3.py

@@ -18,6 +18,18 @@
 
 import logging
 
+
+# Certifi is Mozilla's certificate bundle. Urllib3 needs a certificate bundle
+# to verify HTTPS requests, and certifi is the recommended and most reliable
+# way to get a root certificate bundle. See
+# http://urllib3.readthedocs.io/en/latest/user-guide.html\
+#   #certificate-verification
+# For more details.
+try:
+    import certifi
+except ImportError:  # pragma: NO COVER
+    certifi = None
+
 import urllib3
 import urllib3.exceptions
 
@@ -27,7 +39,7 @@
 _LOGGER = logging.getLogger(__name__)
 
 
-class Response(transport.Response):
+class _Response(transport.Response):
     """urllib3 transport response adapter.
 
     Args:
@@ -50,7 +62,22 @@ def data(self):
 
 
 class Request(transport.Request):
-    """urllib3 request adapter
+    """urllib3 request adapter.
+
+    This class is used internally for making requests using various transports
+    in a consistent way. If you use :class:`AuthorizedHttp` you do not need
+    to construct or use this class directly.
+
+    This class can be useful if you want to manually refresh a
+    :class:`~google.auth.credentials.Credentials` instance::
+
+        import google.auth.transport.urllib3
+        import urllib3
+
+        http = urllib3.PoolManager()
+        request = google.auth.transport.urllib3.Request(http)
+
+        credentials.refresh(request)
 
     Args:
         http (urllib3.request.RequestMethods): An instance of any urllib3
@@ -79,7 +106,7 @@ def __call__(self, url, method='GET', body=None, headers=None,
                 urllib3 :meth:`urlopen` method.
 
         Returns:
-            Response: The HTTP response.
+            google.auth.transport.Response: The HTTP response.
 
         Raises:
             google.auth.exceptions.TransportError: If any exception occurred.
@@ -93,6 +120,129 @@ def __call__(self, url, method='GET', body=None, headers=None,
             _LOGGER.debug('Making request: %s %s', method, url)
             response = self.http.request(
                 method, url, body=body, headers=headers, **kwargs)
-            return Response(response)
+            return _Response(response)
         except urllib3.exceptions.HTTPError as exc:
             raise exceptions.TransportError(exc)
+
+
+def _make_default_http():
+    if certifi is not None:
+        return urllib3.PoolManager(
+            cert_reqs='CERT_REQUIRED',
+            ca_certs=certifi.where())
+    else:
+        return urllib3.PoolManager()
+
+
+class AuthorizedHttp(urllib3.request.RequestMethods):
+    """A urllib3 HTTP class with credentials.
+
+    This class is used to perform requests to API endpoints that require
+    authorization::
+
+        from google.auth.transport.urllib3 import AuthorizedHttp
+
+        authed_http = AuthorizedHttp(credentials)
+
+        response = authed_http.request(
+            'GET', 'https://www.googleapis.com/storage/v1/b')
+
+    This class implements :class:`urllib3.request.RequestMethods` and can be
+    used just like any other :class:`urllib3.PoolManager`.
+
+    The underlying :meth:`urlopen` implementation handles adding the
+    credentials' headers to the request and refreshing credentials as needed.
+
+    Args:
+        credentials (google.auth.credentials.Credentials): The credentials to
+            add to the request.
+        http (urllib3.PoolManager): The underlying HTTP object to
+            use to make requests. If not specified, a
+            :class:`urllib3.PoolManager` instance will be constructed with
+            sane defaults.
+        refresh_status_codes (Sequence[int]): Which HTTP status codes indicate
+            that credentials should be refreshed and the request should be
+            retried.
+        max_refresh_attempts (int): The maximum number of times to attempt to
+            refresh the credentials and retry the request.
+    """
+    def __init__(self, credentials, http=None,
+                 refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
+                 max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS):
+
+        if http is None:
+            http = _make_default_http()
+
+        self.http = http
+        self.credentials = credentials
+        self._refresh_status_codes = refresh_status_codes
+        self._max_refresh_attempts = max_refresh_attempts
+        # Request instance used by internal methods (for example,
+        # credentials.refresh).
+        self._request = Request(self.http)
+
+    def urlopen(self, method, url, body=None, headers=None, **kwargs):
+        """Implementation of urllib3's urlopen."""
+
+        # Use a kwarg for this instead of an attribute to maintain
+        # thread-safety.
+        _credential_refresh_attempt = kwargs.pop(
+            '_credential_refresh_attempt', 0)
+
+        if headers is None:
+            headers = self.headers
+
+        # Make a copy of the headers. They will be modified by the credentials
+        # and we want to pass the original headers if we recurse.
+        request_headers = headers.copy()
+
+        self.credentials.before_request(
+            self._request, method, url, request_headers)
+
+        response = self.http.urlopen(
+            method, url, body=body, headers=request_headers, **kwargs)
+
+        # If the response indicated that the credentials needed to be
+        # refreshed, then refresh the credentials and re-attempt the
+        # request.
+        # A stored token may expire between the time it is retrieved and
+        # the time the request is made, so we may need to try twice.
+        # The reason urllib3's retries aren't used is because they
+        # don't allow you to modify the request headers. :/
+        if (response.status in self._refresh_status_codes
+                and _credential_refresh_attempt < self._max_refresh_attempts):
+
+            _LOGGER.info(
+                'Refreshing credentials due to a %s response. Attempt %s/%s.',
+                response.status, _credential_refresh_attempt + 1,
+                self._max_refresh_attempts)
+
+            self.credentials.refresh(self._request)
+
+            # Recurse. Pass in the original headers, not our modified set.
+            return self.urlopen(
+                method, url, body=body, headers=headers,
+                _credential_refresh_attempt=_credential_refresh_attempt + 1,
+                **kwargs)
+
+        return response
+
+    # Proxy methods for compliance with the urllib3.PoolManager interface
+
+    def __enter__(self):
+        """Proxy to ``self.http``."""
+        return self.http.__enter__()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Proxy to ``self.http``."""
+        return self.http.__exit__(exc_type, exc_val, exc_tb)
+
+    @property
+    def headers(self):
+        """Proxy to ``self.http``."""
+        return self.http.headers
+
+    @headers.setter
+    def headers(self, value):
+        """Proxy to ``self.http``."""
+        self.http.headers = value

tests/transport/test_urllib3.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import mock
+from six.moves import http_client
 import urllib3
 
 import google.auth.transport.urllib3
@@ -24,10 +25,113 @@ def make_request(self):
         http = urllib3.PoolManager()
         return google.auth.transport.urllib3.Request(http)
 
+    def test_timeout(self):
+        http = mock.Mock()
+        request = google.auth.transport.urllib3.Request(http)
+        request(url='http://example.com', method='GET', timeout=5)
 
-def test_timeout():
-    http = mock.Mock()
-    request = google.auth.transport.urllib3.Request(http)
-    request(url='http://example.com', method='GET', timeout=5)
+        assert http.request.call_args[1]['timeout'] == 5
 
-    assert http.request.call_args[1]['timeout'] == 5
+
+def test__make_default_http_with_certfi():
+    http = google.auth.transport.urllib3._make_default_http()
+    assert 'cert_reqs' in http.connection_pool_kw
+
+
+@mock.patch.object(google.auth.transport.urllib3, 'certifi', new=None)
+def test__make_default_http_without_certfi():
+    http = google.auth.transport.urllib3._make_default_http()
+    assert 'cert_reqs' not in http.connection_pool_kw
+
+
+class MockCredentials(object):
+    def __init__(self, token='token'):
+        self.token = token
+
+    def apply(self, headers):
+        headers['authorization'] = self.token
+
+    def before_request(self, request, method, url, headers):
+        self.apply(headers)
+
+    def refresh(self, request):
+        self.token += '1'
+
+
+class MockHttp(object):
+    def __init__(self, responses, headers=None):
+        self.responses = responses
+        self.requests = []
+        self.headers = headers or {}
+
+    def urlopen(self, method, url, body=None, headers=None, **kwargs):
+        self.requests.append((method, url, body, headers, kwargs))
+        return self.responses.pop(0)
+
+
+class MockResponse(object):
+    def __init__(self, status=http_client.OK, data=None):
+        self.status = status
+        self.data = data
+
+
+class TestAuthorizedHttp(object):
+    TEST_URL = 'http://example.com'
+
+    def test_authed_http_defaults(self):
+        authed_http = google.auth.transport.urllib3.AuthorizedHttp(
+            mock.sentinel.credentials)
+
+        assert authed_http.credentials == mock.sentinel.credentials
+        assert isinstance(authed_http.http, urllib3.PoolManager)
+
+    def test_urlopen_no_refresh(self):
+        mock_credentials = mock.Mock(wraps=MockCredentials())
+        mock_response = MockResponse()
+        mock_http = MockHttp([mock_response])
+
+        authed_http = google.auth.transport.urllib3.AuthorizedHttp(
+            mock_credentials, http=mock_http)
+
+        response = authed_http.urlopen('GET', self.TEST_URL)
+
+        assert response == mock_response
+        assert mock_credentials.before_request.called
+        assert not mock_credentials.refresh.called
+        assert mock_http.requests == [
+            ('GET', self.TEST_URL, None, {'authorization': 'token'}, {})]
+
+    def test_urlopen_refresh(self):
+        mock_credentials = mock.Mock(wraps=MockCredentials())
+        mock_final_response = MockResponse(status=http_client.OK)
+        # First request will 401, second request will succeed.
+        mock_http = MockHttp([
+            MockResponse(status=http_client.UNAUTHORIZED),
+            mock_final_response])
+
+        authed_http = google.auth.transport.urllib3.AuthorizedHttp(
+            mock_credentials, http=mock_http)
+
+        response = authed_http.urlopen('GET', 'http://example.com')
+
+        assert response == mock_final_response
+        assert mock_credentials.before_request.call_count == 2
+        assert mock_credentials.refresh.called
+        assert mock_http.requests == [
+            ('GET', self.TEST_URL, None, {'authorization': 'token'}, {}),
+            ('GET', self.TEST_URL, None, {'authorization': 'token1'}, {})]
+
+    def test_proxies(self):
+        mock_http = mock.MagicMock()
+
+        authed_http = google.auth.transport.urllib3.AuthorizedHttp(
+            None, http=mock_http)
+
+        with authed_http:
+            pass
+
+        assert mock_http.__enter__.called
+        assert mock_http.__exit__.called
+
+        authed_http.headers = mock.sentinel.headers
+        assert authed_http.headers == mock_http.headers

tox.ini

@@ -9,6 +9,7 @@ deps =
   pytest-cov
   pytest-localserver
   urllib3
+  certifi
 commands =
   py.test --cov=google.auth --cov=google.oauth2 --cov=tests {posargs:tests}