More integration testing

docs/history.rst

@@ -8,6 +8,7 @@ Version History
  - Rework error processing to support ``application/json``,
     ``application/x-amz-json-1.0``, ``application/x-amz-json-1.0`` and
     XML based responses (S3, E2, others)
+ - Move to 100% test coverage \o/
 0.7.3 (2017-06-06)
 ------------------

tests/client_tests.py

@@ -388,6 +388,14 @@ class ClientFetchTestCase(MockTestCase):
                 self.assertEqual(request.headers['Content-Type'],
                                  'application/x-amz-json-1.0')
+    def test_fetch_os_error(self):
+        with self.client_with_default_creds('s3') as obj:
+            with mock.patch.object(obj._client, 'fetch') as fetch:
+                fetch.side_effect = OSError()
+                body = json.dumps({'foo': 'bar'})
+                with self.assertRaises(exceptions.AWSClientException):
+                    obj.fetch('POST', '/', body=body)
+
     def test_fetch_no_headers(self):
         with self.client_with_default_creds('s3') as obj:
             with mock.patch.object(obj._client, 'fetch') as fetch:
@@ -620,6 +628,17 @@ class AsyncClientFetchTestCase(MockTestCase, utils.AsyncHTTPTestCase):
                         with self.assertRaises(exceptions.AWSError):
                             yield obj.fetch('GET', '/api')
+    @testing.gen_test
+    def test_fetch_os_error(self):
+        with self.client_with_default_creds(
+                's3', endpoint=self.get_url('/api')) as obj:
+            with mock.patch.object(obj._client, 'fetch') as fetch:
+                future = concurrent.Future()
+                future.set_exception(OSError)
+                fetch.return_value = future
+                with self.assertRaises(exceptions.AWSClientException):
+                    yield obj.fetch('GET', '/api')
+
 class NoCurlAsyncTestCase(unittest.TestCase):

tests/integration_tests.py

 import copy
 import json
+import logging
 import os
 import unittest
 import uuid
-from tornado import httpclient, testing
+from tornado import gen, httpclient, testing
 try:
     from tornado import curl_httpclient
@@ -15,6 +16,14 @@ import tornado_aws
 from . import utils
+LOGGER = logging.getLogger(__name__)
+
+CREATE_BUCKET_BODY = """\
+<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <LocationConstraint>test</LocationConstraint>
+</CreateBucketConfiguration>
+"""
+
 class DynamoDBTestCase(testing.AsyncTestCase):
@@ -121,3 +130,45 @@ class UseCurlDynamoDBTestCase(DynamoDBTestCase):
                 headers={
                     'x-amz-target': 'DynamoDB_20120810.CreateTable',
                     'Content-Type': 'application/x-amz-json-1.0'})
+
+
+class S3TestCase(testing.AsyncTestCase):
+
+    def setUp(self):
+        super(S3TestCase, self).setUp()
+        utils.clear_environment()
+        os.environ['AWS_ACCESS_KEY_ID'] = str(uuid.uuid4())
+        os.environ['AWS_SECRET_ACCESS_KEY'] = str(uuid.uuid4())
+        os.environ['AWS_DEFAULT_REGION'] = 'local'
+        self.client = tornado_aws.AsyncAWSClient(
+            's3', endpoint='http://localhost:4567')
+        self.bucket = uuid.uuid4().hex
+        self.headers = {'Host': '{}.s3.amazonaws.com'.format(self.bucket)}
+
+    def get(self, key):
+        LOGGER.debug('Getting object from s3://%s/%s', self.bucket, key)
+        return self.client.fetch('GET', '/{}'.format(key), headers=self.headers)
+
+    @gen.coroutine
+    def store(self, key, value):
+        LOGGER.debug('Storing revision to s3://%s/%s', self.bucket, key)
+        headers = dict(self.headers)
+        headers['x-amx-storage-class'] = 'STANDARD_IA'
+        headers['x-amz-server-side-encryption'] = 'AES256'
+        response = yield self.client.fetch(
+            'PUT', '/{}'.format(key), headers=headers, body=value)
+        return response.code == 200
+
+    @testing.gen_test
+    def test_store_and_get(self):
+        result = yield self.client.fetch(
+            'PUT', '/', self.headers, body=CREATE_BUCKET_BODY.encode('utf-8'))
+        record = {
+            'id': str(uuid.uuid4()),
+            'version': 2,
+            'data': str(uuid.uuid4())}
+        response = yield self.store(
+            record['id'], json.dumps(record).encode('utf-8'))
+        self.assertTrue(response)
+        response = yield self.get(record['id'])
+        self.assertDictEqual(record, json.loads(response.body.decode('utf-8')))

tornado_aws/client.py

@@ -11,6 +11,7 @@ import hmac
 import json
 import logging
 import os
+import socket
 try:
     from urllib.parse import urlparse
 except ImportError:  # pragma: nocover
@@ -168,9 +169,11 @@ class AWSClient(object):
         try:
             result = self._client.fetch(request, raise_error=True)
             return result
+        except (OSError, socket.error) as error:
+            LOGGER.error('Error making request: %s', error)
+            raise exceptions.AWSClientException()
         except httpclient.HTTPError as error:
             need_credentials, aws_error = self._process_error(error)
-            LOGGER.debug('err: %r, %r, %r', need_credentials, aws_error, self._auth_config.local_credentials)
             if need_credentials and not self._auth_config.local_credentials:
                 self._auth_config.reset()
                 if not recursed:
@@ -304,6 +307,7 @@ class AWSClient(object):
             headers = {}
         signed_headers, signed_url = self._signed_request(
             method, path, query_args or {}, dict(headers), body or b'')
+        LOGGER.debug('Signed URL: %s', signed_url)
         return httpclient.HTTPRequest(signed_url, method,
                                       signed_headers, body,
                                       connect_timeout=self.CONNECT_TIMEOUT,
@@ -580,20 +584,24 @@ class AsyncAWSClient(AWSClient):
         def on_response(response):
             exception = response.exception()
             if exception:
-                need_credentials, aws_error = self._process_error(exception)
-                if need_credentials and \
-                        not self._auth_config.local_credentials:
-                    self._auth_config.reset()
-                    if not recursed:
-                        def on_retry(retry):
-                            if not self._future_exception(retry, future):
-                                future.set_result(retry.result())
-
-                        request = self.fetch(method, path, query_args,
-                                             headers, body, True)
-                        self._ioloop.add_future(request, on_retry)
-                        return
-                future.set_exception(aws_error if aws_error else exception)
+                if isinstance(exception, httpclient.HTTPError):
+                    need_credentials, aws_error = self._process_error(exception)
+                    if need_credentials and \
+                            not self._auth_config.local_credentials:
+                        self._auth_config.reset()
+                        if not recursed:
+                            def on_retry(retry):
+                                if not self._future_exception(retry, future):
+                                    future.set_result(retry.result())
+
+                            request = self.fetch(method, path, query_args,
+                                                 headers, body, True)
+                            self._ioloop.add_future(request, on_retry)
+                            return
+                    future.set_exception(aws_error if aws_error else exception)
+                else:
+                    LOGGER.error('Error making request: %s', exception)
+                    future.set_exception(exceptions.AWSClientException())
             else:
                 future.set_result(response.result())