[SVCS-826] S3 Compatablilty

requirements.txt

@@ -1,6 +1,7 @@
 agent==0.1.2
 aiohttp==0.18.2
 git+https://github.com/felliott/boto.git@feature/gen-url-query-params-6#egg=boto
+boto3==1.7.36
 celery==3.1.17
 furl==0.4.2
 google-auth==1.4.1
@@ -15,7 +16,7 @@ raven==5.27.0
 setuptools==37.0.0
 stevedore==1.2.0
 tornado==4.3
-xmltodict==0.9.0
+xmltodict==0.11.0
 
 # Issue: certifi-2015.9.6.1 and 2015.9.6.2 fail verification (https://github.com/certifi/python-certifi/issues/26)
 certifi==2015.4.28

waterbutler/auth/osf/handler.py

@@ -40,6 +40,7 @@ def build_payload(self, bundle, view_only=None, cookie=None):
         return query_params
 
     async def make_request(self, params, headers, cookies):
+        data = None
         try:
             response = await aiohttp.request(
                 'get',

waterbutler/core/streams/http.py

@@ -179,6 +179,7 @@ def __init__(self, request, inner):
         super().__init__()
         self.inner = inner
         self.request = request
+        self.offset = 0
 
     @property
     def size(self):
@@ -187,12 +188,16 @@ def size(self):
     def at_eof(self):
         return self.inner.at_eof()
 
+    def tell(self):
+        return self.offset
+
     async def _read(self, size):
         if self.inner.at_eof():
             return b''
         if size < 0:
             return (await self.inner.read(size))
         try:
+            self.offset += size
             return (await self.inner.readexactly(size))
         except asyncio.IncompleteReadError as e:
             return e.partial

waterbutler/core/utils.py

@@ -105,15 +105,17 @@ async def send_signed_request(method, url, payload):
     ))
 
 
-def normalize_datetime(date_string):
-    if date_string is None:
+def normalize_datetime(date):
+    if date is None:
         return None
-    parsed_datetime = dateutil.parser.parse(date_string)
-    if not parsed_datetime.tzinfo:
-        parsed_datetime = parsed_datetime.replace(tzinfo=pytz.UTC)
-    parsed_datetime = parsed_datetime.astimezone(tz=pytz.UTC)
-    parsed_datetime = parsed_datetime.replace(microsecond=0)
-    return parsed_datetime.isoformat()
+    if isinstance(date, str):
+        date = dateutil.parser.parse(date)
+    if not date.tzinfo:
+        date = date.replace(tzinfo=pytz.UTC)
+    date = date.astimezone(tz=pytz.UTC)
+    date = date.replace(microsecond=0)
+    return date.isoformat()
+
 
 
 class ZipStreamGenerator:

waterbutler/providers/s3/metadata.py

@@ -1,6 +1,7 @@
 import os
 
 from waterbutler.core import metadata
+from waterbutler.core import utils
 
 
 class S3Metadata(metadata.BaseMetadata):
@@ -16,11 +17,13 @@ def name(self):
 
 class S3FileMetadataHeaders(S3Metadata, metadata.BaseFileMetadata):
 
-    def __init__(self, path, headers):
+    def __init__(self, path, s3_object=None, headers=None):
         self._path = path
+        self.obj = headers
+        self._etag = None
         # Cast to dict to clone as the headers will
         # be destroyed when the request leaves scope
-        super().__init__(dict(headers))
+        super().__init__(headers)
 
     @property
     def path(self):
@@ -44,16 +47,17 @@ def created_utc(self):
 
     @property
     def etag(self):
-        return self.raw['ETAG'].replace('"', '')
+        if self._etag is None:
+            self._etag = self.raw['ETAG'].replace('"', '')
+        return self._etag
 
     @property
     def extra(self):
-        md5 = self.raw['ETAG'].replace('"', '')
         return {
-            'md5': md5,
+            'md5': self.etag,
             'encryption': self.raw.get('X-AMZ-SERVER-SIDE-ENCRYPTION', ''),
             'hashes': {
-                'md5': md5,
+                'md5': self.etag,
             },
         }
 
@@ -132,7 +136,7 @@ def version(self):
 
     @property
     def modified(self):
-        return self.raw['LastModified']
+        return utils.normalize_datetime(self.raw['LastModified'])
 
     @property
     def extra(self):