From dd1aeb4ef383de65408da1085ce4d1ee6e2da03c Mon Sep 17 00:00:00 2001 From: Dongmin Jang Date: Tue, 28 Apr 2015 17:16:11 +0900 Subject: [PATCH 1/2] Add original_prefix and reproducible_prefix option to store classes --- sqlalchemy_imageattach/context.py | 22 ++++++++++------ sqlalchemy_imageattach/store.py | 38 ++++++++++++++++++++++------ sqlalchemy_imageattach/stores/fs.py | 33 ++++++++++++++++-------- sqlalchemy_imageattach/stores/s3.py | 39 ++++++++++++++++++++++------- 4 files changed, 97 insertions(+), 35 deletions(-) diff --git a/sqlalchemy_imageattach/context.py b/sqlalchemy_imageattach/context.py index 5b656e0..267cb72 100644 --- a/sqlalchemy_imageattach/context.py +++ b/sqlalchemy_imageattach/context.py @@ -195,25 +195,31 @@ def __init__(self, get_current_object, repr_string=None): self.repr_string = repr_string def put_file(self, file, object_type, object_id, width, height, - mimetype, reproducible): + mimetype, reproducible=False): self.get_current_object().put_file( file, object_type, object_id, width, height, - mimetype, reproducible + mimetype, reproducible=reproducible ) - def delete_file(self, object_type, object_id, width, height, mimetype): + def delete_file(self, object_type, object_id, width, height, + mimetype, reproducible=False): self.get_current_object().delete_file( - object_type, object_id, width, height, mimetype + object_type, object_id, width, height, + mimetype, reproducible=reproducible ) - def get_file(self, object_type, object_id, width, height, mimetype): + def get_file(self, object_type, object_id, width, height, + mimetype, reproducible=False): return self.get_current_object().get_file( - object_type, object_id, width, height, mimetype + object_type, object_id, width, height, + mimetype, reproducible=reproducible ) - def get_url(self, object_type, object_id, width, height, mimetype): + def get_url(self, object_type, object_id, width, height, + mimetype, reproducible=False): return self.get_current_object().get_url( - object_type, object_id, width, height, mimetype + object_type, object_id, width, height, + mimetype, reproducible=reproducible ) def __eq__(self, other): diff --git a/sqlalchemy_imageattach/store.py b/sqlalchemy_imageattach/store.py index 6e02565..f75a24b 100644 --- a/sqlalchemy_imageattach/store.py +++ b/sqlalchemy_imageattach/store.py @@ -27,7 +27,7 @@ class Store(object): """ def put_file(self, file, object_type, object_id, width, height, mimetype, - reproducible): + reproducible=False): """Puts the ``file`` of the image. :param file: the image file to put @@ -48,6 +48,7 @@ def put_file(self, file, object_type, object_id, width, height, mimetype, computing e.g. resized thumbnails. ``False`` if it cannot be reproduced e.g. original images + default is ``False`` :type reproducible: :class:`bool` .. note:: @@ -61,7 +62,8 @@ def put_file(self, file, object_type, object_id, width, height, mimetype, """ raise NotImplementedError('put_file() has to be implemented') - def delete_file(self, object_type, object_id, width, height, mimetype): + def delete_file(self, object_type, object_id, width, height, mimetype, + reproducible=False): """Deletes all reproducible files related to the image. It doesn't raise any exception even if there's no such file. @@ -77,11 +79,18 @@ def delete_file(self, object_type, object_id, width, height, mimetype): :param mimetype: the mimetype of the image to delete e.g. ``'image/jpeg'`` :type mimetype: :class:`basestring` + :param reproducible: ``True`` only if it's reproducible by + computing e.g. resized thumbnails. + ``False`` if it cannot be reproduced + e.g. original images + default is ``False`` + :type reproducible: :class:`bool` """ raise NotImplementedError('delete_file() has to be implemented') - def get_file(self, object_type, object_id, width, height, mimetype): + def get_file(self, object_type, object_id, width, height, mimetype, + reproducible=False): """Gets the file-like object of the given criteria. :param object_type: the object type of the image to find @@ -96,6 +105,12 @@ def get_file(self, object_type, object_id, width, height, mimetype): :param mimetype: the mimetype of the image to find e.g. ``'image/jpeg'`` :type mimetype: :class:`basestring` + :param reproducible: ``True`` only if it's reproducible by + computing e.g. resized thumbnails. + ``False`` if it cannot be reproduced + e.g. original images + default is ``False`` + :type reproducible: :class:`bool` :returns: the file of the image :rtype: file-like object, :class:`file` :raises exceptions.IOError: when such file doesn't exist @@ -111,7 +126,8 @@ def get_file(self, object_type, object_id, width, height, mimetype): """ raise NotImplementedError('get_file() has to be implemented') - def get_url(self, object_type, object_id, width, height, mimetype): + def get_url(self, object_type, object_id, width, height, mimetype, + reproducible=False): """Gets the file-like object of the given criteria. :param object_type: the object type of the image to find @@ -126,6 +142,12 @@ def get_url(self, object_type, object_id, width, height, mimetype): :param mimetype: the mimetype of the image to find e.g. ``'image/jpeg'`` :type mimetype: :class:`basestring` + :param reproducible: ``True`` only if it's reproducible by + computing e.g. resized thumbnails. + ``False`` if it cannot be reproduced + e.g. original images + default is ``False`` + :type reproducible: :class:`bool` :returns: the url locating the image :rtype: :class:`basestring` @@ -162,7 +184,7 @@ def store(self, image, file): 'implements read() method, not ' + repr(file)) self.put_file(file, image.object_type, image.object_id, image.width, image.height, image.mimetype, - not image.original) + reproducible=not image.original) def delete(self, image): """Delete the file of the given ``image``. @@ -176,7 +198,8 @@ def delete(self, image): raise TypeError('image must be a sqlalchemy_imageattach.entity.' 'Image instance, not ' + repr(image)) self.delete_file(image.object_type, image.object_id, - image.width, image.height, image.mimetype) + image.width, image.height, image.mimetype, + reproducible=not image.original) def open(self, image, use_seek=False): """Opens the file-like object of the given ``image``. @@ -267,7 +290,8 @@ def locate(self, image): raise TypeError('image must be a sqlalchemy_imageattach.entity.' 'Image instance, not ' + repr(image)) url = self.get_url(image.object_type, image.object_id, - image.width, image.height, image.mimetype) + image.width, image.height, image.mimetype, + reproducible=not image.original) if '?' in url: fmt = '{0}&_ts={1}' else: diff --git a/sqlalchemy_imageattach/stores/fs.py b/sqlalchemy_imageattach/stores/fs.py index 5b6178f..d39b850 100644 --- a/sqlalchemy_imageattach/stores/fs.py +++ b/sqlalchemy_imageattach/stores/fs.py @@ -56,19 +56,25 @@ class BaseFileSystemStore(Store): """ - def __init__(self, path): + def __init__(self, path, original_prefix='', reproducible_prefix=''): self.path = path - - def get_path(self, object_type, object_id, width, height, mimetype): + if original_prefix.endswith('/'): + original_prefix = original_prefix.rstrip('/') + self.original_prefix = original_prefix + if reproducible_prefix.endswith('/'): + reproducible_prefix = reproducible_prefix.rstrip('/') + self.reproducible_prefix = reproducible_prefix + + def get_path(self, object_type, object_id, width, height, mimetype, reproducible=False): id_segment_a = str(object_id % 1000) id_segment_b = str(object_id // 1000) suffix = guess_extension(mimetype) filename = '{0}.{1}x{2}{3}'.format(object_id, width, height, suffix) - return object_type, id_segment_a, id_segment_b, filename + prefix = self.reproducible_prefix if reproducible else self.original_prefix + return prefix, id_segment_a, id_segment_b, filename - def put_file(self, file, object_type, object_id, width, height, mimetype, - reproducible): - path = self.get_path(object_type, object_id, width, height, mimetype) + def put_file(self, file, object_type, object_id, width, height, mimetype, reproducible=False): + path = self.get_path(object_type, object_id, width, height, mimetype, reproducible=reproducible) for i in range(len(path)): d = os.path.join(self.path, *path[:i]) if not os.path.isdir(d): @@ -104,8 +110,10 @@ class FileSystemStore(BaseFileSystemStore): """ - def __init__(self, path, base_url): - super(FileSystemStore, self).__init__(path) + def __init__(self, path, base_url, original_prefix='', reproducible_prefix=''): + super(FileSystemStore, self).__init__(path, + original_prefix=original_prefix, + reproducible_prefix=reproducible_prefix) if not base_url.endswith('/'): base_url += '/' self.base_url = base_url @@ -174,10 +182,13 @@ class HttpExposedFileSystemStore(BaseFileSystemStore): """ - def __init__(self, path, prefix='__images__', host_url_getter=None): + def __init__(self, path, prefix='__images__', host_url_getter=None, + original_prefix='', reproducible_prefix=''): if not (callable(host_url_getter) or host_url_getter is None): raise TypeError('host_url_getter must be callable') - super(HttpExposedFileSystemStore, self).__init__(path) + super(HttpExposedFileSystemStore, self).__init__(path, + original_prefix=original_prefix, + reproducible_prefix=reproducible_prefix) if prefix.startswith('/'): prefix = prefix[1:] if prefix.endswith('/'): diff --git a/sqlalchemy_imageattach/stores/s3.py b/sqlalchemy_imageattach/stores/s3.py index be529d4..be661f8 100644 --- a/sqlalchemy_imageattach/stores/s3.py +++ b/sqlalchemy_imageattach/stores/s3.py @@ -15,6 +15,7 @@ import hashlib import hmac import logging +import os.path try: from urllib import request as urllib2 except ImportError: @@ -140,6 +141,12 @@ class S3Store(Store): :param prefix: the optional key prefix to logically separate stores with the same bucket. not used by default :type prefix: :class:`basestring` + :param original_prefix: the optional key prefix to logically separate stores + with the same bucket. not used by default + :type original_prefix: :class:`basestring` + :param reproducible_prefix: the optional key prefix to logically separate stores + with the same bucket. not used by default + :type reproducible_prefix: :class:`basestring` :param public_base_url: an optional url base for public urls. useful when used with cdn :type public_base_url: :class:`basestring` @@ -164,6 +171,14 @@ class S3Store(Store): #: stores with the same bucket. prefix = None + #: (:class:`basestring`) The optional key prefix to logically separate + #: stores with the same bucket. + original_prefix = None + + #: (:class:`basestring`) The optional key prefix to logically separate + #: stores with the same bucket. + reproducible_prefix = None + #: (:class:`basestring`) The optional url base for public urls. public_base_url = None @@ -177,6 +192,12 @@ def __init__(self, bucket, access_key=None, secret_key=None, self.prefix = prefix.strip() if self.prefix.endswith('/'): self.prefix = self.prefix.rstrip('/') + self.original_prefix = original_prefix.strip() + if self.original_prefix.endswith('/'): + self.original_prefix = self.original_prefix.rstrip('/') + self.reproducible_prefix = reproducible_prefix.strip() + if self.reproducible_prefix.endswith('/'): + self.reproducible_prefix = self.reproducible_prefix.rstrip('/') if public_base_url is None: self.public_base_url = self.base_url elif public_base_url.endswith('/'): @@ -184,13 +205,14 @@ def __init__(self, bucket, access_key=None, secret_key=None, else: self.public_base_url = public_base_url - def get_key(self, object_type, object_id, width, height, mimetype): + def get_key(self, object_type, object_id, width, height, mimetype, reproducible=False): key = '{0}/{1}/{2}x{3}{4}'.format( object_type, object_id, width, height, guess_extension(mimetype) ) - if self.prefix: - return '{0}/{1}'.format(self.prefix, key) + prefix = os.path.join(self.prefix, self.reproducible_prefix if reproducible else self.original_prefix) + if prefix: + return '{0}/{1}'.format(prefix, key) return key def get_file(self, *args, **kwargs): @@ -246,9 +268,8 @@ def upload_file(self, url, data, content_type, rrs, acl='public-read'): else: break - def put_file(self, file, object_type, object_id, width, height, mimetype, - reproducible): - url = self.get_s3_url(object_type, object_id, width, height, mimetype) + def put_file(self, file, object_type, object_id, width, height, mimetype, reproducible=False): + url = self.get_s3_url(object_type, object_id, width, height, mimetype, reproducible=reproducible) self.upload_file(url, file.read(), mimetype, rrs=reproducible) def delete_file(self, *args, **kwargs): @@ -337,10 +358,10 @@ def get_url(self, *args, **kwargs): def put_file(self, *args, **kwargs): self.overriding.put_file(*args, **kwargs) - def delete_file(self, object_type, object_id, width, height, mimetype): + def delete_file(self, object_type, object_id, width, height, mimetype, reproducible=False): args = object_type, object_id, width, height, mimetype - self.overriding.delete_file(*args) - url = self.overriding.get_s3_url(*args) + self.overriding.delete_file(*args, reproducible=reproducible) + url = self.overriding.get_s3_url(*args, reproducible=reproducible) self.overriding.upload_file( url, data=b'', From cae32dbb593dc21c032698a17b29f8bf66b35033 Mon Sep 17 00:00:00 2001 From: Dongmin Jang Date: Tue, 28 Apr 2015 17:32:55 +0900 Subject: [PATCH 2/2] Add acl option to S3Store It's useful for serving private contents with AWS CloudFront. --- sqlalchemy_imageattach/stores/s3.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sqlalchemy_imageattach/stores/s3.py b/sqlalchemy_imageattach/stores/s3.py index be661f8..55407a0 100644 --- a/sqlalchemy_imageattach/stores/s3.py +++ b/sqlalchemy_imageattach/stores/s3.py @@ -138,6 +138,8 @@ class S3Store(Store): :param max_age: the ``max-age`` seconds of :mailheader:`Cache-Control`. default is :const:`DEFAULT_MAX_AGE` :type max_age: :class:`numbers.Integral` + :param acl: the ``acl`` option of uploaded key. default is :const:`public-read` + :type acl: :class:`basestring` :param prefix: the optional key prefix to logically separate stores with the same bucket. not used by default :type prefix: :class:`basestring` @@ -167,6 +169,9 @@ class S3Store(Store): #: :mailheader:`Cache-Control`. max_age = None + #: (:class:`basestring`) The ``acl`` option of uploaded key. + acl = None + #: (:class:`basestring`) The optional key prefix to logically separate #: stores with the same bucket. prefix = None @@ -182,13 +187,14 @@ class S3Store(Store): #: (:class:`basestring`) The optional url base for public urls. public_base_url = None - def __init__(self, bucket, access_key=None, secret_key=None, - max_age=DEFAULT_MAX_AGE, prefix='', public_base_url=None): + def __init__(self, bucket, access_key=None, secret_key=None, max_age=DEFAULT_MAX_AGE, acl='public-read', + prefix='', original_prefix='', reproducible_prefix='', public_base_url=None): self.bucket = bucket self.access_key = access_key self.secret_key = secret_key self.base_url = BASE_URL_FORMAT.format(bucket) self.max_age = max_age + self.acl = acl self.prefix = prefix.strip() if self.prefix.endswith('/'): self.prefix = self.prefix.rstrip('/') @@ -239,10 +245,10 @@ def make_request(self, url, *args, **kwargs): secret_key=self.secret_key, **kwargs) - def upload_file(self, url, data, content_type, rrs, acl='public-read'): + def upload_file(self, url, data, content_type, rrs): headers = { 'Cache-Control': 'max-age=' + str(self.max_age), - 'x-amz-acl': acl, + 'x-amz-acl': self.acl, 'x-amz-storage-class': 'REDUCED_REDUNDANCY' if rrs else 'STANDARD' } request = self.make_request(