Merge pull request #26 from maxmind/greg/packed-bytearray-fix

Fix incompatibility with py2-ipaddress 

Author: rafl

HISTORY.rst

@@ -3,6 +3,14 @@
 History
 -------
 
+1.2.3
+++++++++++++++++++
+
+* Improve compatibility with other Python 2 ``ipaddress`` backports. Although
+  ```ipaddress`` <https://pypi.python.org/pypi/ipaddress/1.0.18>`_ is highly
+  recommended, ``py2-ipaddress`` should now work. Incompatibility reported by
+  John Zadroga. ``geoip2`` GitHub issue #41.
+
 1.2.2 (2016-11-21)
 ++++++++++++++++++
 

maxminddb/__init__.py

@@ -39,6 +39,7 @@ def Reader(database):  # pylint: disable=invalid-name
     """This exists for backwards compatibility. Use open_database instead"""
     return open_database(database)
 
+
 __title__ = 'maxminddb'
 __version__ = '1.2.2'
 __author__ = 'Gregory Oschwald'

maxminddb/compat.py

@@ -5,6 +5,7 @@
 # pylint: skip-file
 
 if sys.version_info[0] == 2:
+
     def compat_ip_address(address):
         if isinstance(address, bytes):
             address = address.decode()
@@ -21,6 +22,7 @@ def int_from_bytes(b):
 
     byte_from_int = chr
 else:
+
     def compat_ip_address(address):
         return ipaddress.ip_address(address)
 

maxminddb/decoder.py

@@ -14,7 +14,6 @@
 
 
 class Decoder(object):  # pylint: disable=too-few-public-methods
-
     """Decoder for the data section of the MaxMind DB"""
 
     def __init__(self, database_buffer, pointer_base=0, pointer_test=False):
@@ -55,8 +54,9 @@ def unpack_type(self, size, offset):
             packed_bytes = self._buffer[offset:new_offset]
             if pad:
                 packed_bytes = packed_bytes.rjust(type_size, b'\x00')
-            (value,) = struct.unpack(type_code, packed_bytes)
+            (value, ) = struct.unpack(type_code, packed_bytes)
             return value, new_offset
+
         return unpack_type
 
     def _decode_map(self, size, offset):
@@ -105,7 +105,8 @@ def _decode_utf8_string(self, size, offset):
         5: _decode_uint,  # uint16
         6: _decode_uint,  # uint32
         7: _decode_map,
-        8: _decode_packed_type(b'!i', 4, pad=True),  # int32
+        8: _decode_packed_type(
+            b'!i', 4, pad=True),  # int32
         9: _decode_uint,  # uint64
         10: _decode_uint,  # uint128
         11: _decode_array,
@@ -120,7 +121,7 @@ def decode(self, offset):
         offset -- the location of the data structure to decode
         """
         new_offset = offset + 1
-        (ctrl_byte,) = struct.unpack(b'!B', self._buffer[offset:new_offset])
+        (ctrl_byte, ) = struct.unpack(b'!B', self._buffer[offset:new_offset])
         type_num = ctrl_byte >> 5
         # Extended type
         if not type_num:
@@ -130,12 +131,12 @@ def decode(self, offset):
             raise InvalidDatabaseError('Unexpected type number ({type}) '
                                        'encountered'.format(type=type_num))
 
-        (size, new_offset) = self._size_from_ctrl_byte(
-            ctrl_byte, new_offset, type_num)
+        (size, new_offset) = self._size_from_ctrl_byte(ctrl_byte, new_offset,
+                                                       type_num)
         return self._type_decoder[type_num](self, size, new_offset)
 
     def _read_extended(self, offset):
-        (next_byte,) = struct.unpack(b'!B', self._buffer[offset:offset + 1])
+        (next_byte, ) = struct.unpack(b'!B', self._buffer[offset:offset + 1])
         type_num = next_byte + 7
         if type_num < 7:
             raise InvalidDatabaseError(
@@ -148,8 +149,7 @@ def _verify_size(self, expected, actual):
         if expected != actual:
             raise InvalidDatabaseError(
                 'The MaxMind DB file\'s data section contains bad data '
-                '(unknown data type or corrupt data)'
-            )
+                '(unknown data type or corrupt data)')
 
     def _size_from_ctrl_byte(self, ctrl_byte, offset, type_num):
         size = ctrl_byte & 0x1f
@@ -167,7 +167,7 @@ def _size_from_ctrl_byte(self, ctrl_byte, offset, type_num):
         elif size == 30:
             size = 285 + struct.unpack(b'!H', size_bytes)[0]
         elif size > 30:
-            size = struct.unpack(
-                b'!I', size_bytes.rjust(4, b'\x00'))[0] + 65821
+            size = struct.unpack(b'!I', size_bytes.rjust(4,
+                                                         b'\x00'))[0] + 65821
 
         return size, new_offset

maxminddb/errors.py

@@ -7,5 +7,4 @@
 
 
 class InvalidDatabaseError(RuntimeError):
-
     """This error is thrown when unexpected data is found in the database."""

maxminddb/file.py

@@ -10,7 +10,6 @@
 
 
 class FileBuffer(object):
-
     """A slice-able file reader"""
 
     def __init__(self, database):

maxminddb/reader.py

@@ -15,15 +15,14 @@
 
 import struct
 
-from maxminddb.compat import byte_from_int, int_from_byte, compat_ip_address
+from maxminddb.compat import byte_from_int, compat_ip_address
 from maxminddb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY
 from maxminddb.decoder import Decoder
 from maxminddb.errors import InvalidDatabaseError
 from maxminddb.file import FileBuffer
 
 
 class Reader(object):
-
     """
     Instances of this class provide a reader for the MaxMind DB format. IP
     addresses can be looked up using the ``get`` method.
@@ -60,13 +59,14 @@ def __init__(self, database, mode=MODE_AUTO):
                 self._buffer = db_file.read()
                 self._buffer_size = len(self._buffer)
         else:
-            raise ValueError('Unsupported open mode ({0}). Only MODE_AUTO, '
-                             ' MODE_FILE, and MODE_MEMORY are support by the pure Python '
-                             'Reader'.format(mode))
+            raise ValueError(
+                'Unsupported open mode ({0}). Only MODE_AUTO, '
+                ' MODE_FILE, and MODE_MEMORY are support by the pure Python '
+                'Reader'.format(mode))
 
-        metadata_start = self._buffer.rfind(self._METADATA_START_MARKER,
-                                            max(0, self._buffer_size
-                                                - 128 * 1024))
+        metadata_start = self._buffer.rfind(
+            self._METADATA_START_MARKER, max(0,
+                                             self._buffer_size - 128 * 1024))
 
         if metadata_start == -1:
             self.close()
@@ -77,11 +77,10 @@ def __init__(self, database, mode=MODE_AUTO):
         metadata_start += len(self._METADATA_START_MARKER)
         metadata_decoder = Decoder(self._buffer, metadata_start)
         (metadata, _) = metadata_decoder.decode(metadata_start)
-        self._metadata = Metadata(
-            **metadata)  # pylint: disable=bad-option-value
+        self._metadata = Metadata(**metadata)  # pylint: disable=bad-option-value
 
-        self._decoder = Decoder(self._buffer, self._metadata.search_tree_size
-                                + self._DATA_SECTION_SEPARATOR_SIZE)
+        self._decoder = Decoder(self._buffer, self._metadata.search_tree_size +
+                                self._DATA_SECTION_SEPARATOR_SIZE)
 
     def metadata(self):
         """Return the metadata associated with the MaxMind DB file"""
@@ -98,23 +97,23 @@ def get(self, ip_address):
         address = compat_ip_address(ip_address)
 
         if address.version == 6 and self._metadata.ip_version == 4:
-            raise ValueError('Error looking up {0}. You attempted to look up '
-                             'an IPv6 address in an IPv4-only database.'.format(
-                                 ip_address))
+            raise ValueError(
+                'Error looking up {0}. You attempted to look up '
+                'an IPv6 address in an IPv4-only database.'.format(ip_address))
         pointer = self._find_address_in_tree(address)
 
         return self._resolve_data_pointer(pointer) if pointer else None
 
     def _find_address_in_tree(self, ip_address):
-        packed = ip_address.packed
+        packed = bytearray(ip_address.packed)
 
         bit_count = len(packed) * 8
         node = self._start_node(bit_count)
 
         for i in range(bit_count):
             if node >= self._metadata.node_count:
                 break
-            bit = 1 & (int_from_byte(packed[i >> 3]) >> 7 - (i % 8))
+            bit = 1 & (packed[i >> 3] >> 7 - (i % 8))
             node = self._read_node(node, bit)
         if node == self._metadata.node_count:
             # Record is empty
@@ -149,21 +148,21 @@ def _read_node(self, node_number, index):
             offset = base_offset + index * 3
             node_bytes = b'\x00' + self._buffer[offset:offset + 3]
         elif record_size == 28:
-            (middle,) = struct.unpack(
+            (middle, ) = struct.unpack(
                 b'!B', self._buffer[base_offset + 3:base_offset + 4])
             if index:
                 middle &= 0x0F
             else:
                 middle = (0xF0 & middle) >> 4
             offset = base_offset + index * 4
-            node_bytes = byte_from_int(
-                middle) + self._buffer[offset:offset + 3]
+            node_bytes = byte_from_int(middle) + self._buffer[offset:offset +
+                                                              3]
         elif record_size == 32:
             offset = base_offset + index * 4
             node_bytes = self._buffer[offset:offset + 4]
         else:
-            raise InvalidDatabaseError(
-                'Unknown record size: {0}'.format(record_size))
+            raise InvalidDatabaseError('Unknown record size: {0}'.format(
+                record_size))
         return struct.unpack(b'!I', node_bytes)[0]
 
     def _resolve_data_pointer(self, pointer):
@@ -185,7 +184,6 @@ def close(self):
 
 
 class Metadata(object):
-
     """Metadata for the MaxMind DB reader"""
 
     # pylint: disable=too-many-instance-attributes