Merge pull request #32 from maxmind/greg/object-input

 No longer allow ipaddress objects in pure Python implementation

Author: andyjack

.travis-yapf.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+diff=$(yapf -rd maxminddb tests)
+
+if [[ $? != 0 ]]; then
+    echo "yapf failed to run."
+    echo "$diff"
+    exit $?
+elif [[ $diff ]]; then
+    echo "$diff"
+    exit 1
+else
+    exit 0
+fi

.travis.yml

@@ -5,6 +5,8 @@ python:
   - 2.7
   - 3.3
   - 3.4
+  - 3.5
+  - 3.6
   - pypy
 
 before_install:
@@ -17,13 +19,15 @@ before_install:
   - sudo make install
   - sudo ldconfig
   - cd ..
-  - pip install pylint coveralls
+  - pip install coverage coveralls
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pip install pylint yapf; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install unittest2; fi
 
 script:
   - if [[ $TRAVIS_PYTHON_VERSION != 'pypy' ]]; then export MM_FORCE_EXT_TESTS=1; fi
-  - CFLAGS="-Werror -Wall -Wextra" python setup.py test
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pylint --rcfile .pylintrc maxminddb/*.py; fi
+  - CFLAGS="-Werror -Wall -Wextra" coverage run --source maxminddb setup.py test
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pylint --rcfile .pylintrc maxminddb/*.py; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then ./.travis-yapf.sh; fi
 
 after_success:
   - coveralls

HISTORY.rst

@@ -3,6 +3,20 @@
 History
 -------
 
+1.4.0
+++++++++++++++++++
+
+* IMPORTANT: Previously, the pure Python reader would allow
+  `ipaddress.IPv4Address` and `ipaddress.IPv6Address` objects when calling
+  `.get()`. This would fail with the C extension. The fact that these objects
+  worked at all was an implementation detail and has varied with different
+  releases. This release makes the pure Python implementation consistent
+  with the extension. A `TypeError` will now be thrown if you attempt to
+  use these types with either the pure Python implementation or the
+  extension. The IP address passed to `.get()` should be a string type.
+* Fix issue where incorrect size was used when unpacking some types with the
+  pure Python reader. Reported by Lee Symes. GitHub #30.
+
 1.3.0 (2017-03-13)
 ++++++++++++++++++
 

docs/conf.py

@@ -18,7 +18,6 @@
 sys.path.insert(0, os.path.abspath('..'))
 import maxminddb
 
-
 __version__ = maxminddb.__version__
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -33,8 +32,10 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest',
-              'sphinx.ext.intersphinx', 'sphinx.ext.coverage']
+extensions = [
+    'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx',
+    'sphinx.ext.coverage'
+]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -95,7 +96,6 @@
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []
 
-
 # -- Options for HTML output ---------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -175,7 +175,6 @@
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'maxminddbdoc'
 
-
 # -- Options for LaTeX output --------------------------------------------
 
 latex_elements = {
@@ -192,8 +191,8 @@
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-    ('index', 'maxminddb.tex', 'maxminddb Documentation',
-     'Gregory Oschwald', 'manual'),
+    ('index', 'maxminddb.tex', 'maxminddb Documentation', 'Gregory Oschwald',
+     'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -216,29 +215,24 @@
 # If false, no module index is generated.
 #latex_domain_indices = True
 
-
 # -- Options for manual page output --------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'maxminddb', 'maxminddb Documentation',
-     ['Gregory Oschwald'], 1)
-]
+man_pages = [('index', 'maxminddb', 'maxminddb Documentation',
+              ['Gregory Oschwald'], 1)]
 
 # If true, show URL addresses after external links.
 #man_show_urls = False
 
-
 # -- Options for Texinfo output ------------------------------------------
 
 # Grouping the document tree into Texinfo files. List of tuples
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    ('index', 'maxminddb', 'maxminddb Documentation',
-     'Gregory Oschwald', 'maxminddb', 'MaxMind DB Reader',
-     'Miscellaneous'),
+    ('index', 'maxminddb', 'maxminddb Documentation', 'Gregory Oschwald',
+     'maxminddb', 'MaxMind DB Reader', 'Miscellaneous'),
 ]
 
 # Documents to append as an appendix to all manuals.
@@ -250,6 +244,5 @@
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'
 
-
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'http://docs.python.org/': None}

examples/benchmark.py

@@ -11,12 +11,11 @@
 import timeit
 
 parser = argparse.ArgumentParser(description='Benchmark maxminddb.')
-parser.add_argument('--count', default=250000, type=int,
-                    help='number of lookups')
-parser.add_argument('--mode', default=0, type=int,
-                    help='reader mode to use')
-parser.add_argument('--file', default='GeoIP2-City.mmdb',
-                    help='path to mmdb file')
+parser.add_argument(
+    '--count', default=250000, type=int, help='number of lookups')
+parser.add_argument('--mode', default=0, type=int, help='reader mode to use')
+parser.add_argument(
+    '--file', default='GeoIP2-City.mmdb', help='path to mmdb file')
 
 args = parser.parse_args()
 
@@ -28,8 +27,9 @@ def lookup_ip_address():
     record = reader.get(str(ip))
 
 
-elapsed = timeit.timeit('lookup_ip_address()',
-                        setup='from __main__ import lookup_ip_address',
-                        number=args.count)
+elapsed = timeit.timeit(
+    'lookup_ip_address()',
+    setup='from __main__ import lookup_ip_address',
+    number=args.count)
 
 print(args.count / elapsed, 'lookups per second')

maxminddb/__init__.py

@@ -27,11 +27,13 @@ def open_database(database, mode=MODE_AUTO):
             * MODE_AUTO - tries MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that
                           order. Default mode.
     """
-    has_extension = maxminddb.extension and hasattr(maxminddb.extension, 'Reader')
+    has_extension = maxminddb.extension and hasattr(maxminddb.extension,
+                                                    'Reader')
     if (mode == MODE_AUTO and has_extension) or mode == MODE_MMAP_EXT:
         if not has_extension:
             raise ValueError(
-                "MODE_MMAP_EXT requires the maxminddb.extension module to be available")
+                "MODE_MMAP_EXT requires the maxminddb.extension module to be available"
+            )
         return maxminddb.extension.Reader(database)
     elif mode in (MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY):
         return maxminddb.reader.Reader(database, mode)

maxminddb/compat.py

@@ -21,6 +21,10 @@ def int_from_bytes(b):
         return 0
 
     byte_from_int = chr
+
+    string_type = basestring
+
+    string_type_name = 'string'
 else:
 
     def compat_ip_address(address):
@@ -33,3 +37,7 @@ def compat_ip_address(address):
     int_from_bytes = lambda x: int.from_bytes(x, 'big')
 
     byte_from_int = lambda x: bytes([x])
+
+    string_type = str
+
+    string_type_name = string_type.__name__

maxminddb/decoder.py

@@ -50,7 +50,7 @@ def _decode_packed_type(type_code, type_size, pad=False):
         def unpack_type(self, size, offset):
             if not pad:
                 self._verify_size(size, type_size)
-            new_offset = offset + type_size
+            new_offset = offset + size
             packed_bytes = self._buffer[offset:new_offset]
             if pad:
                 packed_bytes = packed_bytes.rjust(type_size, b'\x00')
@@ -105,8 +105,7 @@ def _decode_utf8_string(self, size, offset):
         5: _decode_uint,  # uint16
         6: _decode_uint,  # uint32
         7: _decode_map,
-        8: _decode_packed_type(
-            b'!i', 4, pad=True),  # int32
+        8: _decode_packed_type(b'!i', 4, pad=True),  # int32
         9: _decode_uint,  # uint64
         10: _decode_uint,  # uint128
         11: _decode_array,

maxminddb/reader.py

@@ -15,7 +15,8 @@
 
 import struct
 
-from maxminddb.compat import byte_from_int, compat_ip_address
+from maxminddb.compat import (byte_from_int, compat_ip_address, string_type,
+                              string_type_name)
 from maxminddb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY
 from maxminddb.decoder import Decoder
 from maxminddb.errors import InvalidDatabaseError
@@ -93,6 +94,9 @@ def get(self, ip_address):
         Arguments:
         ip_address -- an IP address in the standard string notation
         """
+        if not isinstance(ip_address, string_type):
+            raise TypeError('argument 1 must be %s, not %s' %
+                            (string_type_name, type(ip_address).__name__))
 
         address = compat_ip_address(ip_address)
 
@@ -155,14 +159,13 @@ def _read_node(self, node_number, index):
             else:
                 middle = (0xF0 & middle) >> 4
             offset = base_offset + index * 4
-            node_bytes = byte_from_int(middle) + self._buffer[offset:offset +
-                                                              3]
+            node_bytes = byte_from_int(middle) + self._buffer[offset:offset + 3]
         elif record_size == 32:
             offset = base_offset + index * 4
             node_bytes = self._buffer[offset:offset + 4]
         else:
-            raise InvalidDatabaseError('Unknown record size: {0}'.format(
-                record_size))
+            raise InvalidDatabaseError(
+                'Unknown record size: {0}'.format(record_size))
         return struct.unpack(b'!I', node_bytes)[0]
 
     def _resolve_data_pointer(self, pointer):

setup.py

@@ -16,8 +16,8 @@
 JYTHON = sys.platform.startswith('java')
 requirements = []
 
-if sys.version_info[0] == 2 or (sys.version_info[0] == 3 and
-                                sys.version_info[1] < 3):
+if sys.version_info[0] == 2 or (sys.version_info[0] == 3
+                                and sys.version_info[1] < 3):
     requirements.append('ipaddress')
 
 compile_args = ['-Wall', '-Wextra']
@@ -30,7 +30,8 @@
         'maxminddb.extension',
         libraries=['maxminddb'],
         sources=['maxminddb/extension/maxminddb.c'],
-        extra_compile_args=compile_args, )
+        extra_compile_args=compile_args,
+    )
 ]
 
 # Cargo cult code for installing extension with pure Python fallback.
@@ -95,8 +96,8 @@ def status_msgs(*msgs):
 def find_packages(location):
     packages = []
     for pkg in ['maxminddb']:
-        for _dir, subdirectories, files in (
-                os.walk(os.path.join(location, pkg))):
+        for _dir, subdirectories, files in (os.walk(
+                os.path.join(location, pkg))):
             if '__init__.py' in files:
                 tokens = _dir.split(os.sep)[len(location.split(os.sep)):]
                 packages.append(".".join(tokens))
@@ -108,7 +109,8 @@ def run_setup(with_cext):
     if with_cext:
         if Feature:
             kwargs['features'] = {
-                'extension': Feature(
+                'extension':
+                Feature(
                     "optional C implementation",
                     standard=True,
                     ext_modules=ext_module)