Merge pull request #126 from maxmind/greg/windows-ext

Windows support for C extension

Author: horgh

.github/workflows/create-wheels.yml

@@ -8,9 +8,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        # The extension currently does not build on windows-latest.
-        # We have a follow-up story for that.
-        os: [ubuntu-latest, macos-latest]
+        os: [macos-latest, ubuntu-latest, windows-latest]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/test-libmaxminddb.yml

@@ -0,0 +1,57 @@
+name: Python tests (system libmaxminddb)
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '3 15 * * SUN'
+
+jobs:
+  build:
+
+    strategy:
+      matrix:
+        # We don't test on Windows currently due to issues
+        # build libmaxminddb there.
+        platform: [macos-latest, ubuntu-latest]
+        python-version: [3.8, 3.9, "3.10", 3.11, 3.12]
+
+    name: Python ${{ matrix.python-version }} on ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+    env:
+      MAXMINDDB_REQUIRE_EXTENSION: 1
+      MAXMINDDB_USE_SYSTEM_LIBMAXMINDDB: 1
+      MM_FORCE_EXT_TESTS: 1
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+              python -m pip install --upgrade pip
+              pip install setuptools tox tox-gh-actions wheel
+
+
+      - name: Install libmaxminddb
+        run: sudo apt install libmaxminddb-dev
+        if: matrix.platform == 'ubuntu-latest'
+
+      - name: Install libmaxminddb
+        run: brew install libmaxminddb
+        if: matrix.platform == 'macos-latest'
+
+      - name: Build with Werror and Wall
+        run: python setup.py build
+        env:
+          CFLAGS: "-Werror -Wall -Wextra"
+
+      - name: Test with tox (system libmaxminddb)
+        run: tox

.github/workflows/test.yml

@@ -11,9 +11,7 @@ jobs:
 
     strategy:
       matrix:
-        # We don't test on Windows currently due to issues
-        # build libmaxminddb there.
-        platform: [macos-latest, ubuntu-latest]
+        platform: [macos-latest, ubuntu-latest, windows-latest]
         python-version: [3.8, 3.9, "3.10", 3.11, 3.12]
 
     name: Python ${{ matrix.python-version }} on ${{ matrix.platform }}
@@ -33,31 +31,10 @@ jobs:
       - name: Install dependencies
         run: |
               python -m pip install --upgrade pip
-              pip install setuptools tox tox-gh-actions wheel
-
-      - name: Build with Werror and Wall
-        run: python setup.py build
-        env:
-          CFLAGS: "-Werror -Wall -Wextra"
-          MAXMINDDB_REQUIRE_EXTENSION: 1
+              pip install tox tox-gh-actions
 
       - name: Test with tox (vendored libmaxminddb)
         run: tox
         env:
           MAXMINDDB_REQUIRE_EXTENSION: 1
           MM_FORCE_EXT_TESTS: 1
-
-      - name: Install libmaxminddb
-        run: sudo apt install libmaxminddb-dev
-        if: matrix.platform == 'ubuntu-latest'
-
-      - name: Install libmaxminddb
-        run: brew install libmaxminddb
-        if: matrix.platform == 'macos-latest'
-
-      - name: Test with tox (system libmaxminddb)
-        run: tox
-        env:
-          MAXMINDDB_REQUIRE_EXTENSION: 1
-          MAXMINDDB_USE_SYSTEM_LIBMAXMINDDB: 1
-          MM_FORCE_EXT_TESTS: 1

HISTORY.rst

@@ -8,6 +8,7 @@ History
 
 * IMPORTANT: Python 3.8 or greater is required. If you are using an older
   version, please use an earlier release.
+* Windows is now supported by the C extension.
 * The ``Reader`` class now implements the ``__iter__`` method. This will
   return an iterator that iterates over all records in the database,
   excluding repeated aliased of the IPv4 network. Requested by

extension/maxminddb.c

@@ -1,11 +1,17 @@
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
+#ifdef MS_WINDOWS
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#else
 #include <arpa/inet.h>
-#include <maxminddb.h>
 #include <netinet/in.h>
-#include <structmember.h>
 #include <sys/socket.h>
 #include <unistd.h>
+#endif
+#include <maxminddb.h>
+#include <stdbool.h>
+#include <structmember.h>
 
 #define __STDC_FORMAT_MACROS
 #include <inttypes.h>
@@ -53,6 +59,7 @@ typedef struct {
 } Metadata_obj;
 // clang-format on
 
+static bool can_read(const char *path);
 static int get_record(PyObject *self, PyObject *args, PyObject **record);
 static bool format_sockaddr(struct sockaddr *addr, char *dst);
 static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list);
@@ -97,8 +104,7 @@ static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) {
         return -1;
     }
 
-    if (0 != access(filename, R_OK)) {
-
+    if (!can_read(filename)) {
         PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, filepath);
         Py_XDECREF(filepath);
         return -1;
@@ -737,7 +743,7 @@ static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) {
 
     const uint32_t map_size = (*entry_data_list)->entry_data.data_size;
 
-    uint i;
+    uint32_t i;
     // entry_data_list cannot start out NULL (see from_entry_data_list). We
     // check it in the loop because it may become NULL.
     // coverity[check_after_deref]
@@ -778,7 +784,7 @@ static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list) {
         return NULL;
     }
 
-    uint i;
+    uint32_t i;
     // entry_data_list cannot start out NULL (see from_entry_data_list). We
     // check it in the loop because it may become NULL.
     // coverity[check_after_deref]
@@ -826,6 +832,16 @@ static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list) {
     return py_obj;
 }
 
+static bool can_read(const char *path) {
+#ifdef MS_WINDOWS
+    int rv = _access(path, 04);
+#else
+    int rv = access(path, R_OK);
+#endif
+
+    return rv == 0;
+}
+
 static PyMethodDef Reader_methods[] = {
     {"get",
      Reader_get,
@@ -873,7 +889,7 @@ static PyMethodDef ReaderIter_methods[] = {{NULL, NULL, 0, NULL}};
 
 // clang-format off
 static PyTypeObject ReaderIter_Type = {
-    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    PyVarObject_HEAD_INIT(NULL, 0)
     .tp_basicsize = sizeof(ReaderIter_obj),
     .tp_dealloc = ReaderIter_dealloc,
     .tp_doc = "Iterator for Reader object",

maxminddb/file.py

@@ -47,7 +47,7 @@ def close(self) -> None:
         def _read(self, buffersize: int, offset: int) -> bytes:
             """read that uses pread"""
             # pylint: disable=no-member
-            return os.pread(self._handle.fileno(), buffersize, offset)
+            return os.pread(self._handle.fileno(), buffersize, offset)  # type: ignore
 
     else:
 

setup.py

@@ -22,16 +22,19 @@
 JYTHON = sys.platform.startswith("java")
 
 if os.name == "nt":
-    compile_args = []
+    # Disable unknown pragma warning
+    compile_args = ["-wd4068"]
+    libraries = ["Ws2_32"]
 else:
     compile_args = ["-Wall", "-Wextra", "-Wno-unknown-pragmas"]
+    libraries = []
 
 
 if os.getenv("MAXMINDDB_USE_SYSTEM_LIBMAXMINDDB"):
     ext_module = [
         Extension(
             "maxminddb.extension",
-            libraries=["maxminddb"],
+            libraries=["maxminddb"] + libraries,
             sources=["extension/maxminddb.c"],
             extra_compile_args=compile_args,
         )
@@ -40,6 +43,7 @@
     ext_module = [
         Extension(
             "maxminddb.extension",
+            libraries=libraries,
             sources=[
                 "extension/maxminddb.c",
                 "extension/libmaxminddb/src/data-pool.c",

tests/reader_test.py

@@ -46,7 +46,11 @@ class BaseTestReader(object):
         Type["maxminddb.extension.Reader"], Type["maxminddb.reader.Reader"]
     ]
     use_ip_objects = False
-    mp = multiprocessing.get_context("fork")
+
+    # fork doesn't work on Windows and spawn would involve pickling the reader,
+    # which isn't possible.
+    if os.name != "nt":
+        mp = multiprocessing.get_context("fork")
 
     def ipf(self, ip):
         if self.use_ip_objects:
@@ -490,41 +494,43 @@ def test_closed_metadata(self):
         else:
             self.assertIsNotNone(metadata, "pure Python implementation returns value")
 
-    def test_multiprocessing(self):
-        self._check_concurrency(self.mp.Process)
-
-    def test_threading(self):
-        self._check_concurrency(threading.Thread)
+    if os.name != "nt":
 
-    def _check_concurrency(self, worker_class):
-        reader = open_database(
-            "tests/data/test-data/GeoIP2-Domain-Test.mmdb", self.mode
-        )
+        def test_multiprocessing(self):
+            self._check_concurrency(self.mp.Process)
 
-        def lookup(pipe):
-            try:
-                for i in range(32):
-                    reader.get(self.ipf(f"65.115.240.{i}"))
-                pipe.send(1)
-            except:
-                pipe.send(0)
-            finally:
-                if worker_class is self.mp.Process:
-                    reader.close()
-                pipe.close()
-
-        pipes = [self.mp.Pipe() for _ in range(32)]
-        procs = [worker_class(target=lookup, args=(c,)) for (p, c) in pipes]
-        for proc in procs:
-            proc.start()
-        for proc in procs:
-            proc.join()
+        def test_threading(self):
+            self._check_concurrency(threading.Thread)
 
-        reader.close()
-
-        count = sum([p.recv() for (p, c) in pipes])
+        def _check_concurrency(self, worker_class):
+            reader = open_database(
+                "tests/data/test-data/GeoIP2-Domain-Test.mmdb", self.mode
+            )
 
-        self.assertEqual(count, 32, "expected number of successful lookups")
+            def lookup(pipe):
+                try:
+                    for i in range(32):
+                        reader.get(self.ipf(f"65.115.240.{i}"))
+                    pipe.send(1)
+                except:
+                    pipe.send(0)
+                finally:
+                    if worker_class is self.mp.Process:
+                        reader.close()
+                    pipe.close()
+
+            pipes = [self.mp.Pipe() for _ in range(32)]
+            procs = [worker_class(target=lookup, args=(c,)) for (p, c) in pipes]
+            for proc in procs:
+                proc.start()
+            for proc in procs:
+                proc.join()
+
+            reader.close()
+
+            count = sum([p.recv() for (p, c) in pipes])
+
+            self.assertEqual(count, 32, "expected number of successful lookups")
 
     def _check_metadata(self, reader, ip_version, record_size):
         metadata = reader.metadata()