Introduce explicit thread pool management

docs/reference/IO.md

@@ -5,4 +5,4 @@
 ::: osmium.io.Header
 ::: osmium.io.Reader
 ::: osmium.io.Writer
-
+::: osmium.io.ThreadPool

lib/base_handler.h

@@ -2,7 +2,7 @@
  *
  * This file is part of pyosmium. (https://osmcode.org/pyosmium/)
  *
- * Copyright (C) 2024 Sarah Hoffmann <lonvia@denofr.de> and others.
+ * Copyright (C) 2025 Sarah Hoffmann <lonvia@denofr.de> and others.
  * For a full list of authors see the git log.
  */
 #ifndef PYOSMIUM_BASE_HANDLER_HPP
@@ -41,7 +41,6 @@ class BaseHandler
     osmium::osm_entity_bits::type m_enabled_for = osmium::osm_entity_bits::all;
 };
 
-void apply(osmium::io::Reader &reader, BaseHandler &handler);
 void apply_item(osmium::OSMEntity &item, BaseHandler &handler);
 
 } // namespace

lib/file_iterator.cc

@@ -2,7 +2,7 @@
  *
  * This file is part of pyosmium. (https://osmcode.org/pyosmium/)
  *
- * Copyright (C) 2024 Sarah Hoffmann <lonvia@denofr.de> and others.
+ * Copyright (C) 2025 Sarah Hoffmann <lonvia@denofr.de> and others.
  * For a full list of authors see the git log.
  */
 #include <pybind11/pybind11.h>
@@ -15,6 +15,7 @@
 #include "osm_base_objects.h"
 #include "handler_chain.h"
 #include "python_handler.h"
+#include "io.h"
 
 namespace py = pybind11;
 
@@ -23,8 +24,8 @@ namespace {
 class OsmFileIterator
 {
 public:
-    OsmFileIterator(osmium::io::Reader *reader, py::args args)
-    : m_reader(reader), m_handler(args)
+    OsmFileIterator(pyosmium::PyReader &reader, py::args args)
+    : m_reader(reader.get()), m_handler(args)
     {
         m_buffer = m_reader->read();
 
@@ -140,12 +141,12 @@ namespace pyosmium {
 void init_osm_file_iterator(py::module &m)
 {
     py::class_<OsmFileIterator>(m, "OsmFileIterator")
-        .def(py::init<osmium::io::Reader *, py::args>(),
-             py::keep_alive<0, 1>())
+        .def(py::init<pyosmium::PyReader &, py::args>(),
+             py::keep_alive<1, 2>())
         .def("set_filtered_handler", &OsmFileIterator::set_filtered_handler,
-             py::keep_alive<0, 1>())
+             py::keep_alive<1, 2>())
         .def("set_filtered_handler", &OsmFileIterator::set_filtered_python_handler,
-             py::keep_alive<0, 1>())
+             py::keep_alive<1, 2>())
         .def("__iter__", [](py::object const &self) { return self; })
         .def("__next__", &OsmFileIterator::next)
         ;

lib/id_tracker.cc

@@ -2,7 +2,7 @@
  *
  * This file is part of pyosmium. (https://osmcode.org/pyosmium/)
  *
- * Copyright (C) 2024 Sarah Hoffmann <lonvia@denofr.de> and others.
+ * Copyright (C) 2025 Sarah Hoffmann <lonvia@denofr.de> and others.
  * For a full list of authors see the git log.
  */
 #include <pybind11/pybind11.h>
@@ -12,6 +12,7 @@
 #include <osmium/io/any_input.hpp>
 #include <osmium/index/nwr_array.hpp>
 #include <osmium/index/id_set.hpp>
+#include <osmium/thread/pool.hpp>
 
 #include "base_filter.h"
 #include "osmium_module.h"
@@ -100,10 +101,11 @@ class IdTracker
 
     void complete_backward_references(osmium::io::File file, int relation_depth)
     {
+        osmium::thread::Pool thread_pool{};
         // first pass: relations
         while (relation_depth > 0 && !m_ids.relations().empty()) {
             bool need_recurse = false;
-            osmium::io::Reader rd{file, osmium::osm_entity_bits::relation};
+            osmium::io::Reader rd{file, osmium::osm_entity_bits::relation, thread_pool};
             while (auto const buffer = rd.read()) {
                 for (auto const &rel: buffer.select<osmium::Relation>()) {
                     if (m_ids.relations().get(rel.id())) {
@@ -125,7 +127,7 @@ class IdTracker
 
         // second pass: ways
         if (!m_ids.ways().empty()) {
-            osmium::io::Reader rd{file, osmium::osm_entity_bits::way};
+            osmium::io::Reader rd{file, osmium::osm_entity_bits::way, thread_pool};
             while (auto const buffer = rd.read()) {
                 for (auto const &way: buffer.select<osmium::Way>()) {
                     if (m_ids.ways().get(way.id())) {
@@ -141,13 +143,14 @@ class IdTracker
 
     void complete_forward_references(osmium::io::File file, int relation_depth)
     {
+        osmium::thread::Pool thread_pool{};
         // standard pass: find directly referenced ways and relations
         {
             auto entities = osmium::osm_entity_bits::way;
             if (relation_depth >= 0) {
                 entities |= osmium::osm_entity_bits::relation;
             }
-            osmium::io::Reader rd{file, entities};
+            osmium::io::Reader rd{file, entities, thread_pool};
             while (auto const buffer = rd.read()) {
                 for (auto const &object: buffer.select<osmium::OSMObject>()) {
                     if (object.type() == osmium::item_type::way) {
@@ -176,7 +179,7 @@ class IdTracker
         // recursive passes: find additional referenced relations
         while (relation_depth > 0 && !m_ids.relations().empty()) {
             bool need_recurse = false;
-            osmium::io::Reader rd{file, osmium::osm_entity_bits::relation};
+            osmium::io::Reader rd{file, osmium::osm_entity_bits::relation, thread_pool};
             while (auto const buffer = rd.read()) {
                 for (auto const &rel: buffer.select<osmium::Relation>()) {
                     if (!m_ids.relations().get(rel.id())) {

lib/io.cc

@@ -10,9 +10,12 @@
 
 #include <osmium/io/any_input.hpp>
 #include <osmium/io/any_output.hpp>
+#include <osmium/thread/pool.hpp>
 
 #include <filesystem>
 
+#include "io.h"
+
 namespace py = pybind11;
 
 namespace {
@@ -78,34 +81,65 @@ PYBIND11_MODULE(io, m)
              py::return_value_policy::reference_internal)
     ;
 
-    py::class_<osmium::io::Reader>(m, "Reader")
-        .def(py::init<std::string>())
-        .def(py::init<std::string, osmium::osm_entity_bits::type>())
-        .def(py::init<>([] (std::filesystem::path const &file) {
-                 return new osmium::io::Reader(file.string());
-             }))
-        .def(py::init<>([] (std::filesystem::path const &file, osmium::osm_entity_bits::type etype) {
-                 return new osmium::io::Reader(file.string(), etype);
-             }))
-        .def(py::init<osmium::io::File>(),
-             py::keep_alive<1, 2>())
-        .def(py::init<osmium::io::File, osmium::osm_entity_bits::type>(),
-             py::keep_alive<1, 2>())
-        .def("eof", &osmium::io::Reader::eof)
-        .def("close", &osmium::io::Reader::close)
-        .def("header", &osmium::io::Reader::header)
+    py::class_<pyosmium::PyReader>(m, "Reader")
+        .def(py::init<osmium::io::File, osmium::osm_entity_bits::type const *,
+                      osmium::thread::Pool *>(),
+             py::keep_alive<1, 2>(), py::keep_alive<1, 4>(),
+             py::arg("file"), py::arg("types") = nullptr, py::arg("thread_pool") = nullptr
+            )
+        .def(py::init<>([] (std::string file,
+                            osmium::osm_entity_bits::type const *types,
+                            osmium::thread::Pool *pool) {
+                 return new pyosmium::PyReader(osmium::io::File(std::move(file)),
+                                               types, pool); }),
+             py::keep_alive<1, 2>(), py::keep_alive<1, 4>(),
+             py::arg("file"), py::arg("types") = nullptr, py::arg("thread_pool") = nullptr
+            )
+        .def(py::init<>([] (std::filesystem::path const &file,
+                            osmium::osm_entity_bits::type const *types,
+                            osmium::thread::Pool *pool) {
+                 return new pyosmium::PyReader(osmium::io::File(file.string()),
+                                               types, pool); }),
+             py::keep_alive<1, 2>(), py::keep_alive<1, 4>(),
+             py::arg("file"), py::arg("types") = nullptr, py::arg("thread_pool") = nullptr
+            )
+        .def("eof", [](pyosmium::PyReader const &self) { return self.get()->eof(); })
+        .def("close", [](pyosmium::PyReader &self) { self.get()->close(); })
+        .def("header", [](pyosmium::PyReader &self) { return self.get()->header(); })
         .def("__enter__", [](py::object const &self) { return self; })
-        .def("__exit__", [](osmium::io::Reader &self, py::args args) { self.close(); })
+        .def("__exit__", [](pyosmium::PyReader &self, py::args args) { self.get()->close(); })
     ;
 
-    py::class_<osmium::io::Writer>(m, "Writer")
-        .def(py::init<std::string>())
-        .def(py::init<>([] (std::filesystem::path const &file) {
-                 return new osmium::io::Writer(file.string());
-             }))
-        .def(py::init<osmium::io::File>())
-        .def(py::init<std::string, osmium::io::Header>())
-        .def(py::init<osmium::io::File, osmium::io::Header>())
-        .def("close", &osmium::io::Writer::close)
+    py::class_<pyosmium::PyWriter>(m, "Writer")
+        .def(py::init<osmium::io::File, osmium::io::Header const *, bool, osmium::thread::Pool *>(),
+             py::keep_alive<1, 5>(),
+             py::arg("file"), py::arg("header") = nullptr,
+             py::arg("overwrite") = false, py::arg("thread_pool") = nullptr
+            )
+        .def(py::init<>([] (std::filesystem::path const &file, osmium::io::Header const *header,
+                            bool overwrite, osmium::thread::Pool *pool) {
+                 return new pyosmium::PyWriter(osmium::io::File(file.string()),
+                                               header, overwrite, pool); }),
+             py::keep_alive<1, 5>(),
+             py::arg("file"), py::arg("header") = nullptr,
+             py::arg("overwrite") = false, py::arg("thread_pool") = nullptr
+            )
+        .def(py::init<>([] (std::string filename, osmium::io::Header const *header,
+                            bool overwrite, osmium::thread::Pool *pool) {
+                 return new pyosmium::PyWriter(osmium::io::File(std::move(filename)),
+                                               header, overwrite, pool); }),
+             py::keep_alive<1, 5>(),
+             py::arg("file"), py::arg("header") = nullptr,
+             py::arg("overwrite") = false, py::arg("thread_pool") = nullptr
+            )
+        .def("close", [](pyosmium::PyWriter &self) { self.get()->close(); })
     ;
+
+    py::class_<osmium::thread::Pool>(m, "ThreadPool")
+        .def(py::init<int, std::size_t>(),
+             py::arg("num_threads")=0, py::arg("max_queue_size")=0U)
+        .def_property_readonly("num_threads", &osmium::thread::Pool::num_threads)
+        .def("queue_size", &osmium::thread::Pool::queue_size)
+        .def("queue_empty", &osmium::thread::Pool::queue_empty)
+     ;
 }

lib/io.h

@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: BSD-2-Clause
+ *
+ * This file is part of pyosmium. (https://osmcode.org/pyosmium/)
+ *
+ * Copyright (C) 2025 Sarah Hoffmann <lonvia@denofr.de> and others.
+ * For a full list of authors see the git log.
+ */
+#ifndef PYOSMIUM_IO_H
+#define PYOSMIUM_IO_H
+
+#include <osmium/thread/pool.hpp>
+#include <osmium/io/any_input.hpp>
+#include <osmium/io/any_output.hpp>
+
+namespace pyosmium {
+
+class PyReader
+{
+public:
+    explicit PyReader(osmium::io::File fname)
+    : thread_pool(std::make_unique<osmium::thread::Pool>()),
+      reader(std::move(fname))
+    {}
+
+    PyReader(osmium::io::File fname, osmium::osm_entity_bits::type const *etype,
+             osmium::thread::Pool *pool)
+    : thread_pool(pool ? std::unique_ptr<osmium::thread::Pool>()
+                       : std::make_unique<osmium::thread::Pool>()),
+      reader(std::move(fname),
+             etype ? *etype : osmium::osm_entity_bits::all,
+             *(pool ? pool : thread_pool.get()))
+    {}
+
+    osmium::io::Reader const *get() const { return &reader; }
+    osmium::io::Reader *get() { return &reader; }
+
+private:
+    std::unique_ptr<osmium::thread::Pool> thread_pool;
+    osmium::io::Reader reader;
+};
+
+
+class PyWriter
+{
+public:
+    PyWriter(osmium::io::File file, osmium::io::Header const *header,
+             bool overwrite, osmium::thread::Pool *pool)
+    : thread_pool(pool ? std::unique_ptr<osmium::thread::Pool>()
+                       : std::make_unique<osmium::thread::Pool>()),
+      writer(std::move(file),
+             header ? *header : osmium::io::Header(),
+             overwrite ? osmium::io::overwrite::allow : osmium::io::overwrite::no,
+             *(pool ? pool : thread_pool.get()))
+    {}
+
+    osmium::io::Writer const *get() const { return &writer; }
+    osmium::io::Writer *get() { return &writer; }
+
+private:
+    std::unique_ptr<osmium::thread::Pool> thread_pool;
+    osmium::io::Writer writer;
+};
+
+}
+
+#endif // PYOSMIUM_IO_H

lib/merge_input_reader.cc

@@ -2,24 +2,26 @@
  *
  * This file is part of pyosmium. (https://osmcode.org/pyosmium/)
  *
- * Copyright (C) 2024 Sarah Hoffmann <lonvia@denofr.de> and others.
+ * Copyright (C) 2025 Sarah Hoffmann <lonvia@denofr.de> and others.
  * For a full list of authors see the git log.
  */
 #include <pybind11/pybind11.h>
 
 #include <vector>
 
-#include <boost/function_output_iterator.hpp>
+#include <boost/iterator/function_output_iterator.hpp>
 
 #include <osmium/osm/object_comparisons.hpp>
 #include <osmium/io/any_input.hpp>
 #include <osmium/io/any_output.hpp>
 #include <osmium/io/output_iterator.hpp>
 #include <osmium/object_pointer_collection.hpp>
 #include <osmium/visitor.hpp>
+#include <osmium/thread/pool.hpp>
 
 #include "osmium_module.h"
 #include "handler_chain.h"
+#include "io.h"
 
 namespace py = pybind11;
 
@@ -82,16 +84,16 @@ class MergeInputReader
         changes.clear();
     }
 
-    void apply_to_reader(osmium::io::Reader &reader, osmium::io::Writer &writer,
+    void apply_to_reader(pyosmium::PyReader &reader, pyosmium::PyWriter &writer,
                          bool with_history)
     {
-        auto input = osmium::io::make_input_iterator_range<osmium::OSMObject>(reader);
+        auto input = osmium::io::make_input_iterator_range<osmium::OSMObject>(*reader.get());
         if (with_history) {
             // For history files this is a straightforward sort of the change
             // files followed by a merge with the input file.
             objects.sort(osmium::object_order_type_id_version());
 
-            auto out = osmium::io::make_output_iterator(writer);
+            auto out = osmium::io::make_output_iterator(*writer.get());
             std::set_union(objects.begin(),
                     objects.end(),
                     input.begin(),
@@ -111,7 +113,7 @@ class MergeInputReader
             objects.sort(osmium::object_order_type_id_reverse_version());
 
             auto output_it = boost::make_function_output_iterator(
-                    copy_first_with_id(writer)
+                    copy_first_with_id(*writer.get())
                     );
 
             std::set_union(objects.begin(),
@@ -144,8 +146,9 @@ class MergeInputReader
     size_t internal_add(osmium::io::File change_file)
     {
         size_t sz = 0;
+        osmium::thread::Pool thread_pool{};
 
-        osmium::io::Reader reader(change_file, osmium::osm_entity_bits::nwr);
+        osmium::io::Reader reader(change_file, osmium::osm_entity_bits::nwr, thread_pool);
         while (osmium::memory::Buffer buffer = reader.read()) {
             osmium::apply(buffer, objects);
             sz += buffer.committed();