diff --git a/docs/reference/IO.md b/docs/reference/IO.md index 576be3d..7dfc53e 100644 --- a/docs/reference/IO.md +++ b/docs/reference/IO.md @@ -5,4 +5,4 @@ ::: osmium.io.Header ::: osmium.io.Reader ::: osmium.io.Writer - +::: osmium.io.ThreadPool diff --git a/lib/base_handler.h b/lib/base_handler.h index 3c92f53..1ed17bb 100644 --- a/lib/base_handler.h +++ b/lib/base_handler.h @@ -2,7 +2,7 @@ * * This file is part of pyosmium. (https://osmcode.org/pyosmium/) * - * Copyright (C) 2024 Sarah Hoffmann and others. + * Copyright (C) 2025 Sarah Hoffmann and others. * For a full list of authors see the git log. */ #ifndef PYOSMIUM_BASE_HANDLER_HPP @@ -41,7 +41,6 @@ class BaseHandler osmium::osm_entity_bits::type m_enabled_for = osmium::osm_entity_bits::all; }; -void apply(osmium::io::Reader &reader, BaseHandler &handler); void apply_item(osmium::OSMEntity &item, BaseHandler &handler); } // namespace diff --git a/lib/file_iterator.cc b/lib/file_iterator.cc index 045de77..0c7d0f1 100644 --- a/lib/file_iterator.cc +++ b/lib/file_iterator.cc @@ -2,7 +2,7 @@ * * This file is part of pyosmium. (https://osmcode.org/pyosmium/) * - * Copyright (C) 2024 Sarah Hoffmann and others. + * Copyright (C) 2025 Sarah Hoffmann and others. * For a full list of authors see the git log. */ #include @@ -15,6 +15,7 @@ #include "osm_base_objects.h" #include "handler_chain.h" #include "python_handler.h" +#include "io.h" namespace py = pybind11; @@ -23,8 +24,8 @@ namespace { class OsmFileIterator { public: - OsmFileIterator(osmium::io::Reader *reader, py::args args) - : m_reader(reader), m_handler(args) + OsmFileIterator(pyosmium::PyReader &reader, py::args args) + : m_reader(reader.get()), m_handler(args) { m_buffer = m_reader->read(); @@ -140,12 +141,12 @@ namespace pyosmium { void init_osm_file_iterator(py::module &m) { py::class_(m, "OsmFileIterator") - .def(py::init(), - py::keep_alive<0, 1>()) + .def(py::init(), + py::keep_alive<1, 2>()) .def("set_filtered_handler", &OsmFileIterator::set_filtered_handler, - py::keep_alive<0, 1>()) + py::keep_alive<1, 2>()) .def("set_filtered_handler", &OsmFileIterator::set_filtered_python_handler, - py::keep_alive<0, 1>()) + py::keep_alive<1, 2>()) .def("__iter__", [](py::object const &self) { return self; }) .def("__next__", &OsmFileIterator::next) ; diff --git a/lib/id_tracker.cc b/lib/id_tracker.cc index 490828d..8405b42 100644 --- a/lib/id_tracker.cc +++ b/lib/id_tracker.cc @@ -2,7 +2,7 @@ * * This file is part of pyosmium. (https://osmcode.org/pyosmium/) * - * Copyright (C) 2024 Sarah Hoffmann and others. + * Copyright (C) 2025 Sarah Hoffmann and others. * For a full list of authors see the git log. */ #include @@ -12,6 +12,7 @@ #include #include #include +#include #include "base_filter.h" #include "osmium_module.h" @@ -100,10 +101,11 @@ class IdTracker void complete_backward_references(osmium::io::File file, int relation_depth) { + osmium::thread::Pool thread_pool{}; // first pass: relations while (relation_depth > 0 && !m_ids.relations().empty()) { bool need_recurse = false; - osmium::io::Reader rd{file, osmium::osm_entity_bits::relation}; + osmium::io::Reader rd{file, osmium::osm_entity_bits::relation, thread_pool}; while (auto const buffer = rd.read()) { for (auto const &rel: buffer.select()) { if (m_ids.relations().get(rel.id())) { @@ -125,7 +127,7 @@ class IdTracker // second pass: ways if (!m_ids.ways().empty()) { - osmium::io::Reader rd{file, osmium::osm_entity_bits::way}; + osmium::io::Reader rd{file, osmium::osm_entity_bits::way, thread_pool}; while (auto const buffer = rd.read()) { for (auto const &way: buffer.select()) { if (m_ids.ways().get(way.id())) { @@ -141,13 +143,14 @@ class IdTracker void complete_forward_references(osmium::io::File file, int relation_depth) { + osmium::thread::Pool thread_pool{}; // standard pass: find directly referenced ways and relations { auto entities = osmium::osm_entity_bits::way; if (relation_depth >= 0) { entities |= osmium::osm_entity_bits::relation; } - osmium::io::Reader rd{file, entities}; + osmium::io::Reader rd{file, entities, thread_pool}; while (auto const buffer = rd.read()) { for (auto const &object: buffer.select()) { if (object.type() == osmium::item_type::way) { @@ -176,7 +179,7 @@ class IdTracker // recursive passes: find additional referenced relations while (relation_depth > 0 && !m_ids.relations().empty()) { bool need_recurse = false; - osmium::io::Reader rd{file, osmium::osm_entity_bits::relation}; + osmium::io::Reader rd{file, osmium::osm_entity_bits::relation, thread_pool}; while (auto const buffer = rd.read()) { for (auto const &rel: buffer.select()) { if (!m_ids.relations().get(rel.id())) { diff --git a/lib/io.cc b/lib/io.cc index d36a2dc..9b34d29 100644 --- a/lib/io.cc +++ b/lib/io.cc @@ -10,9 +10,12 @@ #include #include +#include #include +#include "io.h" + namespace py = pybind11; namespace { @@ -78,34 +81,65 @@ PYBIND11_MODULE(io, m) py::return_value_policy::reference_internal) ; - py::class_(m, "Reader") - .def(py::init()) - .def(py::init()) - .def(py::init<>([] (std::filesystem::path const &file) { - return new osmium::io::Reader(file.string()); - })) - .def(py::init<>([] (std::filesystem::path const &file, osmium::osm_entity_bits::type etype) { - return new osmium::io::Reader(file.string(), etype); - })) - .def(py::init(), - py::keep_alive<1, 2>()) - .def(py::init(), - py::keep_alive<1, 2>()) - .def("eof", &osmium::io::Reader::eof) - .def("close", &osmium::io::Reader::close) - .def("header", &osmium::io::Reader::header) + py::class_(m, "Reader") + .def(py::init(), + py::keep_alive<1, 2>(), py::keep_alive<1, 4>(), + py::arg("file"), py::arg("types") = nullptr, py::arg("thread_pool") = nullptr + ) + .def(py::init<>([] (std::string file, + osmium::osm_entity_bits::type const *types, + osmium::thread::Pool *pool) { + return new pyosmium::PyReader(osmium::io::File(std::move(file)), + types, pool); }), + py::keep_alive<1, 2>(), py::keep_alive<1, 4>(), + py::arg("file"), py::arg("types") = nullptr, py::arg("thread_pool") = nullptr + ) + .def(py::init<>([] (std::filesystem::path const &file, + osmium::osm_entity_bits::type const *types, + osmium::thread::Pool *pool) { + return new pyosmium::PyReader(osmium::io::File(file.string()), + types, pool); }), + py::keep_alive<1, 2>(), py::keep_alive<1, 4>(), + py::arg("file"), py::arg("types") = nullptr, py::arg("thread_pool") = nullptr + ) + .def("eof", [](pyosmium::PyReader const &self) { return self.get()->eof(); }) + .def("close", [](pyosmium::PyReader &self) { self.get()->close(); }) + .def("header", [](pyosmium::PyReader &self) { return self.get()->header(); }) .def("__enter__", [](py::object const &self) { return self; }) - .def("__exit__", [](osmium::io::Reader &self, py::args args) { self.close(); }) + .def("__exit__", [](pyosmium::PyReader &self, py::args args) { self.get()->close(); }) ; - py::class_(m, "Writer") - .def(py::init()) - .def(py::init<>([] (std::filesystem::path const &file) { - return new osmium::io::Writer(file.string()); - })) - .def(py::init()) - .def(py::init()) - .def(py::init()) - .def("close", &osmium::io::Writer::close) + py::class_(m, "Writer") + .def(py::init(), + py::keep_alive<1, 5>(), + py::arg("file"), py::arg("header") = nullptr, + py::arg("overwrite") = false, py::arg("thread_pool") = nullptr + ) + .def(py::init<>([] (std::filesystem::path const &file, osmium::io::Header const *header, + bool overwrite, osmium::thread::Pool *pool) { + return new pyosmium::PyWriter(osmium::io::File(file.string()), + header, overwrite, pool); }), + py::keep_alive<1, 5>(), + py::arg("file"), py::arg("header") = nullptr, + py::arg("overwrite") = false, py::arg("thread_pool") = nullptr + ) + .def(py::init<>([] (std::string filename, osmium::io::Header const *header, + bool overwrite, osmium::thread::Pool *pool) { + return new pyosmium::PyWriter(osmium::io::File(std::move(filename)), + header, overwrite, pool); }), + py::keep_alive<1, 5>(), + py::arg("file"), py::arg("header") = nullptr, + py::arg("overwrite") = false, py::arg("thread_pool") = nullptr + ) + .def("close", [](pyosmium::PyWriter &self) { self.get()->close(); }) ; + + py::class_(m, "ThreadPool") + .def(py::init(), + py::arg("num_threads")=0, py::arg("max_queue_size")=0U) + .def_property_readonly("num_threads", &osmium::thread::Pool::num_threads) + .def("queue_size", &osmium::thread::Pool::queue_size) + .def("queue_empty", &osmium::thread::Pool::queue_empty) + ; } diff --git a/lib/io.h b/lib/io.h new file mode 100644 index 0000000..fbfced0 --- /dev/null +++ b/lib/io.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-2-Clause + * + * This file is part of pyosmium. (https://osmcode.org/pyosmium/) + * + * Copyright (C) 2025 Sarah Hoffmann and others. + * For a full list of authors see the git log. + */ +#ifndef PYOSMIUM_IO_H +#define PYOSMIUM_IO_H + +#include +#include +#include + +namespace pyosmium { + +class PyReader +{ +public: + explicit PyReader(osmium::io::File fname) + : thread_pool(std::make_unique()), + reader(std::move(fname)) + {} + + PyReader(osmium::io::File fname, osmium::osm_entity_bits::type const *etype, + osmium::thread::Pool *pool) + : thread_pool(pool ? std::unique_ptr() + : std::make_unique()), + reader(std::move(fname), + etype ? *etype : osmium::osm_entity_bits::all, + *(pool ? pool : thread_pool.get())) + {} + + osmium::io::Reader const *get() const { return &reader; } + osmium::io::Reader *get() { return &reader; } + +private: + std::unique_ptr thread_pool; + osmium::io::Reader reader; +}; + + +class PyWriter +{ +public: + PyWriter(osmium::io::File file, osmium::io::Header const *header, + bool overwrite, osmium::thread::Pool *pool) + : thread_pool(pool ? std::unique_ptr() + : std::make_unique()), + writer(std::move(file), + header ? *header : osmium::io::Header(), + overwrite ? osmium::io::overwrite::allow : osmium::io::overwrite::no, + *(pool ? pool : thread_pool.get())) + {} + + osmium::io::Writer const *get() const { return &writer; } + osmium::io::Writer *get() { return &writer; } + +private: + std::unique_ptr thread_pool; + osmium::io::Writer writer; +}; + +} + +#endif // PYOSMIUM_IO_H diff --git a/lib/merge_input_reader.cc b/lib/merge_input_reader.cc index 52cd722..3691a95 100644 --- a/lib/merge_input_reader.cc +++ b/lib/merge_input_reader.cc @@ -2,14 +2,14 @@ * * This file is part of pyosmium. (https://osmcode.org/pyosmium/) * - * Copyright (C) 2024 Sarah Hoffmann and others. + * Copyright (C) 2025 Sarah Hoffmann and others. * For a full list of authors see the git log. */ #include #include -#include +#include #include #include @@ -17,9 +17,11 @@ #include #include #include +#include #include "osmium_module.h" #include "handler_chain.h" +#include "io.h" namespace py = pybind11; @@ -82,16 +84,16 @@ class MergeInputReader changes.clear(); } - void apply_to_reader(osmium::io::Reader &reader, osmium::io::Writer &writer, + void apply_to_reader(pyosmium::PyReader &reader, pyosmium::PyWriter &writer, bool with_history) { - auto input = osmium::io::make_input_iterator_range(reader); + auto input = osmium::io::make_input_iterator_range(*reader.get()); if (with_history) { // For history files this is a straightforward sort of the change // files followed by a merge with the input file. objects.sort(osmium::object_order_type_id_version()); - auto out = osmium::io::make_output_iterator(writer); + auto out = osmium::io::make_output_iterator(*writer.get()); std::set_union(objects.begin(), objects.end(), input.begin(), @@ -111,7 +113,7 @@ class MergeInputReader objects.sort(osmium::object_order_type_id_reverse_version()); auto output_it = boost::make_function_output_iterator( - copy_first_with_id(writer) + copy_first_with_id(*writer.get()) ); std::set_union(objects.begin(), @@ -144,8 +146,9 @@ class MergeInputReader size_t internal_add(osmium::io::File change_file) { size_t sz = 0; + osmium::thread::Pool thread_pool{}; - osmium::io::Reader reader(change_file, osmium::osm_entity_bits::nwr); + osmium::io::Reader reader(change_file, osmium::osm_entity_bits::nwr, thread_pool); while (osmium::memory::Buffer buffer = reader.read()) { osmium::apply(buffer, objects); sz += buffer.committed(); diff --git a/lib/osmium.cc b/lib/osmium.cc index 084f10d..7cfb3e8 100644 --- a/lib/osmium.cc +++ b/lib/osmium.cc @@ -20,6 +20,7 @@ #include "python_handler.h" #include "handler_chain.h" #include "buffer_iterator.h" +#include "io.h" #include #include @@ -62,9 +63,11 @@ void pyosmium::apply_item(osmium::OSMEntity &obj, pyosmium::BaseHandler &handler } } -void pyosmium::apply(osmium::io::Reader &reader, pyosmium::BaseHandler &handler) +namespace { + +void pyosmium_apply(pyosmium::PyReader &reader, pyosmium::BaseHandler &handler) { - while (auto buffer = reader.read()) { + while (auto buffer = reader.get()->read()) { for (auto &obj : buffer.select()) { pyosmium::apply_item(obj, handler); } @@ -72,6 +75,8 @@ void pyosmium::apply(osmium::io::Reader &reader, pyosmium::BaseHandler &handler) handler.flush(); } +} + #ifdef Py_GIL_DISABLED PYBIND11_MODULE(_osmium, m, py::mod_gil_not_used()) #else @@ -87,52 +92,52 @@ PYBIND11_MODULE(_osmium, m) } }); - m.def("apply", &pyosmium::apply, + m.def("apply", &pyosmium_apply, py::arg("reader"), py::arg("handler")); - m.def("apply", [](osmium::io::Reader &rd, py::args args) - { - pyosmium::HandlerChain handler{args}; - pyosmium::apply(rd, handler); - }, + m.def("apply", [](pyosmium::PyReader &rd, py::args args) + { + pyosmium::HandlerChain handler{args}; + pyosmium_apply(rd, handler); + }, py::arg("reader")); m.def("apply", [](std::string fn, pyosmium::BaseHandler &h) { - osmium::io::Reader rd{fn}; - pyosmium::apply(rd, h); + pyosmium::PyReader rd{osmium::io::File(std::move(fn))}; + pyosmium_apply(rd, h); }, py::arg("filename"), py::arg("handler")); m.def("apply", [](std::string fn, py::args args) - { - pyosmium::HandlerChain handler{args}; - osmium::io::Reader rd{fn}; - pyosmium::apply(rd, handler); - }, + { + pyosmium::HandlerChain handler{args}; + pyosmium::PyReader rd{osmium::io::File(std::move(fn))}; + pyosmium_apply(rd, handler); + }, py::arg("filename")); m.def("apply", [](std::filesystem::path const &fn, pyosmium::BaseHandler &h) { - osmium::io::Reader rd{fn.string()}; - pyosmium::apply(rd, h); + pyosmium::PyReader rd{osmium::io::File(fn.string())}; + pyosmium_apply(rd, h); }, py::arg("filename"), py::arg("handler")); m.def("apply", [](std::filesystem::path const &fn, py::args args) - { - pyosmium::HandlerChain handler{args}; - osmium::io::Reader rd{fn.string()}; - pyosmium::apply(rd, handler); - }, + { + pyosmium::HandlerChain handler{args}; + pyosmium::PyReader rd{osmium::io::File(fn.string())}; + pyosmium_apply(rd, handler); + }, py::arg("filename")); m.def("apply", [](osmium::io::File fn, pyosmium::BaseHandler &h) { - osmium::io::Reader rd{fn}; - pyosmium::apply(rd, h); + pyosmium::PyReader rd{fn}; + pyosmium_apply(rd, h); }, py::arg("filename"), py::arg("handler")); m.def("apply", [](osmium::io::File fn, py::args args) - { - pyosmium::HandlerChain handler{args}; - osmium::io::Reader rd{fn}; - pyosmium::apply(rd, handler); - }, + { + pyosmium::HandlerChain handler{args}; + pyosmium::PyReader rd{fn}; + pyosmium_apply(rd, handler); + }, py::arg("filename")); py::class_(m, "BaseHandler"); diff --git a/lib/replication.cc b/lib/replication.cc index 8a9c189..b3cb120 100644 --- a/lib/replication.cc +++ b/lib/replication.cc @@ -6,12 +6,16 @@ * For a full list of authors see the git log. */ #include +#include #include -#include #include #include + +#include + #include "cast.h" +#include "io.h" namespace py = pybind11; @@ -29,6 +33,12 @@ struct LastChangeHandler : public osmium::handler::Handler } }; +osmium::Timestamp newest_change_from_file(pyosmium::PyReader &reader) { + LastChangeHandler handler; + osmium::apply(*reader.get(), handler); + return handler.last_change; +} + } // namespace #ifdef Py_GIL_DISABLED @@ -37,14 +47,18 @@ PYBIND11_MODULE(_replication, m, py::mod_gil_not_used()) PYBIND11_MODULE(_replication, m) #endif { - m.def("newest_change_from_file", [](char const *filename) - { - osmium::io::Reader reader(filename, osmium::osm_entity_bits::nwr); - - LastChangeHandler handler; - osmium::apply(reader, handler); - reader.close(); - - return handler.last_change; - }); + m.def("newest_change_from_file", &newest_change_from_file, + py::arg("reader")); + m.def("newest_change_from_file", [](std::string file) { + pyosmium::PyReader reader(osmium::io::File(std::move(file))); + return newest_change_from_file(reader); + }); + m.def("newest_change_from_file", [](std::filesystem::path const &file) { + pyosmium::PyReader reader(osmium::io::File(file.string())); + return newest_change_from_file(reader); + }); + m.def("newest_change_from_file", [](osmium::io::File file) { + pyosmium::PyReader reader(std::move(file)); + return newest_change_from_file(reader); + }); } diff --git a/lib/simple_writer.cc b/lib/simple_writer.cc index 5126c77..e7900ca 100644 --- a/lib/simple_writer.cc +++ b/lib/simple_writer.cc @@ -2,7 +2,7 @@ * * This file is part of pyosmium. (https://osmcode.org/pyosmium/) * - * Copyright (C) 2024 Sarah Hoffmann and others. + * Copyright (C) 2025 Sarah Hoffmann and others. * For a full list of authors see the git log. */ #include @@ -14,10 +14,12 @@ #include #include #include +#include #include "cast.h" #include "osm_base_objects.h" #include "base_handler.h" +#include "io.h" #include @@ -30,20 +32,10 @@ class SimpleWriter : public pyosmium::BaseHandler enum { BUFFER_WRAP = 4096 }; public: - SimpleWriter(const char* filename, size_t bufsz, osmium::io::Header const *header, - bool overwrite, const std::string &filetype) - : writer(osmium::io::File(filename, filetype), - header ? *header : osmium::io::Header(), - overwrite ? osmium::io::overwrite::allow : osmium::io::overwrite::no), - buffer(bufsz < 2 * BUFFER_WRAP ? 2 * BUFFER_WRAP : bufsz, - osmium::memory::Buffer::auto_grow::yes), - buffer_size(buffer.capacity()) // same rounding to BUFFER_WRAP - {} - - SimpleWriter(osmium::io::File file, size_t bufsz, osmium::io::Header const *header, - bool overwrite) - : writer(file, header ? *header : osmium::io::Header(), - overwrite ? osmium::io::overwrite::allow : osmium::io::overwrite::no), + SimpleWriter(osmium::io::File file, unsigned long bufsz, + osmium::io::Header const *header, bool overwrite, + osmium::thread::Pool *pool) + : writer(file, header, overwrite, pool), buffer(bufsz < 2 * BUFFER_WRAP ? 2 * BUFFER_WRAP : bufsz, osmium::memory::Buffer::auto_grow::yes), buffer_size(buffer.capacity()) // same rounding to BUFFER_WRAP @@ -154,8 +146,8 @@ class SimpleWriter : public pyosmium::BaseHandler void close() { if (buffer) { - writer(std::move(buffer)); - writer.close(); + (*writer.get())(std::move(buffer)); + writer.get()->close(); buffer = osmium::memory::Buffer(); } } @@ -335,11 +327,11 @@ class SimpleWriter : public pyosmium::BaseHandler osmium::memory::Buffer new_buffer(buffer_size, osmium::memory::Buffer::auto_grow::yes); using std::swap; swap(buffer, new_buffer); - writer(std::move(new_buffer)); + (*writer.get())(std::move(new_buffer)); } } - osmium::io::Writer writer; + pyosmium::PyWriter writer; osmium::memory::Buffer buffer; size_t buffer_size; }; @@ -351,22 +343,38 @@ namespace pyosmium { void init_simple_writer(pybind11::module &m) { py::class_(m, "SimpleWriter") - .def(py::init(), + .def(py::init<>([] (std::string file, unsigned long bufsz, + osmium::io::Header const *header, bool overwrite, + osmium::thread::Pool *pool) { + return new SimpleWriter(osmium::io::File(std::move(file)), bufsz, + header, overwrite, pool); + }), + py::keep_alive<1, 6>(), py::arg("filename"), py::arg("bufsz") = 4096*1024, py::arg("header") = nullptr, py::arg("overwrite") = false, - py::arg("filetype") = "") + py::arg("thread_pool") = nullptr + ) .def(py::init<>([] (std::filesystem::path const &file, unsigned long bufsz, - osmium::io::Header const *header, bool overwrite) { - return new SimpleWriter(file.string().c_str(), bufsz, header, overwrite, ""); + osmium::io::Header const *header, bool overwrite, + osmium::thread::Pool *pool) { + return new SimpleWriter(osmium::io::File(file.string()), bufsz, + header, overwrite, pool); }), + py::keep_alive<1, 6>(), py::arg("filename"), py::arg("bufsz") = 4096*1024, py::arg("header") = nullptr, - py::arg("overwrite") = false) - .def(py::init(), + py::arg("overwrite") = false, + py::arg("thread_pool") = nullptr + ) + .def(py::init(), + py::keep_alive<1, 6>(), py::arg("filename"), py::arg("bufsz") = 4096*1024, py::arg("header") = nullptr, - py::arg("overwrite") = false) + py::arg("overwrite") = false, + py::arg("thread_pool") = nullptr + ) .def("add_node", &SimpleWriter::add_node, py::arg("node")) .def("add_way", &SimpleWriter::add_way, py::arg("way")) .def("add_relation", &SimpleWriter::add_relation, py::arg("relation")) diff --git a/src/osmium/_osmium.pyi b/src/osmium/_osmium.pyi index b0f4b3c..4f5cf16 100644 --- a/src/osmium/_osmium.pyi +++ b/src/osmium/_osmium.pyi @@ -2,7 +2,7 @@ # # This file is part of pyosmium. (https://osmcode.org/pyosmium/) # -# Copyright (C) 2024 Sarah Hoffmann and others. +# Copyright (C) 2025 Sarah Hoffmann and others. # For a full list of authors see the git log. from typing import ByteString, Union, Optional, Any import os @@ -10,7 +10,7 @@ import os from .osm import osm_entity_bits from .osm.types import OSMEntity from .index import LocationTable, IdSet -from .io import Reader, Writer, Header, File, FileBuffer +from .io import Reader, Writer, Header, File, FileBuffer, ThreadPool # Placeholder for more narrow type definition to come HandlerLike = object @@ -127,9 +127,9 @@ class SimpleWriter(BaseHandler): when writing is finished. """ def __init__(self, file: Union[str, 'os.PathLike[str]', File], - bufsz: int= ..., - header: Optional[Header]= ..., overwrite: bool= ..., - filetype: str= ...) -> None: + bufsz: int=4096*1024, + header: Optional[Header]=None, overwrite: bool=False, + thread_pool: Optional[ThreadPool]=None) -> None: """ Initiate a new writer for the given file. The writer will refuse to overwrite an already existing file unless _overwrite_ is explicitly set to `True`. @@ -137,8 +137,6 @@ class SimpleWriter(BaseHandler): The file type is usually determined from the file extension. If you want to explicitly set the filetype (for example, when writing to standard output '-'), then use a File object. - Using the _filetype_ parameter to set the file type is deprecated - and only works when the file is a string. The _header_ parameter can be used to set a custom header in the output file. What kind of information can be written into @@ -149,6 +147,14 @@ class SimpleWriter(BaseHandler): size is 4MB. Larger buffers are normally better but you should be aware that there are normally multiple buffers in use during the write process. + + The writer implicitly creates a private + [ThreadPool][osmium.io.ThreadPool] which it may + use to parallelize writing to the output. Alternatively you + may hand in an externally created thread pool. This may be useful + when you create many writers in parallel and want them to share + a single thread pool or when you want to customize the size + of the thread pool. """ def add_node(self, node: object) -> None: """ Add a new node to the file. The node may be a diff --git a/src/osmium/back_reference_writer.py b/src/osmium/back_reference_writer.py index 5de2b49..2782e51 100644 --- a/src/osmium/back_reference_writer.py +++ b/src/osmium/back_reference_writer.py @@ -2,15 +2,15 @@ # # This file is part of pyosmium. (https://osmcode.org/pyosmium/) # -# Copyright (C) 2024 Sarah Hoffmann and others. +# Copyright (C) 2025 Sarah Hoffmann and others. # For a full list of authors see the git log. -from typing import Any, Union +from typing import Any, Union, Optional from pathlib import Path from tempfile import TemporaryDirectory import os from osmium._osmium import SimpleWriter -from osmium.io import File, FileBuffer +from osmium.io import File, FileBuffer, ThreadPool from osmium.file_processor import FileProcessor, zip_processors from osmium import IdTracker @@ -30,7 +30,7 @@ class BackReferenceWriter: def __init__(self, outfile: Union[str, 'os.PathLike[str]', File], ref_src: Union[str, 'os.PathLike[str]', File, FileBuffer], overwrite: bool = False, remove_tags: bool = True, - relation_depth: int = 0): + relation_depth: int = 0, thread_pool: Optional[ThreadPool] = None): """ Create a new writer. `outfile` is the name of the output file to write. The file must @@ -47,10 +47,17 @@ def __init__(self, outfile: Union[str, 'os.PathLike[str]', File], The writer will not complete nested relations by default. If you need nested relations, set `relation_depth` to the minimum depth to which relations shall be completed. + + The writer implicitly creates a private + [ThreadPool][osmium.io.ThreadPool] which it + uses to parallelize IO operations. Alternatively you + may hand in an externally created thread pool. """ self.outfile = outfile self.tmpdir = TemporaryDirectory() - self.writer = SimpleWriter(str(Path(self.tmpdir.name, 'back_writer.osm.pbf'))) + self.thread_pool = thread_pool or ThreadPool() + self.writer = SimpleWriter(Path(self.tmpdir.name, 'back_writer.osm.pbf'), + thread_pool=self.thread_pool) self.overwrite = overwrite self.remove_tags = remove_tags self.id_tracker = IdTracker() @@ -102,10 +109,13 @@ def close(self) -> None: self.id_tracker.complete_backward_references(self.ref_src, relation_depth=self.relation_depth) - fp1 = FileProcessor(str(Path(self.tmpdir.name, 'back_writer.osm.pbf'))) - fp2 = FileProcessor(self.ref_src).with_filter(self.id_tracker.id_filter()) + fp1 = FileProcessor(Path(self.tmpdir.name, 'back_writer.osm.pbf'), + thread_pool=self.thread_pool) + fp2 = FileProcessor(self.ref_src, thread_pool=self.thread_pool + ).with_filter(self.id_tracker.id_filter()) - with SimpleWriter(self.outfile, overwrite=self.overwrite) as writer: + with SimpleWriter(self.outfile, overwrite=self.overwrite, + thread_pool=self.thread_pool) as writer: for o1, o2 in zip_processors(fp1, fp2): if o1: writer.add(o1) diff --git a/src/osmium/file_processor.py b/src/osmium/file_processor.py index dfb02a0..befb1b9 100644 --- a/src/osmium/file_processor.py +++ b/src/osmium/file_processor.py @@ -9,7 +9,7 @@ import osmium from osmium.index import LocationTable -from osmium.io import File, FileBuffer +from osmium.io import File, FileBuffer, ThreadPool, Reader from osmium.osm.types import OSMEntity @@ -20,7 +20,8 @@ class FileProcessor: """ def __init__(self, indata: Union[File, FileBuffer, str, 'os.PathLike[str]'], - entities: osmium.osm.osm_entity_bits = osmium.osm.ALL) -> None: + entities: osmium.osm.osm_entity_bits = osmium.osm.ALL, + thread_pool: Optional[ThreadPool] = None) -> None: """ Initialise a new file processor for the given input source _indata_. This may either be a filename, an instance of [File](IO.md#osmium.io.File) or buffered data in form of a [FileBuffer](IO.md#osmium.io.FileBuffer). @@ -30,6 +31,13 @@ def __init__(self, indata: Union[File, FileBuffer, str, 'os.PathLike[str]'], directly at the source file and will never be passed to any filters including the location and area processors. You usually should not be restricting objects, when using those. + + By default, pyosmium will create a private thread pool, which is + used to parallelize reading of the file. Alternatively you may + explicitly create a thread pool and hand it to the FileProcessor. + This may be necessary if you plan to run many processors in + parallel and want them to share a common thread pool or if you + want to change the default settings of the thread pool. """ self._file = indata self._entities = entities @@ -38,13 +46,14 @@ def __init__(self, indata: Union[File, FileBuffer, str, 'os.PathLike[str]'], self._filters: List['osmium._osmium.HandlerLike'] = [] self._area_filters: List['osmium._osmium.HandlerLike'] = [] self._filtered_handler: Optional['osmium._osmium.HandlerLike'] = None + self._thread_pool = thread_pool or ThreadPool() @property def header(self) -> osmium.io.Header: """ (read-only) [Header](IO.md#osmium.io.Header) information for the file to be read. """ - return osmium.io.Reader(self._file, osmium.osm.NOTHING).header() + return Reader(self._file, osmium.osm.NOTHING, self._thread_pool).header() @property def node_location_storage(self) -> Optional[LocationTable]: @@ -153,31 +162,28 @@ def __iter__(self) -> Iterator[OSMEntity]: handlers.append(lh) if self._area_handler is None: - reader = osmium.io.Reader(self._file, self._entities) - it = osmium.OsmFileIterator(reader, *handlers, *self._filters) - if self._filtered_handler: - it.set_filtered_handler(self._filtered_handler) - yield from it + with Reader(self._file, self._entities, thread_pool=self._thread_pool) as reader: + it = osmium.OsmFileIterator(reader, *handlers, *self._filters) + if self._filtered_handler: + it.set_filtered_handler(self._filtered_handler) + yield from it return # need areas, do two pass handling - rd = osmium.io.Reader(self._file, osmium.osm.RELATION) - try: + with Reader(self._file, osmium.osm.RELATION, thread_pool=self._thread_pool) as rd: osmium.apply(rd, *self._area_filters, self._area_handler.first_pass_handler()) - finally: - rd.close() buffer_it = osmium.BufferIterator(*self._filters) handlers.append(self._area_handler.second_pass_to_buffer(buffer_it)) - reader = osmium.io.Reader(self._file, self._entities) - it = osmium.OsmFileIterator(reader, *handlers, *self._filters) - if self._filtered_handler: - it.set_filtered_handler(self._filtered_handler) - for obj in it: - yield obj - if buffer_it: - yield from buffer_it + with Reader(self._file, self._entities, thread_pool=self._thread_pool) as reader: + it = osmium.OsmFileIterator(reader, *handlers, *self._filters) + if self._filtered_handler: + it.set_filtered_handler(self._filtered_handler) + for obj in it: + yield obj + if buffer_it: + yield from buffer_it # catch anything after the final flush if buffer_it: diff --git a/src/osmium/forward_reference_writer.py b/src/osmium/forward_reference_writer.py index b65e32c..335a26b 100644 --- a/src/osmium/forward_reference_writer.py +++ b/src/osmium/forward_reference_writer.py @@ -2,7 +2,7 @@ # # This file is part of pyosmium. (https://osmcode.org/pyosmium/) # -# Copyright (C) 2024 Sarah Hoffmann and others. +# Copyright (C) 2025 Sarah Hoffmann and others. # For a full list of authors see the git log. from typing import Any, Optional, Union from pathlib import Path @@ -11,7 +11,7 @@ from osmium._osmium import SimpleWriter from osmium import IdTracker -from osmium.io import File, FileBuffer +from osmium.io import File, FileBuffer, ThreadPool from osmium.file_processor import FileProcessor, zip_processors @@ -32,7 +32,8 @@ def __init__(self, outfile: Union[str, 'os.PathLike[str]', File], ref_src: Union[str, 'os.PathLike[str]', File, FileBuffer], overwrite: bool = False, back_references: bool = True, remove_tags: bool = True, forward_relation_depth: int = 0, - backward_relation_depth: int = 1) -> None: + backward_relation_depth: int = 1, + thread_pool: Optional[ThreadPool] = None) -> None: """ Create a new writer. `outfile` is the name of the output file to write. The file must @@ -52,7 +53,9 @@ def __init__(self, outfile: Union[str, 'os.PathLike[str]', File], """ self.outfile = outfile self.tmpdir: Optional['TemporaryDirectory[Any]'] = TemporaryDirectory() - self.writer = SimpleWriter(str(Path(self.tmpdir.name, 'forward_writer.osm.pbf'))) + self.thread_pool = thread_pool or ThreadPool() + self.writer = SimpleWriter(Path(self.tmpdir.name, 'forward_writer.osm.pbf'), + thread_pool=self.thread_pool) self.overwrite = overwrite self.back_references = back_references self.id_tracker = IdTracker() @@ -118,10 +121,13 @@ def close(self) -> None: self.ref_src, relation_depth=self.backward_relation_depth) - fp1 = FileProcessor(Path(self.tmpdir.name, 'forward_writer.osm.pbf')) - fp2 = FileProcessor(self.ref_src).with_filter(self.id_tracker.id_filter()) + fp1 = FileProcessor(Path(self.tmpdir.name, 'forward_writer.osm.pbf'), + thread_pool=self.thread_pool) + fp2 = FileProcessor(self.ref_src, thread_pool=self.thread_pool + ).with_filter(self.id_tracker.id_filter()) - with SimpleWriter(self.outfile, overwrite=self.overwrite) as writer: + with SimpleWriter(self.outfile, overwrite=self.overwrite, + thread_pool=self.thread_pool) as writer: for o1, o2 in zip_processors(fp1, fp2): if o1: writer.add(o1) diff --git a/src/osmium/helper.py b/src/osmium/helper.py index 82f7047..44e1150 100644 --- a/src/osmium/helper.py +++ b/src/osmium/helper.py @@ -56,8 +56,8 @@ class WriteHandler(SimpleWriter): documentation. """ - def __init__(self, filename: str, bufsz: int = 4096*1024, filetype: str = "") -> None: - super().__init__(filename, bufsz=bufsz, filetype=filetype) + def __init__(self, filename: str, bufsz: int = 4096*1024) -> None: + super().__init__(filename, bufsz=bufsz) def _merge_apply(self: MergeInputReader, *handlers: 'HandlerLike', diff --git a/src/osmium/io.pyi b/src/osmium/io.pyi index 59efed4..7096845 100644 --- a/src/osmium/io.pyi +++ b/src/osmium/io.pyi @@ -2,9 +2,9 @@ # # This file is part of pyosmium. (https://osmcode.org/pyosmium/) # -# Copyright (C) 2024 Sarah Hoffmann and others. +# Copyright (C) 2025 Sarah Hoffmann and others. # For a full list of authors see the git log. -from typing import Any, Union +from typing import Any, Union, Optional import os from typing_extensions import Buffer @@ -88,6 +88,50 @@ class Header: """ Set the value of header option _key_ to _value_. """ + +class ThreadPool: + """ A thread-pool for parallelizing IO operations in pyosmium. + + By default pyosmium manages thread pools for readers and writers + transparently using the default settings. For more fine-grained + control over the threads created you can instantiate a thread pool + explicitly and hand it to Readers, Writers, FileProcessors and + some other functions. + """ + + def __init__(self, num_threads: int = 0, max_queue_size: int = 0) -> None: + """ Create a new ThreadPool with the given number of threads and + a worker queue with the given maximum size. + + A negative value for 'num_threads' sets the number of cores + in the system that should be left unused. The minimum number + to use is 1. + When 'num_threads' is 0, then pyosmium tries to use the + content of OSMIUM_POOL_THREADS environment variable. When it + does not exist, -2 is used as the default (use all available + cores in the system except two). + + 'max_queue_size' is the number of data buffers that should + put at maximum in the queue for processing. When set to '0', + the size is read from the environment variable + OSMIUM_MAX_WORK_QUEUE_SIZE if available. Otherwise the default is 10. + """ + + @property + def num_threads(self) -> int: + """ Return the number of threads configured for this pool. + """ + + def queue_size(self) -> int: + """ Return the current size of the worker queue for this pool. + """ + + def queue_empty(self) -> bool: + """ Return true when there is currently no data available in the + work queue. + """ + + class Reader: """ Low-level object for reading data from an OSM file. @@ -95,20 +139,29 @@ class Reader: from the file. Use [apply][osmium.apply] for that purpose. """ def __init__(self, filename: Union[str, 'os.PathLike[str]', FileBuffer, File], - types: osm_entity_bits = ...) -> None: + types: Optional[osm_entity_bits] = None, + thread_pool: Optional[ThreadPool] = None) -> None: """ Create a new reader object. The input may either be a filename or a [File][osmium.io.File] or - [FileBuffer][osmium.io.FileBuffer] object. The _types_ parameter - defines which kinds of objects will be read from the input. Any - types not present will be skipped completely when reading the - file. Depending on the type of input, this can save quite a bit - of time. However, be careful to not skip over types that may - be referenced by other objects. For example, ways need + [FileBuffer][osmium.io.FileBuffer] object. The 'types' parameter + defines which kinds of objects will be read from the input. When + set, then any types not present will be skipped completely when + reading the file. Depending on the type of input, this can save + quite a bit of time. However, be careful to not skip over types + that may be referenced by other objects. For example, ways need nodes in order to compute their geometry. Readers may be used as a context manager. In that case, the `close()` function will be called automatically when the reader leaves the scope. + + The reader implicitly creates a private + [ThreadPool][osmium.io.ThreadPool] which it + uses to parallelize reading from the input. Alternatively you + may hand in an externally created thread pool. This may be useful + when you create many readers in parallel and want them to share + a single thread pool or when you want to customize the size + of the thread pool. """ def close(self) -> None: @@ -137,12 +190,25 @@ class Writer: for writing data. """ def __init__(self, ffile: Union[str, 'os.PathLike[str]', File], - header: Header = ...) -> None: + header: Optional[Header] = None, + overwrite: bool = False, + thread_pool: Optional[ThreadPool] = None) -> None: """ Create a new Writer. The output may either be a simple filename or a [File][osmium.io.File] object. A custom [Header][osmium.io.Header] object may be given, to customize the global file information that is written out. Be aware that not all file formats support writing out all header information. + + pyosmium will refuse to overwrite to existing files by default. + Set 'overwrite' to True to allow overwriting. + + The writer implicitly creates a private + [ThreadPool][osmium.io.ThreadPool] which it may + use to parallelize writing to the output. Alternatively you + may hand in an externally created thread pool. This may be useful + when you create many writers in parallel and want them to share + a single thread pool or when you want to customize the size + of the thread pool. """ def close(self) -> int: diff --git a/src/osmium/replication/_replication.pyi b/src/osmium/replication/_replication.pyi index f66aeb1..80c93ad 100644 --- a/src/osmium/replication/_replication.pyi +++ b/src/osmium/replication/_replication.pyi @@ -2,10 +2,15 @@ # # This file is part of pyosmium. (https://osmcode.org/pyosmium/) # -# Copyright (C) 2024 Sarah Hoffmann and others. +# Copyright (C) 2025 Sarah Hoffmann and others. # For a full list of authors see the git log. +from typing import Union import datetime +import os -def newest_change_from_file(filename: str) -> datetime.datetime: - """ Find the data of the most recent change in a file. +from ..io import Reader, File + +def newest_change_from_file(file: Union[str, 'os.PathLike[str]', File, Reader] + ) -> datetime.datetime: + """ Find the date of the most recent change in a file. """ diff --git a/src/osmium/replication/server.py b/src/osmium/replication/server.py index 22e84a4..37f2e64 100644 --- a/src/osmium/replication/server.py +++ b/src/osmium/replication/server.py @@ -288,7 +288,8 @@ def apply_diffs_to_file(self, infile: str, outfile: str, start_id: int, if diffs is None: return None - reader = oio.Reader(infile) + thread_pool = oio.ThreadPool() + reader = oio.Reader(infile, thread_pool=thread_pool) has_history = reader.header().has_multiple_object_versions h = oio.Header() @@ -310,7 +311,7 @@ def apply_diffs_to_file(self, infile: str, outfile: str, start_id: int, of = oio.File(outfile, outformat) of.has_multiple_object_versions = has_history - writer = oio.Writer(of, h) + writer = oio.Writer(of, h, thread_pool=thread_pool) LOG.debug("Merging changes into OSM file.") diff --git a/src/osmium/simple_handler.py b/src/osmium/simple_handler.py index 511a05d..240eda6 100644 --- a/src/osmium/simple_handler.py +++ b/src/osmium/simple_handler.py @@ -12,7 +12,7 @@ from ._osmium import HandlerLike from ._osmium import apply, NodeLocationsForWays -from .io import Reader, File, FileBuffer +from .io import Reader, File, FileBuffer, ThreadPool from .osm import osm_entity_bits from .area import AreaManager from .index import create_map @@ -65,16 +65,19 @@ def apply_buffer(self, buffer: 'Buffer', format: str, filters: List['HandlerLike'] = []) -> None: """Apply the handler to a string buffer. The buffer must be a byte string. + + The other parameters have the same meaning as for `apply_file()`. """ self._apply_object(FileBuffer(buffer, format), locations, idx, filters) def _apply_object(self, obj: Union[str, 'os.PathLike[str]', File, FileBuffer], locations: bool, idx: str, filters: List['HandlerLike']) -> None: + thread_pool = ThreadPool() entities = self.enabled_for() if entities & osm_entity_bits.AREA: area = AreaManager() - with Reader(obj, osm_entity_bits.RELATION) as rd: + with Reader(obj, osm_entity_bits.RELATION, thread_pool=thread_pool) as rd: apply(rd, *filters, area.first_pass_handler()) entities |= osm_entity_bits.OBJECT @@ -89,5 +92,5 @@ def _apply_object(self, obj: Union[str, 'os.PathLike[str]', File, FileBuffer], else: handlers = [*filters, self] - with Reader(obj, entities) as rd: + with Reader(obj, entities, thread_pool=thread_pool) as rd: apply(rd, *handlers) diff --git a/src/osmium/tools/pyosmium_get_changes.py b/src/osmium/tools/pyosmium_get_changes.py index 7f9d874..b837f9b 100644 --- a/src/osmium/tools/pyosmium_get_changes.py +++ b/src/osmium/tools/pyosmium_get_changes.py @@ -39,6 +39,7 @@ from ..replication import server as rserv from ..version import pyosmium_release from .. import SimpleWriter +from .. import io as oio from .common import ReplicationStart @@ -180,10 +181,11 @@ def pyosmium_get_changes(args: List[str]) -> int: log.debug("Starting download at ID %d (max %f MB)" % (startseq, options.outsize or float('inf'))) + if options.outformat is not None: - outhandler = SimpleWriter(options.outfile, filetype=options.outformat) + outfile = oio.File(options.outfile, options.outformat) else: - outhandler = SimpleWriter(options.outfile) + outfile = options.outfile if options.outsize is not None: max_size = options.outsize * 1024 @@ -200,9 +202,9 @@ def pyosmium_get_changes(args: List[str]) -> int: log.error("Cannot find the end date/ID on the server.") return 1 - endseq = svr.apply_diffs(outhandler, startseq, max_size=max_size, - end_id=end_id, simplify=options.simplify) - outhandler.close() + with SimpleWriter(outfile) as outhandler: + endseq = svr.apply_diffs(outhandler, startseq, max_size=max_size, + end_id=end_id, simplify=options.simplify) # save cookies if options.cookie: diff --git a/test/test_io.py b/test/test_io.py index 31d6d65..f14cc88 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -11,17 +11,6 @@ from helpers import CountingHandler -class NullHandler: - - def node(self, n): - pass - - -def _run_file(fn): - with osmium.io.Reader(fn) as rd: - osmium.apply(rd, NullHandler()) - - @pytest.mark.parametrize('as_string', [True, False]) def test_file_simple(tmp_path, as_string): fn = tmp_path / f"{uuid.uuid4()}.opl" @@ -48,36 +37,12 @@ def test_file_with_format(tmp_path, as_string): assert n.id == 1 -def test_node_only(test_data): - _run_file(test_data('n1')) - - -def test_way_only(test_data): - _run_file(test_data('w1 Nn1,n2,n3')) - - -def test_relation_only(test_data): - _run_file(test_data('r573 Mw1@')) - - -def test_node_with_tags(test_data): - _run_file(test_data('n32 Tbar=xx')) - - -def test_way_with_tags(test_data): - _run_file(test_data('w5666 Nn1,n2,n3 Tbar=xx')) - - -def test_relation_with_tags(test_data): - _run_file(test_data('r573 Mw1@ Tbar=xx')) - - def test_broken_timestamp(test_data): fn = test_data('n1 tx') with osmium.io.Reader(fn) as rd: with pytest.raises(RuntimeError): - osmium.apply(rd, NullHandler()) + osmium.apply(rd, CountingHandler()) @pytest.mark.parametrize('as_string', [True, False]) @@ -101,8 +66,43 @@ def test_file_header(tmp_path, as_string): def test_reader_with_filebuffer(): rd = osmium.io.Reader(osmium.io.FileBuffer('n1 x4 y1'.encode('utf-8'), 'opl')) - handler = CountingHandler() + try: + handler = CountingHandler() + + osmium.apply(rd, handler) + + assert handler.counts == [1, 0, 0, 0] + assert rd.eof() + finally: + rd.close() + + +def test_reader_with_separate_thread_pool(test_data): + with osmium.io.Reader(test_data('n1 x1 y1'), thread_pool=osmium.io.ThreadPool()) as rd: + for obj in osmium.OsmFileIterator(rd): + assert obj.id == 1 + + +@pytest.mark.parametrize("entities,expected", [(osmium.osm.NODE, [2, 0, 0, 0]), + (osmium.osm.ALL, [2, 1, 1, 0])]) +def test_reader_with_entity_filter(test_data, entities, expected): + fn = test_data("""\ + n1 x1 y2 + n2 x1 y3 + w34 Nn1,n2 + r67 Mw34@ + """) + + h = CountingHandler() + with osmium.io.Reader(fn, entities) as rd: + osmium.apply(rd, h) + + assert h.counts == expected + - osmium.apply(rd, handler) +def test_thread_pool(): + pool = osmium.io.ThreadPool(2, 15) - assert handler.counts == [1, 0, 0, 0] + assert pool.num_threads == 2 + assert pool.queue_size() == 0 + assert pool.queue_empty() diff --git a/test/test_replication.py b/test/test_replication.py index 67cdc00..bc5710d 100644 --- a/test/test_replication.py +++ b/test/test_replication.py @@ -48,14 +48,28 @@ def test_get_diff_url(inp, outp): assert outp, svr.get_diff_url(inp) -def test_get_newest_change_from_file(tmp_path): +@pytest.mark.parametrize("as_string", [True, False]) +def test_get_newest_change_from_file(tmp_path, as_string): fn = tmp_path / f"{uuid.uuid4()}.opl" fn.write_text('n6365 v1 c63965061 t2018-10-29T03:56:07Z i8369524 ux x1 y7') - val = osmium.replication.newest_change_from_file(str(fn)) + if as_string: + fn = str(fn) + + val = osmium.replication.newest_change_from_file(fn) assert val == mkdate(2018, 10, 29, 3, 56, 7) +def test_get_newest_change_from_reader(): + fb = osmium.io.FileBuffer( + 'n6365 v1 t2018-10-29T03:56:07Z x1 y7\n' + 'n6366 v1 t2018-10-29T04:56:07Z x1 y7\n'.encode('utf-8'), 'opl') + + with osmium.io.Reader(fb, thread_pool=osmium.io.ThreadPool()) as rd: + val = osmium.replication.newest_change_from_file(rd) + assert val == mkdate(2018, 10, 29, 4, 56, 7) + + def test_get_state_valid(httpserver): httpserver.expect_request('/state.txt').respond_with_data("""\ #Sat Aug 26 11:04:04 UTC 2017 diff --git a/test/test_writer.py b/test/test_writer.py index 53ea613..39d78bf 100644 --- a/test/test_writer.py +++ b/test/test_writer.py @@ -20,7 +20,7 @@ def test_writer(tmp_path): @contextmanager def _WriteExpect(filename, expected): - with osmium.SimpleWriter(str(filename), 1024*1024) as writer: + with osmium.SimpleWriter(filename, 1024*1024) as writer: yield writer assert filename.read_text().strip() == expected @@ -373,3 +373,10 @@ def test_write_to_file(tmp_path): with osmium.SimpleWriter(osmium.io.File(test_file, 'opl'), bufsz=4000) as writer: writer.add_node(osmium.osm.mutable.Node(id=123)) + + +def test_write_with_pool(tmp_path): + test_file = tmp_path / f"{uuid.uuid4()}.opl" + + with osmium.SimpleWriter(test_file, bufsz=400, thread_pool=osmium.io.ThreadPool()) as writer: + writer.add_node(osmium.osm.mutable.Node(id=123))