Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/build_wheels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ jobs:
CIBW_TEST_REQUIRES: pytest pytest-httpserver
CIBW_TEST_COMMAND: pytest {project}/test
CIBW_BUILD_FRONTEND: build
CIBW_BEFORE_BUILD_LINUX: yum install -y expat-devel boost-devel zlib-devel bzip2-devel lz4-devel
CIBW_BEFORE_BUILD_MACOS: brew install boost
CIBW_BEFORE_BUILD_LINUX: yum install -y expat-devel zlib-devel bzip2-devel lz4-devel
CIBW_BEFORE_BUILD_MACOS: brew install
CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=11.0 SKBUILD_CMAKE_ARGS=-DWITH_LZ4=OFF
CIBW_BEFORE_BUILD_WINDOWS: vcpkg install bzip2 expat zlib boost-variant boost-iterator lz4
CIBW_BEFORE_BUILD_WINDOWS: vcpkg install bzip2 expat zlib lz4
CIBW_ENVIRONMENT_WINDOWS: 'CMAKE_TOOLCHAIN_FILE="C:/vcpkg/scripts/buildsystems/vcpkg.cmake"'
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: 'pipx run delvewheel repair --add-path ${{ matrix.vcpkg }}/bin/ --add-path ${{ matrix.vcpkg }}/debug/bin -w {dest_dir} {wheel}'

Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
- name: Install packages
run: |
sudo apt-get update -y -qq
sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev pipx
sudo apt-get install -y -qq libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev pipx
pipx install mypy
pipx inject mypy types-requests
pipx install flake8
Expand Down Expand Up @@ -116,7 +116,7 @@ jobs:
- name: Install packages
run: |
sudo apt-get update -y -qq
sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev pipx
sudo apt-get install -y -qq libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev pipx
pipx install mypy
pipx inject mypy types-requests
pipx install flake8
Expand Down Expand Up @@ -329,11 +329,11 @@ jobs:
- name: Install packages
run: |
sudo apt-get update -y -qq
sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev virtualenv
sudo apt-get install -y -qq libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev virtualenv
if: ${{ matrix.flavour == 'linux' }}

- name: Install packages
run: brew install boost geos virtualenv
run: brew install geos virtualenv
shell: bash
if: ${{ matrix.flavour == 'macos' }}

Expand Down Expand Up @@ -404,7 +404,7 @@ jobs:
shell: bash

- name: Install packages
run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows
run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows lz4:x86-windows
shell: bash

- name: Set up Python 3.8
Expand Down Expand Up @@ -531,7 +531,7 @@ jobs:

- name: Install packages
run: |
vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows
vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows lz4:x86-windows
shell: bash

- name: Set up Python 3.13t
Expand Down
3 changes: 0 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,6 @@ endif()

message(STATUS "Building in C++${CMAKE_CXX_STANDARD} mode")

find_package(Boost 1.66 REQUIRED COMPONENTS)
include_directories(SYSTEM ${Boost_INCLUDE_DIRS})

# Modules without any Python code and just one source file.
foreach(PYMOD geom index io area)
pybind11_add_module(${PYMOD} lib/${PYMOD}.cc)
Expand Down
7 changes: 2 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ For other versions, a source wheel is provided. Make sure to install all
external dependencies first. On Debian/Ubuntu-like systems, the following
command installs all required packages:

sudo apt-get install build-essential cmake libboost-dev \
libexpat1-dev zlib1g-dev libbz2-dev
sudo apt-get install build-essential cmake libexpat1-dev zlib1g-dev libbz2-dev


### Installing from source
Expand All @@ -45,16 +44,14 @@ pyosmium has the following dependencies:
* [expat](https://libexpat.github.io/)
* [libz](https://www.zlib.net/)
* [libbz2](https://www.sourceware.org/bzip2/)
* [Boost](https://www.boost.org/) variant and iterator >= 1.70
* [Python Requests](https://docs.python-requests.org/)
* [scikit-build-core](https://scikit-build-core.readthedocs.io)
* a C++17-compatible compiler (Clang 13+, GCC 10+ are supported)


### Compiling from Source

Make sure to install the development packages for expat, libz, libbz2
and boost.
Make sure to install the development packages for expat, libz and libbz2.

The appropriate versions for Libosmium and Protozero will be downloaded into
the `contrib` directory when building the source package:
Expand Down
5 changes: 2 additions & 3 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,10 @@ pyosmium can be installed with pip:

The Pypi source package already comes bundled with a matching version of
libosmium, protozero and pybind11. Pyosmium additionally depends on
expat, libz, libbz2 and Boost variant and iterator. You need to install
expat, libz and libbz2. You need to install
development packages for these libraries. On Debian/Ubuntu do::

sudo apt-get install build-essential cmake libboost-dev \
libexpat1-dev zlib1g-dev libbz2-dev
sudo apt-get install build-essential cmake libexpat1-dev zlib1g-dev libbz2-dev


Python >= 3.8 is supported. Pypy is known not to work.
Expand Down
3 changes: 1 addition & 2 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ the following additional dependencies need to be available:
* [expat](https://libexpat.github.io/)
* [libz](https://www.zlib.net/)
* [libbz2](https://www.sourceware.org/bzip2/)
* [Boost](https://www.boost.org/) variant and iterator >= 1.41
* [Python Requests](https://docs.python-requests.org/en/master/)
* a recent C++ compiler (Clang 3.4+, GCC 4.8+)

Expand All @@ -41,7 +40,7 @@ of the build process:
On Debian/Ubuntu-like systems, the following command installs all required
packages:

sudo apt-get install python3-dev build-essential cmake libboost-dev \
sudo apt-get install python3-dev build-essential cmake \
libexpat1-dev zlib1g-dev libbz2-dev

Compatible versions of libosmium and protozero are shipped with the source
Expand Down
19 changes: 9 additions & 10 deletions lib/merge_input_reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,13 @@
*
* This file is part of pyosmium. (https://osmcode.org/pyosmium/)
*
* Copyright (C) 2025 Sarah Hoffmann <lonvia@denofr.de> and others.
* Copyright (C) 2026 Sarah Hoffmann <lonvia@denofr.de> and others.
* For a full list of authors see the git log.
*/
#include <pybind11/pybind11.h>

#include <vector>

#include <boost/iterator/function_output_iterator.hpp>
#include <iterator>

#include <osmium/osm/object_comparisons.hpp>
#include <osmium/io/any_input.hpp>
Expand All @@ -30,20 +29,21 @@ namespace {
/**
* Copy the first OSM object with a given Id to the output. Keep
* track of the Id of each object to do this.
*
* We are using this functor class instead of a simple lambda, because the
* lambda doesn't build on MSVC.
*/
class copy_first_with_id {
osmium::io::Writer* writer;
osmium::object_id_type id = 0;

public:
using iterator_category = std::output_iterator_tag;
using value_type = const osmium::OSMObject;
using reference = const osmium::OSMObject&;

explicit copy_first_with_id(osmium::io::Writer& w) :
writer(&w) {
}

void operator()(const osmium::OSMObject& obj) {
void push_back(const osmium::OSMObject& obj) {
if (obj.id() != id) {
if (obj.visible()) {
(*writer)(obj);
Expand Down Expand Up @@ -112,9 +112,8 @@ class MergeInputReader
std::reverse(objects.ptr_begin(), objects.ptr_end());
objects.sort(osmium::object_order_type_id_reverse_version());

auto output_it = boost::make_function_output_iterator(
copy_first_with_id(*writer.get())
);
copy_first_with_id copy_first{*writer.get()};
auto output_it = std::back_inserter(copy_first);

std::set_union(objects.begin(),
objects.end(),
Expand Down
101 changes: 101 additions & 0 deletions test/test_merge_input_reader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
# SPDX-License-Identifier: BSD-2-Clause
#
# This file is part of pyosmium. (https://osmcode.org/pyosmium/)
#
# Copyright (C) 2026 Sarah Hoffmann <lonvia@denofr.de> and others.
# For a full list of authors see the git log.
from textwrap import dedent
import uuid

import pytest

import osmium


class ListHandler:
def __init__(self):
self.data = []

def node(self, o):
self.data.append(f"N{o.id}/{o.version}")

def way(self, o):
self.data.append(f"W{o.id}/{o.version}")

def relation(self, o):
self.data.append(f"R{o.id}/{o.version}")

def area(self, _):
assert not "Area handler should not be called"


def add_as_buffer(mir, opl, tmp_path):
mir.add_buffer(dedent(opl).encode('utf-8'), format='opl')


def add_as_file(mir, opl, tmp_path):
fn = tmp_path / (str(uuid.uuid4()) + '.opl')
fn.write_text(dedent(opl))

mir.add_file(str(fn))


@pytest.mark.parametrize('adder', [add_as_buffer, add_as_file])
def test_simple_input(adder, tmp_path):
mir = osmium.MergeInputReader()

opl = """\
n1 v1 x1 y1
w3 v34 Nn1
"""

adder(mir, opl, tmp_path)

h = ListHandler()
mir.apply(h)

assert h.data == ['N1/1', 'W3/34']


@pytest.mark.parametrize('adder', [add_as_buffer, add_as_file])
def test_multibuffer_no_simplify(adder, tmp_path):
mir = osmium.MergeInputReader()

opls = ["""\
n1 v1 x1 y1
w3 v2 Nn1
""",
"""\
n2 v1 x1 y1
w3 v1 Nn2
"""]

for opl in opls:
adder(mir, opl, tmp_path)

h = ListHandler()
mir.apply(h, simplify=False)

assert h.data == ['N1/1', 'N2/1', 'W3/1', 'W3/2']


@pytest.mark.parametrize('adder', [add_as_buffer, add_as_file])
def test_multibuffer_simplify(adder, tmp_path):
mir = osmium.MergeInputReader()

opls = ["""\
n1 v1 x1 y1
w3 v2 Nn1
""",
"""\
n2 v1 x1 y1
w3 v1 Nn2
"""]

for opl in opls:
adder(mir, opl, tmp_path)

h = ListHandler()
mir.apply(h, simplify=True)

assert h.data == ['N1/1', 'N2/1', 'W3/2']