Skip to content

Commit b00c770

Browse files
Alex-PLACETAlex-PLACET
authored
Add integration tests (#61)
1 parent 93c2fe8 commit b00c770

40 files changed

+2624
-121
lines changed

‎.github/copilot-instructions.md‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Sparrow-IPC AI Agent Instructions
2+
3+
C++20 library for Arrow IPC serialization/deserialization using FlatBuffers. See [../examples/write_and_read_streams.cpp](../examples/write_and_read_streams.cpp) for usage patterns.
4+
5+
## Architecture
6+
7+
- **Serialization**: `record_batch` → `serializer` → FlatBuffer metadata + body → stream (continuation bytes + length + message + padding + data)
8+
- **Deserialization**: Binary stream → `extract_encapsulated_message()` → parse FlatBuffer → reconstruct `record_batch`
9+
- **Critical**: All record batches in a stream must have identical schemas (validated in `serialize_record_batches_to_ipc_stream`)
10+
- **Memory model**: Deserialized arrays use `std::span<const uint8_t>` - source buffer must outlive arrays
11+
12+
## Build System
13+
14+
**Dependency fetching** (unique pattern in `cmake/external_dependencies.cmake`):
15+
- `FETCH_DEPENDENCIES_WITH_CMAKE=OFF` - require via `find_package()` (CI default)
16+
- `FETCH_DEPENDENCIES_WITH_CMAKE=MISSING` - auto-fetch missing (local dev)
17+
- All binaries/libs → `${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}` (not standard locations)
18+
19+
**FlatBuffer schemas**: Auto-downloaded from Apache Arrow during configure → `${CMAKE_BINARY_DIR}/generated/*_generated.h`. Never edit generated headers.
20+
21+
**Build**:
22+
```bash
23+
mamba env create -f environment-dev.yml && mamba activate sparrow-ipc
24+
cmake -B build -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -DSPARROW_IPC_BUILD_TESTS=ON
25+
cmake --build build -j12
26+
cmake --build build --target run_tests
27+
```
28+
29+
## Platform-Specific Patterns
30+
31+
**Linux executables linking sparrow-ipc**: Must set RPATH (libs in same dir):
32+
```cmake
33+
set_target_properties(my_exe PROPERTIES
34+
BUILD_RPATH_USE_ORIGIN ON
35+
BUILD_RPATH "$ORIGIN"
36+
INSTALL_RPATH "$ORIGIN")
37+
```
38+
See `integration_tests/CMakeLists.txt` for examples. Missing this causes "cannot open shared object file" errors.
39+
40+
**Windows**: Explicit DLL copying in CMakeLists (see `tests/CMakeLists.txt:32-47`).
41+
42+
## Testing
43+
44+
- Arrow test data: Auto-fetched from `apache/arrow-testing`, `.json.gz` files extracted during configure
45+
- Unit tests: `cmake --build build --target run_tests`
46+
- Integration tests: `integration_tests/` tools integrate with Apache Arrow's Archery framework via Docker
47+
48+
## Naming & Style
49+
50+
- `snake_case` for everything (types, functions)
51+
- `m_` prefix for members
52+
- Namespace: `sparrow_ipc`
53+
- Format: `cmake --build build --target clang-format` (requires `ACTIVATE_LINTER=ON`)

‎.github/workflows/conan.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
uses: mozilla-actions/sccache-action@v0.0.9
1515

1616
- name: Checkout repository
17-
uses: actions/checkout@v4
17+
uses: actions/checkout@v6
1818

1919
- name: Setup Conan Environment
2020
uses: hankhsu1996/setup-conan@v1.1.0

‎.github/workflows/docs.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010

1111
steps:
1212
- name: Checkout repository
13-
uses: actions/checkout@v4
13+
uses: actions/checkout@v6
1414

1515
- name: Set conda environment
1616
uses: mamba-org/setup-micromamba@main

‎.github/workflows/integration_tests.yaml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: Integration tests
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
push:
7+
branches: [main]
8+
9+
jobs:
10+
build_integration_container_and_run_tests:
11+
runs-on: ubuntu-22.04
12+
steps:
13+
- name: Install dependencies
14+
run: |
15+
sudo apt-get update
16+
sudo apt-get install -y mold libpthread-stubs0-dev libboost-thread-dev doctest-dev
17+
18+
- name: Checkout code
19+
uses: actions/checkout@v6
20+
21+
- name: Configure using CMake
22+
run: |
23+
cmake -G Ninja \
24+
-Bbuild \
25+
-DCMAKE_BUILD_TYPE:STRING=RELEASE \
26+
-DSPARROW_IPC_BUILD_INTEGRATION_TESTS=ON \
27+
-DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING \
28+
-DSPARROW_IPC_BUILD_SHARED=OFF
29+
30+
- name: Build arrow_file_to_stream target
31+
working-directory: build
32+
run: cmake --build . --config Release --target arrow_file_to_stream
33+
34+
- name: Build arrow_stream_to_file target
35+
working-directory: build
36+
run: cmake --build . --config Release --target arrow_stream_to_file
37+
38+
- name: Build arrow_json_to_file target
39+
working-directory: build
40+
run: cmake --build . --config Release --target arrow_json_to_file
41+
42+
- name: Build arrow_validate target
43+
working-directory: build
44+
run: cmake --build . --config Release --target arrow_validate
45+
46+
47+
- name: Build Docker image
48+
run: docker build -t sparrow/integration-tests -f ci/docker/integration.dockerfile .
49+
50+
- name: Run Integration tests
51+
run: |
52+
docker run --rm \
53+
-e ARCHERY_INTEGRATION_WITH_EXTERNAL_LIBRARY=/workspace/build/bin/RELEASE/ \
54+
-e ARCHERY_INTEGRATION_EXTERNAL_LIBRARY_IPC_PRODUCER=true \
55+
-e ARCHERY_INTEGRATION_EXTERNAL_LIBRARY_IPC_CONSUMER=true \
56+
-v ${{ github.workspace }}:/workspace \
57+
-w /arrow-integration sparrow/integration-tests \
58+
"/arrow-integration/ci/scripts/integration_arrow.sh /arrow-integration /build"

‎.github/workflows/linux.yml‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
build_shared: [ON, OFF]
2020
steps:
2121
- name: Checkout repository
22-
uses: actions/checkout@v4
22+
uses: actions/checkout@v6
2323

2424
- name: Create build environment
2525
uses: mamba-org/setup-micromamba@v2
@@ -37,6 +37,7 @@ jobs:
3737
-DCMAKE_PREFIX_PATH=$CONDA_PREFIX \
3838
-DSPARROW_IPC_BUILD_SHARED=${{ matrix.build_shared }} \
3939
-DSPARROW_IPC_BUILD_TESTS=ON \
40+
-DSPARROW_IPC_BUILD_INTEGRATION_TESTS=ON \
4041
-DSPARROW_IPC_BUILD_EXAMPLES=ON
4142
4243
- name: Build sparrow-ipc
@@ -59,6 +60,14 @@ jobs:
5960
working-directory: build
6061
run: cmake --build . --target run_example
6162

63+
- name: Build test_integration_tools
64+
working-directory: build
65+
run: cmake --build . --target test_integration_tools
66+
67+
- name: Run test_integration_tools
68+
working-directory: build
69+
run: cmake --build . --target run_test_integration_tools
70+
6271
- name: Install
6372
working-directory: build
6473
run: cmake --install .
@@ -71,7 +80,7 @@ jobs:
7180
build_shared: [ON, OFF]
7281
steps:
7382
- name: Checkout repository
74-
uses: actions/checkout@v4
83+
uses: actions/checkout@v6
7584

7685
- name: Configure using cmake
7786
run: |
@@ -81,6 +90,7 @@ jobs:
8190
-DSPARROW_IPC_BUILD_SHARED=${{ matrix.build_shared }} \
8291
-DSPARROW_IPC_BUILD_TESTS=ON \
8392
-DSPARROW_IPC_BUILD_EXAMPLES=ON \
93+
-DSPARROW_IPC_BUILD_INTEGRATION_TESTS=ON \
8494
-DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING
8595
8696
- name: Build sparrow-ipc
@@ -103,6 +113,14 @@ jobs:
103113
working-directory: build
104114
run: cmake --build . --target run_example
105115

116+
- name: Build test_integration_tools
117+
working-directory: build
118+
run: cmake --build . --target test_integration_tools
119+
120+
- name: Run test_integration_tools
121+
working-directory: build
122+
run: cmake --build . --target run_test_integration_tools
123+
106124
- name: Install
107125
working-directory: build
108126
run: sudo cmake --install .

‎.github/workflows/osx.yml‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
build_shared: [ON, OFF]
2020
steps:
2121
- name: Checkout repository
22-
uses: actions/checkout@v4
22+
uses: actions/checkout@v6
2323

2424
- name: Select XCode version
2525
run: |
@@ -42,6 +42,7 @@ jobs:
4242
-DCMAKE_PREFIX_PATH=$CONDA_PREFIX \
4343
-DSPARROW_IPC_BUILD_SHARED=${{ matrix.build_shared }} \
4444
-DSPARROW_IPC_BUILD_TESTS=ON \
45+
-DSPARROW_IPC_BUILD_INTEGRATION_TESTS=ON \
4546
-DSPARROW_IPC_BUILD_EXAMPLES=ON
4647
4748
- name: Build sparrow-ipc
@@ -64,6 +65,14 @@ jobs:
6465
working-directory: build
6566
run: cmake --build . --target run_example
6667

68+
- name: Build test_integration_tools
69+
working-directory: build
70+
run: cmake --build . --target test_integration_tools
71+
72+
- name: Run test_integration_tools
73+
working-directory: build
74+
run: cmake --build . --target run_test_integration_tools
75+
6776
- name: Install
6877
working-directory: build
6978
run: cmake --install .
@@ -76,7 +85,7 @@ jobs:
7685
build_shared: [ON, OFF]
7786
steps:
7887
- name: Checkout repository
79-
uses: actions/checkout@v4
88+
uses: actions/checkout@v6
8089

8190
- name: Select XCode version
8291
run: |
@@ -90,6 +99,7 @@ jobs:
9099
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
91100
-DSPARROW_IPC_BUILD_SHARED=${{ matrix.build_shared }} \
92101
-DSPARROW_IPC_BUILD_TESTS=ON \
102+
-DSPARROW_IPC_BUILD_INTEGRATION_TESTS=ON \
93103
-DSPARROW_IPC_BUILD_EXAMPLES=ON \
94104
-DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING
95105
@@ -113,6 +123,14 @@ jobs:
113123
working-directory: build
114124
run: cmake --build . --target run_example
115125

126+
- name: Build test_integration_tools
127+
working-directory: build
128+
run: cmake --build . --target test_integration_tools
129+
130+
- name: Run test_integration_tools
131+
working-directory: build
132+
run: cmake --build . --target run_test_integration_tools
133+
116134
- name: Install
117135
working-directory: build
118136
run: sudo cmake --install .

‎.github/workflows/release.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030

3131
steps:
3232
- name: Checkout
33-
uses: actions/checkout@v4
33+
uses: actions/checkout@v6
3434
with:
3535
token: ${{ secrets.GITHUB_TOKEN }}
3636

‎.github/workflows/windows.yml‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
build_shared: [ON, OFF]
2020
steps:
2121
- name: Checkout repository
22-
uses: actions/checkout@v4
22+
uses: actions/checkout@v6
2323

2424
- name: Create build environment
2525
uses: mamba-org/setup-micromamba@v2
@@ -41,6 +41,7 @@ jobs:
4141
-DCMAKE_PREFIX_PATH=$GLOB_PREFIX_PATH \
4242
-DSPARROW_IPC_BUILD_SHARED=${{ matrix.build_shared }} \
4343
-DSPARROW_IPC_BUILD_TESTS=ON \
44+
-DSPARROW_IPC_BUILD_INTEGRATION_TESTS=ON \
4445
-DSPARROW_IPC_BUILD_EXAMPLES=ON
4546
4647
- name: Build sparrow-ipc
@@ -64,6 +65,14 @@ jobs:
6465
working-directory: build
6566
run: cmake --build . --config ${{ matrix.build_type }} --target run_example
6667

68+
- name: Build test_integration_tools
69+
working-directory: build
70+
run: cmake --build . --config ${{ matrix.build_type }} --target test_integration_tools
71+
72+
- name: Run test_integration_tools
73+
working-directory: build
74+
run: cmake --build . --config ${{ matrix.build_type }} --target run_test_integration_tools
75+
6776
- name: Install
6877
working-directory: build
6978
run: cmake --install . --config ${{ matrix.build_type }}
@@ -76,7 +85,7 @@ jobs:
7685
build_shared: [ON, OFF]
7786
steps:
7887
- name: Checkout repository
79-
uses: actions/checkout@v4
88+
uses: actions/checkout@v6
8089

8190
- name: Enable tests coverage
8291
if: matrix.build_type == 'Debug'
@@ -90,6 +99,7 @@ jobs:
9099
-DSPARROW_IPC_BUILD_SHARED=${{ matrix.build_shared }} \
91100
-DSPARROW_IPC_BUILD_TESTS=ON \
92101
-DSPARROW_IPC_BUILD_EXAMPLES=ON \
102+
-DSPARROW_IPC_BUILD_INTEGRATION_TESTS=ON \
93103
-DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING \
94104
$TEST_COVERAGE_ACTIVATION
95105
@@ -113,6 +123,14 @@ jobs:
113123
working-directory: build
114124
run: cmake --build . --config ${{ matrix.build_type }} --target run_example
115125

126+
- name: Build test_integration_tools
127+
working-directory: build
128+
run: cmake --build . --config ${{ matrix.build_type }} --target test_integration_tools
129+
130+
- name: Run test_integration_tools
131+
working-directory: build
132+
run: cmake --build . --config ${{ matrix.build_type }} --target run_test_integration_tools
133+
116134
- name: Install
117135
working-directory: build
118136
run: cmake --install . --config ${{ matrix.build_type }}

‎CMakeLists.txt‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ include(CMakeDependentOption)
1111
list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
1212
message(DEBUG "CMake module path: ${CMAKE_MODULE_PATH}")
1313

14-
include(external_dependencies)
15-
1614
set(SPARROW_IPC_COMPILE_DEFINITIONS "" CACHE STRING "List of public compile definitions of the sparrow-ipc target")
1715

1816
set(SPARROW_IPC_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
@@ -85,11 +83,27 @@ MESSAGE(STATUS "🔧 Build docs: ${SPARROW_IPC_BUILD_DOCS}")
8583
OPTION(SPARROW_IPC_BUILD_EXAMPLES "Build sparrow-ipc examples" OFF)
8684
MESSAGE(STATUS "🔧 Build examples: ${SPARROW_IPC_BUILD_EXAMPLES}")
8785

86+
OPTION(SPARROW_IPC_BUILD_INTEGRATION_TESTS "Build sparrow-ipc integration tests" OFF)
87+
MESSAGE(STATUS "🔧 Build integration tests: ${SPARROW_IPC_BUILD_INTEGRATION_TESTS}")
88+
8889
# Code coverage
8990
# =============
9091
OPTION(SPARROW_IPC_ENABLE_COVERAGE "Enable sparrow-ipc test coverage" OFF)
9192
MESSAGE(STATUS "🔧 Enable coverage: ${SPARROW_IPC_ENABLE_COVERAGE}")
9293

94+
include(external_dependencies)
95+
96+
# Build
97+
# =====
98+
set(BINARY_BUILD_DIR "${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE}")
99+
100+
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG "${BINARY_BUILD_DIR}")
101+
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE "${BINARY_BUILD_DIR}")
102+
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG "${BINARY_BUILD_DIR}")
103+
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "${BINARY_BUILD_DIR}")
104+
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG "${BINARY_BUILD_DIR}")
105+
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE "${BINARY_BUILD_DIR}")
106+
93107
if(SPARROW_IPC_ENABLE_COVERAGE)
94108
include(code_coverage)
95109
endif()
@@ -287,6 +301,13 @@ if(SPARROW_IPC_BUILD_EXAMPLES)
287301
add_subdirectory(examples)
288302
endif()
289303

304+
# Integration tests
305+
# =================
306+
if(SPARROW_IPC_BUILD_INTEGRATION_TESTS)
307+
message(STATUS "🔨 Create integration tests targets")
308+
add_subdirectory(integration_tests)
309+
endif()
310+
290311
# Installation
291312
# ============
292313
include(GNUInstallDirs)

0 commit comments

Comments
 (0)