From c12e4c8241f41e52f39b83930cac878064ea16f5 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 09:23:25 -0300 Subject: [PATCH 01/14] Add Codspeed, update CMakeLists files and create github action file for it --- .github/workflows/codspeed.yml | 33 +++++++++++++++++++++++++++++++++ CMakeLists.txt | 5 +++-- benchmarks/CMakeLists.txt | 15 +++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/codspeed.yml diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..fcff137d4 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,33 @@ +name: CodSpeed Benchmarks + +on: + push: + branches: + - "main" + paths-ignore: + - '**.md' + - 'docs/**' + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +permissions: + contents: read + +jobs: + benchmarks: + name: Run benchmarks + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v5 + - name: Build the benchmark target(s) + run: | + cmake -DADA_TESTING=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build + cmake --build build -j + - name: Run the benchmarks + uses: CodSpeedHQ/action@v4 + with: + mode: simulation + run: cmake --build build --target run_all_benchmarks + diff --git a/CMakeLists.txt b/CMakeLists.txt index 565035b89..a0793a3ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,8 +45,9 @@ if(ADA_TESTING OR ADA_BENCHMARKS OR ADA_TOOLS) if(ADA_BENCHMARKS AND (CMAKE_SIZEOF_VOID_P EQUAL 8)) CPMAddPackage( NAME benchmark - GITHUB_REPOSITORY google/benchmark - VERSION 1.9.0 + GITHUB_REPOSITORY CodSpeedHQ/codspeed-cpp + VERSION 2.0.0 + SOURCE_SUBDIR google_benchmark OPTIONS "BENCHMARK_ENABLE_TESTING OFF" "BENCHMARK_ENABLE_INSTALL OFF" "BENCHMARK_ENABLE_WERROR OFF" diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index c54eb5d06..b8f84d159 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -64,6 +64,21 @@ target_link_libraries(percent_encode PRIVATE benchmark::benchmark) target_link_libraries(bench_search_params PRIVATE benchmark::benchmark) target_link_libraries(urlpattern PRIVATE benchmark::benchmark) +set(BENCHMARKS wpt_bench bench benchdata bbc_bench percent_encode bench_search_params urlpattern) + +add_custom_target(run_all_benchmarks + COMMAND ${CMAKE_COMMAND} -E echo "Running all benchmarks..." +) +foreach(benchmark IN LISTS BENCHMARKS) + add_custom_command( + TARGET run_all_benchmarks + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E echo "Running ${benchmark}..." + COMMAND $ + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) +endforeach() + option(ADA_COMPETITION "Whether to install various competitors." OFF) # We only build url_whatwg if ICU is found, so we need to make From f54e6b8cea8bb74dcbbd342c7d0d03b178832134 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 09:32:35 -0300 Subject: [PATCH 02/14] use specific compiler version --- .github/workflows/codspeed.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index fcff137d4..03e7f6197 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -25,6 +25,8 @@ jobs: run: | cmake -DADA_TESTING=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j + env: + CXX: g++-12 - name: Run the benchmarks uses: CodSpeedHQ/action@v4 with: From 95ea32317b6be0c0022aacae104d5729bf0719b3 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 09:55:51 -0300 Subject: [PATCH 03/14] only use codspeed if CODSPEED_MODE flag is provided --- CMakeLists.txt | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a0793a3ea..03c79c620 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,16 +43,28 @@ if(ADA_TESTING OR ADA_BENCHMARKS OR ADA_TOOLS) endif() # We use Google Benchmark, but it does not build under several 32-bit systems. if(ADA_BENCHMARKS AND (CMAKE_SIZEOF_VOID_P EQUAL 8)) - CPMAddPackage( - NAME benchmark - GITHUB_REPOSITORY CodSpeedHQ/codspeed-cpp - VERSION 2.0.0 - SOURCE_SUBDIR google_benchmark - OPTIONS "BENCHMARK_ENABLE_TESTING OFF" - "BENCHMARK_ENABLE_INSTALL OFF" - "BENCHMARK_ENABLE_WERROR OFF" - - ) + if(DEFINED CODSPEED_MODE) + message(STATUS "Using CodSpeed-instrumented Google Benchmark") + CPMAddPackage( + NAME benchmark + GITHUB_REPOSITORY CodSpeedHQ/codspeed-cpp + VERSION 2.0.0 + SOURCE_SUBDIR google_benchmark + OPTIONS "BENCHMARK_ENABLE_TESTING OFF" + "BENCHMARK_ENABLE_INSTALL OFF" + "BENCHMARK_ENABLE_WERROR OFF" + ) + else() + message(STATUS "Using standard Google Benchmark") + CPMAddPackage( + NAME benchmark + GITHUB_REPOSITORY google/benchmark + VERSION 1.9.0 + OPTIONS "BENCHMARK_ENABLE_TESTING OFF" + "BENCHMARK_ENABLE_INSTALL OFF" + "BENCHMARK_ENABLE_WERROR OFF" + ) + endif() endif() if (ADA_TESTING AND NOT EMSCRIPTEN) From b24c1bccc7886c22c81ec8f058da87eb540713c1 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 14:42:13 -0300 Subject: [PATCH 04/14] update ubuntu version, actions/checkout, and add -DADA_DEVELOPMENT_CHECKS=OFF --- .github/workflows/codspeed.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 03e7f6197..52f532aac 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -18,12 +18,12 @@ permissions: jobs: benchmarks: name: Run benchmarks - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6.0.0 - name: Build the benchmark target(s) run: | - cmake -DADA_TESTING=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build + cmake -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j env: CXX: g++-12 From a07d7ac0bb74bcee636307b4c4f9cadbfaaf0c48 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 14:52:08 -0300 Subject: [PATCH 05/14] add ADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 52f532aac..ec63418aa 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v6.0.0 - name: Build the benchmark target(s) run: | - cmake -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build + cmake -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j env: CXX: g++-12 From 3a285b0962b3c20ec380fc2cbea7949f44fd9a9d Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 14:56:35 -0300 Subject: [PATCH 06/14] use more recent clang version --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index ec63418aa..82a2a6834 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -26,7 +26,7 @@ jobs: cmake -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j env: - CXX: g++-12 + CXX: clang++-15 - name: Run the benchmarks uses: CodSpeedHQ/action@v4 with: From d514a10134b796b438df5ee00f19d5f75a9627ba Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 15:01:32 -0300 Subject: [PATCH 07/14] update actions/checkout version --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 82a2a6834..bf2a7e510 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -20,7 +20,7 @@ jobs: name: Run benchmarks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Build the benchmark target(s) run: | cmake -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build From cecd98ffad867a1a7d3c239929e6a9c1153675b0 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 15:02:44 -0300 Subject: [PATCH 08/14] use clang14 --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index bf2a7e510..150e5624e 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -26,7 +26,7 @@ jobs: cmake -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j env: - CXX: clang++-15 + CXX: clang++-14 - name: Run the benchmarks uses: CodSpeedHQ/action@v4 with: From 15cf7a0702aeb80b19b5e375bd186bb2e8bb3538 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 15:30:59 -0300 Subject: [PATCH 09/14] add ADA_USE_UNSAFE_STD_REGEX_PROVIDER --- .github/workflows/codspeed.yml | 2 +- CMakeLists.txt | 2 +- benchmarks/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 150e5624e..06cdeb7eb 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Build the benchmark target(s) run: | - cmake -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build + cmake -DCMAKE_BUILD_TYPE=Release -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j env: CXX: clang++-14 diff --git a/CMakeLists.txt b/CMakeLists.txt index 03c79c620..9ee0fea2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,7 +103,7 @@ endif() add_library(ada::ada ALIAS ada) -if(ADA_TESTING) +if(ADA_TESTING OR ADA_USE_UNSAFE_STD_REGEX_PROVIDER) # IMPORTANT! # # We enable std_regex_provider for testing purposes diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index b8f84d159..4d85f5196 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -74,7 +74,7 @@ foreach(benchmark IN LISTS BENCHMARKS) TARGET run_all_benchmarks POST_BUILD COMMAND ${CMAKE_COMMAND} -E echo "Running ${benchmark}..." - COMMAND $ + COMMAND $ --benchmark_min_time=1.0 WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) endforeach() From 4ed8780413b2338ec22c126710803662196e667a Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 15:34:37 -0300 Subject: [PATCH 10/14] use clang++-17 --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 06cdeb7eb..ac9729d85 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -26,7 +26,7 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j env: - CXX: clang++-14 + CXX: clang++-17 - name: Run the benchmarks uses: CodSpeedHQ/action@v4 with: From bb9058b05cc8278e1c29e9b2e63bc7224a7abca9 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 15:37:54 -0300 Subject: [PATCH 11/14] use g++-12 --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index ac9729d85..586973af3 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -26,7 +26,7 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j env: - CXX: clang++-17 + CXX: g++-12 - name: Run the benchmarks uses: CodSpeedHQ/action@v4 with: From 7b8e625588fa66a8fe974521b3f55b0d1e28421e Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 15:41:09 -0300 Subject: [PATCH 12/14] fix value passed to benchmark_min_time --- benchmarks/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 4d85f5196..d9af9e637 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -74,7 +74,7 @@ foreach(benchmark IN LISTS BENCHMARKS) TARGET run_all_benchmarks POST_BUILD COMMAND ${CMAKE_COMMAND} -E echo "Running ${benchmark}..." - COMMAND $ --benchmark_min_time=1.0 + COMMAND $ --benchmark_min_time=1s WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) endforeach() From 5373bafaed7545741c781d25998adaf39429f5e7 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 15:50:36 -0300 Subject: [PATCH 13/14] use just clang++ --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 586973af3..d9bc76960 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -26,7 +26,7 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j env: - CXX: g++-12 + CXX: clang++ - name: Run the benchmarks uses: CodSpeedHQ/action@v4 with: From b30de36414bd6998cf7b56ce7bf5586f6cfb3f86 Mon Sep 17 00:00:00 2001 From: Carlos Sousa Date: Wed, 3 Dec 2025 15:54:07 -0300 Subject: [PATCH 14/14] back to g++-12 --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index d9bc76960..586973af3 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -26,7 +26,7 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release -DADA_TESTING=OFF -DADA_DEVELOPMENT_CHECKS=OFF -DADA_USE_UNSAFE_STD_REGEX_PROVIDER=ON -DADA_BENCHMARKS=ON -DCODSPEED_MODE=simulation -G Ninja -B build cmake --build build -j env: - CXX: clang++ + CXX: g++-12 - name: Run the benchmarks uses: CodSpeedHQ/action@v4 with: