From 3f61b63b6f344bbb9c7cfb1c7e27996d28276646 Mon Sep 17 00:00:00 2001 From: konard Date: Sat, 27 Dec 2025 12:10:44 +0100 Subject: [PATCH 1/5] Initial commit with task details Adding CLAUDE.md with task information for AI processing. This file will be removed when the task is complete. Issue: https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/issues/13 --- CLAUDE.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..82df866 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +Issue to solve: https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/issues/13 +Your prepared branch: issue-13-23f296f3c251 +Your prepared working directory: /tmp/gh-issue-solver-1766833843184 + +Proceed. \ No newline at end of file From b8573e91a0651496397c44c4fd3be14ae1b79bf8 Mon Sep 17 00:00:00 2001 From: konard Date: Sat, 27 Dec 2025 12:15:26 +0100 Subject: [PATCH 2/5] Apply best practices from Comparisons.Neo4jVSDoublets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: - Update rust.yml workflow: - Use actions/checkout@v4 (from v2) - Use dtolnay/rust-toolchain (from deprecated actions-rs/toolchain) - Run cargo directly instead of deprecated actions-rs/cargo - Add PostgreSQL readiness wait step - Add BENCHMARK_LINK_COUNT env var (1000 for main, 10 for PRs) - Only run benchmark result prep on main/master push - Auto-commit benchmark results to Docs/ and README.md - Change image storage from gh-pages to main branch Docs/ - Update out.py files (rust and cpp): - Add results.md output for CI to update README - Improve graph visibility with minimum bar width - Add proper debug logging - Fix annotation format (use "x faster" instead of "+ times faster") - Improve chart titles to be more descriptive - Update README.md: - Change image links from gh-pages to main branch - Add Docs/ directory for storing benchmark result images 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/rust.yml | 97 ++++++++++++++++++++++++++------ Docs/.gitkeep | 0 README.md | 4 +- cpp/out.py | 111 ++++++++++++++++++++++++------------- rust/out.py | 56 ++++++++++++++----- 5 files changed, 197 insertions(+), 71 deletions(-) create mode 100644 Docs/.gitkeep diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index efacb1b..7d35fd1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,6 +1,14 @@ name: Benchmark Rust version -on: [push, pull_request] +on: + push: + branches: + - main + - master + pull_request: + branches: + - main + - master env: toolchain: nightly-2022-08-22 @@ -27,23 +35,45 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 + - name: Setup Rust - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@master with: - profile: minimal toolchain: ${{env.toolchain}} components: rustfmt, clippy - default: true + + - name: Wait for PostgreSQL to be ready + run: | + echo "Waiting for PostgreSQL to be fully ready..." + for i in {1..30}; do + if pg_isready -h localhost -p 5432 -U postgres > /dev/null 2>&1; then + echo "PostgreSQL is ready!" + break + fi + echo "Attempt $i: PostgreSQL not ready yet..." + sleep 2 + done + - name: Build benchmark - uses: actions-rs/cargo@v1 - with: - command: build - args: --release --all-features --manifest-path rust/Cargo.toml + run: cargo build --release --all-features --manifest-path rust/Cargo.toml + - name: Run benchmark working-directory: rust - run: cargo bench --bench bench -- --output-format bencher | tee out.txt + env: + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + # Use 1000 links for main/master branch benchmarks, 10 for pull requests + BENCHMARK_LINK_COUNT: ${{ (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')) && '1000' || '10' }} + run: | + set -o pipefail + cargo bench --bench bench -- --output-format bencher | tee out.txt + - name: Prepare benchmark results + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') run: | git config --global user.email "linksplatform@gmail.com" git config --global user.name "LinksPlatformBencher" @@ -51,13 +81,45 @@ jobs: pip install numpy matplotlib python3 out.py cd .. - git fetch - git checkout gh-pages - mv -f rust/bench_rust.png Docs - mv -f rust/bench_rust_log_scale.png Docs - git add Docs - git commit -m "Publish new benchmark results" - git push origin gh-pages + + # Create Docs directory if it doesn't exist + mkdir -p Docs + + # Copy generated images + cp -f rust/bench_rust.png Docs/ + cp -f rust/bench_rust_log_scale.png Docs/ + + # Update README with latest results + if [ -f rust/results.md ]; then + # Replace the results section in README.md + python3 -c " + import re + + with open('rust/results.md', 'r') as f: + results = f.read() + + with open('README.md', 'r') as f: + readme = f.read() + + # Pattern to find and replace the results table + pattern = r'(\| Operation.*?\n\|[-|]+\n(?:\|.*?\n)*)' + if re.search(pattern, readme): + readme = re.sub(pattern, results.strip() + '\n', readme) + + with open('README.md', 'w') as f: + f.write(readme) + " + fi + + # Commit changes if any + git add Docs README.md + if git diff --staged --quiet; then + echo "No changes to commit" + else + git commit -m "Update benchmark results" + git push origin HEAD + fi + - name: Save benchmark results uses: actions/upload-artifact@v4 with: @@ -65,3 +127,4 @@ jobs: path: | rust/bench_rust.png rust/bench_rust_log_scale.png + rust/out.txt diff --git a/Docs/.gitkeep b/Docs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md index fda3bbb..9cda4aa 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ The results below represent the amount of time (ns) the operation takes per iter - Second picture shows time in a logarithmic scale (to see diffrence clearly, because it is around 2-3 orders of magnitude). ### Rust -![Image of Rust benchmark (pixel scale)](https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/blob/gh-pages/Docs/bench_rust.png?raw=true) -![Image of Rust benchmark (log scale)](https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/blob/gh-pages/Docs/bench_rust_log_scale.png?raw=true) +![Image of Rust benchmark (pixel scale)](https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/blob/main/Docs/bench_rust.png?raw=true) +![Image of Rust benchmark (log scale)](https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/blob/main/Docs/bench_rust_log_scale.png?raw=true) ### Raw benchmark results (all numbers are in nanoseconds) diff --git a/cpp/out.py b/cpp/out.py index 8a122b2..861792d 100644 --- a/cpp/out.py +++ b/cpp/out.py @@ -1,8 +1,16 @@ import re +import logging import matplotlib.pyplot as plt import numpy as np +# Enable detailed tracing. Set to False to disable verbose output. +DEBUG = True +logging.basicConfig(level=logging.INFO if DEBUG else logging.WARNING, + format="%(message)s") + data = open('out.txt').read() +if DEBUG: + logging.info("Loaded out.txt, length: %d characters", len(data)) patterns = [ r"BM_(PSQL)/\w+\/?\w+?/(\w+)/\d+/min_warmup_time:20\.000\s+(\d+)\sns\s+\d+\sns\s+\d+", @@ -18,15 +26,21 @@ for pattern in patterns: matches = re.findall(pattern, data) + if DEBUG: + logging.info("Pattern %s matched %d entries", pattern, len(matches)) for match in matches: if match[0] == 'PSQL': _category, transaction, time = match + if DEBUG: + logging.info("PSQL - %s: %s ns", transaction, time) if transaction == "Transaction": PSQL_Transaction.append(int(time)) else: PSQL_NonTransaction.append(int(time)) else: _category, trees, storage, time = match + if DEBUG: + logging.info("Doublets %s %s: %s ns", trees, storage, time) if trees == 'United': if storage == 'Volatile': Doublets_United_Volatile.append(int(time)) @@ -38,69 +52,92 @@ else: Doublets_Split_NonVolatile.append(int(time)) +if DEBUG: + logging.info("\nFinal lists (after parsing):") + logging.info("PSQL_Transaction: %s", PSQL_Transaction) + logging.info("PSQL_NonTransaction: %s", PSQL_NonTransaction) + logging.info("Doublets_United_Volatile: %s", Doublets_United_Volatile) + logging.info("Doublets_United_NonVolatile: %s", Doublets_United_NonVolatile) + logging.info("Doublets_Split_Volatile: %s", Doublets_Split_Volatile) + logging.info("Doublets_Split_NonVolatile: %s", Doublets_Split_NonVolatile) + labels = ['Create', 'Delete', 'Each Identity', 'Each Concrete', 'Each Outgoing', 'Each Incoming', 'Each All', 'Update'] + +# ───────────────────────────────────────────────────────────────────────────── +# Plots +# ───────────────────────────────────────────────────────────────────────────── +def ensure_min_visible(arr, min_val): + """Ensure non-zero values are at least min_val for visibility on graph.""" + return [max(v, min_val) if v > 0 else 0 for v in arr] + def bench1(): - Doublets_United_Volatile_Pixels = [max(1, x // 10000000) for x in Doublets_United_Volatile] - Doublets_United_NonVolatile_Pixels = [max(1, x // 10000000) for x in Doublets_United_NonVolatile] - Doublets_Split_Volatile_Pixels = [max(1, x // 10000000) for x in Doublets_Split_Volatile] - Doublets_Split_NonVolatile_Pixels = [max(1, x // 10000000) for x in Doublets_Split_NonVolatile] - PSQL_NonTransaction_Pixels = [max(1, x // 10000000) for x in PSQL_NonTransaction] - PSQL_Transaction_Pixels = [max(1, x // 10000000) for x in PSQL_Transaction] - + """Horizontal bars – raw values (pixel scale).""" y = np.arange(len(labels)) - width = 0.1 - fig, ax = plt.subplots(figsize=(12, 8)) - - rects1 = ax.barh(y - 2*width, Doublets_United_Volatile_Pixels, width, label='Doublets United Volatile', color='salmon') - rects2 = ax.barh(y - width, Doublets_United_NonVolatile_Pixels, width, label='Doublets United NonVolatile', color='red') - - rects3 = ax.barh(y, Doublets_Split_Volatile_Pixels, width, label='Doublets Split Volatile', color='lightgreen') - rects4 = ax.barh(y + width, Doublets_Split_NonVolatile_Pixels, width, label='Doublets Split NonVolatile', color='green') - - rects5 = ax.barh(y + 2*width, PSQL_NonTransaction_Pixels, width, label='PSQL NonTransaction', color='lightblue') - rects6 = ax.barh(y + 3*width, PSQL_Transaction_Pixels, width, label='PSQL Transaction', color='blue') - - ax.set_xlabel('Time (ns) - Scaled to Pixels') - ax.set_title('Benchmark comparison for Doublets and PostgreSQL') + # Calculate maximum value across all data series to determine scale + all_values = (Doublets_United_Volatile + Doublets_United_NonVolatile + + Doublets_Split_Volatile + Doublets_Split_NonVolatile + + PSQL_NonTransaction + PSQL_Transaction) + max_val = max(all_values) if all_values else 1 + + # Minimum visible bar width: ~0.5% of max value ensures at least 2 pixels + # on typical 12-inch wide figure at 100 DPI (~900px plot area) + min_visible = max_val * 0.005 + if DEBUG: + logging.info("bench1: max_val=%d, min_visible=%d", max_val, min_visible) + + # Apply minimum visibility to all data series + du_volatile_vis = ensure_min_visible(Doublets_United_Volatile, min_visible) + du_nonvolatile_vis = ensure_min_visible(Doublets_United_NonVolatile, min_visible) + ds_volatile_vis = ensure_min_visible(Doublets_Split_Volatile, min_visible) + ds_nonvolatile_vis = ensure_min_visible(Doublets_Split_NonVolatile, min_visible) + psql_non_vis = ensure_min_visible(PSQL_NonTransaction, min_visible) + psql_trans_vis = ensure_min_visible(PSQL_Transaction, min_visible) + + ax.barh(y - 2*width, du_volatile_vis, width, label='Doublets United Volatile', color='salmon') + ax.barh(y - width, du_nonvolatile_vis, width, label='Doublets United NonVolatile', color='red') + ax.barh(y, ds_volatile_vis, width, label='Doublets Split Volatile', color='lightgreen') + ax.barh(y + width, ds_nonvolatile_vis, width, label='Doublets Split NonVolatile', color='green') + ax.barh(y + 2*width, psql_non_vis, width, label='PSQL NonTransaction', color='lightblue') + ax.barh(y + 3*width, psql_trans_vis, width, label='PSQL Transaction', color='blue') + + ax.set_xlabel('Time (ns)') + ax.set_title('Benchmark Comparison: PostgreSQL vs Doublets (C++)') ax.set_yticks(y) ax.set_yticklabels(labels) ax.legend() fig.tight_layout() - - plt.savefig("Bench1.png") + plt.savefig("bench_cpp.png") plt.close(fig) + if DEBUG: logging.info("bench_cpp.png saved.") def bench2(): + """Horizontal bars – raw values on a log scale.""" y = np.arange(len(labels)) - width = 0.1 fig, ax = plt.subplots(figsize=(12, 8)) - rects1 = ax.barh(y - 2*width, Doublets_United_Volatile, width, label='Doublets United Volatile', color='salmon') - rects2 = ax.barh(y - width, Doublets_United_NonVolatile, width, label='Doublets United NonVolatile', color='red') + ax.barh(y - 2*width, Doublets_United_Volatile, width, label='Doublets United Volatile', color='salmon') + ax.barh(y - width, Doublets_United_NonVolatile, width, label='Doublets United NonVolatile', color='red') + ax.barh(y, Doublets_Split_Volatile, width, label='Doublets Split Volatile', color='lightgreen') + ax.barh(y + width, Doublets_Split_NonVolatile, width, label='Doublets Split NonVolatile', color='green') + ax.barh(y + 2*width, PSQL_NonTransaction, width, label='PSQL NonTransaction', color='lightblue') + ax.barh(y + 3*width, PSQL_Transaction, width, label='PSQL Transaction', color='blue') - rects3 = ax.barh(y, Doublets_Split_Volatile, width, label='Doublets Split Volatile', color='lightgreen') - rects4 = ax.barh(y + width, Doublets_Split_NonVolatile, width, label='Doublets Split NonVolatile', color='green') - - rects5 = ax.barh(y + 2*width, PSQL_NonTransaction, width, label='PSQL NonTransaction', color='lightblue') - rects6 = ax.barh(y + 3*width, PSQL_Transaction, width, label='PSQL Transaction', color='blue') - - ax.set_xlabel('Time (ns) - Logarithmic Scale') - ax.set_title('Benchmark comparison for Doublets and PostgreSQL') + ax.set_xlabel('Time (ns) – log scale') + ax.set_title('Benchmark Comparison: PostgreSQL vs Doublets (C++)') ax.set_yticks(y) ax.set_yticklabels(labels) ax.legend() - ax.set_xscale('log') fig.tight_layout() - - plt.savefig("Bench2.png") + plt.savefig("bench_cpp_log_scale.png") plt.close(fig) + if DEBUG: logging.info("bench_cpp_log_scale.png saved.") bench1() bench2() diff --git a/rust/out.py b/rust/out.py index 94667e8..58cf3d6 100644 --- a/rust/out.py +++ b/rust/out.py @@ -34,9 +34,8 @@ if DEBUG: logging.info("Pattern %s matched %d entries", pattern, len(matches)) for match in matches: - # ─── FIX: normalise name ──────────────────────────────────────────── - op = match[0].replace("_", " ") # Create, Each All, … - # ──────────────────────────────────────────────────────────────────── + # Normalise name + op = match[0].replace("_", " ") # Create, Each All, … if match[1] == 'PSQL': # (operation, 'PSQL', transaction, time) transaction = match[2] @@ -109,7 +108,7 @@ def print_results_markdown(): def annotate(v): if v == 0: return "N/A" if min_psql == float('inf'): return f"{v}" - return f"{v} ({min_psql / v:.1f}+ times faster)" + return f"{v} ({min_psql / v:.1f}x faster)" row = ( f"| {op:<13} | {annotate(du_volatile_arr[i]):<24} | " @@ -122,26 +121,53 @@ def annotate(v): table_md = "\n".join(lines) print(table_md) + + # Save to file for CI to use + with open("results.md", "w") as f: + f.write(table_md) + if DEBUG: logging.info("\nGenerated Markdown Table:\n%s", table_md) # ───────────────────────────────────────────────────────────────────────────── # Plots # ───────────────────────────────────────────────────────────────────────────── +def ensure_min_visible(arr, min_val): + """Ensure non-zero values are at least min_val for visibility on graph.""" + return [max(v, min_val) if v > 0 else 0 for v in arr] + def bench1(): - """Horizontal bars – scaled (divide by 10 000 000).""" - scale = lambda arr: [max(1, x // 10_000_000) for x in arr] + """Horizontal bars – raw values (pixel scale).""" y, w = np.arange(len(ordered_ops)), 0.1 fig, ax = plt.subplots(figsize=(12, 8)) - ax.barh(y - 2*w, scale(du_volatile_arr), w, label='Doublets United Volatile', color='salmon') - ax.barh(y - w, scale(du_nonvolatile_arr),w, label='Doublets United NonVolatile',color='red') - ax.barh(y , scale(ds_volatile_arr), w, label='Doublets Split Volatile', color='lightgreen') - ax.barh(y + w, scale(ds_nonvolatile_arr), w, label='Doublets Split NonVolatile', color='green') - ax.barh(y + 2*w, scale(psql_non_arr), w, label='PSQL NonTransaction', color='lightblue') - ax.barh(y + 3*w, scale(psql_trans_arr), w, label='PSQL Transaction', color='blue') + # Calculate maximum value across all data series to determine scale + all_values = (du_volatile_arr + du_nonvolatile_arr + ds_volatile_arr + + ds_nonvolatile_arr + psql_non_arr + psql_trans_arr) + max_val = max(all_values) if all_values else 1 - ax.set_xlabel('Time (ns) – scaled') - ax.set_title ('Benchmark Comparison (Rust)') + # Minimum visible bar width: ~0.5% of max value ensures at least 2 pixels + # on typical 12-inch wide figure at 100 DPI (~900px plot area) + min_visible = max_val * 0.005 + if DEBUG: + logging.info("bench1: max_val=%d, min_visible=%d", max_val, min_visible) + + # Apply minimum visibility to all data series + du_volatile_vis = ensure_min_visible(du_volatile_arr, min_visible) + du_nonvolatile_vis = ensure_min_visible(du_nonvolatile_arr, min_visible) + ds_volatile_vis = ensure_min_visible(ds_volatile_arr, min_visible) + ds_nonvolatile_vis = ensure_min_visible(ds_nonvolatile_arr, min_visible) + psql_non_vis = ensure_min_visible(psql_non_arr, min_visible) + psql_trans_vis = ensure_min_visible(psql_trans_arr, min_visible) + + ax.barh(y - 2*w, du_volatile_vis, w, label='Doublets United Volatile', color='salmon') + ax.barh(y - w, du_nonvolatile_vis,w, label='Doublets United NonVolatile',color='red') + ax.barh(y , ds_volatile_vis, w, label='Doublets Split Volatile', color='lightgreen') + ax.barh(y + w, ds_nonvolatile_vis, w, label='Doublets Split NonVolatile', color='green') + ax.barh(y + 2*w, psql_non_vis, w, label='PSQL NonTransaction', color='lightblue') + ax.barh(y + 3*w, psql_trans_vis, w, label='PSQL Transaction', color='blue') + + ax.set_xlabel('Time (ns)') + ax.set_title ('Benchmark Comparison: PostgreSQL vs Doublets (Rust)') ax.set_yticks(y); ax.set_yticklabels(ordered_ops); ax.legend() fig.tight_layout(); plt.savefig("bench_rust.png"); plt.close(fig) if DEBUG: logging.info("bench_rust.png saved.") @@ -159,7 +185,7 @@ def bench2(): ax.barh(y + 3*w, psql_trans_arr, w, label='PSQL Transaction', color='blue') ax.set_xlabel('Time (ns) – log scale') - ax.set_title ('Benchmark Comparison (Rust)') + ax.set_title ('Benchmark Comparison: PostgreSQL vs Doublets (Rust)') ax.set_yticks(y); ax.set_yticklabels(ordered_ops); ax.set_xscale('log'); ax.legend() fig.tight_layout(); plt.savefig("bench_rust_log_scale.png"); plt.close(fig) if DEBUG: logging.info("bench_rust_log_scale.png saved.") From 1a6550b271d85d27e70e6c128cd812070115a944 Mon Sep 17 00:00:00 2001 From: konard Date: Sat, 27 Dec 2025 12:29:06 +0100 Subject: [PATCH 3/5] Revert "Initial commit with task details" This reverts commit 3f61b63b6f344bbb9c7cfb1c7e27996d28276646. --- CLAUDE.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 82df866..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,5 +0,0 @@ -Issue to solve: https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/issues/13 -Your prepared branch: issue-13-23f296f3c251 -Your prepared working directory: /tmp/gh-issue-solver-1766833843184 - -Proceed. \ No newline at end of file From e2eeed257140cb3454a63827e4567ec16cd6ee05 Mon Sep 17 00:00:00 2001 From: konard Date: Sat, 27 Dec 2025 13:32:12 +0100 Subject: [PATCH 4/5] Fix CodeFactor style issues in Python output scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove multiple spaces before/after operators and commas - Fix alignment in rust/out.py and cpp/out.py - All spacing now follows PEP 8 style guidelines These changes address the CodeFactor warnings mentioned in PR #14 comments. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- cpp/out.py | 8 ++++---- rust/out.py | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/cpp/out.py b/cpp/out.py index 861792d..4843214 100644 --- a/cpp/out.py +++ b/cpp/out.py @@ -89,12 +89,12 @@ def bench1(): logging.info("bench1: max_val=%d, min_visible=%d", max_val, min_visible) # Apply minimum visibility to all data series - du_volatile_vis = ensure_min_visible(Doublets_United_Volatile, min_visible) + du_volatile_vis = ensure_min_visible(Doublets_United_Volatile, min_visible) du_nonvolatile_vis = ensure_min_visible(Doublets_United_NonVolatile, min_visible) - ds_volatile_vis = ensure_min_visible(Doublets_Split_Volatile, min_visible) + ds_volatile_vis = ensure_min_visible(Doublets_Split_Volatile, min_visible) ds_nonvolatile_vis = ensure_min_visible(Doublets_Split_NonVolatile, min_visible) - psql_non_vis = ensure_min_visible(PSQL_NonTransaction, min_visible) - psql_trans_vis = ensure_min_visible(PSQL_Transaction, min_visible) + psql_non_vis = ensure_min_visible(PSQL_NonTransaction, min_visible) + psql_trans_vis = ensure_min_visible(PSQL_Transaction, min_visible) ax.barh(y - 2*width, du_volatile_vis, width, label='Doublets United Volatile', color='salmon') ax.barh(y - width, du_nonvolatile_vis, width, label='Doublets United NonVolatile', color='red') diff --git a/rust/out.py b/rust/out.py index 58cf3d6..7241de3 100644 --- a/rust/out.py +++ b/rust/out.py @@ -82,12 +82,12 @@ # Assemble series in the desired order. def get_series(d): return [d.get(op, 0) for op in ordered_ops] -du_volatile_arr = get_series(Doublets_United_Volatile) -du_nonvolatile_arr= get_series(Doublets_United_NonVolatile) -ds_volatile_arr = get_series(Doublets_Split_Volatile) -ds_nonvolatile_arr= get_series(Doublets_Split_NonVolatile) -psql_non_arr = get_series(PSQL_NonTransaction) -psql_trans_arr = get_series(PSQL_Transaction) +du_volatile_arr = get_series(Doublets_United_Volatile) +du_nonvolatile_arr = get_series(Doublets_United_NonVolatile) +ds_volatile_arr = get_series(Doublets_Split_Volatile) +ds_nonvolatile_arr = get_series(Doublets_Split_NonVolatile) +psql_non_arr = get_series(PSQL_NonTransaction) +psql_trans_arr = get_series(PSQL_Transaction) # ───────────────────────────────────────────────────────────────────────────── # Markdown Table @@ -101,9 +101,9 @@ def print_results_markdown(): lines = [header] for i, op in enumerate(ordered_ops): - psql_val1 = psql_non_arr[i] if psql_non_arr[i] else float('inf') + psql_val1 = psql_non_arr[i] if psql_non_arr[i] else float('inf') psql_val2 = psql_trans_arr[i] if psql_trans_arr[i] else float('inf') - min_psql = min(psql_val1, psql_val2) + min_psql = min(psql_val1, psql_val2) def annotate(v): if v == 0: return "N/A" @@ -137,7 +137,7 @@ def ensure_min_visible(arr, min_val): def bench1(): """Horizontal bars – raw values (pixel scale).""" - y, w = np.arange(len(ordered_ops)), 0.1 + y, w = np.arange(len(ordered_ops)), 0.1 fig, ax = plt.subplots(figsize=(12, 8)) # Calculate maximum value across all data series to determine scale @@ -152,19 +152,19 @@ def bench1(): logging.info("bench1: max_val=%d, min_visible=%d", max_val, min_visible) # Apply minimum visibility to all data series - du_volatile_vis = ensure_min_visible(du_volatile_arr, min_visible) + du_volatile_vis = ensure_min_visible(du_volatile_arr, min_visible) du_nonvolatile_vis = ensure_min_visible(du_nonvolatile_arr, min_visible) - ds_volatile_vis = ensure_min_visible(ds_volatile_arr, min_visible) + ds_volatile_vis = ensure_min_visible(ds_volatile_arr, min_visible) ds_nonvolatile_vis = ensure_min_visible(ds_nonvolatile_arr, min_visible) - psql_non_vis = ensure_min_visible(psql_non_arr, min_visible) - psql_trans_vis = ensure_min_visible(psql_trans_arr, min_visible) + psql_non_vis = ensure_min_visible(psql_non_arr, min_visible) + psql_trans_vis = ensure_min_visible(psql_trans_arr, min_visible) - ax.barh(y - 2*w, du_volatile_vis, w, label='Doublets United Volatile', color='salmon') - ax.barh(y - w, du_nonvolatile_vis,w, label='Doublets United NonVolatile',color='red') - ax.barh(y , ds_volatile_vis, w, label='Doublets Split Volatile', color='lightgreen') - ax.barh(y + w, ds_nonvolatile_vis, w, label='Doublets Split NonVolatile', color='green') - ax.barh(y + 2*w, psql_non_vis, w, label='PSQL NonTransaction', color='lightblue') - ax.barh(y + 3*w, psql_trans_vis, w, label='PSQL Transaction', color='blue') + ax.barh(y - 2*w, du_volatile_vis, w, label='Doublets United Volatile', color='salmon') + ax.barh(y - w, du_nonvolatile_vis, w, label='Doublets United NonVolatile', color='red') + ax.barh(y, ds_volatile_vis, w, label='Doublets Split Volatile', color='lightgreen') + ax.barh(y + w, ds_nonvolatile_vis, w, label='Doublets Split NonVolatile', color='green') + ax.barh(y + 2*w, psql_non_vis, w, label='PSQL NonTransaction', color='lightblue') + ax.barh(y + 3*w, psql_trans_vis, w, label='PSQL Transaction', color='blue') ax.set_xlabel('Time (ns)') ax.set_title ('Benchmark Comparison: PostgreSQL vs Doublets (Rust)') @@ -174,15 +174,15 @@ def bench1(): def bench2(): """Horizontal bars – raw values on a log scale.""" - y, w = np.arange(len(ordered_ops)), 0.1 + y, w = np.arange(len(ordered_ops)), 0.1 fig, ax = plt.subplots(figsize=(12, 8)) - ax.barh(y - 2*w, du_volatile_arr, w, label='Doublets United Volatile', color='salmon') - ax.barh(y - w, du_nonvolatile_arr,w, label='Doublets United NonVolatile',color='red') - ax.barh(y , ds_volatile_arr, w, label='Doublets Split Volatile', color='lightgreen') - ax.barh(y + w, ds_nonvolatile_arr, w, label='Doublets Split NonVolatile', color='green') - ax.barh(y + 2*w, psql_non_arr, w, label='PSQL NonTransaction', color='lightblue') - ax.barh(y + 3*w, psql_trans_arr, w, label='PSQL Transaction', color='blue') + ax.barh(y - 2*w, du_volatile_arr, w, label='Doublets United Volatile', color='salmon') + ax.barh(y - w, du_nonvolatile_arr, w, label='Doublets United NonVolatile', color='red') + ax.barh(y, ds_volatile_arr, w, label='Doublets Split Volatile', color='lightgreen') + ax.barh(y + w, ds_nonvolatile_arr, w, label='Doublets Split NonVolatile', color='green') + ax.barh(y + 2*w, psql_non_arr, w, label='PSQL NonTransaction', color='lightblue') + ax.barh(y + 3*w, psql_trans_arr, w, label='PSQL Transaction', color='blue') ax.set_xlabel('Time (ns) – log scale') ax.set_title ('Benchmark Comparison: PostgreSQL vs Doublets (Rust)') From e99c5cdbb7a6689efe2b0f6924d7d0ef035a5830 Mon Sep 17 00:00:00 2001 From: konard Date: Sat, 27 Dec 2025 16:21:25 +0100 Subject: [PATCH 5/5] Make benchmark parameters configurable via environment variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove unused import: doublets::Doublets in lib.rs - Replace hardcoded BACKGROUND_LINKS constant with background_links() function - Add benchmark_links() function for configurable operation count - Update all benchmark files to use the new functions - Rename BENCHMARK_LINK_COUNT to BENCHMARK_LINKS in workflow - Add BENCHMARK_BACKGROUND_LINKS (100 for PRs, 100000 for main branch) Environment variables: - BENCHMARK_LINKS: number of links per operation (10 for PRs, 1000 for main) - BENCHMARK_BACKGROUND_LINKS: background links count (100 for PRs, 100000 for main) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/rust.yml | 4 +++- rust/benches/benchmarks/create.rs | 5 +++-- rust/benches/benchmarks/delete.rs | 8 ++++--- rust/benches/benchmarks/each/concrete.rs | 11 +++------- rust/benches/benchmarks/each/identity.rs | 11 +++------- rust/benches/benchmarks/each/incoming.rs | 11 +++------- rust/benches/benchmarks/each/outgoing.rs | 11 +++------- rust/benches/benchmarks/update.rs | 6 ++++-- rust/src/lib.rs | 27 +++++++++++++++++++----- 9 files changed, 49 insertions(+), 45 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7d35fd1..0639608 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -67,7 +67,9 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres # Use 1000 links for main/master branch benchmarks, 10 for pull requests - BENCHMARK_LINK_COUNT: ${{ (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')) && '1000' || '10' }} + BENCHMARK_LINKS: ${{ (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')) && '1000' || '10' }} + # Use 100000 background links for main/master branch, 100 for pull requests + BENCHMARK_BACKGROUND_LINKS: ${{ (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')) && '100000' || '100' }} run: | set -o pipefail cargo bench --bench bench -- --output-format bencher | tee out.txt diff --git a/rust/benches/benchmarks/create.rs b/rust/benches/benchmarks/create.rs index 281eea4..8b51cbb 100644 --- a/rust/benches/benchmarks/create.rs +++ b/rust/benches/benchmarks/create.rs @@ -8,7 +8,7 @@ use { parts::LinkPart, split::{self, DataPart, IndexPart}, unit, }, - linkspsql::{bench, Benched, Client, connect, Exclusive, Fork, Transaction}, + linkspsql::{bench, benchmark_links, Benched, Client, connect, Exclusive, Fork, Transaction}, std::{alloc::Global, time::{Duration, Instant}}, }; @@ -17,9 +17,10 @@ fn bench>( id: &str, mut benched: B, ) { + let links = benchmark_links(); group.bench_function(id, |bencher| { bench!(|fork| as B { - for _ in 0..1_000 { + for _ in 0..links { let _ = elapsed! {fork.create_point()?}; } })(bencher, &mut benched); diff --git a/rust/benches/benchmarks/delete.rs b/rust/benches/benchmarks/delete.rs index 6e67926..01c9425 100644 --- a/rust/benches/benchmarks/delete.rs +++ b/rust/benches/benchmarks/delete.rs @@ -7,7 +7,7 @@ use { parts::LinkPart, split::{self, DataPart, IndexPart}, unit, }, - linkspsql::{bench, Benched, Client, connect, Exclusive, Fork, Transaction, BACKGROUND_LINKS}, + linkspsql::{bench, background_links, benchmark_links, Benched, Client, connect, Exclusive, Fork, Transaction}, std::{alloc::Global, time::{Duration, Instant}}, }; fn bench>( @@ -15,12 +15,14 @@ fn bench>( id: &str, mut benched: B, ) { + let bg_links = background_links(); + let links = benchmark_links(); group.bench_function(id, |bencher| { bench!(|fork| as B { - for _prepare in BACKGROUND_LINKS..BACKGROUND_LINKS + 1_000 { + for _prepare in bg_links..bg_links + links { let _ = fork.create_point(); } - for id in (BACKGROUND_LINKS..=BACKGROUND_LINKS + 1_000).rev() { + for id in (bg_links..=bg_links + links).rev() { let _ = elapsed! {fork.delete(id)?}; } })(bencher, &mut benched); diff --git a/rust/benches/benchmarks/each/concrete.rs b/rust/benches/benchmarks/each/concrete.rs index 12a87a2..a10457c 100644 --- a/rust/benches/benchmarks/each/concrete.rs +++ b/rust/benches/benchmarks/each/concrete.rs @@ -8,7 +8,7 @@ use { split::{self, DataPart, IndexPart}, unit, Doublets, }, - linkspsql::{bench, connect, Benched, Client, Exclusive, Fork, Transaction}, + linkspsql::{background_links, bench, connect, Benched, Client, Exclusive, Fork, Transaction}, std::{ alloc::Global, time::{Duration, Instant}, @@ -21,15 +21,10 @@ fn bench>( ) { let handler = |_| Flow::Continue; let any = LinksConstants::new().any; + let bg_links = background_links(); group.bench_function(id, |bencher| { bench!(|fork| as B { - for index in 1..=1_000 { - elapsed! {fork.each_by([any, index, index], handler)}; - } - for index in 1_001..=2_000 { - elapsed! {fork.each_by([any, index, index], handler)}; - } - for index in 2_001..=BACKGROUND_LINKS { + for index in 1..=bg_links { elapsed! {fork.each_by([any, index, index], handler)}; } })(bencher, &mut benched); diff --git a/rust/benches/benchmarks/each/identity.rs b/rust/benches/benchmarks/each/identity.rs index a76dce9..40386de 100644 --- a/rust/benches/benchmarks/each/identity.rs +++ b/rust/benches/benchmarks/each/identity.rs @@ -8,7 +8,7 @@ use { split::{self, DataPart, IndexPart}, unit, Doublets, }, - linkspsql::{bench, connect, Benched, Client, Exclusive, Fork, Transaction}, + linkspsql::{background_links, bench, connect, Benched, Client, Exclusive, Fork, Transaction}, std::{ alloc::Global, time::{Duration, Instant}, @@ -22,15 +22,10 @@ fn bench>( ) { let handler = |_| Flow::Continue; let any = LinksConstants::new().any; + let bg_links = background_links(); group.bench_function(id, |bencher| { bench!(|fork| as B { - for index in 1..=1_000 { - elapsed! {fork.each_by([index, any, any], handler)}; - } - for index in 1_001..=2_000 { - elapsed! {fork.each_by([index, any, any], handler)}; - } - for index in 2_001..=BACKGROUND_LINKS { + for index in 1..=bg_links { elapsed! {fork.each_by([index, any, any], handler)}; } })(bencher, &mut benched); diff --git a/rust/benches/benchmarks/each/incoming.rs b/rust/benches/benchmarks/each/incoming.rs index 108bfc1..3b13bda 100644 --- a/rust/benches/benchmarks/each/incoming.rs +++ b/rust/benches/benchmarks/each/incoming.rs @@ -8,7 +8,7 @@ use { split::{self, DataPart, IndexPart}, unit, Doublets, }, - linkspsql::{bench, connect, Benched, Client, Exclusive, Fork, Transaction}, + linkspsql::{background_links, bench, connect, Benched, Client, Exclusive, Fork, Transaction}, std::{ alloc::Global, time::{Duration, Instant}, @@ -22,15 +22,10 @@ fn bench>( ) { let handler = |_| Flow::Continue; let any = LinksConstants::new().any; + let bg_links = background_links(); group.bench_function(id, |bencher| { bench!(|fork| as B { - for index in 1..=1_000 { - elapsed! {fork.each_by([any, any, index], handler)}; - } - for index in 1_001..=2_000 { - elapsed! {fork.each_by([any, any, index], handler)}; - } - for index in 2_001..=BACKGROUND_LINKS { + for index in 1..=bg_links { elapsed! {fork.each_by([any, any, index], handler)}; } })(bencher, &mut benched); diff --git a/rust/benches/benchmarks/each/outgoing.rs b/rust/benches/benchmarks/each/outgoing.rs index c798309..cb27384 100644 --- a/rust/benches/benchmarks/each/outgoing.rs +++ b/rust/benches/benchmarks/each/outgoing.rs @@ -8,7 +8,7 @@ use { split::{self, DataPart, IndexPart}, unit, Doublets, }, - linkspsql::{bench, connect, Benched, Client, Exclusive, Fork, Transaction}, + linkspsql::{background_links, bench, connect, Benched, Client, Exclusive, Fork, Transaction}, std::{ alloc::Global, time::{Duration, Instant}, @@ -22,15 +22,10 @@ fn bench>( ) { let handler = |_| Flow::Continue; let any = LinksConstants::new().any; + let bg_links = background_links(); group.bench_function(id, |bencher| { bench!(|fork| as B { - for index in 1..=1_000 { - let _ = elapsed! {fork.each_by([any, index, any], handler)}; - } - for index in 1_001..=2_000 { - let _ = elapsed! {fork.each_by([any, index, any], handler)}; - } - for index in 2_001..=BACKGROUND_LINKS { + for index in 1..=bg_links { let _ = elapsed! {fork.each_by([any, index, any], handler)}; } })(bencher, &mut benched); diff --git a/rust/benches/benchmarks/update.rs b/rust/benches/benchmarks/update.rs index 68b6879..75cb273 100644 --- a/rust/benches/benchmarks/update.rs +++ b/rust/benches/benchmarks/update.rs @@ -6,7 +6,7 @@ use { mem::{Alloc, FileMapped}, split, split::{DataPart, IndexPart}, unit, unit::LinkPart, }, - linkspsql::{BACKGROUND_LINKS, bench, Benched, Transaction, Exclusive, connect, Client, Fork}, + linkspsql::{background_links, benchmark_links, bench, Benched, Transaction, Exclusive, connect, Client, Fork}, std::{alloc::Global, time::{Duration, Instant}}, }; @@ -15,9 +15,11 @@ fn bench>( id: &str, mut benched: B, ) { + let bg_links = background_links(); + let links = benchmark_links(); group.bench_function(id, |bencher| { bench!(|fork| as B { - for id in BACKGROUND_LINKS - 999..=BACKGROUND_LINKS { + for id in bg_links - (links - 1)..=bg_links { let _ = elapsed! {fork.update(id, 0, 0)?}; let _ = elapsed! {fork.update(id, id, id)?}; } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index cbba133..3743ea2 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -15,10 +15,10 @@ macro_rules! bench { }}; } crate::tri! { - use linkspsql::BACKGROUND_LINKS; + let background_links = linkspsql::background_links(); for _iter in 0..iters { let mut $fork: Fork<$B> = Benched::fork(&mut *benched); - for _ in 0..BACKGROUND_LINKS { + for _ in 0..background_links { let _ = $fork.create_point()?; } $($body)* @@ -30,7 +30,6 @@ macro_rules! bench { } } -use doublets::Doublets; pub use { benched::Benched, client::Client, exclusive::Exclusive, fork::Fork, transaction::Transaction, }; @@ -38,7 +37,7 @@ pub use { use { doublets::{data::LinkType, mem::FileMapped}, postgres::NoTls, - std::{error, fs::File, io, result}, + std::{env, error, fs::File, io, result}, }; mod benched; @@ -49,7 +48,25 @@ mod transaction; pub type Result> = result::Result; -pub const BACKGROUND_LINKS: usize = 3_000; +/// Number of background links to create before each benchmark iteration. +/// Configurable via BENCHMARK_BACKGROUND_LINKS environment variable. +/// Default: 3000 (for local testing), CI uses 100 for PRs and 100000 for main branch. +pub fn background_links() -> usize { + env::var("BENCHMARK_BACKGROUND_LINKS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(3_000) +} + +/// Number of links to create/update/delete in each benchmark operation. +/// Configurable via BENCHMARK_LINKS environment variable. +/// Default: 1000 (for local testing), CI uses 10 for PRs and 1000 for main branch. +pub fn benchmark_links() -> usize { + env::var("BENCHMARK_LINKS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(1_000) +} const PARAMS: &str = "user=postgres dbname=postgres password=postgres host=localhost port=5432"; pub fn connect() -> Result> {