From c7832642c98cd023bbe2ccf84b69e0e937b5bc61 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 2 Dec 2025 15:29:07 +0000 Subject: [PATCH 1/6] Write a Ractor benchmark that mirrors the process structure The previous ractor knucleotide benchmark ran the whole benchmark in parallel inside multiple ractors. This is a good test that the benchmark can be parallelised but isn't directly comparable to the original Knucleotide implementation which uses processes to partition the work done by the benchmark itself. This implementation uses ractors to parallelise the work in the same way that the original uses Process.fork so it's more directly comparable. This needs to be run with the regular benchmark harness instead of the ractor harness, otherwise the ractor harness will attempt to wrap this benchmark run in multiple ractors too. --- benchmarks.yml | 2 + benchmarks/knucleotide-ractor/benchmark.rb | 71 ++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 benchmarks/knucleotide-ractor/benchmark.rb diff --git a/benchmarks.yml b/benchmarks.yml index 7c0297d6..58221ec3 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -91,6 +91,8 @@ graphql-native: desc: GraphQL gem parsing a large file, but using a native parser knucleotide: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Process.fork +knucleotide-ractor: + desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors lee: desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way matmul: diff --git a/benchmarks/knucleotide-ractor/benchmark.rb b/benchmarks/knucleotide-ractor/benchmark.rb new file mode 100644 index 00000000..41d265d6 --- /dev/null +++ b/benchmarks/knucleotide-ractor/benchmark.rb @@ -0,0 +1,71 @@ +# The Computer Language Benchmarks Game +# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/ +# +# k-nucleotide benchmark - Ractor implementation +# Mirrors the Process.fork version: spawns 7 ractors (one per task) + +Warning[:experimental] = false + +require_relative '../../harness/loader' + +def frequency(seq, length) + frequencies = Hash.new(0) + last_index = seq.length - length + + i = 0 + while i <= last_index + frequencies[seq.byteslice(i, length)] += 1 + i += 1 + end + + [seq.length - length + 1, frequencies] +end + +def sort_by_freq(seq, length) + n, table = frequency(seq, length) + + table.sort { |a, b| + cmp = b[1] <=> a[1] + cmp == 0 ? a[0] <=> b[0] : cmp + }.map { |seq, count| + "#{seq} #{'%.3f' % ((count * 100.0) / n)}" + }.join("\n") + "\n\n" +end + +def find_seq(seq, s) + _, table = frequency(seq, s.length) + "#{table[s] || 0}\t#{s}\n" +end + +def generate_test_sequence(size) + alu = "GGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCGGGCGGATCACCTGAGGTCA" + + "GGAGTTCGAGACCAGCCTGGCCAACATGGTGAAACCCCGTCTCTACTAAAAATACAAAAATTAGCCGGGCGTGG" + + "TGGCGCGCGCCTGTAATCCCAGCTACTCGGGAGGCTGAGGCAGGAGAATCGCTTGAACCCGGGAGGCGGAGGTT" + + "GCAGTGAGCCGAGATCGCGCCACTGCACTCCAGCCTGGGCGACAGAGCGAGACTCCGTCTCAAAAA" + + sequence = "" + full_copies = size / alu.length + remainder = size % alu.length + + full_copies.times { sequence << alu } + sequence << alu[0, remainder] if remainder > 0 + + sequence.upcase +end + +TEST_SEQUENCE = Ractor.make_shareable(generate_test_sequence(100_000)) + +run_benchmark(5) do + freqs = [1, 2] + nucleos = %w(GGT GGTA GGTATT GGTATTTTAATT GGTATTTTAATTTATAGT) + + ractors = freqs.map { |i| + Ractor.new(TEST_SEQUENCE, i) { |seq, len| sort_by_freq(seq, len) } + } + ractors += nucleos.map { |s| + Ractor.new(TEST_SEQUENCE, s) { |seq, nucleo| find_seq(seq, nucleo) } + } + + results = ractors.map(&:value) + results +end From ad34b578c736f43a6aac6a63c4df1d0a23e24110 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 2 Dec 2025 16:05:33 +0000 Subject: [PATCH 2/6] Allow pinning to be specified in the benchmarks.yml --- benchmarks.yml | 1 + lib/argument_parser.rb | 6 ++++++ lib/benchmark_runner/cli.rb | 3 ++- lib/benchmark_suite.rb | 19 ++++++++++++++----- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/benchmarks.yml b/benchmarks.yml index 58221ec3..6041618c 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -93,6 +93,7 @@ knucleotide: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Process.fork knucleotide-ractor: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors + no_pinning: true lee: desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way matmul: diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 35de87c1..528c4c9e 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -15,6 +15,7 @@ class ArgumentParser :rss, :graph, :no_pinning, + :force_pinning, :turbo, :skip_yjit, :with_pre_init, @@ -140,6 +141,10 @@ def parse(argv) args.no_pinning = true end + opts.on("--force-pinning", "force pinning even for benchmarks marked no_pinning") do + args.force_pinning = true + end + opts.on("--turbo", "don't disable CPU turbo boost") do args.turbo = true end @@ -183,6 +188,7 @@ def default_args rss: false, graph: false, no_pinning: false, + force_pinning: false, turbo: false, skip_yjit: false, with_pre_init: nil, diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index 92206a48..639c2014 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -35,7 +35,8 @@ def run out_path: args.out_path, harness: args.harness, pre_init: args.with_pre_init, - no_pinning: args.no_pinning + no_pinning: args.no_pinning, + force_pinning: args.force_pinning ) # Benchmark with and without YJIT diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 93d9bbf9..b60d1acd 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -19,9 +19,9 @@ class BenchmarkSuite RACTOR_CATEGORY = ["ractor"].freeze RACTOR_HARNESS = "harness-ractor" - attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir + attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :force_pinning, :bench_dir, :ractor_bench_dir - def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false) + def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false, force_pinning: false) @categories = categories @name_filters = name_filters @excludes = excludes @@ -29,6 +29,7 @@ def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pr @harness = harness @pre_init = pre_init ? expand_pre_init(pre_init) : nil @no_pinning = no_pinning + @force_pinning = force_pinning @ractor_only = (categories == RACTOR_ONLY_CATEGORY) setup_benchmark_directories @@ -41,7 +42,6 @@ def run(ruby:, ruby_description:) bench_failures = {} benchmark_entries = discover_benchmarks - cmd_prefix = base_cmd(ruby_description) env = benchmark_env(ruby) caller_json_path = ENV["RESULT_JSON_PATH"] @@ -49,6 +49,7 @@ def run(ruby:, ruby_description:) puts("Running benchmark \"#{entry.name}\" (#{idx+1}/#{benchmark_entries.length})") result_json_path = caller_json_path || File.join(out_path, "temp#{Process.pid}.json") + cmd_prefix = base_cmd(ruby_description, entry.name) result = run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env) if result[:success] @@ -199,13 +200,13 @@ def linux? end # Set up the base command with CPU pinning if needed - def base_cmd(ruby_description) + def base_cmd(ruby_description, benchmark_name) if linux? cmd = setarch_prefix # Pin the process to one given core to improve caching and reduce variance on CRuby # Other Rubies need to use multiple cores, e.g., for JIT threads - if ruby_description.start_with?('ruby ') && !no_pinning + if ruby_description.start_with?('ruby ') && should_pin?(benchmark_name) # The last few cores of Intel CPU may be slow E-Cores, so avoid using the last one. cpu = [(Etc.nprocessors / 2) - 1, 0].max cmd.concat(["taskset", "-c", "#{cpu}"]) @@ -217,6 +218,14 @@ def base_cmd(ruby_description) end end + def should_pin?(benchmark_name) + return false if no_pinning + return true if force_pinning + + benchmark_meta = benchmarks_metadata[benchmark_name] || {} + !benchmark_meta["no_pinning"] + end + # Generate setarch prefix for Linux def setarch_prefix # Disable address space randomization (for determinism) From d5a843482e8ed38b0e35d1813ad245d276f2f5d5 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 3 Dec 2025 12:50:08 +0000 Subject: [PATCH 3/6] Don't fail tests if the user has a ruby version installed already --- lib/argument_parser.rb | 15 +++++-- test/argument_parser_test.rb | 82 ++++++++++++++++++------------------ 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 528c4c9e..6e73e338 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -53,10 +53,8 @@ def parse(argv) name = name.shellsplit.first end version, *options = version.shellsplit - rubies_dir = ENV["RUBIES_DIR"] || "#{ENV["HOME"]}/.rubies" - unless executable = ["/opt/rubies/#{version}/bin/ruby", "#{rubies_dir}/#{version}/bin/ruby"].find { |path| File.executable?(path) } - abort "Cannot find '#{version}' in /opt/rubies or #{rubies_dir}" - end + executable = find_chruby_ruby(version) + abort "Cannot find '#{version}' in chruby paths" unless executable args.executables[name] = [executable, *options] end end @@ -170,6 +168,15 @@ def parse(argv) private + def find_chruby_ruby(version) + rubies_dir = ENV["RUBIES_DIR"] || "#{ENV["HOME"]}/.rubies" + chruby_search_paths(version, rubies_dir).find { |path| File.executable?(path) } + end + + def chruby_search_paths(version, rubies_dir) + ["/opt/rubies/#{version}/bin/ruby", "#{rubies_dir}/#{version}/bin/ruby"] + end + def have_yjit?(ruby) ruby_version = `#{ruby} -v --yjit 2> #{File::NULL}`.strip ruby_version.downcase.include?("yjit") diff --git a/test/argument_parser_test.rb b/test/argument_parser_test.rb index 7008dcef..d19ae240 100644 --- a/test/argument_parser_test.rb +++ b/test/argument_parser_test.rb @@ -103,15 +103,11 @@ def setup_mock_ruby(path) ruby_path = File.join(tmpdir, 'opt/rubies/3.2.0/bin/ruby') setup_mock_ruby(ruby_path) - File.stub :executable?, ->(path) { - if path == "/opt/rubies/3.2.0/bin/ruby" - File.exist?(ruby_path) && File.stat(ruby_path).executable? - end - } do - parser = ArgumentParser.new + parser = ArgumentParser.new + parser.stub :chruby_search_paths, ->(version, rubies_dir) { [ruby_path] } do args = parser.parse(['--chruby=ruby-3.2.0::3.2.0']) - assert_equal '/opt/rubies/3.2.0/bin/ruby', args.executables['ruby-3.2.0'].first + assert_equal ruby_path, args.executables['ruby-3.2.0'].first end end end @@ -126,9 +122,11 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=my-ruby::3.3.0']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=my-ruby::3.3.0']) - assert_equal ruby_path, args.executables['my-ruby'].first + assert_equal ruby_path, args.executables['my-ruby'].first + end end end @@ -143,20 +141,11 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir - File.stub :executable?, ->(path) { - case path - when "/opt/rubies/3.2.0/bin/ruby" - File.exist?(opt_ruby) && File.stat(opt_ruby).executable? - when "#{tmpdir}/.rubies/3.2.0/bin/ruby" - File.exist?(home_ruby) && File.stat(home_ruby).executable? - else - File.method(:executable?).super_method.call(path) - end - } do - parser = ArgumentParser.new + parser = ArgumentParser.new + parser.stub :chruby_search_paths, ->(version, rd) { [opt_ruby, home_ruby] } do args = parser.parse(['--chruby=test::3.2.0']) - assert_equal '/opt/rubies/3.2.0/bin/ruby', args.executables['test'].first + assert_equal opt_ruby, args.executables['test'].first end end end @@ -171,9 +160,11 @@ def setup_mock_ruby(path) ENV['RUBIES_DIR'] = custom_rubies parser = ArgumentParser.new - args = parser.parse(['--chruby=custom::3.4.0']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=custom::3.4.0']) - assert_equal ruby_path, args.executables['custom'].first + assert_equal ruby_path, args.executables['custom'].first + end end end @@ -183,10 +174,11 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - - assert_raises(SystemExit) do - capture_io do - parser.parse(['--chruby=nonexistent::nonexistent-version-999']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + assert_raises(SystemExit) do + capture_io do + parser.parse(['--chruby=nonexistent::nonexistent-version-999']) + end end end end @@ -202,10 +194,12 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=yjit::3.2.0 --yjit']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=yjit::3.2.0 --yjit']) - assert_equal ruby_path, args.executables['yjit'].first - assert_equal '--yjit', args.executables['yjit'].last + assert_equal ruby_path, args.executables['yjit'].first + assert_equal '--yjit', args.executables['yjit'].last + end end end @@ -219,10 +213,12 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=3.2.0 --yjit']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=3.2.0 --yjit']) - assert args.executables.key?('3.2.0') - assert_equal ruby_path, args.executables['3.2.0'].first + assert args.executables.key?('3.2.0') + assert_equal ruby_path, args.executables['3.2.0'].first + end end end @@ -238,12 +234,14 @@ def setup_mock_ruby(path) ENV['HOME'] = tmpdir parser = ArgumentParser.new - args = parser.parse(['--chruby=ruby32::3.2.0;ruby33::3.3.0 --yjit']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=ruby32::3.2.0;ruby33::3.3.0 --yjit']) - assert_equal 2, args.executables.size - assert_equal ruby_path_32, args.executables['ruby32'].first - assert_equal ruby_path_33, args.executables['ruby33'].first - assert_equal '--yjit', args.executables['ruby33'].last + assert_equal 2, args.executables.size + assert_equal ruby_path_32, args.executables['ruby32'].first + assert_equal ruby_path_33, args.executables['ruby33'].first + assert_equal '--yjit', args.executables['ruby33'].last + end end end end @@ -589,10 +587,12 @@ def setup_mock_ruby(path) parser = ArgumentParser.new(ruby_executable: mock_ruby) parser.stub :have_yjit?, true do - args = parser.parse(['--chruby=test::3.2.0']) + parser.stub :chruby_search_paths, ->(version, rd) { ["#{rd}/#{version}/bin/ruby"] } do + args = parser.parse(['--chruby=test::3.2.0']) - assert_equal 1, args.executables.size - assert_equal ruby_path, args.executables['test'].first + assert_equal 1, args.executables.size + assert_equal ruby_path, args.executables['test'].first + end end end end From ffa30de4987ce080666d3d43c985a167e4046643 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 3 Dec 2025 14:35:18 +0000 Subject: [PATCH 4/6] Remove old knucleotide ractor benchmark. I don't think there is value in keeping this around now that we have a Ractor enabled version that has feature parity with the old Process based one --- benchmarks-ractor/knucleotide/benchmark.rb | 66 ---------------------- benchmarks.yml | 2 - 2 files changed, 68 deletions(-) delete mode 100644 benchmarks-ractor/knucleotide/benchmark.rb diff --git a/benchmarks-ractor/knucleotide/benchmark.rb b/benchmarks-ractor/knucleotide/benchmark.rb deleted file mode 100644 index ca5b9cd9..00000000 --- a/benchmarks-ractor/knucleotide/benchmark.rb +++ /dev/null @@ -1,66 +0,0 @@ -# The Computer Language Benchmarks Game -# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/ -# -# k-nucleotide benchmark - Ractor implementation -# Mirrors the Process.fork version structure as closely as possible - -require_relative "../../harness/loader" - -def frequency(seq, length) - frequencies = Hash.new(0) - last_index = seq.length - length - - i = 0 - while i <= last_index - frequencies[seq.byteslice(i, length)] += 1 - i += 1 - end - - [seq.length - length + 1, frequencies] -end - -def sort_by_freq(seq, length) - n, table = frequency(seq, length) - - table.sort { |a, b| - cmp = b[1] <=> a[1] - cmp == 0 ? a[0] <=> b[0] : cmp - }.map! { |seq, count| - "#{seq} #{'%.3f' % ((count * 100.0) / n)}" - }.join("\n") << "\n\n" -end - -def find_seq(seq, s) - _, table = frequency(seq, s.length) - "#{table[s] || 0}\t#{s}\n" -end - -def generate_test_sequence(size) - alu = "GGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCGGGCGGATCACCTGAGGTCA" + - "GGAGTTCGAGACCAGCCTGGCCAACATGGTGAAACCCCGTCTCTACTAAAAATACAAAAATTAGCCGGGCGTGG" + - "TGGCGCGCGCCTGTAATCCCAGCTACTCGGGAGGCTGAGGCAGGAGAATCGCTTGAACCCGGGAGGCGGAGGTT" + - "GCAGTGAGCCGAGATCGCGCCACTGCACTCCAGCCTGGGCGACAGAGCGAGACTCCGTCTCAAAAA" - - sequence = "" - full_copies = size / alu.length - remainder = size % alu.length - - full_copies.times { sequence << alu } - sequence << alu[0, remainder] if remainder > 0 - - sequence.upcase.freeze -end - -# Make sequence shareable for Ractors -TEST_SEQUENCE = make_shareable(generate_test_sequence(100_000)) - -run_benchmark(5) do |num_ractors, ractor_args| - freqs = [1, 2] - nucleos = %w(GGT GGTA GGTATT GGTATTTTAATT GGTATTTTAATTTATAGT) - - # Sequential version - mirrors Process version but without Workers - results = [] - freqs.each { |i| results << sort_by_freq(TEST_SEQUENCE, i) } - nucleos.each { |s| results << find_seq(TEST_SEQUENCE, s) } - results -end diff --git a/benchmarks.yml b/benchmarks.yml index 6041618c..d020dc3b 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -236,8 +236,6 @@ throw: # # Ractor-only benchmarks # -ractor/knucleotide: - desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables. Counts groups in parallel using Ractors. ractor/gvl_release_acquire: desc: microbenchmark designed to test how fast the gvl can be acquired and released between ractors. ractor/json_parse_float: From fae5127afcce72e63903668dca3aad83a29bd58e Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 3 Dec 2025 14:38:28 +0000 Subject: [PATCH 5/6] Restructure the category selection For benchmarks like the knucleotide ractor benchmark, which use Ractors internally so need to be included in the ractor benchmarks, but which must not use the ractor harness New category selection is as follows: - No category: Runs all benchmarks except those with `ractor_only: true`. Uses each benchmark's `default_harness`, falling back to the `default` harness. - `--category=ractor`: Runs benchmarks with ractor: true or `ractor_only: true`. Uses each benchmark's `default_harness`, falling back to `harness-ractor`. - `--category=ractor-only`: Runs only benchmarks with `ractor_only: true`. Uses each benchmark's `default_harness`, falling back to `harness-ractor`. --- benchmarks.yml | 21 ++- .../gvl_release_acquire/benchmark.rb | 0 .../json_parse_float/Gemfile | 0 .../json_parse_float/Gemfile.lock | 0 .../json_parse_float/benchmark.rb | 0 .../json_parse_string/Gemfile | 0 .../json_parse_string/Gemfile.lock | 0 .../json_parse_string/benchmark.rb | 0 burn_in.rb | 36 ++--- lib/argument_parser.rb | 3 + lib/benchmark_filter.rb | 14 +- lib/benchmark_runner/cli.rb | 1 + lib/benchmark_suite.rb | 79 +++++----- test/benchmark_filter_test.rb | 17 +++ test/benchmark_suite_test.rb | 137 +++++++++++------- 15 files changed, 192 insertions(+), 116 deletions(-) rename {benchmarks-ractor => benchmarks}/gvl_release_acquire/benchmark.rb (100%) rename {benchmarks-ractor => benchmarks}/json_parse_float/Gemfile (100%) rename {benchmarks-ractor => benchmarks}/json_parse_float/Gemfile.lock (100%) rename {benchmarks-ractor => benchmarks}/json_parse_float/benchmark.rb (100%) rename {benchmarks-ractor => benchmarks}/json_parse_string/Gemfile (100%) rename {benchmarks-ractor => benchmarks}/json_parse_string/Gemfile.lock (100%) rename {benchmarks-ractor => benchmarks}/json_parse_string/benchmark.rb (100%) diff --git a/benchmarks.yml b/benchmarks.yml index d020dc3b..5b61fd2f 100644 --- a/benchmarks.yml +++ b/benchmarks.yml @@ -93,7 +93,9 @@ knucleotide: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Process.fork knucleotide-ractor: desc: k-nucleotide from the Computer Language Benchmarks Game - counts nucleotide frequencies using hash tables in parallel using Ractors - no_pinning: true + ractor: true + ractor_only: true + default_harness: harness lee: desc: lee is a circuit-board layout solver, deployed in a plausibly reality-like way matmul: @@ -234,11 +236,20 @@ throw: ractor: true # -# Ractor-only benchmarks +# Ractor scaling benchmarks # -ractor/gvl_release_acquire: +gvl_release_acquire: desc: microbenchmark designed to test how fast the gvl can be acquired and released between ractors. -ractor/json_parse_float: + ractor: true + ractor_only: true + default_harness: harness-ractor +json_parse_float: desc: test the performance of parsing multiple lists of json floats with ractors. -ractor/json_parse_string: + ractor: true + ractor_only: true + default_harness: harness-ractor +json_parse_string: desc: test the performance of parsing multiple lists of strings with ractors. + ractor: true + ractor_only: true + default_harness: harness-ractor diff --git a/benchmarks-ractor/gvl_release_acquire/benchmark.rb b/benchmarks/gvl_release_acquire/benchmark.rb similarity index 100% rename from benchmarks-ractor/gvl_release_acquire/benchmark.rb rename to benchmarks/gvl_release_acquire/benchmark.rb diff --git a/benchmarks-ractor/json_parse_float/Gemfile b/benchmarks/json_parse_float/Gemfile similarity index 100% rename from benchmarks-ractor/json_parse_float/Gemfile rename to benchmarks/json_parse_float/Gemfile diff --git a/benchmarks-ractor/json_parse_float/Gemfile.lock b/benchmarks/json_parse_float/Gemfile.lock similarity index 100% rename from benchmarks-ractor/json_parse_float/Gemfile.lock rename to benchmarks/json_parse_float/Gemfile.lock diff --git a/benchmarks-ractor/json_parse_float/benchmark.rb b/benchmarks/json_parse_float/benchmark.rb similarity index 100% rename from benchmarks-ractor/json_parse_float/benchmark.rb rename to benchmarks/json_parse_float/benchmark.rb diff --git a/benchmarks-ractor/json_parse_string/Gemfile b/benchmarks/json_parse_string/Gemfile similarity index 100% rename from benchmarks-ractor/json_parse_string/Gemfile rename to benchmarks/json_parse_string/Gemfile diff --git a/benchmarks-ractor/json_parse_string/Gemfile.lock b/benchmarks/json_parse_string/Gemfile.lock similarity index 100% rename from benchmarks-ractor/json_parse_string/Gemfile.lock rename to benchmarks/json_parse_string/Gemfile.lock diff --git a/benchmarks-ractor/json_parse_string/benchmark.rb b/benchmarks/json_parse_string/benchmark.rb similarity index 100% rename from benchmarks-ractor/json_parse_string/benchmark.rb rename to benchmarks/json_parse_string/benchmark.rb diff --git a/burn_in.rb b/burn_in.rb index ea305eb8..91b1ada1 100755 --- a/burn_in.rb +++ b/burn_in.rb @@ -59,14 +59,10 @@ def free_file_path(parent_dir, name_prefix) end end -def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version) - # Determine the path to the benchmark script - bench_name = bench_id.sub('ractor/', '') - bench_dir, harness = if bench_name == bench_id - ['benchmarks', 'harness'] - else - ['benchmarks-ractor', 'harness-ractor'] - end +def run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata) + bench_dir = 'benchmarks' + entry = metadata[bench_name] || {} + harness = entry.fetch('default_harness', 'harness') script_path = File.join(bench_dir, bench_name, 'benchmark.rb') if not File.exist?(script_path) @@ -153,12 +149,12 @@ def run_benchmark(bench_id, no_yjit, logs_path, run_time, ruby_version) return false end -def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) +def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version, metadata) error_found = false while true bench_name = bench_names.sample() - error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version) + error = run_benchmark(bench_name, no_yjit, logs_path, run_time, ruby_version, metadata) error_found ||= error if error_found @@ -201,12 +197,16 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) bench_names = [] if args.categories.include?('ractor-only') - # Only include benchmarks with ractor/ prefix (from benchmarks-ractor directory) - bench_names = metadata.keys.select { |name| name.start_with?('ractor/') } + # Include only benchmarks with ractor_only: true + metadata.each do |name, entry| + if entry['ractor_only'] + bench_names << name + end + end elsif args.categories.include?('ractor') - # Include both ractor/ prefixed benchmarks and those with ractor: true + # Include benchmarks with ractor: true or ractor_only: true metadata.each do |name, entry| - if name.start_with?('ractor/') || entry['ractor'] + if entry['ractor'] || entry['ractor_only'] bench_names << name end end @@ -221,10 +221,12 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) end end else - # Regular category filtering + # Regular category filtering - exclude ractor-only and ractor harness benchmarks metadata.each do |name, entry| category = entry.fetch('category', 'other') - if args.categories.include?(category) + is_ractor_only = entry['ractor_only'] || + (entry['ractor'] && entry['default_harness'] == 'harness-ractor') + if args.categories.include?(category) && !is_ractor_only bench_names << name end end @@ -237,7 +239,7 @@ def test_loop(bench_names, no_yjit, logs_path, run_time, ruby_version) args.num_procs.times do |i| pid = Process.fork do run_time = (i < args.num_long_runs)? (3600 * 2):10 - test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version) + test_loop(bench_names, args.no_yjit, args.logs_path, run_time, ruby_version, metadata) end end diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 6e73e338..e7a4cf4e 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -8,6 +8,7 @@ class ArgumentParser :out_path, :out_override, :harness, + :harness_explicit, :yjit_opts, :categories, :name_filters, @@ -93,6 +94,7 @@ def parse(argv) opts.on("--harness=HARNESS_DIR", "which harness to use") do |v| v = "harness-#{v}" unless v.start_with?('harness') args.harness = v + args.harness_explicit = true end opts.on("--warmup=N", "the number of warmup iterations for the default harness (default: 15)") do |n| @@ -188,6 +190,7 @@ def default_args out_path: File.expand_path("./data"), out_override: nil, harness: "harness", + harness_explicit: false, yjit_opts: "", categories: [], name_filters: [], diff --git a/lib/benchmark_filter.rb b/lib/benchmark_filter.rb index 3219d711..a9ed5586 100644 --- a/lib/benchmark_filter.rb +++ b/lib/benchmark_filter.rb @@ -18,12 +18,21 @@ def match?(name) private def matches_category?(name) - return true if @categories.empty? + if @categories.empty? + return false if ractor_harness_benchmark?(name) + return true + end benchmark_categories = get_benchmark_categories(name) @categories.intersect?(benchmark_categories) end + def ractor_harness_benchmark?(name) + benchmark_metadata = @metadata[name] || {} + benchmark_metadata['ractor_only'] || + (benchmark_metadata['ractor'] && benchmark_metadata['default_harness'] == 'harness-ractor') + end + def matches_name_filter?(name) return true if @name_filters.empty? @@ -58,7 +67,8 @@ def get_benchmark_categories(name) @category_cache[name] ||= begin benchmark_metadata = @metadata[name] || {} categories = [benchmark_metadata.fetch('category', 'other')] - categories << 'ractor' if benchmark_metadata['ractor'] + categories << 'ractor' if benchmark_metadata['ractor'] || benchmark_metadata['ractor_only'] + categories << 'ractor-only' if benchmark_metadata['ractor_only'] categories end end diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index 639c2014..d4c011c2 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -34,6 +34,7 @@ def run excludes: args.excludes, out_path: args.out_path, harness: args.harness, + harness_explicit: args.harness_explicit, pre_init: args.with_pre_init, no_pinning: args.no_pinning, force_pinning: args.force_pinning diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index b60d1acd..e9f72519 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -14,25 +14,23 @@ # BenchmarkSuite runs a collection of benchmarks and collects their results class BenchmarkSuite BENCHMARKS_DIR = "benchmarks" - RACTOR_BENCHMARKS_DIR = "benchmarks-ractor" - RACTOR_ONLY_CATEGORY = ["ractor-only"].freeze RACTOR_CATEGORY = ["ractor"].freeze + RACTOR_ONLY_CATEGORY = ["ractor-only"].freeze RACTOR_HARNESS = "harness-ractor" - attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :force_pinning, :bench_dir, :ractor_bench_dir + attr_reader :categories, :name_filters, :excludes, :out_path, :harness, :harness_explicit, :pre_init, :no_pinning, :force_pinning, :bench_dir - def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false, force_pinning: false) + def initialize(categories:, name_filters:, excludes: [], out_path:, harness:, harness_explicit: false, pre_init: nil, no_pinning: false, force_pinning: false) @categories = categories @name_filters = name_filters @excludes = excludes @out_path = out_path @harness = harness + @harness_explicit = harness_explicit @pre_init = pre_init ? expand_pre_init(pre_init) : nil @no_pinning = no_pinning @force_pinning = force_pinning - @ractor_only = (categories == RACTOR_ONLY_CATEGORY) - - setup_benchmark_directories + @bench_dir = BENCHMARKS_DIR end # Run all the benchmarks and record execution times @@ -50,7 +48,7 @@ def run(ruby:, ruby_description:) result_json_path = caller_json_path || File.join(out_path, "temp#{Process.pid}.json") cmd_prefix = base_cmd(ruby_description, entry.name) - result = run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env) + result = run_single_benchmark(entry.script_path, result_json_path, ruby, cmd_prefix, env, entry.name) if result[:success] bench_data[entry.name] = process_benchmark_result(result_json_path, result[:command], delete_file: !caller_json_path) @@ -91,47 +89,24 @@ def discover_benchmarks end def discover_all_benchmark_entries - main_discovery = BenchmarkDiscovery.new(bench_dir) - main_entries = main_discovery.discover - - ractor_entries = if benchmark_ractor_directory? - ractor_discovery = BenchmarkDiscovery.new(ractor_bench_dir) - ractor_discovery.discover - else - [] - end - - { main: main_entries, ractor: ractor_entries } + discovery = BenchmarkDiscovery.new(bench_dir) + { main: discovery.discover } end def build_directory_map(all_entries) - combined_entries = all_entries[:main] + all_entries[:ractor] - combined_entries.each_with_object({}) do |entry, map| + all_entries[:main].each_with_object({}) do |entry, map| map[entry.name] = entry.directory end end def filter_benchmarks(all_entries, directory_map) - main_benchmarks = filter_entries( + filter_entries( all_entries[:main], categories: categories, name_filters: name_filters, excludes: excludes, directory_map: directory_map ) - - if benchmark_ractor_directory? - ractor_benchmarks = filter_entries( - all_entries[:ractor], - categories: [], - name_filters: name_filters, - excludes: excludes, - directory_map: directory_map - ) - main_benchmarks + ractor_benchmarks - else - main_benchmarks - end end def filter_entries(entries, categories:, name_filters:, excludes:, directory_map:) @@ -145,17 +120,22 @@ def filter_entries(entries, categories:, name_filters:, excludes:, directory_map entries.select { |entry| filter.match?(entry.name) } end - def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) + def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env, benchmark_name) # Fix for jruby/jruby#7394 in JRuby 9.4.2.0 script_path = File.expand_path(script_path) - # Set up the environment for the benchmarking command + # Save and restore ENV["RESULT_JSON_PATH"] to avoid polluting the environment + # for subsequent runs (e.g., when running multiple executables) + original_result_json_path = ENV["RESULT_JSON_PATH"] ENV["RESULT_JSON_PATH"] = result_json_path + # Use per-benchmark default_harness if set, otherwise use global harness + benchmark_harness = benchmark_harness_for(benchmark_name) + # Set up the benchmarking command cmd = cmd_prefix + [ *ruby, - "-I", harness, + "-I", benchmark_harness, *pre_init, script_path, ].compact @@ -164,6 +144,20 @@ def run_single_benchmark(script_path, result_json_path, ruby, cmd_prefix, env) result = BenchmarkRunner.check_call(cmd.shelljoin, env: env, raise_error: false) result[:command] = cmd.shelljoin result + ensure + if original_result_json_path + ENV["RESULT_JSON_PATH"] = original_result_json_path + else + ENV.delete("RESULT_JSON_PATH") + end + end + + def benchmark_harness_for(benchmark_name) + return harness if harness_explicit + + benchmark_meta = benchmarks_metadata[benchmark_name] || {} + default = ractor_category_run? ? RACTOR_HARNESS : harness + benchmark_meta.fetch('default_harness', default) end def benchmark_env(ruby) @@ -190,10 +184,6 @@ def benchmarks_metadata @benchmarks_metadata ||= YAML.load_file('benchmarks.yml') end - def benchmark_ractor_directory? - categories == RACTOR_CATEGORY - end - # Check if running on Linux def linux? @linux ||= RbConfig::CONFIG['host_os'] =~ /linux/ @@ -221,11 +211,16 @@ def base_cmd(ruby_description, benchmark_name) def should_pin?(benchmark_name) return false if no_pinning return true if force_pinning + return false if ractor_category_run? benchmark_meta = benchmarks_metadata[benchmark_name] || {} !benchmark_meta["no_pinning"] end + def ractor_category_run? + categories == RACTOR_CATEGORY || categories == RACTOR_ONLY_CATEGORY + end + # Generate setarch prefix for Linux def setarch_prefix # Disable address space randomization (for determinism) diff --git a/test/benchmark_filter_test.rb b/test/benchmark_filter_test.rb index 5b623d10..1abdfada 100644 --- a/test/benchmark_filter_test.rb +++ b/test/benchmark_filter_test.rb @@ -47,6 +47,23 @@ assert_equal true, filter.match?('ractor_bench') end + it 'excludes ractor harness benchmarks from default runs' do + metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'harness-ractor' }) + filter = BenchmarkFilter.new(categories: [], name_filters: [], excludes: [], metadata: metadata) + + assert_equal true, filter.match?('fib') + assert_equal true, filter.match?('ractor_bench') # ractor: true without harness-ractor runs in default + assert_equal false, filter.match?('ractor_harness_bench') # ractor: true with harness-ractor excluded + end + + it 'includes ractor harness benchmarks in ractor category' do + metadata = @metadata.merge('ractor_harness_bench' => { 'category' => 'other', 'ractor' => true, 'default_harness' => 'harness-ractor' }) + filter = BenchmarkFilter.new(categories: ['ractor'], name_filters: [], excludes: [], metadata: metadata) + + assert_equal true, filter.match?('ractor_harness_bench') + assert_equal true, filter.match?('ractor_bench') + end + it 'handles regex filters' do filter = BenchmarkFilter.new(categories: [], name_filters: ['/rails/'], excludes: [], metadata: @metadata) diff --git a/test/benchmark_suite_test.rb b/test/benchmark_suite_test.rb index ab9abe15..aa6ac32b 100644 --- a/test/benchmark_suite_test.rb +++ b/test/benchmark_suite_test.rb @@ -15,7 +15,6 @@ # Create mock benchmarks directory structure FileUtils.mkdir_p('benchmarks') - FileUtils.mkdir_p('benchmarks-ractor') FileUtils.mkdir_p('harness') # Create a simple benchmark file @@ -75,7 +74,7 @@ assert_equal true, suite.no_pinning end - it 'sets bench_dir to BENCHMARKS_DIR by default' do + it 'sets bench_dir to BENCHMARKS_DIR' do suite = BenchmarkSuite.new( categories: ['micro'], name_filters: [], @@ -84,37 +83,100 @@ ) assert_equal 'benchmarks', suite.bench_dir - assert_equal 'benchmarks-ractor', suite.ractor_bench_dir assert_equal 'harness', suite.harness assert_equal ['micro'], suite.categories end - it 'sets bench_dir to ractor directory and updates harness when ractor-only category is used' do + it 'keeps bench_dir as BENCHMARKS_DIR when ractor category is used' do suite = BenchmarkSuite.new( - categories: ['ractor-only'], + categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness' ) - assert_equal 'benchmarks-ractor', suite.bench_dir - assert_equal 'benchmarks-ractor', suite.ractor_bench_dir - assert_equal 'harness-ractor', suite.harness - assert_equal [], suite.categories + assert_equal 'benchmarks', suite.bench_dir + assert_equal 'harness', suite.harness + assert_equal ['ractor'], suite.categories end - it 'keeps bench_dir as BENCHMARKS_DIR when ractor category is used' do + it 'tracks harness_explicit flag' do + suite_explicit = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'custom-harness', + harness_explicit: true + ) + assert_equal true, suite_explicit.harness_explicit + + suite_auto = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'harness-ractor' + ) + assert_equal false, suite_auto.harness_explicit + end + end + + describe '#benchmark_harness_for' do + before do + @metadata_with_harness = { + 'simple' => { 'category' => 'micro' }, + 'custom_harness_bench' => { 'category' => 'other', 'default_harness' => 'harness' } + } + File.write('benchmarks.yml', YAML.dump(@metadata_with_harness)) + end + + it 'returns default_harness when set and harness not explicit' do + suite = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'harness-ractor', + harness_explicit: false + ) + + assert_equal 'harness', suite.send(:benchmark_harness_for, 'custom_harness_bench') + assert_equal 'harness-ractor', suite.send(:benchmark_harness_for, 'simple') + end + + it 'ignores default_harness when harness is explicit' do + suite = BenchmarkSuite.new( + categories: [], + name_filters: [], + out_path: @out_path, + harness: 'custom-harness', + harness_explicit: true + ) + + assert_equal 'custom-harness', suite.send(:benchmark_harness_for, 'custom_harness_bench') + assert_equal 'custom-harness', suite.send(:benchmark_harness_for, 'simple') + end + end + + describe '#ractor_category_run?' do + it 'returns true for ractor category' do suite = BenchmarkSuite.new( categories: ['ractor'], name_filters: [], out_path: @out_path, + harness: 'harness-ractor' + ) + + assert_equal true, suite.send(:ractor_category_run?) + end + + it 'returns false for other categories' do + suite = BenchmarkSuite.new( + categories: ['micro'], + name_filters: [], + out_path: @out_path, harness: 'harness' ) - assert_equal 'benchmarks', suite.bench_dir - assert_equal 'benchmarks-ractor', suite.ractor_bench_dir - assert_equal 'harness', suite.harness - assert_equal ['ractor'], suite.categories + assert_equal false, suite.send(:ractor_category_run?) end end @@ -234,9 +296,9 @@ assert_empty bench_failures end - it 'handles ractor-only category' do + it 'handles ractor category with ractor benchmarks' do # Create a ractor benchmark - File.write('benchmarks-ractor/ractor_test.rb', <<~RUBY) + File.write('benchmarks/ractor_test.rb', <<~RUBY) require 'json' result = { 'warmup' => [0.001], @@ -246,8 +308,14 @@ File.write(ENV['RESULT_JSON_PATH'], JSON.generate(result)) RUBY + metadata = { + 'ractor_test' => { 'category' => 'other', 'ractor' => true }, + 'simple' => { 'category' => 'micro' } + } + File.write('benchmarks.yml', YAML.dump(metadata)) + suite = BenchmarkSuite.new( - categories: ['ractor-only'], + categories: ['ractor'], name_filters: [], out_path: @out_path, harness: 'harness', @@ -259,41 +327,10 @@ bench_data, bench_failures = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') end - # When ractor-only is specified, it should use benchmarks-ractor directory + # Should only include ractor benchmarks when ractor category specified assert_includes bench_data, 'ractor_test' + refute_includes bench_data, 'simple' assert_empty bench_failures - - # harness should be updated to harness-ractor - assert_equal 'harness-ractor', suite.harness - end - - it 'includes both regular and ractor benchmarks with ractor category' do - File.write('benchmarks-ractor/ractor_bench.rb', <<~RUBY) - require 'json' - result = { - 'warmup' => [0.001], - 'bench' => [0.001], - 'rss' => 10485760 - } - File.write(ENV['RESULT_JSON_PATH'], JSON.generate(result)) - RUBY - - suite = BenchmarkSuite.new( - categories: ['ractor'], - name_filters: [], - out_path: @out_path, - harness: 'harness', - no_pinning: true - ) - - bench_data = nil - capture_io do - bench_data, _ = suite.run(ruby: [RbConfig.ruby], ruby_description: 'ruby 3.2.0') - end - - # With ractor category, both directories should be scanned - # but we need appropriate filters - assert_instance_of Hash, bench_data end it 'expands pre_init when provided' do From 7343ad5474e7df6ec324f9fd08471ccb20dd0bfe Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 3 Dec 2025 22:50:33 +0000 Subject: [PATCH 6/6] Don't run ractor benchmarks on stable ruby The API has changed between 3.4.7 and HEAD. Until 4.0 is realeased we shouldn't test 3.4.7 --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7d61b29..9b77dd8d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: [ruby, head] + ruby: [head] if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} steps: - uses: actions/checkout@v3 @@ -85,7 +85,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: [ruby, head] + ruby: [head] if: ${{ github.event_name != 'schedule' || github.repository == 'ruby/ruby-bench' }} steps: - uses: actions/checkout@v3