From f1c59d81df1eb2b4c0d372059520d0f3e94d5c0a Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Sat, 22 Nov 2025 13:48:04 +0100 Subject: [PATCH] Add --excludes to run_benchmarks.rb to easily exclude benchmarks --- lib/argument_parser.rb | 6 ++++++ lib/benchmark_filter.rb | 9 +++++++-- lib/benchmark_runner/cli.rb | 1 + lib/benchmark_suite.rb | 7 +++++-- test/benchmark_filter_test.rb | 29 ++++++++++++++++++----------- test/benchmark_runner_cli_test.rb | 1 + 6 files changed, 38 insertions(+), 15 deletions(-) diff --git a/lib/argument_parser.rb b/lib/argument_parser.rb index 5b961af9..35de87c1 100644 --- a/lib/argument_parser.rb +++ b/lib/argument_parser.rb @@ -11,6 +11,7 @@ class ArgumentParser :yjit_opts, :categories, :name_filters, + :excludes, :rss, :graph, :no_pinning, @@ -82,6 +83,10 @@ def parse(argv) args.name_filters = list end + opts.on("--excludes=x,y,z", Array, "excludes the listed benchmarks") do |list| + args.excludes = list + end + opts.on("--skip-yjit", "Don't run with yjit after interpreter") do args.skip_yjit = true end @@ -174,6 +179,7 @@ def default_args yjit_opts: "", categories: [], name_filters: [], + excludes: [], rss: false, graph: false, no_pinning: false, diff --git a/lib/benchmark_filter.rb b/lib/benchmark_filter.rb index 36b73a37..abbd0392 100644 --- a/lib/benchmark_filter.rb +++ b/lib/benchmark_filter.rb @@ -2,16 +2,17 @@ # Filters benchmarks based on categories and name patterns class BenchmarkFilter - def initialize(categories:, name_filters:, metadata:) + def initialize(categories:, name_filters:, excludes:, metadata:) @categories = categories @name_filters = process_name_filters(name_filters) + @excludes = excludes @metadata = metadata @category_cache = {} end def match?(entry) name = entry.sub(/\.rb\z/, '') - matches_category?(name) && matches_name_filter?(name) + matches_category?(name) && matches_name_filter?(name) && !matches_excludes?(name) end private @@ -29,6 +30,10 @@ def matches_name_filter?(name) @name_filters.any? { |filter| filter === name } end + def matches_excludes?(name) + @excludes.include?(name) + end + def get_benchmark_categories(name) @category_cache[name] ||= begin benchmark_metadata = @metadata[name] || {} diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index c5343fd8..ab0d0692 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -40,6 +40,7 @@ def run ruby_description: ruby_descriptions[name], categories: args.categories, name_filters: args.name_filters, + excludes: args.excludes, out_path: args.out_path, harness: args.harness, pre_init: args.with_pre_init, diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 83cc8bb0..45c5159d 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -18,13 +18,14 @@ class BenchmarkSuite RACTOR_CATEGORY = ["ractor"].freeze RACTOR_HARNESS = "harness-ractor" - attr_reader :ruby, :ruby_description, :categories, :name_filters, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir + attr_reader :ruby, :ruby_description, :categories, :name_filters, :excludes, :out_path, :harness, :pre_init, :no_pinning, :bench_dir, :ractor_bench_dir - def initialize(ruby:, ruby_description:, categories:, name_filters:, out_path:, harness:, pre_init: nil, no_pinning: false) + def initialize(ruby:, ruby_description:, categories:, name_filters:, excludes: [], out_path:, harness:, pre_init: nil, no_pinning: false) @ruby = ruby @ruby_description = ruby_description @categories = categories @name_filters = name_filters + @excludes = excludes @out_path = out_path @harness = harness @pre_init = pre_init ? expand_pre_init(pre_init) : nil @@ -148,6 +149,7 @@ def main_benchmark_filter @main_benchmark_filter ||= BenchmarkFilter.new( categories: categories, name_filters: name_filters, + excludes: excludes, metadata: benchmarks_metadata ) end @@ -156,6 +158,7 @@ def ractor_benchmark_filter @ractor_benchmark_filter ||= BenchmarkFilter.new( categories: [], name_filters: name_filters, + excludes: excludes, metadata: benchmarks_metadata ) end diff --git a/test/benchmark_filter_test.rb b/test/benchmark_filter_test.rb index 4e18cd48..d24490cc 100644 --- a/test/benchmark_filter_test.rb +++ b/test/benchmark_filter_test.rb @@ -14,73 +14,80 @@ describe '#match?' do it 'matches when no filters provided' do - filter = BenchmarkFilter.new(categories: [], name_filters: [], metadata: @metadata) + filter = BenchmarkFilter.new(categories: [], name_filters: [], excludes: [], metadata: @metadata) assert_equal true, filter.match?('fib.rb') end it 'matches by category' do - filter = BenchmarkFilter.new(categories: ['micro'], name_filters: [], metadata: @metadata) + filter = BenchmarkFilter.new(categories: ['micro'], name_filters: [], excludes: [], metadata: @metadata) assert_equal true, filter.match?('fib.rb') assert_equal false, filter.match?('railsbench.rb') end it 'matches by name filter' do - filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], metadata: @metadata) + filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], excludes: [], metadata: @metadata) assert_equal true, filter.match?('fib.rb') assert_equal false, filter.match?('railsbench.rb') end + it 'applies excludes' do + filter = BenchmarkFilter.new(categories: ['headline'], name_filters: [], excludes: ['railsbench'], metadata: @metadata) + + assert_equal true, filter.match?('optcarrot.rb') + assert_equal false, filter.match?('railsbench.rb') + end + it 'matches ractor category' do - filter = BenchmarkFilter.new(categories: ['ractor'], name_filters: [], metadata: @metadata) + filter = BenchmarkFilter.new(categories: ['ractor'], name_filters: [], excludes: [], metadata: @metadata) assert_equal true, filter.match?('ractor_bench.rb') end it 'strips .rb extension from entry name' do - filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], metadata: @metadata) + filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], excludes: [], metadata: @metadata) assert_equal true, filter.match?('fib.rb') end it 'handles regex filters' do - filter = BenchmarkFilter.new(categories: [], name_filters: ['/rails/'], metadata: @metadata) + filter = BenchmarkFilter.new(categories: [], name_filters: ['/rails/'], excludes: [], metadata: @metadata) assert_equal true, filter.match?('railsbench.rb') assert_equal false, filter.match?('fib.rb') end it 'handles case-insensitive regex filters' do - filter = BenchmarkFilter.new(categories: [], name_filters: ['/RAILS/i'], metadata: @metadata) + filter = BenchmarkFilter.new(categories: [], name_filters: ['/RAILS/i'], excludes: [], metadata: @metadata) assert_equal true, filter.match?('railsbench.rb') end it 'handles multiple categories' do - filter = BenchmarkFilter.new(categories: ['micro', 'headline'], name_filters: [], metadata: @metadata) + filter = BenchmarkFilter.new(categories: ['micro', 'headline'], name_filters: [], excludes: [], metadata: @metadata) assert_equal true, filter.match?('fib.rb') assert_equal true, filter.match?('railsbench.rb') end it 'requires both category and name filter to match when both provided' do - filter = BenchmarkFilter.new(categories: ['micro'], name_filters: ['rails'], metadata: @metadata) + filter = BenchmarkFilter.new(categories: ['micro'], name_filters: ['rails'], excludes: [], metadata: @metadata) assert_equal false, filter.match?('fib.rb') # matches category but not name assert_equal false, filter.match?('railsbench.rb') # matches name but not category end it 'handles complex regex patterns' do - filter = BenchmarkFilter.new(categories: [], name_filters: ['/opt.*rot/'], metadata: @metadata) + filter = BenchmarkFilter.new(categories: [], name_filters: ['/opt.*rot/'], excludes: [], metadata: @metadata) assert_equal true, filter.match?('optcarrot.rb') assert_equal false, filter.match?('fib.rb') end it 'handles mixed string and regex filters' do - filter = BenchmarkFilter.new(categories: [], name_filters: ['fib', '/rails/'], metadata: @metadata) + filter = BenchmarkFilter.new(categories: [], name_filters: ['fib', '/rails/'], excludes: [], metadata: @metadata) assert_equal true, filter.match?('fib.rb') assert_equal true, filter.match?('railsbench.rb') diff --git a/test/benchmark_runner_cli_test.rb b/test/benchmark_runner_cli_test.rb index 4b155494..d3e525c6 100644 --- a/test/benchmark_runner_cli_test.rb +++ b/test/benchmark_runner_cli_test.rb @@ -41,6 +41,7 @@ def create_args(overrides = {}) yjit_opts: '', categories: [], name_filters: [], + excludes: [], rss: false, graph: false, no_pinning: true,