diff --git a/box.c b/box.c index 81f285b3212ccd..3fbc4e9fe23a14 100644 --- a/box.c +++ b/box.c @@ -149,6 +149,7 @@ box_entry_initialize(rb_box_t *box) box->loading_table = st_init_strtable(); box->ruby_dln_libmap = rb_hash_new_with_size(0); box->gvar_tbl = rb_hash_new_with_size(0); + box->classext_cow_classes = st_init_numtable(); box->is_user = true; box->is_optional = true; @@ -199,6 +200,9 @@ rb_box_entry_mark(void *ptr) } rb_gc_mark(box->ruby_dln_libmap); rb_gc_mark(box->gvar_tbl); + if (box->classext_cow_classes) { + rb_mark_tbl(box->classext_cow_classes); + } } static int @@ -233,9 +237,36 @@ box_root_free(void *ptr) } } +static int +free_classext_for_box(st_data_t _key, st_data_t obj_value, st_data_t box_arg) +{ + rb_classext_t *ext; + VALUE obj = (VALUE)obj_value; + const rb_box_t *box = (const rb_box_t *)box_arg; + + if (RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)) { + ext = rb_class_unlink_classext(obj, box); + rb_class_classext_free(obj, ext, false); + } + else if (RB_TYPE_P(obj, T_ICLASS)) { + ext = rb_class_unlink_classext(obj, box); + rb_iclass_classext_free(obj, ext, false); + } + else { + rb_bug("Invalid type of object in classext_cow_classes: %s", rb_type_str(BUILTIN_TYPE(obj))); + } + return ST_CONTINUE; +} + static void box_entry_free(void *ptr) { + const rb_box_t *box = (const rb_box_t *)ptr; + + if (box->classext_cow_classes) { + st_foreach(box->classext_cow_classes, free_classext_for_box, (st_data_t)box); + } + box_root_free(ptr); xfree(ptr); } @@ -250,7 +281,7 @@ box_entry_memsize(const void *ptr) } const rb_data_type_t rb_box_data_type = { - "Namespace::Entry", + "Ruby::Box::Entry", { rb_box_entry_mark, box_entry_free, @@ -261,7 +292,7 @@ const rb_data_type_t rb_box_data_type = { }; const rb_data_type_t rb_root_box_data_type = { - "Namespace::Root", + "Ruby::Box::Root", { rb_box_entry_mark, box_root_free, @@ -750,6 +781,7 @@ initialize_root_box(void) root->ruby_dln_libmap = rb_hash_new_with_size(0); root->gvar_tbl = rb_hash_new_with_size(0); + root->classext_cow_classes = NULL; // classext CoW never happen on the root box vm->root_box = root; diff --git a/class.c b/class.c index 2f94b801471104..0cec526b74fe98 100644 --- a/class.c +++ b/class.c @@ -86,6 +86,17 @@ cvar_table_free_i(VALUE value, void *ctx) return ID_TABLE_CONTINUE; } +rb_classext_t * +rb_class_unlink_classext(VALUE klass, const rb_box_t *box) +{ + st_data_t ext; + st_data_t key = (st_data_t)box->box_object; + VALUE obj_id = rb_obj_id(klass); + st_delete(box->classext_cow_classes, &obj_id, 0); + st_delete(RCLASS_CLASSEXT_TBL(klass), &key, &ext); + return (rb_classext_t *)ext; +} + void rb_class_classext_free(VALUE klass, rb_classext_t *ext, bool is_prime) { @@ -156,7 +167,7 @@ struct rb_class_set_box_classext_args { }; static int -rb_class_set_box_classext_update(st_data_t *key_ptr, st_data_t *val_ptr, st_data_t a, int existing) +set_box_classext_update(st_data_t *key_ptr, st_data_t *val_ptr, st_data_t a, int existing) { struct rb_class_set_box_classext_args *args = (struct rb_class_set_box_classext_args *)a; @@ -182,7 +193,10 @@ rb_class_set_box_classext(VALUE obj, const rb_box_t *box, rb_classext_t *ext) .ext = ext, }; - st_update(RCLASS_CLASSEXT_TBL(obj), (st_data_t)box->box_object, rb_class_set_box_classext_update, (st_data_t)&args); + VM_ASSERT(BOX_USER_P(box)); + + st_update(RCLASS_CLASSEXT_TBL(obj), (st_data_t)box->box_object, set_box_classext_update, (st_data_t)&args); + st_insert(box->classext_cow_classes, (st_data_t)rb_obj_id(obj), obj); // FIXME: This is done here because this is the first time the objects in // the classext are exposed via this class. It's likely that if GC diff --git a/ext/psych/lib/psych/visitors/to_ruby.rb b/ext/psych/lib/psych/visitors/to_ruby.rb index 580a74e9fb034d..2814ce1a695df0 100644 --- a/ext/psych/lib/psych/visitors/to_ruby.rb +++ b/ext/psych/lib/psych/visitors/to_ruby.rb @@ -219,7 +219,8 @@ def visit_Psych_Nodes_Mapping o revive_data_members(members, o) end data ||= allocate_anon_data(o, members) - init_struct(data, **members) + values = data.members.map { |m| members[m] } + init_data(data, values) data.freeze data diff --git a/ext/psych/psych_to_ruby.c b/ext/psych/psych_to_ruby.c index d473a5f840a1d7..0132b2c94e28c7 100644 --- a/ext/psych/psych_to_ruby.c +++ b/ext/psych/psych_to_ruby.c @@ -10,7 +10,11 @@ static VALUE build_exception(VALUE self, VALUE klass, VALUE mesg) { VALUE e = rb_obj_alloc(klass); +#ifdef TRUFFLERUBY + rb_exc_set_message(e, mesg); +#else rb_iv_set(e, "mesg", mesg); +#endif return e; } @@ -24,12 +28,9 @@ static VALUE path2class(VALUE self, VALUE path) return rb_path_to_class(path); } -static VALUE init_struct(VALUE self, VALUE data, VALUE attrs) +static VALUE init_data(VALUE self, VALUE data, VALUE values) { - VALUE args = rb_ary_new2(1); - rb_ary_push(args, attrs); - rb_struct_initialize(data, args); - + rb_struct_initialize(data, values); return data; } @@ -42,7 +43,7 @@ void Init_psych_to_ruby(void) VALUE visitor = rb_define_class_under(visitors, "Visitor", rb_cObject); cPsychVisitorsToRuby = rb_define_class_under(visitors, "ToRuby", visitor); - rb_define_private_method(cPsychVisitorsToRuby, "init_struct", init_struct, 2); + rb_define_private_method(cPsychVisitorsToRuby, "init_data", init_data, 2); rb_define_private_method(cPsychVisitorsToRuby, "build_exception", build_exception, 2); rb_define_private_method(class_loader, "path2class", path2class, 1); } diff --git a/gc.c b/gc.c index 4f6751316f07da..97b7362c9fbb85 100644 --- a/gc.c +++ b/gc.c @@ -3154,12 +3154,18 @@ rb_gc_mark_children(void *objspace, VALUE obj) foreach_args.objspace = objspace; foreach_args.obj = obj; rb_class_classext_foreach(obj, gc_mark_classext_module, (void *)&foreach_args); + if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) { + gc_mark_internal(RCLASS_PRIME_BOX(obj)->box_object); + } break; case T_ICLASS: foreach_args.objspace = objspace; foreach_args.obj = obj; rb_class_classext_foreach(obj, gc_mark_classext_iclass, (void *)&foreach_args); + if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) { + gc_mark_internal(RCLASS_PRIME_BOX(obj)->box_object); + } break; case T_ARRAY: diff --git a/internal/box.h b/internal/box.h index 41cad634823f71..c341f046d3e147 100644 --- a/internal/box.h +++ b/internal/box.h @@ -34,6 +34,7 @@ struct rb_box_struct { VALUE ruby_dln_libmap; VALUE gvar_tbl; + struct st_table *classext_cow_classes; bool is_user; bool is_optional; diff --git a/internal/class.h b/internal/class.h index f122d2f189c580..296a07ae29e9f3 100644 --- a/internal/class.h +++ b/internal/class.h @@ -513,6 +513,7 @@ void rb_undef_methods_from(VALUE klass, VALUE super); VALUE rb_class_inherited(VALUE, VALUE); VALUE rb_keyword_error_new(const char *, VALUE); +rb_classext_t *rb_class_unlink_classext(VALUE klass, const rb_box_t *box); void rb_class_classext_free(VALUE klass, rb_classext_t *ext, bool is_prime); void rb_iclass_classext_free(VALUE klass, rb_classext_t *ext, bool is_prime); diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index 6ea98fc1eaa598..e488b7c5cf0c72 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -71,7 +71,7 @@ def self.parse(src, filename = "(ripper)", lineno = 1) # [[1, 13], :on_kw, "end", END ]] # def self.lex(src, filename = "-", lineno = 1, raise_errors: false) - result = Prism.lex_compat(src, filepath: filename, line: lineno) + result = Prism.lex_compat(src, filepath: filename, line: lineno, version: "current") if result.failure? && raise_errors raise SyntaxError, result.errors.first.message @@ -3295,7 +3295,7 @@ def visit_yield_node(node) # Lazily initialize the parse result. def result - @result ||= Prism.parse(source, partial_script: true) + @result ||= Prism.parse(source, partial_script: true, version: "current") end ########################################################################## diff --git a/load.c b/load.c index c519efe2a13f8d..5a0697a2626707 100644 --- a/load.c +++ b/load.c @@ -1344,16 +1344,7 @@ require_internal(rb_execution_context_t *ec, VALUE fname, int exception, bool wa else { switch (found) { case 'r': - // iseq_eval_in_box will be called with the loading box eventually - if (BOX_OPTIONAL_P(box)) { - // check with BOX_OPTIONAL_P (not BOX_USER_P) for NS1::xxx naming - // it is not expected for the main box - // TODO: no need to use load_wrapping() here? - load_wrapping(saved.ec, path, box->box_object); - } - else { - load_iseq_eval(saved.ec, path); - } + load_iseq_eval(saved.ec, path); break; case 's': diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index c01ed678fcfda4..9600f4be8d15d0 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -883,6 +883,15 @@ def test_json_generate_as_json_convert_to_proc assert_equal object.object_id.to_json, JSON.generate(object, strict: true, as_json: -> (o, is_key) { o.object_id }) end + def test_as_json_nan_does_not_call_to_json + def (obj = Object.new).to_json(*) + "null" + end + assert_raise(JSON::GeneratorError) do + JSON.generate(Float::NAN, strict: true, as_json: proc { obj }) + end + end + def assert_float_roundtrip(expected, actual) assert_equal(expected, JSON.generate(actual)) assert_equal(actual, JSON.parse(JSON.generate(actual)), "JSON: #{JSON.generate(actual)}") diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index bd7a8a638166c5..706b7395574e05 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -88,7 +88,7 @@ def assert_errors(filepath, version) expected = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) source = expected.lines.grep_v(/^\s*\^/).join.gsub(/\n*\z/, "") - refute_valid_syntax(source) if current_major_minor == version + refute_valid_syntax(source) if CURRENT_MAJOR_MINOR == version result = Prism.parse(source, version: version) errors = result.errors diff --git a/test/prism/fixtures_test.rb b/test/prism/fixtures_test.rb index 2aebb1847782dc..7df97029d3aa52 100644 --- a/test/prism/fixtures_test.rb +++ b/test/prism/fixtures_test.rb @@ -24,23 +24,9 @@ class FixturesTest < TestCase except << "whitequark/ruby_bug_19281.txt" end - if RUBY_VERSION < "3.4.0" - except << "3.4/circular_parameters.txt" - end - - # Valid only on Ruby 3.3 - except << "3.3-3.3/block_args_in_array_assignment.txt" - except << "3.3-3.3/it_with_ordinary_parameter.txt" - except << "3.3-3.3/keyword_args_in_array_assignment.txt" - except << "3.3-3.3/return_in_sclass.txt" - - # Leaving these out until they are supported by parse.y. - except << "4.0/leading_logical.txt" - except << "4.0/endless_methods_command_call.txt" - # https://bugs.ruby-lang.org/issues/21168#note-5 except << "command_method_call_2.txt" - Fixture.each(except: except) do |fixture| + Fixture.each_for_current_ruby(except: except) do |fixture| define_method(fixture.test_name) { assert_valid_syntax(fixture.read) } end end diff --git a/test/prism/lex_test.rb b/test/prism/lex_test.rb index 19dd845d755621..68e47a096408cd 100644 --- a/test/prism/lex_test.rb +++ b/test/prism/lex_test.rb @@ -7,19 +7,13 @@ module Prism class LexTest < TestCase except = [ - # It seems like there are some oddities with nested heredocs and ripper. - # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. - "seattlerb/heredoc_nested.txt", - "whitequark/dedenting_heredoc.txt", - # Ripper seems to have a bug that the regex portions before and after - # the heredoc are combined into a single token. See - # https://bugs.ruby-lang.org/issues/19838. + # https://bugs.ruby-lang.org/issues/21756 "spanning_heredoc.txt", - "spanning_heredoc_newlines.txt", - # Prism emits a single :on_tstring_content in <<- style heredocs when there - # is a line continuation preceded by escaped backslashes. It should emit two, same - # as if the backslashes are not present. + # Prism emits a single string in some cases when ripper splits them up + "whitequark/dedenting_heredoc.txt", "heredocs_with_fake_newlines.txt", + # Prism emits BEG for `on_regexp_end` + "spanning_heredoc_newlines.txt", ] if RUBY_VERSION < "3.3.0" @@ -42,17 +36,11 @@ class LexTest < TestCase except << "whitequark/ruby_bug_19281.txt" end - # https://bugs.ruby-lang.org/issues/20925 - except << "4.0/leading_logical.txt" - - # https://bugs.ruby-lang.org/issues/17398#note-12 - except << "4.0/endless_methods_command_call.txt" - # https://bugs.ruby-lang.org/issues/21168#note-5 except << "command_method_call_2.txt" - Fixture.each_with_version(except: except) do |fixture, version| - define_method(fixture.test_name(version)) { assert_lex(fixture, version) } + Fixture.each_for_current_ruby(except: except) do |fixture| + define_method(fixture.test_name) { assert_lex(fixture) } end def test_lex_file @@ -97,12 +85,10 @@ def test_parse_lex_file private - def assert_lex(fixture, version) - return unless current_major_minor == version - + def assert_lex(fixture) source = fixture.read - result = Prism.lex_compat(source, version: version) + result = Prism.lex_compat(source, version: "current") assert_equal [], result.errors Prism.lex_ripper(source).zip(result.value).each do |(ripper, prism)| diff --git a/test/prism/locals_test.rb b/test/prism/locals_test.rb index 814c9a9978d84a..4f8d6080e8075d 100644 --- a/test/prism/locals_test.rb +++ b/test/prism/locals_test.rb @@ -13,11 +13,6 @@ # in comparing the locals because they will be the same. return if RubyVM::InstructionSequence.compile("").to_a[4][:parser] == :prism -# In Ruby 3.4.0, the local table for method forwarding changed. But 3.4.0 can -# refer to the dev version, so while 3.4.0 still isn't released, we need to -# check if we have a high enough revision. -return if RubyVM::InstructionSequence.compile("def foo(...); end").to_a[13][2][2][10].length != 1 - # Omit tests if running on a 32-bit machine because there is a bug with how # Ruby is handling large ISeqs on 32-bit machines return if RUBY_PLATFORM =~ /i686/ @@ -31,19 +26,11 @@ class LocalsTest < TestCase # CRuby is eliminating dead code. "whitequark/ruby_bug_10653.txt", - # Valid only on Ruby 3.3 - "3.3-3.3/block_args_in_array_assignment.txt", - "3.3-3.3/it_with_ordinary_parameter.txt", - "3.3-3.3/keyword_args_in_array_assignment.txt", - "3.3-3.3/return_in_sclass.txt", - - # Leaving these out until they are supported by parse.y. - "4.0/leading_logical.txt", - "4.0/endless_methods_command_call.txt", - "command_method_call_2.txt" + # https://bugs.ruby-lang.org/issues/21168#note-5 + "command_method_call_2.txt", ] - Fixture.each(except: except) do |fixture| + Fixture.each_for_current_ruby(except: except) do |fixture| define_method(fixture.test_name) { assert_locals(fixture) } end diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb index 648c44e77a6573..c972f0962bb6f9 100644 --- a/test/prism/ruby/parser_test.rb +++ b/test/prism/ruby/parser_test.rb @@ -65,15 +65,6 @@ class ParserTest < TestCase # 1.. && 2 "ranges.txt", - # https://bugs.ruby-lang.org/issues/20478 - "3.4/circular_parameters.txt", - - # Cannot yet handling leading logical operators. - "4.0/leading_logical.txt", - - # Ruby >= 4.0 specific syntax - "4.0/endless_methods_command_call.txt", - # https://bugs.ruby-lang.org/issues/21168#note-5 "command_method_call_2.txt", ] @@ -148,7 +139,7 @@ class ParserTest < TestCase "whitequark/space_args_block.txt" ] - Fixture.each(except: skip_syntax_error) do |fixture| + Fixture.each_for_version(except: skip_syntax_error, version: "3.3") do |fixture| define_method(fixture.test_name) do assert_equal_parses( fixture, @@ -171,7 +162,7 @@ def test_non_prism_builder_class_deprecated if RUBY_VERSION >= "3.3" def test_current_parser_for_current_ruby - major, minor = current_major_minor.split(".") + major, minor = CURRENT_MAJOR_MINOR.split(".") # Let's just hope there never is a Ruby 3.10 or similar expected = major.to_i * 10 + minor.to_i assert_equal(expected, Translation::ParserCurrent.new.version) diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 400139acc03d01..bd63302efcf908 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -8,44 +8,34 @@ module Prism class RipperTest < TestCase # Skip these tests that Ripper is reporting the wrong results for. incorrect = [ - # Not yet supported. - "4.0/leading_logical.txt", - # Ripper incorrectly attributes the block to the keyword. - "seattlerb/block_break.txt", - "seattlerb/block_next.txt", "seattlerb/block_return.txt", - "whitequark/break_block.txt", - "whitequark/next_block.txt", "whitequark/return_block.txt", - # Ripper is not accounting for locals created by patterns using the ** - # operator within an `in` clause. - "seattlerb/parse_pattern_058.txt", - # Ripper cannot handle named capture groups in regular expressions. "regex.txt", - "regex_char_width.txt", - "whitequark/lvar_injecting_match.txt", # Ripper fails to understand some structures that span across heredocs. "spanning_heredoc.txt", - "3.3-3.3/block_args_in_array_assignment.txt", - "3.3-3.3/it_with_ordinary_parameter.txt", - "3.3-3.3/keyword_args_in_array_assignment.txt", - "3.3-3.3/return_in_sclass.txt", - - # https://bugs.ruby-lang.org/issues/20478 + # Ripper interprets circular keyword arguments as method calls. "3.4/circular_parameters.txt", - # https://bugs.ruby-lang.org/issues/17398#note-12 + # Ripper doesn't emit `args_add_block` when endless method is prefixed by modifier. "4.0/endless_methods_command_call.txt", # https://bugs.ruby-lang.org/issues/21168#note-5 "command_method_call_2.txt", ] + if RUBY_VERSION.start_with?("3.3.") + incorrect += [ + "whitequark/lvar_injecting_match.txt", + "seattlerb/parse_pattern_058.txt", + "regex_char_width.txt", + ] + end + # Skip these tests that we haven't implemented yet. omitted = [ "dos_endings.txt", @@ -68,7 +58,7 @@ class RipperTest < TestCase "whitequark/slash_newline_in_heredocs.txt" ] - Fixture.each(except: incorrect | omitted) do |fixture| + Fixture.each_for_current_ruby(except: incorrect | omitted) do |fixture| define_method(fixture.test_name) { assert_ripper(fixture.read) } end diff --git a/test/prism/snippets_test.rb b/test/prism/snippets_test.rb index 3160442cc07653..3c28d27a250e22 100644 --- a/test/prism/snippets_test.rb +++ b/test/prism/snippets_test.rb @@ -18,7 +18,7 @@ class SnippetsTest < TestCase "whitequark/multiple_pattern_matches.txt" ] - Fixture.each_with_version(except: except) do |fixture, version| + Fixture.each_with_all_versions(except: except) do |fixture, version| define_method(fixture.test_name(version)) { assert_snippets(fixture, version) } end diff --git a/test/prism/test_helper.rb b/test/prism/test_helper.rb index 42555738cf318f..f78e68e87c107a 100644 --- a/test/prism/test_helper.rb +++ b/test/prism/test_helper.rb @@ -72,7 +72,18 @@ def self.each(except: [], &block) paths.each { |path| yield Fixture.new(path) } end - def self.each_with_version(except: [], &block) + def self.each_for_version(except: [], version:, &block) + each(except: except) do |fixture| + next unless TestCase.ruby_versions_for(fixture.path).include?(version) + yield fixture + end + end + + def self.each_for_current_ruby(except: [], &block) + each_for_version(except: except, version: CURRENT_MAJOR_MINOR, &block) + end + + def self.each_with_all_versions(except: [], &block) each(except: except) do |fixture| TestCase.ruby_versions_for(fixture.path).each do |version| yield fixture, version @@ -232,6 +243,9 @@ def self.windows? # All versions that prism can parse SYNTAX_VERSIONS = %w[3.3 3.4 4.0] + # `RUBY_VERSION` with the patch version excluded + CURRENT_MAJOR_MINOR = RUBY_VERSION.split(".")[0, 2].join(".") + # Returns an array of ruby versions that a given filepath should test against: # test.txt # => all available versions # 3.4/test.txt # => versions since 3.4 (inclusive) @@ -250,13 +264,9 @@ def self.ruby_versions_for(filepath) end end - def current_major_minor - RUBY_VERSION.split(".")[0, 2].join(".") - end - if RUBY_VERSION >= "3.3.0" def test_all_syntax_versions_present - assert_include(SYNTAX_VERSIONS, current_major_minor) + assert_include(SYNTAX_VERSIONS, CURRENT_MAJOR_MINOR) end end