diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index adbfe328cc..99ab85ddbe 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -3807,6 +3807,36 @@ CODE end end + def test_encode_fallback_not_string_memory_leak + { + "hash" => <<~RUBY, + fallback = Hash.new { Object.new } + RUBY + "proc" => <<~RUBY, + fallback = proc { Object.new } + RUBY + "method" => <<~RUBY, + def my_method(_str) = Object.new + fallback = method(:my_method) + RUBY + "aref" => <<~RUBY, + fallback = Object.new + def fallback.[](_str) = Object.new + RUBY + }.each do |type, code| + assert_no_memory_leak([], '', <<~RUBY, "fallback type is #{type}", rss: true) + class MyError < StandardError; end + + #{code} + + 100_000.times do |i| + "\\ufffd".encode(Encoding::US_ASCII, fallback:) + rescue TypeError + end + RUBY + end + end + private def assert_bytesplice_result(expected, s, *args) diff --git a/transcode.c b/transcode.c index 20b92b66f7..86e828c479 100644 --- a/transcode.c +++ b/transcode.c @@ -2360,7 +2360,13 @@ transcode_loop_fallback_try(VALUE a) { struct transcode_loop_fallback_args *args = (struct transcode_loop_fallback_args *)a; - return args->fallback_func(args->fallback, args->rep); + VALUE ret = args->fallback_func(args->fallback, args->rep); + + if (!UNDEF_P(ret) && !NIL_P(ret)) { + StringValue(ret); + } + + return ret; } static void @@ -2428,7 +2434,6 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos, } if (!UNDEF_P(rep) && !NIL_P(rep)) { - StringValue(rep); ret = rb_econv_insert_output(ec, (const unsigned char *)RSTRING_PTR(rep), RSTRING_LEN(rep), rb_enc_name(rb_enc_get(rep))); if ((int)ret == -1) {