diff --git a/lib/rubygems/commands/setup_command.rb b/lib/rubygems/commands/setup_command.rb
index 8fcece5d5c..175599967c 100644
--- a/lib/rubygems/commands/setup_command.rb
+++ b/lib/rubygems/commands/setup_command.rb
@@ -402,8 +402,10 @@ By default, this RubyGems will install gem as:
           install_dir: default_dir,
           wrappers: true
         )
-        installer.install
-        File.delete installer.spec_file
+        # We need to install only executable and default spec files.
+        # lib/bundler.rb and lib/bundler/* are available under the site_ruby directory.
+        installer.extract_bin
+        installer.generate_bin
         installer.write_default_spec
       ensure
         FileUtils.rm_f built_gem
diff --git a/test/rubygems/test_gem_commands_setup_command.rb b/test/rubygems/test_gem_commands_setup_command.rb
index dfd951268d..b33e05ab28 100644
--- a/test/rubygems/test_gem_commands_setup_command.rb
+++ b/test/rubygems/test_gem_commands_setup_command.rb
@@ -31,6 +31,7 @@ class TestGemCommandsSetupCommand < Gem::TestCase
     gemspec = util_spec "bundler", "9.9.9" do |s|
       s.bindir = "exe"
       s.executables = ["bundle", "bundler"]
+      s.files = ["lib/bundler.rb"]
     end
 
     File.open "bundler/bundler.gemspec", "w" do |io|
@@ -222,6 +223,9 @@ class TestGemCommandsSetupCommand < Gem::TestCase
 
     assert_path_exist "#{Gem.dir}/gems/bundler-#{bundler_version}"
     assert_path_exist "#{Gem.dir}/gems/bundler-audit-1.0.0"
+
+    assert_path_exist "#{Gem.dir}/gems/bundler-#{bundler_version}/exe/bundle"
+    assert_path_not_exist "#{Gem.dir}/gems/bundler-#{bundler_version}/lib/bundler.rb"
   end
 
   def test_install_default_bundler_gem_with_default_gems_not_installed_at_default_dir