From 9f593156b6184718a65f99905e6b07002058ebe6 Mon Sep 17 00:00:00 2001 From: Edouard CHIN Date: Mon, 8 Dec 2025 22:02:03 +0100 Subject: [PATCH] [ruby/rubygems] Pass down value of `BUNDLE_JOBS` to RubyGems before compiling: - ### Problem Since https://github.com/ruby/rubygems/pull/9131, we are now compiling make rules simultaneously. The number of jobs is equal to the number of processors. This may be problematic for some users as they want to control this value. ### Solution The number of jobs passed to `make` will now be equal to the `BUNDLE_JOBS` value. ### Side note It's also worth to note that since Bundler installs gems in parallel, we may end up running multiple `make -j` in parallel which would cause exhaust the number of processors we have. This problem can be fixed by implementing a GNU jobserver, which I plan to do. But I felt that this would be too much change in one PR. https://github.com/ruby/rubygems/commit/d51995deb9 --- lib/bundler/rubygems_gem_installer.rb | 4 ++ lib/rubygems/ext/builder.rb | 19 +++++-- lib/rubygems/ext/cargo_builder.rb | 2 +- lib/rubygems/ext/cmake_builder.rb | 2 +- lib/rubygems/ext/configure_builder.rb | 4 +- lib/rubygems/ext/ext_conf_builder.rb | 7 +-- lib/rubygems/ext/rake_builder.rb | 2 +- lib/rubygems/installer.rb | 7 ++- spec/bundler/commands/install_spec.rb | 73 +++++++++++++++++++++++++++ 9 files changed, 105 insertions(+), 15 deletions(-) diff --git a/lib/bundler/rubygems_gem_installer.rb b/lib/bundler/rubygems_gem_installer.rb index 0da5ed236b..64ce6193d3 100644 --- a/lib/bundler/rubygems_gem_installer.rb +++ b/lib/bundler/rubygems_gem_installer.rb @@ -103,6 +103,10 @@ module Bundler end end + def build_jobs + Bundler.settings[:jobs] || super + end + def build_extensions extension_cache_path = options[:bundler_extension_cache_path] extension_dir = spec.extension_dir diff --git a/lib/rubygems/ext/builder.rb b/lib/rubygems/ext/builder.rb index 600a6a5ff6..350daf1e16 100644 --- a/lib/rubygems/ext/builder.rb +++ b/lib/rubygems/ext/builder.rb @@ -22,7 +22,7 @@ class Gem::Ext::Builder end def self.make(dest_path, results, make_dir = Dir.pwd, sitedir = nil, targets = ["clean", "", "install"], - target_rbconfig: Gem.target_rbconfig) + target_rbconfig: Gem.target_rbconfig, n_jobs: nil) unless File.exist? File.join(make_dir, "Makefile") # No makefile exists, nothing to do. raise NoMakefileError, "No Makefile found in #{make_dir}" @@ -34,8 +34,18 @@ class Gem::Ext::Builder make_program_name ||= RUBY_PLATFORM.include?("mswin") ? "nmake" : "make" make_program = shellsplit(make_program_name) + is_nmake = /\bnmake/i.match?(make_program_name) # The installation of the bundled gems is failed when DESTDIR is empty in mswin platform. - destdir = /\bnmake/i !~ make_program_name || ENV["DESTDIR"] && ENV["DESTDIR"] != "" ? format("DESTDIR=%s", ENV["DESTDIR"]) : "" + destdir = !is_nmake || ENV["DESTDIR"] && ENV["DESTDIR"] != "" ? format("DESTDIR=%s", ENV["DESTDIR"]) : "" + + # nmake doesn't support parallel build + unless is_nmake + have_make_arguments = make_program.size > 1 + + if !have_make_arguments && !ENV["MAKEFLAGS"] && n_jobs + make_program << "-j#{n_jobs}" + end + end env = [destdir] @@ -147,11 +157,12 @@ class Gem::Ext::Builder # have build arguments, saved, set +build_args+ which is an ARGV-style # array. - def initialize(spec, build_args = spec.build_args, target_rbconfig = Gem.target_rbconfig) + def initialize(spec, build_args = spec.build_args, target_rbconfig = Gem.target_rbconfig, build_jobs = nil) @spec = spec @build_args = build_args @gem_dir = spec.full_gem_path @target_rbconfig = target_rbconfig + @build_jobs = build_jobs @ran_rake = false end @@ -208,7 +219,7 @@ EOF FileUtils.mkdir_p dest_path results = builder.build(extension, dest_path, - results, @build_args, lib_dir, extension_dir, @target_rbconfig) + results, @build_args, lib_dir, extension_dir, @target_rbconfig, n_jobs: @build_jobs) verbose { results.join("\n") } diff --git a/lib/rubygems/ext/cargo_builder.rb b/lib/rubygems/ext/cargo_builder.rb index 6bf3b405ad..42dca3b102 100644 --- a/lib/rubygems/ext/cargo_builder.rb +++ b/lib/rubygems/ext/cargo_builder.rb @@ -15,7 +15,7 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder end def build(extension, dest_path, results, args = [], lib_dir = nil, cargo_dir = Dir.pwd, - target_rbconfig = Gem.target_rbconfig) + target_rbconfig = Gem.target_rbconfig, n_jobs: nil) require "tempfile" require "fileutils" diff --git a/lib/rubygems/ext/cmake_builder.rb b/lib/rubygems/ext/cmake_builder.rb index 2915568b39..e660ed558b 100644 --- a/lib/rubygems/ext/cmake_builder.rb +++ b/lib/rubygems/ext/cmake_builder.rb @@ -37,7 +37,7 @@ class Gem::Ext::CmakeBuilder < Gem::Ext::Builder end def build(extension, dest_path, results, args = [], lib_dir = nil, cmake_dir = Dir.pwd, - target_rbconfig = Gem.target_rbconfig) + target_rbconfig = Gem.target_rbconfig, n_jobs: nil) if target_rbconfig.path warn "--target-rbconfig is not yet supported for CMake extensions. Ignoring" end diff --git a/lib/rubygems/ext/configure_builder.rb b/lib/rubygems/ext/configure_builder.rb index 76c1cd8b19..230b214b3c 100644 --- a/lib/rubygems/ext/configure_builder.rb +++ b/lib/rubygems/ext/configure_builder.rb @@ -8,7 +8,7 @@ class Gem::Ext::ConfigureBuilder < Gem::Ext::Builder def self.build(extension, dest_path, results, args = [], lib_dir = nil, configure_dir = Dir.pwd, - target_rbconfig = Gem.target_rbconfig) + target_rbconfig = Gem.target_rbconfig, n_jobs: nil) if target_rbconfig.path warn "--target-rbconfig is not yet supported for configure-based extensions. Ignoring" end @@ -19,7 +19,7 @@ class Gem::Ext::ConfigureBuilder < Gem::Ext::Builder run cmd, results, class_name, configure_dir end - make dest_path, results, configure_dir, target_rbconfig: target_rbconfig + make dest_path, results, configure_dir, target_rbconfig: target_rbconfig, n_jobs: n_jobs results end diff --git a/lib/rubygems/ext/ext_conf_builder.rb b/lib/rubygems/ext/ext_conf_builder.rb index 81491eac79..822454355d 100644 --- a/lib/rubygems/ext/ext_conf_builder.rb +++ b/lib/rubygems/ext/ext_conf_builder.rb @@ -8,7 +8,7 @@ class Gem::Ext::ExtConfBuilder < Gem::Ext::Builder def self.build(extension, dest_path, results, args = [], lib_dir = nil, extension_dir = Dir.pwd, - target_rbconfig = Gem.target_rbconfig) + target_rbconfig = Gem.target_rbconfig, n_jobs: nil) require "fileutils" require "tempfile" @@ -40,11 +40,8 @@ class Gem::Ext::ExtConfBuilder < Gem::Ext::Builder end ENV["DESTDIR"] = nil - unless RUBY_PLATFORM.include?("mswin") && RbConfig::CONFIG["configure_args"]&.include?("nmake") - ENV["MAKEFLAGS"] ||= "-j#{Etc.nprocessors + 1}" - end - make dest_path, results, extension_dir, tmp_dest_relative, target_rbconfig: target_rbconfig + make dest_path, results, extension_dir, tmp_dest_relative, target_rbconfig: target_rbconfig, n_jobs: n_jobs full_tmp_dest = File.join(extension_dir, tmp_dest_relative) diff --git a/lib/rubygems/ext/rake_builder.rb b/lib/rubygems/ext/rake_builder.rb index 0eac5a180c..d702d7f339 100644 --- a/lib/rubygems/ext/rake_builder.rb +++ b/lib/rubygems/ext/rake_builder.rb @@ -8,7 +8,7 @@ class Gem::Ext::RakeBuilder < Gem::Ext::Builder def self.build(extension, dest_path, results, args = [], lib_dir = nil, extension_dir = Dir.pwd, - target_rbconfig = Gem.target_rbconfig) + target_rbconfig = Gem.target_rbconfig, n_jobs: nil) if target_rbconfig.path warn "--target-rbconfig is not yet supported for Rake extensions. Ignoring" end diff --git a/lib/rubygems/installer.rb b/lib/rubygems/installer.rb index 4c3038770d..90aa25dc07 100644 --- a/lib/rubygems/installer.rb +++ b/lib/rubygems/installer.rb @@ -635,6 +635,7 @@ class Gem::Installer @build_root = options[:build_root] @build_args = options[:build_args] + @build_jobs = options[:build_jobs] @gem_home = @install_dir || user_install_dir || Gem.dir @@ -803,7 +804,7 @@ class Gem::Installer # configure scripts and rakefiles or mkrf_conf files. def build_extensions - builder = Gem::Ext::Builder.new spec, build_args, Gem.target_rbconfig + builder = Gem::Ext::Builder.new spec, build_args, Gem.target_rbconfig, build_jobs builder.build_extensions end @@ -941,6 +942,10 @@ class Gem::Installer end end + def build_jobs + @build_jobs ||= Etc.nprocessors + 1 + end + def rb_config Gem.target_rbconfig end diff --git a/spec/bundler/commands/install_spec.rb b/spec/bundler/commands/install_spec.rb index 41b3a865cd..ae651bf981 100644 --- a/spec/bundler/commands/install_spec.rb +++ b/spec/bundler/commands/install_spec.rb @@ -1306,6 +1306,79 @@ RSpec.describe "bundle install with gem sources" do end end + describe "parallel make" do + before do + unless Gem::Installer.private_method_defined?(:build_jobs) + skip "This example is runnable when RubyGems::Installer implements `build_jobs`" + end + + @old_makeflags = ENV["MAKEFLAGS"] + @gemspec = nil + + extconf_code = <<~CODE + require "mkmf" + create_makefile("foo") + CODE + + build_repo4 do + build_gem "mypsych", "4.0.6" do |s| + @gemspec = s + extension = "ext/mypsych/extconf.rb" + s.extensions = extension + + s.write(extension, extconf_code) + end + end + end + + after do + if @old_makeflags + ENV["MAKEFLAGS"] = @old_makeflags + else + ENV.delete("MAKEFLAGS") + end + end + + it "doesn't pass down -j to make when MAKEFLAGS is set" do + ENV["MAKEFLAGS"] = "-j1" + + install_gemfile(<<~G, env: { "BUNDLE_JOBS" => "8" }) + source "https://gem.repo4" + gem "mypsych" + G + + gem_make_out = File.read(File.join(@gemspec.extension_dir, "gem_make.out")) + + expect(gem_make_out).not_to include("make -j8") + end + + it "pass down the BUNDLE_JOBS to RubyGems when running the compilation of an extension" do + ENV.delete("MAKEFLAGS") + + install_gemfile(<<~G, env: { "BUNDLE_JOBS" => "8" }) + source "https://gem.repo4" + gem "mypsych" + G + + gem_make_out = File.read(File.join(@gemspec.extension_dir, "gem_make.out")) + + expect(gem_make_out).to include("make -j8") + end + + it "uses nprocessors by default" do + ENV.delete("MAKEFLAGS") + + install_gemfile(<<~G) + source "https://gem.repo4" + gem "mypsych" + G + + gem_make_out = File.read(File.join(@gemspec.extension_dir, "gem_make.out")) + + expect(gem_make_out).to include("make -j#{Etc.nprocessors + 1}") + end + end + describe "when configured path is UTF-8 and a file inside a gem package too" do let(:app_path) do path = tmp("♥")