[ruby/rubygems] Pass down value of BUNDLE_JOBS to RubyGems before compiling:

- ### Problem

  Since https://github.com/ruby/rubygems/pull/9131, we are now
  compiling make rules simultaneously. The number of jobs
  is equal to the number of processors.
  This may be problematic for some users as they want to control
  this value.

  ### Solution

  The number of jobs passed to `make` will now be equal to the
  `BUNDLE_JOBS` value.

  ### Side note

  It's also worth to note that since Bundler installs gems in
  parallel, we may end up running multiple `make -j<JOB>` in parallel
  which would cause exhaust the number of processors we have.
  This problem can be fixed by implementing a GNU jobserver, which I
  plan to do. But I felt that this would be too much change in one PR.

https://github.com/ruby/rubygems/commit/d51995deb9

lib/bundler/rubygems_gem_installer.rb

@@ -103,6 +103,10 @@ module Bundler
       end
     end
 
+    def build_jobs
+      Bundler.settings[:jobs] || super
+    end
+
     def build_extensions
       extension_cache_path = options[:bundler_extension_cache_path]
       extension_dir = spec.extension_dir

lib/rubygems/ext/builder.rb

@@ -22,7 +22,7 @@ class Gem::Ext::Builder
   end
 
   def self.make(dest_path, results, make_dir = Dir.pwd, sitedir = nil, targets = ["clean", "", "install"],
-    target_rbconfig: Gem.target_rbconfig)
+    target_rbconfig: Gem.target_rbconfig, n_jobs: nil)
     unless File.exist? File.join(make_dir, "Makefile")
       # No makefile exists, nothing to do.
       raise NoMakefileError, "No Makefile found in #{make_dir}"
@@ -34,8 +34,18 @@ class Gem::Ext::Builder
     make_program_name ||= RUBY_PLATFORM.include?("mswin") ? "nmake" : "make"
     make_program = shellsplit(make_program_name)
 
+    is_nmake = /\bnmake/i.match?(make_program_name)
     # The installation of the bundled gems is failed when DESTDIR is empty in mswin platform.
-    destdir = /\bnmake/i !~ make_program_name || ENV["DESTDIR"] && ENV["DESTDIR"] != "" ? format("DESTDIR=%s", ENV["DESTDIR"]) : ""
+    destdir = !is_nmake || ENV["DESTDIR"] && ENV["DESTDIR"] != "" ? format("DESTDIR=%s", ENV["DESTDIR"]) : ""
+
+    # nmake doesn't support parallel build
+    unless is_nmake
+      have_make_arguments = make_program.size > 1
+
+      if !have_make_arguments && !ENV["MAKEFLAGS"] && n_jobs
+        make_program << "-j#{n_jobs}"
+      end
+    end
 
     env = [destdir]
 
@@ -147,11 +157,12 @@ class Gem::Ext::Builder
   # have build arguments, saved, set +build_args+ which is an ARGV-style
   # array.
 
-  def initialize(spec, build_args = spec.build_args, target_rbconfig = Gem.target_rbconfig)
+  def initialize(spec, build_args = spec.build_args, target_rbconfig = Gem.target_rbconfig, build_jobs = nil)
     @spec       = spec
     @build_args = build_args
     @gem_dir    = spec.full_gem_path
     @target_rbconfig = target_rbconfig
+    @build_jobs = build_jobs
 
     @ran_rake = false
   end
@@ -208,7 +219,7 @@ EOF
       FileUtils.mkdir_p dest_path
 
       results = builder.build(extension, dest_path,
-                              results, @build_args, lib_dir, extension_dir, @target_rbconfig)
+                              results, @build_args, lib_dir, extension_dir, @target_rbconfig, n_jobs: @build_jobs)
 
       verbose { results.join("\n") }
 

lib/rubygems/ext/cargo_builder.rb

@@ -15,7 +15,7 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder
   end
 
   def build(extension, dest_path, results, args = [], lib_dir = nil, cargo_dir = Dir.pwd,
-    target_rbconfig = Gem.target_rbconfig)
+    target_rbconfig = Gem.target_rbconfig, n_jobs: nil)
     require "tempfile"
     require "fileutils"
 

lib/rubygems/ext/cmake_builder.rb

@@ -37,7 +37,7 @@ class Gem::Ext::CmakeBuilder < Gem::Ext::Builder
   end
 
   def build(extension, dest_path, results, args = [], lib_dir = nil, cmake_dir = Dir.pwd,
-    target_rbconfig = Gem.target_rbconfig)
+    target_rbconfig = Gem.target_rbconfig, n_jobs: nil)
     if target_rbconfig.path
       warn "--target-rbconfig is not yet supported for CMake extensions. Ignoring"
     end

lib/rubygems/ext/configure_builder.rb

@@ -8,7 +8,7 @@
 
 class Gem::Ext::ConfigureBuilder < Gem::Ext::Builder
   def self.build(extension, dest_path, results, args = [], lib_dir = nil, configure_dir = Dir.pwd,
-    target_rbconfig = Gem.target_rbconfig)
+    target_rbconfig = Gem.target_rbconfig, n_jobs: nil)
     if target_rbconfig.path
       warn "--target-rbconfig is not yet supported for configure-based extensions. Ignoring"
     end
@@ -19,7 +19,7 @@ class Gem::Ext::ConfigureBuilder < Gem::Ext::Builder
       run cmd, results, class_name, configure_dir
     end
 
-    make dest_path, results, configure_dir, target_rbconfig: target_rbconfig
+    make dest_path, results, configure_dir, target_rbconfig: target_rbconfig, n_jobs: n_jobs
 
     results
   end

lib/rubygems/ext/ext_conf_builder.rb

@@ -8,7 +8,7 @@
 
 class Gem::Ext::ExtConfBuilder < Gem::Ext::Builder
   def self.build(extension, dest_path, results, args = [], lib_dir = nil, extension_dir = Dir.pwd,
-    target_rbconfig = Gem.target_rbconfig)
+    target_rbconfig = Gem.target_rbconfig, n_jobs: nil)
     require "fileutils"
     require "tempfile"
 
@@ -40,11 +40,8 @@ class Gem::Ext::ExtConfBuilder < Gem::Ext::Builder
       end
 
       ENV["DESTDIR"] = nil
-      unless RUBY_PLATFORM.include?("mswin") && RbConfig::CONFIG["configure_args"]&.include?("nmake")
-        ENV["MAKEFLAGS"] ||= "-j#{Etc.nprocessors + 1}"
-      end
 
-      make dest_path, results, extension_dir, tmp_dest_relative, target_rbconfig: target_rbconfig
+      make dest_path, results, extension_dir, tmp_dest_relative, target_rbconfig: target_rbconfig, n_jobs: n_jobs
 
       full_tmp_dest = File.join(extension_dir, tmp_dest_relative)
 

lib/rubygems/ext/rake_builder.rb

@@ -8,7 +8,7 @@
 
 class Gem::Ext::RakeBuilder < Gem::Ext::Builder
   def self.build(extension, dest_path, results, args = [], lib_dir = nil, extension_dir = Dir.pwd,
-    target_rbconfig = Gem.target_rbconfig)
+    target_rbconfig = Gem.target_rbconfig, n_jobs: nil)
     if target_rbconfig.path
       warn "--target-rbconfig is not yet supported for Rake extensions. Ignoring"
     end

lib/rubygems/installer.rb

@@ -635,6 +635,7 @@ class Gem::Installer
     @build_root          = options[:build_root]
 
     @build_args = options[:build_args]
+    @build_jobs = options[:build_jobs]
 
     @gem_home = @install_dir || user_install_dir || Gem.dir
 
@@ -803,7 +804,7 @@ class Gem::Installer
   # configure scripts and rakefiles or mkrf_conf files.
 
   def build_extensions
-    builder = Gem::Ext::Builder.new spec, build_args, Gem.target_rbconfig
+    builder = Gem::Ext::Builder.new spec, build_args, Gem.target_rbconfig, build_jobs
 
     builder.build_extensions
   end
@@ -941,6 +942,10 @@ class Gem::Installer
                     end
   end
 
+  def build_jobs
+    @build_jobs ||= Etc.nprocessors + 1
+  end
+
   def rb_config
     Gem.target_rbconfig
   end

spec/bundler/commands/install_spec.rb

@@ -1306,6 +1306,79 @@ RSpec.describe "bundle install with gem sources" do
     end
   end
 
+  describe "parallel make" do
+    before do
+      unless Gem::Installer.private_method_defined?(:build_jobs)
+        skip "This example is runnable when RubyGems::Installer implements `build_jobs`"
+      end
+
+      @old_makeflags = ENV["MAKEFLAGS"]
+      @gemspec = nil
+
+      extconf_code = <<~CODE
+        require "mkmf"
+        create_makefile("foo")
+      CODE
+
+      build_repo4 do
+        build_gem "mypsych", "4.0.6" do |s|
+          @gemspec = s
+          extension = "ext/mypsych/extconf.rb"
+          s.extensions = extension
+
+          s.write(extension, extconf_code)
+        end
+      end
+    end
+
+    after do
+      if @old_makeflags
+        ENV["MAKEFLAGS"] = @old_makeflags
+      else
+        ENV.delete("MAKEFLAGS")
+      end
+    end
+
+    it "doesn't pass down -j to make when MAKEFLAGS is set" do
+      ENV["MAKEFLAGS"] = "-j1"
+
+      install_gemfile(<<~G, env: { "BUNDLE_JOBS" => "8" })
+        source "https://gem.repo4"
+        gem "mypsych"
+      G
+
+      gem_make_out = File.read(File.join(@gemspec.extension_dir, "gem_make.out"))
+
+      expect(gem_make_out).not_to include("make -j8")
+    end
+
+    it "pass down the BUNDLE_JOBS to RubyGems when running the compilation of an extension" do
+      ENV.delete("MAKEFLAGS")
+
+      install_gemfile(<<~G, env: { "BUNDLE_JOBS" => "8" })
+        source "https://gem.repo4"
+        gem "mypsych"
+      G
+
+      gem_make_out = File.read(File.join(@gemspec.extension_dir, "gem_make.out"))
+
+      expect(gem_make_out).to include("make -j8")
+    end
+
+    it "uses nprocessors by default" do
+      ENV.delete("MAKEFLAGS")
+
+      install_gemfile(<<~G)
+        source "https://gem.repo4"
+        gem "mypsych"
+      G
+
+      gem_make_out = File.read(File.join(@gemspec.extension_dir, "gem_make.out"))
+
+      expect(gem_make_out).to include("make -j#{Etc.nprocessors + 1}")
+    end
+  end
+
   describe "when configured path is UTF-8 and a file inside a gem package too" do
     let(:app_path) do
       path = tmp("♥")