mirror of
https://github.com/ruby/ruby.git
synced 2026-01-26 20:19:19 +00:00
948 lines
32 KiB
Ruby
Executable File
948 lines
32 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
# Sync upstream github repositories to ruby repository.
|
|
# See `tool/sync_default_gems.rb --help` for how to use this.
|
|
|
|
require 'fileutils'
|
|
require "rbconfig"
|
|
require "find"
|
|
require "tempfile"
|
|
|
|
module SyncDefaultGems
|
|
include FileUtils
|
|
extend FileUtils
|
|
|
|
module_function
|
|
|
|
# upstream: "owner/repo"
|
|
# branch: "branch_name"
|
|
# mappings: [ ["path_in_upstream", "path_in_ruby"], ... ]
|
|
# NOTE: path_in_ruby is assumed to be "owned" by this gem, and the contents
|
|
# will be removed before sync
|
|
# exclude: [ "fnmatch_pattern_after_mapping", ... ]
|
|
Repository = Data.define(:upstream, :branch, :mappings, :exclude) do
|
|
def excluded?(newpath)
|
|
p = newpath
|
|
until p == "."
|
|
return true if exclude.any? {|pat| File.fnmatch?(pat, p, File::FNM_PATHNAME|File::FNM_EXTGLOB)}
|
|
p = File.dirname(p)
|
|
end
|
|
false
|
|
end
|
|
|
|
def rewrite_for_ruby(path)
|
|
newpath = mappings.find do |src, dst|
|
|
if path == src || path.start_with?(src + "/")
|
|
break path.sub(src, dst)
|
|
end
|
|
end
|
|
return nil unless newpath
|
|
return nil if excluded?(newpath)
|
|
newpath
|
|
end
|
|
end
|
|
|
|
CLASSICAL_DEFAULT_BRANCH = "master"
|
|
|
|
def repo((upstream, branch), mappings, exclude: [])
|
|
branch ||= CLASSICAL_DEFAULT_BRANCH
|
|
exclude += ["ext/**/depend"]
|
|
Repository.new(upstream:, branch:, mappings:, exclude:)
|
|
end
|
|
|
|
def lib((upstream, branch), gemspec_in_subdir: false)
|
|
_org, name = upstream.split("/")
|
|
gemspec_dst = gemspec_in_subdir ? "lib/#{name}/#{name}.gemspec" : "lib/#{name}.gemspec"
|
|
repo([upstream, branch], [
|
|
["lib/#{name}.rb", "lib/#{name}.rb"],
|
|
["lib/#{name}", "lib/#{name}"],
|
|
["test/test_#{name}.rb", "test/test_#{name}.rb"],
|
|
["test/#{name}", "test/#{name}"],
|
|
["#{name}.gemspec", gemspec_dst],
|
|
])
|
|
end
|
|
|
|
# Note: tool/auto_review_pr.rb also depends on these constants.
|
|
NO_UPSTREAM = [
|
|
"lib/unicode_normalize", # not to match with "lib/un"
|
|
]
|
|
REPOSITORIES = {
|
|
Onigmo: repo("k-takata/Onigmo", [
|
|
["regcomp.c", "regcomp.c"],
|
|
["regenc.c", "regenc.c"],
|
|
["regenc.h", "regenc.h"],
|
|
["regerror.c", "regerror.c"],
|
|
["regexec.c", "regexec.c"],
|
|
["regint.h", "regint.h"],
|
|
["regparse.c", "regparse.c"],
|
|
["regparse.h", "regparse.h"],
|
|
["regsyntax.c", "regsyntax.c"],
|
|
["onigmo.h", "include/ruby/onigmo.h"],
|
|
["enc", "enc"],
|
|
]),
|
|
"io-console": repo("ruby/io-console", [
|
|
["ext/io/console", "ext/io/console"],
|
|
["test/io/console", "test/io/console"],
|
|
["lib/io/console", "ext/io/console/lib/console"],
|
|
["io-console.gemspec", "ext/io/console/io-console.gemspec"],
|
|
]),
|
|
"io-nonblock": repo("ruby/io-nonblock", [
|
|
["ext/io/nonblock", "ext/io/nonblock"],
|
|
["test/io/nonblock", "test/io/nonblock"],
|
|
["io-nonblock.gemspec", "ext/io/nonblock/io-nonblock.gemspec"],
|
|
]),
|
|
"io-wait": repo("ruby/io-wait", [
|
|
["ext/io/wait", "ext/io/wait"],
|
|
["test/io/wait", "test/io/wait"],
|
|
["io-wait.gemspec", "ext/io/wait/io-wait.gemspec"],
|
|
]),
|
|
"net-http": repo("ruby/net-http", [
|
|
["lib/net/http.rb", "lib/net/http.rb"],
|
|
["lib/net/http", "lib/net/http"],
|
|
["test/net/http", "test/net/http"],
|
|
["net-http.gemspec", "lib/net/http/net-http.gemspec"],
|
|
]),
|
|
"net-protocol": repo("ruby/net-protocol", [
|
|
["lib/net/protocol.rb", "lib/net/protocol.rb"],
|
|
["test/net/protocol", "test/net/protocol"],
|
|
["net-protocol.gemspec", "lib/net/net-protocol.gemspec"],
|
|
]),
|
|
"open-uri": lib("ruby/open-uri"),
|
|
"win32-registry": repo("ruby/win32-registry", [
|
|
["lib/win32/registry.rb", "ext/win32/lib/win32/registry.rb"],
|
|
["test/win32/test_registry.rb", "test/win32/test_registry.rb"],
|
|
["win32-registry.gemspec", "ext/win32/win32-registry.gemspec"],
|
|
]),
|
|
English: lib("ruby/English"),
|
|
cgi: repo("ruby/cgi", [
|
|
["ext/cgi", "ext/cgi"],
|
|
["lib/cgi/escape.rb", "lib/cgi/escape.rb"],
|
|
["test/cgi/test_cgi_escape.rb", "test/cgi/test_cgi_escape.rb"],
|
|
["test/cgi/update_env.rb", "test/cgi/update_env.rb"],
|
|
]),
|
|
date: repo("ruby/date", [
|
|
["doc/date", "doc/date"],
|
|
["ext/date", "ext/date"],
|
|
["lib", "ext/date/lib"],
|
|
["test/date", "test/date"],
|
|
["date.gemspec", "ext/date/date.gemspec"],
|
|
], exclude: [
|
|
"ext/date/lib/date_core.bundle",
|
|
]),
|
|
delegate: lib("ruby/delegate"),
|
|
did_you_mean: repo("ruby/did_you_mean", [
|
|
["lib/did_you_mean.rb", "lib/did_you_mean.rb"],
|
|
["lib/did_you_mean", "lib/did_you_mean"],
|
|
["test", "test/did_you_mean"],
|
|
["did_you_mean.gemspec", "lib/did_you_mean/did_you_mean.gemspec"],
|
|
], exclude: [
|
|
"test/did_you_mean/lib",
|
|
"test/did_you_mean/tree_spell/test_explore.rb",
|
|
]),
|
|
digest: repo("ruby/digest", [
|
|
["ext/digest/lib/digest/sha2", "ext/digest/sha2/lib/sha2"],
|
|
["ext/digest", "ext/digest"],
|
|
["lib/digest.rb", "ext/digest/lib/digest.rb"],
|
|
["lib/digest/version.rb", "ext/digest/lib/digest/version.rb"],
|
|
["lib/digest/sha2.rb", "ext/digest/sha2/lib/sha2.rb"],
|
|
["test/digest", "test/digest"],
|
|
["digest.gemspec", "ext/digest/digest.gemspec"],
|
|
]),
|
|
erb: repo("ruby/erb", [
|
|
["ext/erb", "ext/erb"],
|
|
["lib/erb", "lib/erb"],
|
|
["lib/erb.rb", "lib/erb.rb"],
|
|
["test/erb", "test/erb"],
|
|
["erb.gemspec", "lib/erb/erb.gemspec"],
|
|
["libexec/erb", "libexec/erb"],
|
|
]),
|
|
error_highlight: repo("ruby/error_highlight", [
|
|
["lib/error_highlight.rb", "lib/error_highlight.rb"],
|
|
["lib/error_highlight", "lib/error_highlight"],
|
|
["test", "test/error_highlight"],
|
|
["error_highlight.gemspec", "lib/error_highlight/error_highlight.gemspec"],
|
|
]),
|
|
etc: repo("ruby/etc", [
|
|
["ext/etc", "ext/etc"],
|
|
["test/etc", "test/etc"],
|
|
["etc.gemspec", "ext/etc/etc.gemspec"],
|
|
]),
|
|
fcntl: repo("ruby/fcntl", [
|
|
["ext/fcntl", "ext/fcntl"],
|
|
["fcntl.gemspec", "ext/fcntl/fcntl.gemspec"],
|
|
]),
|
|
fileutils: lib("ruby/fileutils"),
|
|
find: lib("ruby/find"),
|
|
forwardable: lib("ruby/forwardable", gemspec_in_subdir: true),
|
|
ipaddr: lib("ruby/ipaddr"),
|
|
json: repo("ruby/json", [
|
|
["ext/json/ext", "ext/json"],
|
|
["test/json", "test/json"],
|
|
["lib", "ext/json/lib"],
|
|
["json.gemspec", "ext/json/json.gemspec"],
|
|
], exclude: [
|
|
"ext/json/lib/json/ext/.keep",
|
|
"ext/json/lib/json/pure.rb",
|
|
"ext/json/lib/json/pure",
|
|
"ext/json/lib/json/truffle_ruby",
|
|
"test/json/lib",
|
|
"ext/json/extconf.rb",
|
|
]),
|
|
mmtk: repo(["ruby/mmtk", "main"], [
|
|
["gc/mmtk", "gc/mmtk"],
|
|
]),
|
|
open3: lib("ruby/open3", gemspec_in_subdir: true).tap {
|
|
it.exclude << "lib/open3/jruby_windows.rb"
|
|
},
|
|
openssl: repo("ruby/openssl", [
|
|
["ext/openssl", "ext/openssl"],
|
|
["lib", "ext/openssl/lib"],
|
|
["test/openssl", "test/openssl"],
|
|
["sample", "sample/openssl"],
|
|
["openssl.gemspec", "ext/openssl/openssl.gemspec"],
|
|
["History.md", "ext/openssl/History.md"],
|
|
], exclude: [
|
|
"test/openssl/envutil.rb",
|
|
"ext/openssl/depend",
|
|
]),
|
|
optparse: lib("ruby/optparse", gemspec_in_subdir: true).tap {
|
|
it.mappings << ["doc/optparse", "doc/optparse"]
|
|
},
|
|
pathname: repo("ruby/pathname", [
|
|
["ext/pathname/pathname.c", "pathname.c"],
|
|
["lib/pathname_builtin.rb", "pathname_builtin.rb"],
|
|
["lib/pathname.rb", "lib/pathname.rb"],
|
|
["test/pathname", "test/pathname"],
|
|
]),
|
|
pp: lib("ruby/pp"),
|
|
prettyprint: lib("ruby/prettyprint"),
|
|
prism: repo(["ruby/prism", "main"], [
|
|
["ext/prism", "prism"],
|
|
["lib/prism.rb", "lib/prism.rb"],
|
|
["lib/prism", "lib/prism"],
|
|
["test/prism", "test/prism"],
|
|
["src", "prism"],
|
|
["prism.gemspec", "lib/prism/prism.gemspec"],
|
|
["include/prism", "prism"],
|
|
["include/prism.h", "prism/prism.h"],
|
|
["config.yml", "prism/config.yml"],
|
|
["templates", "prism/templates"],
|
|
], exclude: [
|
|
"prism/templates/{javascript,java,rbi,sig}",
|
|
"test/prism/snapshots_test.rb",
|
|
"test/prism/snapshots",
|
|
"prism/extconf.rb",
|
|
"prism/srcs.mk*",
|
|
]),
|
|
psych: repo("ruby/psych", [
|
|
["ext/psych", "ext/psych"],
|
|
["lib", "ext/psych/lib"],
|
|
["test/psych", "test/psych"],
|
|
["psych.gemspec", "ext/psych/psych.gemspec"],
|
|
], exclude: [
|
|
"ext/psych/lib/org",
|
|
"ext/psych/lib/psych.jar",
|
|
"ext/psych/lib/psych_jars.rb",
|
|
"ext/psych/lib/psych.{bundle,so}",
|
|
"ext/psych/lib/2.*",
|
|
"ext/psych/yaml/LICENSE",
|
|
"ext/psych/.gitignore",
|
|
]),
|
|
resolv: repo("ruby/resolv", [
|
|
["lib/resolv.rb", "lib/resolv.rb"],
|
|
["test/resolv", "test/resolv"],
|
|
["resolv.gemspec", "lib/resolv.gemspec"],
|
|
["ext/win32/resolv/lib/resolv.rb", "ext/win32/lib/win32/resolv.rb"],
|
|
["ext/win32/resolv", "ext/win32/resolv"],
|
|
]),
|
|
rubygems: repo("ruby/rubygems", [
|
|
["lib/rubygems.rb", "lib/rubygems.rb"],
|
|
["lib/rubygems", "lib/rubygems"],
|
|
["test/rubygems", "test/rubygems"],
|
|
["bundler/lib/bundler.rb", "lib/bundler.rb"],
|
|
["bundler/lib/bundler", "lib/bundler"],
|
|
["bundler/exe/bundle", "libexec/bundle"],
|
|
["bundler/exe/bundler", "libexec/bundler"],
|
|
["bundler/bundler.gemspec", "lib/bundler/bundler.gemspec"],
|
|
["bundler/spec", "spec/bundler"],
|
|
*["bundle", "parallel_rspec", "rspec"].map {|binstub|
|
|
["bundler/bin/#{binstub}", "spec/bin/#{binstub}"]
|
|
},
|
|
*%w[dev_gems test_gems rubocop_gems standard_gems].flat_map {|gemfile|
|
|
["rb.lock", "rb"].map do |ext|
|
|
["tool/bundler/#{gemfile}.#{ext}", "tool/bundler/#{gemfile}.#{ext}"]
|
|
end
|
|
},
|
|
], exclude: [
|
|
"spec/bundler/bin",
|
|
"spec/bundler/support/artifice/vcr_cassettes",
|
|
"spec/bundler/support/artifice/used_cassettes.txt",
|
|
"lib/{bundler,rubygems}/**/{COPYING,LICENSE,README}{,.{md,txt,rdoc}}",
|
|
]),
|
|
securerandom: lib("ruby/securerandom"),
|
|
shellwords: lib("ruby/shellwords"),
|
|
singleton: lib("ruby/singleton"),
|
|
stringio: repo("ruby/stringio", [
|
|
["ext/stringio", "ext/stringio"],
|
|
["test/stringio", "test/stringio"],
|
|
["stringio.gemspec", "ext/stringio/stringio.gemspec"],
|
|
["doc/stringio", "doc/stringio"],
|
|
], exclude: [
|
|
"ext/stringio/README.md",
|
|
]),
|
|
strscan: repo("ruby/strscan", [
|
|
["ext/strscan", "ext/strscan"],
|
|
["lib", "ext/strscan/lib"],
|
|
["test/strscan", "test/strscan"],
|
|
["strscan.gemspec", "ext/strscan/strscan.gemspec"],
|
|
["doc/strscan", "doc/strscan"],
|
|
], exclude: [
|
|
"ext/strscan/regenc.h",
|
|
"ext/strscan/regint.h",
|
|
]),
|
|
syntax_suggest: lib(["ruby/syntax_suggest", "main"], gemspec_in_subdir: true),
|
|
tempfile: lib("ruby/tempfile"),
|
|
time: lib("ruby/time"),
|
|
timeout: lib("ruby/timeout"),
|
|
tmpdir: lib("ruby/tmpdir"),
|
|
tsort: lib("ruby/tsort"),
|
|
un: lib("ruby/un"),
|
|
uri: lib("ruby/uri", gemspec_in_subdir: true),
|
|
weakref: lib("ruby/weakref"),
|
|
yaml: lib("ruby/yaml", gemspec_in_subdir: true),
|
|
zlib: repo("ruby/zlib", [
|
|
["ext/zlib", "ext/zlib"],
|
|
["test/zlib", "test/zlib"],
|
|
["zlib.gemspec", "ext/zlib/zlib.gemspec"],
|
|
]),
|
|
}.transform_keys(&:to_s)
|
|
|
|
class << Repository
|
|
def find_upstream(file)
|
|
return if NO_UPSTREAM.any? {|dst| file.start_with?(dst) }
|
|
REPOSITORIES.find do |repo_name, repository|
|
|
if repository.mappings.any? {|_src, dst| file.start_with?(dst) }
|
|
break repo_name
|
|
end
|
|
end
|
|
end
|
|
|
|
def group(files)
|
|
files.group_by {|file| find_upstream(file)}
|
|
end
|
|
end
|
|
|
|
# Allow synchronizing commits up to this FETCH_DEPTH. We've historically merged PRs
|
|
# with about 250 commits to ruby/ruby, so we use this depth for ruby/ruby in general.
|
|
FETCH_DEPTH = 500
|
|
|
|
def pipe_readlines(args, rs: "\0", chomp: true)
|
|
IO.popen(args) do |f|
|
|
f.readlines(rs, chomp: chomp)
|
|
end
|
|
end
|
|
|
|
def porcelain_status(*pattern)
|
|
pipe_readlines(%W"git status --porcelain --no-renames -z --" + pattern)
|
|
end
|
|
|
|
def replace_rdoc_ref(file)
|
|
src = File.binread(file)
|
|
changed = false
|
|
changed |= src.gsub!(%r[\[\Khttps://docs\.ruby-lang\.org/en/master(?:/doc)?/(([A-Z]\w+(?:/[A-Z]\w+)*)|\w+_rdoc)\.html(\#\S+)?(?=\])]) do
|
|
name, mod, label = $1, $2, $3
|
|
mod &&= mod.gsub('/', '::')
|
|
if label && (m = label.match(/\A\#(?:method-([ci])|(?:(?:class|module)-#{mod}-)?label)-([-+\w]+)\z/))
|
|
scope, label = m[1], m[2]
|
|
scope = scope ? scope.tr('ci', '.#') : '@'
|
|
end
|
|
"rdoc-ref:#{mod || name.chomp("_rdoc") + ".rdoc"}#{scope}#{label}"
|
|
end
|
|
changed or return false
|
|
File.binwrite(file, src)
|
|
return true
|
|
end
|
|
|
|
def replace_rdoc_ref_all
|
|
result = porcelain_status("*.c", "*.rb", "*.rdoc")
|
|
result.map! {|line| line[/\A.M (.*)/, 1]}
|
|
result.compact!
|
|
return if result.empty?
|
|
result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + result)
|
|
result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)}
|
|
end
|
|
|
|
def replace_rdoc_ref_all_full
|
|
Dir.glob("**/*.{c,rb,rdoc}").inject(false) {|changed, file| changed | replace_rdoc_ref(file)}
|
|
end
|
|
|
|
def rubygems_do_fixup
|
|
gemspec_content = File.readlines("lib/bundler/bundler.gemspec").map do |line|
|
|
next if line =~ /LICENSE\.md/
|
|
|
|
line.gsub("bundler.gemspec", "lib/bundler/bundler.gemspec")
|
|
end.compact.join
|
|
File.write("lib/bundler/bundler.gemspec", gemspec_content)
|
|
|
|
["bundle", "parallel_rspec", "rspec"].each do |binstub|
|
|
path = "spec/bin/#{binstub}"
|
|
next unless File.exist?(path)
|
|
content = File.read(path).gsub("../spec", "../bundler")
|
|
File.write(path, content)
|
|
chmod("+x", path)
|
|
end
|
|
end
|
|
|
|
# We usually don't use this. Please consider using #sync_default_gems_with_commits instead.
|
|
def sync_default_gems(gem)
|
|
config = REPOSITORIES[gem]
|
|
puts "Sync #{config.upstream}"
|
|
|
|
upstream = File.join("..", "..", config.upstream)
|
|
|
|
config.mappings.each do |src, dst|
|
|
rm_rf(dst)
|
|
end
|
|
|
|
copied = Set.new
|
|
config.mappings.each do |src, dst|
|
|
prefix = File.join(upstream, src)
|
|
# Maybe mapping needs to be updated?
|
|
next unless File.exist?(prefix)
|
|
Find.find(prefix) do |path|
|
|
next if File.directory?(path)
|
|
if copied.add?(path)
|
|
newpath = config.rewrite_for_ruby(path.sub(%r{\A#{Regexp.escape(upstream)}/}, ""))
|
|
next unless newpath
|
|
mkdir_p(File.dirname(newpath))
|
|
cp(path, newpath)
|
|
end
|
|
end
|
|
end
|
|
|
|
porcelain_status().each do |line|
|
|
/\A(?:.)(?:.) (?<path>.*)\z/ =~ line or raise
|
|
if config.excluded?(path)
|
|
puts "Restoring excluded file: #{path}"
|
|
IO.popen(%W"git checkout --" + [path], "rb", &:read)
|
|
end
|
|
end
|
|
|
|
# RubyGems/Bundler needs special care
|
|
if gem == "rubygems"
|
|
rubygems_do_fixup
|
|
end
|
|
|
|
check_prerelease_version(gem)
|
|
|
|
# Architecture-dependent files must not pollute libdir.
|
|
rm_rf(Dir["lib/**/*.#{RbConfig::CONFIG['DLEXT']}"])
|
|
replace_rdoc_ref_all
|
|
end
|
|
|
|
def check_prerelease_version(gem)
|
|
return if ["rubygems", "mmtk", "cgi", "pathname", "Onigmo"].include?(gem)
|
|
|
|
require "net/https"
|
|
require "json"
|
|
require "uri"
|
|
|
|
uri = URI("https://rubygems.org/api/v1/versions/#{gem.downcase}/latest.json")
|
|
response = Net::HTTP.get(uri)
|
|
latest_version = JSON.parse(response)["version"]
|
|
|
|
gemspec = [
|
|
"lib/#{gem}/#{gem}.gemspec",
|
|
"lib/#{gem}.gemspec",
|
|
"ext/#{gem}/#{gem}.gemspec",
|
|
"ext/#{gem.split("-").join("/")}/#{gem}.gemspec",
|
|
"lib/#{gem.split("-").first}/#{gem}.gemspec",
|
|
"ext/#{gem.split("-").first}/#{gem}.gemspec",
|
|
"lib/#{gem.split("-").join("/")}/#{gem}.gemspec",
|
|
].find{|gemspec| File.exist?(gemspec)}
|
|
spec = Gem::Specification.load(gemspec)
|
|
puts "#{gem}-#{spec.version} is not latest version of rubygems.org" if spec.version.to_s != latest_version
|
|
end
|
|
|
|
def message_filter(repo, sha, log, context: nil)
|
|
unless repo.count("/") == 1 and /\A\S+\z/ =~ repo
|
|
raise ArgumentError, "invalid repository: #{repo}"
|
|
end
|
|
unless /\A\h{10,40}\z/ =~ sha
|
|
raise ArgumentError, "invalid commit-hash: #{sha}"
|
|
end
|
|
repo_url = "https://github.com/#{repo}"
|
|
|
|
# Log messages generated by GitHub web UI have inconsistent line endings
|
|
log = log.delete("\r")
|
|
log << "\n" if !log.end_with?("\n")
|
|
|
|
# Split the subject from the log message according to git conventions.
|
|
# SPECIAL TREAT: when the first line ends with a dot `.` (which is not
|
|
# obeying the conventions too), takes only that line.
|
|
subject, log = log.split(/\A.+\.\K\n(?=\S)|\n(?:[ \t]*(?:\n|\z))/, 2)
|
|
conv = proc do |s|
|
|
mod = true if s.gsub!(/\b(?:(?i:fix(?:e[sd])?|close[sd]?|resolve[sd]?) +)\K#(?=\d+\b)|\bGH-#?(?=\d+\b)|\(\K#(?=\d+\))/) {
|
|
"#{repo_url}/pull/"
|
|
}
|
|
mod |= true if s.gsub!(%r{(?<![-\[\](){}\w@/])(?:(\w+(?:-\w+)*/\w+(?:-\w+)*)@)?(\h{10,40})\b}) {|c|
|
|
"https://github.com/#{$1 || repo}/commit/#{$2[0,12]}"
|
|
}
|
|
mod
|
|
end
|
|
subject = "[#{repo}] #{subject}"
|
|
subject.gsub!(/\s*\n\s*/, " ")
|
|
if conv[subject]
|
|
if subject.size > 68
|
|
subject.gsub!(/\G.{,67}[^\s.,][.,]*\K\s+/, "\n")
|
|
end
|
|
end
|
|
commit_url = "#{repo_url}/commit/#{sha[0,10]}\n"
|
|
sync_note = context ? "#{commit_url}\n#{context}" : commit_url
|
|
if log and !log.empty?
|
|
log.sub!(/(?<=\n)\n+\z/, '') # drop empty lines at the last
|
|
conv[log]
|
|
log.sub!(/(?:(\A\s*)|\s*\n)(?=((?i:^Co-authored-by:.*\n?)+)?\Z)/) {
|
|
($~.begin(1) ? "" : "\n\n") + sync_note + ($~.begin(2) ? "\n" : "")
|
|
}
|
|
else
|
|
log = sync_note
|
|
end
|
|
"#{subject}\n\n#{log}"
|
|
end
|
|
|
|
def log_format(format, args, &block)
|
|
IO.popen(%W[git -c core.autocrlf=false -c core.eol=lf
|
|
log --no-show-signature --format=#{format}] + args, "rb", &block)
|
|
end
|
|
|
|
def commits_in_range(upto, exclude, toplevel:)
|
|
args = [upto, *exclude.map {|s|"^#{s}"}]
|
|
log_format('%H,%P,%s', %W"--first-parent" + args) do |f|
|
|
f.read.split("\n").reverse.flat_map {|commit|
|
|
hash, parents, subject = commit.split(',', 3)
|
|
parents = parents.split
|
|
|
|
# Non-merge commit
|
|
if parents.size <= 1
|
|
puts "#{hash} #{subject}"
|
|
next [[hash, subject]]
|
|
end
|
|
|
|
# Clean 2-parent merge commit: follow the other parent as long as it
|
|
# contains no potentially-non-clean merges
|
|
if parents.size == 2 &&
|
|
IO.popen(%W"git diff-tree --remerge-diff #{hash}", "rb", &:read).empty?
|
|
puts "\e[2mChecking the other parent of #{hash} #{subject}\e[0m"
|
|
ret = catch(:quit) {
|
|
commits_in_range(parents[1], exclude + [parents[0]], toplevel: false)
|
|
}
|
|
next ret if ret
|
|
end
|
|
|
|
unless toplevel
|
|
puts "\e[1mMerge commit with possible conflict resolution #{hash} #{subject}\e[0m"
|
|
throw :quit
|
|
end
|
|
|
|
puts "#{hash} #{subject} " \
|
|
"\e[1m[merge commit with possible conflicts, will do a squash merge]\e[0m"
|
|
[[hash, subject]]
|
|
}
|
|
end
|
|
end
|
|
|
|
# Returns commit list as array of [commit_hash, subject, sync_note].
|
|
def commits_in_ranges(ranges)
|
|
ranges.flat_map do |range|
|
|
exclude, upto = range.include?("..") ? range.split("..", 2) : ["#{range}~1", range]
|
|
puts "Looking for commits in range #{exclude}..#{upto}"
|
|
commits_in_range(upto, exclude.empty? ? [] : [exclude], toplevel: true)
|
|
end.uniq
|
|
end
|
|
|
|
#--
|
|
# Following methods used by sync_default_gems_with_commits return
|
|
# true: success
|
|
# false: skipped
|
|
# nil: failed
|
|
#++
|
|
|
|
def resolve_conflicts(gem, sha, edit)
|
|
# Discover unmerged files: any unstaged changes
|
|
changes = porcelain_status()
|
|
conflict = changes.grep(/\A(?:.[^ ?]) /) {$'}
|
|
# If -e option is given, open each conflicted file with an editor
|
|
unless conflict.empty?
|
|
if edit
|
|
case
|
|
when (editor = ENV["GIT_EDITOR"] and !editor.empty?)
|
|
when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?))
|
|
end
|
|
if editor
|
|
system([editor, conflict].join(' '))
|
|
conflict.delete_if {|f| !File.exist?(f)}
|
|
return true if conflict.empty?
|
|
return system(*%w"git add --", *conflict)
|
|
end
|
|
end
|
|
return false
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
def collect_cacheinfo(tree)
|
|
pipe_readlines(%W"git ls-tree -r -t -z #{tree}").filter_map do |line|
|
|
fields, path = line.split("\t", 2)
|
|
mode, type, object = fields.split(" ", 3)
|
|
next unless type == "blob"
|
|
[mode, type, object, path]
|
|
end
|
|
end
|
|
|
|
def rewrite_cacheinfo(gem, blobs)
|
|
config = REPOSITORIES[gem]
|
|
rewritten = []
|
|
ignored = blobs.dup
|
|
ignored.delete_if do |mode, type, object, path|
|
|
newpath = config.rewrite_for_ruby(path)
|
|
next unless newpath
|
|
rewritten << [mode, type, object, newpath]
|
|
end
|
|
[rewritten, ignored]
|
|
end
|
|
|
|
def make_commit_info(gem, sha)
|
|
config = REPOSITORIES[gem]
|
|
headers, orig = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2)
|
|
/^author (?<author_name>.+?) <(?<author_email>.*?)> (?<author_date>.+?)$/ =~ headers or
|
|
raise "unable to parse author info for commit #{sha}"
|
|
author = {
|
|
"GIT_AUTHOR_NAME" => author_name,
|
|
"GIT_AUTHOR_EMAIL" => author_email,
|
|
"GIT_AUTHOR_DATE" => author_date,
|
|
}
|
|
context = nil
|
|
if /^parent (?<first_parent>.{40})\nparent .{40}$/ =~ headers
|
|
# Squashing a merge commit: keep authorship information
|
|
context = IO.popen(%W"git shortlog #{first_parent}..#{sha} --", "rb", &:read)
|
|
end
|
|
message = message_filter(config.upstream, sha, orig, context: context)
|
|
[author, message]
|
|
end
|
|
|
|
def fixup_commit(gem, commit)
|
|
wt = File.join("tmp", "sync_default_gems-fixup-worktree")
|
|
if File.directory?(wt)
|
|
IO.popen(%W"git -C #{wt} clean -xdf", "rb", &:read)
|
|
IO.popen(%W"git -C #{wt} reset --hard #{commit}", "rb", &:read)
|
|
else
|
|
IO.popen(%W"git worktree remove --force #{wt}", "rb", err: File::NULL, &:read)
|
|
IO.popen(%W"git worktree add --detach #{wt} #{commit}", "rb", &:read)
|
|
end
|
|
raise "git worktree prepare failed for commit #{commit}" unless $?.success?
|
|
|
|
Dir.chdir(wt) do
|
|
if gem == "rubygems"
|
|
rubygems_do_fixup
|
|
end
|
|
replace_rdoc_ref_all_full
|
|
end
|
|
|
|
IO.popen(%W"git -C #{wt} add -u", "rb", &:read)
|
|
IO.popen(%W"git -C #{wt} commit --amend --no-edit", "rb", &:read)
|
|
IO.popen(%W"git -C #{wt} rev-parse HEAD", "rb", &:read).chomp
|
|
end
|
|
|
|
def make_and_fixup_commit(gem, original_commit, cacheinfo, parent: nil, message: nil, author: nil)
|
|
tree = Tempfile.create("sync_default_gems-#{gem}-index") do |f|
|
|
File.unlink(f.path)
|
|
IO.popen({"GIT_INDEX_FILE" => f.path},
|
|
%W"git update-index --index-info", "wb", out: IO::NULL) do |io|
|
|
cacheinfo.each do |mode, type, object, path|
|
|
io.puts("#{mode} #{type} #{object}\t#{path}")
|
|
end
|
|
end
|
|
raise "git update-index failed" unless $?.success?
|
|
|
|
IO.popen({"GIT_INDEX_FILE" => f.path}, %W"git write-tree --missing-ok", "rb", &:read).chomp
|
|
end
|
|
|
|
args = ["-m", message || "Rewriten commit for #{original_commit}"]
|
|
args += ["-p", parent] if parent
|
|
commit = IO.popen({**author}, %W"git commit-tree #{tree}" + args, "rb", &:read).chomp
|
|
|
|
# Apply changes that require a working tree
|
|
commit = fixup_commit(gem, commit)
|
|
|
|
commit
|
|
end
|
|
|
|
def rewrite_commit(gem, sha)
|
|
author, message = make_commit_info(gem, sha)
|
|
new_blobs = collect_cacheinfo("#{sha}")
|
|
new_rewritten, new_ignored = rewrite_cacheinfo(gem, new_blobs)
|
|
|
|
headers, _ = IO.popen(%W[git cat-file commit #{sha}], "rb", &:read).split("\n\n", 2)
|
|
first_parent = headers[/^parent (.{40})$/, 1]
|
|
unless first_parent
|
|
# Root commit, first time to sync this repo
|
|
return make_and_fixup_commit(gem, sha, new_rewritten, message: message, author: author)
|
|
end
|
|
|
|
old_blobs = collect_cacheinfo(first_parent)
|
|
old_rewritten, old_ignored = rewrite_cacheinfo(gem, old_blobs)
|
|
if old_ignored != new_ignored
|
|
paths = (old_ignored + new_ignored - (old_ignored & new_ignored))
|
|
.map {|*_, path| path}.uniq
|
|
puts "\e\[1mIgnoring file changes not in mappings: #{paths.join(" ")}\e\[0m"
|
|
end
|
|
changed_paths = (old_rewritten + new_rewritten - (old_rewritten & new_rewritten))
|
|
.map {|*_, path| path}.uniq
|
|
if changed_paths.empty?
|
|
puts "Skip commit only for tools or toplevel"
|
|
return false
|
|
end
|
|
|
|
# Build commit objects from "cacheinfo"
|
|
new_parent = make_and_fixup_commit(gem, first_parent, old_rewritten)
|
|
new_commit = make_and_fixup_commit(gem, sha, new_rewritten, parent: new_parent, message: message, author: author)
|
|
puts "Created a temporary commit for cherry-pick: #{new_commit}"
|
|
new_commit
|
|
end
|
|
|
|
def pickup_commit(gem, sha, edit)
|
|
rewritten = rewrite_commit(gem, sha)
|
|
|
|
# No changes remaining after rewriting
|
|
return false unless rewritten
|
|
|
|
# Attempt to cherry-pick a commit
|
|
result = IO.popen(%W"git cherry-pick #{rewritten}", "rb", err: [:child, :out], &:read)
|
|
unless $?.success?
|
|
if result =~ /The previous cherry-pick is now empty/
|
|
system(*%w"git cherry-pick --skip")
|
|
puts "Skip empty commit #{sha}"
|
|
return false
|
|
end
|
|
|
|
# If the cherry-pick attempt failed, try to resolve conflicts.
|
|
# Skip the commit, if it contains unresolved conflicts or no files to pick up.
|
|
unless resolve_conflicts(gem, sha, edit)
|
|
system(*%w"git --no-pager diff") if !edit # If failed, show `git diff` unless editing
|
|
`git reset` && `git checkout .` && `git clean -fd` # Clean up un-committed diffs
|
|
return nil # Fail unless cherry-picked
|
|
end
|
|
|
|
# Commit cherry-picked commit
|
|
if porcelain_status().empty?
|
|
system(*%w"git cherry-pick --skip")
|
|
return false
|
|
else
|
|
system(*%w"git cherry-pick --continue --no-edit")
|
|
return nil unless $?.success?
|
|
end
|
|
end
|
|
|
|
new_head = IO.popen(%W"git rev-parse HEAD", "rb", &:read).chomp
|
|
puts "Committed cherry-pick as #{new_head}"
|
|
return true
|
|
end
|
|
|
|
# @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository.
|
|
# @param ranges [Array<String>, true] "commit", "before..after", or true. Note that it will NOT sync "before" (but commits after that).
|
|
# @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this.
|
|
def sync_default_gems_with_commits(gem, ranges, edit: nil)
|
|
config = REPOSITORIES[gem]
|
|
repo, default_branch = config.upstream, config.branch
|
|
puts "Sync #{repo} with commit history."
|
|
|
|
# Fetch the repository to be synchronized
|
|
IO.popen(%W"git remote") do |f|
|
|
unless f.read.split.include?(gem)
|
|
`git remote add #{gem} https://github.com/#{repo}.git`
|
|
end
|
|
end
|
|
system(*%W"git fetch --no-tags --depth=#{FETCH_DEPTH} #{gem} #{default_branch}")
|
|
|
|
# If -a is given, discover all commits since the last picked commit
|
|
if ranges == true
|
|
pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$"
|
|
log = log_format('%B', %W"-E --grep=#{pattern} -n1 --", &:read)
|
|
ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"]
|
|
end
|
|
commits = commits_in_ranges(ranges)
|
|
if commits.empty?
|
|
puts "No commits to pick"
|
|
return true
|
|
end
|
|
|
|
failed_commits = []
|
|
commits.each do |sha, subject|
|
|
puts "----"
|
|
puts "Pick #{sha} #{subject}"
|
|
case pickup_commit(gem, sha, edit)
|
|
when false
|
|
# skipped
|
|
when nil
|
|
failed_commits << [sha, subject]
|
|
end
|
|
end
|
|
|
|
unless failed_commits.empty?
|
|
puts "---- failed commits ----"
|
|
failed_commits.each do |sha, subject|
|
|
puts "#{sha} #{subject}"
|
|
end
|
|
return false
|
|
end
|
|
return true
|
|
end
|
|
|
|
def sync_lib(repo, upstream = nil)
|
|
unless upstream and File.directory?(upstream) or File.directory?(upstream = "../#{repo}")
|
|
abort %[Expected '#{upstream}' \(#{File.expand_path("#{upstream}")}\) to be a directory, but it wasn't.]
|
|
end
|
|
rm_rf(["lib/#{repo}.rb", "lib/#{repo}/*", "test/test_#{repo}.rb"])
|
|
cp_r(Dir.glob("#{upstream}/lib/*"), "lib")
|
|
tests = if File.directory?("test/#{repo}")
|
|
"test/#{repo}"
|
|
else
|
|
"test/test_#{repo}.rb"
|
|
end
|
|
cp_r("#{upstream}/#{tests}", "test") if File.exist?("#{upstream}/#{tests}")
|
|
gemspec = if File.directory?("lib/#{repo}")
|
|
"lib/#{repo}/#{repo}.gemspec"
|
|
else
|
|
"lib/#{repo}.gemspec"
|
|
end
|
|
cp_r("#{upstream}/#{repo}.gemspec", "#{gemspec}")
|
|
end
|
|
|
|
def update_default_gems(gem, release: false)
|
|
config = REPOSITORIES[gem]
|
|
author, repository = config.upstream.split('/')
|
|
default_branch = config.branch
|
|
|
|
puts "Update #{author}/#{repository}"
|
|
|
|
unless File.exist?("../../#{author}/#{repository}")
|
|
mkdir_p("../../#{author}")
|
|
`git clone git@github.com:#{author}/#{repository}.git ../../#{author}/#{repository}`
|
|
end
|
|
|
|
Dir.chdir("../../#{author}/#{repository}") do
|
|
unless `git remote`.match(/ruby\-core/)
|
|
`git remote add ruby-core git@github.com:ruby/ruby.git`
|
|
end
|
|
`git fetch ruby-core master --no-tags`
|
|
unless `git branch`.match(/ruby\-core/)
|
|
`git checkout ruby-core/master`
|
|
`git branch ruby-core`
|
|
end
|
|
`git checkout ruby-core`
|
|
`git rebase ruby-core/master`
|
|
`git fetch origin --tags`
|
|
|
|
if release
|
|
last_release = `git tag | sort -V`.chomp.split.delete_if{|v| v =~ /pre|beta/ }.last
|
|
`git checkout #{last_release}`
|
|
else
|
|
`git checkout #{default_branch}`
|
|
`git rebase origin/#{default_branch}`
|
|
end
|
|
end
|
|
end
|
|
|
|
case ARGV[0]
|
|
when "up"
|
|
if ARGV[1]
|
|
update_default_gems(ARGV[1])
|
|
else
|
|
REPOSITORIES.each_key {|gem| update_default_gems(gem)}
|
|
end
|
|
when "all"
|
|
if ARGV[1] == "release"
|
|
REPOSITORIES.each_key do |gem|
|
|
update_default_gems(gem, release: true)
|
|
sync_default_gems(gem)
|
|
end
|
|
else
|
|
REPOSITORIES.each_key {|gem| sync_default_gems(gem)}
|
|
end
|
|
when "list"
|
|
ARGV.shift
|
|
pattern = Regexp.new(ARGV.join('|'))
|
|
REPOSITORIES.each do |gem, config|
|
|
next unless pattern =~ gem or pattern =~ config.upstream
|
|
printf "%-15s https://github.com/%s\n", gem, config.upstream
|
|
end
|
|
when "rdoc-ref"
|
|
ARGV.shift
|
|
pattern = ARGV.empty? ? %w[*.c *.rb *.rdoc] : ARGV
|
|
result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + pattern)
|
|
result.inject(false) do |changed, file|
|
|
if replace_rdoc_ref(file)
|
|
puts "replaced rdoc-ref in #{file}"
|
|
changed = true
|
|
end
|
|
changed
|
|
end
|
|
when nil, "-h", "--help"
|
|
puts <<-HELP
|
|
\e[1mSync with upstream code of default libraries\e[0m
|
|
|
|
\e[1mImport all default gems through `git clone` and `cp -rf` (git commits are lost)\e[0m
|
|
ruby #$0 all
|
|
|
|
\e[1mImport all released version of default gems\e[0m
|
|
ruby #$0 all release
|
|
|
|
\e[1mImport a default gem with specific gem same as all command\e[0m
|
|
ruby #$0 rubygems
|
|
|
|
\e[1mPick a single commit from the upstream repository\e[0m
|
|
ruby #$0 rubygems 97e9768612
|
|
|
|
\e[1mPick a commit range from the upstream repository\e[0m
|
|
ruby #$0 rubygems 97e9768612..9e53702832
|
|
|
|
\e[1mPick all commits since the last picked commit\e[0m
|
|
ruby #$0 -a rubygems
|
|
|
|
\e[1mUpdate repositories of default gems\e[0m
|
|
ruby #$0 up
|
|
|
|
\e[1mList known libraries\e[0m
|
|
ruby #$0 list
|
|
|
|
\e[1mList known libraries matching with patterns\e[0m
|
|
ruby #$0 list read
|
|
HELP
|
|
|
|
exit
|
|
else
|
|
while /\A-/ =~ ARGV[0]
|
|
case ARGV[0]
|
|
when "-e"
|
|
edit = true
|
|
ARGV.shift
|
|
when "-a"
|
|
auto = true
|
|
ARGV.shift
|
|
else
|
|
$stderr.puts "Unknown command line option: #{ARGV[0]}"
|
|
exit 1
|
|
end
|
|
end
|
|
gem = ARGV.shift
|
|
if ARGV[0]
|
|
exit sync_default_gems_with_commits(gem, ARGV, edit: edit)
|
|
elsif auto
|
|
exit sync_default_gems_with_commits(gem, true, edit: edit)
|
|
else
|
|
sync_default_gems(gem)
|
|
end
|
|
end if $0 == __FILE__
|
|
end
|