5 files changed, 106 insertions, 12 deletions
diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb
index 1900b7088d..ece6ca1dc8 100644
--- a/tool/lib/core_assertions.rb
+++ b/tool/lib/core_assertions.rb
@@ -97,11 +97,12 @@ module Test
       end
 
       def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil,
-                            success: nil, **opt)
+                            success: nil, failed: nil, **opt)
         args = Array(args).dup
         args.insert((Hash === args[0] ? 1 : 0), '--disable=gems')
         stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt)
-        desc = FailDesc[status, message, stderr]
+        desc = failed[status, message, stderr] if failed
+        desc ||= FailDesc[status, message, stderr]
         if block_given?
           raise "test_stdout ignored, use block only or without block" if test_stdout != []
           raise "test_stderr ignored, use block only or without block" if test_stderr != []
diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb
new file mode 100644
index 0000000000..56b420a546
--- /dev/null
+++ b/tool/lib/dump.gdb
@@ -0,0 +1,17 @@
+set height 0
+set width 0
+set confirm off
+
+echo \n>>> Threads\n\n
+info threads
+
+echo \n>>> Machine level backtrace\n\n
+thread apply all info stack full
+
+echo \n>>> Dump Ruby level backtrace (if possible)\n\n
+call rb_vmdebug_stack_dump_all_threads()
+call fflush(stderr)
+
+echo ">>> Finish\n"
+detach
+quit
diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb
new file mode 100644
index 0000000000..ed9cb89010
--- /dev/null
+++ b/tool/lib/dump.lldb
@@ -0,0 +1,13 @@
+script print("\n>>> Threads\n\n")
+thread list
+
+script print("\n>>> Machine level backtrace\n\n")
+thread backtrace all
+
+script print("\n>>> Dump Ruby level backtrace (if possible)\n\n")
+call rb_vmdebug_stack_dump_all_threads()
+call fflush(stderr)
+
+script print(">>> Finish\n")
+detach
+quit
diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb
index 65c86c1685..573fd5122c 100644
--- a/tool/lib/envutil.rb
+++ b/tool/lib/envutil.rb
@@ -79,6 +79,70 @@ module EnvUtil
   end
   module_function :timeout
 
+  class Debugger
+    @list = []
+
+    attr_accessor :name
+
+    def self.register(name, &block)
+      @list << new(name, &block)
+    end
+
+    def initialize(name, &block)
+      @name = name
+      instance_eval(&block)
+    end
+
+    def usable?; false; end
+
+    def start(pid, *args) end
+
+    def dump(pid, timeout: 60, reprieve: timeout&.div(4))
+      dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")))
+    rescue Errno::ENOENT
+      return
+    else
+      return unless dpid
+      [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig|
+        return EnvUtil.timeout(t) {Process.wait(dpid)}
+      rescue Timeout::Error
+        Process.kill(sig, dpid)
+      end
+      true
+    end
+
+    # sudo -n: --non-interactive
+    PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM)
+
+    def spawn(*args, **opts)
+      super(*PRECOMMAND, *args, **opts)
+    end
+
+    register("gdb") do
+      class << self
+        def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end
+        def start(pid, *args)
+          spawn(*%w[gdb --batch --quiet --pid #{pid}], *args)
+        end
+        def command_file(file) "--command=#{file}"; end
+      end
+    end
+
+    register("lldb") do
+      class << self
+        def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end
+        def start(pid, *args)
+          spawn(*%w[lldb --batch -Q --attach-pid #{pid}])
+        end
+        def command_file(file) ["--source", file]; end
+      end
+    end
+
+    def self.search
+      @debugger ||= @list.find(&:usable?)
+    end
+  end
+
   def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1)
     reprieve = apply_timeout_scale(reprieve) if reprieve
 
@@ -94,17 +158,10 @@ module EnvUtil
       pgroup = pid
     end
 
-    lldb = true if /darwin/ =~ RUBY_PLATFORM
-
     while signal = signals.shift
 
-      if lldb and [:ABRT, :KILL].include?(signal)
-        lldb = false
-        # sudo -n: --non-interactive
-        # lldb -p: attach
-        #      -o: run command
-        system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit])
-        true
+      if (dbg = Debugger.search) and [:ABRT, :KILL].include?(signal)
+        dbg.dump(pid)
       end
 
       begin
diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb
index 9ca29b6e64..7d43e825e1 100644
--- a/tool/lib/test/unit.rb
+++ b/tool/lib/test/unit.rb
@@ -421,6 +421,7 @@ module Test
         end
 
         def kill
+          EnvUtil::Debugger.search&.dump(@pid)
           signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV
           Process.kill(signal, @pid)
           warn "worker #{to_s} does not respond; #{signal} is sent"
@@ -1298,10 +1299,15 @@ module Test
         parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n|
           options[:repeat_count] = n
         end
+        options[:keep_repeating] = false
+        parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n|
+          options[:keep_repeating] = true
+        end
       end
 
       def _run_anything(type)
         @repeat_count = @options[:repeat_count]
+        @keep_repeating = @options[:keep_repeating]
         super
       end
     end
@@ -1623,7 +1629,7 @@ module Test
               [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type,
                 t, @test_count.fdiv(t), @assertion_count.fdiv(t)]
         end while @repeat_count && @@current_repeat_count < @repeat_count &&
-                  report.empty? && failures.zero? && errors.zero?
+                  (@keep_repeating || report.empty? && failures.zero? && errors.zero?)
 
         output.sync = old_sync if sync