Integrate Launchable into make btest

author: Naoto Ono <[email protected]> 2024-07-06 18:20:41 +0900
committer: Hiroshi SHIBATA <[email protected]> 2024-07-08 10:15:04 +0900
commit: 5b7892545542c0d451cb1e14a1d85474ac46b537 (patch)
tree: 5867397f0cf6362e8f78066bbc614685fec43e0e
parent: dface4427da99561a4578a4b92a8321bc2d6c023 (diff)
6 files changed, 185 insertions, 95 deletions
diff --git a/.github/actions/launchable/setup/action.yml b/.github/actions/launchable/setup/action.yml
index 155604983e..5060e4f7a3 100644
--- a/.github/actions/launchable/setup/action.yml
+++ b/.github/actions/launchable/setup/action.yml
@@ -38,6 +38,19 @@ inputs:
       Directory to (re-)checkout source codes. Launchable retrives the commit information
       from the directory.
 
+  launchable-workspace:
+    required: true
+    default: ${{ github.event.repository.name }}
+    description: >-
+      A workspace name in Launchable
+
+  test-task:
+    required: true
+    default: ${{ matrix.test_task }}
+    description: >-
+      A test task that determine which tests are executed.
+      This value is used in the Launchable flavor.
+
 runs:
   using: composite
 
@@ -77,7 +90,7 @@ runs:
         : # The following envs are necessary in Launchable tokenless authentication.
         : # https://github.com/launchableinc/cli/blob/v1.80.1/launchable/utils/authentication.py#L20
         echo "LAUNCHABLE_ORGANIZATION=${{ github.repository_owner }}" >> $GITHUB_ENV
-        echo "LAUNCHABLE_WORKSPACE=${{ github.event.repository.name }}" >> $GITHUB_ENV
+        echo "LAUNCHABLE_WORKSPACE=${{ inputs.launchable-workspace }}" >> $GITHUB_ENV
         : # https://github.com/launchableinc/cli/blob/v1.80.1/launchable/utils/authentication.py#L71
         echo "GITHUB_PR_HEAD_SHA=${{ github.event.pull_request.head.sha || github.sha }}" >> $GITHUB_ENV
         echo "LAUNCHABLE_TOKEN=${{ inputs.launchable-token }}" >> $GITHUB_ENV
@@ -135,7 +148,7 @@ runs:
         working-directory: ${{ inputs.srcdir }}
         post: |
           : # record
-          launchable record tests --flavor os=${{ inputs.os }} --flavor test_task=${{ matrix.test_task }} --flavor test_opts=${test_opts} raw ${report_path}
+          launchable record tests --flavor os=${{ inputs.os }} --flavor test_task=${{ inputs.test-task }} --flavor test_opts=${test_opts} raw ${report_path}
           rm -f ${report_path}
       if: ${{ always() && steps.enable-launchable.outputs.enable-launchable }}
       env:
diff --git a/.github/workflows/rjit.yml b/.github/workflows/rjit.yml
index 75c8152ce5..99fb3b658e 100644
--- a/.github/workflows/rjit.yml
+++ b/.github/workflows/rjit.yml
@@ -67,6 +67,8 @@ jobs:
           srcdir: src
           builddir: build
           makeup: true
+          # Set fetch-depth: 10 so that Launchable can receive commits information.
+          fetch-depth: 10
 
       - name: Run configure
         env:
@@ -77,13 +79,27 @@ jobs:
 
       - run: $SETARCH make
 
+      - name: Set up Launchable
+        uses: ./.github/actions/launchable/setup
+        with:
+          os: ubuntu-22.04
+          test-task: test
+          launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }}
+          builddir: build
+          srcdir: src
+          launchable-workspace: ruby-make-btest
+          test-opts: ${{ matrix.run_opts }}
+        continue-on-error: true
+
       - name: make test
         run: |
           $SETARCH make -s test RUN_OPTS="$RUN_OPTS"
         timeout-minutes: 30
         env:
           GNUMAKEFLAGS: ''
-          RUBY_TESTOPTS: '--tty=no'
+          RUBY_TESTOPTS: >-
+            ${{ env.TESTS }}
+            --tty=no
           RUN_OPTS: ${{ matrix.run_opts }}
 
       - name: make test-all
diff --git a/bootstraptest/runner.rb b/bootstraptest/runner.rb
index 764c4898dd..70805c3523 100755
--- a/bootstraptest/runner.rb
+++ b/bootstraptest/runner.rb
@@ -78,6 +78,7 @@ bt = Struct.new(:ruby,
                 :platform,
                 :timeout,
                 :timeout_scale,
+                :launchable_test_reports
                 )
 BT = Class.new(bt) do
   def indent=(n)
@@ -229,6 +230,19 @@ End
       exit true
     when /\A-j/
       true
+    when /--launchable-test-reports=(.*)/
+      if File.exist?($1)
+        # To protect files from overwritten, do nothing when the file exists.
+        return true
+      end
+
+      require_relative '../tool/lib/test/unit/launchable'
+      BT.launchable_test_reports = writer = Launchable::JsonStreamWriter.new($1)
+      writer.write_array('testCases')
+      at_exit {
+        writer.close
+      }
+      true
     else
       false
     end
@@ -345,6 +359,45 @@ def concurrent_exec_test
   end
 end
 
+module Launchable
+  def show_progress(message = '')
+    faildesc, t = super
+
+    if writer = BT.launchable_test_reports
+      if !faildesc
+        status = 'TEST_PASSED'
+      else
+        status = 'TEST_FAILED'
+      end
+      repo_path = File.expand_path("#{__dir__}/../")
+      relative_path = self.path.delete_prefix("#{repo_path}/")
+      # The test path is a URL-encoded representation.
+      # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18
+      test_path = {file: relative_path, testcase: self.id}.map{|key, val|
+        "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}"
+      }.join('#')
+      writer.write_object(
+        {
+          testPath: test_path,
+          status: status,
+          duration: t,
+          createdAt: Time.now.to_s,
+          stderr: faildesc,
+          stdout: nil,
+          data: {
+            lineNumber: self.lineno
+          }
+        }
+      )
+    end
+  end
+
+  private
+  def encode_test_path_component component
+    component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26')
+  end
+end
+
 def exec_test(paths)
   # setup
   load_test paths
@@ -421,6 +474,7 @@ def target_platform
 end
 
 class Assertion < Struct.new(:src, :path, :lineno, :proc)
+  prepend Launchable
   @count = 0
   @all = Hash.new{|h, k| h[k] = []}
   @errbuf = []
@@ -495,9 +549,9 @@ class Assertion < Struct.new(:src, :path, :lineno, :proc)
       $stderr.print "#{BT.progress_bs}#{BT.progress[BT_STATE.count % BT.progress.size]}"
     end
 
-    t = Time.now if BT.verbose
+    t = Time.now if BT.verbose || BT.launchable_test_reports
     faildesc, errout = with_stderr {yield}
-    t = Time.now - t if BT.verbose
+    t = Time.now - t if BT.verbose || BT.launchable_test_reports
 
     if !faildesc
       # success
@@ -524,6 +578,8 @@ class Assertion < Struct.new(:src, :path, :lineno, :proc)
         $stderr.printf("%-*s%s", BT.width, path, BT.progress[BT_STATE.count % BT.progress.size])
       end
     end
+
+    [faildesc, t]
   rescue Interrupt
     $stderr.puts "\##{@id} #{path}:#{lineno}"
     raise
diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb
index 5e5b34f5b6..0d32abd18a 100644
--- a/tool/lib/test/unit.rb
+++ b/tool/lib/test/unit.rb
@@ -1450,9 +1450,8 @@ module Test
       def setup_options(opts, options)
         super
         opts.on_tail '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path|
-          require 'json'
-          require 'uri'
-          options[:launchable_test_reports] = writer = JsonStreamWriter.new(path)
+          require_relative '../test/unit/launchable'
+          options[:launchable_test_reports] = writer = Launchable::JsonStreamWriter.new(path)
           writer.write_array('testCases')
           main_pid = Process.pid
           at_exit {
@@ -1469,92 +1468,6 @@ module Test
           component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26')
         end
       end
-
-      ##
-      # JsonStreamWriter writes a JSON file using a stream.
-      # By utilizing a stream, we can minimize memory usage, especially for large files.
-      class JsonStreamWriter
-        def initialize(path)
-          @file = File.open(path, "w")
-          @file.write("{")
-          @indent_level = 0
-          @is_first_key_val = true
-          @is_first_obj = true
-          write_new_line
-        end
-
-        def write_object obj
-          if @is_first_obj
-            @is_first_obj = false
-          else
-            write_comma
-            write_new_line
-          end
-          @indent_level += 1
-          @file.write(to_json_str(obj))
-          @indent_level -= 1
-          @is_first_key_val = true
-          # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified.
-          # {
-          #   "testPath": "file=test%2Ftest_timeout.rb&class=TestTimeout&testcase=test_allows_zero_seconds",
-          #   "status": "TEST_PASSED",
-          #   "duration": 2.7e-05,
-          #   "createdAt": "2024-02-09 12:21:07 +0000",
-          #   "stderr": null,
-          #   "stdout": null
-          # }: null <- here
-          # },
-          # To prevent this, IO#flush is called here.
-          @file.flush
-        end
-
-        def write_array(key)
-          @indent_level += 1
-          @file.write(to_json_str(key))
-          write_colon
-          @file.write(" ", "[")
-          write_new_line
-        end
-
-        def close
-          return if @file.closed?
-          close_array
-          @indent_level -= 1
-          write_new_line
-          @file.write("}", "\n")
-          @file.flush
-          @file.close
-        end
-
-        private
-        def to_json_str(obj)
-          json = JSON.pretty_generate(obj)
-          json.gsub(/^/, ' ' * (2 * @indent_level))
-        end
-
-        def write_indent
-          @file.write(" " * 2 * @indent_level)
-        end
-
-        def write_new_line
-          @file.write("\n")
-        end
-
-        def write_comma
-          @file.write(',')
-        end
-
-        def write_colon
-          @file.write(":")
-        end
-
-        def close_array
-          write_new_line
-          write_indent
-          @file.write("]")
-          @indent_level -= 1
-        end
-      end
     end
 
     class Runner # :nodoc: all
diff --git a/tool/lib/test/unit/launchable.rb b/tool/lib/test/unit/launchable.rb
new file mode 100644
index 0000000000..38f4fe92b3
--- /dev/null
+++ b/tool/lib/test/unit/launchable.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+require 'json'
+require 'uri'
+
+module Launchable
+  ##
+  # JsonStreamWriter writes a JSON file using a stream.
+  # By utilizing a stream, we can minimize memory usage, especially for large files.
+  class JsonStreamWriter
+    def initialize(path)
+      @file = File.open(path, "w")
+      @file.write("{")
+      @indent_level = 0
+      @is_first_key_val = true
+      @is_first_obj = true
+      write_new_line
+    end
+
+    def write_object obj
+      if @is_first_obj
+        @is_first_obj = false
+      else
+        write_comma
+        write_new_line
+      end
+      @indent_level += 1
+      @file.write(to_json_str(obj))
+      @indent_level -= 1
+      @is_first_key_val = true
+      # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified.
+      # {
+      #   "testPath": "file=test%2Ftest_timeout.rb&class=TestTimeout&testcase=test_allows_zero_seconds",
+      #   "status": "TEST_PASSED",
+      #   "duration": 2.7e-05,
+      #   "createdAt": "2024-02-09 12:21:07 +0000",
+      #   "stderr": null,
+      #   "stdout": null
+      # }: null <- here
+      # },
+      # To prevent this, IO#flush is called here.
+      @file.flush
+    end
+
+    def write_array(key)
+      @indent_level += 1
+      @file.write(to_json_str(key))
+      write_colon
+      @file.write(" ", "[")
+      write_new_line
+    end
+
+    def close
+      return if @file.closed?
+      close_array
+      @indent_level -= 1
+      write_new_line
+      @file.write("}", "\n")
+      @file.flush
+      @file.close
+    end
+
+    private
+    def to_json_str(obj)
+      json = JSON.pretty_generate(obj)
+      json.gsub(/^/, ' ' * (2 * @indent_level))
+    end
+
+    def write_indent
+      @file.write(" " * 2 * @indent_level)
+    end
+
+    def write_new_line
+      @file.write("\n")
+    end
+
+    def write_comma
+      @file.write(',')
+    end
+
+    def write_colon
+      @file.write(":")
+    end
+
+    def close_array
+      write_new_line
+      write_indent
+      @file.write("]")
+      @indent_level -= 1
+    end
+  end
+end
diff --git a/tool/test/testunit/test_launchable.rb b/tool/test/testunit/test_launchable.rb
index 70c371e212..3d3c5c29de 100644
--- a/tool/test/testunit/test_launchable.rb
+++ b/tool/test/testunit/test_launchable.rb
@@ -2,11 +2,12 @@
 require 'test/unit'
 require 'tempfile'
 require 'json'
+require_relative '../../lib/test/unit/launchable'
 
 class TestLaunchable < Test::Unit::TestCase
   def test_json_stream_writer
     Tempfile.create(['launchable-test-', '.json']) do |f|
-      json_stream_writer = Test::Unit::LaunchableOption::JsonStreamWriter.new(f.path)
+      json_stream_writer = Launchable::JsonStreamWriter.new(f.path)
       json_stream_writer.write_array('testCases')
       json_stream_writer.write_object(
         {
author	Naoto Ono <[email protected]>	2024-07-06 18:20:41 +0900
committer	Hiroshi SHIBATA <[email protected]>	2024-07-08 10:15:04 +0900
commit	5b7892545542c0d451cb1e14a1d85474ac46b537 (patch)
tree	5867397f0cf6362e8f78066bbc614685fec43e0e
parent	dface4427da99561a4578a4b92a8321bc2d6c023 (diff)