Launchable: Start recording test-spec results (#12302)

author: Naoto Ono <[email protected]> 2024-12-12 16:32:35 +0900
committer: GitHub <[email protected]> 2024-12-12 16:32:35 +0900
commit: 46e8a26c17d20d19838a3b55316e06229ac1e2fd (patch)
tree: db75a95798f39a648c09cca99c8ab79514c84512
parent: 3fda6e92addd4e1799b20529beb1733a7e2148e3 (diff)
7 files changed, 146 insertions, 3 deletions
diff --git a/.github/actions/launchable/setup/action.yml b/.github/actions/launchable/setup/action.yml
index cd9d1b6fc9..164eb1db5d 100644
--- a/.github/actions/launchable/setup/action.yml
+++ b/.github/actions/launchable/setup/action.yml
@@ -96,12 +96,16 @@ runs:
       run: |
         test_all_enabled="${{ inputs.test-task == 'check' ||  inputs.test-task == 'test-all' || contains(fromJSON(inputs.test-tasks), 'test-all') }}"
         btest_enabled="${{ inputs.test-task == 'check' ||  inputs.test-task == 'test' || contains(fromJSON(inputs.test-tasks), 'test') }}"
+        test_spec_enabled="${{ inputs.test-task == 'check' ||  inputs.test-task == 'test-spec' || contains(fromJSON(inputs.test-tasks), 'test-spec') }}"
         echo test_all_enabled="${test_all_enabled}" >> $GITHUB_OUTPUT
         echo btest_enabled="${btest_enabled}" >> $GITHUB_OUTPUT
+        echo test_spec_enabled="${test_spec_enabled}" >> $GITHUB_OUTPUT
         echo test_all_session_file='launchable_test_all_session.txt' >> $GITHUB_OUTPUT
         echo btest_session_file='launchable_btest_session.txt' >> $GITHUB_OUTPUT
+        echo test_spec_session_file='launchable_test_spec_session.txt' >> $GITHUB_OUTPUT
         echo test_all_report_file='launchable_test_all_report.json' >> $GITHUB_OUTPUT
         echo btest_report_file='launchable_btest_report.json' >> $GITHUB_OUTPUT
+        echo test_spec_report_dir='launchable_test_spec_report' >> $GITHUB_OUTPUT
       if: steps.enable-launchable.outputs.enable-launchable
 
     - name: Set environment variables for Launchable
@@ -144,9 +148,11 @@ runs:
         test_opts="${test_opts//=/:}"
         test_all_test_suite='test-all'
         btest_test_suite='btest'
+        test_spec_test_suite='test-spec'
         if [ "${{ inputs.is-yjit }}" = "true" ]; then
           test_all_test_suite="yjit-${test_all_test_suite}"
           btest_test_suite="yjit-${btest_test_suite}"
+          test_spec_test_suite="yjit-${test_spec_test_suite}"
         fi
         launchable record build --name "${build_name}"
         if [ "${test_all_enabled}" = "true" ]; then
@@ -169,18 +175,32 @@ runs:
             > "${btest_session_file}"
           echo "BTESTS=${BTESTS} --launchable-test-reports=${btest_report_file}" >> $GITHUB_ENV
         fi
+        if [ "${test_spec_enabled}" = "true" ]; then
+          launchable record session \
+            --build "${build_name}" \
+            --flavor os=${{ inputs.os }} \
+            --flavor test_task=${{ inputs.test-task }} \
+            --flavor test_opts=${test_opts} \
+            --test-suite ${test_spec_test_suite} \
+            > "${test_spec_session_file}"
+          echo "SPECOPTS=${SPECOPTS} --launchable-test-reports=${test_spec_report_dir}" >> $GITHUB_ENV
+        fi
       if: steps.enable-launchable.outputs.enable-launchable
       env:
         test_all_enabled: ${{ steps.global.outputs.test_all_enabled }}
         btest_enabled: ${{ steps.global.outputs.btest_enabled }}
+        test_spec_enabled: ${{ steps.global.outputs.test_spec_enabled }}
         test_all_session_file: ${{ steps.global.outputs.test_all_session_file }}
         btest_session_file: ${{ steps.global.outputs.btest_session_file }}
+        test_spec_session_file: ${{ steps.global.outputs.test_spec_session_file }}
         test_all_report_file: ${{ steps.global.outputs.test_all_report_file }}
         btest_report_file: ${{ steps.global.outputs.btest_report_file }}
+        test_spec_report_dir: ${{ steps.global.outputs.test_spec_report_dir }}
 
     - name: Variables to report Launchable
       id: variables
       shell: bash
+      working-directory: ${{ inputs.srcdir }}
       run: |
         set -x
         : # report-path from srcdir
@@ -198,14 +218,21 @@ runs:
           btest_report_path="${dir}${builddir:+${builddir}/}${btest_report_file}"
           echo btest_report_path="${btest_report_path}" >> $GITHUB_OUTPUT
         fi
+        if [ "${test_spec_enabled}" = "true" ]; then
+          test_spec_report_path="${dir}${builddir:+${builddir}/}${test_spec_report_dir}"
+          mkdir "${test_spec_report_path}"
+          echo test_spec_report_path="${test_spec_report_path}" >> $GITHUB_OUTPUT
+        fi
       if: steps.enable-launchable.outputs.enable-launchable
       env:
         srcdir: ${{ inputs.srcdir }}
         builddir: ${{ inputs.builddir }}
         test_all_enabled: ${{ steps.global.outputs.test_all_enabled }}
         btest_enabled: ${{ steps.global.outputs.btest_enabled }}
+        test_spec_enabled: ${{ steps.global.outputs.test_spec_enabled }}
         test_all_report_file: ${{ steps.global.outputs.test_all_report_file }}
         btest_report_file: ${{ steps.global.outputs.btest_report_file }}
+        test_spec_report_dir: ${{ steps.global.outputs.test_spec_report_dir }}
 
     - name: Record test results in Launchable
       uses: gacts/run-and-post-run@674528335da98a7afc80915ff2b4b860a0b3553a # v1.4.0
@@ -223,15 +250,25 @@ runs:
               --session "$(cat "${btest_session_file}")" \
               raw "${btest_report_path}" || true
 
+          [[ "${test_spec_enabled}" = "true" ]] && \
+            launchable record tests \
+              --session "$(cat "${test_spec_session_file}")" \
+              raw ${test_spec_report_path}/* || true
+
           rm -f "${test_all_session_file}"
           rm -f "${btest_session_file}"
+          rm -f "${test_spec_session_file}"
           rm -f "${test_report_path}"
           rm -f "${btest_report_path}"
+          rm -fr "${test_spec_report_path}"
       if: ${{ always() && steps.enable-launchable.outputs.enable-launchable }}
       env:
         test_report_path: ${{ steps.variables.outputs.test_report_path }}
         btest_report_path: ${{ steps.variables.outputs.btest_report_path }}
+        test_spec_report_path: ${{ steps.variables.outputs.test_spec_report_path }}
         test_all_enabled: ${{ steps.global.outputs.test_all_enabled }}
         btest_enabled: ${{ steps.global.outputs.btest_enabled }}
+        test_spec_enabled: ${{ steps.global.outputs.test_spec_enabled }}
         test_all_session_file: ${{ steps.global.outputs.test_all_session_file }}
         btest_session_file: ${{ steps.global.outputs.btest_session_file }}
+        test_spec_session_file: ${{ steps.global.outputs.test_spec_session_file }}
diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml
index e03a8d2838..898a51b63f 100644
--- a/.github/workflows/mingw.yml
+++ b/.github/workflows/mingw.yml
@@ -126,7 +126,7 @@ jobs:
           launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }}
           builddir: build
           srcdir: src
-          test-tasks: '["test", "test-all"]'
+          test-tasks: '["test", "test-all", "test-spec"]'
         continue-on-error: true
 
       - name: test
diff --git a/.github/workflows/rjit.yml b/.github/workflows/rjit.yml
index 320534d99c..a2358668f0 100644
--- a/.github/workflows/rjit.yml
+++ b/.github/workflows/rjit.yml
@@ -87,7 +87,7 @@ jobs:
           builddir: build
           srcdir: src
           test-opts: ${{ matrix.run_opts }}
-          test-tasks: '["test", "test-all"]'
+          test-tasks: '["test", "test-all", "test-spec"]'
         continue-on-error: true
 
       - name: make test
diff --git a/spec/mspec/lib/mspec/commands/mspec-run.rb b/spec/mspec/lib/mspec/commands/mspec-run.rb
index 064c177d69..36463d8c6f 100644
--- a/spec/mspec/lib/mspec/commands/mspec-run.rb
+++ b/spec/mspec/lib/mspec/commands/mspec-run.rb
@@ -53,6 +53,9 @@ class MSpecRun < MSpecScript
     options.doc "\n When to perform it"
     options.action_filters
 
+    options.doc "\n Launchable"
+    options.launchable
+
     options.doc "\n Help!"
     options.debug
     options.version MSpec::VERSION
diff --git a/spec/mspec/lib/mspec/runner/formatters/launchable.rb b/spec/mspec/lib/mspec/runner/formatters/launchable.rb
new file mode 100644
index 0000000000..f738781c71
--- /dev/null
+++ b/spec/mspec/lib/mspec/runner/formatters/launchable.rb
@@ -0,0 +1,88 @@
+module LaunchableFormatter
+  def self.extend_object(obj)
+    super
+    obj.init
+  end
+
+  def self.setDir(dir)
+    @@path = File.join(dir, "#{rand.to_s}.json")
+    self
+  end
+
+  def init
+    @timer = nil
+    @tests = []
+  end
+
+  def before(state = nil)
+    super
+    @timer = TimerAction.new
+    @timer.start
+  end
+
+  def after(state = nil)
+    super
+    @timer.finish
+    file = MSpec.file
+    return if file.nil? || state&.example.nil? || exception?
+
+    @tests << {:test => state, :file => file, :exception => false, duration: @timer.elapsed}
+  end
+
+  def exception(exception)
+    super
+    @timer.finish
+    file = MSpec.file
+    return if file.nil?
+
+    @tests << {:test => exception, :file => file, :exception => true, duration: @timer.elapsed}
+  end
+
+  def finish
+    super
+
+    require_relative '../../../../../../tool/lib/launchable'
+
+    @writer = writer = Launchable::JsonStreamWriter.new(@@path)
+    @writer.write_array('testCases')
+    at_exit {
+      @writer.close
+    }
+
+    repo_path = File.expand_path("#{__dir__}/../../../../../../")
+
+    @tests.each do |t|
+      testcase = t[:test].description
+      relative_path = t[:file].delete_prefix("#{repo_path}/")
+      # The test path is a URL-encoded representation.
+      # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18
+      test_path = {file: relative_path, testcase: testcase}.map{|key, val|
+        "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}"
+      }.join('#')
+
+      status = 'TEST_PASSED'
+      if t[:exception]
+        message = t[:test].message
+        backtrace = t[:test].backtrace
+        e = "#{message}\n#{backtrace}"
+        status = 'TEST_FAILED'
+      end
+
+      @writer.write_object(
+        {
+          testPath: test_path,
+          status: status,
+          duration: t[:duration],
+          createdAt: Time.now.to_s,
+          stderr: e,
+          stdout: nil
+        }
+      )
+    end
+  end
+
+  private
+  def encode_test_path_component component
+    component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26').tr("\x00-\x08", "")
+  end
+end
diff --git a/spec/mspec/lib/mspec/utils/options.rb b/spec/mspec/lib/mspec/utils/options.rb
index 23cf915e66..adeafa1f81 100644
--- a/spec/mspec/lib/mspec/utils/options.rb
+++ b/spec/mspec/lib/mspec/utils/options.rb
@@ -489,6 +489,14 @@ class MSpecOptions
     end
   end
 
+  def launchable
+    on("--launchable-test-reports", "DIR",
+       "DIR The directory for reporting test results in Launchable JSON format") do |o|
+      require 'mspec/runner/formatters/launchable'
+      config[:launchable] = LaunchableFormatter.setDir(o)
+    end
+  end
+
   def all
     configure {}
     env
@@ -508,5 +516,6 @@ class MSpecOptions
     action_filters
     actions
     debug
+    launchable
   end
 end
diff --git a/spec/mspec/lib/mspec/utils/script.rb b/spec/mspec/lib/mspec/utils/script.rb
index e86beaab86..15fd23fabf 100644
--- a/spec/mspec/lib/mspec/utils/script.rb
+++ b/spec/mspec/lib/mspec/utils/script.rb
@@ -159,8 +159,14 @@ class MSpecScript
     end
 
     if config[:formatter]
-      config[:formatter].new(config[:output])
+      config[:formatter] = config[:formatter].new(config[:output])
     end
+
+    if config[:launchable]
+      config[:formatter].extend config[:launchable]
+    end
+
+    config[:formatter]
   end
 
   # Callback for enabling custom actions, etc. This version is a
author	Naoto Ono <[email protected]>	2024-12-12 16:32:35 +0900
committer	GitHub <[email protected]>	2024-12-12 16:32:35 +0900
commit	46e8a26c17d20d19838a3b55316e06229ac1e2fd (patch)
tree	db75a95798f39a648c09cca99c8ab79514c84512
parent	3fda6e92addd4e1799b20529beb1733a7e2148e3 (diff)