diff --git a/.github/workflows/test-ruby.yml b/.github/workflows/test-ruby.yml
index 0d0215bc06..f8280a22f4 100644
--- a/.github/workflows/test-ruby.yml
+++ b/.github/workflows/test-ruby.yml
@@ -113,6 +113,7 @@ jobs:
       CAS_ENABLED: true
       BUNDLE_WITH: 'pam_authentication test'
       CI_JOBS: ${{ matrix.ci_job }}/4
+      GITHUB_RSPEC: ${{ matrix.ruby-version == '.ruby-version' }}
 
     strategy:
       fail-fast: false
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 4d3c234a0e..6ff0a8f842 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -38,7 +38,7 @@ RSpec.configure do |config|
   end
 
   # Use the GitHub Annotations formatter for CI
-  if ENV['GITHUB_ACTIONS'] == 'true'
+  if ENV['GITHUB_ACTIONS'] == 'true' && ENV['GITHUB_RSPEC'] == 'true'
     require 'rspec/github'
     config.add_formatter RSpec::Github::Formatter
   end