From f860582e60ff5ed891f5232a441fe079cb993e8e Mon Sep 17 00:00:00 2001 From: Gyula Petrovics Date: Thu, 17 Sep 2026 16:05:02 +0200 Subject: [PATCH 01/16] fix: stop xcodebuild collecting a sysdiagnose the user did not ask for Since Xcode 26, xcodebuild collects a simulator sysdiagnose of its own after a failing test run, by shelling out to `simctl diagnose --timeout=600`. This is a separate mechanism from the diagnostics this Step collects during teardown, and it runs even when the user set `collect_simulator_diagnostics: never` - the default. It usually presents as tests "hanging" at the end of the run. Measured on osx-xcode-27.0.x with a two-file SPM package and a single XCTFail, the tests themselves take ~1s either way: default -collect-test-diagnostics never IDETestOperationsObserverDebug 618.801s 21.746s Xcode Test step 10.7 min 45.62 sec .xcresult.zip artifact 16.58 MiB 55.92 KiB The collection does not even succeed; it ends with IDETestOperationsObserverDebug: Failure collecting diagnostics from simulator: Timed out after 600.0 seconds while waiting for a response from the invoked process so the default spends ten minutes and 16 MB per failing build to produce a timeout error. Rather than add a third diagnostics knob, map the existing `collect_simulator_diagnostics` input onto xcodebuild's option, so the Step honours the contract that input already advertises: never (default) -> -collect-test-diagnostics never on_failure -> -collect-test-diagnostics on-failure always -> -collect-test-diagnostics on-failure xcodebuild accepts `on-failure|never` only - there is no `always` - and its collection is failure-triggered anyway, so `always` maps onto `on-failure`. Guarded two ways: - The option does not exist before Xcode 26, so it is only passed for major >= 26. main.go already treats an unreadable Xcode version as non-fatal, leaving major at 0, which skips the option. - An explicit `-collect-test-diagnostics` in `xcodebuild_options` wins; the Step does not add its own. User options are also still appended last. Users who want xcodebuild's collection back set `collect_simulator_diagnostics` to `on_failure` or `always`, or pass the option directly. Co-Authored-By: Claude Opus 5 (1M context) --- main.go | 2 +- step.yml | 15 +++++ step/collecttestdiagnostics_test.go | 92 +++++++++++++++++++++++++++++ step/step.go | 12 +++- step/step_test.go | 7 ++- step/utils.go | 55 ++++++++++++++++- xcodebuild/utils.go | 14 ++++- xcodebuild/xcodebuild_test.go | 5 ++ 8 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 step/collecttestdiagnostics_test.go diff --git a/main.go b/main.go index abbc6dd4..23b86e26 100644 --- a/main.go +++ b/main.go @@ -79,7 +79,7 @@ func createConfigParser(logger log.Logger) step.XcodeTestConfigParser { deviceFinder := destination.NewDeviceFinder(logger, commandFactory, xcodeVersion) utils := step.NewUtils(logger) - return step.NewXcodeTestConfigParser(inputParser, logger, deviceFinder, pathModifier, utils) + return step.NewXcodeTestConfigParser(inputParser, logger, deviceFinder, pathModifier, utils, xcodeVersion) } func createStep(logger log.Logger, logFormatter string) (step.XcodeTestRunner, error) { diff --git a/step.yml b/step.yml index 7dd64d70..259d345d 100644 --- a/step.yml +++ b/step.yml @@ -259,6 +259,21 @@ inputs: category: Debugging title: Collect Simulator diagnostics summary: If this input is set, the simulator verbose logging will be enabled and the simulator diagnostics log will be exported. + description: |- + If this input is set, the simulator verbose logging will be enabled and the simulator diagnostics log will be exported. + + On Xcode 26 and later this input also controls xcodebuild's own `-collect-test-diagnostics` option. + Since Xcode 26, xcodebuild collects a simulator sysdiagnose itself after a failing test run, which + can add up to ten minutes to a failing build and often ends in a timeout without collecting anything. + Setting this input to `never` (the default) now also tells xcodebuild not to collect them. + + Set it to `on_failure` or `always` to keep xcodebuild's collection enabled. You can also override the + option directly in `xcodebuild_options`, which takes precedence over this input. + + Available options: + - `always`: collect diagnostics after every run. + - `on_failure`: collect diagnostics only when the tests fail. + - `never`: do not collect diagnostics. value_options: - always - on_failure diff --git a/step/collecttestdiagnostics_test.go b/step/collecttestdiagnostics_test.go new file mode 100644 index 00000000..7e99e3df --- /dev/null +++ b/step/collecttestdiagnostics_test.go @@ -0,0 +1,92 @@ +package step + +import ( + "testing" + + "github.com/bitrise-steplib/steps-xcode-test/xcodebuild" + "github.com/stretchr/testify/require" +) + +func Test_collectTestDiagnosticsValue(t *testing.T) { + tests := []struct { + name string + condition exportCondition + xcodeMajorVersion int64 + additionalOptions []string + want string + }{ + { + name: "Xcode 26, never - suppresses xcodebuild's own collection", + condition: never, + xcodeMajorVersion: 26, + want: "never", + }, + { + name: "Xcode 27, never - suppresses xcodebuild's own collection", + condition: never, + xcodeMajorVersion: 27, + want: "never", + }, + { + name: "Xcode 27, on_failure - keeps xcodebuild's collection", + condition: onFailure, + xcodeMajorVersion: 27, + want: "on-failure", + }, + { + // xcodebuild has no "always"; its collection is failure-triggered regardless. + name: "Xcode 27, always - maps onto on-failure", + condition: always, + xcodeMajorVersion: 27, + want: "on-failure", + }, + { + // The option does not exist before Xcode 26; passing it would be a usage error. + name: "Xcode 25, never - option not passed", + condition: never, + xcodeMajorVersion: 25, + want: "", + }, + { + name: "Xcode 15, always - option not passed", + condition: always, + xcodeMajorVersion: 15, + want: "", + }, + { + // main.go treats an unreadable Xcode version as non-fatal and leaves Major at 0. + name: "unknown Xcode version - option not passed", + condition: never, + xcodeMajorVersion: 0, + want: "", + }, + { + name: "user set the option explicitly - theirs wins", + condition: never, + xcodeMajorVersion: 27, + additionalOptions: []string{xcodebuild.CollectTestDiagnosticsFlag, "on-failure"}, + want: "", + }, + { + name: "user set the option with = syntax - theirs wins", + condition: never, + xcodeMajorVersion: 27, + additionalOptions: []string{xcodebuild.CollectTestDiagnosticsFlag + "=on-failure"}, + want: "", + }, + { + name: "unrelated additional options are ignored", + condition: never, + xcodeMajorVersion: 27, + additionalOptions: []string{"-quiet", "-parallel-testing-enabled", "NO"}, + want: "never", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := collectTestDiagnosticsValue(tt.condition, tt.xcodeMajorVersion, tt.additionalOptions) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/step/step.go b/step/step.go index b8e102c4..a2f2a601 100644 --- a/step/step.go +++ b/step/step.go @@ -20,6 +20,7 @@ import ( "github.com/bitrise-io/go-xcode/v2/simulator" cache "github.com/bitrise-io/go-xcode/v2/xcodecache" "github.com/bitrise-io/go-xcode/v2/xcodecommand" + "github.com/bitrise-io/go-xcode/v2/xcodeversion" "github.com/bitrise-steplib/steps-xcode-test/output" "github.com/bitrise-steplib/steps-xcode-test/xcodebuild" "github.com/kballard/go-shellquote" @@ -99,7 +100,10 @@ type Config struct { SkipTesting []string CollectSimulatorDiagnostics exportCondition - HeadlessMode bool + // CollectTestDiagnostics is the resolved value for xcodebuild's -collect-test-diagnostics + // option, derived from CollectSimulatorDiagnostics. Empty means do not pass the option. + CollectTestDiagnostics string + HeadlessMode bool DeployDir string } @@ -110,15 +114,17 @@ type XcodeTestConfigParser struct { deviceFinder destination.DeviceFinder pathModifier pathutil.PathModifier utils Utils + xcodeVersion xcodeversion.Version } -func NewXcodeTestConfigParser(inputParser stepconf.InputParser, logger log.Logger, deviceFinder destination.DeviceFinder, pathModifier pathutil.PathModifier, utils Utils) XcodeTestConfigParser { +func NewXcodeTestConfigParser(inputParser stepconf.InputParser, logger log.Logger, deviceFinder destination.DeviceFinder, pathModifier pathutil.PathModifier, utils Utils, xcodeVersion xcodeversion.Version) XcodeTestConfigParser { return XcodeTestConfigParser{ logger: logger, inputParser: inputParser, deviceFinder: deviceFinder, pathModifier: pathModifier, utils: utils, + xcodeVersion: xcodeVersion, } } @@ -205,7 +211,7 @@ func (s XcodeTestConfigParser) ProcessConfig() (Config, error) { return Config{}, fmt.Errorf("failed to process quarentined tests: %w", err) } - return s.utils.CreateConfig(input, projectPath, sim, additionalOptions, additionalLogFormatterOptions, skipTesting), nil + return s.utils.CreateConfig(input, projectPath, sim, additionalOptions, additionalLogFormatterOptions, skipTesting, s.xcodeVersion.Major), nil } /* diff --git a/step/step_test.go b/step/step_test.go index 41798295..7bc6b167 100644 --- a/step/step_test.go +++ b/step/step_test.go @@ -9,6 +9,7 @@ import ( "github.com/bitrise-io/go-steputils/v2/stepconf" "github.com/bitrise-io/go-utils/v2/log" "github.com/bitrise-io/go-xcode/v2/destination" + "github.com/bitrise-io/go-xcode/v2/xcodeversion" commonMocks "github.com/bitrise-steplib/steps-xcode-test/mocks" "github.com/bitrise-steplib/steps-xcode-test/step/mocks" "github.com/hashicorp/go-version" @@ -264,6 +265,7 @@ func defaultConfigs() Config { CacheLevel: "swift_packages", CollectSimulatorDiagnostics: never, + CollectTestDiagnostics: "never", HeadlessMode: true, } } @@ -304,7 +306,10 @@ func createConfigParser(t *testing.T, envValues map[string]string) (XcodeTestCon pathModifier := mocks.NewPathModifier(t) utils := NewUtils(logger) - configParser := NewXcodeTestConfigParser(inputParser, logger, deviceFinder, pathModifier, utils) + // Xcode 27 so that the -collect-test-diagnostics mapping is exercised by the existing cases. + xcodeVersion := xcodeversion.Version{Version: "27.0", BuildVersion: "27A266a", Major: 27, Minor: 0} + + configParser := NewXcodeTestConfigParser(inputParser, logger, deviceFinder, pathModifier, utils, xcodeVersion) mocks := configParserMocks{ deviceFinder: deviceFinder, pathModifier: pathModifier, diff --git a/step/utils.go b/step/utils.go index e8023e37..c8083068 100644 --- a/step/utils.go +++ b/step/utils.go @@ -2,6 +2,7 @@ package step import ( "fmt" + "strings" "github.com/bitrise-io/go-utils/v2/log" "github.com/bitrise-io/go-utils/v2/log/colorstring" @@ -12,7 +13,7 @@ import ( type Utils interface { PrintLastLinesOfXcodebuildTestLog(rawXcodebuildOutput string, isRunSuccess bool) - CreateConfig(input Input, projectPath string, sim destination.Device, additionalOptions, additionalLogFormatterOptions []string, skipTesting []string) Config + CreateConfig(input Input, projectPath string, sim destination.Device, additionalOptions, additionalLogFormatterOptions []string, skipTesting []string, xcodeMajorVersion int64) Config CreateTestParams(cfg Config, xcresultPath, swiftPackagesPath string) xcodebuild.TestRunParams } @@ -52,7 +53,8 @@ that will attach the file to your build as an artifact!`)) func (u utils) CreateConfig(input Input, projectPath string, sim destination.Device, - additionalOptions, additionalLogFormatterOptions []string, skipTesting []string) Config { + additionalOptions, additionalLogFormatterOptions []string, skipTesting []string, + xcodeMajorVersion int64) Config { return Config{ ProjectPath: projectPath, Scheme: input.Scheme, @@ -76,7 +78,9 @@ func (u utils) CreateConfig(input Input, SkipTesting: skipTesting, CollectSimulatorDiagnostics: exportCondition(input.CollectSimulatorDiagnostics), - HeadlessMode: input.HeadlessMode, + CollectTestDiagnostics: collectTestDiagnosticsValue( + exportCondition(input.CollectSimulatorDiagnostics), xcodeMajorVersion, additionalOptions), + HeadlessMode: input.HeadlessMode, DeployDir: input.DeployDir, } @@ -95,6 +99,7 @@ func (u utils) CreateTestParams(cfg Config, xcresultPath, swiftPackagesPath stri XCConfigContent: cfg.XCConfigContent, PerformCleanAction: cfg.PerformCleanAction, SkipTesting: cfg.SkipTesting, + CollectTestDiagnostics: cfg.CollectTestDiagnostics, AdditionalOptions: cfg.XcodebuildOptions, } @@ -106,3 +111,47 @@ func (u utils) CreateTestParams(cfg Config, xcresultPath, swiftPackagesPath stri SwiftPackagesPath: swiftPackagesPath, } } + +// minimumCollectTestDiagnosticsXcodeMajor is the first Xcode version that understands +// xcodebuild's -collect-test-diagnostics option. Passing it to anything older is a usage error. +const minimumCollectTestDiagnosticsXcodeMajor = 26 + +// collectTestDiagnosticsValue maps the Step's collect_simulator_diagnostics input onto +// xcodebuild's -collect-test-diagnostics option. +// +// Since Xcode 26 xcodebuild collects a simulator sysdiagnose of its own after a failing test run, +// by shelling out to `simctl diagnose --timeout=600`. That is a separate mechanism from the +// diagnostics this Step collects during teardown, and it runs even when the user asked for no +// diagnostics at all. Measured on a two-file SPM package with a single failing test, it added ten +// minutes to the run and 16 MB to the result bundle, and then gave up with +// +// IDETestOperationsObserverDebug: Failure collecting diagnostics from simulator: +// Timed out after 600.0 seconds while waiting for a response from the invoked process +// +// It commonly presents as tests "hanging" at the end of the run. So honour the input the Step +// already has: if the user does not want simulator diagnostics, do not let xcodebuild collect +// them either. +// +// Returns an empty string when the option must not be passed at all. +func collectTestDiagnosticsValue(condition exportCondition, xcodeMajorVersion int64, additionalOptions []string) string { + // An unreadable Xcode version reads as major 0, which lands here and leaves xcodebuild alone. + if xcodeMajorVersion < minimumCollectTestDiagnosticsXcodeMajor { + return "" + } + + // An explicit -collect-test-diagnostics in xcodebuild_options wins. + for _, option := range additionalOptions { + if option == xcodebuild.CollectTestDiagnosticsFlag || + strings.HasPrefix(option, xcodebuild.CollectTestDiagnosticsFlag+"=") { + return "" + } + } + + if condition == never { + return "never" + } + + // xcodebuild accepts on-failure|never only - there is no "always" - and its collection is + // failure-triggered anyway, so both always and on_failure map onto on-failure. + return "on-failure" +} diff --git a/xcodebuild/utils.go b/xcodebuild/utils.go index 86998b32..ed6c0c6f 100644 --- a/xcodebuild/utils.go +++ b/xcodebuild/utils.go @@ -40,6 +40,10 @@ var testRunnerErrorPatterns = []string{ failedToOpenTestRunner, } +// CollectTestDiagnosticsFlag is xcodebuild's option for controlling whether it collects a +// simulator sysdiagnose after a failing test run. Added in Xcode 26; it accepts on-failure|never. +const CollectTestDiagnosticsFlag = "-collect-test-diagnostics" + // TestParams ... type TestParams struct { ProjectPath string @@ -53,7 +57,10 @@ type TestParams struct { XCConfigContent string PerformCleanAction bool SkipTesting []string - AdditionalOptions []string + // CollectTestDiagnostics is the value for xcodebuild's -collect-test-diagnostics option. + // Empty means the option is not passed at all, leaving xcodebuild at its own default. + CollectTestDiagnostics string + AdditionalOptions []string } func (b *xcodebuild) createXcodebuildTestArgs(params TestParams) ([]string, error) { @@ -104,6 +111,11 @@ func (b *xcodebuild) createXcodebuildTestArgs(params TestParams) ([]string, erro xcodebuildArgs = append(xcodebuildArgs, fmt.Sprintf("-skip-testing:%s", test)) } + if params.CollectTestDiagnostics != "" { + xcodebuildArgs = append(xcodebuildArgs, CollectTestDiagnosticsFlag, params.CollectTestDiagnostics) + } + + // Appended last so that anything the user passes in xcodebuild_options takes precedence. xcodebuildArgs = append(xcodebuildArgs, params.AdditionalOptions...) return xcodebuildArgs, nil diff --git a/xcodebuild/xcodebuild_test.go b/xcodebuild/xcodebuild_test.go index 6441dad8..e8c82ff9 100644 --- a/xcodebuild/xcodebuild_test.go +++ b/xcodebuild/xcodebuild_test.go @@ -260,6 +260,7 @@ func runParameters() TestRunParams { RelaunchTestsForEachRepetition: true, XCConfigContent: "XCConfigContent", PerformCleanAction: false, + CollectTestDiagnostics: "never", AdditionalOptions: []string{"AdditionalOptions"}, } @@ -316,6 +317,10 @@ func argumentsFromRunParameters(parameters TestRunParams) []string { arguments = append(arguments, fmt.Sprintf("-skip-testing:%s", test)) } + if parameters.TestParams.CollectTestDiagnostics != "" { + arguments = append(arguments, CollectTestDiagnosticsFlag, parameters.TestParams.CollectTestDiagnostics) + } + arguments = append(arguments, parameters.TestParams.AdditionalOptions...) return arguments From dd24e56d27d234407b5ceb2dd133c2ce741c00da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:21:44 +0200 Subject: [PATCH 02/16] comments cleanup --- step/step.go | 6 ++---- step/utils.go | 16 ++++------------ xcodebuild/utils.go | 6 ++---- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/step/step.go b/step/step.go index 27fcf89a..e32c8ebe 100644 --- a/step/step.go +++ b/step/step.go @@ -100,10 +100,8 @@ type Config struct { SkipTesting []string CollectSimulatorDiagnostics exportCondition - // CollectTestDiagnostics is the resolved value for xcodebuild's -collect-test-diagnostics - // option, derived from CollectSimulatorDiagnostics. Empty means do not pass the option. - CollectTestDiagnostics string - HeadlessMode bool + CollectTestDiagnostics string + HeadlessMode bool DeployDir string } diff --git a/step/utils.go b/step/utils.go index c8083068..0e489007 100644 --- a/step/utils.go +++ b/step/utils.go @@ -112,13 +112,6 @@ func (u utils) CreateTestParams(cfg Config, xcresultPath, swiftPackagesPath stri } } -// minimumCollectTestDiagnosticsXcodeMajor is the first Xcode version that understands -// xcodebuild's -collect-test-diagnostics option. Passing it to anything older is a usage error. -const minimumCollectTestDiagnosticsXcodeMajor = 26 - -// collectTestDiagnosticsValue maps the Step's collect_simulator_diagnostics input onto -// xcodebuild's -collect-test-diagnostics option. -// // Since Xcode 26 xcodebuild collects a simulator sysdiagnose of its own after a failing test run, // by shelling out to `simctl diagnose --timeout=600`. That is a separate mechanism from the // diagnostics this Step collects during teardown, and it runs even when the user asked for no @@ -132,9 +125,11 @@ const minimumCollectTestDiagnosticsXcodeMajor = 26 // already has: if the user does not want simulator diagnostics, do not let xcodebuild collect // them either. // -// Returns an empty string when the option must not be passed at all. +// Returns an empty string when the option must not be passed. func collectTestDiagnosticsValue(condition exportCondition, xcodeMajorVersion int64, additionalOptions []string) string { - // An unreadable Xcode version reads as major 0, which lands here and leaves xcodebuild alone. + // first Xcode version that understands -collect-test-diagnostics + const minimumCollectTestDiagnosticsXcodeMajor = 26 + // earlier Xcode (or unknown version: 0) if xcodeMajorVersion < minimumCollectTestDiagnosticsXcodeMajor { return "" } @@ -150,8 +145,5 @@ func collectTestDiagnosticsValue(condition exportCondition, xcodeMajorVersion in if condition == never { return "never" } - - // xcodebuild accepts on-failure|never only - there is no "always" - and its collection is - // failure-triggered anyway, so both always and on_failure map onto on-failure. return "on-failure" } diff --git a/xcodebuild/utils.go b/xcodebuild/utils.go index ed6c0c6f..b03da88b 100644 --- a/xcodebuild/utils.go +++ b/xcodebuild/utils.go @@ -57,10 +57,8 @@ type TestParams struct { XCConfigContent string PerformCleanAction bool SkipTesting []string - // CollectTestDiagnostics is the value for xcodebuild's -collect-test-diagnostics option. - // Empty means the option is not passed at all, leaving xcodebuild at its own default. - CollectTestDiagnostics string - AdditionalOptions []string + CollectTestDiagnostics string + AdditionalOptions []string } func (b *xcodebuild) createXcodebuildTestArgs(params TestParams) ([]string, error) { From dec8d33137e96632211c22bfdb5ec48fa58799ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:31:56 +0200 Subject: [PATCH 03/16] Do not treat -collect-test-diagnostics=value as a user override xcodebuild only understands the two-token form, `-collect-test-diagnostics never`. It accepts `-collect-test-diagnostics=never` (and any `-name=value` token) without an error and silently ignores it, verified on Xcode 26.5 and 27.0 by reading DiagnosticCollectionPolicy back from the generated xctestrun. With the `=` form counted as an override the Step dropped its own flag, xcodebuild dropped the user's, and the default on-failure collection ran even though both asked for `never`. Only the exact two-token form suppresses the Step's flag now. Co-Authored-By: Claude Fable 5.1 --- step/collecttestdiagnostics_test.go | 5 +++-- step/utils.go | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/step/collecttestdiagnostics_test.go b/step/collecttestdiagnostics_test.go index 7e99e3df..e6eb1735 100644 --- a/step/collecttestdiagnostics_test.go +++ b/step/collecttestdiagnostics_test.go @@ -68,11 +68,12 @@ func Test_collectTestDiagnosticsValue(t *testing.T) { want: "", }, { - name: "user set the option with = syntax - theirs wins", + // xcodebuild silently drops the -option=value form, so it must not count as an override. + name: "user wrote the option with = syntax - Step still passes its own", condition: never, xcodeMajorVersion: 27, additionalOptions: []string{xcodebuild.CollectTestDiagnosticsFlag + "=on-failure"}, - want: "", + want: "never", }, { name: "unrelated additional options are ignored", diff --git a/step/utils.go b/step/utils.go index 0e489007..8a49a650 100644 --- a/step/utils.go +++ b/step/utils.go @@ -2,7 +2,6 @@ package step import ( "fmt" - "strings" "github.com/bitrise-io/go-utils/v2/log" "github.com/bitrise-io/go-utils/v2/log/colorstring" @@ -134,10 +133,10 @@ func collectTestDiagnosticsValue(condition exportCondition, xcodeMajorVersion in return "" } - // An explicit -collect-test-diagnostics in xcodebuild_options wins. + // An explicit -collect-test-diagnostics in xcodebuild_options wins. Only the two-token form counts: + // xcodebuild silently ignores -collect-test-diagnostics=value, so that form must not suppress the Step's own. for _, option := range additionalOptions { - if option == xcodebuild.CollectTestDiagnosticsFlag || - strings.HasPrefix(option, xcodebuild.CollectTestDiagnosticsFlag+"=") { + if option == xcodebuild.CollectTestDiagnosticsFlag { return "" } } From c64a7bd3281fc9b4ad7727f14bb89a7f87b9e69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:35:55 +0200 Subject: [PATCH 04/16] Skip the Step's own Simulator diagnostics collection on Xcode 26 and later Since Xcode 26 xcodebuild collects the Simulator diagnostics itself after a failing run, driven by -collect-test-diagnostics, and stores them in the xcresult the Step already exports. The Step's own `simctl diagnose` during teardown produced a second copy of the same data as a separate zip and doubled the time spent collecting on a failing build. On Xcode 26 and later the Step no longer collects on its own. Simulator verbose logging is still enabled when the input is not `never`, so xcodebuild's bundle benefits from it. Xcode 16 and earlier keep the previous behaviour. `always` consequently behaves like `on_failure` on Xcode 26 and later, which the input description now states. Rewrote the input description so the Xcode 26 behaviour is the primary text and older Xcode is the exception, as requested in review, and regenerated README.md. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- step.yml | 22 +++++++++++++--------- step/collecttestdiagnostics_test.go | 29 +++++++++++++++++++++++++++++ step/step.go | 10 +++++++--- step/step_test.go | 1 + step/utils.go | 23 ++++++++++++++++++----- 6 files changed, 69 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 40ae783c..117a4638 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Run tests with custom xcconfig file path: | `xcpretty_options` | Additional options to be added to the executed xcpretty command. | | `--color --report html --output "${BITRISE_DEPLOY_DIR}/xcode-test-results-${BITRISE_SCHEME}.html"` | | `cache_level` | Defines what cache content should be automatically collected. Use key-based caching instead for better performance. Available options: - `none`: Disable collecting cache content. - `swift_packages`: Collect Swift PM packages added to the Xcode project. With key-based caching, you only need the Restore SPM cache and the Save SPM cache Steps to cache your Swift packages. [See devcenter for more information.](https://devcenter.bitrise.io/en/dependencies-and-caching/managing-dependencies-for-ios-apps/managing-dependencies-with-spm.html#caching-swift-packages) | | `none` | | `verbose_log` | If this input is set, the Step will print additional logs for debugging. | | `no` | -| `collect_simulator_diagnostics` | If this input is set, the simulator verbose logging will be enabled and the simulator diagnostics log will be exported. | | `never` | +| `collect_simulator_diagnostics` | Enables Simulator verbose logging and controls whether Simulator diagnostics are collected after the test run. Unless set to `never`, Simulator verbose logging is enabled before the tests run. xcodebuild collects the Simulator diagnostics (a `simctl diagnose` bundle) itself after a failing test run and stores them in the `.xcresult`; extract them with `xcrun xcresulttool export diagnostics`. This input maps onto xcodebuild's `-collect-test-diagnostics` option: `never` (the default) disables that collection, which can otherwise add up to ten minutes to a failing build, while `on_failure` and `always` keep it enabled. The Step does not collect diagnostics on its own, so `always` behaves like `on_failure`. A `-collect-test-diagnostics` option passed in `xcodebuild_options` takes precedence over this input. On Xcode 16 and earlier, xcodebuild has no such option and the Step collects the diagnostics itself after the tests and exports them as a zip artifact: after every run with `always`, only after a failed run with `on_failure`. Available options: - `always`: collect diagnostics after every run (Xcode 16 and earlier) or after a failed run (Xcode 26 and later). - `on_failure`: collect diagnostics only when the tests fail. - `never`: do not collect diagnostics. | | `never` | | `headless_mode` | In headless mode the simulator is not launched in the foreground. If this input is set, the simulator will not be visible but tests (even the screenshots) will run just like if you run a simulator in foreground. | | `yes` | | `quarantined_tests` | JSON list of tests added to quarantine on Bitrise.io, quarantined tests are excluded from test runs. | | `$BITRISE_QUARANTINED_TESTS_JSON` | diff --git a/step.yml b/step.yml index 259d345d..93c790ac 100644 --- a/step.yml +++ b/step.yml @@ -258,20 +258,24 @@ inputs: opts: category: Debugging title: Collect Simulator diagnostics - summary: If this input is set, the simulator verbose logging will be enabled and the simulator diagnostics log will be exported. + summary: Enables Simulator verbose logging and controls whether Simulator diagnostics are collected after the test run. description: |- - If this input is set, the simulator verbose logging will be enabled and the simulator diagnostics log will be exported. + Enables Simulator verbose logging and controls whether Simulator diagnostics are collected after the test run. - On Xcode 26 and later this input also controls xcodebuild's own `-collect-test-diagnostics` option. - Since Xcode 26, xcodebuild collects a simulator sysdiagnose itself after a failing test run, which - can add up to ten minutes to a failing build and often ends in a timeout without collecting anything. - Setting this input to `never` (the default) now also tells xcodebuild not to collect them. + Unless set to `never`, Simulator verbose logging is enabled before the tests run. - Set it to `on_failure` or `always` to keep xcodebuild's collection enabled. You can also override the - option directly in `xcodebuild_options`, which takes precedence over this input. + xcodebuild collects the Simulator diagnostics (a `simctl diagnose` bundle) itself after a failing test run + and stores them in the `.xcresult`; extract them with `xcrun xcresulttool export diagnostics`. This input + maps onto xcodebuild's `-collect-test-diagnostics` option: `never` (the default) disables that collection, + which can otherwise add up to ten minutes to a failing build, while `on_failure` and `always` keep it enabled. + The Step does not collect diagnostics on its own, so `always` behaves like `on_failure`. A + `-collect-test-diagnostics` option passed in `xcodebuild_options` takes precedence over this input. + + On Xcode 16 and earlier, xcodebuild has no such option and the Step collects the diagnostics itself after the + tests and exports them as a zip artifact: after every run with `always`, only after a failed run with `on_failure`. Available options: - - `always`: collect diagnostics after every run. + - `always`: collect diagnostics after every run (Xcode 16 and earlier) or after a failed run (Xcode 26 and later). - `on_failure`: collect diagnostics only when the tests fail. - `never`: do not collect diagnostics. value_options: diff --git a/step/collecttestdiagnostics_test.go b/step/collecttestdiagnostics_test.go index e6eb1735..8852243e 100644 --- a/step/collecttestdiagnostics_test.go +++ b/step/collecttestdiagnostics_test.go @@ -91,3 +91,32 @@ func Test_collectTestDiagnosticsValue(t *testing.T) { }) } } + +func Test_shouldCollectSimulatorDiagnostics(t *testing.T) { + tests := []struct { + name string + condition exportCondition + testFailed bool + xcodeMajorVersion int64 + want bool + }{ + {name: "Xcode 16, always, passed", condition: always, testFailed: false, xcodeMajorVersion: 16, want: true}, + {name: "Xcode 16, always, failed", condition: always, testFailed: true, xcodeMajorVersion: 16, want: true}, + {name: "Xcode 16, on_failure, passed", condition: onFailure, testFailed: false, xcodeMajorVersion: 16, want: false}, + {name: "Xcode 16, on_failure, failed", condition: onFailure, testFailed: true, xcodeMajorVersion: 16, want: true}, + {name: "Xcode 16, never, failed", condition: never, testFailed: true, xcodeMajorVersion: 16, want: false}, + // main.go leaves the major at 0 when the Xcode version cannot be read; keep collecting there. + {name: "unknown Xcode, on_failure, failed", condition: onFailure, testFailed: true, xcodeMajorVersion: 0, want: true}, + // Since Xcode 26 xcodebuild collects into the xcresult itself; the Step must not collect a second copy. + {name: "Xcode 26, on_failure, failed", condition: onFailure, testFailed: true, xcodeMajorVersion: 26, want: false}, + {name: "Xcode 27, always, failed", condition: always, testFailed: true, xcodeMajorVersion: 27, want: false}, + {name: "Xcode 27, always, passed", condition: always, testFailed: false, xcodeMajorVersion: 27, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := shouldCollectSimulatorDiagnostics(tt.condition, tt.testFailed, tt.xcodeMajorVersion) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/step/step.go b/step/step.go index e32c8ebe..f8be8222 100644 --- a/step/step.go +++ b/step/step.go @@ -102,6 +102,7 @@ type Config struct { CollectSimulatorDiagnostics exportCondition CollectTestDiagnostics string HeadlessMode bool + XcodeMajorVersion int64 DeployDir string } @@ -301,7 +302,7 @@ func (s XcodeTestRunner) Run(cfg Config) (Result, error) { testExitCode = code } - result.SimulatorDiagnosticsPath = s.teardownSimulator(cfg.Simulator.UDID, cfg.CollectSimulatorDiagnostics, cfg.IsSimulatorBooted, testErr) + result.SimulatorDiagnosticsPath = s.teardownSimulator(cfg.Simulator.UDID, cfg.CollectSimulatorDiagnostics, cfg.IsSimulatorBooted, testErr, cfg.XcodeMajorVersion) if testErr != nil { s.logger.Println() @@ -484,10 +485,10 @@ func (s XcodeTestRunner) runTests(cfg Config) (Result, int, error) { return result, exitCode, testErr } -func (s XcodeTestRunner) teardownSimulator(simulatorID string, simulatorDebug exportCondition, isSimulatorBooted bool, testErr error) string { +func (s XcodeTestRunner) teardownSimulator(simulatorID string, simulatorDebug exportCondition, isSimulatorBooted bool, testErr error, xcodeMajorVersion int64) string { var simulatorDiagnosticsPath string - if simulatorDebug == always || (simulatorDebug == onFailure && testErr != nil) { + if shouldCollectSimulatorDiagnostics(simulatorDebug, testErr != nil, xcodeMajorVersion) { s.logger.Println() s.logger.Infof("Collecting Simulator diagnostics") @@ -498,6 +499,9 @@ func (s XcodeTestRunner) teardownSimulator(simulatorID string, simulatorDebug ex s.logger.Donef("Simulator diagnostics are available as an artifact (%s)", diagnosticsPath) simulatorDiagnosticsPath = diagnosticsPath } + } else if simulatorDebug != never && testErr != nil { + s.logger.Println() + s.logger.Infof("Simulator diagnostics are collected by xcodebuild into the xcresult on Xcode 26 and later (xcrun xcresulttool export diagnostics)") } // Shut down the simulator if it was started by the step for diagnostic logs. diff --git a/step/step_test.go b/step/step_test.go index b9578483..002eecfa 100644 --- a/step/step_test.go +++ b/step/step_test.go @@ -339,6 +339,7 @@ func defaultConfigs() Config { CollectSimulatorDiagnostics: never, CollectTestDiagnostics: "never", HeadlessMode: true, + XcodeMajorVersion: 27, } } func defaultSimulator() destination.Device { diff --git a/step/utils.go b/step/utils.go index 8a49a650..b5eb3187 100644 --- a/step/utils.go +++ b/step/utils.go @@ -79,7 +79,8 @@ func (u utils) CreateConfig(input Input, CollectSimulatorDiagnostics: exportCondition(input.CollectSimulatorDiagnostics), CollectTestDiagnostics: collectTestDiagnosticsValue( exportCondition(input.CollectSimulatorDiagnostics), xcodeMajorVersion, additionalOptions), - HeadlessMode: input.HeadlessMode, + HeadlessMode: input.HeadlessMode, + XcodeMajorVersion: xcodeMajorVersion, DeployDir: input.DeployDir, } @@ -111,6 +112,21 @@ func (u utils) CreateTestParams(cfg Config, xcresultPath, swiftPackagesPath stri } } +// minimumCollectTestDiagnosticsXcodeMajor is the first Xcode version that understands +// -collect-test-diagnostics and collects Simulator diagnostics itself after a failing test run. +const minimumCollectTestDiagnosticsXcodeMajor = 26 + +// shouldCollectSimulatorDiagnostics decides whether the Step runs its own `simctl diagnose` after the tests. +// Since Xcode 26 xcodebuild collects the same diagnostics into the xcresult after a failing run +// (see collectTestDiagnosticsValue), so the Step does not collect on those Xcode versions. +func shouldCollectSimulatorDiagnostics(condition exportCondition, testFailed bool, xcodeMajorVersion int64) bool { + if xcodeMajorVersion >= minimumCollectTestDiagnosticsXcodeMajor { + return false + } + + return condition == always || (condition == onFailure && testFailed) +} + // Since Xcode 26 xcodebuild collects a simulator sysdiagnose of its own after a failing test run, // by shelling out to `simctl diagnose --timeout=600`. That is a separate mechanism from the // diagnostics this Step collects during teardown, and it runs even when the user asked for no @@ -126,15 +142,12 @@ func (u utils) CreateTestParams(cfg Config, xcresultPath, swiftPackagesPath stri // // Returns an empty string when the option must not be passed. func collectTestDiagnosticsValue(condition exportCondition, xcodeMajorVersion int64, additionalOptions []string) string { - // first Xcode version that understands -collect-test-diagnostics - const minimumCollectTestDiagnosticsXcodeMajor = 26 // earlier Xcode (or unknown version: 0) if xcodeMajorVersion < minimumCollectTestDiagnosticsXcodeMajor { return "" } - // An explicit -collect-test-diagnostics in xcodebuild_options wins. Only the two-token form counts: - // xcodebuild silently ignores -collect-test-diagnostics=value, so that form must not suppress the Step's own. + // An explicit -collect-test-diagnostics in xcodebuild_options wins. for _, option := range additionalOptions { if option == xcodebuild.CollectTestDiagnosticsFlag { return "" From a666d273e6a2cdd16b9263a5ce7fa8aefde6a57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:00:59 +0200 Subject: [PATCH 05/16] Document that always equals on_failure on Xcode 26 and later xcodebuild's -collect-test-diagnostics knows on-failure and never only, and the Step no longer collects on its own on Xcode 26 and later, so `always` cannot mean more than `on_failure` there. Keep the value accepted (dropping it would break existing configs) and say what it does per Xcode version. README regenerated. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- step.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 117a4638..6488fa88 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Run tests with custom xcconfig file path: | `xcpretty_options` | Additional options to be added to the executed xcpretty command. | | `--color --report html --output "${BITRISE_DEPLOY_DIR}/xcode-test-results-${BITRISE_SCHEME}.html"` | | `cache_level` | Defines what cache content should be automatically collected. Use key-based caching instead for better performance. Available options: - `none`: Disable collecting cache content. - `swift_packages`: Collect Swift PM packages added to the Xcode project. With key-based caching, you only need the Restore SPM cache and the Save SPM cache Steps to cache your Swift packages. [See devcenter for more information.](https://devcenter.bitrise.io/en/dependencies-and-caching/managing-dependencies-for-ios-apps/managing-dependencies-with-spm.html#caching-swift-packages) | | `none` | | `verbose_log` | If this input is set, the Step will print additional logs for debugging. | | `no` | -| `collect_simulator_diagnostics` | Enables Simulator verbose logging and controls whether Simulator diagnostics are collected after the test run. Unless set to `never`, Simulator verbose logging is enabled before the tests run. xcodebuild collects the Simulator diagnostics (a `simctl diagnose` bundle) itself after a failing test run and stores them in the `.xcresult`; extract them with `xcrun xcresulttool export diagnostics`. This input maps onto xcodebuild's `-collect-test-diagnostics` option: `never` (the default) disables that collection, which can otherwise add up to ten minutes to a failing build, while `on_failure` and `always` keep it enabled. The Step does not collect diagnostics on its own, so `always` behaves like `on_failure`. A `-collect-test-diagnostics` option passed in `xcodebuild_options` takes precedence over this input. On Xcode 16 and earlier, xcodebuild has no such option and the Step collects the diagnostics itself after the tests and exports them as a zip artifact: after every run with `always`, only after a failed run with `on_failure`. Available options: - `always`: collect diagnostics after every run (Xcode 16 and earlier) or after a failed run (Xcode 26 and later). - `on_failure`: collect diagnostics only when the tests fail. - `never`: do not collect diagnostics. | | `never` | +| `collect_simulator_diagnostics` | Enables Simulator verbose logging and controls whether Simulator diagnostics are collected after the test run. Unless set to `never`, Simulator verbose logging is enabled before the tests run. xcodebuild collects the Simulator diagnostics (a `simctl diagnose` bundle) itself after a failing test run and stores them in the `.xcresult`; extract them with `xcrun xcresulttool export diagnostics`. This input maps onto xcodebuild's `-collect-test-diagnostics` option: `never` (the default) disables that collection, which can otherwise add up to ten minutes to a failing build, while `on_failure` keeps it enabled. The Step does not collect diagnostics on its own. A `-collect-test-diagnostics` option passed in `xcodebuild_options` takes precedence over this input. On Xcode 16 and earlier, xcodebuild has no such option and the Step collects the diagnostics itself after the tests and exports them as a zip artifact. Available options: - `always`: same as `on_failure` on Xcode 26 and later, where xcodebuild collects the diagnostics itself. On Xcode 16 and earlier the Step also collects after passing runs. - `on_failure`: collect diagnostics when the tests fail. - `never`: do not collect diagnostics. | | `never` | | `headless_mode` | In headless mode the simulator is not launched in the foreground. If this input is set, the simulator will not be visible but tests (even the screenshots) will run just like if you run a simulator in foreground. | | `yes` | | `quarantined_tests` | JSON list of tests added to quarantine on Bitrise.io, quarantined tests are excluded from test runs. | | `$BITRISE_QUARANTINED_TESTS_JSON` | diff --git a/step.yml b/step.yml index 93c790ac..76cdb618 100644 --- a/step.yml +++ b/step.yml @@ -267,16 +267,16 @@ inputs: xcodebuild collects the Simulator diagnostics (a `simctl diagnose` bundle) itself after a failing test run and stores them in the `.xcresult`; extract them with `xcrun xcresulttool export diagnostics`. This input maps onto xcodebuild's `-collect-test-diagnostics` option: `never` (the default) disables that collection, - which can otherwise add up to ten minutes to a failing build, while `on_failure` and `always` keep it enabled. - The Step does not collect diagnostics on its own, so `always` behaves like `on_failure`. A - `-collect-test-diagnostics` option passed in `xcodebuild_options` takes precedence over this input. + which can otherwise add up to ten minutes to a failing build, while `on_failure` keeps it enabled. The Step + does not collect diagnostics on its own. A `-collect-test-diagnostics` option passed in `xcodebuild_options` + takes precedence over this input. On Xcode 16 and earlier, xcodebuild has no such option and the Step collects the diagnostics itself after the - tests and exports them as a zip artifact: after every run with `always`, only after a failed run with `on_failure`. + tests and exports them as a zip artifact. Available options: - - `always`: collect diagnostics after every run (Xcode 16 and earlier) or after a failed run (Xcode 26 and later). - - `on_failure`: collect diagnostics only when the tests fail. + - `always`: same as `on_failure` on Xcode 26 and later, where xcodebuild collects the diagnostics itself. On Xcode 16 and earlier the Step also collects after passing runs. + - `on_failure`: collect diagnostics when the tests fail. - `never`: do not collect diagnostics. value_options: - always From 0355622c3d583189660be36fa13a40d1e058a3a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:17:48 +0200 Subject: [PATCH 06/16] Add project_setting to collect_simulator_diagnostics The Step's default `never` overrides whatever the project's test plan says about diagnostics collection, since the -collect-test-diagnostics option wins over the plan in both directions (verified on Xcode 26.5 and 27.0). Projects that opted in through their plan had no way to keep that choice short of passing the option in xcodebuild_options. `project_setting` passes no -collect-test-diagnostics at all, so xcodebuild follows the test plan: collect on failure unless the plan says otherwise. Simulator verbose logging is still enabled, as for every value other than `never`. On Xcode 16 and earlier there is nothing to follow and the Step does not collect on its own for this value. Input description rewritten around the new default behaviour; README regenerated. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- step.yml | 17 +++++++++-------- step/collecttestdiagnostics_test.go | 10 ++++++++++ step/step.go | 11 ++++++----- step/step_test.go | 15 +++++++++++++++ step/utils.go | 5 +++++ 6 files changed, 46 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 6488fa88..376350d6 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Run tests with custom xcconfig file path: | `xcpretty_options` | Additional options to be added to the executed xcpretty command. | | `--color --report html --output "${BITRISE_DEPLOY_DIR}/xcode-test-results-${BITRISE_SCHEME}.html"` | | `cache_level` | Defines what cache content should be automatically collected. Use key-based caching instead for better performance. Available options: - `none`: Disable collecting cache content. - `swift_packages`: Collect Swift PM packages added to the Xcode project. With key-based caching, you only need the Restore SPM cache and the Save SPM cache Steps to cache your Swift packages. [See devcenter for more information.](https://devcenter.bitrise.io/en/dependencies-and-caching/managing-dependencies-for-ios-apps/managing-dependencies-with-spm.html#caching-swift-packages) | | `none` | | `verbose_log` | If this input is set, the Step will print additional logs for debugging. | | `no` | -| `collect_simulator_diagnostics` | Enables Simulator verbose logging and controls whether Simulator diagnostics are collected after the test run. Unless set to `never`, Simulator verbose logging is enabled before the tests run. xcodebuild collects the Simulator diagnostics (a `simctl diagnose` bundle) itself after a failing test run and stores them in the `.xcresult`; extract them with `xcrun xcresulttool export diagnostics`. This input maps onto xcodebuild's `-collect-test-diagnostics` option: `never` (the default) disables that collection, which can otherwise add up to ten minutes to a failing build, while `on_failure` keeps it enabled. The Step does not collect diagnostics on its own. A `-collect-test-diagnostics` option passed in `xcodebuild_options` takes precedence over this input. On Xcode 16 and earlier, xcodebuild has no such option and the Step collects the diagnostics itself after the tests and exports them as a zip artifact. Available options: - `always`: same as `on_failure` on Xcode 26 and later, where xcodebuild collects the diagnostics itself. On Xcode 16 and earlier the Step also collects after passing runs. - `on_failure`: collect diagnostics when the tests fail. - `never`: do not collect diagnostics. | | `never` | +| `collect_simulator_diagnostics` | Enables Simulator verbose logging and controls whether Simulator diagnostics are collected after the test run. xcodebuild collects the Simulator diagnostics (a `simctl diagnose` bundle) itself after a failing test run and stores them in the `.xcresult`; extract them with `xcrun xcresulttool export diagnostics`. Simulator verbose logging is enabled before the tests run unless this Input is `never`. The default value is `never`, which prevents intermittent hangs in a CI environment. To let xcodebuild follow the test plan (collect on failure unless the plan says otherwise), set this Input to `project_setting`. A `-collect-test-diagnostics` option passed in `xcodebuild_options` takes precedence over this Input. On Xcode 16 and earlier, xcodebuild has no such option and the Step collects the diagnostics itself after the tests and exports them as a zip artifact. Available options: - `project_setting`: use the option set in the Xcode project test plan. - `always`: same as `on_failure` on Xcode 26 and later, where xcodebuild collects the diagnostics itself. On Xcode 16 and earlier the Step also collects after passing runs. - `on_failure`: collect diagnostics when the tests fail. - `never`: overrides any test plan setting and prevents the collection of diagnostics. | | `never` | | `headless_mode` | In headless mode the simulator is not launched in the foreground. If this input is set, the simulator will not be visible but tests (even the screenshots) will run just like if you run a simulator in foreground. | | `yes` | | `quarantined_tests` | JSON list of tests added to quarantine on Bitrise.io, quarantined tests are excluded from test runs. | | `$BITRISE_QUARANTINED_TESTS_JSON` | diff --git a/step.yml b/step.yml index 76cdb618..76e5a254 100644 --- a/step.yml +++ b/step.yml @@ -262,23 +262,24 @@ inputs: description: |- Enables Simulator verbose logging and controls whether Simulator diagnostics are collected after the test run. - Unless set to `never`, Simulator verbose logging is enabled before the tests run. - xcodebuild collects the Simulator diagnostics (a `simctl diagnose` bundle) itself after a failing test run - and stores them in the `.xcresult`; extract them with `xcrun xcresulttool export diagnostics`. This input - maps onto xcodebuild's `-collect-test-diagnostics` option: `never` (the default) disables that collection, - which can otherwise add up to ten minutes to a failing build, while `on_failure` keeps it enabled. The Step - does not collect diagnostics on its own. A `-collect-test-diagnostics` option passed in `xcodebuild_options` - takes precedence over this input. + and stores them in the `.xcresult`; extract them with `xcrun xcresulttool export diagnostics`. + Simulator verbose logging is enabled before the tests run unless this Input is `never`. + + The default value is `never`, which prevents intermittent hangs in a CI environment. + To let xcodebuild follow the test plan (collect on failure unless the plan says otherwise), set this Input to `project_setting`. + A `-collect-test-diagnostics` option passed in `xcodebuild_options` takes precedence over this Input. On Xcode 16 and earlier, xcodebuild has no such option and the Step collects the diagnostics itself after the tests and exports them as a zip artifact. Available options: + - `project_setting`: use the option set in the Xcode project test plan. - `always`: same as `on_failure` on Xcode 26 and later, where xcodebuild collects the diagnostics itself. On Xcode 16 and earlier the Step also collects after passing runs. - `on_failure`: collect diagnostics when the tests fail. - - `never`: do not collect diagnostics. + - `never`: overrides any test plan setting and prevents the collection of diagnostics. value_options: + - project_setting - always - on_failure - never diff --git a/step/collecttestdiagnostics_test.go b/step/collecttestdiagnostics_test.go index 8852243e..c7806c88 100644 --- a/step/collecttestdiagnostics_test.go +++ b/step/collecttestdiagnostics_test.go @@ -40,6 +40,13 @@ func Test_collectTestDiagnosticsValue(t *testing.T) { xcodeMajorVersion: 27, want: "on-failure", }, + { + // project_setting leaves the decision to the test plan, so nothing is passed. + name: "Xcode 27, project_setting - option not passed", + condition: projectSetting, + xcodeMajorVersion: 27, + want: "", + }, { // The option does not exist before Xcode 26; passing it would be a usage error. name: "Xcode 25, never - option not passed", @@ -105,6 +112,9 @@ func Test_shouldCollectSimulatorDiagnostics(t *testing.T) { {name: "Xcode 16, on_failure, passed", condition: onFailure, testFailed: false, xcodeMajorVersion: 16, want: false}, {name: "Xcode 16, on_failure, failed", condition: onFailure, testFailed: true, xcodeMajorVersion: 16, want: true}, {name: "Xcode 16, never, failed", condition: never, testFailed: true, xcodeMajorVersion: 16, want: false}, + // project_setting has nothing to follow before Xcode 26, the Step does not collect on its own. + {name: "Xcode 16, project_setting, failed", condition: projectSetting, testFailed: true, xcodeMajorVersion: 16, want: false}, + {name: "Xcode 27, project_setting, failed", condition: projectSetting, testFailed: true, xcodeMajorVersion: 27, want: false}, // main.go leaves the major at 0 when the Xcode version cannot be read; keep collecting there. {name: "unknown Xcode, on_failure, failed", condition: onFailure, testFailed: true, xcodeMajorVersion: 0, want: true}, // Since Xcode 26 xcodebuild collects into the xcresult itself; the Step must not collect a second copy. diff --git a/step/step.go b/step/step.go index f8be8222..d1a3b298 100644 --- a/step/step.go +++ b/step/step.go @@ -55,7 +55,7 @@ type Input struct { // Debugging VerboseLog bool `env:"verbose_log,opt[yes,no]"` QuarantinedTests string `env:"quarantined_tests"` - CollectSimulatorDiagnostics string `env:"collect_simulator_diagnostics,opt[always,on_failure,never]"` + CollectSimulatorDiagnostics string `env:"collect_simulator_diagnostics,opt[project_setting,always,on_failure,never]"` HeadlessMode bool `env:"headless_mode,opt[yes,no]"` // Output export @@ -65,9 +65,10 @@ type Input struct { type exportCondition string const ( - always = "always" - never = "never" - onFailure = "on_failure" + always = "always" + never = "never" + onFailure = "on_failure" + projectSetting = "project_setting" ) // Output tools @@ -501,7 +502,7 @@ func (s XcodeTestRunner) teardownSimulator(simulatorID string, simulatorDebug ex } } else if simulatorDebug != never && testErr != nil { s.logger.Println() - s.logger.Infof("Simulator diagnostics are collected by xcodebuild into the xcresult on Xcode 26 and later (xcrun xcresulttool export diagnostics)") + s.logger.Infof("On Xcode 26 and later xcodebuild collects the Simulator diagnostics into the xcresult (xcrun xcresulttool export diagnostics), the Step does not collect them separately") } // Shut down the simulator if it was started by the step for diagnostic logs. diff --git a/step/step_test.go b/step/step_test.go index 002eecfa..50201b1e 100644 --- a/step/step_test.go +++ b/step/step_test.go @@ -111,6 +111,21 @@ func Test_GivenConfigParser_WhenParsesConfig(t *testing.T) { return config }, }, + { + name: "collect_simulator_diagnostics_project_setting", + envsFunc: func() map[string]string { + envValues := defaultEnvValues() + envValues["collect_simulator_diagnostics"] = "project_setting" + return envValues + }, + expectedConfig: func() Config { + config := defaultConfigs() + config.CollectSimulatorDiagnostics = projectSetting + // Nothing is passed to xcodebuild, the test plan decides. + config.CollectTestDiagnostics = "" + return config + }, + }, { name: "skip_tests", envsFunc: func() map[string]string { diff --git a/step/utils.go b/step/utils.go index b5eb3187..7721d044 100644 --- a/step/utils.go +++ b/step/utils.go @@ -154,6 +154,11 @@ func collectTestDiagnosticsValue(condition exportCondition, xcodeMajorVersion in } } + // project_setting: do not override anything, xcodebuild follows the test plan. + if condition == projectSetting { + return "" + } + if condition == never { return "never" } From 2442790d0df49c5a51edc0ac128e1ef2869783a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:22:57 +0200 Subject: [PATCH 07/16] Inline the -collect-test-diagnostics option name Co-Authored-By: Claude Fable 5.1 --- step/collecttestdiagnostics_test.go | 5 ++--- step/utils.go | 2 +- xcodebuild/utils.go | 6 +----- xcodebuild/xcodebuild_test.go | 2 +- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/step/collecttestdiagnostics_test.go b/step/collecttestdiagnostics_test.go index c7806c88..2f5c45e4 100644 --- a/step/collecttestdiagnostics_test.go +++ b/step/collecttestdiagnostics_test.go @@ -3,7 +3,6 @@ package step import ( "testing" - "github.com/bitrise-steplib/steps-xcode-test/xcodebuild" "github.com/stretchr/testify/require" ) @@ -71,7 +70,7 @@ func Test_collectTestDiagnosticsValue(t *testing.T) { name: "user set the option explicitly - theirs wins", condition: never, xcodeMajorVersion: 27, - additionalOptions: []string{xcodebuild.CollectTestDiagnosticsFlag, "on-failure"}, + additionalOptions: []string{"-collect-test-diagnostics", "on-failure"}, want: "", }, { @@ -79,7 +78,7 @@ func Test_collectTestDiagnosticsValue(t *testing.T) { name: "user wrote the option with = syntax - Step still passes its own", condition: never, xcodeMajorVersion: 27, - additionalOptions: []string{xcodebuild.CollectTestDiagnosticsFlag + "=on-failure"}, + additionalOptions: []string{"-collect-test-diagnostics=on-failure"}, want: "never", }, { diff --git a/step/utils.go b/step/utils.go index 7721d044..ca72fb3c 100644 --- a/step/utils.go +++ b/step/utils.go @@ -149,7 +149,7 @@ func collectTestDiagnosticsValue(condition exportCondition, xcodeMajorVersion in // An explicit -collect-test-diagnostics in xcodebuild_options wins. for _, option := range additionalOptions { - if option == xcodebuild.CollectTestDiagnosticsFlag { + if option == "-collect-test-diagnostics" { return "" } } diff --git a/xcodebuild/utils.go b/xcodebuild/utils.go index b03da88b..15b9b9dc 100644 --- a/xcodebuild/utils.go +++ b/xcodebuild/utils.go @@ -40,10 +40,6 @@ var testRunnerErrorPatterns = []string{ failedToOpenTestRunner, } -// CollectTestDiagnosticsFlag is xcodebuild's option for controlling whether it collects a -// simulator sysdiagnose after a failing test run. Added in Xcode 26; it accepts on-failure|never. -const CollectTestDiagnosticsFlag = "-collect-test-diagnostics" - // TestParams ... type TestParams struct { ProjectPath string @@ -110,7 +106,7 @@ func (b *xcodebuild) createXcodebuildTestArgs(params TestParams) ([]string, erro } if params.CollectTestDiagnostics != "" { - xcodebuildArgs = append(xcodebuildArgs, CollectTestDiagnosticsFlag, params.CollectTestDiagnostics) + xcodebuildArgs = append(xcodebuildArgs, "-collect-test-diagnostics", params.CollectTestDiagnostics) } // Appended last so that anything the user passes in xcodebuild_options takes precedence. diff --git a/xcodebuild/xcodebuild_test.go b/xcodebuild/xcodebuild_test.go index e8c82ff9..3c2747dd 100644 --- a/xcodebuild/xcodebuild_test.go +++ b/xcodebuild/xcodebuild_test.go @@ -318,7 +318,7 @@ func argumentsFromRunParameters(parameters TestRunParams) []string { } if parameters.TestParams.CollectTestDiagnostics != "" { - arguments = append(arguments, CollectTestDiagnosticsFlag, parameters.TestParams.CollectTestDiagnostics) + arguments = append(arguments, "-collect-test-diagnostics", parameters.TestParams.CollectTestDiagnostics) } arguments = append(arguments, parameters.TestParams.AdditionalOptions...) From 4b4f7657f8a1dbd68d347c9a404b37f4c66b9026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:22:57 +0200 Subject: [PATCH 08/16] Migrate to the shared steps-check workflow Replace the inline check workflow with the steps-check include, as in bitrise-steplib/steps-git-clone#250. The shared checks surfaced a few things, fixed here so the first run is green: staticcheck ST1005 (capitalised error strings in main.go), QF1004 (strings.ReplaceAll in testaddon), and yamlfmt (`{}` spelling in e2e/bitrise.yml). Verified locally with `bitrise merge`, the shared .golangci.yml (0 issues), the shared .ymlfmt (clean) and stepman audit. Co-Authored-By: Claude Fable 5.1 --- bitrise.yml | 9 ++++++--- e2e/bitrise.yml | 2 +- main.go | 8 ++++---- testaddon/utils.go | 4 ++-- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/bitrise.yml b/bitrise.yml index 89b04e10..091a41da 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -1,14 +1,17 @@ format_version: "11" default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +include: +- repository: steps-check + branch: master + path: steps.bitrise.yml + app: envs: - ORIG_BITRISE_SOURCE_DIR: $BITRISE_SOURCE_DIR workflows: - check: - steps: - - git::https://github.com/bitrise-steplib/steps-check.git: { } + # check: included from steps-check/steps.bitrise.yml e2e: steps: diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 7ca5447c..5c884106 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -209,7 +209,7 @@ workflows: - repository_url: $TEST_APP_URL - clone_into_dir: ./_tmp - branch: $TEST_APP_BRANCH - - certificate-and-profile-installer: { } + - certificate-and-profile-installer: {} - script: title: Set 'collect_simulator_diagnostics' input inputs: diff --git a/main.go b/main.go index 6894e0c9..e40054c0 100644 --- a/main.go +++ b/main.go @@ -38,13 +38,13 @@ func run() int { configParser := createConfigParser(logger) config, err := configParser.ProcessConfig() if err != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to process Step inputs: %w", err))) + logger.Errorf(errorutil.FormattedError(fmt.Errorf("failed to process Step inputs: %w", err))) return 1 } xcodeTestRunner, err := createStep(logger, config.LogFormatter) if err != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to process Step inputs: %w", err))) + logger.Errorf(errorutil.FormattedError(fmt.Errorf("failed to process Step inputs: %w", err))) return 1 } @@ -54,12 +54,12 @@ func run() int { exportErr := xcodeTestRunner.Export(res, runErr != nil) if runErr != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to execute Step: %w", runErr))) + logger.Errorf(errorutil.FormattedError(fmt.Errorf("failed to execute Step: %w", runErr))) return 1 } if exportErr != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to export Step outputs: %w", exportErr))) + logger.Errorf(errorutil.FormattedError(fmt.Errorf("failed to export Step outputs: %w", exportErr))) return 1 } diff --git a/testaddon/utils.go b/testaddon/utils.go index 9dd24b0b..4e7d349f 100644 --- a/testaddon/utils.go +++ b/testaddon/utils.go @@ -30,8 +30,8 @@ func NewTestAddon(logger log.Logger) TestAddon { // ReplaceUnsupportedFilenameCharacters Replaces characters '/' and ':', which are unsupported in filnenames on macOS func (t testAddon) ReplaceUnsupportedFilenameCharacters(s string) string { - s = strings.Replace(s, "/", "-", -1) - s = strings.Replace(s, ":", "-", -1) + s = strings.ReplaceAll(s, "/", "-") + s = strings.ReplaceAll(s, ":", "-") return s } From c2a4eaa0c3b4db3a6e94b5ca9e8871ebaf6c863f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:23:36 +0200 Subject: [PATCH 09/16] Keep the top-level error messages capitalised They are user-facing log lines, not wrapped errors, so exempt them from staticcheck ST1005 instead of lowercasing them. Co-Authored-By: Claude Fable 5.1 --- main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index e40054c0..a01ecf45 100644 --- a/main.go +++ b/main.go @@ -38,13 +38,13 @@ func run() int { configParser := createConfigParser(logger) config, err := configParser.ProcessConfig() if err != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("failed to process Step inputs: %w", err))) + logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to process Step inputs: %w", err))) //nolint:staticcheck // top-level messages are user facing return 1 } xcodeTestRunner, err := createStep(logger, config.LogFormatter) if err != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("failed to process Step inputs: %w", err))) + logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to process Step inputs: %w", err))) //nolint:staticcheck // top-level messages are user facing return 1 } @@ -54,12 +54,12 @@ func run() int { exportErr := xcodeTestRunner.Export(res, runErr != nil) if runErr != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("failed to execute Step: %w", runErr))) + logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to execute Step: %w", runErr))) //nolint:staticcheck // top-level messages are user facing return 1 } if exportErr != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("failed to export Step outputs: %w", exportErr))) + logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to export Step outputs: %w", exportErr))) //nolint:staticcheck // top-level messages are user facing return 1 } From f3085ed3ee2304b74a3da8b24fc1979103aab8f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:26:07 +0200 Subject: [PATCH 10/16] Pass formatted errors to the logger as arguments logger.Errorf received the formatted error chain as its format string, so a `%` inside an error message would have been misrendered. Use "%s" as in steps-xcode-archive. Also fix the Xcode version warning, which used the %w verb in a logger call where only %s is valid. Co-Authored-By: Claude Fable 5.1 --- main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index a01ecf45..78178f65 100644 --- a/main.go +++ b/main.go @@ -38,13 +38,13 @@ func run() int { configParser := createConfigParser(logger) config, err := configParser.ProcessConfig() if err != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to process Step inputs: %w", err))) //nolint:staticcheck // top-level messages are user facing + logger.Errorf("%s", errorutil.FormattedError(fmt.Errorf("Failed to process Step inputs: %w", err))) //nolint:staticcheck // top-level messages are user facing return 1 } xcodeTestRunner, err := createStep(logger, config.LogFormatter) if err != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to process Step inputs: %w", err))) //nolint:staticcheck // top-level messages are user facing + logger.Errorf("%s", errorutil.FormattedError(fmt.Errorf("Failed to process Step inputs: %w", err))) //nolint:staticcheck // top-level messages are user facing return 1 } @@ -54,12 +54,12 @@ func run() int { exportErr := xcodeTestRunner.Export(res, runErr != nil) if runErr != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to execute Step: %w", runErr))) //nolint:staticcheck // top-level messages are user facing + logger.Errorf("%s", errorutil.FormattedError(fmt.Errorf("Failed to execute Step: %w", runErr))) //nolint:staticcheck // top-level messages are user facing return 1 } if exportErr != nil { - logger.Errorf(errorutil.FormattedError(fmt.Errorf("Failed to export Step outputs: %w", exportErr))) //nolint:staticcheck // top-level messages are user facing + logger.Errorf("%s", errorutil.FormattedError(fmt.Errorf("Failed to export Step outputs: %w", exportErr))) //nolint:staticcheck // top-level messages are user facing return 1 } @@ -73,7 +73,7 @@ func createConfigParser(logger log.Logger) step.XcodeTestConfigParser { xcodeVersionProvider := xcodeversion.NewXcodeVersionProvider(commandFactory) xcodeVersion, err := xcodeVersionProvider.GetVersion() if err != nil { // Not a fatal error, continuing with empty version - logger.Errorf("failed to read Xcode version: %w", err) + logger.Errorf("Failed to read Xcode version: %s", err) } pathModifier := pathutil.NewPathModifier() deviceFinder := destination.NewDeviceFinder(logger, commandFactory, xcodeVersion) From 79d17bdb5089ffca5fa0b5ca77efbdde02f29359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:29:13 +0200 Subject: [PATCH 11/16] Compact the collectTestDiagnosticsValue doc comment Describe the mapping as it behaves after this change instead of the history behind it. Co-Authored-By: Claude Fable 5.1 --- step/utils.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/step/utils.go b/step/utils.go index ca72fb3c..4ed16cb1 100644 --- a/step/utils.go +++ b/step/utils.go @@ -127,20 +127,10 @@ func shouldCollectSimulatorDiagnostics(condition exportCondition, testFailed boo return condition == always || (condition == onFailure && testFailed) } -// Since Xcode 26 xcodebuild collects a simulator sysdiagnose of its own after a failing test run, -// by shelling out to `simctl diagnose --timeout=600`. That is a separate mechanism from the -// diagnostics this Step collects during teardown, and it runs even when the user asked for no -// diagnostics at all. Measured on a two-file SPM package with a single failing test, it added ten -// minutes to the run and 16 MB to the result bundle, and then gave up with -// -// IDETestOperationsObserverDebug: Failure collecting diagnostics from simulator: -// Timed out after 600.0 seconds while waiting for a response from the invoked process -// -// It commonly presents as tests "hanging" at the end of the run. So honour the input the Step -// already has: if the user does not want simulator diagnostics, do not let xcodebuild collect -// them either. -// -// Returns an empty string when the option must not be passed. +// collectTestDiagnosticsValue maps the collect_simulator_diagnostics input onto the value of xcodebuild's +// -collect-test-diagnostics option (on-failure|never). It returns an empty string when the option must not be +// passed: on Xcode versions without the option, when xcodebuild_options already sets it, or for project_setting, +// which leaves the decision to the test plan. func collectTestDiagnosticsValue(condition exportCondition, xcodeMajorVersion int64, additionalOptions []string) string { // earlier Xcode (or unknown version: 0) if xcodeMajorVersion < minimumCollectTestDiagnosticsXcodeMajor { From 43a65361633f355c2948b975782a3fa489c99c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:31:56 +0200 Subject: [PATCH 12/16] Name the diagnostics helpers after who collects shouldStepCollectDiagnostics decides whether the Step runs its own simctl diagnose; xcodebuildDiagnosticsOverride is the value the Step passes to xcodebuild's -collect-test-diagnostics, carried as Config.XcodebuildDiagnosticsOverride and TestParams.XcodebuildDiagnosticsOverride. Co-Authored-By: Claude Fable 5.1 --- ...iagnostics_test.go => diagnostics_test.go} | 8 +++---- step/step.go | 12 +++++----- step/step_test.go | 10 ++++----- step/utils.go | 22 +++++++++---------- xcodebuild/utils.go | 6 ++--- xcodebuild/xcodebuild_test.go | 6 ++--- 6 files changed, 32 insertions(+), 32 deletions(-) rename step/{collecttestdiagnostics_test.go => diagnostics_test.go} (93%) diff --git a/step/collecttestdiagnostics_test.go b/step/diagnostics_test.go similarity index 93% rename from step/collecttestdiagnostics_test.go rename to step/diagnostics_test.go index 2f5c45e4..23a7273f 100644 --- a/step/collecttestdiagnostics_test.go +++ b/step/diagnostics_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" ) -func Test_collectTestDiagnosticsValue(t *testing.T) { +func Test_xcodebuildDiagnosticsOverride(t *testing.T) { tests := []struct { name string condition exportCondition @@ -92,13 +92,13 @@ func Test_collectTestDiagnosticsValue(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := collectTestDiagnosticsValue(tt.condition, tt.xcodeMajorVersion, tt.additionalOptions) + got := xcodebuildDiagnosticsOverride(tt.condition, tt.xcodeMajorVersion, tt.additionalOptions) require.Equal(t, tt.want, got) }) } } -func Test_shouldCollectSimulatorDiagnostics(t *testing.T) { +func Test_shouldStepCollectDiagnostics(t *testing.T) { tests := []struct { name string condition exportCondition @@ -124,7 +124,7 @@ func Test_shouldCollectSimulatorDiagnostics(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := shouldCollectSimulatorDiagnostics(tt.condition, tt.testFailed, tt.xcodeMajorVersion) + got := shouldStepCollectDiagnostics(tt.condition, tt.testFailed, tt.xcodeMajorVersion) require.Equal(t, tt.want, got) }) } diff --git a/step/step.go b/step/step.go index d1a3b298..5ed6b0b0 100644 --- a/step/step.go +++ b/step/step.go @@ -99,11 +99,11 @@ type Config struct { CacheLevel string - SkipTesting []string - CollectSimulatorDiagnostics exportCondition - CollectTestDiagnostics string - HeadlessMode bool - XcodeMajorVersion int64 + SkipTesting []string + CollectSimulatorDiagnostics exportCondition + XcodebuildDiagnosticsOverride string + HeadlessMode bool + XcodeMajorVersion int64 DeployDir string } @@ -489,7 +489,7 @@ func (s XcodeTestRunner) runTests(cfg Config) (Result, int, error) { func (s XcodeTestRunner) teardownSimulator(simulatorID string, simulatorDebug exportCondition, isSimulatorBooted bool, testErr error, xcodeMajorVersion int64) string { var simulatorDiagnosticsPath string - if shouldCollectSimulatorDiagnostics(simulatorDebug, testErr != nil, xcodeMajorVersion) { + if shouldStepCollectDiagnostics(simulatorDebug, testErr != nil, xcodeMajorVersion) { s.logger.Println() s.logger.Infof("Collecting Simulator diagnostics") diff --git a/step/step_test.go b/step/step_test.go index 50201b1e..32c47b4a 100644 --- a/step/step_test.go +++ b/step/step_test.go @@ -122,7 +122,7 @@ func Test_GivenConfigParser_WhenParsesConfig(t *testing.T) { config := defaultConfigs() config.CollectSimulatorDiagnostics = projectSetting // Nothing is passed to xcodebuild, the test plan decides. - config.CollectTestDiagnostics = "" + config.XcodebuildDiagnosticsOverride = "" return config }, }, @@ -351,10 +351,10 @@ func defaultConfigs() Config { CacheLevel: "swift_packages", - CollectSimulatorDiagnostics: never, - CollectTestDiagnostics: "never", - HeadlessMode: true, - XcodeMajorVersion: 27, + CollectSimulatorDiagnostics: never, + XcodebuildDiagnosticsOverride: "never", + HeadlessMode: true, + XcodeMajorVersion: 27, } } func defaultSimulator() destination.Device { diff --git a/step/utils.go b/step/utils.go index 4ed16cb1..446f1faf 100644 --- a/step/utils.go +++ b/step/utils.go @@ -77,7 +77,7 @@ func (u utils) CreateConfig(input Input, SkipTesting: skipTesting, CollectSimulatorDiagnostics: exportCondition(input.CollectSimulatorDiagnostics), - CollectTestDiagnostics: collectTestDiagnosticsValue( + XcodebuildDiagnosticsOverride: xcodebuildDiagnosticsOverride( exportCondition(input.CollectSimulatorDiagnostics), xcodeMajorVersion, additionalOptions), HeadlessMode: input.HeadlessMode, XcodeMajorVersion: xcodeMajorVersion, @@ -99,7 +99,7 @@ func (u utils) CreateTestParams(cfg Config, xcresultPath, swiftPackagesPath stri XCConfigContent: cfg.XCConfigContent, PerformCleanAction: cfg.PerformCleanAction, SkipTesting: cfg.SkipTesting, - CollectTestDiagnostics: cfg.CollectTestDiagnostics, + XcodebuildDiagnosticsOverride: cfg.XcodebuildDiagnosticsOverride, AdditionalOptions: cfg.XcodebuildOptions, } @@ -112,28 +112,28 @@ func (u utils) CreateTestParams(cfg Config, xcresultPath, swiftPackagesPath stri } } -// minimumCollectTestDiagnosticsXcodeMajor is the first Xcode version that understands +// minimumXcodeMajorWithDiagnosticsOption is the first Xcode version that understands // -collect-test-diagnostics and collects Simulator diagnostics itself after a failing test run. -const minimumCollectTestDiagnosticsXcodeMajor = 26 +const minimumXcodeMajorWithDiagnosticsOption = 26 -// shouldCollectSimulatorDiagnostics decides whether the Step runs its own `simctl diagnose` after the tests. +// shouldStepCollectDiagnostics decides whether the Step runs its own `simctl diagnose` after the tests. // Since Xcode 26 xcodebuild collects the same diagnostics into the xcresult after a failing run -// (see collectTestDiagnosticsValue), so the Step does not collect on those Xcode versions. -func shouldCollectSimulatorDiagnostics(condition exportCondition, testFailed bool, xcodeMajorVersion int64) bool { - if xcodeMajorVersion >= minimumCollectTestDiagnosticsXcodeMajor { +// (see xcodebuildDiagnosticsOverride), so the Step does not collect on those Xcode versions. +func shouldStepCollectDiagnostics(condition exportCondition, testFailed bool, xcodeMajorVersion int64) bool { + if xcodeMajorVersion >= minimumXcodeMajorWithDiagnosticsOption { return false } return condition == always || (condition == onFailure && testFailed) } -// collectTestDiagnosticsValue maps the collect_simulator_diagnostics input onto the value of xcodebuild's +// xcodebuildDiagnosticsOverride maps the collect_simulator_diagnostics input onto the value of xcodebuild's // -collect-test-diagnostics option (on-failure|never). It returns an empty string when the option must not be // passed: on Xcode versions without the option, when xcodebuild_options already sets it, or for project_setting, // which leaves the decision to the test plan. -func collectTestDiagnosticsValue(condition exportCondition, xcodeMajorVersion int64, additionalOptions []string) string { +func xcodebuildDiagnosticsOverride(condition exportCondition, xcodeMajorVersion int64, additionalOptions []string) string { // earlier Xcode (or unknown version: 0) - if xcodeMajorVersion < minimumCollectTestDiagnosticsXcodeMajor { + if xcodeMajorVersion < minimumXcodeMajorWithDiagnosticsOption { return "" } diff --git a/xcodebuild/utils.go b/xcodebuild/utils.go index 15b9b9dc..dc30af15 100644 --- a/xcodebuild/utils.go +++ b/xcodebuild/utils.go @@ -53,7 +53,7 @@ type TestParams struct { XCConfigContent string PerformCleanAction bool SkipTesting []string - CollectTestDiagnostics string + XcodebuildDiagnosticsOverride string AdditionalOptions []string } @@ -105,8 +105,8 @@ func (b *xcodebuild) createXcodebuildTestArgs(params TestParams) ([]string, erro xcodebuildArgs = append(xcodebuildArgs, fmt.Sprintf("-skip-testing:%s", test)) } - if params.CollectTestDiagnostics != "" { - xcodebuildArgs = append(xcodebuildArgs, "-collect-test-diagnostics", params.CollectTestDiagnostics) + if params.XcodebuildDiagnosticsOverride != "" { + xcodebuildArgs = append(xcodebuildArgs, "-collect-test-diagnostics", params.XcodebuildDiagnosticsOverride) } // Appended last so that anything the user passes in xcodebuild_options takes precedence. diff --git a/xcodebuild/xcodebuild_test.go b/xcodebuild/xcodebuild_test.go index 3c2747dd..150b588e 100644 --- a/xcodebuild/xcodebuild_test.go +++ b/xcodebuild/xcodebuild_test.go @@ -260,7 +260,7 @@ func runParameters() TestRunParams { RelaunchTestsForEachRepetition: true, XCConfigContent: "XCConfigContent", PerformCleanAction: false, - CollectTestDiagnostics: "never", + XcodebuildDiagnosticsOverride: "never", AdditionalOptions: []string{"AdditionalOptions"}, } @@ -317,8 +317,8 @@ func argumentsFromRunParameters(parameters TestRunParams) []string { arguments = append(arguments, fmt.Sprintf("-skip-testing:%s", test)) } - if parameters.TestParams.CollectTestDiagnostics != "" { - arguments = append(arguments, "-collect-test-diagnostics", parameters.TestParams.CollectTestDiagnostics) + if parameters.TestParams.XcodebuildDiagnosticsOverride != "" { + arguments = append(arguments, "-collect-test-diagnostics", parameters.TestParams.XcodebuildDiagnosticsOverride) } arguments = append(arguments, parameters.TestParams.AdditionalOptions...) From b6e7dfb916451fa12d881201279ff43a11cd666b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:54:32 +0200 Subject: [PATCH 13/16] Replace the xcodebuildDiagnosticsOverride doc comment with inline notes Co-Authored-By: Claude Fable 5.1 --- step/utils.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/step/utils.go b/step/utils.go index 446f1faf..8fc430bb 100644 --- a/step/utils.go +++ b/step/utils.go @@ -127,25 +127,18 @@ func shouldStepCollectDiagnostics(condition exportCondition, testFailed bool, xc return condition == always || (condition == onFailure && testFailed) } -// xcodebuildDiagnosticsOverride maps the collect_simulator_diagnostics input onto the value of xcodebuild's -// -collect-test-diagnostics option (on-failure|never). It returns an empty string when the option must not be -// passed: on Xcode versions without the option, when xcodebuild_options already sets it, or for project_setting, -// which leaves the decision to the test plan. func xcodebuildDiagnosticsOverride(condition exportCondition, xcodeMajorVersion int64, additionalOptions []string) string { - // earlier Xcode (or unknown version: 0) - if xcodeMajorVersion < minimumXcodeMajorWithDiagnosticsOption { + if xcodeMajorVersion < minimumXcodeMajorWithDiagnosticsOption { // no such option yet return "" } - // An explicit -collect-test-diagnostics in xcodebuild_options wins. for _, option := range additionalOptions { - if option == "-collect-test-diagnostics" { + if option == "-collect-test-diagnostics" { // user's option wins return "" } } - // project_setting: do not override anything, xcodebuild follows the test plan. - if condition == projectSetting { + if condition == projectSetting { // leave it to the test plan return "" } From 78952908420d21f91cbdd7ed6a472d9eb81d7e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:55:30 +0200 Subject: [PATCH 14/16] Replace the shouldStepCollectDiagnostics doc comment with an inline note Co-Authored-By: Claude Fable 5.1 --- step/utils.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/step/utils.go b/step/utils.go index 8fc430bb..7a05486d 100644 --- a/step/utils.go +++ b/step/utils.go @@ -116,11 +116,8 @@ func (u utils) CreateTestParams(cfg Config, xcresultPath, swiftPackagesPath stri // -collect-test-diagnostics and collects Simulator diagnostics itself after a failing test run. const minimumXcodeMajorWithDiagnosticsOption = 26 -// shouldStepCollectDiagnostics decides whether the Step runs its own `simctl diagnose` after the tests. -// Since Xcode 26 xcodebuild collects the same diagnostics into the xcresult after a failing run -// (see xcodebuildDiagnosticsOverride), so the Step does not collect on those Xcode versions. func shouldStepCollectDiagnostics(condition exportCondition, testFailed bool, xcodeMajorVersion int64) bool { - if xcodeMajorVersion >= minimumXcodeMajorWithDiagnosticsOption { + if xcodeMajorVersion >= minimumXcodeMajorWithDiagnosticsOption { // xcodebuild collects return false } From 1bd9be2f0519bcc4de95038bb4711646fe402c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:56:09 +0200 Subject: [PATCH 15/16] Only mention xcodebuild's diagnostics collection where it exists The teardown notice that xcodebuild collected the diagnostics into the xcresult also fired for a failed project_setting run on Xcode 16 and earlier, where nothing collects anything. Gate it on the Xcode version. Co-Authored-By: Claude Fable 5.1 --- step/step.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/step/step.go b/step/step.go index 5ed6b0b0..df06deee 100644 --- a/step/step.go +++ b/step/step.go @@ -500,9 +500,9 @@ func (s XcodeTestRunner) teardownSimulator(simulatorID string, simulatorDebug ex s.logger.Donef("Simulator diagnostics are available as an artifact (%s)", diagnosticsPath) simulatorDiagnosticsPath = diagnosticsPath } - } else if simulatorDebug != never && testErr != nil { + } else if simulatorDebug != never && testErr != nil && xcodeMajorVersion >= minimumXcodeMajorWithDiagnosticsOption { s.logger.Println() - s.logger.Infof("On Xcode 26 and later xcodebuild collects the Simulator diagnostics into the xcresult (xcrun xcresulttool export diagnostics), the Step does not collect them separately") + s.logger.Infof("xcodebuild collects the Simulator diagnostics into the xcresult (xcrun xcresulttool export diagnostics), the Step does not collect them separately") } // Shut down the simulator if it was started by the step for diagnostic logs. From 82be39b85765840f47e82b26d5d4114303dc0598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20P=C3=BCs=C3=B6k?= <7979773+lpusok@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:58:18 +0200 Subject: [PATCH 16/16] Name the xcodebuild-collects decision in teardown Co-Authored-By: Claude Fable 5.1 --- step/diagnostics_test.go | 26 ++++++++++++++++++++++++++ step/step.go | 2 +- step/utils.go | 4 ++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/step/diagnostics_test.go b/step/diagnostics_test.go index 23a7273f..831e5dff 100644 --- a/step/diagnostics_test.go +++ b/step/diagnostics_test.go @@ -129,3 +129,29 @@ func Test_shouldStepCollectDiagnostics(t *testing.T) { }) } } + +func Test_xcodebuildCollectsDiagnostics(t *testing.T) { + tests := []struct { + name string + condition exportCondition + testFailed bool + xcodeMajorVersion int64 + want bool + }{ + {name: "Xcode 27, on_failure, failed", condition: onFailure, testFailed: true, xcodeMajorVersion: 27, want: true}, + {name: "Xcode 27, always, failed", condition: always, testFailed: true, xcodeMajorVersion: 27, want: true}, + {name: "Xcode 27, project_setting, failed", condition: projectSetting, testFailed: true, xcodeMajorVersion: 27, want: true}, + {name: "Xcode 27, never, failed", condition: never, testFailed: true, xcodeMajorVersion: 27, want: false}, + {name: "Xcode 27, on_failure, passed", condition: onFailure, testFailed: false, xcodeMajorVersion: 27, want: false}, + // xcodebuild has no collection of its own before Xcode 26. + {name: "Xcode 16, on_failure, failed", condition: onFailure, testFailed: true, xcodeMajorVersion: 16, want: false}, + {name: "Xcode 16, project_setting, failed", condition: projectSetting, testFailed: true, xcodeMajorVersion: 16, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := xcodebuildCollectsDiagnostics(tt.condition, tt.testFailed, tt.xcodeMajorVersion) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/step/step.go b/step/step.go index df06deee..0babd6f5 100644 --- a/step/step.go +++ b/step/step.go @@ -500,7 +500,7 @@ func (s XcodeTestRunner) teardownSimulator(simulatorID string, simulatorDebug ex s.logger.Donef("Simulator diagnostics are available as an artifact (%s)", diagnosticsPath) simulatorDiagnosticsPath = diagnosticsPath } - } else if simulatorDebug != never && testErr != nil && xcodeMajorVersion >= minimumXcodeMajorWithDiagnosticsOption { + } else if xcodebuildCollectsDiagnostics(simulatorDebug, testErr != nil, xcodeMajorVersion) { s.logger.Println() s.logger.Infof("xcodebuild collects the Simulator diagnostics into the xcresult (xcrun xcresulttool export diagnostics), the Step does not collect them separately") } diff --git a/step/utils.go b/step/utils.go index 7a05486d..8cf81748 100644 --- a/step/utils.go +++ b/step/utils.go @@ -124,6 +124,10 @@ func shouldStepCollectDiagnostics(condition exportCondition, testFailed bool, xc return condition == always || (condition == onFailure && testFailed) } +func xcodebuildCollectsDiagnostics(condition exportCondition, testFailed bool, xcodeMajorVersion int64) bool { + return xcodeMajorVersion >= minimumXcodeMajorWithDiagnosticsOption && condition != never && testFailed +} + func xcodebuildDiagnosticsOverride(condition exportCondition, xcodeMajorVersion int64, additionalOptions []string) string { if xcodeMajorVersion < minimumXcodeMajorWithDiagnosticsOption { // no such option yet return ""