mirror of
https://github.com/jmcorgan/fips.git
synced 2026-10-05 11:08:25 +00:00
Johnathan Corgan
da9a12a3dd
The ci nextest profile retries a failing test twice, so a test that fails and then passes leaves the job green, with the retry visible only as "(1 flaky)" in the summary line and a FLAKY line in the log. Recent macOS runs have carried such flakes unnoticed. The Linux job's two junit reporters ignore nextest's <flakyFailure> elements, and the macOS and Windows jobs have no reporter, so no job surfaced them. testing/check-nextest-flaky.sh reads the ci profile's JUnit report and emits a warning annotation and a step-summary line for each test that passed only on retry. It exits 0 so a flake does not red the run, and 2 when there is no report to read. Each of the three unit-test jobs runs it after nextest, on success or failure but not on a cancelled run. Fixture tests over recorded nextest reports (clean, flaky, and a test that failed every attempt) run in local CI and in the GitHub CI parity job. The unit-test jobs cache the whole target directory, so a restored target/nextest/ci/junit.xml from an earlier run could sit where the check reads and hide a run that failed before writing one. Delete the report before nextest runs.
73 lines
3.2 KiB
XML
73 lines
3.2 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites name="nextest-run" tests="4" skipped="0" failures="1" errors="0" uuid="f7324cab-d0eb-42ec-aa2e-bb588c9f47b5" timestamp="2026-10-01T03:24:55.659+00:00" time="0.052">
|
|
<testsuite name="flakydemo" tests="4" skipped="0" errors="0" failures="1">
|
|
<testcase name="tests::steady" classname="flakydemo" timestamp="2026-10-01T03:24:55.659+00:00" time="0.009"/>
|
|
<testcase name="tests::also_flips" classname="flakydemo" timestamp="2026-10-01T03:24:55.660+00:00" time="0.009"/>
|
|
<testcase name="tests::flips_once" classname="flakydemo" timestamp="2026-10-01T03:24:55.660+00:00" time="0.015"/>
|
|
<testcase name="tests::always_fails" classname="flakydemo" timestamp="2026-10-01T03:24:55.659+00:00" time="0.015">
|
|
<failure message="thread 'tests::always_fails' (191023) panicked at src/lib.rs:22:67" type="test failure with exit code 101">thread 'tests::always_fails' (191023) panicked at src/lib.rs:22:67:
|
|
nope
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace</failure>
|
|
<rerunFailure timestamp="2026-10-01T03:24:55.679+00:00" time="0.016" message="thread 'tests::always_fails' (191028) panicked at src/lib.rs:22:67" type="test failure with exit code 101">thread 'tests::always_fails' (191028) panicked at src/lib.rs:22:67:
|
|
nope
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
|
<system-out>
|
|
running 1 test
|
|
test tests::always_fails ... FAILED
|
|
|
|
failures:
|
|
|
|
failures:
|
|
tests::always_fails
|
|
|
|
test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 3 filtered out; finished in 0.00s
|
|
|
|
</system-out>
|
|
<system-err>
|
|
thread 'tests::always_fails' (191028) panicked at src/lib.rs:22:67:
|
|
nope
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
|
</system-err>
|
|
</rerunFailure>
|
|
<rerunFailure timestamp="2026-10-01T03:24:55.695+00:00" time="0.016" message="thread 'tests::always_fails' (191032) panicked at src/lib.rs:22:67" type="test failure with exit code 101">thread 'tests::always_fails' (191032) panicked at src/lib.rs:22:67:
|
|
nope
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
|
<system-out>
|
|
running 1 test
|
|
test tests::always_fails ... FAILED
|
|
|
|
failures:
|
|
|
|
failures:
|
|
tests::always_fails
|
|
|
|
test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 3 filtered out; finished in 0.01s
|
|
|
|
</system-out>
|
|
<system-err>
|
|
thread 'tests::always_fails' (191032) panicked at src/lib.rs:22:67:
|
|
nope
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
|
</system-err>
|
|
</rerunFailure>
|
|
<system-out>
|
|
running 1 test
|
|
test tests::always_fails ... FAILED
|
|
|
|
failures:
|
|
|
|
failures:
|
|
tests::always_fails
|
|
|
|
test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 3 filtered out; finished in 0.00s
|
|
|
|
</system-out>
|
|
<system-err>
|
|
thread 'tests::always_fails' (191023) panicked at src/lib.rs:22:67:
|
|
nope
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
|
</system-err>
|
|
</testcase>
|
|
</testsuite>
|
|
</testsuites>
|