mirror of
https://github.com/jmcorgan/fips.git
synced 2026-10-05 11:08:25 +00:00
Johnathan Corgan
da9a12a3dd
The ci nextest profile retries a failing test twice, so a test that fails and then passes leaves the job green, with the retry visible only as "(1 flaky)" in the summary line and a FLAKY line in the log. Recent macOS runs have carried such flakes unnoticed. The Linux job's two junit reporters ignore nextest's <flakyFailure> elements, and the macOS and Windows jobs have no reporter, so no job surfaced them. testing/check-nextest-flaky.sh reads the ci profile's JUnit report and emits a warning annotation and a step-summary line for each test that passed only on retry. It exits 0 so a flake does not red the run, and 2 when there is no report to read. Each of the three unit-test jobs runs it after nextest, on success or failure but not on a cancelled run. Fixture tests over recorded nextest reports (clean, flaky, and a test that failed every attempt) run in local CI and in the GitHub CI parity job. The unit-test jobs cache the whole target directory, so a restored target/nextest/ci/junit.xml from an earlier run could sit where the check reads and hide a run that failed before writing one. Delete the report before nextest runs.
54 lines
2.6 KiB
XML
54 lines
2.6 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites name="nextest-run" tests="4" skipped="0" failures="0" errors="0" uuid="3a0fa717-c76b-4698-8d37-2a9d18fce812" timestamp="2026-10-01T03:24:08.148+00:00" time="0.029">
|
|
<testsuite name="flakydemo" tests="4" skipped="0" errors="0" failures="0">
|
|
<testcase name="tests::always_fails" classname="flakydemo" timestamp="2026-10-01T03:24:08.148+00:00" time="0.012"/>
|
|
<testcase name="tests::steady" classname="flakydemo" timestamp="2026-10-01T03:24:08.148+00:00" time="0.013"/>
|
|
<testcase name="tests::flips_once" classname="flakydemo" timestamp="2026-10-01T03:24:08.162+00:00" time="0.014">
|
|
<flakyFailure timestamp="2026-10-01T03:24:08.148+00:00" time="0.013" message="thread 'tests::flips_once' (174785) panicked at src/lib.rs:10:13" type="test failure with exit code 101">thread 'tests::flips_once' (174785) panicked at src/lib.rs:10:13:
|
|
first attempt fails
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
|
<system-out>
|
|
running 1 test
|
|
test tests::flips_once ... FAILED
|
|
|
|
failures:
|
|
|
|
failures:
|
|
tests::flips_once
|
|
|
|
test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 3 filtered out; finished in 0.00s
|
|
|
|
</system-out>
|
|
<system-err>
|
|
thread 'tests::flips_once' (174785) panicked at src/lib.rs:10:13:
|
|
first attempt fails
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
|
</system-err>
|
|
</flakyFailure>
|
|
</testcase>
|
|
<testcase name="tests::also_flips" classname="flakydemo" timestamp="2026-10-01T03:24:08.168+00:00" time="0.009">
|
|
<flakyFailure timestamp="2026-10-01T03:24:08.148+00:00" time="0.019" message="thread 'tests::also_flips' (174790) panicked at src/lib.rs:18:13" type="test failure with exit code 101">thread 'tests::also_flips' (174790) panicked at src/lib.rs:18:13:
|
|
first attempt fails <&> "quoted"
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
|
<system-out>
|
|
running 1 test
|
|
test tests::also_flips ... FAILED
|
|
|
|
failures:
|
|
|
|
failures:
|
|
tests::also_flips
|
|
|
|
test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 3 filtered out; finished in 0.00s
|
|
|
|
</system-out>
|
|
<system-err>
|
|
thread 'tests::also_flips' (174790) panicked at src/lib.rs:18:13:
|
|
first attempt fails <&> "quoted"
|
|
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
|
|
</system-err>
|
|
</flakyFailure>
|
|
</testcase>
|
|
</testsuite>
|
|
</testsuites>
|