Fail a chaos run whose nodes logged errors

The simulation exit ladder reported panics and failed assertions but said
nothing about ERROR-level log lines, so a run in which every node errored
on every line still exited 0.

Add a max_errors assertion and apply it to every scenario by default
rather than having each one opt in. This is a floor on what a green run
means rather than a property of an individual scenario, and a scenario
that has to ask for the floor is one that can forget to.

The default ceiling of 0 is what the archived corpus supports: across
2416 result directories no node log contains an ERROR-level line, while
the WARN counter extracted by the same code path ranges from 0 to 1135,
so the counter is known to discriminate rather than merely known to read
zero. A scenario that legitimately induces errors raises the ceiling in
its own YAML and says there why.

Reject rust_log: off alongside it, since that is the one level that would
leave the ceiling counting zero whatever the mesh did.

testing/chaos/sim/assertions.py

@@ -20,6 +20,7 @@ from dataclasses import dataclass
 from .control import snapshot_all_bloom
 from .scenario import (
     BloomSendRateAssertion,
+    MaxErrorsAssertion,
     MaxParentSwitchesAssertion,
     MinParentSwitchesAssertion,
 )
@@ -170,6 +171,47 @@ def evaluate_max_parent_switches(
     )
 
 
+def evaluate_max_errors(
+    cfg: MaxErrorsAssertion,
+    errors: list[tuple[str, str]],
+) -> AssertionOutcome:
+    """Ceiling on ERROR-level log lines across the whole mesh.
+
+    ``errors`` is the ``AnalysisResult.errors`` list of ``(source, line)``
+    pairs rather than a bare count, so a failure can name the nodes and
+    quote the lines. A ceiling breach that only reports a number sends the
+    reader back to the logs it was supposed to save them reading.
+    """
+    count = len(errors)
+    if count <= cfg.max_total:
+        return AssertionOutcome(
+            name="max_errors",
+            passed=True,
+            detail=(
+                f"PASS max_errors: {count} ERROR line(s) mesh-wide <= "
+                f"ceiling {cfg.max_total}"
+            ),
+        )
+
+    per_node: dict[str, int] = {}
+    for source, _line in errors:
+        per_node[source] = per_node.get(source, 0) + 1
+    worst = sorted(per_node.items(), key=lambda kv: -kv[1])
+    breakdown = ", ".join(f"{src}={n}" for src, n in worst)
+    samples = "\n".join(
+        f"    [{src}] {line.strip()}" for src, line in errors[:5]
+    )
+    return AssertionOutcome(
+        name="max_errors",
+        passed=False,
+        detail=(
+            f"FAIL max_errors: {count} ERROR line(s) mesh-wide > ceiling "
+            f"{cfg.max_total} — per node: {breakdown}. First "
+            f"{min(5, count)}:\n{samples}"
+        ),
+    )
+
+
 def evaluate_min_parent_switches(
     cfg: MinParentSwitchesAssertion,
     parent_switch_count: int,

testing/chaos/sim/runner.py

@@ -15,6 +15,7 @@ from datetime import datetime
 from .assertions import (
     AssertionOutcome,
     BloomSendRateMonitor,
+    evaluate_max_errors,
     evaluate_max_parent_switches,
     evaluate_min_parent_switches,
 )
@@ -604,6 +605,18 @@ class SimRunner:
                 else:
                     log.error("%s", outcome.detail)
 
+            # Applied to every scenario by default, so this is the one
+            # assertion that is present even when the YAML declares no
+            # assertions block at all.
+            err_cfg = self.scenario.assertions.max_errors
+            if err_cfg is not None:
+                outcome = evaluate_max_errors(err_cfg, result.errors)
+                self.assertion_outcomes.append(outcome)
+                if outcome.passed:
+                    log.info("%s", outcome.detail)
+                else:
+                    log.error("%s", outcome.detail)
+
             # Write assertion outcomes
             if self.assertion_outcomes:
                 assertions_path = os.path.join(self.output_dir, "assertions.txt")

testing/chaos/sim/scenario.py

@@ -218,6 +218,28 @@ class MaxParentSwitchesAssertion:
     node: str | None = None
 
 
+@dataclass
+class MaxErrorsAssertion:
+    """Ceiling on ERROR-level lines across every node's log.
+
+    Unlike the other assertions this one is applied to every scenario
+    whether or not the YAML asks for it, because it is a floor on what a
+    green run means rather than a property of one scenario: without it a
+    run in which every node errors on every line still exits 0.
+
+    ``max_total`` defaults to 0, which is what the archived corpus
+    supports. Across 2416 archived run directories no node log contains a
+    single ERROR-level line, while the sibling WARN counter extracted by
+    the same code path ranges from 0 to 1135 — so 0 is an observed value
+    and not an aspiration, and the counter is known to discriminate.
+
+    A scenario that legitimately induces errors raises the ceiling in its
+    own YAML and must say at that site why the errors are expected.
+    """
+
+    max_total: int = 0
+
+
 @dataclass
 class AssertionsConfig:
     """Optional post-run assertions evaluated against control-socket data."""
@@ -225,6 +247,7 @@ class AssertionsConfig:
     bloom_send_rate: BloomSendRateAssertion | None = None
     min_parent_switches: MinParentSwitchesAssertion | None = None
     max_parent_switches: MaxParentSwitchesAssertion | None = None
+    max_errors: MaxErrorsAssertion | None = None
 
 
 @dataclass
@@ -301,6 +324,7 @@ _SECTION_KEYS = {
     "link_swap.edges[]": {"edge", "policy"},
     "assertions": {
         "bloom_send_rate", "min_parent_switches", "max_parent_switches",
+        "max_errors",
     },
     "logging": {"rust_log", "output_dir"},
 }
@@ -308,6 +332,7 @@ _ASSERTION_KEYS = {
     "bloom_send_rate": {"window_secs", "max_per_node"},
     "min_parent_switches": {"min_total"},
     "max_parent_switches": {"max_total", "node"},
+    "max_errors": {"max_total"},
 }
 _NETEM_POLICY_KEYS = {
     "delay_ms", "jitter_ms", "loss_pct", "duplicate_pct", "reorder_pct",
@@ -589,6 +614,34 @@ def load_scenario(path: str) -> Scenario:
             max_total=max_total,
             node=node,
         )
+    if "max_errors" in asrt:
+        mev = asrt["max_errors"]
+        _reject_unknown(
+            mev, _ASSERTION_KEYS["max_errors"], "assertions.max_errors",
+        )
+        if "max_total" not in mev:
+            raise ValueError(
+                "assertions.max_errors: max_total is required (the point of "
+                "overriding the default ceiling is to name the new number)"
+            )
+        err_total = mev["max_total"]
+        if isinstance(err_total, bool) or not isinstance(err_total, int):
+            raise ValueError(
+                f"assertions.max_errors: max_total must be a non-negative "
+                f"integer, got {err_total!r}"
+            )
+        if err_total < 0:
+            raise ValueError(
+                f"assertions.max_errors: max_total must be non-negative, "
+                f"got {err_total}"
+            )
+        s.assertions.max_errors = MaxErrorsAssertion(max_total=err_total)
+    else:
+        # Default-on. See MaxErrorsAssertion for why this one assertion is
+        # applied without being asked for: it is the floor on what a green
+        # run means, and a scenario that has to opt in is a scenario that
+        # can forget to.
+        s.assertions.max_errors = MaxErrorsAssertion()
 
     # Logging section
     lg = raw.get("logging", {})
@@ -639,9 +692,31 @@ def _validate_parent_switch_observability(s: Scenario):
         )
 
 
+def _validate_error_observability(s: Scenario):
+    """Refuse an error ceiling the log level cannot observe.
+
+    Narrower than the parent-switch guard on purpose: ERROR lines survive
+    every level except ``off``, so ``off`` is the only setting that turns
+    this assertion into one that counts zero whatever the mesh did. Since
+    the ceiling is applied by default, a scenario silencing its logs would
+    otherwise acquire an assertion that cannot fail.
+    """
+    if s.assertions.max_errors is None:
+        return
+    default_level = s.logging.rust_log.split(",")[0].strip().lower()
+    if default_level == "off":
+        raise ValueError(
+            "logging.rust_log is 'off', which suppresses the ERROR-level "
+            "lines the max_errors assertion counts. The assertion would see "
+            "zero errors regardless of what the mesh did. Raise the level, "
+            "or state a deliberate override in assertions.max_errors."
+        )
+
+
 def _validate(s: Scenario):
     """Validate scenario constraints."""
     _validate_parent_switch_observability(s)
+    _validate_error_observability(s)
     if s.topology.num_nodes < 2:
         raise ValueError("topology.num_nodes must be >= 2")
     if s.topology.num_nodes > 250: