fix: stop the strict policy for checking warnings and metrics

AGENTS.md

@@ -139,9 +139,12 @@ warning checks, and metric collection.
   configured success marker, and the log contains no `FAILED`, `FATAL`,
   `assertion failed`, or `%Error` marker.
 - New self-owned C compiler warnings fail the regression. New or increased EDA
-  warning signatures fail `check-warnings`; resolved warnings are reported.
+  warning signatures make `check-warnings` fail when invoked directly, but the
+  regression runner records them as non-blocking observations so CI uploads the
+  warning and metric reports; resolved warnings are reported.
 - `check-metrics` records firmware size, area, cell count, timing, and flow
-  duration, but its current policy is observational.
+  duration. It is non-blocking in the regression runner, including if a future
+  policy changes the standalone command into a gate.
 - Verilator currently runs through `scripts/run_flow.py`, which captures the
   emulator stdout in a pipe. UART output can therefore appear only after a
   newline or when simulation ends. Treat `sim.log` and `result-sim*.json` as

docs/engineering.md

@@ -138,8 +138,11 @@ behavior. It is not asynchronous CDC/RDC signoff; implementation timing,
 clock-tree checks, and the technology PLL model remain separate signoff work.
 
 `make check-warnings` normalizes tool messages and compares them with
-`quality/warnings/<profile>/<tool>.json`. New signatures and increased counts fail; resolved warnings
-are reported but do not fail. To refresh one baseline after review:
+`quality/warnings/<profile>/<tool>.json`. New signatures and increased counts make the standalone
+command fail; resolved warnings are reported but do not fail. Regression suites run warning and
+metric checks as non-blocking observations so their JSON reports are generated and uploaded even
+when a baseline delta is present. Compilation, simulation, synthesis, and STA failures remain
+blocking. To refresh one baseline after review:
 
 ```sh
 python3 scripts/analyze_warnings.py baseline \
@@ -151,7 +154,7 @@ python3 scripts/analyze_warnings.py baseline \
 ## Metrics Promotion
 
 `make check-metrics` currently uses `quality/metrics/policy.json` in `observe` mode. Regression
-artifacts retain the metrics from each successful run. After ten successful `main` runs for the same
+artifacts retain the metrics from every completed regression run. After ten successful `main` runs for the same
 profile and locked tools, create the median baseline:
 
 ```sh

scripts/regress.py

@@ -16,6 +16,7 @@ from scripts.check_c_warnings import self_owned_warnings  # noqa: E402
 
 
 RTL_LINT_VALUES = ("SIMU=VERILATOR", "HAVE_SVA=YES", "rtl-lint", "check-rtl-lint")
+OBSERVATION_TARGETS = ("check-warnings", "check-metrics")
 
 
 PR_COMMANDS = (
@@ -132,6 +133,23 @@ def run_command(
     return result.stdout
 
 
+def run_observation(command: list[str], root: Path, environment: dict[str, str]) -> None:
+    """Run a report-producing quality check without changing the regression verdict."""
+    try:
+        result = subprocess.run(command, cwd=root, env=environment, check=False)
+    except OSError as error:
+        print(
+            f"non-blocking observation could not start: {error}: {' '.join(command)}",
+            file=sys.stderr,
+        )
+        return
+    if result.returncode:
+        print(
+            f"non-blocking observation failed (exit {result.returncode}): {' '.join(command)}",
+            file=sys.stderr,
+        )
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description="Run a supported retroSoC regression suite")
     parser.add_argument("--root", type=Path, required=True)
@@ -155,15 +173,11 @@ def main() -> int:
                 print("\n".join(warnings), flush=True)
                 return 1
     for profile in profiles:
-        command = [
-            "make",
-            f"CONFIG={profile}",
-            "check-warnings",
-            "check-metrics",
-        ]
-        print("+ " + " ".join(command), flush=True)
-        if not args.dry_run:
-            subprocess.run(command, cwd=args.root, env=environment, check=True)
+        for target in OBSERVATION_TARGETS:
+            command = ["make", f"CONFIG={profile}", target]
+            print("+ " + " ".join(command), flush=True)
+            if not args.dry_run:
+                run_observation(command, args.root, environment)
     return 0
 
 

tests/test_script_tools.py

@@ -37,6 +37,7 @@ from scripts.generate_mpw import (  # noqa: E402
     remove_legacy_picorv32_entry,
 )
 from scripts.install_toolchain import safe_extract  # noqa: E402
+from scripts import regress  # noqa: E402
 from scripts.regress import (  # noqa: E402
     NIGHTLY_COMMANDS,
     PDK_PR_PROFILES,
@@ -675,6 +676,38 @@ def test_pdk_pr_regressions_cover_firmware_rtl_and_netlist() -> None:
         assert any("SIMU=IVERILOG" in values and "netsim" in values for values in command_values)
 
 
+def test_regression_observations_do_not_block_or_skip_metrics(
+    monkeypatch, tmp_path: Path, capsys
+) -> None:
+    calls: list[list[str]] = []
+
+    monkeypatch.setattr(regress, "select_regression", lambda _suite, _pdk: ((), ("unit-profile",)))
+    monkeypatch.setattr(regress, "regression_environment", lambda: {})
+
+    def fail_quality_check(
+        command: list[str], *, cwd: Path, env: dict[str, str], check: bool
+    ) -> subprocess.CompletedProcess[str]:
+        assert cwd == tmp_path
+        assert env == {}
+        assert check is False
+        calls.append(command)
+        return subprocess.CompletedProcess(command, returncode=1)
+
+    monkeypatch.setattr(regress.subprocess, "run", fail_quality_check)
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        ["regress.py", "--root", str(tmp_path), "--suite", "pr"],
+    )
+
+    assert regress.main() == 0
+    assert calls == [
+        ["make", "CONFIG=unit-profile", "check-warnings"],
+        ["make", "CONFIG=unit-profile", "check-metrics"],
+    ]
+    assert capsys.readouterr().err.count("non-blocking observation failed") == 2
+
+
 def test_rtl_lint_warning_baseline_is_independent(tmp_path: Path) -> None:
     profile = "unit-profile"
     lint_log = tmp_path / "variant/lint/verilator/lint.log"