fix(flow): retain terminal evidence across interrupted runs

Record parent and child process IDs for diagnostics, persist interruption
before cleanup, and ignore repeated termination signals during cleanup.
Close unexpected failures with a terminal result and retain the child's
exit status without treating namespace-local PID absence as completion.

Require a successful terminal run_flow result when checking simulation
logs with an associated result file. Cover repeated cancellation and
reject running, failed or inconsistent terminal records despite PASS text.

This supports the TINY-R2-P7 campaign. Interrupted and timed-out runs remain
non-acceptance evidence; UART output alone cannot qualify a Tiny simulation.
This commit is contained in:
maksyuki committed 2026-10-14 16:55:12 +08:00
1 parent a14fe12724
commit 6fbc6ec521
3 files changed
+102 -10

No files matched your search

+15
View File
@@ -39,6 +39,19 @@ def main() -> int:
if default_failure:
rejected.append(default_failure.group(0))
flow_result_path = args.result.with_name("result-sim.json")
flow_status = None
if flow_result_path.is_file():
try:
flow_result = json.loads(flow_result_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
flow_result = {}
flow_status = flow_result.get("status", "invalid")
# PIDs are namespace-local and can be reused. A durable terminal
# verdict, not a liveness guess from this process, establishes success.
if flow_status != "passed" or flow_result.get("exit_code") != 0:
missing.append("successful terminal run_flow result")
passed = not missing and not rejected
report = {
"schema_version": 1,
@@ -48,6 +61,8 @@ def main() -> int:
"missing_markers": missing,
"rejected_matches": sorted(set(rejected)),
}
if flow_status is not None:
report["flow_status"] = flow_status
atomic_write(args.result, json.dumps(report, indent=2, sort_keys=True) + "\n")
print(f"simulation check: {report['status']} ({args.result.resolve()})")
return int(not passed)
+29 -10
View File
@@ -124,24 +124,22 @@ def main() -> int:
args.result.parent.mkdir(parents=True, exist_ok=True)
started = datetime.now(timezone.utc)
start_clock = time.monotonic()
atomic_write(
args.result,
json.dumps(
{
running_result = {
"schema_version": 1,
"tool": args.tool,
"command": command,
"started_at": started.isoformat(),
"status": "running",
"parent_pid": os.getpid(),
"log": str(args.log.resolve()),
},
indent=2,
sort_keys=True,
)
+ "\n",
)
}
atomic_write(args.result, json.dumps(running_result, indent=2, sort_keys=True) + "\n")
def interrupt_flow(_signum: int, _frame: object) -> None:
# A launcher can deliver SIGINT followed by SIGTERM. Do not let a
# second signal interrupt process-group cleanup or result persistence.
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
raise KeyboardInterrupt
signal.signal(signal.SIGINT, interrupt_flow)
@@ -169,6 +167,8 @@ def main() -> int:
pass_fds=jobserver_fds(),
start_new_session=True,
)
running_result["child_pid"] = process.pid
atomic_write(args.result, json.dumps(running_result, indent=2, sort_keys=True) + "\n")
assert process.stdout is not None
if args.stream_bytes:
while chunk := os.read(process.stdout.fileno(), 4096):
@@ -193,11 +193,25 @@ def main() -> int:
returncode = process.wait()
except KeyboardInterrupt:
error_message = "interrupted"
# Persist cancellation before the bounded cleanup wait. If the outer
# executor subsequently kills this wrapper, the file must not imply
# that the job is still running or that its child finished normally.
cancelled = dict(running_result, status="interrupted", exit_code=130,
interrupted_at=datetime.now(timezone.utc).isoformat(),
child_cleanup_pending=process is not None)
atomic_write(args.result, json.dumps(cancelled, indent=2, sort_keys=True) + "\n")
if process is not None:
terminate_process_group(process)
returncode = 130
except (OSError, subprocess.SubprocessError) as error:
error_message = str(error)
if process is not None:
terminate_process_group(process)
except BaseException as error:
error_message = f"unexpected {type(error).__name__}: {error}"
if process is not None:
terminate_process_group(process)
returncode = 1
finished = datetime.now(timezone.utc)
result = {
@@ -211,8 +225,13 @@ def main() -> int:
"peak_rss_kib": resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss,
"exit_code": returncode,
"status": "interrupted" if returncode == 130 else ("passed" if returncode == 0 else "failed"),
"parent_pid": os.getpid(),
"log": str(args.log.resolve()),
}
if process is not None:
result["child_pid"] = process.pid
result["child_exit_code"] = process.returncode
result["child_cleanup_pending"] = process.returncode is None
if error_message:
result["error"] = error_message
if args.success_marker:
+58
View File
@@ -1969,6 +1969,8 @@ def test_run_flow_writes_structured_result(tmp_path: Path) -> None:
assert data["status"] == "passed"
assert data["exit_code"] == 0
assert data["duration_seconds"] >= 0
assert isinstance(data["parent_pid"], int)
assert isinstance(data["child_pid"], int)
assert isinstance(data["peak_rss_kib"], int)
assert data["peak_rss_kib"] > 0
assert log.read_text(encoding="utf-8") == "flow output\n"
@@ -2120,6 +2122,41 @@ def test_run_flow_streams_bytes_without_waiting_for_newline(tmp_path: Path) -> N
assert json.loads(result.read_text(encoding="utf-8"))["status"] == "passed"
def test_run_flow_persists_cancel_before_cleanup_and_ignores_second_signal(tmp_path: Path) -> None:
result = tmp_path / "cancelled.json"
process = subprocess.Popen([
sys.executable, str(ROOT / "scripts/run_flow.py"), "--tool", "unit",
"--log", str(tmp_path / "cancelled.log"), "--result", str(result), "--",
sys.executable, "-u", "-c",
"import signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); "
"print('READY',flush=True); time.sleep(30)",
], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
assert process.stdout is not None
assert process.stdout.readline().strip() == "READY"
process.send_signal(signal.SIGINT)
for _ in range(100):
checkpoint = json.loads(result.read_text())
if checkpoint["status"] == "interrupted":
break
time.sleep(0.01)
assert checkpoint["status"] == "interrupted"
assert checkpoint["child_cleanup_pending"] is True
assert "finished_at" not in checkpoint
process.send_signal(signal.SIGTERM)
_, stderr = process.communicate(timeout=10)
assert process.returncode == 130, stderr
final = json.loads(result.read_text())
assert final["status"] == "interrupted"
assert final["child_cleanup_pending"] is False
assert final["child_exit_code"] == -signal.SIGKILL
assert final["finished_at"] >= final["started_at"]
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=5)
def test_run_flow_records_interruption(tmp_path: Path) -> None:
log = tmp_path / "interrupted.log"
result = tmp_path / "interrupted.json"
@@ -2192,6 +2229,27 @@ def test_simulation_success_marker_and_failure_detection(tmp_path: Path) -> None
assert json.loads(result.read_text(encoding="utf-8"))["status"] == "failed"
@pytest.mark.parametrize("status,exit_code", [("running", None), ("failed", 1),
("passed", 124), ("invalid", None)])
def test_check_simulation_requires_terminal_flow(tmp_path: Path, status: str,
exit_code: int | None) -> None:
log = tmp_path / "sim.log"
result = tmp_path / "result-sim-check.json"
log.write_text("SIM_TEST_PASS code=0\n", encoding="utf-8")
(tmp_path / "result-sim.json").write_text(
json.dumps({"status": status, "exit_code": exit_code}),
encoding="utf-8",
)
checked = subprocess.run(
[sys.executable, str(ROOT / "scripts/check_simulation.py"), "--log", str(log),
"--result", str(result)], text=True, capture_output=True,
)
report = json.loads(result.read_text(encoding="utf-8"))
assert checked.returncode != 0
assert report["status"] == "failed"
assert report["flow_status"] == status
def test_warning_baseline_rejects_new_signature(tmp_path: Path) -> None:
profile = "unit-profile"
log = tmp_path / "variant/sim/verilator/verilating.log"