mirror of
https://github.com/retroSoC/retroSoC.git
synced 2026-11-04 22:32:36 +00:00
fix(flow): retain terminal evidence across interrupted runs
Record parent and child process IDs for diagnostics, persist interruption before cleanup, and ignore repeated termination signals during cleanup. Close unexpected failures with a terminal result and retain the child's exit status without treating namespace-local PID absence as completion. Require a successful terminal run_flow result when checking simulation logs with an associated result file. Cover repeated cancellation and reject running, failed or inconsistent terminal records despite PASS text. This supports the TINY-R2-P7 campaign. Interrupted and timed-out runs remain non-acceptance evidence; UART output alone cannot qualify a Tiny simulation.
This commit is contained in:
3 files changed
+102
-10
No files matched your search
@@ -39,6 +39,19 @@ def main() -> int:
|
||||
if default_failure:
|
||||
rejected.append(default_failure.group(0))
|
||||
|
||||
flow_result_path = args.result.with_name("result-sim.json")
|
||||
flow_status = None
|
||||
if flow_result_path.is_file():
|
||||
try:
|
||||
flow_result = json.loads(flow_result_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
flow_result = {}
|
||||
flow_status = flow_result.get("status", "invalid")
|
||||
# PIDs are namespace-local and can be reused. A durable terminal
|
||||
# verdict, not a liveness guess from this process, establishes success.
|
||||
if flow_status != "passed" or flow_result.get("exit_code") != 0:
|
||||
missing.append("successful terminal run_flow result")
|
||||
|
||||
passed = not missing and not rejected
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
@@ -48,6 +61,8 @@ def main() -> int:
|
||||
"missing_markers": missing,
|
||||
"rejected_matches": sorted(set(rejected)),
|
||||
}
|
||||
if flow_status is not None:
|
||||
report["flow_status"] = flow_status
|
||||
atomic_write(args.result, json.dumps(report, indent=2, sort_keys=True) + "\n")
|
||||
print(f"simulation check: {report['status']} ({args.result.resolve()})")
|
||||
return int(not passed)
|
||||
|
||||
+29
-10
@@ -124,24 +124,22 @@ def main() -> int:
|
||||
args.result.parent.mkdir(parents=True, exist_ok=True)
|
||||
started = datetime.now(timezone.utc)
|
||||
start_clock = time.monotonic()
|
||||
atomic_write(
|
||||
args.result,
|
||||
json.dumps(
|
||||
{
|
||||
running_result = {
|
||||
"schema_version": 1,
|
||||
"tool": args.tool,
|
||||
"command": command,
|
||||
"started_at": started.isoformat(),
|
||||
"status": "running",
|
||||
"parent_pid": os.getpid(),
|
||||
"log": str(args.log.resolve()),
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n",
|
||||
)
|
||||
}
|
||||
atomic_write(args.result, json.dumps(running_result, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
def interrupt_flow(_signum: int, _frame: object) -> None:
|
||||
# A launcher can deliver SIGINT followed by SIGTERM. Do not let a
|
||||
# second signal interrupt process-group cleanup or result persistence.
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
raise KeyboardInterrupt
|
||||
|
||||
signal.signal(signal.SIGINT, interrupt_flow)
|
||||
@@ -169,6 +167,8 @@ def main() -> int:
|
||||
pass_fds=jobserver_fds(),
|
||||
start_new_session=True,
|
||||
)
|
||||
running_result["child_pid"] = process.pid
|
||||
atomic_write(args.result, json.dumps(running_result, indent=2, sort_keys=True) + "\n")
|
||||
assert process.stdout is not None
|
||||
if args.stream_bytes:
|
||||
while chunk := os.read(process.stdout.fileno(), 4096):
|
||||
@@ -193,11 +193,25 @@ def main() -> int:
|
||||
returncode = process.wait()
|
||||
except KeyboardInterrupt:
|
||||
error_message = "interrupted"
|
||||
# Persist cancellation before the bounded cleanup wait. If the outer
|
||||
# executor subsequently kills this wrapper, the file must not imply
|
||||
# that the job is still running or that its child finished normally.
|
||||
cancelled = dict(running_result, status="interrupted", exit_code=130,
|
||||
interrupted_at=datetime.now(timezone.utc).isoformat(),
|
||||
child_cleanup_pending=process is not None)
|
||||
atomic_write(args.result, json.dumps(cancelled, indent=2, sort_keys=True) + "\n")
|
||||
if process is not None:
|
||||
terminate_process_group(process)
|
||||
returncode = 130
|
||||
except (OSError, subprocess.SubprocessError) as error:
|
||||
error_message = str(error)
|
||||
if process is not None:
|
||||
terminate_process_group(process)
|
||||
except BaseException as error:
|
||||
error_message = f"unexpected {type(error).__name__}: {error}"
|
||||
if process is not None:
|
||||
terminate_process_group(process)
|
||||
returncode = 1
|
||||
|
||||
finished = datetime.now(timezone.utc)
|
||||
result = {
|
||||
@@ -211,8 +225,13 @@ def main() -> int:
|
||||
"peak_rss_kib": resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss,
|
||||
"exit_code": returncode,
|
||||
"status": "interrupted" if returncode == 130 else ("passed" if returncode == 0 else "failed"),
|
||||
"parent_pid": os.getpid(),
|
||||
"log": str(args.log.resolve()),
|
||||
}
|
||||
if process is not None:
|
||||
result["child_pid"] = process.pid
|
||||
result["child_exit_code"] = process.returncode
|
||||
result["child_cleanup_pending"] = process.returncode is None
|
||||
if error_message:
|
||||
result["error"] = error_message
|
||||
if args.success_marker:
|
||||
|
||||
@@ -1969,6 +1969,8 @@ def test_run_flow_writes_structured_result(tmp_path: Path) -> None:
|
||||
assert data["status"] == "passed"
|
||||
assert data["exit_code"] == 0
|
||||
assert data["duration_seconds"] >= 0
|
||||
assert isinstance(data["parent_pid"], int)
|
||||
assert isinstance(data["child_pid"], int)
|
||||
assert isinstance(data["peak_rss_kib"], int)
|
||||
assert data["peak_rss_kib"] > 0
|
||||
assert log.read_text(encoding="utf-8") == "flow output\n"
|
||||
@@ -2120,6 +2122,41 @@ def test_run_flow_streams_bytes_without_waiting_for_newline(tmp_path: Path) -> N
|
||||
assert json.loads(result.read_text(encoding="utf-8"))["status"] == "passed"
|
||||
|
||||
|
||||
def test_run_flow_persists_cancel_before_cleanup_and_ignores_second_signal(tmp_path: Path) -> None:
|
||||
result = tmp_path / "cancelled.json"
|
||||
process = subprocess.Popen([
|
||||
sys.executable, str(ROOT / "scripts/run_flow.py"), "--tool", "unit",
|
||||
"--log", str(tmp_path / "cancelled.log"), "--result", str(result), "--",
|
||||
sys.executable, "-u", "-c",
|
||||
"import signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); "
|
||||
"print('READY',flush=True); time.sleep(30)",
|
||||
], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
try:
|
||||
assert process.stdout is not None
|
||||
assert process.stdout.readline().strip() == "READY"
|
||||
process.send_signal(signal.SIGINT)
|
||||
for _ in range(100):
|
||||
checkpoint = json.loads(result.read_text())
|
||||
if checkpoint["status"] == "interrupted":
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert checkpoint["status"] == "interrupted"
|
||||
assert checkpoint["child_cleanup_pending"] is True
|
||||
assert "finished_at" not in checkpoint
|
||||
process.send_signal(signal.SIGTERM)
|
||||
_, stderr = process.communicate(timeout=10)
|
||||
assert process.returncode == 130, stderr
|
||||
final = json.loads(result.read_text())
|
||||
assert final["status"] == "interrupted"
|
||||
assert final["child_cleanup_pending"] is False
|
||||
assert final["child_exit_code"] == -signal.SIGKILL
|
||||
assert final["finished_at"] >= final["started_at"]
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
|
||||
|
||||
def test_run_flow_records_interruption(tmp_path: Path) -> None:
|
||||
log = tmp_path / "interrupted.log"
|
||||
result = tmp_path / "interrupted.json"
|
||||
@@ -2192,6 +2229,27 @@ def test_simulation_success_marker_and_failure_detection(tmp_path: Path) -> None
|
||||
assert json.loads(result.read_text(encoding="utf-8"))["status"] == "failed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status,exit_code", [("running", None), ("failed", 1),
|
||||
("passed", 124), ("invalid", None)])
|
||||
def test_check_simulation_requires_terminal_flow(tmp_path: Path, status: str,
|
||||
exit_code: int | None) -> None:
|
||||
log = tmp_path / "sim.log"
|
||||
result = tmp_path / "result-sim-check.json"
|
||||
log.write_text("SIM_TEST_PASS code=0\n", encoding="utf-8")
|
||||
(tmp_path / "result-sim.json").write_text(
|
||||
json.dumps({"status": status, "exit_code": exit_code}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
checked = subprocess.run(
|
||||
[sys.executable, str(ROOT / "scripts/check_simulation.py"), "--log", str(log),
|
||||
"--result", str(result)], text=True, capture_output=True,
|
||||
)
|
||||
report = json.loads(result.read_text(encoding="utf-8"))
|
||||
assert checked.returncode != 0
|
||||
assert report["status"] == "failed"
|
||||
assert report["flow_status"] == status
|
||||
|
||||
|
||||
def test_warning_baseline_rejects_new_signature(tmp_path: Path) -> None:
|
||||
profile = "unit-profile"
|
||||
log = tmp_path / "variant/sim/verilator/verilating.log"
|
||||
|
||||
Reference in new issue
Block a user