mirror of
https://github.com/retroSoC/retroSoC.git
synced 2026-11-04 22:32:36 +00:00
fix(flow): retain terminal evidence across interrupted runs
Record parent and child process IDs for diagnostics, persist interruption before cleanup, and ignore repeated termination signals during cleanup. Close unexpected failures with a terminal result and retain the child's exit status without treating namespace-local PID absence as completion. Require a successful terminal run_flow result when checking simulation logs with an associated result file. Cover repeated cancellation and reject running, failed or inconsistent terminal records despite PASS text. This supports the TINY-R2-P7 campaign. Interrupted and timed-out runs remain non-acceptance evidence; UART output alone cannot qualify a Tiny simulation.
This commit is contained in:
3 files changed
+108
-16
No files matched your search
@@ -39,6 +39,19 @@ def main() -> int:
|
|||||||
if default_failure:
|
if default_failure:
|
||||||
rejected.append(default_failure.group(0))
|
rejected.append(default_failure.group(0))
|
||||||
|
|
||||||
|
flow_result_path = args.result.with_name("result-sim.json")
|
||||||
|
flow_status = None
|
||||||
|
if flow_result_path.is_file():
|
||||||
|
try:
|
||||||
|
flow_result = json.loads(flow_result_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
flow_result = {}
|
||||||
|
flow_status = flow_result.get("status", "invalid")
|
||||||
|
# PIDs are namespace-local and can be reused. A durable terminal
|
||||||
|
# verdict, not a liveness guess from this process, establishes success.
|
||||||
|
if flow_status != "passed" or flow_result.get("exit_code") != 0:
|
||||||
|
missing.append("successful terminal run_flow result")
|
||||||
|
|
||||||
passed = not missing and not rejected
|
passed = not missing and not rejected
|
||||||
report = {
|
report = {
|
||||||
"schema_version": 1,
|
"schema_version": 1,
|
||||||
@@ -48,6 +61,8 @@ def main() -> int:
|
|||||||
"missing_markers": missing,
|
"missing_markers": missing,
|
||||||
"rejected_matches": sorted(set(rejected)),
|
"rejected_matches": sorted(set(rejected)),
|
||||||
}
|
}
|
||||||
|
if flow_status is not None:
|
||||||
|
report["flow_status"] = flow_status
|
||||||
atomic_write(args.result, json.dumps(report, indent=2, sort_keys=True) + "\n")
|
atomic_write(args.result, json.dumps(report, indent=2, sort_keys=True) + "\n")
|
||||||
print(f"simulation check: {report['status']} ({args.result.resolve()})")
|
print(f"simulation check: {report['status']} ({args.result.resolve()})")
|
||||||
return int(not passed)
|
return int(not passed)
|
||||||
|
|||||||
+35
-16
@@ -124,24 +124,22 @@ def main() -> int:
|
|||||||
args.result.parent.mkdir(parents=True, exist_ok=True)
|
args.result.parent.mkdir(parents=True, exist_ok=True)
|
||||||
started = datetime.now(timezone.utc)
|
started = datetime.now(timezone.utc)
|
||||||
start_clock = time.monotonic()
|
start_clock = time.monotonic()
|
||||||
atomic_write(
|
running_result = {
|
||||||
args.result,
|
"schema_version": 1,
|
||||||
json.dumps(
|
"tool": args.tool,
|
||||||
{
|
"command": command,
|
||||||
"schema_version": 1,
|
"started_at": started.isoformat(),
|
||||||
"tool": args.tool,
|
"status": "running",
|
||||||
"command": command,
|
"parent_pid": os.getpid(),
|
||||||
"started_at": started.isoformat(),
|
"log": str(args.log.resolve()),
|
||||||
"status": "running",
|
}
|
||||||
"log": str(args.log.resolve()),
|
atomic_write(args.result, json.dumps(running_result, indent=2, sort_keys=True) + "\n")
|
||||||
},
|
|
||||||
indent=2,
|
|
||||||
sort_keys=True,
|
|
||||||
)
|
|
||||||
+ "\n",
|
|
||||||
)
|
|
||||||
|
|
||||||
def interrupt_flow(_signum: int, _frame: object) -> None:
|
def interrupt_flow(_signum: int, _frame: object) -> None:
|
||||||
|
# A launcher can deliver SIGINT followed by SIGTERM. Do not let a
|
||||||
|
# second signal interrupt process-group cleanup or result persistence.
|
||||||
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||||
|
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||||
raise KeyboardInterrupt
|
raise KeyboardInterrupt
|
||||||
|
|
||||||
signal.signal(signal.SIGINT, interrupt_flow)
|
signal.signal(signal.SIGINT, interrupt_flow)
|
||||||
@@ -169,6 +167,8 @@ def main() -> int:
|
|||||||
pass_fds=jobserver_fds(),
|
pass_fds=jobserver_fds(),
|
||||||
start_new_session=True,
|
start_new_session=True,
|
||||||
)
|
)
|
||||||
|
running_result["child_pid"] = process.pid
|
||||||
|
atomic_write(args.result, json.dumps(running_result, indent=2, sort_keys=True) + "\n")
|
||||||
assert process.stdout is not None
|
assert process.stdout is not None
|
||||||
if args.stream_bytes:
|
if args.stream_bytes:
|
||||||
while chunk := os.read(process.stdout.fileno(), 4096):
|
while chunk := os.read(process.stdout.fileno(), 4096):
|
||||||
@@ -193,11 +193,25 @@ def main() -> int:
|
|||||||
returncode = process.wait()
|
returncode = process.wait()
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
error_message = "interrupted"
|
error_message = "interrupted"
|
||||||
|
# Persist cancellation before the bounded cleanup wait. If the outer
|
||||||
|
# executor subsequently kills this wrapper, the file must not imply
|
||||||
|
# that the job is still running or that its child finished normally.
|
||||||
|
cancelled = dict(running_result, status="interrupted", exit_code=130,
|
||||||
|
interrupted_at=datetime.now(timezone.utc).isoformat(),
|
||||||
|
child_cleanup_pending=process is not None)
|
||||||
|
atomic_write(args.result, json.dumps(cancelled, indent=2, sort_keys=True) + "\n")
|
||||||
if process is not None:
|
if process is not None:
|
||||||
terminate_process_group(process)
|
terminate_process_group(process)
|
||||||
returncode = 130
|
returncode = 130
|
||||||
except (OSError, subprocess.SubprocessError) as error:
|
except (OSError, subprocess.SubprocessError) as error:
|
||||||
error_message = str(error)
|
error_message = str(error)
|
||||||
|
if process is not None:
|
||||||
|
terminate_process_group(process)
|
||||||
|
except BaseException as error:
|
||||||
|
error_message = f"unexpected {type(error).__name__}: {error}"
|
||||||
|
if process is not None:
|
||||||
|
terminate_process_group(process)
|
||||||
|
returncode = 1
|
||||||
|
|
||||||
finished = datetime.now(timezone.utc)
|
finished = datetime.now(timezone.utc)
|
||||||
result = {
|
result = {
|
||||||
@@ -211,8 +225,13 @@ def main() -> int:
|
|||||||
"peak_rss_kib": resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss,
|
"peak_rss_kib": resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss,
|
||||||
"exit_code": returncode,
|
"exit_code": returncode,
|
||||||
"status": "interrupted" if returncode == 130 else ("passed" if returncode == 0 else "failed"),
|
"status": "interrupted" if returncode == 130 else ("passed" if returncode == 0 else "failed"),
|
||||||
|
"parent_pid": os.getpid(),
|
||||||
"log": str(args.log.resolve()),
|
"log": str(args.log.resolve()),
|
||||||
}
|
}
|
||||||
|
if process is not None:
|
||||||
|
result["child_pid"] = process.pid
|
||||||
|
result["child_exit_code"] = process.returncode
|
||||||
|
result["child_cleanup_pending"] = process.returncode is None
|
||||||
if error_message:
|
if error_message:
|
||||||
result["error"] = error_message
|
result["error"] = error_message
|
||||||
if args.success_marker:
|
if args.success_marker:
|
||||||
|
|||||||
@@ -1969,6 +1969,8 @@ def test_run_flow_writes_structured_result(tmp_path: Path) -> None:
|
|||||||
assert data["status"] == "passed"
|
assert data["status"] == "passed"
|
||||||
assert data["exit_code"] == 0
|
assert data["exit_code"] == 0
|
||||||
assert data["duration_seconds"] >= 0
|
assert data["duration_seconds"] >= 0
|
||||||
|
assert isinstance(data["parent_pid"], int)
|
||||||
|
assert isinstance(data["child_pid"], int)
|
||||||
assert isinstance(data["peak_rss_kib"], int)
|
assert isinstance(data["peak_rss_kib"], int)
|
||||||
assert data["peak_rss_kib"] > 0
|
assert data["peak_rss_kib"] > 0
|
||||||
assert log.read_text(encoding="utf-8") == "flow output\n"
|
assert log.read_text(encoding="utf-8") == "flow output\n"
|
||||||
@@ -2120,6 +2122,41 @@ def test_run_flow_streams_bytes_without_waiting_for_newline(tmp_path: Path) -> N
|
|||||||
assert json.loads(result.read_text(encoding="utf-8"))["status"] == "passed"
|
assert json.loads(result.read_text(encoding="utf-8"))["status"] == "passed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_flow_persists_cancel_before_cleanup_and_ignores_second_signal(tmp_path: Path) -> None:
|
||||||
|
result = tmp_path / "cancelled.json"
|
||||||
|
process = subprocess.Popen([
|
||||||
|
sys.executable, str(ROOT / "scripts/run_flow.py"), "--tool", "unit",
|
||||||
|
"--log", str(tmp_path / "cancelled.log"), "--result", str(result), "--",
|
||||||
|
sys.executable, "-u", "-c",
|
||||||
|
"import signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); "
|
||||||
|
"print('READY',flush=True); time.sleep(30)",
|
||||||
|
], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
try:
|
||||||
|
assert process.stdout is not None
|
||||||
|
assert process.stdout.readline().strip() == "READY"
|
||||||
|
process.send_signal(signal.SIGINT)
|
||||||
|
for _ in range(100):
|
||||||
|
checkpoint = json.loads(result.read_text())
|
||||||
|
if checkpoint["status"] == "interrupted":
|
||||||
|
break
|
||||||
|
time.sleep(0.01)
|
||||||
|
assert checkpoint["status"] == "interrupted"
|
||||||
|
assert checkpoint["child_cleanup_pending"] is True
|
||||||
|
assert "finished_at" not in checkpoint
|
||||||
|
process.send_signal(signal.SIGTERM)
|
||||||
|
_, stderr = process.communicate(timeout=10)
|
||||||
|
assert process.returncode == 130, stderr
|
||||||
|
final = json.loads(result.read_text())
|
||||||
|
assert final["status"] == "interrupted"
|
||||||
|
assert final["child_cleanup_pending"] is False
|
||||||
|
assert final["child_exit_code"] == -signal.SIGKILL
|
||||||
|
assert final["finished_at"] >= final["started_at"]
|
||||||
|
finally:
|
||||||
|
if process.poll() is None:
|
||||||
|
process.kill()
|
||||||
|
process.wait(timeout=5)
|
||||||
|
|
||||||
|
|
||||||
def test_run_flow_records_interruption(tmp_path: Path) -> None:
|
def test_run_flow_records_interruption(tmp_path: Path) -> None:
|
||||||
log = tmp_path / "interrupted.log"
|
log = tmp_path / "interrupted.log"
|
||||||
result = tmp_path / "interrupted.json"
|
result = tmp_path / "interrupted.json"
|
||||||
@@ -2192,6 +2229,27 @@ def test_simulation_success_marker_and_failure_detection(tmp_path: Path) -> None
|
|||||||
assert json.loads(result.read_text(encoding="utf-8"))["status"] == "failed"
|
assert json.loads(result.read_text(encoding="utf-8"))["status"] == "failed"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("status,exit_code", [("running", None), ("failed", 1),
|
||||||
|
("passed", 124), ("invalid", None)])
|
||||||
|
def test_check_simulation_requires_terminal_flow(tmp_path: Path, status: str,
|
||||||
|
exit_code: int | None) -> None:
|
||||||
|
log = tmp_path / "sim.log"
|
||||||
|
result = tmp_path / "result-sim-check.json"
|
||||||
|
log.write_text("SIM_TEST_PASS code=0\n", encoding="utf-8")
|
||||||
|
(tmp_path / "result-sim.json").write_text(
|
||||||
|
json.dumps({"status": status, "exit_code": exit_code}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
checked = subprocess.run(
|
||||||
|
[sys.executable, str(ROOT / "scripts/check_simulation.py"), "--log", str(log),
|
||||||
|
"--result", str(result)], text=True, capture_output=True,
|
||||||
|
)
|
||||||
|
report = json.loads(result.read_text(encoding="utf-8"))
|
||||||
|
assert checked.returncode != 0
|
||||||
|
assert report["status"] == "failed"
|
||||||
|
assert report["flow_status"] == status
|
||||||
|
|
||||||
|
|
||||||
def test_warning_baseline_rejects_new_signature(tmp_path: Path) -> None:
|
def test_warning_baseline_rejects_new_signature(tmp_path: Path) -> None:
|
||||||
profile = "unit-profile"
|
profile = "unit-profile"
|
||||||
log = tmp_path / "variant/sim/verilator/verilating.log"
|
log = tmp_path / "variant/sim/verilator/verilating.log"
|
||||||
|
|||||||
Reference in new issue
Block a user