feat: add byte-by-byte output streaming to run_flow script and update related tests

docs/engineering.md

@@ -98,10 +98,11 @@ capped at 16 and can be set with `JOBS=<n>` or `MAX_JOBS=<n>`.
 ## Result Policy
 
 Every EDA and simulation flow command is run through `scripts/run_flow.py`, which streams a log and writes a
-result JSON containing the command, timestamps, duration, exit code, and status. A simulation passes
-only when the command succeeds, its log contains the firmware startup marker, and no fatal/failure
-marker is present. Icarus regressions use the assembly self-test as `retrosoc_asm`, while the normal
-`retrosoc_fw` image remains available for firmware size tracking and Verilator regressions.
+result JSON containing the command, timestamps, duration, exit code, and status. The Verilator simulation
+path streams UART output to an interactive terminal byte-by-byte while preserving the raw output in `sim.log`.
+A simulation passes only when the command succeeds, its log contains the firmware startup marker, and no
+fatal/failure marker is present. Icarus regressions use the assembly self-test as `retrosoc_asm`, while the
+normal `retrosoc_fw` image remains available for firmware size tracking and Verilator regressions.
 
 ## Formal Protocol Proofs
 

rtl/mini/dv/verilator/csrc/main.cpp

@@ -1,4 +1,5 @@
 #include <csignal>
+#include <cstdio>
 #include <iostream>
 
 #include "verilated.h"
@@ -32,6 +33,10 @@ void parseArgs(int argc, char *argv[]) {
 
 extern void env_init();
 int main(int argc, char *argv[]) {
+    if (std::setvbuf(stdout, nullptr, _IONBF, 0) != 0) {
+        std::fputs("failed to disable Verilator stdout buffering\n", stderr);
+        return 1;
+    }
     env_init();
     atexit(release);
     parseArgs(argc, argv);

rtl/mini/mk/verilator.mk

@@ -94,7 +94,7 @@ comp: $(VERILATOR_EMU)
 sim: comp
 	@test -f $(SW_BUILD_DIR)/$(SIM_FIRMWARE_NAME).bin || { \
 		echo "firmware image missing; run 'make firmware' first" >&2; exit 1; }
-	python3 $(ROOT_PATH)/scripts/run_flow.py --tool verilator-sim \
+	python3 $(ROOT_PATH)/scripts/run_flow.py --tool verilator-sim --stream-bytes \
 		--log $(BUILD_DIR)/sim.log --result $(BUILD_DIR)/result-sim.json \
 		--cwd $(BUILD_DIR) -- $(BUILD_DIR)/emu -i $(SW_BUILD_DIR)/$(SIM_FIRMWARE_NAME).bin \
 		-t $(SOC_SIM_TIME)

scripts/run_flow.py

@@ -65,6 +65,13 @@ def write_console_output(line: str) -> None:
     sys.stdout.flush()
 
 
+def write_console_bytes(chunk: bytes) -> None:
+    if terminal_requires_crlf():
+        chunk = chunk.replace(b"\n", b"\r\n")
+    sys.stdout.buffer.write(chunk)
+    sys.stdout.buffer.flush()
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description="Run a flow command with structured results")
     parser.add_argument("--tool", required=True)
@@ -72,6 +79,11 @@ def main() -> int:
     parser.add_argument("--result", type=Path, required=True)
     parser.add_argument("--cwd", type=Path)
     parser.add_argument("--env", action="append", default=[])
+    parser.add_argument(
+        "--stream-bytes",
+        action="store_true",
+        help="stream child output byte-by-byte and preserve raw log bytes",
+    )
     parser.add_argument("command", nargs=argparse.REMAINDER)
     args = parser.parse_args()
     command = args.command[1:] if args.command[:1] == ["--"] else args.command
@@ -88,26 +100,36 @@ def main() -> int:
     start_clock = time.monotonic()
     returncode = 127
     error_message = None
-    process: subprocess.Popen[str] | None = None
+    process: subprocess.Popen[str] | subprocess.Popen[bytes] | None = None
     try:
-        with args.log.open("w", encoding="utf-8", errors="replace") as log:
+        if args.stream_bytes:
+            log_context = args.log.open("wb")
+        else:
+            log_context = args.log.open("w", encoding="utf-8", errors="replace")
+        with log_context as log:
             process = subprocess.Popen(
                 command,
                 cwd=args.cwd,
                 env=environment,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
-                text=True,
-                errors="replace",
-                bufsize=1,
+                text=not args.stream_bytes,
+                errors=None if args.stream_bytes else "replace",
+                bufsize=0 if args.stream_bytes else 1,
                 pass_fds=jobserver_fds(),
             )
             assert process.stdout is not None
-            for line in process.stdout:
-                log.write(line)
-                log.flush()
-                if should_display(args.tool, line):
-                    write_console_output(line)
+            if args.stream_bytes:
+                while chunk := os.read(process.stdout.fileno(), 4096):
+                    log.write(chunk)
+                    log.flush()
+                    write_console_bytes(chunk)
+            else:
+                for line in process.stdout:
+                    log.write(line)
+                    log.flush()
+                    if should_display(args.tool, line):
+                        write_console_output(line)
             returncode = process.wait()
     except KeyboardInterrupt:
         error_message = "interrupted"

tests/test_script_tools.py

@@ -845,6 +845,67 @@ def test_run_flow_keeps_newlines_when_terminal_maps_them_to_carriage_return(monk
     assert terminal.getvalue() == "first\nsecond\n"
 
 
+def test_run_flow_byte_output_adds_carriage_return_only_for_terminal(monkeypatch) -> None:
+    class Terminal(io.StringIO):
+        def __init__(self) -> None:
+            super().__init__()
+            self.buffer = io.BytesIO()
+
+        def isatty(self) -> bool:
+            return True
+
+        def fileno(self) -> int:
+            return 42
+
+    terminal = Terminal()
+    monkeypatch.setattr(run_flow.sys, "stdout", terminal)
+    monkeypatch.setattr(
+        run_flow.termios,
+        "tcgetattr",
+        lambda descriptor: [0, run_flow.termios.OPOST, 0, 0, 0, 0, 0],
+    )
+
+    run_flow.write_console_bytes(b"first\nsecond\n")
+
+    assert terminal.buffer.getvalue() == b"first\r\nsecond\r\n"
+
+
+def test_run_flow_streams_bytes_without_waiting_for_newline(tmp_path: Path) -> None:
+    log = tmp_path / "flow.log"
+    result = tmp_path / "result.json"
+    process = subprocess.Popen(
+        [
+            sys.executable,
+            str(ROOT / "scripts/run_flow.py"),
+            "--tool",
+            "unit",
+            "--stream-bytes",
+            "--log",
+            str(log),
+            "--result",
+            str(result),
+            "--",
+            sys.executable,
+            "-c",
+            "import os, time; os.write(1, b'A'); time.sleep(2); os.write(1, b'\\xffB')",
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    assert process.stdout is not None
+    started = time.monotonic()
+
+    assert process.stdout.read(1) == b"A"
+    assert time.monotonic() - started < 1
+
+    remaining, stderr = process.communicate(timeout=10)
+    assert process.returncode == 0
+    assert stderr == b""
+    assert remaining == b"\xffB"
+    assert log.read_bytes() == b"A\xffB"
+    assert json.loads(result.read_text(encoding="utf-8"))["status"] == "passed"
+
+
 def test_run_flow_records_interruption(tmp_path: Path) -> None:
     log = tmp_path / "interrupted.log"
     result = tmp_path / "interrupted.json"