mirror of
https://github.com/retroSoC/retroSoC.git
synced 2026-11-04 22:32:36 +00:00
maksyuki
e6b9a9f43d
- Introduced for deterministic corpus framing, CRC checks, and regression evidence for NPU-P6. - Added to validate NPU-P6 reference execution. - Created a new RTL model for gate-level NPU transactions. - Updated to instantiate and streamline the testbench. - Enhanced with capability checks and added NPU-P6 reference contract tests. - Modified to include in the source list and refactor rejection logic. - Updated README.md to document new test scripts and their purposes.
295 lines
13 KiB
Python
295 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Validate and assemble fail-closed NPU-P6 V015..V018 evidence."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
REJECTED = ("FAILED", "FATAL", "assertion failed", "%Error", "SIM_TEST_FAIL", "SIM_TEST_TIMEOUT")
|
|
|
|
|
|
def load(path: Path) -> dict[str, Any]:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(data, dict):
|
|
raise ValueError(f"evidence is not an object: {path}")
|
|
return data
|
|
|
|
|
|
def identity(path: Path) -> dict[str, object]:
|
|
payload = path.read_bytes()
|
|
return {
|
|
"path": str(path.resolve()),
|
|
"bytes": len(payload),
|
|
"sha256": hashlib.sha256(payload).hexdigest(),
|
|
}
|
|
|
|
|
|
def require_pass(path: Path, *, phase: str | None = None) -> dict[str, Any]:
|
|
data = load(path)
|
|
status = data.get("verdict", data.get("status"))
|
|
if status not in ("PASS", "passed"):
|
|
raise ValueError(f"required evidence did not pass: {path}")
|
|
if phase is not None and data.get("phase") != phase:
|
|
raise ValueError(f"required evidence has wrong phase: {path}")
|
|
summary = data.get("summary", {})
|
|
if data.get("skipped", 0) or (isinstance(summary, dict) and summary.get("skipped", 0)):
|
|
raise ValueError(f"required evidence contains skipped cases: {path}")
|
|
return data
|
|
|
|
|
|
def validate_identity(record: dict[str, Any]) -> None:
|
|
path = Path(record.get("path", ""))
|
|
if not path.is_file():
|
|
raise ValueError(f"missing retained P6 artifact: {path}")
|
|
actual = identity(path)
|
|
if actual["bytes"] != record.get("bytes") or actual["sha256"] != record.get("sha256"):
|
|
raise ValueError(f"retained P6 artifact identity changed: {path}")
|
|
|
|
|
|
def validate_implementation(data: dict[str, Any], label: str) -> None:
|
|
implementation = data.get("implementation")
|
|
if not isinstance(implementation, dict) or not implementation:
|
|
raise ValueError(f"{label} evidence lacks implementation identities")
|
|
for name, record in implementation.items():
|
|
relative = Path(name)
|
|
if relative.is_absolute() or ".." in relative.parts:
|
|
raise ValueError(f"{label} implementation path is not repository-relative: {name}")
|
|
expected = (ROOT / relative).resolve()
|
|
if Path(record.get("path", "")).resolve() != expected:
|
|
raise ValueError(f"{label} implementation path changed: {name}")
|
|
validate_identity(record)
|
|
|
|
|
|
def validate_corpus(path: Path) -> dict[str, Any]:
|
|
data = require_pass(path, phase="NPU-P6")
|
|
workloads = data.get("workloads")
|
|
if data.get("schema") != 2 or not isinstance(workloads, dict) or set(workloads) != {"kws", "vww"}:
|
|
raise ValueError("P6 corpus report has the wrong schema or workload set")
|
|
for workload, record in workloads.items():
|
|
shards = record.get("shards") if isinstance(record, dict) else None
|
|
if (
|
|
record.get("case_count") != 1000
|
|
or record.get("cases_per_shard") != 100
|
|
or record.get("shard_count") != 10
|
|
or not isinstance(shards, list)
|
|
or len(shards) != 10
|
|
):
|
|
raise ValueError(f"{workload} corpus report is incomplete")
|
|
for shard_index, shard in enumerate(shards):
|
|
cases = shard.get("cases")
|
|
expected = list(range(shard_index * 100, (shard_index + 1) * 100))
|
|
if (
|
|
shard.get("index") != shard_index
|
|
or shard.get("first_case") != expected[0]
|
|
or shard.get("case_count") != 100
|
|
or not isinstance(cases, list)
|
|
or [case.get("index") for case in cases] != expected
|
|
):
|
|
raise ValueError(f"{workload} corpus shard {shard_index} changed")
|
|
validate_identity(shard["file"])
|
|
if any(
|
|
set(case) != {
|
|
"index",
|
|
"name",
|
|
"label",
|
|
"input_sha256",
|
|
"terminal_sha256",
|
|
"softmax_sha256",
|
|
}
|
|
for case in cases
|
|
):
|
|
raise ValueError(f"{workload} corpus shard {shard_index} case schema changed")
|
|
return data
|
|
|
|
|
|
def validate_performance(path: Path) -> dict[str, Any]:
|
|
data = require_pass(path, phase="NPU-P6")
|
|
if data.get("verification_ids") != ["NPU-V016"]:
|
|
raise ValueError("P6 performance verification IDs changed")
|
|
workloads = data.get("workloads")
|
|
if not isinstance(workloads, dict) or set(workloads) != {"kws", "vww"}:
|
|
raise ValueError("P6 performance report must contain KWS and VWW")
|
|
for name, record in workloads.items():
|
|
cases = record.get("cases") if isinstance(record, dict) else None
|
|
if not isinstance(cases, list) or len(cases) != 1000:
|
|
raise ValueError(f"{name} performance report must contain 1000 cases")
|
|
if [case.get("index") for case in cases] != list(range(1000)):
|
|
raise ValueError(f"{name} performance cases are incomplete or reordered")
|
|
for case in cases:
|
|
index = case.get("index")
|
|
expected_cache = "cold" if index % 100 == 0 else "steady"
|
|
expected_contention = "ga2d-copy-32x32768-rgb565" if index < 10 else "none"
|
|
if (
|
|
case.get("status") != "PASS"
|
|
or case.get("reference_cycles", 0) <= 0
|
|
or case.get("npu_cycles", 0) <= 0
|
|
or case.get("npu_cycles")
|
|
!= case.get("npu_wait_cycles", -1) + case.get("softmax_cycles", -1)
|
|
or case.get("reference_class") != case.get("npu_class")
|
|
or case.get("cache") != expected_cache
|
|
or case.get("contention") != expected_contention
|
|
):
|
|
raise ValueError(f"{name} performance case failed or has invalid cycles")
|
|
reference = sum(case["reference_cycles"] for case in cases)
|
|
npu = sum(case["npu_cycles"] for case in cases)
|
|
speedup = reference / npu
|
|
if abs(float(record.get("speedup", 0.0)) - speedup) > 1e-9 or speedup < 2.0:
|
|
raise ValueError(f"{name} does not satisfy the 2.0-times P6 target")
|
|
if record.get("skipped", 0) or record.get("mismatches", 0):
|
|
raise ValueError(f"{name} contains skipped or mismatching cases")
|
|
comparison = record.get("compiler_comparison")
|
|
metrics = comparison.get("metrics") if isinstance(comparison, dict) else None
|
|
counter_names = {
|
|
"active_cycles",
|
|
"dma_read_bytes",
|
|
"dma_write_bytes",
|
|
"useful_macs",
|
|
"pack_cycles",
|
|
"retired_descriptors",
|
|
}
|
|
if (
|
|
not isinstance(comparison, dict)
|
|
or comparison.get("material_threshold_percent") != 10
|
|
or not isinstance(comparison.get("cycle_model"), str)
|
|
or not comparison["cycle_model"]
|
|
or not isinstance(metrics, dict)
|
|
or set(metrics) != counter_names
|
|
):
|
|
raise ValueError(f"{name} compiler comparison is incomplete")
|
|
validate_identity(comparison["artifact"])
|
|
for counter_name in counter_names:
|
|
metric = metrics[counter_name]
|
|
expected_per_inference = metric.get("estimated_per_inference", 0)
|
|
expected_total = expected_per_inference * len(cases)
|
|
measured_total = sum(case[counter_name] for case in cases)
|
|
ratio = measured_total / expected_total if expected_total > 0 else 0.0
|
|
material = abs(ratio - 1.0) > 0.10
|
|
if (
|
|
not isinstance(expected_per_inference, int)
|
|
or expected_per_inference <= 0
|
|
or metric.get("expected_total") != expected_total
|
|
or metric.get("measured_total") != measured_total
|
|
or abs(float(metric.get("measured_mean", 0.0)) - (measured_total / len(cases)))
|
|
> 1e-9
|
|
or abs(float(metric.get("measured_to_estimated_ratio", 0.0)) - ratio) > 1e-9
|
|
or metric.get("material_difference") is not material
|
|
or not isinstance(metric.get("explanation"), str)
|
|
or not metric["explanation"]
|
|
):
|
|
raise ValueError(f"{name} compiler comparison changed for {counter_name}")
|
|
return data
|
|
|
|
|
|
def validate_log(path: Path, required: tuple[str, ...]) -> None:
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
if any(marker not in text for marker in required) or any(marker in text for marker in REJECTED):
|
|
raise ValueError(f"P6 log markers failed: {path}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--p0-report", type=Path, required=True)
|
|
parser.add_argument("--p5-report", type=Path, required=True)
|
|
parser.add_argument("--corpus-report", type=Path, required=True)
|
|
parser.add_argument("--performance-report", type=Path, required=True)
|
|
parser.add_argument("--formal-report", type=Path, required=True)
|
|
parser.add_argument("--netlist-report", type=Path, required=True)
|
|
parser.add_argument("--physical-report", type=Path, required=True)
|
|
parser.add_argument("--regression-report", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
revision = subprocess.check_output(["git", "-C", str(ROOT), "rev-parse", "HEAD"], text=True).strip()
|
|
p0 = require_pass(args.p0_report, phase="NPU-P0")
|
|
p5 = require_pass(args.p5_report, phase="NPU-P5")
|
|
corpus = validate_corpus(args.corpus_report)
|
|
performance = validate_performance(args.performance_report)
|
|
formal = require_pass(args.formal_report, phase="NPU-P6")
|
|
netlist = require_pass(args.netlist_report, phase="NPU-P6")
|
|
physical = require_pass(args.physical_report, phase="NPU-P6")
|
|
regression = require_pass(args.regression_report, phase="NPU-P6")
|
|
expected_ids = {
|
|
"formal": ["NPU-V015"],
|
|
"netlist": ["NPU-V017", "NPU-V018"],
|
|
"physical": ["NPU-V017"],
|
|
"regression": ["NPU-V018"],
|
|
}
|
|
for name, data in (
|
|
("formal", formal),
|
|
("netlist", netlist),
|
|
("physical", physical),
|
|
("regression", regression),
|
|
):
|
|
if data.get("verification_ids") != expected_ids[name]:
|
|
raise ValueError(f"{name} evidence has the wrong verification IDs")
|
|
if netlist.get("physical_reused") or physical.get("product_reused"):
|
|
raise ValueError("P6 aggregate report requires fresh physical and netlist flows")
|
|
for name, data in (
|
|
("p0", p0),
|
|
("p5", p5),
|
|
("corpus", corpus),
|
|
("performance", performance),
|
|
("formal", formal),
|
|
("netlist", netlist),
|
|
("physical", physical),
|
|
("regression", regression),
|
|
):
|
|
evidence_revision = data.get("git_revision", data.get("provenance", {}).get("git_revision"))
|
|
if evidence_revision != revision:
|
|
raise ValueError(f"{name} evidence revision {evidence_revision!r} != {revision}")
|
|
if name not in {"p0", "p5"}:
|
|
validate_implementation(data, name)
|
|
if corpus.get("summary") != {"cases": 2000, "skipped": 0}:
|
|
raise ValueError("P6 corpus report is incomplete")
|
|
report = {
|
|
"schema": 1,
|
|
"phase": "NPU-P6",
|
|
"verification_ids": ["NPU-V015", "NPU-V016", "NPU-V017", "NPU-V018"],
|
|
"git_revision": revision,
|
|
"evidence": {
|
|
name: identity(path)
|
|
for name, path in (
|
|
("p0", args.p0_report), ("p5", args.p5_report),
|
|
("corpus", args.corpus_report), ("performance", args.performance_report),
|
|
("formal", args.formal_report), ("netlist", args.netlist_report),
|
|
("physical", args.physical_report), ("regression", args.regression_report),
|
|
)
|
|
},
|
|
"performance": performance["workloads"],
|
|
"implementation": {
|
|
"scripts/npu_p6_report.py": identity(ROOT / "scripts/npu_p6_report.py")
|
|
},
|
|
"maturity": {
|
|
"functional": "qualified",
|
|
"performance": "qualified",
|
|
"physical_integration": "qualified",
|
|
"rtl_readiness": "prototype_metadata_verified",
|
|
"fpga": "NOT_RUN_OPTIONAL",
|
|
"silicon": "NOT_RUN",
|
|
},
|
|
"remaining_commercial_gaps": [
|
|
"post-layout extracted MMMC/PVT closure",
|
|
"qualified CDC/RDC signoff",
|
|
"DFT, scan, MBIST and SRAM repair",
|
|
"power and activity characterization",
|
|
"optional FPGA/board characterization",
|
|
"silicon characterization",
|
|
],
|
|
"summary": {"required_checks": 8, "passes": 8, "failures": 0, "skipped": 0},
|
|
"verdict": "PASS",
|
|
}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(f"NPU-P6 qualification: PASS ({args.output})")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|