mirror of
https://github.com/retroSoC/retroSoC.git
synced 2026-11-04 22:32:36 +00:00
Merge branch 'feat/apu' into dev
# Conflicts: # Makefile # publications/datasheets/ips/apu.typ # rtl/mini/mk/formal.mk # tests/rtl/axi4_target_guard_tb.sv
This commit is contained in:
76 files changed
+38580
-1741
No files matched your search
@@ -16,6 +16,11 @@ disabled below the selected build variant. These inputs never enter firmware,
|
||||
RTL, or shipped APU microcode.
|
||||
`qualify_apu_p5_corpus.py` enumerates that exact corpus, records source and
|
||||
decoded-PCM hashes, and classifies each file against the frozen P5 profile.
|
||||
`run_apu_p5_corpus_rtl.py` then runs every classified file through one shared,
|
||||
verification-only `apb4_apu` fixture compiled for Icarus and Verilator. It
|
||||
records exact production result/accounting and PCM hashes in the same manifest;
|
||||
per-case results are resumable and neither the fixture nor corpus enters a
|
||||
product filelist.
|
||||
|
||||
Scripts are part of the build contract. Prefer existing helpers over ad-hoc
|
||||
shell behavior, preserve structured JSON results, and keep setup/download
|
||||
|
||||
+105
-48
@@ -9,12 +9,23 @@ from apu_p5_coefficients import coefficient_words
|
||||
|
||||
|
||||
class CodecError(ValueError):
|
||||
def __init__(self, code: int, stage: int, reason: int, offset: int, warnings: int = 0):
|
||||
def __init__(
|
||||
self,
|
||||
code: int,
|
||||
stage: int,
|
||||
reason: int,
|
||||
offset: int,
|
||||
warnings: int = 0,
|
||||
*,
|
||||
input_used: int | None = None,
|
||||
):
|
||||
self.code = code
|
||||
self.stage = stage
|
||||
self.reason = reason
|
||||
self.offset = offset
|
||||
self.warnings = warnings
|
||||
self.input_used = offset if input_used is None else input_used
|
||||
self.partial_audio = None
|
||||
super().__init__(f"codec error {code}/{stage} reason 0x{reason:04x} at {offset}")
|
||||
|
||||
|
||||
@@ -45,6 +56,30 @@ class ProcessedPcm:
|
||||
payload: bytes
|
||||
|
||||
|
||||
def _attach_flac_prefix(
|
||||
error: CodecError,
|
||||
meta: _FlacMeta,
|
||||
frames: list[tuple[int, ...]],
|
||||
input_size: int,
|
||||
) -> CodecError:
|
||||
input_used = min(error.input_used, input_size)
|
||||
error.input_used = input_used
|
||||
error.partial_audio = DecodedAudio(
|
||||
AudioInfo(
|
||||
meta.rate,
|
||||
meta.channels,
|
||||
meta.bits,
|
||||
len(frames),
|
||||
meta.frame_offset,
|
||||
max(0, input_used - meta.frame_offset),
|
||||
input_used,
|
||||
error.warnings,
|
||||
),
|
||||
tuple(frames),
|
||||
)
|
||||
return error
|
||||
|
||||
|
||||
_RESAMPLE_RATIOS = (
|
||||
(1, 1),
|
||||
(1, 2),
|
||||
@@ -184,9 +219,7 @@ class _BitReader:
|
||||
bit_offset = self.bit & 7
|
||||
take = min(remaining, 8 - bit_offset)
|
||||
shift = 8 - bit_offset - take
|
||||
value = (value << take) | (
|
||||
(self.data[self.bit // 8] >> shift) & ((1 << take) - 1)
|
||||
)
|
||||
value = (value << take) | ((self.data[self.bit // 8] >> shift) & ((1 << take) - 1))
|
||||
self.bit += take
|
||||
remaining -= take
|
||||
return value
|
||||
@@ -367,8 +400,10 @@ def decode_wav(data: bytes, *, strict: bool = True) -> DecodedAudio:
|
||||
|
||||
|
||||
def inspect_flac(data: bytes, *, strict: bool = True) -> AudioInfo:
|
||||
if len(data) < 8 or data[:4] != b"fLaC":
|
||||
raise CodecError(4, 4, 0x10, 0)
|
||||
if len(data) < 4:
|
||||
raise CodecError(5, 4, 0x20, len(data), 1)
|
||||
if data[:4] != b"fLaC":
|
||||
raise CodecError(4, 4, 0x10, 0, input_used=4)
|
||||
offset = 4
|
||||
blocks = 0
|
||||
metadata_bytes = 0
|
||||
@@ -383,15 +418,26 @@ def inspect_flac(data: bytes, *, strict: bool = True) -> AudioInfo:
|
||||
size = int.from_bytes(data[offset + 1 : offset + 4], "big")
|
||||
payload = offset + 4
|
||||
end = payload + size
|
||||
if end > len(data):
|
||||
raise CodecError(5, 4, 0x20, len(data), 1)
|
||||
blocks += 1
|
||||
if blocks == 1:
|
||||
if block_type != 0:
|
||||
raise CodecError(4, 4, 0x15, offset, 1, input_used=payload)
|
||||
if size != 34:
|
||||
# Only the metadata header has been consumed when the first
|
||||
# STREAMINFO size is invalid; do not report bytes beyond the
|
||||
# supplied input or imply that the payload was inspected.
|
||||
raise CodecError(4, 4, 0x15, offset + 1, 1, input_used=payload)
|
||||
elif block_type == 0 or block_type == 127:
|
||||
raise CodecError(4, 4, 0x16, offset, 1, input_used=payload)
|
||||
elif block_type >= 7:
|
||||
raise CodecError(3, 4, 0x03, offset, 1, input_used=payload)
|
||||
|
||||
metadata_bytes += size
|
||||
if blocks > 128 or metadata_bytes > 1024 * 1024:
|
||||
raise CodecError(3, 4, 0x04, offset, 1)
|
||||
raise CodecError(3, 4, 0x04, offset + 1, 1, input_used=payload)
|
||||
if end > len(data):
|
||||
raise CodecError(5, 4, 0x20, len(data), 1, input_used=payload)
|
||||
if blocks == 1:
|
||||
if block_type != 0 or size != 34:
|
||||
raise CodecError(4, 4, 0x15, offset, 1)
|
||||
minimum = int.from_bytes(data[payload : payload + 2], "big")
|
||||
maximum = int.from_bytes(data[payload + 2 : payload + 4], "big")
|
||||
min_frame = int.from_bytes(data[payload + 4 : payload + 7], "big")
|
||||
@@ -401,23 +447,19 @@ def inspect_flac(data: bytes, *, strict: bool = True) -> AudioInfo:
|
||||
channels = ((packed >> 41) & 7) + 1
|
||||
bits = ((packed >> 36) & 0x1F) + 1
|
||||
samples = packed & ((1 << 36) - 1)
|
||||
if (
|
||||
minimum < 16
|
||||
or maximum < minimum
|
||||
or maximum > (4096 if channels == 1 else 2048)
|
||||
or (min_frame and max_frame and min_frame > max_frame)
|
||||
or max_frame > 65536
|
||||
or channels not in (1, 2)
|
||||
or bits not in (16, 24)
|
||||
or rate < 8000
|
||||
or rate > 96000
|
||||
):
|
||||
raise CodecError(3, 4, 0x04, payload, 1)
|
||||
if minimum < 16:
|
||||
raise CodecError(3, 4, 0x04, payload, 1, input_used=end)
|
||||
if maximum < minimum:
|
||||
raise CodecError(4, 4, 0x15, payload + 2, 1, input_used=end)
|
||||
if channels in (1, 2) and maximum > (4096 if channels == 1 else 2048):
|
||||
raise CodecError(3, 4, 0x04, payload + 2, 1, input_used=end)
|
||||
if min_frame and max_frame and min_frame > max_frame:
|
||||
raise CodecError(4, 4, 0x15, payload + 7, 1, input_used=end)
|
||||
if max_frame > 65536:
|
||||
raise CodecError(3, 4, 0x04, payload + 7, 1, input_used=end)
|
||||
if channels not in (1, 2) or bits not in (16, 24) or not 8000 <= rate <= 96000:
|
||||
raise CodecError(3, 4, 0x02, payload + 10, 1, input_used=end)
|
||||
stream_info = AudioInfo(rate, channels, bits, samples or None, 0, 0, 0, 1)
|
||||
elif block_type == 0 or block_type == 127:
|
||||
raise CodecError(4, 4, 0x16, offset, 1)
|
||||
elif block_type >= 7:
|
||||
raise CodecError(3, 4, 0x03, offset, 1)
|
||||
offset = end
|
||||
if stream_info is None:
|
||||
raise CodecError(4, 4, 0x15, 4, 1)
|
||||
@@ -621,7 +663,7 @@ def _subframe(reader: _BitReader, block_size: int, bits: int) -> list[int]:
|
||||
|
||||
|
||||
def _decode_frame(
|
||||
data: bytes, offset: int, meta: _FlacMeta
|
||||
data: bytes, offset: int, meta: _FlacMeta, expected_number: int | None = None
|
||||
) -> tuple[list[tuple[int, ...]], int, int, int, int]:
|
||||
start = offset
|
||||
reader = _BitReader(data, offset)
|
||||
@@ -639,19 +681,23 @@ def _decode_frame(
|
||||
rate = _sample_rate(rate_code, reader, meta.rate)
|
||||
bits = _sample_bits(bits_code, meta.bits, start)
|
||||
header_end = reader.byte_offset
|
||||
expected_crc8 = reader.get(8, stage=4)
|
||||
if crc8(data[start:header_end]) != expected_crc8:
|
||||
raise CodecError(6, 4, 0x30, header_end, 1)
|
||||
if assignment <= 7:
|
||||
channels = assignment + 1
|
||||
elif assignment <= 10:
|
||||
channels = 2
|
||||
else:
|
||||
raise CodecError(4, 4, 0x17, start, 1)
|
||||
raise CodecError(4, 4, 0x17, start, 1, input_used=header_end)
|
||||
if (rate, channels, bits) != (meta.rate, meta.channels, meta.bits):
|
||||
raise CodecError(4, 4, 0x18, start, 1)
|
||||
raise CodecError(4, 4, 0x18, start, 1, input_used=header_end)
|
||||
if block_size > (4096 if channels == 1 else 2048):
|
||||
raise CodecError(3, 4, 0x04, start, 1)
|
||||
raise CodecError(3, 4, 0x04, start, 1, input_used=header_end)
|
||||
if expected_number is not None and number != expected_number:
|
||||
# Frame numbering is checked before either CRC. This preserves the
|
||||
# frozen first-offender ordering for a bad number plus bad CRC.
|
||||
raise CodecError(4, 4, 0x1A, start, 1, input_used=header_end)
|
||||
expected_crc8 = reader.get(8, stage=4)
|
||||
if crc8(data[start:header_end]) != expected_crc8:
|
||||
raise CodecError(6, 4, 0x30, header_end, 1, input_used=reader.byte_offset)
|
||||
|
||||
channel_bits = [bits] * channels
|
||||
if assignment == 8:
|
||||
@@ -701,31 +747,42 @@ def decode_flac(data: bytes, *, strict: bool = True) -> DecodedAudio:
|
||||
strategy: int | None = None
|
||||
frame_number = 0
|
||||
while offset < len(data) and (meta.total_samples is None or len(frames) < meta.total_samples):
|
||||
decoded, frame_end, block_size, frame_strategy, number = _decode_frame(data, offset, meta)
|
||||
try:
|
||||
expected_number = frame_number if strategy in (None, 0) else len(frames)
|
||||
decoded, frame_end, block_size, frame_strategy, number = _decode_frame(
|
||||
data, offset, meta, expected_number
|
||||
)
|
||||
except CodecError as error:
|
||||
raise _attach_flac_prefix(error, meta, frames, len(data)) from error
|
||||
if strategy is None:
|
||||
strategy = frame_strategy
|
||||
elif strategy != frame_strategy:
|
||||
raise CodecError(4, 4, 0x18, offset, 1)
|
||||
expected_number = frame_number if frame_strategy == 0 else len(frames)
|
||||
if number != expected_number:
|
||||
raise CodecError(4, 4, 0x1A, offset, 1)
|
||||
error = CodecError(4, 4, 0x18, offset, 1, input_used=frame_end)
|
||||
raise _attach_flac_prefix(error, meta, frames, len(data))
|
||||
if meta.total_samples is not None and len(frames) + len(decoded) > meta.total_samples:
|
||||
error = CodecError(4, 4, 0x32, offset, 1, input_used=frame_end)
|
||||
raise _attach_flac_prefix(error, meta, frames, len(data))
|
||||
final_frame = (
|
||||
meta.total_samples is not None and len(frames) + block_size == meta.total_samples
|
||||
) or (meta.total_samples is None and frame_end == len(data))
|
||||
if not final_frame and not meta.minimum_block <= block_size <= meta.maximum_block:
|
||||
error = CodecError(3, 4, 0x04, offset, 1, input_used=frame_end)
|
||||
raise _attach_flac_prefix(error, meta, frames, len(data))
|
||||
if final_frame and block_size > meta.maximum_block:
|
||||
error = CodecError(3, 4, 0x04, offset, 1, input_used=frame_end)
|
||||
raise _attach_flac_prefix(error, meta, frames, len(data))
|
||||
frames.extend(decoded)
|
||||
blocks.append(block_size)
|
||||
frame_number += 1
|
||||
offset = frame_end
|
||||
if meta.total_samples is not None and len(frames) > meta.total_samples:
|
||||
raise CodecError(4, 4, 0x32, offset, 1)
|
||||
if meta.total_samples is not None and len(frames) != meta.total_samples:
|
||||
raise CodecError(4, 4, 0x32, offset, 1)
|
||||
for block_size in blocks[:-1]:
|
||||
if not meta.minimum_block <= block_size <= meta.maximum_block:
|
||||
raise CodecError(4, 4, 0x17, offset, 1)
|
||||
if blocks and blocks[-1] > meta.maximum_block:
|
||||
raise CodecError(3, 4, 0x04, offset, 1)
|
||||
error = CodecError(4, 4, 0x32, offset, 1)
|
||||
raise _attach_flac_prefix(error, meta, frames, len(data))
|
||||
warnings = 1
|
||||
if offset != len(data):
|
||||
if strict:
|
||||
raise CodecError(4, 4, 0x1B, offset, warnings)
|
||||
error = CodecError(4, 4, 0x1B, offset, warnings)
|
||||
raise _attach_flac_prefix(error, meta, frames, len(data))
|
||||
warnings |= 1 << 1
|
||||
info = AudioInfo(
|
||||
meta.rate,
|
||||
|
||||
+29
-15
@@ -8,6 +8,7 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from apu_isa import (
|
||||
APUMC_ABI_V1,
|
||||
APUMC_TARGETS,
|
||||
BitstreamOpcode,
|
||||
ControlOpcode,
|
||||
@@ -71,6 +72,7 @@ class TransportModel:
|
||||
events: int = 0
|
||||
pending: list[TransportCommand] = field(default_factory=list)
|
||||
completed: list[tuple[int, int]] = field(default_factory=list)
|
||||
output_commands: list[dict[str, int]] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def idle(self) -> bool:
|
||||
@@ -88,6 +90,7 @@ class TransportModel:
|
||||
self.events = 0
|
||||
self.pending.clear()
|
||||
self.completed.clear()
|
||||
self.output_commands.clear()
|
||||
|
||||
def abort(self) -> None:
|
||||
self.pending.clear()
|
||||
@@ -146,6 +149,7 @@ class TransportModel:
|
||||
raise PrimitiveFault(8, 8, 9)
|
||||
address = entry.scratch_base + offset
|
||||
payload = primitives.memory.read(address, count, entry)
|
||||
self.output_commands.append({"opcode": int(opcode), "bytes": count})
|
||||
self.pending.append(
|
||||
TransportCommand(
|
||||
opcode, instruction.dst, offset, count, payload, max(1, self.latency)
|
||||
@@ -200,6 +204,12 @@ class TransportModel:
|
||||
self.input_cursor += command.count
|
||||
else:
|
||||
self.output.extend(command.payload)
|
||||
context_output = entry.scratch_base + entry.scratch_bytes - 64 + 44
|
||||
primitives.memory.write(
|
||||
context_output,
|
||||
len(self.output).to_bytes(4, "little"),
|
||||
entry,
|
||||
)
|
||||
self.completed.append((command.destination, command.count))
|
||||
self.pending.remove(command)
|
||||
|
||||
@@ -233,6 +243,7 @@ class Machine:
|
||||
kernel_cycles_remaining: int = field(default=0, init=False)
|
||||
transport_pending_dst: set[int] = field(default_factory=set, init=False)
|
||||
transport_fault: PrimitiveFault | None = field(default=None, init=False)
|
||||
mc_abi: int = APUMC_ABI_V1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.begin_entry(self.entry)
|
||||
@@ -432,14 +443,15 @@ class Machine:
|
||||
self.transport_fault = None
|
||||
raise SequencerTrap(fault.reason, self.pc, instruction)
|
||||
try:
|
||||
validate_instruction(instruction, self.target)
|
||||
validate_instruction(instruction, self.target, self.mc_abi)
|
||||
except ValueError as error:
|
||||
raise SequencerTrap(1, self.pc, instruction) from error
|
||||
if self.retired >= self.entry.max_retired:
|
||||
raise SequencerTrap(8, self.pc, instruction)
|
||||
predicate_true = self._predicate(instruction.predicate)
|
||||
if not predicate_true:
|
||||
if self.pc == self.entry.last_pc or self.pc == 0x7FF:
|
||||
store_last = 0x7FF if self.mc_abi == APUMC_ABI_V1 else 0xFFF
|
||||
if self.pc == self.entry.last_pc or self.pc == store_last:
|
||||
raise SequencerTrap(2, self.pc, instruction)
|
||||
self._retire(instruction, predicate_true)
|
||||
self.pc += 1
|
||||
@@ -582,19 +594,19 @@ class Machine:
|
||||
return False
|
||||
if instruction.instruction_class == InstructionClass.BITSTREAM:
|
||||
opcode = BitstreamOpcode(instruction.opcode)
|
||||
required = (
|
||||
instruction.immediate & 0x3F
|
||||
if opcode
|
||||
in (
|
||||
BitstreamOpcode.REFILL,
|
||||
BitstreamOpcode.PEEK,
|
||||
BitstreamOpcode.GET,
|
||||
BitstreamOpcode.SKIP,
|
||||
)
|
||||
else 1
|
||||
)
|
||||
self.primitives._refill(required)
|
||||
if len(self.primitives.bits) < required and not self.primitives.eof:
|
||||
required = 0
|
||||
if opcode in (
|
||||
BitstreamOpcode.REFILL,
|
||||
BitstreamOpcode.PEEK,
|
||||
BitstreamOpcode.GET,
|
||||
BitstreamOpcode.SKIP,
|
||||
):
|
||||
required = instruction.immediate & 0x3F
|
||||
elif opcode == BitstreamOpcode.FRAME_SYNC:
|
||||
required = instruction.aux & 0x1F
|
||||
if required != 0:
|
||||
self.primitives._refill(required)
|
||||
if required != 0 and len(self.primitives.bits) < required and not self.primitives.eof:
|
||||
self._no_retirement_cycle(instruction, tick_kernel=False, tick_transport=False)
|
||||
return False
|
||||
if instruction.instruction_class == InstructionClass.ENTROPY:
|
||||
@@ -679,6 +691,7 @@ class Machine:
|
||||
"result_stage": self.transport.result_stage,
|
||||
"result_detail": self.transport.result_detail,
|
||||
"events": self.transport.events,
|
||||
"output_commands": self.transport.output_commands,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -713,6 +726,7 @@ def main() -> int:
|
||||
target=args.target,
|
||||
primitives=PrimitiveBam() if args.target in ("p4", "p5") else None,
|
||||
transport=TransportModel() if args.target == "p5" else None,
|
||||
mc_abi=header[1],
|
||||
).run(),
|
||||
sort_keys=True,
|
||||
)
|
||||
|
||||
+72
-37
@@ -1,4 +1,4 @@
|
||||
"""Frozen APU microcode V1 encoding and APUMC bundle helpers."""
|
||||
"""Frozen APU microcode V1/V2 encoding and APUMC bundle helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -10,12 +10,16 @@ from typing import Iterable
|
||||
|
||||
|
||||
APUMC_MAGIC = 0x41504D43
|
||||
APUMC_ABI = 0x00010000
|
||||
APUMC_ABI_V1 = 0x00010000
|
||||
APUMC_ABI_V2 = 0x00020000
|
||||
APUMC_ABI = APUMC_ABI_V1
|
||||
APUMC_HEADER_BYTES = 64
|
||||
APUMC_ENTRY_BYTES = 32
|
||||
APUMC_ENTRY_COUNT = 3
|
||||
APUMC_INSTRUCTION_OFFSET = 192
|
||||
APUMC_MAX_INSTRUCTIONS = 2048
|
||||
APUMC_MAX_INSTRUCTIONS_V1 = 2048
|
||||
APUMC_MAX_INSTRUCTIONS_V2 = 4096
|
||||
APUMC_MAX_INSTRUCTIONS = APUMC_MAX_INSTRUCTIONS_V1
|
||||
APUMC_P3_CAPABILITY_MASK = 0
|
||||
APUMC_P4_CAPABILITY_MASK = 0x0000FFFF
|
||||
APUMC_P5_CAPABILITY_MASK = 0x001FFFFF
|
||||
@@ -205,10 +209,14 @@ def abi_manifest() -> dict[str, object]:
|
||||
"apumc": {
|
||||
"magic": APUMC_MAGIC,
|
||||
"abi": APUMC_ABI,
|
||||
"abi_v1": APUMC_ABI_V1,
|
||||
"abi_v2": APUMC_ABI_V2,
|
||||
"header_bytes": APUMC_HEADER_BYTES,
|
||||
"entry_bytes": APUMC_ENTRY_BYTES,
|
||||
"entry_count": APUMC_ENTRY_COUNT,
|
||||
"max_instructions": APUMC_MAX_INSTRUCTIONS,
|
||||
"max_instructions_v1": APUMC_MAX_INSTRUCTIONS_V1,
|
||||
"max_instructions_v2": APUMC_MAX_INSTRUCTIONS_V2,
|
||||
},
|
||||
"classes": {item.name.lower(): int(item) for item in InstructionClass},
|
||||
"control_opcodes": CONTROL_OPCODES,
|
||||
@@ -308,10 +316,16 @@ class Entry:
|
||||
table_offset: int = 0
|
||||
table_bytes: int = 0
|
||||
|
||||
def words(self) -> tuple[int, ...]:
|
||||
def words(self, abi: int = APUMC_ABI_V1) -> tuple[int, ...]:
|
||||
if abi == APUMC_ABI_V2:
|
||||
entry_mask = 0xFFFF
|
||||
range_mask = 0xFFF
|
||||
else:
|
||||
entry_mask = 0x7FF
|
||||
range_mask = 0x7FF
|
||||
return (
|
||||
self.format_id | (self.entry_pc << 4),
|
||||
self.first_pc | (self.last_pc << 16),
|
||||
self.format_id | ((self.entry_pc & entry_mask) << 4),
|
||||
(self.first_pc & range_mask) | ((self.last_pc & range_mask) << 16),
|
||||
self.scratch_base,
|
||||
self.scratch_bytes,
|
||||
self.max_loop_count,
|
||||
@@ -384,8 +398,8 @@ def instruction_required_mask(instruction: Instruction) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def validate_instruction(instruction: Instruction, target: str = "p3") -> None:
|
||||
"""Validate one frozen V1 instruction for the selected implementation target."""
|
||||
def validate_instruction(instruction: Instruction, target: str = "p3", abi: int = APUMC_ABI_V1) -> None:
|
||||
"""Validate one frozen instruction for the selected target and image ABI."""
|
||||
|
||||
target_mask = _target_mask(target)
|
||||
if instruction.predicate > 11:
|
||||
@@ -400,8 +414,9 @@ def validate_instruction(instruction: Instruction, target: str = "p3") -> None:
|
||||
_require_zero(instruction, ("dst", "src0", "src1", "aux"))
|
||||
elif opcode in (ControlOpcode.JUMP_FWD, ControlOpcode.CALL_FWD):
|
||||
_require_zero(instruction, ("dst", "src0", "src1", "aux"))
|
||||
if not 1 <= instruction.immediate <= 0x7FF:
|
||||
raise ValueError("forward branch delta must be 1..2047")
|
||||
branch_limit = 0xFFF if abi == APUMC_ABI_V2 else 0x7FF
|
||||
if not 1 <= instruction.immediate <= branch_limit:
|
||||
raise ValueError("forward branch delta is outside the image PC width")
|
||||
elif opcode == ControlOpcode.RET:
|
||||
_require_zero(instruction, ("dst", "src0", "src1", "aux", "immediate"))
|
||||
if instruction.predicate != 0:
|
||||
@@ -415,7 +430,7 @@ def validate_instruction(instruction: Instruction, target: str = "p3") -> None:
|
||||
if (
|
||||
instruction.predicate != 0
|
||||
or instruction.aux > 3
|
||||
or not 1 <= instruction.immediate <= 0x7FF
|
||||
or not 1 <= instruction.immediate <= (0xFFF if abi == APUMC_ABI_V2 else 0x7FF)
|
||||
):
|
||||
raise ValueError("LOOP_BACK encoding is invalid")
|
||||
else:
|
||||
@@ -581,10 +596,11 @@ def validate_p5_instruction(instruction: Instruction) -> None:
|
||||
|
||||
|
||||
def validate_entry(entry: Entry, instruction_count: int, target: str = "p3",
|
||||
table_payload_bytes: int = 0) -> None:
|
||||
table_payload_bytes: int = 0, abi: int = APUMC_ABI_V1) -> None:
|
||||
if entry.format_id not in range(APUMC_ENTRY_COUNT):
|
||||
raise ValueError("entry format must be 0, 1, or 2")
|
||||
if not (0 <= entry.first_pc <= entry.entry_pc <= entry.last_pc < instruction_count):
|
||||
pc_limit = APUMC_MAX_INSTRUCTIONS_V2 if abi == APUMC_ABI_V2 else APUMC_MAX_INSTRUCTIONS_V1
|
||||
if not (0 <= entry.first_pc <= entry.entry_pc <= entry.last_pc < instruction_count <= pc_limit):
|
||||
raise ValueError("entry PC range is invalid")
|
||||
if not 1 <= entry.max_loop_count <= 0xFFFF:
|
||||
raise ValueError("maximum loop count must be 1..65535")
|
||||
@@ -617,9 +633,10 @@ def validate_entry(entry: Entry, instruction_count: int, target: str = "p3",
|
||||
|
||||
|
||||
def control_flow_report(
|
||||
instructions: list[Instruction], entry: Entry, target: str = "p3"
|
||||
instructions: list[Instruction], entry: Entry, target: str = "p3",
|
||||
abi: int = APUMC_ABI_V1
|
||||
) -> dict[str, object]:
|
||||
"""Prove bounded V1 control flow and return its deterministic report."""
|
||||
"""Prove bounded control flow for the selected image ABI."""
|
||||
|
||||
loop_pairs: dict[int, int] = {}
|
||||
active_setup: dict[int, int] = {}
|
||||
@@ -728,7 +745,8 @@ def control_flow_report(
|
||||
else:
|
||||
raise ValueError(f"WAIT requires a primitive unavailable in {target.upper()}")
|
||||
pending.extend(successors)
|
||||
if len(visited) > APUMC_MAX_INSTRUCTIONS * 64:
|
||||
proof_limit = (APUMC_MAX_INSTRUCTIONS_V2 if abi == APUMC_ABI_V2 else APUMC_MAX_INSTRUCTIONS_V1) * 64
|
||||
if len(visited) > proof_limit:
|
||||
raise ValueError(f"entry {entry.format_id} control-flow proof exceeded its bound")
|
||||
if not terminals:
|
||||
raise ValueError(f"entry {entry.format_id} has no reachable END or TRAP")
|
||||
@@ -755,11 +773,12 @@ def control_flow_report(
|
||||
|
||||
|
||||
def validate_control_flow(
|
||||
instructions: list[Instruction], entry: Entry, target: str = "p3"
|
||||
instructions: list[Instruction], entry: Entry, target: str = "p3",
|
||||
abi: int = APUMC_ABI_V1
|
||||
) -> None:
|
||||
"""Prove bounded V1 control flow for one entry descriptor."""
|
||||
"""Prove bounded control flow for one entry descriptor."""
|
||||
|
||||
control_flow_report(instructions, entry, target)
|
||||
control_flow_report(instructions, entry, target, abi)
|
||||
|
||||
|
||||
def build_apumc(
|
||||
@@ -769,10 +788,18 @@ def build_apumc(
|
||||
*,
|
||||
target: str = "p3",
|
||||
table_payload: bytes = b"",
|
||||
abi: int | None = None,
|
||||
) -> bytes:
|
||||
if abi is None:
|
||||
abi = APUMC_ABI_V2 if target == "p5" else APUMC_ABI_V1
|
||||
if abi not in (APUMC_ABI_V1, APUMC_ABI_V2):
|
||||
raise ValueError("unsupported APUMC ABI")
|
||||
if abi == APUMC_ABI_V2 and target != "p5":
|
||||
raise ValueError("APUMC V2 is available only for P5")
|
||||
target_mask = _target_mask(target)
|
||||
if not 1 <= len(instructions) <= APUMC_MAX_INSTRUCTIONS:
|
||||
raise ValueError("instruction count must be 1..2048")
|
||||
instruction_limit = APUMC_MAX_INSTRUCTIONS_V2 if abi == APUMC_ABI_V2 else APUMC_MAX_INSTRUCTIONS_V1
|
||||
if not 1 <= len(instructions) <= instruction_limit:
|
||||
raise ValueError("instruction count is outside the selected APUMC ABI")
|
||||
if len(table_payload) & 3 or len(table_payload) > APUMC_P4_TABLE_SCRATCH_BYTES:
|
||||
raise ValueError("table payload must be four-byte aligned and at most 24 KiB")
|
||||
if target == "p3" and table_payload:
|
||||
@@ -782,11 +809,11 @@ def build_apumc(
|
||||
for index, entry in enumerate(entries):
|
||||
if entry.format_id != index:
|
||||
raise ValueError("entry descriptor order must be WAV, MP3, FLAC")
|
||||
validate_entry(entry, len(instructions), target, len(table_payload))
|
||||
validate_entry(entry, len(instructions), target, len(table_payload), abi)
|
||||
for instruction in instructions:
|
||||
validate_instruction(instruction, target)
|
||||
validate_instruction(instruction, target, abi)
|
||||
for entry in entries:
|
||||
validate_control_flow(instructions, entry, target)
|
||||
validate_control_flow(instructions, entry, target, abi)
|
||||
for pc in range(entry.first_pc, entry.last_pc + 1):
|
||||
required = instruction_required_mask(instructions[pc])
|
||||
if instructions[pc].instruction_class == InstructionClass.CONTROL and required:
|
||||
@@ -798,7 +825,7 @@ def build_apumc(
|
||||
elif required & ~entry.primitive_mask:
|
||||
raise ValueError(f"entry {entry.format_id} omits an instruction primitive")
|
||||
|
||||
descriptor = b"".join(struct.pack("<8I", *entry.words()) for entry in entries)
|
||||
descriptor = b"".join(struct.pack("<8I", *entry.words(abi)) for entry in entries)
|
||||
padding = bytes(APUMC_INSTRUCTION_OFFSET - APUMC_HEADER_BYTES - len(descriptor))
|
||||
encoded = b"".join(struct.pack("<Q", instruction.encode()) for instruction in instructions)
|
||||
table_offset = APUMC_INSTRUCTION_OFFSET + len(encoded) if table_payload else 0
|
||||
@@ -815,7 +842,7 @@ def build_apumc(
|
||||
header = struct.pack(
|
||||
"<16I",
|
||||
APUMC_MAGIC,
|
||||
APUMC_ABI,
|
||||
abi,
|
||||
total_bytes,
|
||||
len(instructions),
|
||||
APUMC_INSTRUCTION_OFFSET,
|
||||
@@ -835,15 +862,22 @@ def build_apumc(
|
||||
|
||||
|
||||
def parse_apumc(
|
||||
bundle: bytes, target: str = "p3"
|
||||
bundle: bytes, target: str = "p3", abi: int | None = None
|
||||
) -> tuple[list[Instruction], list[Entry], tuple[int, ...]]:
|
||||
target_mask = _target_mask(target)
|
||||
if len(bundle) < APUMC_HEADER_BYTES:
|
||||
raise ValueError("bundle is smaller than the APUMC header")
|
||||
header = struct.unpack_from("<16I", bundle)
|
||||
image_abi = header[1]
|
||||
if abi is not None and image_abi != abi:
|
||||
raise ValueError("APUMC ABI does not match requested version")
|
||||
if target == "p5":
|
||||
if image_abi not in (APUMC_ABI_V1, APUMC_ABI_V2):
|
||||
raise ValueError("APUMC header is invalid")
|
||||
elif image_abi != APUMC_ABI_V1:
|
||||
raise ValueError("APUMC header is invalid")
|
||||
if (
|
||||
header[0] != APUMC_MAGIC
|
||||
or header[1] != APUMC_ABI
|
||||
or header[2] != len(bundle)
|
||||
or len(bundle) < 160
|
||||
or header[14] != 0
|
||||
@@ -859,7 +893,8 @@ def parse_apumc(
|
||||
entry_offset = header[7]
|
||||
instruction_end = instruction_offset + instruction_count * 8
|
||||
entry_end = entry_offset + APUMC_ENTRY_COUNT * APUMC_ENTRY_BYTES
|
||||
if not 1 <= instruction_count <= APUMC_MAX_INSTRUCTIONS:
|
||||
instruction_limit = APUMC_MAX_INSTRUCTIONS_V2 if image_abi == APUMC_ABI_V2 else APUMC_MAX_INSTRUCTIONS_V1
|
||||
if not 1 <= instruction_count <= instruction_limit:
|
||||
raise ValueError("APUMC instruction count is invalid")
|
||||
common_range_error = (
|
||||
instruction_offset < APUMC_HEADER_BYTES
|
||||
@@ -895,8 +930,8 @@ def parse_apumc(
|
||||
for index in range(APUMC_ENTRY_COUNT):
|
||||
words = struct.unpack_from("<8I", bundle, entry_offset + index * APUMC_ENTRY_BYTES)
|
||||
reserved_error = (
|
||||
words[0] >> 15
|
||||
or (words[1] & 0xF800F800)
|
||||
(words[0] >> (16 if image_abi == APUMC_ABI_V2 else 15))
|
||||
or (words[1] & (0xF000F000 if image_abi == APUMC_ABI_V2 else 0xF800F800))
|
||||
or words[4] >> 16
|
||||
or words[5] >> 24
|
||||
or words[2] >> 17
|
||||
@@ -907,9 +942,9 @@ def parse_apumc(
|
||||
entries.append(
|
||||
Entry(
|
||||
format_id=words[0] & 0xF,
|
||||
entry_pc=(words[0] >> 4) & 0x7FF,
|
||||
first_pc=words[1] & 0x7FF,
|
||||
last_pc=(words[1] >> 16) & 0x7FF,
|
||||
entry_pc=(words[0] >> 4) & (0xFFF if image_abi == APUMC_ABI_V2 else 0x7FF),
|
||||
first_pc=words[1] & (0xFFF if image_abi == APUMC_ABI_V2 else 0x7FF),
|
||||
last_pc=(words[1] >> 16) & (0xFFF if image_abi == APUMC_ABI_V2 else 0x7FF),
|
||||
scratch_base=words[2] & 0x1FFFF,
|
||||
scratch_bytes=words[3] & 0x1FFFF,
|
||||
max_loop_count=words[4] & 0xFFFF,
|
||||
@@ -936,11 +971,11 @@ def parse_apumc(
|
||||
for index, entry in enumerate(entries):
|
||||
if entry.format_id != index:
|
||||
raise ValueError("entry descriptor order must be WAV, MP3, FLAC")
|
||||
validate_entry(entry, instruction_count, target, table_bytes)
|
||||
validate_entry(entry, instruction_count, target, table_bytes, image_abi)
|
||||
for instruction in instructions:
|
||||
validate_instruction(instruction, target)
|
||||
validate_instruction(instruction, target, image_abi)
|
||||
for entry in entries:
|
||||
validate_control_flow(instructions, entry, target)
|
||||
validate_control_flow(instructions, entry, target, image_abi)
|
||||
for pc in range(entry.first_pc, entry.last_pc + 1):
|
||||
required = instruction_required_mask(instructions[pc])
|
||||
if instructions[pc].instruction_class == InstructionClass.CONTROL and required:
|
||||
|
||||
@@ -0,0 +1,587 @@
|
||||
"""Frozen APUM 1.0 KWS container definitions and independent validators."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import struct
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
APUM_MAGIC: Final = 0x4D555041
|
||||
APUM_ABI: Final = 0x00010000
|
||||
APUM_BYTES: Final = 32768
|
||||
APUM_HEADER_BYTES: Final = 64
|
||||
APUM_OPERATOR_OFFSET: Final = 0x0040
|
||||
APUM_OPERATOR_COUNT: Final = 12
|
||||
APUM_OPERATOR_BYTES: Final = 64
|
||||
APUM_TENSOR_OFFSET: Final = 0x0340
|
||||
APUM_TENSOR_COUNT: Final = 13
|
||||
APUM_TENSOR_BYTES: Final = 32
|
||||
APUM_PARAMETER_OFFSET: Final = 0x0500
|
||||
APUM_PARAMETER_BYTES: Final = 29072
|
||||
APUM_PARAMETER_END: Final = APUM_PARAMETER_OFFSET + APUM_PARAMETER_BYTES
|
||||
APUM_PAYLOAD_CRC: Final = 0xB9034B22
|
||||
APUM_SHA256: Final = "87861b5f866182969e34917b4bf57cab61cd2245c3ab9b933797209322e3ce63"
|
||||
APUM_PARAMS_SHA256: Final = "0e4eb210d247a45747049ef9602bbd7e90868e9cb1c04fb2a02f128db44e241e"
|
||||
APUM_MULTIPLIER_SHIFT_SHA256: Final = (
|
||||
"222e05821695f3fdd47be682b2cc75e0e735aa106436bf941cdda3ef8cb03b59"
|
||||
)
|
||||
TFLITE_SHA256: Final = "aeea436800704fce17b17292e4412630ad856e9d777c044c64ef748a880bd0ae"
|
||||
CORPUS_SHA256: Final = "57fa600948ef7a19ed5eb0d0e73c7d09fef7d0dbb12bc8833a0a8caad1d6845e"
|
||||
|
||||
_TENSOR_PROFILE: Final = (
|
||||
(1, 49, 10, 1, 0x4000, 490, 0x3F15AF17, 83),
|
||||
(1, 25, 5, 64, 0x0000, 8000, 0x3DA13AC8, -128),
|
||||
(1, 25, 5, 64, 0x2000, 8000, 0x3DA99AEA, -128),
|
||||
(1, 25, 5, 64, 0x0000, 8000, 0x3D75DD81, -128),
|
||||
(1, 25, 5, 64, 0x2000, 8000, 0x3D808854, -128),
|
||||
(1, 25, 5, 64, 0x0000, 8000, 0x3D19329E, -128),
|
||||
(1, 25, 5, 64, 0x2000, 8000, 0x3D3C02C1, -128),
|
||||
(1, 25, 5, 64, 0x0000, 8000, 0x3D070C5A, -128),
|
||||
(1, 25, 5, 64, 0x2000, 8000, 0x3D3FDF8A, -128),
|
||||
(1, 25, 5, 64, 0x0000, 8000, 0x3DA452DB, -128),
|
||||
(1, 1, 1, 64, 0x2000, 64, 0x3DA452DB, -128),
|
||||
(1, 1, 1, 12, 0x0000, 12, 0x3E142A46, 14),
|
||||
(1, 1, 1, 12, 0x2000, 12, 0x3B800000, -128),
|
||||
)
|
||||
_OPCODES: Final = (1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 4, 5)
|
||||
_WEIGHT_BYTES: Final = (2560, 576, 4096, 576, 4096, 576, 4096, 576, 4096, 0, 768, 0)
|
||||
_OUTPUT_CHANNELS: Final = (64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 12, 12)
|
||||
_KERNELS: Final = (
|
||||
(10, 4, 2, 2),
|
||||
(3, 3, 1, 1),
|
||||
(1, 1, 1, 1),
|
||||
(3, 3, 1, 1),
|
||||
(1, 1, 1, 1),
|
||||
(3, 3, 1, 1),
|
||||
(1, 1, 1, 1),
|
||||
(3, 3, 1, 1),
|
||||
(1, 1, 1, 1),
|
||||
(25, 5, 25, 5),
|
||||
(1, 1, 1, 1),
|
||||
(1, 1, 1, 1),
|
||||
)
|
||||
_PADDING: Final = (
|
||||
(4, 5, 1, 1),
|
||||
(1, 1, 1, 1),
|
||||
(0, 0, 0, 0),
|
||||
(1, 1, 1, 1),
|
||||
(0, 0, 0, 0),
|
||||
(1, 1, 1, 1),
|
||||
(0, 0, 0, 0),
|
||||
(1, 1, 1, 1),
|
||||
(0, 0, 0, 0),
|
||||
(0, 0, 0, 0),
|
||||
(0, 0, 0, 0),
|
||||
(0, 0, 0, 0),
|
||||
)
|
||||
|
||||
|
||||
def crc32_iso_hdlc(data: bytes) -> int:
|
||||
"""Calculate CRC-32/ISO-HDLC with the exact APUM byte ordering."""
|
||||
crc = 0xFFFFFFFF
|
||||
for value in data:
|
||||
crc ^= value
|
||||
for _ in range(8):
|
||||
crc = (crc >> 1) ^ (0xEDB88320 if (crc & 1) != 0 else 0)
|
||||
return crc ^ 0xFFFFFFFF
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApumHeader:
|
||||
magic: int
|
||||
abi: int
|
||||
total_bytes: int
|
||||
operator_set: int
|
||||
mfcc_rows: int
|
||||
mfcc_coefficients: int
|
||||
classes: int
|
||||
operators: int
|
||||
parameter_bytes: int
|
||||
scratch_bytes: int
|
||||
quantization_profile: int
|
||||
crc: int
|
||||
model_id: int
|
||||
threshold: int
|
||||
debounce: int
|
||||
tensor_count: int
|
||||
layout_id: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApumOperator:
|
||||
opcode: int
|
||||
flags: int
|
||||
input_tensor: int
|
||||
output_tensor: int
|
||||
weight_offset: int
|
||||
bias_offset: int
|
||||
multiplier_offset: int
|
||||
shift_offset: int
|
||||
weight_bytes: int
|
||||
output_channels: int
|
||||
kernel_height: int
|
||||
kernel_width: int
|
||||
stride_height: int
|
||||
stride_width: int
|
||||
padding: tuple[int, int, int, int]
|
||||
input_zero: int
|
||||
output_zero: int
|
||||
clamp_min: int
|
||||
clamp_max: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApumTensor:
|
||||
shape: tuple[int, int, int, int]
|
||||
scratch_offset: int
|
||||
byte_count: int
|
||||
element_type: int
|
||||
scale_bits: int
|
||||
zero_point: int
|
||||
|
||||
|
||||
def _invalid(message: str) -> ValueError:
|
||||
return ValueError(f"invalid APUM 1.0: {message}")
|
||||
|
||||
|
||||
def _interval(offset: int, size: int, label: str) -> tuple[int, int, str]:
|
||||
if (offset & 15) != 0 or size <= 0 or offset < APUM_PARAMETER_OFFSET:
|
||||
raise _invalid(f"{label} range/alignment")
|
||||
end = offset + size
|
||||
if end > APUM_PARAMETER_END:
|
||||
raise _invalid(f"{label} outside parameter region")
|
||||
return offset, end, label
|
||||
|
||||
|
||||
def _parse_header(image: bytes) -> ApumHeader:
|
||||
words = struct.unpack_from("<10I", image, 0)
|
||||
magic, abi, total, op_set, mfcc, classes, params, scratch, quant, crc = words
|
||||
model_id = struct.unpack_from("<Q", image, 0x28)[0]
|
||||
defaults, tensor_count, layout_id, reserved = struct.unpack_from("<4I", image, 0x30)
|
||||
if reserved != 0:
|
||||
raise _invalid("header reserved word is nonzero")
|
||||
return ApumHeader(
|
||||
magic=magic,
|
||||
abi=abi,
|
||||
total_bytes=total,
|
||||
operator_set=op_set,
|
||||
mfcc_rows=mfcc & 0xFFFF,
|
||||
mfcc_coefficients=mfcc >> 16,
|
||||
classes=classes & 0xFFFF,
|
||||
operators=classes >> 16,
|
||||
parameter_bytes=params,
|
||||
scratch_bytes=scratch,
|
||||
quantization_profile=quant,
|
||||
crc=crc,
|
||||
model_id=model_id,
|
||||
threshold=defaults & 0xFF,
|
||||
debounce=(defaults >> 8) & 0xFF,
|
||||
tensor_count=tensor_count,
|
||||
layout_id=layout_id,
|
||||
)
|
||||
|
||||
|
||||
def _parse_operator(image: bytes, index: int) -> ApumOperator:
|
||||
offset = APUM_OPERATOR_OFFSET + index * APUM_OPERATOR_BYTES
|
||||
fields = struct.unpack_from("<HHHHIIIIIIHHHH4BiiiiI", image, offset)
|
||||
return ApumOperator(
|
||||
opcode=fields[0],
|
||||
flags=fields[1],
|
||||
input_tensor=fields[2],
|
||||
output_tensor=fields[3],
|
||||
weight_offset=fields[4],
|
||||
bias_offset=fields[5],
|
||||
multiplier_offset=fields[6],
|
||||
shift_offset=fields[7],
|
||||
weight_bytes=fields[8],
|
||||
output_channels=fields[9],
|
||||
kernel_height=fields[10],
|
||||
kernel_width=fields[11],
|
||||
stride_height=fields[12],
|
||||
stride_width=fields[13],
|
||||
padding=(fields[14], fields[15], fields[16], fields[17]),
|
||||
input_zero=fields[18],
|
||||
output_zero=fields[19],
|
||||
clamp_min=fields[20],
|
||||
clamp_max=fields[21],
|
||||
)
|
||||
|
||||
|
||||
def _parse_tensor(image: bytes, index: int) -> ApumTensor:
|
||||
offset = APUM_TENSOR_OFFSET + index * APUM_TENSOR_BYTES
|
||||
n, height, width, channels, scratch, count, element, scale, zero, reserved = struct.unpack_from(
|
||||
"<HHHHIIIIiI", image, offset
|
||||
)
|
||||
if reserved != 0:
|
||||
raise _invalid(f"tensor {index} reserved word is nonzero")
|
||||
return ApumTensor(
|
||||
shape=(n, height, width, channels),
|
||||
scratch_offset=scratch,
|
||||
byte_count=count,
|
||||
element_type=element,
|
||||
scale_bits=scale,
|
||||
zero_point=zero,
|
||||
)
|
||||
|
||||
|
||||
def _validate_header(header: ApumHeader, image: bytes) -> None:
|
||||
expected = (
|
||||
header.magic == APUM_MAGIC,
|
||||
header.abi == APUM_ABI,
|
||||
header.total_bytes == APUM_BYTES,
|
||||
header.operator_set == 1,
|
||||
header.mfcc_rows == 49,
|
||||
header.mfcc_coefficients == 10,
|
||||
header.classes == 12,
|
||||
header.operators == APUM_OPERATOR_COUNT,
|
||||
header.parameter_bytes == APUM_PARAMETER_BYTES,
|
||||
header.scratch_bytes == APUM_BYTES,
|
||||
header.quantization_profile == 1,
|
||||
header.model_id == 0xCE4F70006843EAAE,
|
||||
header.threshold == 128,
|
||||
header.debounce == 3,
|
||||
header.tensor_count == APUM_TENSOR_COUNT,
|
||||
header.layout_id == 1,
|
||||
)
|
||||
if not all(expected):
|
||||
raise _invalid("header profile")
|
||||
if image[0x32:0x34] != b"\x00\x00":
|
||||
raise _invalid("header defaults reserved bits")
|
||||
if crc32_iso_hdlc(image[APUM_HEADER_BYTES:]) != header.crc:
|
||||
raise _invalid("payload CRC")
|
||||
if header.crc != APUM_PAYLOAD_CRC:
|
||||
raise _invalid("non-frozen payload CRC")
|
||||
|
||||
|
||||
def _validate_tensors(image: bytes) -> list[ApumTensor]:
|
||||
tensors = [_parse_tensor(image, index) for index in range(APUM_TENSOR_COUNT)]
|
||||
for index, tensor in enumerate(tensors):
|
||||
n, height, width, channels, scratch, count, scale, zero = _TENSOR_PROFILE[index]
|
||||
if (
|
||||
tensor.shape != (n, height, width, channels)
|
||||
or tensor.scratch_offset != scratch
|
||||
or tensor.byte_count != count
|
||||
or tensor.element_type != 1
|
||||
or tensor.scale_bits != scale
|
||||
or tensor.zero_point != zero
|
||||
):
|
||||
raise _invalid(f"tensor {index} fixed profile")
|
||||
if tensor.byte_count != n * height * width * channels:
|
||||
raise _invalid(f"tensor {index} byte count")
|
||||
if tensor.scratch_offset + tensor.byte_count > APUM_BYTES:
|
||||
raise _invalid(f"tensor {index} scratch range")
|
||||
return tensors
|
||||
|
||||
|
||||
def _validate_operators(image: bytes, tensors: list[ApumTensor]) -> list[ApumOperator]:
|
||||
operators = [_parse_operator(image, index) for index in range(APUM_OPERATOR_COUNT)]
|
||||
intervals: list[tuple[int, int, str]] = []
|
||||
for index, operator in enumerate(operators):
|
||||
if (
|
||||
operator.opcode != _OPCODES[index]
|
||||
or operator.flags != (1 if index < 9 else 0)
|
||||
or operator.input_tensor != index
|
||||
or operator.output_tensor != index + 1
|
||||
or operator.weight_bytes != _WEIGHT_BYTES[index]
|
||||
or operator.output_channels != _OUTPUT_CHANNELS[index]
|
||||
or (
|
||||
operator.kernel_height,
|
||||
operator.kernel_width,
|
||||
operator.stride_height,
|
||||
operator.stride_width,
|
||||
)
|
||||
!= _KERNELS[index]
|
||||
or operator.padding != _PADDING[index]
|
||||
or operator.input_zero != tensors[index].zero_point
|
||||
or operator.output_zero != tensors[index + 1].zero_point
|
||||
or operator.clamp_min != -128
|
||||
or operator.clamp_max != 127
|
||||
):
|
||||
raise _invalid(f"operator {index} fixed profile")
|
||||
if operator.opcode in (3, 5):
|
||||
if any(
|
||||
value != 0
|
||||
for value in (
|
||||
operator.weight_offset,
|
||||
operator.bias_offset,
|
||||
operator.multiplier_offset,
|
||||
operator.shift_offset,
|
||||
operator.weight_bytes,
|
||||
)
|
||||
):
|
||||
raise _invalid(f"operator {index} parameter fields")
|
||||
continue
|
||||
if index == 10 and operator.flags != 0:
|
||||
raise _invalid("fully connected ReLU flag")
|
||||
intervals.extend(
|
||||
(
|
||||
_interval(
|
||||
operator.weight_offset, operator.weight_bytes, f"operator {index} weights"
|
||||
),
|
||||
_interval(
|
||||
operator.bias_offset, operator.output_channels * 4, f"operator {index} biases"
|
||||
),
|
||||
_interval(
|
||||
operator.multiplier_offset,
|
||||
operator.output_channels * 4,
|
||||
f"operator {index} multipliers",
|
||||
),
|
||||
_interval(
|
||||
operator.shift_offset, operator.output_channels * 4, f"operator {index} shifts"
|
||||
),
|
||||
)
|
||||
)
|
||||
intervals.sort()
|
||||
if intervals[0][0] != APUM_PARAMETER_OFFSET or intervals[-1][1] != APUM_PARAMETER_END:
|
||||
raise _invalid("parameter coverage")
|
||||
for previous, current in zip(intervals, intervals[1:]):
|
||||
if previous[1] != current[0]:
|
||||
raise _invalid("parameter overlap or gap")
|
||||
return operators
|
||||
|
||||
|
||||
def parse_apum(image: bytes, *, require_golden_image: bool = True) -> ApumHeader:
|
||||
"""Validate the complete immutable APUM 1.0 profile and return its header."""
|
||||
if len(image) != APUM_BYTES:
|
||||
raise _invalid("image size")
|
||||
header = _parse_header(image)
|
||||
_validate_header(header, image)
|
||||
if image[0x4E0:APUM_PARAMETER_OFFSET] != bytes(APUM_PARAMETER_OFFSET - 0x4E0):
|
||||
raise _invalid("metadata padding")
|
||||
if image[APUM_PARAMETER_END:] != bytes(APUM_BYTES - APUM_PARAMETER_END):
|
||||
raise _invalid("tail padding")
|
||||
tensors = _validate_tensors(image)
|
||||
_validate_operators(image, tensors)
|
||||
if (
|
||||
hashlib.sha256(image[APUM_PARAMETER_OFFSET:APUM_PARAMETER_END]).hexdigest()
|
||||
!= APUM_PARAMS_SHA256
|
||||
):
|
||||
raise _invalid("parameter SHA-256")
|
||||
if require_golden_image and image_sha256(image) != APUM_SHA256:
|
||||
raise _invalid("complete image SHA-256")
|
||||
return header
|
||||
|
||||
|
||||
def apum_layout(image: bytes) -> dict[str, object]:
|
||||
"""Return the reviewable fixed APUM layout after strict validation."""
|
||||
header = parse_apum(image)
|
||||
tensors = _validate_tensors(image)
|
||||
operators = _validate_operators(image, tensors)
|
||||
return {
|
||||
"abi": header.abi,
|
||||
"image_bytes": len(image),
|
||||
"payload_crc": header.crc,
|
||||
"image_sha256": image_sha256(image),
|
||||
"parameter_sha256": hashlib.sha256(
|
||||
image[APUM_PARAMETER_OFFSET:APUM_PARAMETER_END]
|
||||
).hexdigest(),
|
||||
"operators": [operator.__dict__ for operator in operators],
|
||||
"tensors": [tensor.__dict__ for tensor in tensors],
|
||||
}
|
||||
|
||||
|
||||
def image_sha256(image: bytes) -> str:
|
||||
return hashlib.sha256(image).hexdigest()
|
||||
|
||||
|
||||
def quantize_confidence(softmax: int) -> int:
|
||||
"""Apply the frozen signed-INT8 Softmax byte to public unsigned confidence."""
|
||||
return max(0, min(255, int(softmax) + 128))
|
||||
|
||||
|
||||
def select_class(scores: list[int]) -> int:
|
||||
"""Choose the lowest class ID among maximum signed INT8 scores."""
|
||||
if not scores:
|
||||
raise ValueError("at least one class score is required")
|
||||
return min(range(len(scores)), key=lambda index: (-scores[index], index))
|
||||
|
||||
|
||||
def _signed_byte(value: int) -> int:
|
||||
return value - 256 if value >= 128 else value
|
||||
|
||||
|
||||
def _trunc_div(numerator: int, denominator: int) -> int:
|
||||
return numerator // denominator if numerator >= 0 else -((-numerator) // denominator)
|
||||
|
||||
|
||||
def _saturating_high_mul(left: int, right: int) -> int:
|
||||
if left == -(1 << 31) and right == -(1 << 31):
|
||||
return (1 << 31) - 1
|
||||
product = left * right
|
||||
nudge = (1 << 30) if product >= 0 else 1 - (1 << 30)
|
||||
return _trunc_div(product + nudge, 1 << 31)
|
||||
|
||||
|
||||
def _rounding_divide_pot(value: int, exponent: int) -> int:
|
||||
if exponent == 0:
|
||||
return value
|
||||
mask = (1 << exponent) - 1
|
||||
remainder = value & mask
|
||||
threshold = (mask >> 1) + (1 if value < 0 else 0)
|
||||
return (value >> exponent) + (1 if remainder > threshold else 0)
|
||||
|
||||
|
||||
def _multiply_quantized(value: int, multiplier: int, shift: int) -> int:
|
||||
left_shift = max(shift, 0)
|
||||
right_shift = max(-shift, 0)
|
||||
shifted = value << left_shift
|
||||
if not -(1 << 31) <= shifted < (1 << 31):
|
||||
raise OverflowError("APUM requantization left shift overflow")
|
||||
return _rounding_divide_pot(
|
||||
_saturating_high_mul(shifted, multiplier), right_shift
|
||||
)
|
||||
|
||||
|
||||
def _exp_interval_q31(raw: int) -> int:
|
||||
x = raw + (1 << 28)
|
||||
x2 = _saturating_high_mul(x, x)
|
||||
x3 = _saturating_high_mul(x2, x)
|
||||
x4 = _saturating_high_mul(x2, x2)
|
||||
series = _rounding_divide_pot(
|
||||
_saturating_high_mul(_rounding_divide_pot(x4, 2) + x3, 715827883)
|
||||
+ x2,
|
||||
1,
|
||||
)
|
||||
return 1895147668 + _saturating_high_mul(1895147668, x + series)
|
||||
|
||||
|
||||
def _softmax_exp_q31(difference: int) -> int:
|
||||
scaled = _saturating_high_mul((-difference) << 24, 1242899200)
|
||||
quarter = 1 << 24
|
||||
reduced = (scaled & (quarter - 1)) - quarter
|
||||
result = _exp_interval_q31(reduced << 5)
|
||||
remainder = reduced - scaled
|
||||
for exponent, multiplier in (
|
||||
(-2, 1672461947),
|
||||
(-1, 1302514674),
|
||||
(0, 790015084),
|
||||
(1, 290630308),
|
||||
(2, 39332535),
|
||||
(3, 720401),
|
||||
(4, 242),
|
||||
):
|
||||
if (remainder & (1 << (26 + exponent))) != 0:
|
||||
result = _saturating_high_mul(result, multiplier)
|
||||
return (1 << 31) - 1 if scaled == 0 else result
|
||||
|
||||
|
||||
def _reciprocal_q31(value: int) -> int:
|
||||
total = value + ((1 << 31) - 1)
|
||||
half_denominator = _trunc_div(total + (1 if total >= 0 else -1), 2)
|
||||
estimate = 1515870810 + _saturating_high_mul(
|
||||
half_denominator, -1010580540
|
||||
)
|
||||
for _ in range(3):
|
||||
product = _saturating_high_mul(half_denominator, estimate)
|
||||
correction = 0x20000000 - product
|
||||
estimate += _saturating_high_mul(estimate, correction)
|
||||
return max(-(1 << 31), min((1 << 31) - 1, estimate << 1))
|
||||
|
||||
|
||||
def _parameter_i32(image: bytes, offset: int, count: int) -> list[int]:
|
||||
return list(struct.unpack_from(f"<{count}i", image, offset))
|
||||
|
||||
|
||||
def _weighted_operator(
|
||||
image: bytes, operator: ApumOperator, source: list[int], operator_index: int
|
||||
) -> list[int]:
|
||||
output = [0] * operator.output_channels
|
||||
biases = _parameter_i32(image, operator.bias_offset, operator.output_channels)
|
||||
multipliers = _parameter_i32(
|
||||
image, operator.multiplier_offset, operator.output_channels
|
||||
)
|
||||
shifts = _parameter_i32(image, operator.shift_offset, operator.output_channels)
|
||||
weights = image[
|
||||
operator.weight_offset : operator.weight_offset + operator.weight_bytes
|
||||
]
|
||||
output_count = 25 * 5 * operator.output_channels if operator_index < 9 else 12
|
||||
output = [0] * output_count
|
||||
for output_index in range(output_count):
|
||||
channel = output_index % operator.output_channels
|
||||
spatial = output_index // operator.output_channels
|
||||
output_height = spatial // 5
|
||||
output_width = spatial % 5
|
||||
accumulator = biases[channel]
|
||||
if operator_index == 0:
|
||||
for term in range(40):
|
||||
input_height = output_height * 2 + term // 4 - 4
|
||||
input_width = output_width * 2 + term % 4 - 1
|
||||
input_value = operator.input_zero
|
||||
if 0 <= input_height < 49 and 0 <= input_width < 10:
|
||||
input_value = source[input_height * 10 + input_width]
|
||||
weight = _signed_byte(weights[channel * 40 + term])
|
||||
accumulator += (input_value - operator.input_zero) * weight
|
||||
elif operator_index in (1, 3, 5, 7):
|
||||
for term in range(9):
|
||||
input_height = output_height + term // 3 - 1
|
||||
input_width = output_width + term % 3 - 1
|
||||
input_value = operator.input_zero
|
||||
if 0 <= input_height < 25 and 0 <= input_width < 5:
|
||||
input_value = source[
|
||||
(input_height * 5 + input_width) * 64 + channel
|
||||
]
|
||||
weight = _signed_byte(weights[term * 64 + channel])
|
||||
accumulator += (input_value - operator.input_zero) * weight
|
||||
else:
|
||||
for term in range(64):
|
||||
input_value = source[term] if operator_index == 10 else source[spatial * 64 + term]
|
||||
weight = _signed_byte(weights[channel * 64 + term])
|
||||
accumulator += (input_value - operator.input_zero) * weight
|
||||
if not -(1 << 31) <= accumulator < (1 << 31):
|
||||
raise OverflowError(f"APUM operator {operator_index} accumulator overflow")
|
||||
quantized = (
|
||||
_multiply_quantized(accumulator, multipliers[channel], shifts[channel])
|
||||
+ operator.output_zero
|
||||
)
|
||||
clamp_min = max(operator.clamp_min, operator.output_zero) if operator.flags & 1 else -128
|
||||
output[output_index] = max(clamp_min, min(operator.clamp_max, quantized))
|
||||
return output
|
||||
|
||||
|
||||
def _softmax(logits: list[int]) -> list[int]:
|
||||
maximum = max(logits)
|
||||
exponentials = [
|
||||
_softmax_exp_q31(maximum - value) if value - maximum >= -124 else 0
|
||||
for value in logits
|
||||
]
|
||||
accumulation = sum(_rounding_divide_pot(value, 12) for value in exponentials)
|
||||
headroom = 32 - accumulation.bit_length()
|
||||
shifted_sum = ((accumulation << headroom) & 0xFFFFFFFF) - (1 << 31)
|
||||
scale = _reciprocal_q31(shifted_sum)
|
||||
num_bits_over_unit = 12 - headroom
|
||||
result = []
|
||||
for exponential in exponentials:
|
||||
probability = _rounding_divide_pot(
|
||||
_saturating_high_mul(scale, exponential), num_bits_over_unit + 23
|
||||
)
|
||||
result.append(max(-128, min(127, probability - 128)))
|
||||
return result
|
||||
|
||||
|
||||
def infer_apum(image: bytes, features: bytes) -> list[list[int]]:
|
||||
"""Run the frozen scalar INT8 graph and return every published tensor."""
|
||||
parse_apum(image)
|
||||
if len(features) != 490:
|
||||
raise ValueError("APUM inference requires exactly 490 INT8 feature bytes")
|
||||
tensors = [_parse_tensor(image, index) for index in range(APUM_TENSOR_COUNT)]
|
||||
operators = [_parse_operator(image, index) for index in range(APUM_OPERATOR_COUNT)]
|
||||
current = [_signed_byte(value) for value in features]
|
||||
layers: list[list[int]] = []
|
||||
for operator_index, operator in enumerate(operators):
|
||||
if operator_index < 9 or operator_index == 10:
|
||||
current = _weighted_operator(image, operator, current, operator_index)
|
||||
elif operator_index == 9:
|
||||
pooled = []
|
||||
for channel in range(64):
|
||||
total = sum(current[spatial * 64 + channel] for spatial in range(125))
|
||||
pooled.append(
|
||||
_trunc_div(total + (62 if total >= 0 else -62), 125)
|
||||
)
|
||||
current = pooled
|
||||
else:
|
||||
current = _softmax(current)
|
||||
if len(current) != tensors[operator.output_tensor].byte_count:
|
||||
raise ValueError(f"APUM operator {operator_index} output size")
|
||||
layers.append(current)
|
||||
return layers
|
||||
@@ -0,0 +1,526 @@
|
||||
"""Import the locked MLPerf Tiny KWS TFLite graph into frozen APUM 1.0."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import struct
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from apu_kws import (
|
||||
APUM_ABI,
|
||||
APUM_BYTES,
|
||||
APUM_HEADER_BYTES,
|
||||
APUM_MAGIC,
|
||||
APUM_MULTIPLIER_SHIFT_SHA256,
|
||||
APUM_PARAMETER_END,
|
||||
APUM_PARAMETER_OFFSET,
|
||||
APUM_PARAMS_SHA256,
|
||||
APUM_PAYLOAD_CRC,
|
||||
APUM_SHA256,
|
||||
TFLITE_SHA256,
|
||||
apum_layout,
|
||||
crc32_iso_hdlc,
|
||||
image_sha256,
|
||||
parse_apum,
|
||||
)
|
||||
|
||||
_OPERATOR_CODES = {
|
||||
"CONV_2D": 3,
|
||||
"DEPTHWISE_CONV_2D": 4,
|
||||
"AVERAGE_POOL_2D": 1,
|
||||
"RESHAPE": 22,
|
||||
"FULLY_CONNECTED": 9,
|
||||
"SOFTMAX": 25,
|
||||
}
|
||||
_APUM_OPCODES = (1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 4, 5)
|
||||
_ACTIVATIONS = (
|
||||
(22, (1, 25, 5, 64), 0x0000),
|
||||
(23, (1, 25, 5, 64), 0x2000),
|
||||
(24, (1, 25, 5, 64), 0x0000),
|
||||
(25, (1, 25, 5, 64), 0x2000),
|
||||
(26, (1, 25, 5, 64), 0x0000),
|
||||
(27, (1, 25, 5, 64), 0x2000),
|
||||
(28, (1, 25, 5, 64), 0x0000),
|
||||
(29, (1, 25, 5, 64), 0x2000),
|
||||
(30, (1, 25, 5, 64), 0x0000),
|
||||
(31, (1, 1, 1, 64), 0x2000),
|
||||
(33, (1, 1, 1, 12), 0x0000),
|
||||
(34, (1, 1, 1, 12), 0x2000),
|
||||
)
|
||||
|
||||
|
||||
class TfliteError(ValueError):
|
||||
"""The managed input does not have the one frozen graph."""
|
||||
|
||||
|
||||
class _Table:
|
||||
def __init__(self, image: bytes, position: int) -> None:
|
||||
self.image = image
|
||||
self.position = position
|
||||
self.vtable = position - struct.unpack_from("<i", image, position)[0]
|
||||
self.vtable_size = struct.unpack_from("<H", image, self.vtable)[0]
|
||||
|
||||
def field(self, index: int) -> int | None:
|
||||
location = self.vtable + 4 + index * 2
|
||||
if location + 2 > self.vtable + self.vtable_size:
|
||||
return None
|
||||
offset = struct.unpack_from("<H", self.image, location)[0]
|
||||
return self.position + offset if offset != 0 else None
|
||||
|
||||
def scalar(self, index: int, fmt: str, default: int | float = 0) -> int | float:
|
||||
location = self.field(index)
|
||||
return (
|
||||
default if location is None else struct.unpack_from("<" + fmt, self.image, location)[0]
|
||||
)
|
||||
|
||||
def offset(self, index: int) -> int | None:
|
||||
location = self.field(index)
|
||||
if location is None:
|
||||
return None
|
||||
return location + struct.unpack_from("<I", self.image, location)[0]
|
||||
|
||||
def vector(self, index: int, fmt: str) -> tuple[int | float, ...]:
|
||||
location = self.offset(index)
|
||||
if location is None:
|
||||
return ()
|
||||
length = struct.unpack_from("<I", self.image, location)[0]
|
||||
return struct.unpack_from("<" + fmt * length, self.image, location + 4)
|
||||
|
||||
def tables(self, index: int) -> tuple["_Table", ...]:
|
||||
location = self.offset(index)
|
||||
if location is None:
|
||||
return ()
|
||||
length = struct.unpack_from("<I", self.image, location)[0]
|
||||
return tuple(
|
||||
_Table(
|
||||
self.image,
|
||||
location
|
||||
+ 4
|
||||
+ item * 4
|
||||
+ struct.unpack_from("<I", self.image, location + 4 + item * 4)[0],
|
||||
)
|
||||
for item in range(length)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Tensor:
|
||||
shape: tuple[int, ...]
|
||||
kind: int
|
||||
buffer: bytes
|
||||
scales: tuple[float, ...]
|
||||
zero_points: tuple[int, ...]
|
||||
quantized_dimension: int
|
||||
scale_bits: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Operator:
|
||||
builtin: int
|
||||
inputs: tuple[int, ...]
|
||||
outputs: tuple[int, ...]
|
||||
options: _Table | None
|
||||
|
||||
|
||||
def _require(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise TfliteError(message)
|
||||
|
||||
|
||||
def _root(image: bytes) -> _Table:
|
||||
_require(len(image) >= 8 and image[4:8] == b"TFL3", "not a TFLite v3 FlatBuffer")
|
||||
root = struct.unpack_from("<I", image, 0)[0]
|
||||
_require(root < len(image), "root offset")
|
||||
return _Table(image, root)
|
||||
|
||||
|
||||
def _parse_tflite(image: bytes) -> tuple[tuple[_Tensor, ...], tuple[_Operator, ...]]:
|
||||
model = _root(image)
|
||||
operator_codes = tuple(int(code.scalar(0, "b")) for code in model.tables(1))
|
||||
subgraphs = model.tables(2)
|
||||
buffers = model.tables(4)
|
||||
_require(len(subgraphs) == 1, "subgraph count")
|
||||
raw_buffers = tuple(bytes(buffer.vector(0, "B")) for buffer in buffers)
|
||||
subgraph = subgraphs[0]
|
||||
tensors: list[_Tensor] = []
|
||||
for tensor in subgraph.tables(0):
|
||||
quantization = tensor.offset(4)
|
||||
quant = _Table(image, quantization) if quantization is not None else None
|
||||
scales = tuple(float(item) for item in quant.vector(2, "f")) if quant else ()
|
||||
zeroes = tuple(int(item) for item in quant.vector(3, "q")) if quant else ()
|
||||
buffer_index = int(tensor.scalar(2, "I"))
|
||||
_require(buffer_index < len(raw_buffers), "tensor buffer index")
|
||||
tensors.append(
|
||||
_Tensor(
|
||||
shape=tuple(int(item) for item in tensor.vector(0, "i")),
|
||||
kind=int(tensor.scalar(1, "b")),
|
||||
buffer=raw_buffers[buffer_index],
|
||||
scales=scales,
|
||||
zero_points=zeroes,
|
||||
quantized_dimension=int(quant.scalar(5, "i")) if quant else 0,
|
||||
scale_bits=struct.unpack("<I", struct.pack("<f", scales[0]))[0] if scales else 0,
|
||||
)
|
||||
)
|
||||
operators: list[_Operator] = []
|
||||
for operator in subgraph.tables(3):
|
||||
opcode_index = int(operator.scalar(0, "I"))
|
||||
_require(opcode_index < len(operator_codes), "operator-code index")
|
||||
options_offset = operator.offset(4)
|
||||
operators.append(
|
||||
_Operator(
|
||||
builtin=operator_codes[opcode_index],
|
||||
inputs=tuple(int(item) for item in operator.vector(1, "i")),
|
||||
outputs=tuple(int(item) for item in operator.vector(2, "i")),
|
||||
options=_Table(image, options_offset) if options_offset is not None else None,
|
||||
)
|
||||
)
|
||||
return tuple(tensors), tuple(operators)
|
||||
|
||||
|
||||
def _q31_multiplier(value: float) -> tuple[int, int]:
|
||||
_require(value >= 0.0 and math.isfinite(value), "invalid effective quantization multiplier")
|
||||
fraction, shift = math.frexp(value)
|
||||
quantized = int(math.floor(fraction * (1 << 31) + 0.5))
|
||||
if quantized == (1 << 31):
|
||||
quantized //= 2
|
||||
shift += 1
|
||||
if shift < -31:
|
||||
return 0, 0
|
||||
_require(quantized <= 0x7FFFFFFF, "Q31 multiplier overflow")
|
||||
return quantized, shift
|
||||
|
||||
|
||||
def _append_aligned(image: bytearray, cursor: int, values: bytes) -> tuple[int, int]:
|
||||
cursor = (cursor + 15) & ~15
|
||||
end = cursor + len(values)
|
||||
_require(end <= APUM_PARAMETER_END, "parameter image overflow")
|
||||
image[cursor:end] = values
|
||||
return cursor, end
|
||||
|
||||
|
||||
def _expect_options(
|
||||
operator: _Operator, index: int
|
||||
) -> tuple[int, int, int, int, tuple[int, int, int, int]]:
|
||||
_require(operator.options is not None, f"operator {index} options")
|
||||
options = operator.options
|
||||
if index == 9:
|
||||
# Pool2D: padding, stride_w, stride_h, filter_w, filter_h, activation.
|
||||
values = (
|
||||
int(options.scalar(0, "b")),
|
||||
int(options.scalar(1, "i")),
|
||||
int(options.scalar(2, "i")),
|
||||
int(options.scalar(3, "i")),
|
||||
int(options.scalar(4, "i")),
|
||||
int(options.scalar(5, "b")),
|
||||
)
|
||||
_require(values == (1, 5, 25, 5, 25, 0), "pool options")
|
||||
return 25, 5, 25, 5, (0, 0, 0, 0)
|
||||
if index == 10:
|
||||
_require(
|
||||
int(options.scalar(0, "b")) == 0 and int(options.scalar(1, "b")) == 0, "FC options"
|
||||
)
|
||||
return 1, 1, 1, 1, (0, 0, 0, 0)
|
||||
if index == 11:
|
||||
_require(
|
||||
struct.unpack("<I", struct.pack("<f", float(options.scalar(0, "f"))))[0] == 0x3F800000,
|
||||
"softmax beta",
|
||||
)
|
||||
return 1, 1, 1, 1, (0, 0, 0, 0)
|
||||
if index == 0:
|
||||
values = (
|
||||
int(options.scalar(0, "b")),
|
||||
int(options.scalar(1, "i")),
|
||||
int(options.scalar(2, "i")),
|
||||
int(options.scalar(3, "b")),
|
||||
int(options.scalar(4, "i", 1)),
|
||||
int(options.scalar(5, "i", 1)),
|
||||
)
|
||||
_require(values == (0, 2, 2, 1, 1, 1), "first convolution options")
|
||||
return 10, 4, 2, 2, (4, 5, 1, 1)
|
||||
if index in (1, 3, 5, 7):
|
||||
values = (
|
||||
int(options.scalar(0, "b")),
|
||||
int(options.scalar(1, "i")),
|
||||
int(options.scalar(2, "i")),
|
||||
int(options.scalar(3, "i")),
|
||||
int(options.scalar(4, "b")),
|
||||
int(options.scalar(5, "i", 1)),
|
||||
int(options.scalar(6, "i", 1)),
|
||||
)
|
||||
_require(values == (0, 1, 1, 1, 1, 1, 1), f"depthwise options {index}")
|
||||
return 3, 3, 1, 1, (1, 1, 1, 1)
|
||||
values = (
|
||||
int(options.scalar(0, "b")),
|
||||
int(options.scalar(1, "i")),
|
||||
int(options.scalar(2, "i")),
|
||||
int(options.scalar(3, "b")),
|
||||
int(options.scalar(4, "i", 1)),
|
||||
int(options.scalar(5, "i", 1)),
|
||||
)
|
||||
_require(values == (0, 1, 1, 1, 1, 1), f"pointwise options {index}")
|
||||
return 1, 1, 1, 1, (0, 0, 0, 0)
|
||||
|
||||
|
||||
def _weighted_record(
|
||||
image: bytearray,
|
||||
cursor: int,
|
||||
tensors: tuple[_Tensor, ...],
|
||||
operator: _Operator,
|
||||
index: int,
|
||||
input_tensor: int,
|
||||
output_tensor: int,
|
||||
) -> tuple[int, bytes]:
|
||||
_require(len(operator.inputs) == 3 and len(operator.outputs) == 1, f"operator {index} arity")
|
||||
source, weights, bias = (tensors[item] for item in operator.inputs)
|
||||
output = tensors[operator.outputs[0]]
|
||||
expected_output = _ACTIVATIONS[index][1] if index < 10 else _ACTIVATIONS[index][1]
|
||||
if index == 10:
|
||||
expected_output = (1, 1, 1, 12)
|
||||
_require(
|
||||
math.prod(output.shape) == math.prod(expected_output), f"operator {index} output shape"
|
||||
)
|
||||
_require(source.kind == 9 and weights.kind == 9 and bias.kind == 2, f"operator {index} types")
|
||||
_require(
|
||||
len(source.scales) == 1 and len(output.scales) == 1,
|
||||
f"operator {index} activation quantization",
|
||||
)
|
||||
channels = 12 if index == 10 else 64
|
||||
expected_scales = 1 if index == 10 else channels
|
||||
_require(
|
||||
len(weights.scales) == expected_scales and len(weights.zero_points) == expected_scales,
|
||||
f"operator {index} weight scales",
|
||||
)
|
||||
_require(
|
||||
len(bias.buffer) == channels * 4 and len(bias.scales) == expected_scales,
|
||||
f"operator {index} bias",
|
||||
)
|
||||
_require(
|
||||
all(value == 0 for value in weights.zero_points), f"operator {index} weight zero point"
|
||||
)
|
||||
weight_offset, cursor = _append_aligned(image, cursor, weights.buffer)
|
||||
bias_offset, cursor = _append_aligned(image, cursor, bias.buffer)
|
||||
multipliers: list[int] = []
|
||||
shifts: list[int] = []
|
||||
for channel in range(channels):
|
||||
weight_scale = weights.scales[0 if index == 10 else channel]
|
||||
multiplier, shift = _q31_multiplier(
|
||||
float(source.scales[0]) * float(weight_scale) / float(output.scales[0])
|
||||
)
|
||||
multipliers.append(multiplier)
|
||||
shifts.append(shift)
|
||||
multiplier_data = struct.pack("<" + "i" * channels, *multipliers)
|
||||
shift_data = struct.pack("<" + "i" * channels, *shifts)
|
||||
multiplier_offset, cursor = _append_aligned(image, cursor, multiplier_data)
|
||||
shift_offset, cursor = _append_aligned(image, cursor, shift_data)
|
||||
kernel_h, kernel_w, stride_h, stride_w, padding = _expect_options(operator, index)
|
||||
flags = 0 if index == 10 else 1
|
||||
record = struct.pack(
|
||||
"<HHHHIIIIIIHHHH4BiiiiI",
|
||||
_APUM_OPCODES[index],
|
||||
flags,
|
||||
input_tensor,
|
||||
output_tensor,
|
||||
weight_offset,
|
||||
bias_offset,
|
||||
multiplier_offset,
|
||||
shift_offset,
|
||||
len(weights.buffer),
|
||||
channels,
|
||||
kernel_h,
|
||||
kernel_w,
|
||||
stride_h,
|
||||
stride_w,
|
||||
*padding,
|
||||
source.zero_points[0],
|
||||
output.zero_points[0],
|
||||
-128,
|
||||
127,
|
||||
0,
|
||||
)
|
||||
return cursor, record + multiplier_data + shift_data
|
||||
|
||||
|
||||
def import_tflite(model: Path) -> bytes:
|
||||
"""Parse the exact frozen TFLite graph and return a strict APUM 1.0 image."""
|
||||
source = model.read_bytes()
|
||||
_require(hashlib.sha256(source).hexdigest() == TFLITE_SHA256, "unexpected TFLite SHA-256")
|
||||
tensors, operators = _parse_tflite(source)
|
||||
expected = (
|
||||
_OPERATOR_CODES["CONV_2D"],
|
||||
_OPERATOR_CODES["DEPTHWISE_CONV_2D"],
|
||||
_OPERATOR_CODES["CONV_2D"],
|
||||
_OPERATOR_CODES["DEPTHWISE_CONV_2D"],
|
||||
_OPERATOR_CODES["CONV_2D"],
|
||||
_OPERATOR_CODES["DEPTHWISE_CONV_2D"],
|
||||
_OPERATOR_CODES["CONV_2D"],
|
||||
_OPERATOR_CODES["DEPTHWISE_CONV_2D"],
|
||||
_OPERATOR_CODES["CONV_2D"],
|
||||
_OPERATOR_CODES["AVERAGE_POOL_2D"],
|
||||
_OPERATOR_CODES["RESHAPE"],
|
||||
_OPERATOR_CODES["FULLY_CONNECTED"],
|
||||
_OPERATOR_CODES["SOFTMAX"],
|
||||
)
|
||||
_require(tuple(operator.builtin for operator in operators) == expected, "operator graph")
|
||||
image = bytearray(APUM_BYTES)
|
||||
struct.pack_into(
|
||||
"<10I",
|
||||
image,
|
||||
0,
|
||||
APUM_MAGIC,
|
||||
APUM_ABI,
|
||||
APUM_BYTES,
|
||||
1,
|
||||
49 | (10 << 16),
|
||||
12 | (12 << 16),
|
||||
APUM_PARAMETER_END - APUM_PARAMETER_OFFSET,
|
||||
APUM_BYTES,
|
||||
1,
|
||||
0,
|
||||
)
|
||||
struct.pack_into("<Q", image, 0x28, 0xCE4F70006843EAAE)
|
||||
struct.pack_into("<I", image, 0x30, 128 | (3 << 8))
|
||||
struct.pack_into("<I", image, 0x34, 13)
|
||||
struct.pack_into("<I", image, 0x38, 1)
|
||||
activation_sources = (0,) + tuple(item[0] for item in _ACTIVATIONS)
|
||||
_require(len(activation_sources) == 13, "activation count")
|
||||
for index, source_index in enumerate(activation_sources):
|
||||
tensor = tensors[source_index]
|
||||
if index == 0:
|
||||
shape = (1, 49, 10, 1)
|
||||
scratch = 0x4000
|
||||
else:
|
||||
_, shape, scratch = _ACTIVATIONS[index - 1]
|
||||
_require(
|
||||
math.prod(tensor.shape) == math.prod(shape)
|
||||
and tensor.kind == 9
|
||||
and len(tensor.scales) == 1,
|
||||
f"tensor {index}",
|
||||
)
|
||||
struct.pack_into(
|
||||
"<HHHHIIIIiI",
|
||||
image,
|
||||
0x340 + index * 32,
|
||||
*shape,
|
||||
scratch,
|
||||
math.prod(shape),
|
||||
1,
|
||||
tensor.scale_bits,
|
||||
tensor.zero_points[0],
|
||||
0,
|
||||
)
|
||||
cursor = APUM_PARAMETER_OFFSET
|
||||
multiplier_shift = bytearray()
|
||||
source_ops = tuple(operators[index] for index in (*range(10), 11, 12))
|
||||
for index, operator in enumerate(source_ops):
|
||||
if index in (9, 11):
|
||||
kernel_h, kernel_w, stride_h, stride_w, padding = _expect_options(operator, index)
|
||||
flags = 0
|
||||
record = struct.pack(
|
||||
"<HHHHIIIIIIHHHH4BiiiiI",
|
||||
_APUM_OPCODES[index],
|
||||
flags,
|
||||
index,
|
||||
index + 1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
64 if index == 9 else 12,
|
||||
kernel_h,
|
||||
kernel_w,
|
||||
stride_h,
|
||||
stride_w,
|
||||
*padding,
|
||||
tensors[activation_sources[index]].zero_points[0],
|
||||
tensors[activation_sources[index + 1]].zero_points[0],
|
||||
-128,
|
||||
127,
|
||||
0,
|
||||
)
|
||||
else:
|
||||
cursor, packed = _weighted_record(
|
||||
image, cursor, tensors, operator, index, index, index + 1
|
||||
)
|
||||
record, multiplier_data, shift_data = (
|
||||
packed[:64],
|
||||
packed[64 : 64 + (64 if index != 10 else 12) * 4],
|
||||
packed[64 + (64 if index != 10 else 12) * 4 :],
|
||||
)
|
||||
multiplier_shift.extend(multiplier_data)
|
||||
multiplier_shift.extend(shift_data)
|
||||
image[0x40 + index * 64 : 0x40 + (index + 1) * 64] = record
|
||||
_require(cursor == APUM_PARAMETER_END, "parameter size")
|
||||
_require(
|
||||
hashlib.sha256(image[APUM_PARAMETER_OFFSET:APUM_PARAMETER_END]).hexdigest()
|
||||
== APUM_PARAMS_SHA256,
|
||||
"frozen parameter hash",
|
||||
)
|
||||
_require(
|
||||
hashlib.sha256(multiplier_shift).hexdigest() == APUM_MULTIPLIER_SHIFT_SHA256,
|
||||
"frozen multiplier/shift hash",
|
||||
)
|
||||
struct.pack_into("<I", image, 0x24, crc32_iso_hdlc(image[APUM_HEADER_BYTES:]))
|
||||
_require(struct.unpack_from("<I", image, 0x24)[0] == APUM_PAYLOAD_CRC, "frozen payload CRC")
|
||||
_require(image_sha256(image) == APUM_SHA256, "frozen APUM SHA-256")
|
||||
return bytes(image)
|
||||
|
||||
|
||||
def _git_identity(path: Path) -> dict[str, object]:
|
||||
root = path.resolve().parents[1]
|
||||
revision = subprocess.run(
|
||||
["git", "-C", str(root), "rev-parse", "HEAD"], text=True, capture_output=True, check=False
|
||||
)
|
||||
dirty = subprocess.run(
|
||||
["git", "-C", str(root), "status", "--porcelain"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
return {
|
||||
"contract_revision": "apu-kws-convert/1.0.0",
|
||||
"git_revision": revision.stdout.strip() if revision.returncode == 0 else None,
|
||||
"git_dirty": bool(dirty.stdout.strip()),
|
||||
"source_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--target", choices=("p7",), required=True)
|
||||
parser.add_argument("--model", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
image = import_tflite(args.model)
|
||||
parse_apum(image)
|
||||
except (OSError, TfliteError, ValueError) as error:
|
||||
raise SystemExit(str(error)) from error
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_bytes(image)
|
||||
manifest = {
|
||||
**_git_identity(Path(__file__)),
|
||||
"target": args.target,
|
||||
"model": str(args.model.resolve()),
|
||||
"model_sha256": TFLITE_SHA256,
|
||||
"apum_sha256": image_sha256(image),
|
||||
"payload_crc": f"0x{APUM_PAYLOAD_CRC:08x}",
|
||||
"layout": apum_layout(image),
|
||||
}
|
||||
args.manifest.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.manifest.write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(f"APUM image: {args.output} ({len(image)} bytes)")
|
||||
print(f"payload CRC: 0x{APUM_PAYLOAD_CRC:08x}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+37
-7
@@ -41,6 +41,7 @@ class Assembly:
|
||||
build_id: int
|
||||
target: str
|
||||
table_payload: bytes
|
||||
mc_abi: int
|
||||
|
||||
|
||||
def _number(token: str) -> int:
|
||||
@@ -255,7 +256,12 @@ def _encode_instruction(tokens: list[str], pc: int, labels: dict[str, int]) -> I
|
||||
return Instruction(InstructionClass.SCALAR, opcode, **values)
|
||||
|
||||
|
||||
def assemble(source: str, target: str = "p3", table_payload_input: bytes | None = None) -> Assembly:
|
||||
def assemble(
|
||||
source: str,
|
||||
target: str = "p3",
|
||||
table_payload_input: bytes | None = None,
|
||||
mc_abi: int | None = None,
|
||||
) -> Assembly:
|
||||
if target not in APUMC_TARGETS:
|
||||
raise ValueError(f"unknown APU target {target}")
|
||||
lines = []
|
||||
@@ -326,16 +332,32 @@ def assemble(source: str, target: str = "p3", table_payload_input: bytes | None
|
||||
if table_payload_input is not None
|
||||
else b"".join(struct.pack("<I", word) for word in table_words)
|
||||
)
|
||||
bundle = build_apumc(instructions, entries, build_id, target=target, table_payload=table_payload)
|
||||
return Assembly(bundle, dict(sorted(labels.items())), instructions, entries, build_id, target, table_payload)
|
||||
bundle = build_apumc(
|
||||
instructions, entries, build_id, target=target, table_payload=table_payload, abi=mc_abi
|
||||
)
|
||||
selected_abi = mc_abi if mc_abi is not None else (0x00020000 if target == "p5" else 0x00010000)
|
||||
return Assembly(
|
||||
bundle,
|
||||
dict(sorted(labels.items())),
|
||||
instructions,
|
||||
entries,
|
||||
build_id,
|
||||
target,
|
||||
table_payload,
|
||||
selected_abi,
|
||||
)
|
||||
|
||||
|
||||
def _artifact_data(assembly: Assembly) -> dict[str, object]:
|
||||
instruction_limit = 4096 if assembly.mc_abi == 0x00020000 else 2048
|
||||
return {
|
||||
"abi": "1.0",
|
||||
"abi": "2.0" if assembly.mc_abi == 0x00020000 else "1.0",
|
||||
"mc_abi": f"0x{assembly.mc_abi:08x}",
|
||||
"build_id": f"0x{assembly.build_id:016x}",
|
||||
"bundle_bytes": len(assembly.bundle),
|
||||
"instruction_count": len(assembly.instructions),
|
||||
"maximum_instruction_words": instruction_limit,
|
||||
"free_instruction_words": instruction_limit - len(assembly.instructions),
|
||||
"payload_crc32": f"0x{crc32_iso_hdlc(assembly.bundle[64:]):08x}",
|
||||
}
|
||||
|
||||
@@ -344,6 +366,7 @@ def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("source", type=Path)
|
||||
parser.add_argument("--target", choices=APUMC_TARGETS, default="p3")
|
||||
parser.add_argument("--mc-abi", choices=("1.0", "2.0"))
|
||||
parser.add_argument("-o", "--output", type=Path, required=True)
|
||||
parser.add_argument("--symbols", type=Path)
|
||||
parser.add_argument("--cfg-report", type=Path)
|
||||
@@ -353,7 +376,8 @@ def main() -> int:
|
||||
parser.add_argument("--table-binary", type=Path)
|
||||
args = parser.parse_args()
|
||||
table_payload = args.table_binary.read_bytes() if args.table_binary is not None else None
|
||||
assembly = assemble(args.source.read_text(encoding="utf-8"), args.target, table_payload)
|
||||
selected_abi = None if args.mc_abi is None else (0x00010000 if args.mc_abi == "1.0" else 0x00020000)
|
||||
assembly = assemble(args.source.read_text(encoding="utf-8"), args.target, table_payload, selected_abi)
|
||||
args.output.write_bytes(assembly.bundle)
|
||||
artifacts = {
|
||||
args.symbols: assembly.symbols,
|
||||
@@ -361,13 +385,19 @@ def main() -> int:
|
||||
**_artifact_data(assembly),
|
||||
"entries": [entry.__dict__ for entry in assembly.entries],
|
||||
"control_flow": [
|
||||
control_flow_report(assembly.instructions, entry, assembly.target)
|
||||
control_flow_report(
|
||||
assembly.instructions, entry, assembly.target, assembly.mc_abi
|
||||
)
|
||||
for entry in assembly.entries
|
||||
],
|
||||
},
|
||||
args.primitive_manifest: {
|
||||
"target": assembly.target,
|
||||
"implemented_mask": 0 if assembly.target == "p3" else 0x0000FFFF,
|
||||
"implemented_mask": {
|
||||
"p3": 0,
|
||||
"p4": 0x0000FFFF,
|
||||
"p5": 0x001FFFFF,
|
||||
}[assembly.target],
|
||||
"required_mask": __import__("functools").reduce(
|
||||
int.__or__, (entry.primitive_mask for entry in assembly.entries), 0
|
||||
),
|
||||
|
||||
@@ -14,6 +14,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from apu_mcasm import _artifact_data, assemble # noqa: E402
|
||||
from apu_isa import abi_manifest, control_flow_report # noqa: E402
|
||||
from apu_p5_coefficients import coefficient_bytes # noqa: E402
|
||||
|
||||
|
||||
@@ -33,13 +34,64 @@ def main() -> int:
|
||||
(output / "symbols.json").write_text(
|
||||
json.dumps(assembly.symbols, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
reports = {
|
||||
"cfg-report.json": {
|
||||
**_artifact_data(assembly),
|
||||
"entries": [entry.__dict__ for entry in assembly.entries],
|
||||
"control_flow": [
|
||||
control_flow_report(
|
||||
assembly.instructions,
|
||||
entry,
|
||||
assembly.target,
|
||||
assembly.mc_abi,
|
||||
)
|
||||
for entry in assembly.entries
|
||||
],
|
||||
},
|
||||
"primitive-manifest.json": {
|
||||
"target": "p5",
|
||||
"implemented_mask": 0x001FFFFF,
|
||||
"required_mask": __import__("functools").reduce(
|
||||
int.__or__, (entry.primitive_mask for entry in assembly.entries), 0
|
||||
),
|
||||
"entry_masks": [entry.primitive_mask for entry in assembly.entries],
|
||||
},
|
||||
"trace-input.json": {
|
||||
"entries": [entry.entry_pc for entry in assembly.entries],
|
||||
"instructions": [f"0x{item.encode():016x}" for item in assembly.instructions],
|
||||
},
|
||||
"abi-input-manifest.json": {
|
||||
**_artifact_data(assembly),
|
||||
"canonical_abi": abi_manifest(),
|
||||
},
|
||||
}
|
||||
for name, report in reports.items():
|
||||
(output / name).write_text(
|
||||
json.dumps(report, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
high_water = {
|
||||
"apumc_version": f"0x{assembly.mc_abi:08x}",
|
||||
"instruction_words": len(assembly.instructions),
|
||||
"maximum_instruction_words": 4096,
|
||||
"free_instruction_words": 4096 - len(assembly.instructions),
|
||||
"control_store_bytes": len(assembly.instructions) * 8,
|
||||
"table_bytes": len(coefficients),
|
||||
"maximum_scratch_end": max(
|
||||
entry.scratch_base + entry.scratch_bytes for entry in assembly.entries
|
||||
),
|
||||
"loader_proof_workspace": {
|
||||
"pending_record_bits": 64,
|
||||
"pending_path_records_v1": 2048,
|
||||
"pending_path_records_v2": 4096,
|
||||
"pending_storage_bytes_v1": 2048 * 8,
|
||||
"pending_storage_bytes_v2": 4096 * 8,
|
||||
"memo_entries": 8192,
|
||||
"memo_entry_bits": 65,
|
||||
"memo_storage_bytes_ceiling": (8192 * 65 + 7) // 8,
|
||||
"traversal_limit_v1": 131072,
|
||||
"traversal_limit_v2": 262144,
|
||||
},
|
||||
"entries": [
|
||||
{
|
||||
"format_id": entry.format_id,
|
||||
@@ -67,6 +119,12 @@ def main() -> int:
|
||||
"bundle": hashlib.sha256(assembly.bundle).hexdigest(),
|
||||
"coefficients": hashlib.sha256(coefficients).hexdigest(),
|
||||
"source": hashlib.sha256(source.read_bytes()).hexdigest(),
|
||||
"reports": {
|
||||
name: hashlib.sha256(
|
||||
(output / name).read_bytes()
|
||||
).hexdigest()
|
||||
for name in reports
|
||||
},
|
||||
},
|
||||
"high_water": high_water,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the frozen APU-P7 frontend ROM and APUM validation constants."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import struct
|
||||
from decimal import Decimal, ROUND_HALF_EVEN, getcontext
|
||||
from pathlib import Path
|
||||
|
||||
getcontext().prec = 100
|
||||
|
||||
ZERO = Decimal(0)
|
||||
ONE = Decimal(1)
|
||||
TWO = Decimal(2)
|
||||
|
||||
|
||||
def _atan(value: Decimal) -> Decimal:
|
||||
square = value * value
|
||||
term = value
|
||||
result = term
|
||||
denominator = 3
|
||||
negative = True
|
||||
while abs(term) > Decimal(1).scaleb(-105):
|
||||
term *= square
|
||||
addend = term / Decimal(denominator)
|
||||
result = result - addend if negative else result + addend
|
||||
denominator += 2
|
||||
negative = not negative
|
||||
return result
|
||||
|
||||
|
||||
PI = Decimal(16) * _atan(ONE / Decimal(5)) - Decimal(4) * _atan(ONE / Decimal(239))
|
||||
|
||||
|
||||
def _sin(value: Decimal) -> Decimal:
|
||||
value %= TWO * PI
|
||||
if value > PI:
|
||||
value -= TWO * PI
|
||||
term = value
|
||||
result = value
|
||||
index = 1
|
||||
while abs(term) > Decimal(1).scaleb(-105):
|
||||
term *= -(value * value) / Decimal((2 * index) * (2 * index + 1))
|
||||
result += term
|
||||
index += 1
|
||||
return result
|
||||
|
||||
|
||||
def _cos(value: Decimal) -> Decimal:
|
||||
return _sin(value + PI / TWO)
|
||||
|
||||
|
||||
def _rne(value: Decimal) -> int:
|
||||
return int(value.to_integral_value(rounding=ROUND_HALF_EVEN))
|
||||
|
||||
|
||||
def _hann() -> list[int]:
|
||||
return [
|
||||
_rne((ONE - _cos(TWO * PI * Decimal(index) / Decimal(480))) * Decimal(1 << 29))
|
||||
for index in range(480)
|
||||
]
|
||||
|
||||
|
||||
def _twiddle() -> tuple[list[int], list[int]]:
|
||||
real = []
|
||||
imag = []
|
||||
for index in range(256):
|
||||
angle = TWO * PI * Decimal(index) / Decimal(512)
|
||||
real.append(_rne(_cos(angle) * Decimal(1 << 30)))
|
||||
imag.append(_rne(-_sin(angle) * Decimal(1 << 30)))
|
||||
return real, imag
|
||||
|
||||
|
||||
def _mel_weights() -> list[int]:
|
||||
mel_low = Decimal(1127) * (ONE + Decimal(20) / Decimal(700)).ln()
|
||||
mel_high = Decimal(1127) * (ONE + Decimal(4000) / Decimal(700)).ln()
|
||||
edges = []
|
||||
for index in range(42):
|
||||
mel = mel_low + (mel_high - mel_low) * Decimal(index) / Decimal(41)
|
||||
edges.append(Decimal(700) * ((mel / Decimal(1127)).exp() - ONE))
|
||||
result = []
|
||||
for band in range(40):
|
||||
low, center, high = edges[band : band + 3]
|
||||
for bin_index in range(257):
|
||||
frequency = Decimal(16000 * bin_index) / Decimal(512)
|
||||
if bin_index == 0 or frequency <= low or frequency >= high:
|
||||
weight = ZERO
|
||||
elif frequency <= center:
|
||||
weight = (frequency - low) / (center - low)
|
||||
else:
|
||||
weight = (high - frequency) / (high - center)
|
||||
result.append(_rne(max(ZERO, min(ONE, weight)) * Decimal(1 << 30)))
|
||||
return result
|
||||
|
||||
|
||||
def _dct() -> list[int]:
|
||||
scale = (TWO / Decimal(40)).sqrt()
|
||||
return [
|
||||
_rne(
|
||||
scale
|
||||
* _cos(PI * (Decimal(band) + Decimal("0.5")) * Decimal(coefficient) / Decimal(40))
|
||||
* Decimal(1 << 30)
|
||||
)
|
||||
for coefficient in range(10)
|
||||
for band in range(40)
|
||||
]
|
||||
|
||||
|
||||
def _log_rom() -> list[int]:
|
||||
return [
|
||||
_rne((ONE + Decimal(index) / Decimal(1024)).ln() * Decimal(1 << 24))
|
||||
for index in range(1025)
|
||||
]
|
||||
|
||||
|
||||
def _fir(decimation: int) -> list[int]:
|
||||
coefficients = []
|
||||
cutoff = ONE / Decimal(decimation)
|
||||
for tap in range(63):
|
||||
offset = Decimal(tap - 31)
|
||||
window = (ONE - _cos(TWO * PI * Decimal(tap) / Decimal(62))) / TWO
|
||||
sinc = ONE if offset == ZERO else _sin(PI * cutoff * offset) / (PI * cutoff * offset)
|
||||
coefficients.append(cutoff * sinc * window)
|
||||
total = sum(coefficients)
|
||||
quantized = [_rne(value / total * Decimal(1 << 30)) for value in coefficients]
|
||||
quantized[31] += (1 << 30) - sum(quantized)
|
||||
return quantized
|
||||
|
||||
|
||||
def _trunc_div(numerator: int, denominator: int) -> int:
|
||||
return numerator // denominator if numerator >= 0 else -((-numerator) // denominator)
|
||||
|
||||
|
||||
def _srdhm(a: int, b: int) -> int:
|
||||
if a == -(1 << 31) and b == -(1 << 31):
|
||||
return (1 << 31) - 1
|
||||
product = a * b
|
||||
nudge = (1 << 30) if product >= 0 else 1 - (1 << 30)
|
||||
return _trunc_div(product + nudge, 1 << 31)
|
||||
|
||||
|
||||
def _rdbpot(value: int, exponent: int) -> int:
|
||||
if exponent == 0:
|
||||
return value
|
||||
mask = (1 << exponent) - 1
|
||||
remainder = value & mask
|
||||
threshold = (mask >> 1) + (1 if value < 0 else 0)
|
||||
return (value >> exponent) + (1 if remainder > threshold else 0)
|
||||
|
||||
|
||||
def _sat_left(value: int, exponent: int) -> int:
|
||||
result = value << exponent
|
||||
return max(-(1 << 31), min((1 << 31) - 1, result))
|
||||
|
||||
|
||||
def _exp_interval(raw: int) -> int:
|
||||
x = raw + (1 << 28)
|
||||
x2 = _srdhm(x, x)
|
||||
x3 = _srdhm(x2, x)
|
||||
x4 = _srdhm(x2, x2)
|
||||
x4_over_4 = _rdbpot(x4, 2)
|
||||
series = _rdbpot(_srdhm(x4_over_4 + x3, 715827883) + x2, 1)
|
||||
return 1895147668 + _srdhm(1895147668, x + series)
|
||||
|
||||
|
||||
def _exp_negative_q5_26(raw: int) -> int:
|
||||
quarter = 1 << 24
|
||||
mask = quarter - 1
|
||||
reduced = (raw & mask) - quarter
|
||||
result = _exp_interval(_sat_left(reduced, 5))
|
||||
remainder = reduced - raw
|
||||
for exponent, multiplier in (
|
||||
(-2, 1672461947),
|
||||
(-1, 1302514674),
|
||||
(0, 790015084),
|
||||
(1, 290630308),
|
||||
(2, 39332535),
|
||||
(3, 720401),
|
||||
(4, 242),
|
||||
):
|
||||
bit = 26 + exponent
|
||||
if (remainder & (1 << bit)) != 0:
|
||||
result = _srdhm(result, multiplier)
|
||||
return (1 << 31) - 1 if raw == 0 else result
|
||||
|
||||
|
||||
def _softmax_exp() -> list[int]:
|
||||
values = []
|
||||
for difference in range(125):
|
||||
scaled = _srdhm((-difference) << 24, 1242899200)
|
||||
values.append(_exp_negative_q5_26(scaled))
|
||||
return values
|
||||
|
||||
|
||||
def _emit_function(name: str, width: int, values: list[int], *, signed: bool = True) -> str:
|
||||
index_width = max(1, (len(values) - 1).bit_length())
|
||||
qualifier = " signed" if signed else ""
|
||||
lines = [
|
||||
f"function automatic logic{qualifier} [{width - 1}:0] {name}(",
|
||||
f" input logic [{index_width - 1}:0] index_i);",
|
||||
" case (index_i)",
|
||||
]
|
||||
for index, value in enumerate(values):
|
||||
if signed:
|
||||
literal = f"{width}'sd{value}" if value >= 0 else f"-{width}'sd{-value}"
|
||||
else:
|
||||
literal = f"{width}'d{value}"
|
||||
lines.append(f" {index_width}'d{index}: {name} = {literal};")
|
||||
lines.extend((f" default: {name} = '0;", " endcase", "endfunction", ""))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _fixed_apum_words(image: bytes) -> tuple[list[int], list[int]]:
|
||||
offsets = list(range(0, 0x500, 4))
|
||||
for operator in range(12):
|
||||
record = 0x40 + operator * 64
|
||||
for field in (0x10, 0x14):
|
||||
offset = struct.unpack_from("<I", image, record + field)[0]
|
||||
channels = struct.unpack_from("<I", image, record + 0x1C)[0]
|
||||
if offset != 0:
|
||||
offsets.extend(range(offset, offset + channels * 4, 4))
|
||||
offsets = sorted(set(offsets))
|
||||
return offsets, [struct.unpack_from("<I", image, offset)[0] for offset in offsets]
|
||||
|
||||
|
||||
def _emit_profile_words(offsets: list[int], values: list[int]) -> str:
|
||||
ranges: list[tuple[int, int, int]] = []
|
||||
range_start = offsets[0]
|
||||
range_index = 0
|
||||
previous = offsets[0]
|
||||
for index, offset in enumerate(offsets[1:], start=1):
|
||||
if offset != previous + 4:
|
||||
ranges.append((range_start, previous, range_index))
|
||||
range_start = offset
|
||||
range_index = index
|
||||
previous = offset
|
||||
ranges.append((range_start, previous, range_index))
|
||||
|
||||
lines = [
|
||||
"function automatic logic [31:0] apu_kws_apum_fixed_word(",
|
||||
" input logic [10:0] index_i);",
|
||||
" case (index_i)",
|
||||
]
|
||||
for index, value in enumerate(values):
|
||||
lines.append(f" 11'd{index}: apu_kws_apum_fixed_word = 32'h{value:08x};")
|
||||
lines.extend(
|
||||
(
|
||||
" default: apu_kws_apum_fixed_word = 32'd0;",
|
||||
" endcase",
|
||||
"endfunction",
|
||||
"",
|
||||
"function automatic logic [1:0] apu_kws_apum_fixed_word_check(",
|
||||
" input logic [14:0] offset_i, input logic [31:0] data_i);",
|
||||
" logic valid;",
|
||||
" logic [10:0] index;",
|
||||
" begin",
|
||||
" valid = 1'b0;",
|
||||
" index = 11'd0;",
|
||||
)
|
||||
)
|
||||
for start, end, index in ranges:
|
||||
keyword = "if" if index == 0 else "else if"
|
||||
lower_bound = "" if start == 0 else f"(offset_i >= 15'h{start:04x}) && "
|
||||
lines.extend(
|
||||
(
|
||||
f" {keyword} ({lower_bound}(offset_i <= 15'h{end:04x}) && "
|
||||
"(offset_i[1:0] == 2'd0)) begin",
|
||||
" valid = 1'b1;",
|
||||
f" index = 11'd{index} + "
|
||||
f"11'((offset_i - 15'h{start:04x}) >> 2);",
|
||||
" end",
|
||||
)
|
||||
)
|
||||
lines.extend(
|
||||
(
|
||||
" apu_kws_apum_fixed_word_check =",
|
||||
" valid ? {1'b1, data_i == apu_kws_apum_fixed_word(index)} : 2'b00;",
|
||||
" end",
|
||||
"endfunction",
|
||||
"",
|
||||
)
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate(apum: Path, output: Path, profile_output: Path) -> None:
|
||||
image = apum.read_bytes()
|
||||
if len(image) != 32768:
|
||||
raise ValueError("APUM image must be exactly 32768 bytes")
|
||||
fixed_offsets, fixed_words = _fixed_apum_words(image)
|
||||
text = [
|
||||
"// Copyright (c) 2026 Yuchi Miao <miaoyuchi@ict.ac.cn>",
|
||||
"// SPDX-License-Identifier: MulanPSL-2.0",
|
||||
"",
|
||||
"// Generated by scripts/generate_apu_kws_rtl_constants.py.",
|
||||
"// Do not edit: regenerate from the frozen APUM and numerical profile.",
|
||||
"",
|
||||
_emit_function("apu_kws_hann_q30", 32, _hann()),
|
||||
]
|
||||
twiddle_real, twiddle_imag = _twiddle()
|
||||
text.extend(
|
||||
(
|
||||
_emit_function("apu_kws_twiddle_real_q30", 32, twiddle_real),
|
||||
_emit_function("apu_kws_twiddle_imag_q30", 32, twiddle_imag),
|
||||
_emit_function("apu_kws_mel_q30", 32, _mel_weights(), signed=False),
|
||||
_emit_function("apu_kws_dct_q30", 32, _dct()),
|
||||
_emit_function("apu_kws_log_q24", 32, _log_rom()),
|
||||
_emit_function("apu_kws_fir3_q30", 32, _fir(3)),
|
||||
_emit_function("apu_kws_fir6_q30", 32, _fir(6)),
|
||||
_emit_function("apu_kws_softmax_exp_q31", 32, _softmax_exp()),
|
||||
f"localparam logic signed [31:0] ApuKwsLn2Q24 = 32'sd{_rne(TWO.ln() * Decimal(1 << 24))};",
|
||||
"",
|
||||
)
|
||||
)
|
||||
output.write_text("\n".join(text), encoding="ascii")
|
||||
profile_output.write_text(
|
||||
"\n".join(
|
||||
(
|
||||
"// Copyright (c) 2026 Yuchi Miao <miaoyuchi@ict.ac.cn>",
|
||||
"// SPDX-License-Identifier: MulanPSL-2.0",
|
||||
"",
|
||||
"// Generated by scripts/generate_apu_kws_rtl_constants.py.",
|
||||
"// Do not edit: regenerate from the frozen APUM.",
|
||||
"",
|
||||
_emit_profile_words(fixed_offsets, fixed_words),
|
||||
)
|
||||
),
|
||||
encoding="ascii",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--apum", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--profile-output", type=Path)
|
||||
args = parser.parse_args()
|
||||
profile_output = args.profile_output
|
||||
if profile_output is None:
|
||||
profile_output = args.output.with_name("apu_kws_apum_profile.svh")
|
||||
generate(args.apum, args.output, profile_output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+3777
File diff suppressed because it is too large.
Load diff
@@ -14,7 +14,7 @@ from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from apu_codecs import CodecError, decode_flac # noqa: E402
|
||||
from apu_codecs import CodecError, decode_flac, process_pcm # noqa: E402
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
@@ -53,8 +53,21 @@ def _classify(flac: Path, corpus: Path, path: Path) -> dict[str, object]:
|
||||
reason = 0
|
||||
geometry: dict[str, int | None] | None = None
|
||||
model_pcm_hash: str | None = None
|
||||
production_pcm_hash = hashlib.sha256(b"").hexdigest()
|
||||
production_input_used = 0
|
||||
production_output_bytes = 0
|
||||
production_frames = 0
|
||||
production_source_info = 0
|
||||
model_error: CodecError | None = None
|
||||
try:
|
||||
decoded = decode_flac(payload, strict=True)
|
||||
except CodecError as error:
|
||||
profile = "unsupported" if error.code == 3 else "malformed"
|
||||
reason = error.reason
|
||||
model_error = error
|
||||
decoded = error.partial_audio
|
||||
|
||||
if decoded is not None:
|
||||
info = decoded.info
|
||||
geometry = {
|
||||
"rate": info.rate,
|
||||
@@ -68,9 +81,34 @@ def _classify(flac: Path, corpus: Path, path: Path) -> dict[str, object]:
|
||||
for sample in frame:
|
||||
pcm.update(sample.to_bytes(sample_bytes, "little", signed=True))
|
||||
model_pcm_hash = pcm.hexdigest()
|
||||
except CodecError as error:
|
||||
profile = "unsupported" if error.code == 3 else "malformed"
|
||||
reason = error.reason
|
||||
production_pcm = process_pcm(
|
||||
decoded,
|
||||
output_rate=info.rate,
|
||||
output_channels=info.channels,
|
||||
output_bits=info.bits,
|
||||
)
|
||||
production_pcm_hash = hashlib.sha256(production_pcm.payload).hexdigest()
|
||||
production_input_used = info.input_used
|
||||
production_output_bytes = len(production_pcm.payload)
|
||||
production_frames = production_pcm.frames
|
||||
production_source_info = info.rate | (info.channels << 17) | (info.bits << 19)
|
||||
|
||||
if model_error is not None:
|
||||
production_input_used = model_error.input_used
|
||||
production_truth = {
|
||||
"status": "success" if model_error is None else "error",
|
||||
"code": 0 if model_error is None else model_error.code,
|
||||
"stage": 0 if model_error is None else model_error.stage,
|
||||
"warnings": decoded.info.warnings if model_error is None else (model_error.warnings | 1),
|
||||
"reason": 0 if model_error is None else model_error.reason,
|
||||
"error_offset": 0 if model_error is None else model_error.offset,
|
||||
"input_used": production_input_used,
|
||||
"output_bytes": production_output_bytes,
|
||||
"frames": production_frames,
|
||||
"source_info": production_source_info,
|
||||
"pcm_sha256": production_pcm_hash,
|
||||
"input_used_policy": "exact",
|
||||
}
|
||||
|
||||
model_result = profile
|
||||
return_code, pcm_hash, stderr = _decode_hash(flac, path)
|
||||
@@ -97,6 +135,7 @@ def _classify(flac: Path, corpus: Path, path: Path) -> dict[str, object]:
|
||||
"geometry": geometry,
|
||||
"reference_pcm_sha256": pcm_hash,
|
||||
"model_pcm_sha256": model_pcm_hash,
|
||||
"production_truth": production_truth,
|
||||
"reference_integrity": "md5_mismatch"
|
||||
if md5_mismatch
|
||||
else ("passed" if return_code == 0 else "decode_failed"),
|
||||
|
||||
@@ -0,0 +1,673 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the pinned APU-P5 FLAC corpus through identical production RTL fixtures."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
INPUT_BASE = 0x40000000
|
||||
EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
|
||||
MIGRATABLE_VERIFICATION_SHA256 = {
|
||||
"ff529cb4ae848f1dfe05e0f2b9b3d9dc93f2d98e9810e0c48a769b38e85d21dd",
|
||||
"5bd8606e74da174cb235eb0fc56b4d5e95a7bf6ab2f0218d466467c052b25ba1",
|
||||
}
|
||||
RESULT_PATTERN = re.compile(
|
||||
r"APU_P5_CORPUS_RESULT "
|
||||
r"status=(?P<status>[0-9a-fA-F]{8}) "
|
||||
r"input_used=(?P<input_used>[0-9a-fA-F]{8}) "
|
||||
r"output_bytes=(?P<output_bytes>[0-9a-fA-F]{8}) "
|
||||
r"frames=(?P<frames>[0-9a-fA-F]{8}) "
|
||||
r"source_info=(?P<source_info>[0-9a-fA-F]{8}) "
|
||||
r"cycles=(?P<cycles>[0-9a-fA-F]{8}) "
|
||||
r"detail=(?P<detail>[0-9a-fA-F]{8}) "
|
||||
r"error_status=(?P<error_status>[0-9a-fA-F]{8}) "
|
||||
r"error_address=(?P<error_address>[0-9a-fA-F]{8}) "
|
||||
r"error_detail=(?P<error_detail>[0-9a-fA-F]{8}) "
|
||||
r"elapsed=(?P<elapsed>[0-9]+)"
|
||||
)
|
||||
RESULT_FIELDS = (
|
||||
"status",
|
||||
"input_used",
|
||||
"output_bytes",
|
||||
"frames",
|
||||
"source_info",
|
||||
"cycles",
|
||||
"detail",
|
||||
"error_status",
|
||||
"error_address",
|
||||
"error_detail",
|
||||
)
|
||||
|
||||
|
||||
def _subprocess_text(value: str | bytes | None) -> str:
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", errors="replace")
|
||||
return value or ""
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for block in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _write_json(path: Path, value: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def _run_command(command: list[str], *, cwd: Path, timeout: int | None = None) -> dict[str, Any]:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
process = subprocess.run(
|
||||
command,
|
||||
cwd=cwd,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
exit_code = process.returncode
|
||||
stdout = process.stdout
|
||||
stderr = process.stderr
|
||||
timed_out = False
|
||||
except subprocess.TimeoutExpired as error:
|
||||
exit_code = 124
|
||||
stdout = _subprocess_text(error.stdout)
|
||||
stderr = _subprocess_text(error.stderr)
|
||||
timed_out = True
|
||||
return {
|
||||
"command": command,
|
||||
"duration_seconds": round(time.monotonic() - started, 3),
|
||||
"exit_code": exit_code,
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
"timed_out": timed_out,
|
||||
}
|
||||
|
||||
|
||||
def _production_sources() -> list[Path]:
|
||||
common = ROOT / "rtl/managed/clusterip/common/rtl"
|
||||
multimedia = ROOT / "rtl/ip/multimedia"
|
||||
names = (
|
||||
"apu_microcode_pkg.sv",
|
||||
"apu_dma.sv",
|
||||
"apu_ring_scheduler.sv",
|
||||
"apu_stream_router.sv",
|
||||
"apu_control_store.sv",
|
||||
"apu_microcode_loader.sv",
|
||||
"apu_local_sram.sv",
|
||||
"apu_kws_engine.sv",
|
||||
"apu_kws_sram_client.sv",
|
||||
"apu_kws_model_loader.sv",
|
||||
"apu_bitstream_engine.sv",
|
||||
"apu_entropy_engine.sv",
|
||||
"apu_reconstruction_engine.sv",
|
||||
"apu_transform_engine.sv",
|
||||
"apu_resampler.sv",
|
||||
"apu_kernel_engine.sv",
|
||||
"apu_primitive_dispatcher.sv",
|
||||
"apu_codec_sequencer.sv",
|
||||
"apu_codec_transport.sv",
|
||||
"apu_codec_controller.sv",
|
||||
"apu_reg.sv",
|
||||
"apb4_apu.sv",
|
||||
)
|
||||
return [
|
||||
common / "interface/apb4_if.sv",
|
||||
common / "interface/axi4_if.sv",
|
||||
common / "interface/axi4_stream_if.sv",
|
||||
common / "utils/register.sv",
|
||||
common / "utils/fifo.sv",
|
||||
ROOT / "rtl/tech/tc_sram.sv",
|
||||
ROOT / "rtl/ip/peripheral/dma_axi4_master.sv",
|
||||
*(multimedia / name for name in names),
|
||||
ROOT / "tests/rtl/apu_p5_corpus_tb.sv",
|
||||
]
|
||||
|
||||
|
||||
def _verification_sha256() -> str:
|
||||
inputs = [
|
||||
*_production_sources(),
|
||||
ROOT / "rtl/ip/multimedia/apu_define.svh",
|
||||
Path(__file__).resolve(),
|
||||
]
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(inputs):
|
||||
digest.update(path.relative_to(ROOT).as_posix().encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(path.read_bytes())
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _truth_sha256(record: dict[str, Any]) -> str:
|
||||
"""Hash the canonical BAM/libFLAC truth fields for one corpus record."""
|
||||
truth_record = dict(record)
|
||||
truth_record.pop("production_rtl", None)
|
||||
canonical = json.dumps(truth_record, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.sha256(canonical).hexdigest()
|
||||
|
||||
|
||||
def _compile(build_dir: Path, bundle: Path) -> tuple[dict[str, Path], dict[str, Any]]:
|
||||
tools = {name: shutil.which(name) for name in ("iverilog", "vvp", "verilator")}
|
||||
missing = [name for name, path in tools.items() if path is None]
|
||||
if missing:
|
||||
raise RuntimeError(f"missing required APU-P5 corpus tools: {', '.join(missing)}")
|
||||
build_dir.mkdir(parents=True, exist_ok=True)
|
||||
multimedia = ROOT / "rtl/ip/multimedia"
|
||||
common = ROOT / "rtl/managed/clusterip/common/rtl"
|
||||
filelist = build_dir / "apu_p5_corpus.fl"
|
||||
filelist.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"+define+SV_ASSRT_DISABLE",
|
||||
f"+incdir+{common}",
|
||||
f"+incdir+{common / 'interface'}",
|
||||
f"+incdir+{multimedia}",
|
||||
*(str(source) for source in _production_sources()),
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
converted = build_dir / "apu_p5_corpus.v"
|
||||
conversion = _run_command(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "rtl/mini/script/convt_sv2v.py"),
|
||||
"-f",
|
||||
str(filelist),
|
||||
"--output",
|
||||
str(converted),
|
||||
],
|
||||
cwd=ROOT,
|
||||
)
|
||||
if conversion["exit_code"] != 0:
|
||||
raise RuntimeError(f"APU-P5 corpus sv2v failed: {conversion['stderr']}")
|
||||
|
||||
image_hex = build_dir / "apu-p5.hex"
|
||||
payload = bundle.read_bytes()
|
||||
image_hex.write_text(
|
||||
"".join(
|
||||
f"{int.from_bytes(payload[offset : offset + 4].ljust(4, bytes(1)), 'little'):08x}\n"
|
||||
for offset in range(0, 16384 * 4, 4)
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
icarus_binary = build_dir / "apu_p5_corpus_icarus"
|
||||
icarus = _run_command(
|
||||
[
|
||||
str(tools["iverilog"]),
|
||||
"-g2012",
|
||||
"-s",
|
||||
"apu_p5_corpus_tb",
|
||||
"-o",
|
||||
str(icarus_binary),
|
||||
str(converted),
|
||||
],
|
||||
cwd=ROOT,
|
||||
)
|
||||
if icarus["exit_code"] != 0:
|
||||
raise RuntimeError(f"APU-P5 corpus Icarus compile failed: {icarus['stderr']}")
|
||||
|
||||
verilator_dir = build_dir / "verilator"
|
||||
ccache_dir = build_dir / "ccache"
|
||||
ccache_dir.mkdir(exist_ok=True)
|
||||
environment = {
|
||||
**os.environ,
|
||||
"CCACHE_DIR": str(ccache_dir),
|
||||
"CCACHE_TEMPDIR": str(ccache_dir),
|
||||
}
|
||||
started = time.monotonic()
|
||||
verilator = subprocess.run(
|
||||
[
|
||||
str(tools["verilator"]),
|
||||
"--binary",
|
||||
"--timing",
|
||||
"-O3",
|
||||
"-Wno-fatal",
|
||||
"--top-module",
|
||||
"apu_p5_corpus_tb",
|
||||
"--Mdir",
|
||||
str(verilator_dir),
|
||||
str(converted),
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
)
|
||||
verilator_result = {
|
||||
"command": verilator.args,
|
||||
"duration_seconds": round(time.monotonic() - started, 3),
|
||||
"exit_code": verilator.returncode,
|
||||
"stdout": verilator.stdout,
|
||||
"stderr": verilator.stderr,
|
||||
}
|
||||
if verilator.returncode != 0:
|
||||
raise RuntimeError(f"APU-P5 corpus Verilator compile failed: {verilator.stderr}")
|
||||
compile_result = {
|
||||
"schema_version": 1,
|
||||
"status": "passed",
|
||||
"bundle_sha256": _sha256_file(bundle),
|
||||
"verification_sha256": _verification_sha256(),
|
||||
"fixture": "tests/rtl/apu_p5_corpus_tb.sv",
|
||||
"production_sources": [str(path.relative_to(ROOT)) for path in _production_sources()],
|
||||
"sv2v": conversion,
|
||||
"icarus": icarus,
|
||||
"verilator": verilator_result,
|
||||
}
|
||||
_write_json(build_dir / "result-compile.json", compile_result)
|
||||
return {
|
||||
"image": image_hex,
|
||||
"vvp": Path(str(tools["vvp"])),
|
||||
"icarus": icarus_binary,
|
||||
"verilator": verilator_dir / "Vapu_p5_corpus_tb",
|
||||
}, compile_result
|
||||
|
||||
|
||||
def _case_id(index: int, record: dict[str, Any]) -> str:
|
||||
digest = record["sha256"][:12]
|
||||
return f"{index:03d}-{digest}"
|
||||
|
||||
|
||||
def _can_reuse_result(
|
||||
cached: dict[str, Any],
|
||||
source_sha256: str,
|
||||
bundle_sha256: str,
|
||||
verification_sha256: str,
|
||||
truth_sha256: str,
|
||||
) -> bool:
|
||||
return (
|
||||
cached.get("status") == "passed"
|
||||
and cached.get("source_sha256") == source_sha256
|
||||
and cached.get("bundle_sha256") == bundle_sha256
|
||||
and cached.get("verification_sha256") == verification_sha256
|
||||
and cached.get("truth_sha256") == truth_sha256
|
||||
)
|
||||
|
||||
|
||||
def _can_migrate_pass_result(
|
||||
cached: dict[str, Any],
|
||||
source_sha256: str,
|
||||
bundle_sha256: str,
|
||||
truth_sha256: str,
|
||||
) -> bool:
|
||||
"""Reuse a pass from the pre-WALLCLOCK_NS fixture after a harness-only fix."""
|
||||
command = cached.get("command", [])
|
||||
return (
|
||||
cached.get("status") == "passed"
|
||||
and cached.get("source_sha256") == source_sha256
|
||||
and cached.get("bundle_sha256") == bundle_sha256
|
||||
and cached.get("verification_sha256") in MIGRATABLE_VERIFICATION_SHA256
|
||||
and cached.get("truth_sha256") == truth_sha256
|
||||
and isinstance(command, list)
|
||||
and "+WALLCLOCK_NS=0" not in command
|
||||
)
|
||||
|
||||
|
||||
def _case_parameters(record: dict[str, Any], source: Path) -> dict[str, int]:
|
||||
truth = record["production_truth"]
|
||||
geometry = record.get("geometry")
|
||||
bits = int(geometry["bits"]) if geometry is not None else 16
|
||||
pcm_format = 1 if bits == 24 else 0
|
||||
samples = int(geometry.get("samples") or 0) if geometry is not None else 0
|
||||
channels = int(geometry["channels"]) if geometry is not None else 2
|
||||
potential_output = samples * channels * (4 if bits == 24 else 2)
|
||||
fallback_output = min(0x0FFFFFFC, max(4, source.stat().st_size * 16))
|
||||
output_capacity = max(4, int(truth["output_bytes"]), potential_output, fallback_output)
|
||||
output_capacity = min(0x0FFFFFFC, (output_capacity + 3) & ~3)
|
||||
estimated_cycles = max(
|
||||
5_000_000,
|
||||
samples * 5000,
|
||||
source.stat().st_size * 200,
|
||||
)
|
||||
return {
|
||||
"input_bytes": source.stat().st_size,
|
||||
"input_config": 0,
|
||||
"output_config": pcm_format << 19,
|
||||
"output_capacity": output_capacity,
|
||||
"max_cycles": min(1_900_000_000, estimated_cycles),
|
||||
}
|
||||
|
||||
|
||||
def _parse_result(stdout: str) -> dict[str, int]:
|
||||
match = RESULT_PATTERN.search(stdout)
|
||||
if match is None:
|
||||
raise RuntimeError("simulator did not emit APU_P5_CORPUS_RESULT")
|
||||
result = {name: int(match.group(name), 16) for name in RESULT_FIELDS}
|
||||
result["elapsed"] = int(match.group("elapsed"), 10)
|
||||
return result
|
||||
|
||||
|
||||
def _record_execution_failure(
|
||||
*,
|
||||
simulator: str,
|
||||
source_sha256: str,
|
||||
bundle_sha256: str,
|
||||
verification_sha256: str,
|
||||
truth_sha256: str,
|
||||
command: list[str],
|
||||
timeout: int,
|
||||
max_cycles: int,
|
||||
log_path: Path,
|
||||
duration_seconds: float,
|
||||
reason: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"status": "failed",
|
||||
"simulator": simulator,
|
||||
"source_sha256": source_sha256,
|
||||
"bundle_sha256": bundle_sha256,
|
||||
"verification_sha256": verification_sha256,
|
||||
"truth_sha256": truth_sha256,
|
||||
"command": command,
|
||||
"timeout_seconds": timeout,
|
||||
"max_cycles": max_cycles,
|
||||
"duration_seconds": duration_seconds,
|
||||
"log": str(log_path),
|
||||
"execution_errors": [reason],
|
||||
}
|
||||
|
||||
|
||||
def _check_truth(record: dict[str, Any], result: dict[str, int], pcm_sha256: str) -> None:
|
||||
truth = record["production_truth"]
|
||||
success = truth["status"] == "success"
|
||||
if bool(result["status"] & (1 << 1)) != success:
|
||||
raise RuntimeError("JOB_STATUS.DONE does not match BAM")
|
||||
if bool(result["status"] & (1 << 2)) == success:
|
||||
raise RuntimeError("JOB_STATUS.ERROR does not match BAM")
|
||||
expected_code = int(truth["code"])
|
||||
expected_stage = int(truth["stage"])
|
||||
expected_detail = (2 << 24) | (int(truth["warnings"]) << 16) | int(truth["reason"])
|
||||
if ((result["status"] >> 8) & 0x3F) != expected_code:
|
||||
raise RuntimeError("JOB_STATUS error code does not match BAM")
|
||||
if ((result["status"] >> 14) & 0xF) != expected_stage:
|
||||
raise RuntimeError("JOB_STATUS stage does not match BAM")
|
||||
if result["detail"] != expected_detail:
|
||||
raise RuntimeError("JOB_DETAIL does not match BAM")
|
||||
for field in ("input_used", "output_bytes", "frames", "source_info"):
|
||||
if truth[field] is not None and result[field] != int(truth[field]):
|
||||
raise RuntimeError(f"{field} does not match BAM")
|
||||
if pcm_sha256 != truth["pcm_sha256"]:
|
||||
raise RuntimeError("PCM SHA-256 does not match BAM")
|
||||
if success:
|
||||
if result["error_status"] & 1:
|
||||
raise RuntimeError("successful corpus job unexpectedly set first error")
|
||||
else:
|
||||
if not result["error_status"] & 1:
|
||||
raise RuntimeError("failed corpus job did not set first error")
|
||||
if ((result["error_status"] >> 1) & 0x3F) != expected_code:
|
||||
raise RuntimeError("ERROR_STATUS code does not match BAM")
|
||||
if ((result["error_status"] >> 7) & 0xF) != expected_stage:
|
||||
raise RuntimeError("ERROR_STATUS stage does not match BAM")
|
||||
if result["error_address"] != INPUT_BASE + int(truth["error_offset"]):
|
||||
raise RuntimeError("ERROR_ADDRESS does not match BAM")
|
||||
if result["error_detail"] != expected_detail:
|
||||
raise RuntimeError("ERROR_DETAIL does not match BAM")
|
||||
|
||||
|
||||
def _run_case(
|
||||
simulator: str,
|
||||
executable: Path,
|
||||
vvp: Path,
|
||||
image: Path,
|
||||
source: Path,
|
||||
record: dict[str, Any],
|
||||
case_dir: Path,
|
||||
timeout: int,
|
||||
bundle_sha256: str,
|
||||
verification_sha256: str,
|
||||
) -> dict[str, Any]:
|
||||
result_path = case_dir / f"result-{simulator}.json"
|
||||
source_sha256 = record["sha256"]
|
||||
truth_sha256 = _truth_sha256(record)
|
||||
if result_path.is_file():
|
||||
cached = json.loads(result_path.read_text(encoding="utf-8"))
|
||||
if _can_reuse_result(
|
||||
cached, source_sha256, bundle_sha256, verification_sha256, truth_sha256
|
||||
):
|
||||
return cached
|
||||
if _can_migrate_pass_result(cached, source_sha256, bundle_sha256, truth_sha256):
|
||||
cached["migrated_from_verification_sha256"] = cached["verification_sha256"]
|
||||
cached["migration_reason"] = "wallclock-and-sequential-output-harness-only"
|
||||
cached["verification_sha256"] = verification_sha256
|
||||
cached["truth_sha256"] = truth_sha256
|
||||
_write_json(result_path, cached)
|
||||
return cached
|
||||
parameters = _case_parameters(record, source)
|
||||
output = case_dir / f"output-{simulator}.pcm"
|
||||
command = ([str(vvp), str(executable)] if simulator == "icarus" else [str(executable)]) + [
|
||||
f"+IMAGE={image}",
|
||||
f"+INPUT={source}",
|
||||
f"+OUTPUT={output}",
|
||||
f"+INPUT_BYTES={parameters['input_bytes']}",
|
||||
f"+INPUT_CONFIG={parameters['input_config']:x}",
|
||||
f"+OUTPUT_CONFIG={parameters['output_config']:x}",
|
||||
f"+OUTPUT_CAPACITY={parameters['output_capacity']}",
|
||||
f"+MAX_CYCLES={parameters['max_cycles']}",
|
||||
"+WALLCLOCK_NS=0",
|
||||
]
|
||||
execution = _run_command(command, cwd=ROOT, timeout=timeout)
|
||||
log_path = case_dir / f"{simulator}.log"
|
||||
log_path.write_text(execution["stdout"] + execution["stderr"], encoding="utf-8")
|
||||
if execution["exit_code"] != 0:
|
||||
result = _record_execution_failure(
|
||||
simulator=simulator,
|
||||
source_sha256=source_sha256,
|
||||
bundle_sha256=bundle_sha256,
|
||||
verification_sha256=verification_sha256,
|
||||
truth_sha256=truth_sha256,
|
||||
command=command,
|
||||
timeout=timeout,
|
||||
max_cycles=parameters["max_cycles"],
|
||||
log_path=log_path,
|
||||
duration_seconds=execution["duration_seconds"],
|
||||
reason="simulator timeout" if execution["timed_out"] else "simulator failed",
|
||||
)
|
||||
_write_json(result_path, result)
|
||||
return result
|
||||
try:
|
||||
parsed = _parse_result(execution["stdout"])
|
||||
except RuntimeError as error:
|
||||
result = _record_execution_failure(
|
||||
simulator=simulator,
|
||||
source_sha256=source_sha256,
|
||||
bundle_sha256=bundle_sha256,
|
||||
verification_sha256=verification_sha256,
|
||||
truth_sha256=truth_sha256,
|
||||
command=command,
|
||||
timeout=timeout,
|
||||
max_cycles=parameters["max_cycles"],
|
||||
log_path=log_path,
|
||||
duration_seconds=execution["duration_seconds"],
|
||||
reason=str(error),
|
||||
)
|
||||
_write_json(result_path, result)
|
||||
return result
|
||||
pcm_sha256 = _sha256_file(output) if output.is_file() else EMPTY_SHA256
|
||||
output_size = output.stat().st_size if output.is_file() else 0
|
||||
if output_size != parsed["output_bytes"]:
|
||||
result = _record_execution_failure(
|
||||
simulator=simulator,
|
||||
source_sha256=source_sha256,
|
||||
bundle_sha256=bundle_sha256,
|
||||
verification_sha256=verification_sha256,
|
||||
truth_sha256=truth_sha256,
|
||||
command=command,
|
||||
timeout=timeout,
|
||||
max_cycles=parameters["max_cycles"],
|
||||
log_path=log_path,
|
||||
duration_seconds=execution["duration_seconds"],
|
||||
reason=(
|
||||
f"{simulator} output size {output_size} does not match "
|
||||
f"RESULT_OUTPUT_BYTES {parsed['output_bytes']}"
|
||||
),
|
||||
)
|
||||
if output.is_file():
|
||||
output.unlink()
|
||||
_write_json(result_path, result)
|
||||
return result
|
||||
truth_errors: list[str] = []
|
||||
try:
|
||||
_check_truth(record, parsed, pcm_sha256)
|
||||
except RuntimeError as error:
|
||||
truth_errors.append(str(error))
|
||||
if output.is_file():
|
||||
output.unlink()
|
||||
result = {
|
||||
"schema_version": 1,
|
||||
"status": "passed" if not truth_errors else "failed",
|
||||
"simulator": simulator,
|
||||
"source_sha256": source_sha256,
|
||||
"bundle_sha256": bundle_sha256,
|
||||
"verification_sha256": verification_sha256,
|
||||
"truth_sha256": truth_sha256,
|
||||
"command": command,
|
||||
"timeout_seconds": timeout,
|
||||
"max_cycles": parameters["max_cycles"],
|
||||
"duration_seconds": execution["duration_seconds"],
|
||||
"log": str(log_path),
|
||||
"pcm_sha256": pcm_sha256,
|
||||
"result": parsed,
|
||||
"truth_errors": truth_errors,
|
||||
}
|
||||
_write_json(result_path, result)
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--bundle", type=Path, required=True)
|
||||
parser.add_argument("--corpus", type=Path, required=True)
|
||||
parser.add_argument("--build-dir", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--case", action="append", default=[])
|
||||
parser.add_argument(
|
||||
"--category", action="append", choices=("supported", "unsupported", "malformed"), default=[]
|
||||
)
|
||||
parser.add_argument("--jobs", type=int, default=1)
|
||||
parser.add_argument("--timeout-seconds", type=int, default=14400)
|
||||
parser.add_argument("--allow-partial", action="store_true")
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads(args.manifest.resolve().read_text(encoding="utf-8"))
|
||||
bundle = args.bundle.resolve()
|
||||
corpus = args.corpus.resolve()
|
||||
build_dir = args.build_dir.resolve()
|
||||
output = args.output.resolve()
|
||||
if args.jobs < 1:
|
||||
raise SystemExit("--jobs must be positive")
|
||||
records = manifest["files"]
|
||||
selected = [
|
||||
record
|
||||
for record in records
|
||||
if (not args.case or record["path"] in args.case)
|
||||
and (not args.category or record["expected"] in args.category)
|
||||
]
|
||||
unknown = sorted(set(args.case) - {record["path"] for record in selected})
|
||||
if unknown:
|
||||
raise SystemExit(f"unknown corpus cases: {', '.join(unknown)}")
|
||||
if len(selected) != len(records) and not args.allow_partial:
|
||||
raise SystemExit("partial corpus selection requires --allow-partial")
|
||||
|
||||
executables, compile_result = _compile(build_dir, bundle)
|
||||
bundle_sha256 = compile_result["bundle_sha256"]
|
||||
verification_sha256 = compile_result["verification_sha256"]
|
||||
work: list[tuple[int, dict[str, Any], str]] = []
|
||||
indices = {record["path"]: index for index, record in enumerate(records)}
|
||||
for record in selected:
|
||||
for simulator in ("icarus", "verilator"):
|
||||
work.append((indices[record["path"]], record, simulator))
|
||||
|
||||
results: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
with ThreadPoolExecutor(max_workers=args.jobs) as executor:
|
||||
futures = {}
|
||||
for index, record, simulator in work:
|
||||
case_dir = build_dir / "cases" / _case_id(index, record)
|
||||
case_dir.mkdir(parents=True, exist_ok=True)
|
||||
future = executor.submit(
|
||||
_run_case,
|
||||
simulator,
|
||||
executables[simulator],
|
||||
executables["vvp"],
|
||||
executables["image"],
|
||||
corpus / record["path"],
|
||||
record,
|
||||
case_dir,
|
||||
args.timeout_seconds,
|
||||
bundle_sha256,
|
||||
verification_sha256,
|
||||
)
|
||||
futures[future] = (record["path"], simulator)
|
||||
for future in as_completed(futures):
|
||||
key = futures[future]
|
||||
results[key] = future.result()
|
||||
print(f"APU-P5 corpus RTL: {key[1]} {results[key]['status']} {key[0]}", flush=True)
|
||||
|
||||
failures: list[str] = []
|
||||
for record in records:
|
||||
if record["path"] not in {item["path"] for item in selected}:
|
||||
continue
|
||||
icarus = results[(record["path"], "icarus")]
|
||||
verilator = results[(record["path"], "verilator")]
|
||||
agreement = (
|
||||
"result" in icarus
|
||||
and "result" in verilator
|
||||
and all(
|
||||
icarus["result"][field] == verilator["result"][field]
|
||||
for field in (*RESULT_FIELDS, "elapsed")
|
||||
)
|
||||
and icarus["pcm_sha256"] == verilator["pcm_sha256"]
|
||||
)
|
||||
if not agreement:
|
||||
failures.append(f"Icarus/Verilator disagreement for {record['path']}")
|
||||
for simulator, result in (("icarus", icarus), ("verilator", verilator)):
|
||||
if result["status"] != "passed":
|
||||
errors = result.get("truth_errors", result.get("execution_errors", []))
|
||||
failures.append(
|
||||
f"{simulator} truth mismatch for {record['path']}: " + ", ".join(errors)
|
||||
)
|
||||
record["production_rtl"] = {
|
||||
"agreement": agreement,
|
||||
"icarus": icarus,
|
||||
"verilator": verilator,
|
||||
}
|
||||
manifest["production_rtl"] = {
|
||||
"schema_version": 1,
|
||||
"complete": len(selected) == len(records),
|
||||
"case_count": len(selected),
|
||||
"simulator_runs": len(work),
|
||||
"bundle_sha256": bundle_sha256,
|
||||
"fixture": "tests/rtl/apu_p5_corpus_tb.sv",
|
||||
"compile_result": str(build_dir / "result-compile.json"),
|
||||
"status": "passed" if not failures else "failed",
|
||||
"failure_count": len(failures),
|
||||
"failures": failures,
|
||||
}
|
||||
_write_json(output, manifest)
|
||||
print(f"APU-P5 production RTL corpus: {len(selected)} files -> {output}")
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Validate the locked APU-P7 KWS reference inputs without network access."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
REQUIRED_SOURCES = {
|
||||
"apu_mlperf_tiny": "4addd0fa08d216e20637637874e084895f289da4",
|
||||
"apu_kws_mfcc": "cf7c2f2634608a7c0ea7458ab7cb3379f2863424",
|
||||
"apu_tfds": "8997c4140cd4fc145f0693787b1da78691930459",
|
||||
"apu_tensorflow": "fcc4b966f1265f466e82617020af93670141b009",
|
||||
"apu_gemmlowp": "fda83bdc38b118cc6b56753bd540caa49e570745",
|
||||
}
|
||||
REQUIRED_ARCHIVE = {
|
||||
"apu_kws_pcm": "cc2a00c1147c2254e9be3fa0f779d8c17421dc349b86366567a8edfa9acd51df"
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--lock", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
lock = json.loads(args.lock.read_text(encoding="utf-8"))
|
||||
sources = lock.get("sources", {})
|
||||
archives = lock.get("archives", {})
|
||||
for name, revision in REQUIRED_SOURCES.items():
|
||||
if sources.get(name, {}).get("revision") != revision:
|
||||
raise SystemExit(f"missing or stale locked source: {name}")
|
||||
for name, checksum in REQUIRED_ARCHIVE.items():
|
||||
if archives.get(name, {}).get("sha256") != checksum:
|
||||
raise SystemExit(f"missing or stale locked archive: {name}")
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest = args.output_dir / "provenance.json"
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
{"sources": REQUIRED_SOURCES, "archives": REQUIRED_ARCHIVE}, indent=2, sort_keys=True
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"APU-P7 locked references validated: {manifest}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -4,6 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -17,7 +18,18 @@ sys.path.insert(0, str(ROOT))
|
||||
|
||||
from scripts.dependency_lock import archive, source # noqa: E402
|
||||
from scripts.install_toolchain import safe_extract # noqa: E402
|
||||
from scripts.setup_helpers import atomic_write, download_file # noqa: E402
|
||||
from scripts.setup_helpers import atomic_write, download_file, ensure_git_repo, sha256 # noqa: E402
|
||||
|
||||
|
||||
P7_SOURCES = (
|
||||
"apu_mlperf_tiny",
|
||||
"apu_kws_mfcc",
|
||||
"apu_tfds",
|
||||
"apu_tensorflow",
|
||||
"apu_gemmlowp",
|
||||
)
|
||||
P7_MODEL = Path("benchmark/training/keyword_spotting/trained_models/kws_ref_model.tflite")
|
||||
P7_MODEL_SHA256 = "aeea436800704fce17b17292e4412630ad856e9d777c044c64ef748a880bd0ae"
|
||||
|
||||
|
||||
def _install_archive(name: str, *, update: bool, timeout: int) -> Path:
|
||||
@@ -90,12 +102,53 @@ def _build_libflac(source_dir: Path, build_dir: Path) -> None:
|
||||
subprocess.run(["cmake", "--build", str(build_dir), "--target", "flac"], check=True)
|
||||
|
||||
|
||||
def _p7_provenance(build_dir: Path, *, update: bool, doctor: bool) -> None:
|
||||
records: dict[str, object] = {"target": "p7", "sources": {}, "model_sha256": P7_MODEL_SHA256}
|
||||
for name in P7_SOURCES:
|
||||
spec = source(name)
|
||||
destination = ROOT / spec["destination"]
|
||||
if doctor:
|
||||
if not (destination / ".git").is_dir():
|
||||
raise RuntimeError(f"missing P7 managed source: {destination}")
|
||||
revision = subprocess.check_output(
|
||||
["git", "-C", str(destination), "rev-parse", "HEAD"], text=True
|
||||
).strip()
|
||||
if revision != spec["revision"]:
|
||||
raise RuntimeError(f"P7 source revision mismatch for {name}: {revision}")
|
||||
else:
|
||||
ensure_git_repo(
|
||||
spec["url"],
|
||||
destination,
|
||||
spec["revision"],
|
||||
recursive=spec.get("recursive", False),
|
||||
update=update,
|
||||
)
|
||||
records["sources"][name] = {
|
||||
"revision": spec["revision"],
|
||||
"destination": spec["destination"],
|
||||
}
|
||||
model = ROOT / source("apu_mlperf_tiny")["destination"] / P7_MODEL
|
||||
if not model.is_file() or sha256(model) != P7_MODEL_SHA256:
|
||||
raise RuntimeError("P7 locked kws_ref_model.tflite hash mismatch")
|
||||
build_dir.mkdir(parents=True, exist_ok=True)
|
||||
atomic_write(
|
||||
build_dir / "provenance.json", json.dumps(records, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--build-dir", type=Path, required=True)
|
||||
parser.add_argument("--target", choices=("p5", "p7"), default="p5")
|
||||
parser.add_argument("--doctor", action="store_true")
|
||||
parser.add_argument("--update", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.target == "p7":
|
||||
_p7_provenance(args.build_dir.resolve(), update=args.update, doctor=args.doctor)
|
||||
print(f"APU P7 KWS references: {args.build_dir.resolve() / 'provenance.json'}")
|
||||
return 0
|
||||
|
||||
libflac = _install_archive("apu_libflac", update=args.update, timeout=120)
|
||||
corpus = _install_archive("apu_flac_corpus", update=args.update, timeout=1800)
|
||||
_validate_notices(libflac, corpus)
|
||||
|
||||
Reference in new issue
Block a user