feat: update APU register testbench and add codec transport tests

- Modified the capability register value in apu_reg_tb.sv to 0x01bd.
- Simplified rejected write checks for the COMMAND register in apu_reg_tb.sv.
- Added backend build ID input to the ring scheduler testbench.
- Updated test cases in test_apu.py to include new codec transport and controller files.
- Created new test file test_apu_codec_transport.py for APU-P5 transport assembler and parser tests.
- Added codec profile and dependency tests in test_apu_codecs.py.
- Enhanced register parity tests to include P5 primitive mask.
- Improved script tools tests for file download functionality and dependency helpers.
This commit is contained in:
maksyuki committed 2026-09-06 07:43:16 +08:00
1 parent bdeaf7e22c
commit ef754cd0ac
59 files changed
+7854 -704

No files matched your search

+8 -1
View File
@@ -5,11 +5,18 @@ build manifests, flow execution, simulation checks, regression orchestration,
quality checks, metrics, packaging, and cleanup.
`apu_mcasm.py`, `apu_isa.py`, `apu_interpreter.py`, and `apu_primitives.py` are
the frozen APU microcode V1 assembler/ABI model, P3/P4 target interpreters, and
the frozen APU microcode V1 assembler/ABI model, P3/P4/P5 target interpreters, and
bit-accurate P4 primitive model. They accept only the APU instruction language
and deliberately have no C, ELF, RV32, dynamic linking, or runtime
code-generation path.
`setup_apu_reference.py` installs the checksum-pinned libFLAC and official
FLAC corpus as host-only P5 verification inputs and builds libFLAC with Ogg
disabled below the selected build variant. These inputs never enter firmware,
RTL, or shipped APU microcode.
`qualify_apu_p5_corpus.py` enumerates that exact corpus, records source and
decoded-PCM hashes, and classifies each file against the frozen P5 profile.
Scripts are part of the build contract. Prefer existing helpers over ad-hoc
shell behavior, preserve structured JSON results, and keep setup/download
behavior controlled by `dependencies/dependencies.lock.json`.
+740
View File
@@ -0,0 +1,740 @@
"""Bounded APU-P5 container and integer PCM reference helpers."""
from __future__ import annotations
from dataclasses import dataclass
from fractions import Fraction
from apu_p5_coefficients import coefficient_words
class CodecError(ValueError):
def __init__(self, code: int, stage: int, reason: int, offset: int, warnings: int = 0):
self.code = code
self.stage = stage
self.reason = reason
self.offset = offset
self.warnings = warnings
super().__init__(f"codec error {code}/{stage} reason 0x{reason:04x} at {offset}")
@dataclass(frozen=True)
class AudioInfo:
rate: int
channels: int
bits: int
samples: int | None
data_offset: int
data_bytes: int
input_used: int
warnings: int = 0
@dataclass(frozen=True)
class DecodedAudio:
info: AudioInfo
samples: tuple[tuple[int, ...], ...]
@dataclass(frozen=True)
class ProcessedPcm:
rate: int
channels: int
bits: int
frames: int
payload: bytes
_RESAMPLE_RATIOS = (
(1, 1),
(1, 2),
(80, 147),
(160, 147),
(3, 2),
(2, 1),
(3, 1),
(4, 1),
(6, 1),
(8, 1),
(12, 1),
(320, 147),
(640, 147),
(1280, 147),
(2, 3),
(4, 3),
)
def _rne(value: int, shift: int) -> int:
magnitude = abs(value)
quotient, remainder = divmod(magnitude, 1 << shift)
halfway = 1 << (shift - 1)
if remainder > halfway or (remainder == halfway and quotient & 1):
quotient += 1
return -quotient if value < 0 else quotient
def _sat(value: int, bits: int) -> int:
return min(max(value, -(1 << (bits - 1))), (1 << (bits - 1)) - 1)
def _profile(output_rate: int, input_rate: int) -> int:
ratio = Fraction(output_rate, input_rate)
for index, item in enumerate(_RESAMPLE_RATIOS):
if ratio == Fraction(*item):
return index
raise CodecError(3, 0, 0x50, 0)
def _resample(samples: list[tuple[int, ...]], profile: int) -> list[tuple[int, ...]]:
if profile == 0:
return list(samples)
ratio_l, ratio_m = _RESAMPLE_RATIOS[profile]
bank = 2 if profile in (1, 2) else (1 if profile == 14 else 0)
coefficients = coefficient_words()[bank * 512 : (bank + 1) * 512]
channels = len(samples[0]) if samples else 1
zero = (0,) * channels
padded = [zero] * 7 + samples + [zero] * 9
output: list[tuple[int, ...]] = []
numerator = 0
while numerator // ratio_l < len(samples):
source_index, remainder = divmod(numerator, ratio_l)
phase_floor, phase_remainder = divmod(32 * remainder, ratio_l)
phase = phase_floor
twice = phase_remainder * 2
if twice > ratio_l or (twice == ratio_l and phase & 1):
phase += 1
if phase == 32:
phase = 0
source_index += 1
values = []
for channel in range(channels):
total = sum(
padded[source_index + tap][channel] * coefficients[phase * 16 + tap]
for tap in range(16)
)
values.append(_sat(_rne(total, 30), 32))
output.append(tuple(values))
numerator += ratio_m
return output
def process_pcm(
audio: DecodedAudio,
*,
output_rate: int = 0,
output_channels: int = 0,
output_bits: int = 16,
downmix: bool = False,
resample: bool = False,
i2s: bool = False,
) -> ProcessedPcm:
if output_bits not in (16, 24):
raise CodecError(3, 0, 0x50, 0)
target_rate = output_rate or audio.info.rate
logical_channels = output_channels or (2 if i2s else audio.info.channels)
if logical_channels not in (1, 2):
raise CodecError(3, 0, 0x50, 0)
if downmix and (audio.info.channels != 2 or logical_channels != 1):
raise CodecError(3, 0, 0x50, 0)
if not downmix and audio.info.channels == 2 and logical_channels == 1:
raise CodecError(3, 0, 0x50, 0)
if not resample and target_rate != audio.info.rate:
raise CodecError(3, 0, 0x50, 0)
profile = _profile(target_rate, audio.info.rate) if resample else 0
shift = 32 - audio.info.bits
normalized = [tuple(sample << shift for sample in frame) for frame in audio.samples]
if downmix:
normalized = [(_sat(_rne(frame[0] + frame[1], 1), 32),) for frame in normalized]
elif audio.info.channels == 1 and logical_channels == 2:
normalized = [(frame[0], frame[0]) for frame in normalized]
converted = _resample(normalized, profile)
physical_channels = 2 if i2s else logical_channels
if physical_channels == 2 and converted and len(converted[0]) == 1:
converted = [(frame[0], frame[0]) for frame in converted]
payload = bytearray()
target_shift = 32 - output_bits
for frame in converted:
for sample in frame:
value = _sat(sample >> target_shift, output_bits)
if output_bits == 16:
payload.extend(value.to_bytes(2, "little", signed=True))
else:
payload.extend(value.to_bytes(4, "little", signed=True))
return ProcessedPcm(target_rate, physical_channels, output_bits, len(converted), bytes(payload))
class _BitReader:
def __init__(self, data: bytes, offset: int):
self.data = data
self.bit = offset * 8
@property
def byte_offset(self) -> int:
return self.bit // 8
def get(self, width: int, *, stage: int = 5) -> int:
if width < 0 or self.bit + width > len(self.data) * 8:
raise CodecError(5, stage, 0x20, len(self.data), 1)
value = 0
remaining = width
while remaining:
bit_offset = self.bit & 7
take = min(remaining, 8 - bit_offset)
shift = 8 - bit_offset - take
value = (value << take) | (
(self.data[self.bit // 8] >> shift) & ((1 << take) - 1)
)
self.bit += take
remaining -= take
return value
def signed(self, width: int, *, stage: int = 5) -> int:
value = self.get(width, stage=stage)
return value - (1 << width) if value & (1 << (width - 1)) else value
def unary_zeros(self, limit: int = 65535) -> int:
count = 0
while self.get(1) == 0:
count += 1
if count > limit:
raise CodecError(3, 5, 0x04, self.byte_offset, 1)
return count
def align_zero(self) -> None:
while self.bit & 7:
if self.get(1) != 0:
raise CodecError(4, 4, 0x19, self.byte_offset, 1)
def _signed_range(value: int, width: int, offset: int) -> int:
if value < -(1 << (width - 1)) or value > (1 << (width - 1)) - 1:
raise CodecError(22, 6, 0x43, offset, 1)
return value
def _utf8_integer(reader: _BitReader) -> int:
first = reader.get(8, stage=4)
if first < 0x80:
return first
prefix = 0x80
count = 0
while first & prefix:
count += 1
prefix >>= 1
if count < 2 or count > 7:
raise CodecError(4, 4, 0x17, reader.byte_offset - 1, 1)
value = first & ((1 << (7 - count)) - 1)
minimum = 1 << (7 if count == 2 else (5 * count - 4))
for _ in range(count - 1):
byte = reader.get(8, stage=4)
if byte & 0xC0 != 0x80:
raise CodecError(4, 4, 0x17, reader.byte_offset - 1, 1)
value = (value << 6) | (byte & 0x3F)
if value < minimum or value >= (1 << 36):
raise CodecError(4, 4, 0x17, reader.byte_offset - count, 1)
return value
def _u16(data: bytes, offset: int) -> int:
if offset + 2 > len(data):
raise CodecError(5, 4, 0x20, len(data))
return int.from_bytes(data[offset : offset + 2], "little")
def _u32(data: bytes, offset: int) -> int:
if offset + 4 > len(data):
raise CodecError(5, 4, 0x20, len(data))
return int.from_bytes(data[offset : offset + 4], "little")
def parse_wav(data: bytes, *, strict: bool = True) -> AudioInfo:
if len(data) < 12 or data[:4] != b"RIFF" or data[8:12] != b"WAVE":
raise CodecError(4, 4, 0x10, 0)
riff_end = 8 + _u32(data, 4)
if riff_end < 12 or riff_end > len(data):
raise CodecError(4, 4, 0x11, 4)
offset = 12
fmt: tuple[int, int, int, int] | None = None
data_chunk: tuple[int, int] | None = None
skipped = 0
chunks = 0
warnings = 0
while offset < riff_end:
if offset + 8 > riff_end:
raise CodecError(5, 4, 0x20, offset, warnings)
chunk_id = data[offset : offset + 4]
size = _u32(data, offset + 4)
payload = offset + 8
end = payload + size
padded_end = end + (size & 1)
if end < payload or padded_end > riff_end:
raise CodecError(4, 4, 0x13, offset, warnings)
chunks += 1
if chunks > 1024:
raise CodecError(3, 4, 0x04, offset, warnings)
if chunk_id == b"fmt ":
if fmt is not None or data_chunk is not None:
raise CodecError(4, 4, 0x12, offset, warnings)
tag = _u16(data, payload)
channels = _u16(data, payload + 2)
rate = _u32(data, payload + 4)
average = _u32(data, payload + 8)
alignment = _u16(data, payload + 12)
bits = _u16(data, payload + 14)
if tag == 1:
if size not in (16, 18) or (size == 18 and _u16(data, payload + 16) != 0):
raise CodecError(4, 4, 0x14, payload, warnings)
elif tag == 0xFFFE:
pcm_guid = bytes.fromhex("0100000000001000800000aa00389b71")
if (
size != 40
or _u16(data, payload + 16) != 22
or _u16(data, payload + 18) != bits
or data[payload + 24 : payload + 40] != pcm_guid
):
raise CodecError(3, 4, 0x03, payload, warnings)
channel_mask = _u32(data, payload + 20)
if (channels == 1 and channel_mask not in (0, 4)) or (
channels == 2 and channel_mask not in (0, 3)
):
raise CodecError(3, 4, 0x03, payload + 20, warnings)
else:
raise CodecError(3, 4, 0x03, payload, warnings)
expected_alignment = channels * (bits // 8)
if (
channels not in (1, 2)
or rate < 8000
or rate > 96000
or bits not in (8, 16, 24, 32)
or alignment != expected_alignment
or average != rate * expected_alignment
):
raise CodecError(4, 4, 0x14, payload, warnings)
fmt = (rate, channels, bits, alignment)
elif chunk_id == b"data":
if fmt is None or data_chunk is not None:
raise CodecError(4, 4, 0x12, offset, warnings)
if size % fmt[3]:
raise CodecError(4, 4, 0x13, payload, warnings)
data_chunk = (payload, size)
else:
skipped += size + (size & 1)
if skipped > 1024 * 1024:
raise CodecError(3, 4, 0x04, offset, warnings)
if size & 1 and data[end] != 0:
if strict:
raise CodecError(4, 4, 0x19, end, warnings)
warnings |= 1 << 2
offset = padded_end
if fmt is None or data_chunk is None:
raise CodecError(4, 4, 0x12, offset, warnings)
if strict and riff_end != len(data):
raise CodecError(4, 4, 0x1B, riff_end, warnings)
if not strict and riff_end != len(data):
warnings |= 1 << 1
rate, channels, bits, alignment = fmt
return AudioInfo(
rate,
channels,
bits,
data_chunk[1] // alignment,
data_chunk[0],
data_chunk[1],
riff_end,
warnings,
)
def decode_wav(data: bytes, *, strict: bool = True) -> DecodedAudio:
info = parse_wav(data, strict=strict)
bytes_per_sample = info.bits // 8
frames: list[tuple[int, ...]] = []
offset = info.data_offset
for _ in range(info.samples or 0):
channels: list[int] = []
for _ in range(info.channels):
encoded = data[offset : offset + bytes_per_sample]
offset += bytes_per_sample
if info.bits == 8:
channels.append(encoded[0] - 128)
else:
channels.append(int.from_bytes(encoded, "little", signed=True))
frames.append(tuple(channels))
return DecodedAudio(info, tuple(frames))
def inspect_flac(data: bytes, *, strict: bool = True) -> AudioInfo:
if len(data) < 8 or data[:4] != b"fLaC":
raise CodecError(4, 4, 0x10, 0)
offset = 4
blocks = 0
metadata_bytes = 0
stream_info: AudioInfo | None = None
last = False
while not last:
if offset + 4 > len(data):
raise CodecError(5, 4, 0x20, len(data), 1)
header = data[offset]
last = bool(header & 0x80)
block_type = header & 0x7F
size = int.from_bytes(data[offset + 1 : offset + 4], "big")
payload = offset + 4
end = payload + size
if end > len(data):
raise CodecError(5, 4, 0x20, len(data), 1)
blocks += 1
metadata_bytes += size
if blocks > 128 or metadata_bytes > 1024 * 1024:
raise CodecError(3, 4, 0x04, offset, 1)
if blocks == 1:
if block_type != 0 or size != 34:
raise CodecError(4, 4, 0x15, offset, 1)
minimum = int.from_bytes(data[payload : payload + 2], "big")
maximum = int.from_bytes(data[payload + 2 : payload + 4], "big")
min_frame = int.from_bytes(data[payload + 4 : payload + 7], "big")
max_frame = int.from_bytes(data[payload + 7 : payload + 10], "big")
packed = int.from_bytes(data[payload + 10 : payload + 18], "big")
rate = (packed >> 44) & 0xFFFFF
channels = ((packed >> 41) & 7) + 1
bits = ((packed >> 36) & 0x1F) + 1
samples = packed & ((1 << 36) - 1)
if (
minimum < 16
or maximum < minimum
or maximum > (4096 if channels == 1 else 2048)
or (min_frame and max_frame and min_frame > max_frame)
or max_frame > 65536
or channels not in (1, 2)
or bits not in (16, 24)
or rate < 8000
or rate > 96000
):
raise CodecError(3, 4, 0x04, payload, 1)
stream_info = AudioInfo(rate, channels, bits, samples or None, 0, 0, 0, 1)
elif block_type == 0 or block_type == 127:
raise CodecError(4, 4, 0x16, offset, 1)
elif block_type >= 7:
raise CodecError(3, 4, 0x03, offset, 1)
offset = end
if stream_info is None:
raise CodecError(4, 4, 0x15, 4, 1)
if strict and stream_info.samples is None and offset != len(data):
raise CodecError(4, 4, 0x1B, offset, 1)
return AudioInfo(
stream_info.rate,
stream_info.channels,
stream_info.bits,
stream_info.samples,
offset,
len(data) - offset,
offset,
1,
)
def crc8(data: bytes) -> int:
value = 0
for byte in data:
value ^= byte
for _ in range(8):
value = ((value << 1) ^ (0x07 if value & 0x80 else 0)) & 0xFF
return value
def crc16(data: bytes) -> int:
value = 0
for byte in data:
value ^= byte << 8
for _ in range(8):
value = ((value << 1) ^ (0x8005 if value & 0x8000 else 0)) & 0xFFFF
return value
@dataclass(frozen=True)
class _FlacMeta:
minimum_block: int
maximum_block: int
minimum_frame: int
maximum_frame: int
rate: int
channels: int
bits: int
total_samples: int | None
frame_offset: int
def _flac_metadata(data: bytes) -> _FlacMeta:
info = inspect_flac(data, strict=False)
payload = 8
minimum_block = int.from_bytes(data[payload : payload + 2], "big")
maximum_block = int.from_bytes(data[payload + 2 : payload + 4], "big")
minimum_frame = int.from_bytes(data[payload + 4 : payload + 7], "big")
maximum_frame = int.from_bytes(data[payload + 7 : payload + 10], "big")
return _FlacMeta(
minimum_block,
maximum_block,
minimum_frame,
maximum_frame,
info.rate,
info.channels,
info.bits,
info.samples,
info.data_offset,
)
def _block_size(code: int, reader: _BitReader) -> int:
if code == 0:
raise CodecError(4, 4, 0x17, reader.byte_offset, 1)
if code == 1:
return 192
if code <= 5:
return 576 << (code - 2)
if code == 6:
return reader.get(8, stage=4) + 1
if code == 7:
return reader.get(16, stage=4) + 1
return 256 << (code - 8)
def _sample_rate(code: int, reader: _BitReader, stream_rate: int) -> int:
fixed = {
0: stream_rate,
1: 88200,
2: 176400,
3: 192000,
4: 8000,
5: 16000,
6: 22050,
7: 24000,
8: 32000,
9: 44100,
10: 48000,
11: 96000,
}
if code in fixed:
return fixed[code]
if code == 12:
return reader.get(8, stage=4) * 1000
if code == 13:
return reader.get(16, stage=4)
if code == 14:
return reader.get(16, stage=4) * 10
raise CodecError(4, 4, 0x17, reader.byte_offset, 1)
def _sample_bits(code: int, stream_bits: int, offset: int) -> int:
values = {0: stream_bits, 1: 8, 2: 12, 4: 16, 5: 20, 6: 24, 7: 32}
if code not in values:
raise CodecError(4, 4, 0x17, offset, 1)
return values[code]
def _residuals(reader: _BitReader, block_size: int, order: int) -> list[int]:
method = reader.get(2)
if method > 1:
raise CodecError(7, 5, 0x40, reader.byte_offset, 1)
parameter_bits = 4 + method
escape = (1 << parameter_bits) - 1
partition_order = reader.get(4)
partitions = 1 << partition_order
if partition_order > 15 or block_size % partitions:
raise CodecError(7, 5, 0x42, reader.byte_offset, 1)
result: list[int] = []
for partition in range(partitions):
count = block_size // partitions - (order if partition == 0 else 0)
if count < 0:
raise CodecError(7, 5, 0x42, reader.byte_offset, 1)
parameter = reader.get(parameter_bits)
if parameter == escape:
raw_width = reader.get(5)
if raw_width == 0:
result.extend([0] * count)
else:
result.extend(reader.signed(raw_width) for _ in range(count))
else:
for _ in range(count):
quotient = reader.unary_zeros()
remainder = reader.get(parameter) if parameter else 0
unsigned = (quotient << parameter) | remainder
result.append((unsigned >> 1) ^ -(unsigned & 1))
if len(result) != block_size - order:
raise CodecError(7, 5, 0x42, reader.byte_offset, 1)
return result
def _fixed_predict(order: int, history: list[int]) -> int:
if order == 0:
return 0
if order == 1:
return history[-1]
if order == 2:
return 2 * history[-1] - history[-2]
if order == 3:
return 3 * history[-1] - 3 * history[-2] + history[-3]
return 4 * history[-1] - 6 * history[-2] + 4 * history[-3] - history[-4]
def _subframe(reader: _BitReader, block_size: int, bits: int) -> list[int]:
offset = reader.byte_offset
if reader.get(1) != 0:
raise CodecError(4, 5, 0x40, offset, 1)
kind = reader.get(6)
wasted_flag = reader.get(1)
wasted = reader.unary_zeros() + 1 if wasted_flag else 0
coded_bits = bits - wasted
if coded_bits < 1:
raise CodecError(7, 5, 0x41, offset, 1)
if kind == 0:
samples = [reader.signed(coded_bits)] * block_size
elif kind == 1:
samples = [reader.signed(coded_bits) for _ in range(block_size)]
elif 8 <= kind <= 12:
order = kind - 8
if order > block_size:
raise CodecError(7, 5, 0x40, offset, 1)
samples = [reader.signed(coded_bits) for _ in range(order)]
for residual in _residuals(reader, block_size, order):
samples.append(_signed_range(_fixed_predict(order, samples) + residual, 32, offset))
elif 32 <= kind <= 63:
order = kind - 31
if order > block_size:
raise CodecError(7, 5, 0x40, offset, 1)
samples = [reader.signed(coded_bits) for _ in range(order)]
precision = reader.get(4) + 1
if precision == 16:
raise CodecError(7, 5, 0x40, reader.byte_offset, 1)
shift = reader.signed(5)
coefficients = [reader.signed(precision) for _ in range(order)]
for residual in _residuals(reader, block_size, order):
prediction = sum(coefficients[index] * samples[-index - 1] for index in range(order))
prediction = prediction >> shift if shift >= 0 else prediction << -shift
samples.append(_signed_range(prediction + residual, 32, offset))
else:
raise CodecError(7, 5, 0x40, offset, 1)
restored = [_signed_range(sample << wasted, 32, offset) for sample in samples]
return restored
def _decode_frame(
data: bytes, offset: int, meta: _FlacMeta
) -> tuple[list[tuple[int, ...]], int, int, int, int]:
start = offset
reader = _BitReader(data, offset)
if reader.get(14, stage=4) != 0x3FFE or reader.get(1, stage=4) != 0:
raise CodecError(4, 4, 0x17, start, 1)
strategy = reader.get(1, stage=4)
block_code = reader.get(4, stage=4)
rate_code = reader.get(4, stage=4)
assignment = reader.get(4, stage=4)
bits_code = reader.get(3, stage=4)
if reader.get(1, stage=4) != 0:
raise CodecError(4, 4, 0x17, start, 1)
number = _utf8_integer(reader)
block_size = _block_size(block_code, reader)
rate = _sample_rate(rate_code, reader, meta.rate)
bits = _sample_bits(bits_code, meta.bits, start)
header_end = reader.byte_offset
expected_crc8 = reader.get(8, stage=4)
if crc8(data[start:header_end]) != expected_crc8:
raise CodecError(6, 4, 0x30, header_end, 1)
if assignment <= 7:
channels = assignment + 1
elif assignment <= 10:
channels = 2
else:
raise CodecError(4, 4, 0x17, start, 1)
if (rate, channels, bits) != (meta.rate, meta.channels, meta.bits):
raise CodecError(4, 4, 0x18, start, 1)
if block_size > (4096 if channels == 1 else 2048):
raise CodecError(3, 4, 0x04, start, 1)
channel_bits = [bits] * channels
if assignment == 8:
channel_bits[1] += 1
elif assignment == 9:
channel_bits[0] += 1
elif assignment == 10:
channel_bits[1] += 1
planar = [_subframe(reader, block_size, channel_bits[channel]) for channel in range(channels)]
reader.align_zero()
footer = reader.byte_offset
expected_crc16 = reader.get(16, stage=4)
if crc16(data[start:footer]) != expected_crc16:
raise CodecError(6, 4, 0x31, footer, 1)
frame_end = reader.byte_offset
frame_bytes = frame_end - start
if (
(meta.minimum_frame and frame_bytes < meta.minimum_frame)
or (meta.maximum_frame and frame_bytes > meta.maximum_frame)
or frame_bytes > 65536
):
raise CodecError(3, 4, 0x04, start, 1)
frames: list[tuple[int, ...]] = []
for index in range(block_size):
if assignment <= 7:
values = tuple(planar[channel][index] for channel in range(channels))
elif assignment == 8:
left = planar[0][index]
values = (left, left - planar[1][index])
elif assignment == 9:
right = planar[1][index]
values = (planar[0][index] + right, right)
else:
side = planar[1][index]
middle = (planar[0][index] << 1) | (side & 1)
values = ((middle + side) >> 1, (middle - side) >> 1)
frames.append(tuple(_signed_range(value, bits, start) for value in values))
return frames, frame_end, block_size, strategy, number
def decode_flac(data: bytes, *, strict: bool = True) -> DecodedAudio:
meta = _flac_metadata(data)
offset = meta.frame_offset
frames: list[tuple[int, ...]] = []
blocks: list[int] = []
strategy: int | None = None
frame_number = 0
while offset < len(data) and (meta.total_samples is None or len(frames) < meta.total_samples):
decoded, frame_end, block_size, frame_strategy, number = _decode_frame(data, offset, meta)
if strategy is None:
strategy = frame_strategy
elif strategy != frame_strategy:
raise CodecError(4, 4, 0x18, offset, 1)
expected_number = frame_number if frame_strategy == 0 else len(frames)
if number != expected_number:
raise CodecError(4, 4, 0x1A, offset, 1)
frames.extend(decoded)
blocks.append(block_size)
frame_number += 1
offset = frame_end
if meta.total_samples is not None and len(frames) > meta.total_samples:
raise CodecError(4, 4, 0x32, offset, 1)
if meta.total_samples is not None and len(frames) != meta.total_samples:
raise CodecError(4, 4, 0x32, offset, 1)
for block_size in blocks[:-1]:
if not meta.minimum_block <= block_size <= meta.maximum_block:
raise CodecError(4, 4, 0x17, offset, 1)
if blocks and blocks[-1] > meta.maximum_block:
raise CodecError(3, 4, 0x04, offset, 1)
warnings = 1
if offset != len(data):
if strict:
raise CodecError(4, 4, 0x1B, offset, warnings)
warnings |= 1 << 1
info = AudioInfo(
meta.rate,
meta.channels,
meta.bits,
len(frames),
meta.frame_offset,
offset - meta.frame_offset,
offset,
warnings,
)
return DecodedAudio(info, tuple(frames))
+268 -17
View File
@@ -1,4 +1,4 @@
"""Reference interpreter for the enabled APU-P3/P4 microcode targets."""
"""Reference interpreter for the enabled APU-P3/P4/P5 microcode targets."""
from __future__ import annotations
@@ -17,6 +17,7 @@ from apu_isa import (
InstructionClass,
LocalOpcode,
ScalarOpcode,
TransportOpcode,
parse_apumc,
trap_detail,
validate_instruction,
@@ -42,6 +43,167 @@ class PredicateInputs:
transport_done: bool = False
@dataclass
class TransportCommand:
opcode: TransportOpcode
destination: int
offset: int
count: int
payload: bytes
cycles: int
@dataclass
class TransportModel:
input_data: bytes = b""
output_capacity: int = 0xFFFFFFFF
stream_output: bool = False
latency: int = 2
output_frame_bytes: int = 4
input_cursor: int = 0
output: bytearray = field(default_factory=bytearray)
frames: int = 0
logical_input: int = 0
logical_output: int = 0
result_code: int = 0
result_stage: int = 0
result_detail: int = 0
events: int = 0
pending: list[TransportCommand] = field(default_factory=list)
completed: list[tuple[int, int]] = field(default_factory=list)
@property
def idle(self) -> bool:
return not self.pending
def reset(self) -> None:
self.input_cursor = 0
self.output.clear()
self.frames = 0
self.logical_input = 0
self.logical_output = 0
self.result_code = 0
self.result_stage = 0
self.result_detail = 0
self.events = 0
self.pending.clear()
self.completed.clear()
def abort(self) -> None:
self.pending.clear()
self.completed.clear()
def issue(
self,
instruction: Instruction,
registers: list[int],
entry: Entry,
primitives: PrimitiveBam,
) -> bool:
opcode = TransportOpcode(instruction.opcode)
offset = registers[instruction.src0]
count = registers[instruction.src1]
if opcode == TransportOpcode.INPUT_REFILL:
if (
count < 4
or count > 256
or count & 3
or offset & 3
or offset + count > entry.scratch_bytes
):
raise PrimitiveFault(11, 11, 5)
actual = min(count, len(self.input_data) - self.input_cursor)
needed = (actual + 3) // 4
if any(command.opcode == TransportOpcode.INPUT_REFILL for command in self.pending):
return False
if len(primitives.input_fifo.words) + needed > 64:
return False
payload = self.input_data[self.input_cursor : self.input_cursor + actual]
self.pending.append(
TransportCommand(
opcode, instruction.dst, offset, actual, payload, max(1, self.latency)
)
)
return True
if opcode in (TransportOpcode.OUTPUT_COMMIT, TransportOpcode.OUTPUT_STREAM):
if count < 1 or count > 256 or offset & 3 or offset + count > entry.scratch_bytes:
raise PrimitiveFault(11, 11, 5)
if count % self.output_frame_bytes or (
(opcode == TransportOpcode.OUTPUT_STREAM) != self.stream_output
):
raise PrimitiveFault(11, 11, 5)
if not primitives.output_fifo.empty:
raise PrimitiveFault(11, 11, 5)
if any(
command.opcode in (TransportOpcode.OUTPUT_COMMIT, TransportOpcode.OUTPUT_STREAM)
for command in self.pending
):
return False
if (
opcode == TransportOpcode.OUTPUT_COMMIT
and len(self.output) + count > self.output_capacity
):
raise PrimitiveFault(8, 8, 9)
address = entry.scratch_base + offset
payload = primitives.memory.read(address, count, entry)
self.pending.append(
TransportCommand(
opcode, instruction.dst, offset, count, payload, max(1, self.latency)
)
)
return True
if opcode == TransportOpcode.DMA_WAIT:
if not self.idle:
return False
self.completed.append((instruction.dst, 0))
elif opcode == TransportOpcode.FRAME_COMMIT:
if (
count != self.output_frame_bytes
or self.logical_input + offset > self.input_cursor
or self.logical_output + count > len(self.output)
):
raise PrimitiveFault(11, 11, 9)
self.logical_input = min(0xFFFFFFFF, self.logical_input + offset)
self.logical_output = min(0xFFFFFFFF, self.logical_output + count)
self.frames = min(0xFFFFFFFF, self.frames + 1)
self.completed.append((instruction.dst, 0))
elif opcode == TransportOpcode.JOB_RESULT:
code = offset & 0x3F
if offset >> 6 or code > 22:
raise PrimitiveFault(11, 11, 1)
self.result_code = code
self.result_stage = instruction.aux & 0xF
self.result_detail = count
self.completed.append((instruction.dst, 0))
else:
self.events |= instruction.immediate & 0x0C0
self.completed.append((instruction.dst, 0))
return True
def tick(self, entry: Entry, primitives: PrimitiveBam) -> None:
if not self.pending:
return
for command in self.pending:
command.cycles -= 1
ready = [command for command in self.pending if command.cycles == 0]
for command in ready:
if command.opcode == TransportOpcode.INPUT_REFILL:
primitives.memory.write(entry.scratch_base + command.offset, command.payload, entry)
for position in range(0, command.count, 4):
data = command.payload[position : position + 4]
last = self.input_cursor + position + len(data) == len(self.input_data)
primitives.input_fifo.push(
int.from_bytes(data.ljust(4, b"\0"), "little")
| (len(data) << 32)
| (int(last) << 40)
)
self.input_cursor += command.count
else:
self.output.extend(command.payload)
self.completed.append((command.destination, command.count))
self.pending.remove(command)
@dataclass
class Machine:
instructions: list[Instruction]
@@ -62,12 +224,15 @@ class Machine:
trace: list[dict[str, object]] = field(default_factory=list)
target: str = "p3"
primitives: PrimitiveBam | None = None
transport: TransportModel | None = None
primitive_cycles: list[int] = field(default_factory=list)
kernel_result: PrimitiveResult | None = field(default=None, init=False)
kernel_fault: PrimitiveFault | None = field(default=None, init=False)
kernel_pending_fault: PrimitiveFault | None = field(default=None, init=False)
kernel_pending_dst: int | None = field(default=None, init=False)
kernel_cycles_remaining: int = field(default=0, init=False)
transport_pending_dst: set[int] = field(default_factory=set, init=False)
transport_fault: PrimitiveFault | None = field(default=None, init=False)
def __post_init__(self) -> None:
self.begin_entry(self.entry)
@@ -81,7 +246,7 @@ class Machine:
output_ready = self.predicate_inputs.output_ready
kernel_done = self.predicate_inputs.kernel_done
input_exhausted = self.predicate_inputs.input_exhausted
if self.target == "p4" and self.primitives is not None:
if self.target in ("p4", "p5") and self.primitives is not None:
input_ready = not self.primitives.input_fifo.empty
output_ready = not self.primitives.output_fifo.full
kernel_done = self.primitives.kernel_done
@@ -102,7 +267,9 @@ class Machine:
input_ready,
output_ready,
kernel_done,
self.predicate_inputs.transport_done,
self.transport.idle
if self.transport is not None
else self.predicate_inputs.transport_done,
)
if predicate >= len(values):
raise SequencerTrap(1, self.pc, self.instructions[self.pc])
@@ -132,10 +299,29 @@ class Machine:
if self.kernel_cycles_remaining == 0:
self._complete_kernel()
def _tick_transport(self) -> None:
if self.transport is None or self.primitives is None:
return
try:
self.transport.tick(self.entry, self.primitives)
except PrimitiveFault as error:
self.transport_fault = error
return
while self.transport.completed:
register, value = self.transport.completed.pop(0)
self._write(register, value)
self.transport_pending_dst.discard(register)
def _pending_dependency(self, instruction: Instruction) -> bool:
pending = self.kernel_pending_dst
if pending is None:
return False
pending_registers = set(self.transport_pending_dst)
if self.kernel_pending_dst is not None:
pending_registers.add(self.kernel_pending_dst)
return any(
self._uses_pending_register(instruction, pending) for pending in pending_registers
)
@staticmethod
def _uses_pending_register(instruction: Instruction, pending: int) -> bool:
instruction_class = instruction.instruction_class
opcode = instruction.opcode
if instruction_class == InstructionClass.CONTROL:
@@ -175,8 +361,12 @@ class Machine:
self.kernel_pending_fault = None
self.kernel_pending_dst = None
self.kernel_cycles_remaining = 0
self.transport_pending_dst.clear()
self.transport_fault = None
if self.primitives is not None:
self.primitives.reset_mutable()
if self.transport is not None:
self.transport.reset()
def abort(self) -> None:
self.kernel_result = None
@@ -184,6 +374,10 @@ class Machine:
self.kernel_pending_fault = None
self.kernel_pending_dst = None
self.kernel_cycles_remaining = 0
self.transport_pending_dst.clear()
self.transport_fault = None
if self.transport is not None:
self.transport.abort()
self.watchdog = 0
if self.primitives is not None:
self.primitives.reset_mutable()
@@ -203,9 +397,13 @@ class Machine:
}
)
def _no_retirement_cycle(self, instruction: Instruction, *, tick_kernel: bool = True) -> None:
def _no_retirement_cycle(
self, instruction: Instruction, *, tick_kernel: bool = True, tick_transport: bool = True
) -> None:
if tick_kernel:
self._tick_kernel()
if tick_transport:
self._tick_transport()
self.watchdog += 1
if self.timeout <= 0 or self.watchdog >= self.timeout:
raise SequencerTrap(7, self.pc, instruction)
@@ -224,10 +422,15 @@ class Machine:
raise SequencerTrap(2, self.pc, Instruction(0, 0))
instruction = self.instructions[self.pc]
self._tick_kernel()
self._tick_transport()
if self.kernel_fault is not None:
fault = self.kernel_fault
self.kernel_fault = None
raise SequencerTrap(fault.reason, self.pc, instruction)
if self.transport_fault is not None:
fault = self.transport_fault
self.transport_fault = None
raise SequencerTrap(fault.reason, self.pc, instruction)
try:
validate_instruction(instruction, self.target)
except ValueError as error:
@@ -243,7 +446,7 @@ class Machine:
return False
if self._pending_dependency(instruction):
self._no_retirement_cycle(instruction, tick_kernel=False)
self._no_retirement_cycle(instruction, tick_kernel=False, tick_transport=False)
return False
next_pc = self.pc + 1
@@ -251,6 +454,9 @@ class Machine:
if instruction.instruction_class == InstructionClass.CONTROL:
opcode = ControlOpcode(instruction.opcode)
if opcode == ControlOpcode.END:
if self.transport is not None and not self.transport.idle:
self._no_retirement_cycle(instruction, tick_kernel=False, tick_transport=False)
return False
terminal = True
elif opcode == ControlOpcode.TRAP:
raise SequencerTrap(10, self.pc, instruction, instruction.immediate)
@@ -288,12 +494,27 @@ class Machine:
ready = False
if self.primitives is not None:
ready = (
(instruction.aux == 1 and self.primitives.kernel_done)
(
instruction.aux == 0
and self.transport is not None
and self.transport.idle
)
or (instruction.aux == 1 and self.primitives.kernel_done)
or (instruction.aux == 2 and not self.primitives.input_fifo.empty)
or (instruction.aux == 3 and not self.primitives.output_fifo.full)
or (
instruction.aux == 4
and self.transport is not None
and self.transport.idle
)
or (
instruction.aux == 5
and self.transport is not None
and self.transport.idle
)
)
if not ready:
self._no_retirement_cycle(instruction, tick_kernel=False)
self._no_retirement_cycle(instruction, tick_kernel=False, tick_transport=False)
return False
elif instruction.instruction_class == InstructionClass.SCALAR:
opcode = ScalarOpcode(instruction.opcode)
@@ -333,6 +554,22 @@ class Machine:
instruction.dst,
self._saturate(source0, instruction.aux & 0x1F, bool(instruction.aux & 0x20)),
)
elif instruction.instruction_class == InstructionClass.TRANSPORT:
if self.transport is None or self.primitives is None:
raise SequencerTrap(6, self.pc, instruction)
try:
accepted = self.transport.issue(
instruction, self.registers, self.entry, self.primitives
)
except PrimitiveFault as error:
raise SequencerTrap(error.reason, self.pc, instruction) from error
if not accepted:
self._no_retirement_cycle(instruction, tick_kernel=False, tick_transport=False)
return False
if instruction.opcode <= TransportOpcode.OUTPUT_STREAM:
self.transport_pending_dst.add(instruction.dst)
else:
self._tick_transport()
else:
if self.primitives is None:
raise SequencerTrap(6, self.pc, instruction)
@@ -341,7 +578,7 @@ class Machine:
if (opcode == LocalOpcode.FIFO_POP and self.primitives.input_fifo.empty) or (
opcode == LocalOpcode.FIFO_PUSH and self.primitives.output_fifo.full
):
self._no_retirement_cycle(instruction, tick_kernel=False)
self._no_retirement_cycle(instruction, tick_kernel=False, tick_transport=False)
return False
if instruction.instruction_class == InstructionClass.BITSTREAM:
opcode = BitstreamOpcode(instruction.opcode)
@@ -358,7 +595,7 @@ class Machine:
)
self.primitives._refill(required)
if len(self.primitives.bits) < required and not self.primitives.eof:
self._no_retirement_cycle(instruction, tick_kernel=False)
self._no_retirement_cycle(instruction, tick_kernel=False, tick_transport=False)
return False
if instruction.instruction_class == InstructionClass.ENTROPY:
opcode = EntropyOpcode(instruction.opcode)
@@ -367,14 +604,16 @@ class Machine:
elif opcode == EntropyOpcode.UNARY:
required = instruction.immediate + 1
elif opcode == EntropyOpcode.SIGN_RESTORE:
required = 0 if self.registers[instruction.src0] == 0 else (
self.registers[instruction.src1] & 0x1F
) + 1
required = (
0
if self.registers[instruction.src0] == 0
else (self.registers[instruction.src1] & 0x1F) + 1
)
else:
required = 1
self.primitives._refill(required)
if len(self.primitives.bits) < required and not self.primitives.eof:
self._no_retirement_cycle(instruction, tick_kernel=False)
self._no_retirement_cycle(instruction, tick_kernel=False, tick_transport=False)
return False
try:
if instruction.instruction_class == InstructionClass.KERNEL:
@@ -430,6 +669,17 @@ class Machine:
"registers": self.registers,
"trace": self.trace,
"primitive_cycles": self.primitive_cycles,
"transport": None
if self.transport is None
else {
"input_used": self.transport.input_cursor,
"output": bytes(self.transport.output).hex(),
"frames": self.transport.frames,
"result_code": self.transport.result_code,
"result_stage": self.transport.result_stage,
"result_detail": self.transport.result_detail,
"events": self.transport.events,
},
}
@@ -461,7 +711,8 @@ def main() -> int:
inputs,
args.timeout,
target=args.target,
primitives=PrimitiveBam() if args.target == "p4" else None,
primitives=PrimitiveBam() if args.target in ("p4", "p5") else None,
transport=TransportModel() if args.target == "p5" else None,
).run(),
sort_keys=True,
)
+81 -11
View File
@@ -18,8 +18,9 @@ APUMC_INSTRUCTION_OFFSET = 192
APUMC_MAX_INSTRUCTIONS = 2048
APUMC_P3_CAPABILITY_MASK = 0
APUMC_P4_CAPABILITY_MASK = 0x0000FFFF
APUMC_P5_CAPABILITY_MASK = 0x001FFFFF
APUMC_P4_TABLE_SCRATCH_BYTES = 0x6000
APUMC_TARGETS = ("p3", "p4")
APUMC_TARGETS = ("p3", "p4", "p5")
class InstructionClass(IntEnum):
@@ -105,6 +106,16 @@ class KernelOpcode(IntEnum):
PCM_PACK = 9
class TransportOpcode(IntEnum):
INPUT_REFILL = 0
OUTPUT_COMMIT = 1
OUTPUT_STREAM = 2
DMA_WAIT = 3
FRAME_COMMIT = 4
JOB_RESULT = 5
EVENT = 6
PREDICATES = {
"always": 0,
"eq": 1,
@@ -232,6 +243,7 @@ def abi_manifest() -> dict[str, object]:
},
"p3_implemented_primitive_mask": APUMC_P3_CAPABILITY_MASK,
"p4_implemented_primitive_mask": APUMC_P4_CAPABILITY_MASK,
"p5_implemented_primitive_mask": APUMC_P5_CAPABILITY_MASK,
"p4_local_data_bytes": 0x1C000,
"p4_local_table_scratch_bytes": APUMC_P4_TABLE_SCRATCH_BYTES,
}
@@ -334,7 +346,11 @@ def _require_zero(instruction: Instruction, names: Iterable[str]) -> None:
def _target_mask(target: str) -> int:
if target not in APUMC_TARGETS:
raise ValueError(f"unknown APU target {target}")
return APUMC_P3_CAPABILITY_MASK if target == "p3" else APUMC_P4_CAPABILITY_MASK
return {
"p3": APUMC_P3_CAPABILITY_MASK,
"p4": APUMC_P4_CAPABILITY_MASK,
"p5": APUMC_P5_CAPABILITY_MASK,
}[target]
def instruction_required_mask(instruction: Instruction) -> int:
@@ -345,6 +361,10 @@ def instruction_required_mask(instruction: Instruction) -> int:
return 0x0000FFC0
if instruction.aux in (2, 3):
return 1 << 5
if instruction.aux == 4:
return 1 << 18
if instruction.aux == 5:
return 1 << 17
return 0
if instruction_class == InstructionClass.BITSTREAM:
return (1 << 0) | ((1 << 1) if opcode in (6, 7) else 0)
@@ -354,6 +374,13 @@ def instruction_required_mask(instruction: Instruction) -> int:
return 1 << (5 if opcode in (5, 6) else 4)
if instruction_class == InstructionClass.KERNEL:
return 1 << (opcode + 6)
if instruction_class == InstructionClass.TRANSPORT:
if opcode <= TransportOpcode.OUTPUT_STREAM:
return 1 << (opcode + 16)
if opcode in (TransportOpcode.FRAME_COMMIT, TransportOpcode.JOB_RESULT):
return 1 << 19
if opcode == TransportOpcode.EVENT:
return 1 << 20
return 0
@@ -395,7 +422,7 @@ def validate_instruction(instruction: Instruction, target: str = "p3") -> None:
_require_zero(instruction, ("dst", "src0", "src1", "immediate"))
if instruction.predicate != 0 or instruction.aux > 5:
raise ValueError("WAIT encoding is invalid")
if target == "p3" or instruction.aux not in (1, 2, 3):
if target == "p3" or (target == "p4" and instruction.aux not in (1, 2, 3)):
raise ValueError(f"WAIT source is unavailable in {target.upper()}")
return
@@ -432,7 +459,9 @@ def validate_instruction(instruction: Instruction, target: str = "p3") -> None:
if instruction.instruction_class > InstructionClass.TRANSPORT:
raise ValueError("instruction class is invalid in V1")
if target == "p3" or instruction.instruction_class == InstructionClass.TRANSPORT:
if target == "p3" or (
target == "p4" and instruction.instruction_class == InstructionClass.TRANSPORT
):
raise ValueError(f"instruction class is unavailable in {target.upper()}")
if instruction.instruction_class == InstructionClass.BITSTREAM:
@@ -510,6 +539,30 @@ def validate_instruction(instruction: Instruction, target: str = "p3") -> None:
raise ValueError("resampler profile is invalid")
if opcode == KernelOpcode.PCM_PACK and (instruction.aux >> 3 or (instruction.aux & 3) > 1):
raise ValueError("PCM pack format is invalid")
elif instruction.instruction_class == InstructionClass.TRANSPORT:
if instruction.opcode > TransportOpcode.EVENT:
raise ValueError("transport opcode is unallocated")
opcode = TransportOpcode(instruction.opcode)
if opcode in (
TransportOpcode.INPUT_REFILL,
TransportOpcode.OUTPUT_COMMIT,
TransportOpcode.OUTPUT_STREAM,
):
_require_zero(instruction, ("aux", "immediate"))
elif opcode == TransportOpcode.DMA_WAIT:
_require_zero(instruction, ("src0", "src1", "aux", "immediate"))
elif opcode == TransportOpcode.FRAME_COMMIT:
_require_zero(instruction, ("dst", "aux", "immediate"))
elif opcode == TransportOpcode.JOB_RESULT:
_require_zero(instruction, ("dst", "immediate"))
if instruction.aux >> 4:
raise ValueError("JOB_RESULT stage is invalid")
else:
_require_zero(instruction, ("dst", "src0", "src1", "aux"))
if instruction.immediate >> 11 or not instruction.immediate & 0x0C0:
raise ValueError("EVENT mask is invalid")
if instruction.immediate & ~0x0C0:
raise ValueError("EVENT mask contains unavailable IRQ bits")
if instruction_required_mask(instruction) & ~target_mask:
raise ValueError(f"required primitive is unavailable in {target.upper()}")
@@ -523,6 +576,10 @@ def validate_p4_instruction(instruction: Instruction) -> None:
validate_instruction(instruction, "p4")
def validate_p5_instruction(instruction: Instruction) -> None:
validate_instruction(instruction, "p5")
def validate_entry(entry: Entry, instruction_count: int, target: str = "p3",
table_payload_bytes: int = 0) -> None:
if entry.format_id not in range(APUMC_ENTRY_COUNT):
@@ -543,14 +600,14 @@ def validate_entry(entry: Entry, instruction_count: int, target: str = "p3",
)
):
raise ValueError("P3 entries cannot use scratch, table, or primitives")
if target == "p4":
if entry.primitive_mask & ~APUMC_P4_CAPABILITY_MASK:
raise ValueError("P4 entry requires an unavailable primitive")
if target in ("p4", "p5"):
if entry.primitive_mask & ~_target_mask(target):
raise ValueError(f"{target.upper()} entry requires an unavailable primitive")
if entry.scratch_base & 3 or entry.scratch_bytes & 3:
raise ValueError("scratch range must be four-byte aligned")
scratch_end = entry.scratch_base + entry.scratch_bytes
if scratch_end > APUMC_P4_TABLE_SCRATCH_BYTES:
raise ValueError("scratch range exceeds the P4 codec workspace")
raise ValueError("scratch range exceeds the codec workspace")
if entry.scratch_bytes and entry.scratch_base < table_payload_bytes:
raise ValueError("scratch range overlaps the active table payload")
if entry.table_offset & 3 or entry.table_bytes & 3:
@@ -665,7 +722,7 @@ def control_flow_report(
elif opcode == ControlOpcode.NOP:
successors.append((pc + 1, returns, loops))
edges.add((pc, pc + 1, "fallthrough"))
elif opcode == ControlOpcode.WAIT and target == "p4":
elif opcode == ControlOpcode.WAIT and target in ("p4", "p5"):
successors.append((pc + 1, returns, loops))
edges.add((pc, pc + 1, "wait_ready"))
else:
@@ -819,7 +876,7 @@ def parse_apumc(
raise ValueError("APUMC range fields are invalid")
if target == "p3" and (table_bytes or table_offset or header[9] or header[10]):
raise ValueError("APUMC range or P3 capability fields are invalid")
if target == "p4" and (
if target in ("p4", "p5") and (
table_bytes > APUMC_P4_TABLE_SCRATCH_BYTES
or table_bytes & 3
or table_offset > len(bundle)
@@ -831,7 +888,7 @@ def parse_apumc(
or header[9] & ~target_mask
or header[10] > APUMC_P4_TABLE_SCRATCH_BYTES
):
raise ValueError("APUMC range or P4 capability fields are invalid")
raise ValueError(f"APUMC range or {target.upper()} capability fields are invalid")
if crc32_iso_hdlc(bundle[APUMC_HEADER_BYTES:]) != header[11]:
raise ValueError("APUMC CRC mismatch")
entries = []
@@ -892,6 +949,19 @@ def parse_apumc(
raise ValueError("kernel WAIT requires a class-5 primitive")
elif required & ~entry.primitive_mask:
raise ValueError("FIFO WAIT requires the local-FIFO primitive")
elif (
instructions[pc].instruction_class == InstructionClass.CONTROL
and instructions[pc].opcode == ControlOpcode.WAIT
and instructions[pc].aux == 0
and not entry.primitive_mask & 0x00070000
):
raise ValueError("DMA WAIT requires a transport primitive")
elif (
instructions[pc].instruction_class == InstructionClass.TRANSPORT
and instructions[pc].opcode == TransportOpcode.DMA_WAIT
and not entry.primitive_mask & 0x00070000
):
raise ValueError("DMA_WAIT requires a transport primitive")
elif required & ~entry.primitive_mask:
raise ValueError(f"entry {entry.format_id} omits an instruction primitive")
if header[9] != (entries[0].primitive_mask | entries[1].primitive_mask | entries[2].primitive_mask):
+40 -3
View File
@@ -24,6 +24,7 @@ from apu_isa import (
KernelOpcode,
LocalOpcode,
ScalarOpcode,
TransportOpcode,
abi_manifest,
build_apumc,
control_flow_report,
@@ -191,6 +192,34 @@ def _encode_instruction(tokens: list[str], pc: int, labels: dict[str, int]) -> I
aux=_number(args[3]),
immediate=_number(args[4]),
)
elif mnemonic in DEFERRED_OPCODES["transport"]:
opcode = TransportOpcode(DEFERRED_OPCODES["transport"].index(mnemonic))
values = {"predicate": predicate}
if opcode in (
TransportOpcode.INPUT_REFILL,
TransportOpcode.OUTPUT_COMMIT,
TransportOpcode.OUTPUT_STREAM,
):
if len(args) != 3:
raise ValueError(f"{mnemonic} takes destination, address, and count registers")
values.update(dst=_register(args[0]), src0=_register(args[1]), src1=_register(args[2]))
elif opcode == TransportOpcode.DMA_WAIT:
if len(args) != 1:
raise ValueError("dma_wait takes one destination register")
values["dst"] = _register(args[0])
elif opcode == TransportOpcode.FRAME_COMMIT:
if len(args) != 2:
raise ValueError("frame_commit takes input and output count registers")
values.update(src0=_register(args[0]), src1=_register(args[1]))
elif opcode == TransportOpcode.JOB_RESULT:
if len(args) != 3:
raise ValueError("job_result takes code, detail, and stage")
values.update(src0=_register(args[0]), src1=_register(args[1]), aux=_number(args[2]))
else:
if len(args) != 1:
raise ValueError("event takes one IRQ mask")
values["immediate"] = _number(args[0])
return Instruction(InstructionClass.TRANSPORT, opcode, **values)
else:
raise ValueError(f"unknown mnemonic {mnemonic}")
@@ -226,7 +255,7 @@ def _encode_instruction(tokens: list[str], pc: int, labels: dict[str, int]) -> I
return Instruction(InstructionClass.SCALAR, opcode, **values)
def assemble(source: str, target: str = "p3") -> Assembly:
def assemble(source: str, target: str = "p3", table_payload_input: bytes | None = None) -> Assembly:
if target not in APUMC_TARGETS:
raise ValueError(f"unknown APU target {target}")
lines = []
@@ -290,7 +319,13 @@ def assemble(source: str, target: str = "p3") -> Assembly:
except KeyError as error:
raise ValueError(f"unknown entry label {error.args[0]}") from error
entries.sort(key=lambda entry: entry.format_id)
table_payload = b"".join(struct.pack("<I", word) for word in table_words)
if table_payload_input is not None and table_words:
raise ValueError("external table payload cannot be combined with .table32")
table_payload = (
table_payload_input
if table_payload_input is not None
else b"".join(struct.pack("<I", word) for word in table_words)
)
bundle = build_apumc(instructions, entries, build_id, target=target, table_payload=table_payload)
return Assembly(bundle, dict(sorted(labels.items())), instructions, entries, build_id, target, table_payload)
@@ -315,8 +350,10 @@ def main() -> int:
parser.add_argument("--primitive-manifest", type=Path)
parser.add_argument("--trace-input", type=Path)
parser.add_argument("--abi-input-manifest", type=Path)
parser.add_argument("--table-binary", type=Path)
args = parser.parse_args()
assembly = assemble(args.source.read_text(encoding="utf-8"), args.target)
table_payload = args.table_binary.read_bytes() if args.table_binary is not None else None
assembly = assemble(args.source.read_text(encoding="utf-8"), args.target, table_payload)
args.output.write_bytes(assembly.bundle)
artifacts = {
args.symbols: assembly.symbols,
+133
View File
@@ -0,0 +1,133 @@
#!/usr/bin/env python3
"""Generate the frozen APU-P5 32-phase, 16-tap Q2.30 coefficient banks."""
from __future__ import annotations
import argparse
import hashlib
import json
import struct
from decimal import Decimal, ROUND_HALF_EVEN, localcontext
from pathlib import Path
def _pi(precision: int) -> Decimal:
with localcontext() as context:
context.prec = precision + 16
multiplier = Decimal(1)
linear = Decimal(13591409)
exponent = Decimal(1)
total = linear
k_value = Decimal(6)
terms = precision // 14 + 3
for index in range(1, terms):
divisor = Decimal(index) ** 3
multiplier = multiplier * (k_value**3 - Decimal(16) * k_value) / divisor
linear += Decimal(545140134)
exponent *= Decimal(-262537412640768000)
total += multiplier * linear / exponent
k_value += Decimal(12)
value = Decimal(426880) * Decimal(10005).sqrt() / total
context.prec = precision
return +value
def _sin(value: Decimal, pi_value: Decimal, precision: int) -> Decimal:
with localcontext() as context:
context.prec = precision + 12
period = Decimal(2) * pi_value
value %= period
if value > pi_value:
value -= period
term = value
total = value
index = 1
threshold = Decimal(10) ** Decimal(-(precision + 4))
while abs(term) > threshold:
term *= -(value * value) / Decimal((2 * index) * (2 * index + 1))
total += term
index += 1
context.prec = precision
return +total
def _cos(value: Decimal, pi_value: Decimal, precision: int) -> Decimal:
return _sin(value + pi_value / Decimal(2), pi_value, precision)
def coefficient_words(precision: int = 112) -> list[int]:
with localcontext() as context:
context.prec = precision
pi_value = _pi(precision)
banks = (Decimal(1), Decimal(2) / Decimal(3), Decimal(1) / Decimal(2))
words: list[int] = []
for cutoff in banks:
for phase in range(32):
amplitudes: list[Decimal] = []
for tap in range(16):
x_value = Decimal(tap - 7) - Decimal(phase) / Decimal(32)
argument = cutoff * x_value
sinc = (
Decimal(1)
if argument == 0
else _sin(pi_value * argument, pi_value, precision) /
(pi_value * argument)
)
window = (
Decimal(1) -
_cos(
Decimal(2) * pi_value * (Decimal(tap) + Decimal("0.5")) /
Decimal(16),
pi_value,
precision,
)
) / Decimal(2)
amplitudes.append(cutoff * sinc * window)
scale = sum(amplitudes)
phase_words = [
int((Decimal(1 << 30) * value / scale).to_integral_value(
rounding=ROUND_HALF_EVEN
))
for value in amplitudes
]
phase_words[7] += (1 << 30) - sum(phase_words)
words.extend(phase_words)
return words
def coefficient_bytes() -> bytes:
first = coefficient_words(112)
independent = coefficient_words(160)
if first != independent:
raise RuntimeError("coefficient rounding did not converge across precision bounds")
return b"".join(struct.pack("<i", word) for word in first)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--manifest", type=Path, required=True)
args = parser.parse_args()
payload = coefficient_bytes()
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_bytes(payload)
args.manifest.write_text(
json.dumps(
{
"banks": 3,
"phases": 32,
"taps": 16,
"words": len(payload) // 4,
"bytes": len(payload),
"sha256": hashlib.sha256(payload).hexdigest(),
},
indent=2,
sort_keys=True,
) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env python3
"""Build the deterministic APU-P5 codec bundle and evidence reports."""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from apu_mcasm import _artifact_data, assemble # noqa: E402
from apu_p5_coefficients import coefficient_bytes # noqa: E402
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output-dir", type=Path, required=True)
args = parser.parse_args()
output = args.output_dir.resolve()
output.mkdir(parents=True, exist_ok=True)
coefficients = coefficient_bytes()
source = ROOT / "rtl/ip/multimedia/apu_p5_codecs.apus"
assembly = assemble(source.read_text(encoding="utf-8"), "p5", coefficients)
bundle_path = output / "apu-p5.apumc"
coefficient_path = output / "apu-p5-coefficients.bin"
bundle_path.write_bytes(assembly.bundle)
coefficient_path.write_bytes(coefficients)
(output / "symbols.json").write_text(
json.dumps(assembly.symbols, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
high_water = {
"instruction_words": len(assembly.instructions),
"control_store_bytes": len(assembly.instructions) * 8,
"table_bytes": len(coefficients),
"maximum_scratch_end": max(
entry.scratch_base + entry.scratch_bytes for entry in assembly.entries
),
"entries": [
{
"format_id": entry.format_id,
"first_pc": entry.first_pc,
"last_pc": entry.last_pc,
"instruction_span": entry.last_pc - entry.first_pc + 1,
"scratch_end": entry.scratch_base + entry.scratch_bytes,
"maximum_loop_count": entry.max_loop_count,
"maximum_retired": entry.max_retired,
}
for entry in assembly.entries
],
}
(output / "high-water.json").write_text(
json.dumps(high_water, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
(output / "manifest.json").write_text(
json.dumps(
{
**_artifact_data(assembly),
"target": "p5",
"coefficient_words": len(coefficients) // 4,
"entries": [entry.__dict__ for entry in assembly.entries],
"sha256": {
"bundle": hashlib.sha256(assembly.bundle).hexdigest(),
"coefficients": hashlib.sha256(coefficients).hexdigest(),
"source": hashlib.sha256(source.read_bytes()).hexdigest(),
},
"high_water": high_water,
},
indent=2,
sort_keys=True,
)
+ "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
"""Classify the pinned FLAC corpus and record independent PCM truth."""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from apu_codecs import CodecError, decode_flac # noqa: E402
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for block in iter(lambda: source.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def _decode_hash(flac: Path, source: Path) -> tuple[int, str, str]:
command = [
str(flac),
"--decode",
"--stdout",
"--force-raw-format",
"--endian=little",
"--sign=signed",
"--silent",
str(source),
]
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.stdout is None or process.stderr is None:
raise RuntimeError("libFLAC subprocess pipes are unavailable")
digest = hashlib.sha256()
for block in iter(lambda: process.stdout.read(1024 * 1024), b""):
digest.update(block)
stderr = process.stderr.read().decode("utf-8", errors="replace")
return process.wait(), digest.hexdigest(), stderr
def _classify(flac: Path, corpus: Path, path: Path) -> dict[str, object]:
payload = path.read_bytes()
relative = path.relative_to(corpus).as_posix()
profile = "supported"
reason = 0
geometry: dict[str, int | None] | None = None
model_pcm_hash: str | None = None
try:
decoded = decode_flac(payload, strict=True)
info = decoded.info
geometry = {
"rate": info.rate,
"channels": info.channels,
"bits": info.bits,
"samples": info.samples,
}
pcm = hashlib.sha256()
sample_bytes = info.bits // 8
for frame in decoded.samples:
for sample in frame:
pcm.update(sample.to_bytes(sample_bytes, "little", signed=True))
model_pcm_hash = pcm.hexdigest()
except CodecError as error:
profile = "unsupported" if error.code == 3 else "malformed"
reason = error.reason
model_result = profile
return_code, pcm_hash, stderr = _decode_hash(flac, path)
lower_error = stderr.lower()
md5_mismatch = "md5" in lower_error and "mismatch" in lower_error
reference_valid = return_code == 0 or md5_mismatch
if not reference_valid:
profile = "malformed"
if reason == 0:
reason = 0x0030 if "crc" in lower_error else 0x0017
agreement = not (model_result == "supported" and not reference_valid)
if model_result == "supported" and reference_valid and model_pcm_hash != pcm_hash:
agreement = False
if relative.startswith("subset/") and reference_valid and model_result == "malformed":
agreement = False
return {
"path": relative,
"sha256": hashlib.sha256(payload).hexdigest(),
"expected": profile,
"model_result": model_result,
"reference_valid": reference_valid,
"model_reference_agreement": agreement,
"profile_reason": reason,
"geometry": geometry,
"reference_pcm_sha256": pcm_hash,
"model_pcm_sha256": model_pcm_hash,
"reference_integrity": "md5_mismatch"
if md5_mismatch
else ("passed" if return_code == 0 else "decode_failed"),
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--flac", type=Path, required=True)
parser.add_argument("--corpus", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
flac = args.flac.resolve()
corpus = args.corpus.resolve()
output = args.output.resolve()
if not flac.is_file():
raise SystemExit(f"libFLAC executable not found: {flac}")
if not corpus.is_dir():
raise SystemExit(f"FLAC corpus not found: {corpus}")
files = sorted(corpus.rglob("*.flac"))
if not files:
raise SystemExit("pinned FLAC corpus contains no .flac files")
records = [_classify(flac, corpus, path) for path in files]
lock = json.loads((ROOT / "dependencies/dependencies.lock.json").read_text(encoding="utf-8"))
counts = {
category: sum(record["expected"] == category for record in records)
for category in ("supported", "unsupported", "malformed")
}
manifest = {
"schema_version": 1,
"source_revision": lock["sources"]["apu_flac_corpus"]["revision"],
"archive_sha256": lock["archives"]["apu_flac_corpus"]["sha256"],
"corpus_root_sha256": _sha256_file(corpus / "LICENSE.txt"),
"file_count": len(records),
"counts": counts,
"files": records,
}
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"APU-P5 FLAC corpus: {len(records)} files -> {output}")
disagreements = [
record["path"] for record in records if not record["model_reference_agreement"]
]
if disagreements:
print(f"APU-P5 model/libFLAC disagreements: {len(disagreements)}", file=sys.stderr)
for path in disagreements[:20]:
print(f" {path}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
+1
View File
@@ -94,6 +94,7 @@ def main() -> int:
"-Wextra",
"-Werror",
"-fno-builtin",
"-DRS_APU_TEST_MMIO",
"-I",
str(memory_map_root / "include"),
"-I",
+109
View File
@@ -0,0 +1,109 @@
#!/usr/bin/env python3
"""Install and build the pinned host-only APU FLAC references."""
from __future__ import annotations
import argparse
import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from scripts.dependency_lock import archive, source # noqa: E402
from scripts.install_toolchain import safe_extract # noqa: E402
from scripts.setup_helpers import atomic_write, download_file # noqa: E402
def _install_archive(name: str, *, update: bool, timeout: int) -> Path:
source_spec = source(name)
archive_spec = archive(name)
archive_path = ROOT / archive_spec["destination"]
destination = ROOT / source_spec["destination"]
marker = destination / ".retrosoc-archive-sha256"
download_file(
archive_spec["url"],
archive_path,
archive_spec["sha256"],
update=update,
timeout=timeout,
resume=name == "apu_flac_corpus",
)
if marker.is_file() and marker.read_text(encoding="utf-8").strip() == archive_spec["sha256"]:
return destination
if destination.exists() and not update:
raise RuntimeError(f"stale APU reference source: {destination}; rerun with --update")
destination.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(
prefix=f".{destination.name}.", dir=destination.parent
) as temp:
extracted = Path(temp) / "extracted"
extracted.mkdir()
safe_extract(archive_path, extracted)
roots = [path for path in extracted.iterdir() if path.is_dir()]
if len(roots) != 1:
raise RuntimeError(f"{name} archive must contain one source root")
atomic_write(roots[0] / ".retrosoc-archive-sha256", archive_spec["sha256"] + "\n")
if destination.exists():
shutil.rmtree(destination)
os.replace(roots[0], destination)
return destination
def _validate_notices(libflac: Path, corpus: Path) -> None:
for name in ("COPYING.Xiph", "COPYING.GPL", "COPYING.LGPL", "COPYING.FDL"):
if not (libflac / name).is_file():
raise RuntimeError(f"libFLAC notice is missing: {name}")
if not (corpus / "LICENSE.txt").is_file() or not (corpus / "README.txt").is_file():
raise RuntimeError("FLAC corpus license or attribution is missing")
def _build_libflac(source_dir: Path, build_dir: Path) -> None:
build_dir.mkdir(parents=True, exist_ok=True)
subprocess.run(
[
"cmake",
"-S",
str(source_dir),
"-B",
str(build_dir),
"-G",
"Ninja",
"-DWITH_OGG=OFF",
"-DBUILD_CXXLIBS=OFF",
"-DBUILD_PROGRAMS=ON",
"-DBUILD_EXAMPLES=OFF",
"-DBUILD_TESTING=OFF",
"-DBUILD_DOCS=OFF",
"-DINSTALL_MANPAGES=OFF",
"-DBUILD_SHARED_LIBS=OFF",
],
check=True,
)
subprocess.run(["cmake", "--build", str(build_dir), "--target", "flac"], check=True)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--build-dir", type=Path, required=True)
parser.add_argument("--update", action="store_true")
args = parser.parse_args()
libflac = _install_archive("apu_libflac", update=args.update, timeout=120)
corpus = _install_archive("apu_flac_corpus", update=args.update, timeout=1800)
_validate_notices(libflac, corpus)
_build_libflac(libflac, args.build_dir.resolve())
print(f"APU libFLAC reference: {args.build_dir.resolve() / 'src/flac/flac'}")
print(f"APU FLAC corpus: {corpus}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+21 -10
View File
@@ -93,6 +93,7 @@ def download_file(
update: bool = False,
retries: int = 3,
timeout: int = 30,
resume: bool = False,
) -> None:
destination = destination.resolve()
if destination.is_file():
@@ -107,22 +108,31 @@ def download_file(
destination.parent.mkdir(parents=True, exist_ok=True)
last_error: Exception | None = None
partial = destination.with_name(f".{destination.name}.partial") if resume else None
for attempt in range(1, retries + 1):
fd, temporary_name = tempfile.mkstemp(
prefix=f".{destination.name}.", dir=destination.parent
)
os.close(fd)
temporary = Path(temporary_name)
try:
request = urllib.request.Request(
url, headers={"User-Agent": "retroSoC-setup/1"}
if partial is None:
fd, temporary_name = tempfile.mkstemp(
prefix=f".{destination.name}.", dir=destination.parent
)
os.close(fd)
temporary = Path(temporary_name)
else:
temporary = partial
try:
received = temporary.stat().st_size if temporary.is_file() else 0
headers = {"User-Agent": "retroSoC-setup/1"}
if received:
headers["Range"] = f"bytes={received}-"
request = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(request, timeout=timeout) as response:
with temporary.open("wb") as output:
append = received != 0 and response.status == 206
with temporary.open("ab" if append else "wb") as output:
while chunk := response.read(1024 * 1024):
output.write(chunk)
actual = sha256(temporary)
if actual != expected_sha256:
if resume:
temporary.unlink(missing_ok=True)
raise RuntimeError(
f"checksum mismatch for {url}: {actual}, expected {expected_sha256}"
)
@@ -131,7 +141,8 @@ def download_file(
return
except Exception as error:
last_error = error
temporary.unlink(missing_ok=True)
if not resume:
temporary.unlink(missing_ok=True)
if attempt < retries:
time.sleep(attempt)
raise RuntimeError(f"failed to download {url}: {last_error}")