From e901e02694cb322f9e35e08ea70156c35240cb7f Mon Sep 17 00:00:00 2001 From: maksyuki Date: Wed, 30 Sep 2026 16:30:47 +0800 Subject: [PATCH] feat: enhance dependency management with fallback URLs and download functionality --- dependencies/README.md | 5 +++++ dependencies/dependencies.lock.json | 5 ++++- scripts/README.md | 2 ++ scripts/dependency_lock.py | 10 +++++++++ scripts/setup_helpers.py | 32 +++++++++++++++++++++++++++ scripts/setup_npu_reference.py | 12 ++++++++-- tests/test_npu_setup.py | 9 +++++++- tests/test_script_tools.py | 34 ++++++++++++++++++++++++++++- 8 files changed, 104 insertions(+), 5 deletions(-) diff --git a/dependencies/README.md b/dependencies/README.md index 86523fcb..d697eab4 100644 --- a/dependencies/README.md +++ b/dependencies/README.md @@ -55,3 +55,8 @@ verification inputs. Install them with `make setup-apu-reference`; neither is linked into firmware, RTL, or the shipped APUMC bundle. Run `make CONFIG=configs/ci/ihp130.mk apu-p5-corpus` to produce the checksum-pinned per-file profile and independent PCM manifest. + +The NPU Visual Wake Words archive is mirrored as an unmodified release asset in +`retroSoC/artifact`, with the original Silicon Labs URL retained as a fallback. +Both locations resolve to the same required SHA-256; setup accepts neither a +different archive nor a fallback with a non-HTTPS URL. diff --git a/dependencies/dependencies.lock.json b/dependencies/dependencies.lock.json index 12fa8b02..a96ae43c 100644 --- a/dependencies/dependencies.lock.json +++ b/dependencies/dependencies.lock.json @@ -239,7 +239,10 @@ "license": "CC-BY-4.0" }, "npu_vww_dataset": { - "url": "https://www.silabs.com/public/files/github/machine_learning/benchmarks/datasets/vw_coco2014_96.tar.gz", + "url": "https://github.com/retroSoC/artifact/releases/download/npu-vww-dataset-f8746b9e44f8/vw_coco2014_96.tar.gz", + "fallback_urls": [ + "https://www.silabs.com/public/files/github/machine_learning/benchmarks/datasets/vw_coco2014_96.tar.gz" + ], "sha256": "f8746b9e44f8a7a4293f73be9ba6e8da9239fe69798d42364aae62b915cfab58", "destination": ".cache/retrosoc/downloads/npu/vw-coco2014-96.tar.gz", "license": "NOASSERTION" diff --git a/scripts/README.md b/scripts/README.md index 2b3ace72..16e73a1d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -63,6 +63,8 @@ Only raw model parsing is shared with the NPU compiler. Run `make CONFIG=configs/ci/ihp130.mk npu-p0-qualify` after `make CONFIG=configs/ci/ihp130.mk setup-npu-reference` to compare both complete 1000-input corpora against the Python graph reference and compiled executor. +The VWW archive is fetched from the ordered HTTPS URLs in the dependency lock; +every mirror must produce the same locked SHA-256 before extraction. The selected variant's `npu/p0/` directory retains oracle build provenance, per-input golden tensors, three-way hashes, logs and `qualification-p0.json`. The runner uses at most 16 workers (`JOBS`); missing sources/tools, numerical diff --git a/scripts/dependency_lock.py b/scripts/dependency_lock.py index 947c3e3f..bddefaa2 100644 --- a/scripts/dependency_lock.py +++ b/scripts/dependency_lock.py @@ -82,6 +82,16 @@ def _validate_archive(kind: str, name: str, value: dict[str, Any]) -> None: _require_fields(kind, name, value, ("url", "sha256", "destination", "license")) _validate_sha(f"{kind} {name}", value["sha256"]) _validate_url(f"{kind} {name}", value["url"]) + fallback_urls = value.get("fallback_urls", []) + if not isinstance(fallback_urls, list): + raise LockError(f"{kind} {name} fallback_urls must be a list") + for index, url in enumerate(fallback_urls): + if not isinstance(url, str): + raise LockError(f"{kind} {name} fallback_urls must contain URLs") + _validate_url(f"{kind} {name} fallback_urls[{index}]", url) + urls = [value["url"], *fallback_urls] + if len(set(urls)) != len(urls): + raise LockError(f"{kind} {name} download URLs must be unique") _validate_relative_path(f"{kind} {name} destination", value["destination"]) diff --git a/scripts/setup_helpers.py b/scripts/setup_helpers.py index 65708b2a..87d14c12 100644 --- a/scripts/setup_helpers.py +++ b/scripts/setup_helpers.py @@ -148,6 +148,38 @@ def download_file( raise RuntimeError(f"failed to download {url}: {last_error}") +def download_file_from_urls( + urls: Iterable[str], + destination: Path, + expected_sha256: str, + *, + update: bool = False, + retries: int = 3, + timeout: int = 30, + resume: bool = False, +) -> None: + candidates = tuple(urls) + if not candidates: + raise ValueError("at least one download URL is required") + failures: list[str] = [] + for url in candidates: + try: + download_file( + url, + destination, + expected_sha256, + update=update, + retries=retries, + timeout=timeout, + resume=resume, + ) + return + except RuntimeError as error: + failures.append(str(error)) + print(f"[dependency] mirror failed: {url}: {error}") + raise RuntimeError("all download mirrors failed: " + "; ".join(failures)) + + def atomic_write(path: Path, content: str) -> bool: path = path.resolve() if path.exists() and path.read_text(encoding="utf-8") == content: diff --git a/scripts/setup_npu_reference.py b/scripts/setup_npu_reference.py index ee9ac2c6..721fb03d 100644 --- a/scripts/setup_npu_reference.py +++ b/scripts/setup_npu_reference.py @@ -14,7 +14,12 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from scripts.dependency_lock import archive, source # noqa: E402 -from scripts.setup_helpers import atomic_write, download_file, ensure_git_repo, sha256 # noqa: E402 +from scripts.setup_helpers import ( # noqa: E402 + atomic_write, + download_file_from_urls, + ensure_git_repo, + sha256, +) MLCOMMONS_TINY = "apu_mlperf_tiny" KWS_MFCC = "apu_kws_mfcc" @@ -151,7 +156,10 @@ def install(build_dir: Path, *, update: bool) -> dict[str, object]: ensure_git_repo(spec["url"], ROOT / spec["destination"], spec["revision"], update=update) dataset = archive(VWW_DATASET) archive_path = ROOT / dataset["destination"] - download_file(dataset["url"], archive_path, dataset["sha256"], update=update, timeout=600) + urls = (dataset["url"], *dataset.get("fallback_urls", ())) + download_file_from_urls( + urls, archive_path, dataset["sha256"], update=update, timeout=600 + ) models = verify_models(tiny) kws_names = verify_kws_features(mfcc) diff --git a/tests/test_npu_setup.py b/tests/test_npu_setup.py index b7968990..4d93e620 100644 --- a/tests/test_npu_setup.py +++ b/tests/test_npu_setup.py @@ -22,7 +22,14 @@ def test_lock_carries_npu_vww_dataset_entry() -> None: dataset["sha256"] == "f8746b9e44f8a7a4293f73be9ba6e8da9239fe69798d42364aae62b915cfab58" ) - assert dataset["url"].endswith("vw_coco2014_96.tar.gz") + assert dataset["url"] == ( + "https://github.com/retroSoC/artifact/releases/download/" + "npu-vww-dataset-f8746b9e44f8/vw_coco2014_96.tar.gz" + ) + assert dataset["fallback_urls"] == [ + "https://www.silabs.com/public/files/github/machine_learning/benchmarks/" + "datasets/vw_coco2014_96.tar.gz" + ] def test_model_hash_constants_match_extraction_module() -> None: diff --git a/tests/test_script_tools.py b/tests/test_script_tools.py index 8b9863e2..1b16a75d 100644 --- a/tests/test_script_tools.py +++ b/tests/test_script_tools.py @@ -59,7 +59,11 @@ from scripts.regress import ( # noqa: E402 ) from scripts import setup_helpers # noqa: E402 from scripts import setup_apu_reference # noqa: E402 -from scripts.setup_helpers import download_file, ensure_git_repo # noqa: E402 +from scripts.setup_helpers import ( # noqa: E402 + download_file, + download_file_from_urls, + ensure_git_repo, +) def run(*command: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: @@ -935,6 +939,22 @@ def test_dependency_helpers_are_idempotent(tmp_path: Path) -> None: assert not partial.exists() +def test_dependency_helper_uses_fallback_url(tmp_path: Path) -> None: + payload = tmp_path / "payload.bin" + payload.write_bytes(b"verified fallback") + digest = hashlib.sha256(payload.read_bytes()).hexdigest() + downloaded = tmp_path / "downloaded.bin" + + download_file_from_urls( + ((tmp_path / "missing.bin").as_uri(), payload.as_uri()), + downloaded, + digest, + retries=1, + ) + + assert downloaded.read_bytes() == payload.read_bytes() + + def test_dependency_helper_limits_recursive_submodules(monkeypatch, tmp_path: Path) -> None: commands: list[tuple[str, ...]] = [] revision = "a" * 40 @@ -1164,6 +1184,18 @@ def test_dependency_lock_and_config_key_include_a_fixed_timestamp(tmp_path: Path else: raise AssertionError("invalid toolchain download timeout was accepted") + invalid_fallback = json.loads(json.dumps(lock)) + invalid_fallback["archives"]["npu_vww_dataset"]["fallback_urls"] = [ + "http://example.com/vww.tar.gz" + ] + broken.write_text(json.dumps(invalid_fallback), encoding="utf-8") + try: + load_lock(broken) + except LockError as error: + assert "fallback_urls[0] URL must use HTTPS" in str(error) + else: + raise AssertionError("invalid archive fallback URL was accepted") + def test_development_environment_contract_is_lock_pinned(tmp_path: Path) -> None: lock_path = ROOT / "dependencies/dependencies.lock.json"