feat: enhance dependency management with fallback URLs and download functionality

dependencies/README.md

@@ -55,3 +55,8 @@ verification inputs. Install them with `make setup-apu-reference`; neither is
 linked into firmware, RTL, or the shipped APUMC bundle. Run
 `make CONFIG=configs/ci/ihp130.mk apu-p5-corpus` to produce the checksum-pinned
 per-file profile and independent PCM manifest.
+
+The NPU Visual Wake Words archive is mirrored as an unmodified release asset in
+`retroSoC/artifact`, with the original Silicon Labs URL retained as a fallback.
+Both locations resolve to the same required SHA-256; setup accepts neither a
+different archive nor a fallback with a non-HTTPS URL.

dependencies/dependencies.lock.json

@@ -239,7 +239,10 @@
       "license": "CC-BY-4.0"
     },
     "npu_vww_dataset": {
-      "url": "https://www.silabs.com/public/files/github/machine_learning/benchmarks/datasets/vw_coco2014_96.tar.gz",
+      "url": "https://github.com/retroSoC/artifact/releases/download/npu-vww-dataset-f8746b9e44f8/vw_coco2014_96.tar.gz",
+      "fallback_urls": [
+        "https://www.silabs.com/public/files/github/machine_learning/benchmarks/datasets/vw_coco2014_96.tar.gz"
+      ],
       "sha256": "f8746b9e44f8a7a4293f73be9ba6e8da9239fe69798d42364aae62b915cfab58",
       "destination": ".cache/retrosoc/downloads/npu/vw-coco2014-96.tar.gz",
       "license": "NOASSERTION"

scripts/README.md

@@ -63,6 +63,8 @@ Only raw model parsing is shared with the NPU compiler. Run
 `make CONFIG=configs/ci/ihp130.mk npu-p0-qualify` after
 `make CONFIG=configs/ci/ihp130.mk setup-npu-reference` to compare both complete
 1000-input corpora against the Python graph reference and compiled executor.
+The VWW archive is fetched from the ordered HTTPS URLs in the dependency lock;
+every mirror must produce the same locked SHA-256 before extraction.
 The selected variant's `npu/p0/` directory retains oracle build provenance,
 per-input golden tensors, three-way hashes, logs and `qualification-p0.json`.
 The runner uses at most 16 workers (`JOBS`); missing sources/tools, numerical

scripts/dependency_lock.py

@@ -82,6 +82,16 @@ def _validate_archive(kind: str, name: str, value: dict[str, Any]) -> None:
     _require_fields(kind, name, value, ("url", "sha256", "destination", "license"))
     _validate_sha(f"{kind} {name}", value["sha256"])
     _validate_url(f"{kind} {name}", value["url"])
+    fallback_urls = value.get("fallback_urls", [])
+    if not isinstance(fallback_urls, list):
+        raise LockError(f"{kind} {name} fallback_urls must be a list")
+    for index, url in enumerate(fallback_urls):
+        if not isinstance(url, str):
+            raise LockError(f"{kind} {name} fallback_urls must contain URLs")
+        _validate_url(f"{kind} {name} fallback_urls[{index}]", url)
+    urls = [value["url"], *fallback_urls]
+    if len(set(urls)) != len(urls):
+        raise LockError(f"{kind} {name} download URLs must be unique")
     _validate_relative_path(f"{kind} {name} destination", value["destination"])
 
 

scripts/setup_helpers.py

@@ -148,6 +148,38 @@ def download_file(
     raise RuntimeError(f"failed to download {url}: {last_error}")
 
 
+def download_file_from_urls(
+    urls: Iterable[str],
+    destination: Path,
+    expected_sha256: str,
+    *,
+    update: bool = False,
+    retries: int = 3,
+    timeout: int = 30,
+    resume: bool = False,
+) -> None:
+    candidates = tuple(urls)
+    if not candidates:
+        raise ValueError("at least one download URL is required")
+    failures: list[str] = []
+    for url in candidates:
+        try:
+            download_file(
+                url,
+                destination,
+                expected_sha256,
+                update=update,
+                retries=retries,
+                timeout=timeout,
+                resume=resume,
+            )
+            return
+        except RuntimeError as error:
+            failures.append(str(error))
+            print(f"[dependency] mirror failed: {url}: {error}")
+    raise RuntimeError("all download mirrors failed: " + "; ".join(failures))
+
+
 def atomic_write(path: Path, content: str) -> bool:
     path = path.resolve()
     if path.exists() and path.read_text(encoding="utf-8") == content:

scripts/setup_npu_reference.py

@@ -14,7 +14,12 @@ ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT))
 
 from scripts.dependency_lock import archive, source  # noqa: E402
-from scripts.setup_helpers import atomic_write, download_file, ensure_git_repo, sha256  # noqa: E402
+from scripts.setup_helpers import (  # noqa: E402
+    atomic_write,
+    download_file_from_urls,
+    ensure_git_repo,
+    sha256,
+)
 
 MLCOMMONS_TINY = "apu_mlperf_tiny"
 KWS_MFCC = "apu_kws_mfcc"
@@ -151,7 +156,10 @@ def install(build_dir: Path, *, update: bool) -> dict[str, object]:
         ensure_git_repo(spec["url"], ROOT / spec["destination"], spec["revision"], update=update)
     dataset = archive(VWW_DATASET)
     archive_path = ROOT / dataset["destination"]
-    download_file(dataset["url"], archive_path, dataset["sha256"], update=update, timeout=600)
+    urls = (dataset["url"], *dataset.get("fallback_urls", ()))
+    download_file_from_urls(
+        urls, archive_path, dataset["sha256"], update=update, timeout=600
+    )
 
     models = verify_models(tiny)
     kws_names = verify_kws_features(mfcc)

tests/test_npu_setup.py

@@ -22,7 +22,14 @@ def test_lock_carries_npu_vww_dataset_entry() -> None:
         dataset["sha256"]
         == "f8746b9e44f8a7a4293f73be9ba6e8da9239fe69798d42364aae62b915cfab58"
     )
-    assert dataset["url"].endswith("vw_coco2014_96.tar.gz")
+    assert dataset["url"] == (
+        "https://github.com/retroSoC/artifact/releases/download/"
+        "npu-vww-dataset-f8746b9e44f8/vw_coco2014_96.tar.gz"
+    )
+    assert dataset["fallback_urls"] == [
+        "https://www.silabs.com/public/files/github/machine_learning/benchmarks/"
+        "datasets/vw_coco2014_96.tar.gz"
+    ]
 
 
 def test_model_hash_constants_match_extraction_module() -> None:

tests/test_script_tools.py

@@ -59,7 +59,11 @@ from scripts.regress import (  # noqa: E402
 )
 from scripts import setup_helpers  # noqa: E402
 from scripts import setup_apu_reference  # noqa: E402
-from scripts.setup_helpers import download_file, ensure_git_repo  # noqa: E402
+from scripts.setup_helpers import (  # noqa: E402
+    download_file,
+    download_file_from_urls,
+    ensure_git_repo,
+)
 
 
 def run(*command: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
@@ -935,6 +939,22 @@ def test_dependency_helpers_are_idempotent(tmp_path: Path) -> None:
     assert not partial.exists()
 
 
+def test_dependency_helper_uses_fallback_url(tmp_path: Path) -> None:
+    payload = tmp_path / "payload.bin"
+    payload.write_bytes(b"verified fallback")
+    digest = hashlib.sha256(payload.read_bytes()).hexdigest()
+    downloaded = tmp_path / "downloaded.bin"
+
+    download_file_from_urls(
+        ((tmp_path / "missing.bin").as_uri(), payload.as_uri()),
+        downloaded,
+        digest,
+        retries=1,
+    )
+
+    assert downloaded.read_bytes() == payload.read_bytes()
+
+
 def test_dependency_helper_limits_recursive_submodules(monkeypatch, tmp_path: Path) -> None:
     commands: list[tuple[str, ...]] = []
     revision = "a" * 40
@@ -1164,6 +1184,18 @@ def test_dependency_lock_and_config_key_include_a_fixed_timestamp(tmp_path: Path
     else:
         raise AssertionError("invalid toolchain download timeout was accepted")
 
+    invalid_fallback = json.loads(json.dumps(lock))
+    invalid_fallback["archives"]["npu_vww_dataset"]["fallback_urls"] = [
+        "http://example.com/vww.tar.gz"
+    ]
+    broken.write_text(json.dumps(invalid_fallback), encoding="utf-8")
+    try:
+        load_lock(broken)
+    except LockError as error:
+        assert "fallback_urls[0] URL must use HTTPS" in str(error)
+    else:
+        raise AssertionError("invalid archive fallback URL was accepted")
+
 
 def test_development_environment_contract_is_lock_pinned(tmp_path: Path) -> None:
     lock_path = ROOT / "dependencies/dependencies.lock.json"