feat: introduce Docker and Nix support for retroSoC development environment

- Added .dockerignore to exclude unnecessary files from Docker context.
- Updated Makefile to include a new setup-regression target for installing dependencies for all PR PDK profiles.
- Enhanced README with installation instructions for Docker and Nix environments.
- Created Dockerfile for a lock-pinned Ubuntu 22.04 development image.
- Added entrypoint script for Docker container to set up the environment.
- Introduced flake.lock and flake.nix for Nix package management.
- Updated dependency_lock.py to validate Nix inputs and container images in the lock file.
- Implemented development_environment.py for managing the development environment setup.
- Enhanced package.py to include container components in the SBOM.
- Added tests to ensure the development environment contract is lock-pinned and validate container and Nix environment files.

scripts/README.md

@@ -8,6 +8,11 @@ Scripts are part of the build contract. Prefer existing helpers over ad-hoc
 shell behavior, preserve structured JSON results, and keep setup/download
 behavior controlled by `config/dependencies.lock.json`.
 
+`development_environment.py` is the shared Docker, Nix, and manual bootstrap
+entry point. It installs only the checksum-verified open-source tool bundles and
+hash-pinned Python dependencies; project-local PDK and source setup remains under
+the existing Make targets.
+
 `publish_fatfs_artifact.sh` is the manual release helper for the lock-pinned
 FatFs R0.16 archive. It verifies the archive, GitHub authentication, release
 absence, and the published asset checksum before reporting the checksum-pinned

scripts/dependency_lock.py

@@ -12,6 +12,9 @@ from typing import Any
 
 ROOT = Path(__file__).resolve().parents[1]
 DEFAULT_LOCK = ROOT / "config/dependencies.lock.json"
+DEFAULT_FLAKE_LOCK = ROOT / "flake.lock"
+NIX_SRI_HASH = re.compile(r"sha256-[A-Za-z0-9+/]{43}=")
+OCI_DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
 
 
 class LockError(ValueError):
@@ -31,7 +34,7 @@ def load_lock(path: Path = DEFAULT_LOCK) -> dict[str, Any]:
 def validate_lock(data: dict[str, Any]) -> None:
     if data.get("schema_version") != 1:
         raise LockError("dependency lock schema_version must be 1")
-    for section in ("sources", "archives", "toolchains"):
+    for section in ("sources", "archives", "container_images", "nix_inputs", "toolchains"):
         if not isinstance(data.get(section), dict) or not data[section]:
             raise LockError(f"dependency lock section is missing or empty: {section}")
     for name, source in data["sources"].items():
@@ -43,6 +46,20 @@ def validate_lock(data: dict[str, Any]) -> None:
         _validate_source_submodules(name, source)
     for name, archive in data["archives"].items():
         _validate_archive("archive", name, archive)
+    for name, image in data["container_images"].items():
+        _require_fields("container image", name, image, ("image", "digest", "url", "license"))
+        if not re.fullmatch(r"[a-z0-9]+(?:[._/-][a-z0-9]+)*", image["image"]):
+            raise LockError(f"container image {name} has an invalid image name")
+        if OCI_DIGEST.fullmatch(image["digest"]) is None:
+            raise LockError(f"container image {name} digest must be a SHA-256 OCI digest")
+        _validate_url(f"container image {name}", image["url"])
+    for name, nix_input in data["nix_inputs"].items():
+        _require_fields("nix input", name, nix_input, ("url", "revision", "nar_hash", "license"))
+        if not re.fullmatch(r"[0-9a-f]{40}", nix_input["revision"]):
+            raise LockError(f"nix input {name} revision must be a full 40-character commit")
+        _validate_url(f"nix input {name}", nix_input["url"])
+        if NIX_SRI_HASH.fullmatch(nix_input["nar_hash"]) is None:
+            raise LockError(f"nix input {name} nar_hash must be a SHA-256 SRI hash")
     for platform, tools in data["toolchains"].items():
         if not isinstance(tools, dict) or not tools:
             raise LockError(f"toolchain platform is empty: {platform}")
@@ -99,6 +116,33 @@ def _validate_relative_path(label: str, value: str) -> None:
         raise LockError(f"{label} must be a repository-relative path")
 
 
+def validate_flake_lock(data: dict[str, Any], path: Path = DEFAULT_FLAKE_LOCK) -> None:
+    try:
+        flake = json.loads(path.read_text(encoding="utf-8"))
+        nodes = flake["nodes"]
+        root = nodes[flake["root"]]
+    except (KeyError, OSError, json.JSONDecodeError) as error:
+        raise LockError(f"cannot read flake lock {path}: {error}") from error
+    if not isinstance(nodes, dict) or not isinstance(root, dict):
+        raise LockError("flake lock nodes and root must be objects")
+
+    for name, expected in data["nix_inputs"].items():
+        try:
+            node = nodes[root["inputs"][name]]
+            locked = node["locked"]
+            owner = locked["owner"]
+            repository = locked["repo"]
+        except (KeyError, TypeError) as error:
+            raise LockError(f"flake lock is missing nix input {name}") from error
+        actual_url = f"https://github.com/{owner}/{repository}"
+        if actual_url != expected["url"]:
+            raise LockError(f"flake lock URL mismatch for nix input {name}")
+        if locked.get("rev") != expected["revision"]:
+            raise LockError(f"flake lock revision mismatch for nix input {name}")
+        if locked.get("narHash") != expected["nar_hash"]:
+            raise LockError(f"flake lock narHash mismatch for nix input {name}")
+
+
 def lock_digest(path: Path = DEFAULT_LOCK) -> str:
     return hashlib.sha256(path.resolve().read_bytes()).hexdigest()
 
@@ -122,15 +166,18 @@ def archive(name: str, path: Path = DEFAULT_LOCK) -> dict[str, Any]:
 def main() -> int:
     parser = argparse.ArgumentParser(description="Validate the retroSoC dependency lock")
     parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK)
+    parser.add_argument("--flake-lock", type=Path, default=DEFAULT_FLAKE_LOCK)
     parser.add_argument("--digest", action="store_true")
     args = parser.parse_args()
     data = load_lock(args.lock)
+    validate_flake_lock(data, args.flake_lock)
     if args.digest:
         print(lock_digest(args.lock))
     else:
         print(
             f"dependency lock valid: {len(data['sources'])} sources, "
-            f"{len(data['archives'])} archives"
+            f"{len(data['archives'])} archives, {len(data['container_images'])} container images, "
+            f"{len(data['nix_inputs'])} Nix inputs"
         )
     return 0
 

scripts/development_environment.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""Create and verify the lock-pinned open-source development environment."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shlex
+import shutil
+import subprocess
+import sys
+import venv
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from scripts.dependency_lock import DEFAULT_LOCK, lock_digest, load_lock  # noqa: E402
+from scripts.install_toolchain import install  # noqa: E402
+from scripts.setup_helpers import atomic_write  # noqa: E402
+
+
+PLATFORM = "ubuntu-22.04"
+DEFAULT_TOOLS = (
+    "verilator",
+    "verible",
+    "sv2v",
+    "iverilog",
+    "yosys",
+    "sby",
+    "bitwuzla",
+    "opensta",
+    "riscv_gnu",
+)
+TOOL_EXECUTABLES = {
+    "verilator": ("verilator",),
+    "verible": ("verible-verilog-format",),
+    "sv2v": ("sv2v",),
+    "iverilog": ("iverilog", "vvp"),
+    "yosys": ("yosys", "yosys-smtbmc"),
+    "sby": ("sby",),
+    "bitwuzla": ("bitwuzla",),
+    "opensta": ("sta",),
+    "riscv_gnu": ("riscv32-unknown-elf-gcc",),
+}
+PYTHON_IMPORTS = ("pytest", "ruff", "yamllint")
+PYTHON_EXECUTABLES = ("mbake",)
+SYSTEM_EXECUTABLES = ("cc", "ccache", "clang-format-14", "git", "make", "mold", "numactl")
+STAMP_NAME = "development-environment.json"
+ACTIVATE_NAME = "activate.sh"
+
+
+def file_digest(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def requirement_paths(root: Path) -> tuple[Path, Path]:
+    return root / "requirements/build.txt", root / "requirements/ci.txt"
+
+
+def parse_tools(values: list[str] | None, available: dict[str, Any]) -> tuple[str, ...]:
+    tools = tuple(values or DEFAULT_TOOLS)
+    if not tools:
+        raise ValueError("at least one tool must be selected")
+    if len(tools) != len(set(tools)):
+        raise ValueError("selected tools must be unique")
+    unknown = sorted(set(tools) - set(available))
+    if unknown:
+        raise ValueError(f"selected tools are not locked for {PLATFORM}: {', '.join(unknown)}")
+    return tools
+
+
+def tool_paths(cache: Path, tools: tuple[str, ...], available: dict[str, Any]) -> list[Path]:
+    return [
+        cache / "toolchains" / f"{name}-{available[name]['version']}" / available[name]["path"]
+        for name in tools
+    ]
+
+
+def environment_paths(cache: Path, tools: tuple[str, ...], available: dict[str, Any]) -> list[Path]:
+    return [cache / "venv" / "bin", *tool_paths(cache, tools, available)]
+
+
+def stamp_data(
+    root: Path, cache: Path, tools: tuple[str, ...], lock: dict[str, Any], lock_path: Path
+) -> dict[str, Any]:
+    build_requirements, ci_requirements = requirement_paths(root)
+    if not build_requirements.is_file() or not ci_requirements.is_file():
+        raise ValueError("development Python requirement files are missing")
+    available = lock["toolchains"][PLATFORM]
+    return {
+        "schema_version": 1,
+        "cache": str(cache),
+        "dependency_lock_sha256": lock_digest(lock_path),
+        "python_requirements": {
+            str(build_requirements.relative_to(root)): file_digest(build_requirements),
+            str(ci_requirements.relative_to(root)): file_digest(ci_requirements),
+        },
+        "tools": {name: available[name]["version"] for name in tools},
+    }
+
+
+def read_stamp(path: Path) -> dict[str, Any] | None:
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+    return value if isinstance(value, dict) else None
+
+
+def has_environment(
+    cache: Path, tools: tuple[str, ...], available: dict[str, Any], expected_stamp: dict[str, Any]
+) -> bool:
+    if read_stamp(cache / STAMP_NAME) != expected_stamp:
+        return False
+    python = cache / "venv" / "bin" / "python"
+    if not python.is_file():
+        return False
+    result = subprocess.run(
+        [str(python), "-c", "import " + ", ".join(PYTHON_IMPORTS)],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode:
+        return False
+    if any(not (cache / "venv" / "bin" / executable).is_file() for executable in PYTHON_EXECUTABLES):
+        return False
+    for path in tool_paths(cache, tools, available):
+        if not path.is_dir():
+            return False
+    return True
+
+
+def render_activation(cache: Path, paths: list[Path]) -> str:
+    quoted_path = ":".join(shlex.quote(str(path.resolve())) for path in paths)
+    return "\n".join(
+        (
+            "# Generated by scripts/development_environment.py; do not edit.",
+            f"export RETROSOC_DEVELOPMENT_CACHE={shlex.quote(str(cache.resolve()))}",
+            f"export VIRTUAL_ENV={shlex.quote(str((cache / 'venv').resolve()))}",
+            f"export PATH={quoted_path}:$PATH",
+            "",
+        )
+    )
+
+
+def run(command: list[str]) -> None:
+    print("+ " + " ".join(shlex.quote(argument) for argument in command), flush=True)
+    subprocess.run(command, check=True)
+
+
+def create_virtualenv(root: Path, cache: Path) -> None:
+    python = cache / "venv" / "bin" / "python"
+    if not python.is_file():
+        venv.EnvBuilder(with_pip=True, clear=False).create(cache / "venv")
+    build_requirements, ci_requirements = requirement_paths(root)
+    run(
+        [
+            str(python),
+            "-m",
+            "pip",
+            "install",
+            "--disable-pip-version-check",
+            "--require-hashes",
+            "--requirement",
+            str(build_requirements),
+            "--requirement",
+            str(ci_requirements),
+        ]
+    )
+
+
+def bootstrap(args: argparse.Namespace) -> int:
+    root = args.root.resolve()
+    cache = args.cache.resolve()
+    lock_path = args.lock.resolve()
+    lock = load_lock(lock_path)
+    available = lock["toolchains"][PLATFORM]
+    tools = parse_tools(args.tool, available)
+    expected_stamp = stamp_data(root, cache, tools, lock, lock_path)
+
+    if not has_environment(cache, tools, available, expected_stamp):
+        for name in tools:
+            install(name, available[name], cache, update=args.update)
+        create_virtualenv(root, cache)
+        atomic_write(cache / STAMP_NAME, json.dumps(expected_stamp, indent=2, sort_keys=True) + "\n")
+    activation = args.output.resolve() if args.output else cache / ACTIVATE_NAME
+    atomic_write(activation, render_activation(cache, environment_paths(cache, tools, available)))
+    print(f"development environment: {activation}")
+    return check(args, quiet=False)
+
+
+def check(args: argparse.Namespace, *, quiet: bool) -> int:
+    cache = args.cache.resolve()
+    lock_path = args.lock.resolve()
+    lock = load_lock(lock_path)
+    available = lock["toolchains"][PLATFORM]
+    tools = parse_tools(args.tool, available)
+    expected_stamp = stamp_data(args.root.resolve(), cache, tools, lock, lock_path)
+    failures: list[str] = []
+
+    if not has_environment(cache, tools, available, expected_stamp):
+        failures.append("development environment stamp or installed paths do not match the lock")
+    for name in tools:
+        for executable in TOOL_EXECUTABLES[name]:
+            if not (tool_paths(cache, (name,), available)[0] / executable).is_file():
+                failures.append(f"missing executable: {executable}")
+    python = cache / "venv" / "bin" / "python"
+    if python.is_file():
+        result = subprocess.run(
+            [str(python), "-c", "import " + ", ".join(PYTHON_IMPORTS)],
+            check=False,
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode:
+            failures.append("locked Python development packages are unavailable")
+    else:
+        failures.append("missing virtualenv Python executable")
+    for executable in SYSTEM_EXECUTABLES:
+        if shutil.which(executable) is None:
+            failures.append(f"missing system executable: {executable}")
+
+    if not quiet:
+        if failures:
+            print("development environment check failed:")
+            for failure in failures:
+                print(f"  {failure}")
+        else:
+            print(f"development environment check passed: {cache}")
+    return int(bool(failures))
+
+
+def activate(args: argparse.Namespace) -> int:
+    activation = args.output.resolve() if args.output else args.cache.resolve() / ACTIVATE_NAME
+    if not activation.is_file():
+        raise ValueError(f"activation script does not exist: {activation}")
+    print(f"source {shlex.quote(str(activation))}")
+    return 0
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path, default=ROOT)
+    parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK)
+    parser.add_argument("--cache", type=Path, default=ROOT / ".cache/retrosoc/development")
+    parser.add_argument("--output", type=Path)
+    parser.add_argument("--tool", action="append")
+    parser.add_argument("--update", action="store_true")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    subparsers.add_parser("bootstrap")
+    subparsers.add_parser("check")
+    subparsers.add_parser("activate")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    try:
+        if args.command == "bootstrap":
+            return bootstrap(args)
+        if args.command == "check":
+            return check(args, quiet=False)
+        return activate(args)
+    except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as error:
+        print(f"development environment: {error}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/package.py

@@ -33,15 +33,40 @@ def component(name: str, kind: str, spec: dict[str, object]) -> dict[str, object
     if "revision" in spec:
         item["version"] = spec["revision"]
         item["properties"] = [{"name": "vcs:revision", "value": spec["revision"]}]
+        if "nar_hash" in spec:
+            item["properties"].append({"name": "nix:narHash", "value": spec["nar_hash"]})
     else:
         item["version"] = spec.get("version", spec.get("sha256", "unknown"))
         item["hashes"] = [{"alg": "SHA-256", "content": spec["sha256"]}]
     return item
 
 
+def container_component(name: str, spec: dict[str, object]) -> dict[str, object]:
+    license_name = str(spec.get("license", "NOASSERTION"))
+    license_value = (
+        {"name": license_name}
+        if license_name == "NOASSERTION" or license_name.startswith("LicenseRef-")
+        else {"id": license_name}
+    )
+    digest = str(spec["digest"])
+    return {
+        "type": "container",
+        "name": f"container/{name}",
+        "version": digest.removeprefix("sha256:"),
+        "licenses": [{"license": license_value}],
+        "externalReferences": [{"type": "distribution", "url": spec["url"]}],
+        "hashes": [{"alg": "SHA-256", "content": digest.removeprefix("sha256:")}],
+        "properties": [{"name": "oci:image", "value": spec["image"]}],
+    }
+
+
 def make_sbom(lock: dict[str, object]) -> dict[str, object]:
     components = [component(name, "source", spec) for name, spec in lock["sources"].items()]
     components.extend(component(name, "archive", spec) for name, spec in lock["archives"].items())
+    components.extend(
+        container_component(name, spec) for name, spec in lock["container_images"].items()
+    )
+    components.extend(component(f"nix/{name}", "source", spec) for name, spec in lock["nix_inputs"].items())
     for platform, tools in lock["toolchains"].items():
         components.extend(
             component(f"{platform}/{name}", "toolchain", spec) for name, spec in tools.items()