retroSoC/scripts/development_environment.py

#!/usr/bin/env python3
"""Create and verify the lock-pinned open-source development environment."""

from __future__ import annotations

import argparse
import hashlib
import json
import shlex
import shutil
import subprocess
import sys
import venv
from pathlib import Path
from typing import Any

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from scripts.dependency_lock import DEFAULT_LOCK, lock_digest, load_lock  # noqa: E402
from scripts.install_toolchain import install  # noqa: E402
from scripts.setup_helpers import atomic_write  # noqa: E402


PLATFORM = "ubuntu-22.04"
DEFAULT_TOOLS = (
    "verilator",
    "verible",
    "sv2v",
    "iverilog",
    "yosys",
    "sby",
    "bitwuzla",
    "opensta",
    "openocd",
    "sbt",
    "riscv_gnu",
)
TOOL_EXECUTABLES = {
    "verilator": ("verilator",),
    "verible": ("verible-verilog-format",),
    "sv2v": ("sv2v",),
    "iverilog": ("iverilog", "vvp"),
    "yosys": ("yosys", "yosys-smtbmc"),
    "sby": ("sby",),
    "bitwuzla": ("bitwuzla",),
    "opensta": ("sta",),
    "openocd": ("openocd",),
    "sbt": ("sbt",),
    "riscv_gnu": ("riscv32-unknown-elf-gcc", "riscv32-unknown-elf-gdb"),
}
PYTHON_IMPORTS = ("pytest", "ruff", "yamllint")
PYTHON_EXECUTABLES = ("mbake",)
SYSTEM_EXECUTABLES = (
    "cc",
    "ccache",
    "clang-format-14",
    "git",
    "java",
    "make",
    "mold",
    "numactl",
    "perl",
)
STAMP_NAME = "development-environment.json"
ACTIVATE_NAME = "activate.sh"


def file_digest(path: Path) -> str:
    return hashlib.sha256(path.read_bytes()).hexdigest()


def requirement_paths(root: Path) -> tuple[Path, Path]:
    return root / "requirements/build.txt", root / "requirements/ci.txt"


def parse_tools(values: list[str] | None, available: dict[str, Any]) -> tuple[str, ...]:
    tools = tuple(values or DEFAULT_TOOLS)
    if not tools:
        raise ValueError("at least one tool must be selected")
    if len(tools) != len(set(tools)):
        raise ValueError("selected tools must be unique")
    unknown = sorted(set(tools) - set(available))
    if unknown:
        raise ValueError(f"selected tools are not locked for {PLATFORM}: {', '.join(unknown)}")
    return tools


def tool_paths(cache: Path, tools: tuple[str, ...], available: dict[str, Any]) -> list[Path]:
    return [
        cache / "toolchains" / f"{name}-{available[name]['version']}" / available[name]["path"]
        for name in tools
    ]


def environment_paths(cache: Path, tools: tuple[str, ...], available: dict[str, Any]) -> list[Path]:
    return [cache / "venv" / "bin", *tool_paths(cache, tools, available)]


def stamp_data(
    root: Path, cache: Path, tools: tuple[str, ...], lock: dict[str, Any], lock_path: Path
) -> dict[str, Any]:
    build_requirements, ci_requirements = requirement_paths(root)
    if not build_requirements.is_file() or not ci_requirements.is_file():
        raise ValueError("development Python requirement files are missing")
    available = lock["toolchains"][PLATFORM]
    return {
        "schema_version": 1,
        "cache": str(cache),
        "dependency_lock_sha256": lock_digest(lock_path),
        "python_requirements": {
            str(build_requirements.relative_to(root)): file_digest(build_requirements),
            str(ci_requirements.relative_to(root)): file_digest(ci_requirements),
        },
        "tools": {name: available[name]["version"] for name in tools},
    }


def read_stamp(path: Path) -> dict[str, Any] | None:
    try:
        value = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return None
    return value if isinstance(value, dict) else None


def has_environment(
    cache: Path, tools: tuple[str, ...], available: dict[str, Any], expected_stamp: dict[str, Any]
) -> bool:
    if read_stamp(cache / STAMP_NAME) != expected_stamp:
        return False
    python = cache / "venv" / "bin" / "python"
    if not python.is_file():
        return False
    result = subprocess.run(
        [str(python), "-c", "import " + ", ".join(PYTHON_IMPORTS)],
        check=False,
        capture_output=True,
        text=True,
    )
    if result.returncode:
        return False
    if any(not (cache / "venv" / "bin" / executable).is_file() for executable in PYTHON_EXECUTABLES):
        return False
    for path in tool_paths(cache, tools, available):
        if not path.is_dir():
            return False
    return True


def render_activation(cache: Path, paths: list[Path]) -> str:
    quoted_path = ":".join(shlex.quote(str(path.resolve())) for path in paths)
    return "\n".join(
        (
            "# Generated by scripts/development_environment.py; do not edit.",
            f"export RETROSOC_DEVELOPMENT_CACHE={shlex.quote(str(cache.resolve()))}",
            f"export VIRTUAL_ENV={shlex.quote(str((cache / 'venv').resolve()))}",
            f"export PATH={quoted_path}:$PATH",
            "",
        )
    )


def write_activation(path: Path, content: str) -> None:
    atomic_write(path, content)
    path.chmod(0o644)


def run(command: list[str]) -> None:
    print("+ " + " ".join(shlex.quote(argument) for argument in command), flush=True)
    subprocess.run(command, check=True)


def create_virtualenv(root: Path, cache: Path) -> None:
    python = cache / "venv" / "bin" / "python"
    if not python.is_file():
        venv.EnvBuilder(with_pip=True, clear=False).create(cache / "venv")
    build_requirements, ci_requirements = requirement_paths(root)
    run(
        [
            str(python),
            "-m",
            "pip",
            "install",
            "--disable-pip-version-check",
            "--require-hashes",
            "--requirement",
            str(build_requirements),
            "--requirement",
            str(ci_requirements),
        ]
    )


def bootstrap(args: argparse.Namespace) -> int:
    root = args.root.resolve()
    cache = args.cache.resolve()
    lock_path = args.lock.resolve()
    lock = load_lock(lock_path)
    available = lock["toolchains"][PLATFORM]
    tools = parse_tools(args.tool, available)
    expected_stamp = stamp_data(root, cache, tools, lock, lock_path)

    if not has_environment(cache, tools, available, expected_stamp):
        for name in tools:
            install(name, available[name], cache, update=args.update)
        create_virtualenv(root, cache)
        atomic_write(cache / STAMP_NAME, json.dumps(expected_stamp, indent=2, sort_keys=True) + "\n")
    activation = args.output.resolve() if args.output else cache / ACTIVATE_NAME
    write_activation(
        activation,
        render_activation(cache, environment_paths(cache, tools, available)),
    )
    print(f"development environment: {activation}")
    return check(args, quiet=False)


def check(args: argparse.Namespace, *, quiet: bool) -> int:
    cache = args.cache.resolve()
    lock_path = args.lock.resolve()
    lock = load_lock(lock_path)
    available = lock["toolchains"][PLATFORM]
    tools = parse_tools(args.tool, available)
    expected_stamp = stamp_data(args.root.resolve(), cache, tools, lock, lock_path)
    failures: list[str] = []

    if not has_environment(cache, tools, available, expected_stamp):
        failures.append("development environment stamp or installed paths do not match the lock")
    for name in tools:
        for executable in TOOL_EXECUTABLES[name]:
            if not (tool_paths(cache, (name,), available)[0] / executable).is_file():
                failures.append(f"missing executable: {executable}")
    python = cache / "venv" / "bin" / "python"
    if python.is_file():
        result = subprocess.run(
            [str(python), "-c", "import " + ", ".join(PYTHON_IMPORTS)],
            check=False,
            capture_output=True,
            text=True,
        )
        if result.returncode:
            failures.append("locked Python development packages are unavailable")
    else:
        failures.append("missing virtualenv Python executable")
    for executable in SYSTEM_EXECUTABLES:
        if shutil.which(executable) is None:
            failures.append(f"missing system executable: {executable}")

    if not quiet:
        if failures:
            print("development environment check failed:")
            for failure in failures:
                print(f"  {failure}")
        else:
            print(f"development environment check passed: {cache}")
    return int(bool(failures))


def activate(args: argparse.Namespace) -> int:
    activation = args.output.resolve() if args.output else args.cache.resolve() / ACTIVATE_NAME
    if not activation.is_file():
        raise ValueError(f"activation script does not exist: {activation}")
    print(f"source {shlex.quote(str(activation))}")
    return 0


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--root", type=Path, default=ROOT)
    parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK)
    parser.add_argument("--cache", type=Path, default=ROOT / ".cache/retrosoc/development")
    parser.add_argument("--output", type=Path)
    parser.add_argument("--tool", action="append")
    parser.add_argument("--update", action="store_true")
    subparsers = parser.add_subparsers(dest="command", required=True)
    subparsers.add_parser("bootstrap")
    subparsers.add_parser("check")
    subparsers.add_parser("activate")
    return parser.parse_args()


def main() -> int:
    args = parse_args()
    try:
        if args.command == "bootstrap":
            return bootstrap(args)
        if args.command == "check":
            return check(args, quiet=False)
        return activate(args)
    except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as error:
        print(f"development environment: {error}", file=sys.stderr)
        return 1


if __name__ == "__main__":
    raise SystemExit(main())