feat: enhance development environment with Docker and Nix support, add SBT toolchain

This commit is contained in:
maksyuki committed 2026-09-22 19:22:06 +08:00
1 parent d0ec9aeba7
commit d4fba9fb96
13 files changed
+247 -17

No files matched your search

+3
View File
@@ -21,6 +21,9 @@ verifies every locked revision and checksum before tests run.
`workflows/regression-ics55.yml`, `workflows/regression-sky130.yml`, and
`workflows/nightly.yml` call the reusable regression workflow. Smoke enables
`formal_checks`, which runs `formal-doctor` only; it does not run `make formal`.
`workflows/development-environment.yml` builds the Docker environment and
installs the Nix environment in parallel, checks the resulting tool closure,
and runs the same hosted behavioral-only IHP130 PR regression in both.
The reusable workflow currently selects `--behavioral-only`: hosted CI skips SoC Yosys
synthesis, OpenSTA, and netlist simulation while the JPEG synthesis-memory
issue is being resolved.
@@ -0,0 +1,126 @@
name: development-environment
on:
push:
branches: [main, dev]
pull_request:
branches: [main, dev]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: development-environment-${{ github.ref }}
cancel-in-progress: true
jobs:
ihp130-regression:
name: ${{ matrix.environment }} / IHP130 regression
runs-on: ubuntu-22.04
timeout-minutes: 360
strategy:
fail-fast: false
matrix:
environment: [docker, nix]
env:
PIP_DISABLE_PIP_VERSION_CHECK: "1"
IHP130_REGRESSION_COMMAND: |
set -euo pipefail
python3 scripts/development_environment.py \
--cache "${RETROSOC_DEVELOPMENT_CACHE}" check
make CONFIG=configs/ci/ihp130.mk setup
make CONFIG=configs/ci/ihp130.mk \
SIMU=IVERILOG SYNTH=NONE STA=NONE doctor
make CONFIG=configs/ci/ihp130.mk \
SIMU=VERILATOR SYNTH=NONE STA=NONE doctor
make CONFIG=configs/ci/ihp130.mk \
SIMU=IVERILOG SYNTH=YOSYS STA=NONE doctor
make CONFIG=configs/ci/ihp130.mk \
SIMU=IVERILOG SYNTH=NONE STA=OPENSTA doctor
python3 scripts/regress.py --root . --suite pr \
--pdk IHP130 --behavioral-only
steps:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Restore locked IHP130 sources
uses: actions/cache@0057852bfaa89a56745bca8c7296529d2fc39830 # v4.3.0
with:
path: |
app/coremark/coremark-main
app/fatfs/ff16
physical/pdk/IHP-Open-PDK
rtl/managed/clusterip/archinfo
rtl/managed/clusterip/common
rtl/managed/clusterip/crc
rtl/managed/clusterip/ps2
rtl/managed/clusterip/pwm
rtl/managed/clusterip/rng
rtl/managed/clusterip/rtc
rtl/managed/clusterip/wdg
rtl/managed/third_party
rtl/managed/mpw
.cache/retrosoc/pdk
.cache/retrosoc/stamps
key: sources-IHP130-${{ runner.os }}-${{ hashFiles('dependencies/dependencies.lock.json') }}
- name: Restore SBT dependency cache
uses: actions/cache@0057852bfaa89a56745bca8c7296529d2fc39830 # v4.3.0
with:
path: .cache/retrosoc/sbt
key: environment-sbt-${{ runner.os }}-${{ hashFiles('dependencies/dependencies.lock.json') }}
- name: Restore Verilator compiler cache
uses: actions/cache@0057852bfaa89a56745bca8c7296529d2fc39830 # v4.3.0
with:
path: .cache/retrosoc/ccache
key: environment-ccache-${{ matrix.environment }}-${{ runner.os }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
restore-keys: |
environment-ccache-${{ matrix.environment }}-${{ runner.os }}-${{ github.sha }}-
environment-ccache-${{ matrix.environment }}-${{ runner.os }}-
- name: Build Docker development environment
if: matrix.environment == 'docker'
run: docker build --tag retrosoc-dev-ci --file docker/Dockerfile .
- name: Install Nix
if: matrix.environment == 'nix'
uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1
with:
enable_kvm: false
- name: Check Nix flake
if: matrix.environment == 'nix'
run: nix flake check
- name: Run IHP130 regression in development environment
env:
ENVIRONMENT: ${{ matrix.environment }}
run: |
if [[ "${ENVIRONMENT}" == "docker" ]]; then
docker run --rm --init --platform linux/amd64 \
--user "$(id -u):$(id -g)" \
--env GITHUB_ACTIONS=true \
--volume "${GITHUB_WORKSPACE}:/workspace/retrosoc" \
retrosoc-dev-ci bash -c "${IHP130_REGRESSION_COMMAND}"
else
nix run .#dev -- bash -c "${IHP130_REGRESSION_COMMAND}"
fi
- name: Upload reports and structured results
if: always()
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
with:
name: environment-${{ matrix.environment }}-ihp130-${{ github.run_id }}-${{ github.run_attempt }}
path: |
build/**/meta/*.json
build/**/formal/**/*.sby
build/**/formal/**/*.smt2
build/**/formal/**/*.vcd
build/**/formal/**/*.yw
build/**/formal/**/status
build/**/*.log
build/**/*.rpt
if-no-files-found: warn
retention-days: 30
+8 -6
View File
@@ -128,11 +128,12 @@ make regress-rtl
## Prerequisites
The open-source development environment contains the exact Ubuntu 22.04 tool bundles, Python
quality tools, compiler, formatters, simulators, synthesis, STA, and formal tools used by the
current regression. It does not contain PDKs, managed RTL, or application archives; those remain
checkout-local inputs installed and verified through the existing setup targets. Linux x86_64 is
the supported host for the full native environment. On macOS, use Docker with linux/amd64
emulation. Nix support is Linux x86_64 only.
quality tools, compiler, formatters, simulators, synthesis, STA, formal tools, and the locked SBT
launcher used to generate the VexiiRiscv HP core. Java 17 is supplied by the host environment. It
does not contain PDKs, managed RTL, or application archives; those remain checkout-local inputs
installed and verified through the existing setup targets. Linux x86_64 is the supported host for
the full native environment. On macOS, use Docker with linux/amd64 emulation. Nix support is Linux
x86_64 only.
Choose one of the following installation methods. Each uses the locked versions in
[dependencies/dependencies.lock.json](dependencies/dependencies.lock.json) and creates or reuses the local
@@ -178,7 +179,8 @@ downloads only checksum-verified tool bundles and Python packages pinned by the
sudo apt-get update
sudo apt-get install --no-install-recommends --yes \
bzip2 ca-certificates ccache clang-format-14 g++ git libfl2 libgoogle-perftools4 \
libunwind8 make mold numactl python3 python3-pip python3-venv xz-utils zlib1g
libunwind8 make mold numactl openjdk-17-jre-headless python3 python3-pip \
python3-venv xz-utils zlib1g
python3 scripts/development_environment.py bootstrap
source .cache/retrosoc/development/activate.sh
make setup-regression
+6 -3
View File
@@ -30,10 +30,13 @@ before materializing it below `.cache/retrosoc/pdk/sky130/openram/`. Generated
Verilog, Liberty, LEF, GDS, and SPICE views are never committed here.
The HP profile additionally locks VexiiRiscv, OpenSBI, Linux stable, and
Buildroot source revisions. `make setup-hp-linux` installs the software sources
below `.cache/retrosoc/sources/`; VexiiRiscv may be supplied through
Buildroot source revisions and the SBT launcher used by VexiiRiscv generation.
`make setup-hp-linux` installs the software sources below
`.cache/retrosoc/sources/`; VexiiRiscv may be supplied through
`VEXIIRISCV_ROOT`, but its revision is still checked before generated RTL is
accepted. No generated CPU RTL or Linux build output belongs in Git.
accepted. Java 17 is a host runtime supplied by Docker, Nix, or the documented
Ubuntu prerequisites. No generated CPU RTL or Linux build output belongs in
Git.
The libjpeg-turbo source archive is a host-verification input for the JPEG
accelerator. It supplies an implementation-independent interoperability oracle;
+9
View File
@@ -377,6 +377,15 @@
"archive": "xpack-openocd-0.12.0-1-linux-x64.tar.gz",
"path": "xpack-openocd-0.12.0-1/bin"
},
"sbt": {
"version": "1.10.0",
"url": "https://github.com/sbt/sbt/releases/download/v1.10.0/sbt-1.10.0.tgz",
"sha256": "154b7de6c19207c73d0a304f901c8c4b6ead9a9c3a99a98a9d72ac19419d2640",
"archive": "sbt-1.10.0.tgz",
"path": "sbt/bin",
"download_timeout_seconds": 600,
"resume": true
},
"riscv_gnu": {
"version": "2025.05.01",
"url": "https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2025.05.01/riscv32-elf-ubuntu-22.04-gcc-nightly-2025.05.01-nightly.tar.xz",
+1
View File
@@ -22,6 +22,7 @@ RUN apt-get update -qq \
make \
mold \
numactl \
openjdk-17-jre-headless \
python3 \
python3-pip \
python3-venv \
+10 -2
View File
@@ -5,9 +5,10 @@
The Docker image, Nix application, and manual bootstrap expose one Linux x86_64
open-source development environment. It includes the locked Ubuntu 22.04 tool
bundles for Verilator, Verible, sv2v, Icarus Verilog, Yosys, SymbiYosys,
Bitwuzla, OpenSTA, OpenOCD, and the RISC-V GNU toolchain. The GNU bundle
Bitwuzla, OpenSTA, OpenOCD, SBT, and the RISC-V GNU toolchain. The GNU bundle
includes `riscv32-unknown-elf-gdb`, which is used with OpenOCD by the Hazard3
remote-bitbang debug acceptance flow. It also installs the locked Python build
remote-bitbang debug acceptance flow. Docker, Nix, and manual Ubuntu installs
provide Java 17 for SBT. The environment also installs the locked Python build
and quality dependencies, clang-format-14, GNU Make, C/C++ build tools, and
runtime libraries required by those binaries.
@@ -82,3 +83,10 @@ Run the environment check before using a manually shared cache. Build the
Docker image after Dockerfile or bootstrap changes. On a Linux x86_64 host,
run nix flake check and a short nix run .#dev command after flake changes. Then
run make setup-regression and the regression tier appropriate for the change.
The `development-environment` GitHub workflow performs the installation and
runtime validation for both entry points. Each job checks the installed
environment and runs the same hosted behavioral-only IHP130 PR regression as
`regression-ihp130`. Docker and Nix installation are CI-authoritative; local
validation of workflow changes is limited to the repository's Python, YAML,
Actions, dependency-lock, and regression dry-run gates.
+8 -3
View File
@@ -310,9 +310,10 @@ changes after freeze must retain the baseline revision and equivalence evidence.
`quality.yml` validates C, Makefile, and self-owned RTL formatting as well as
Python, YAML, GitHub Actions, the dependency lock, and the complete Pytest
suite. The test environment restores the locked IHP130 PDK and APU reference
corpus, installs the locked third-party simulation models, and installs the
locked Verilator, Icarus, sv2v, and Yosys tools before Pytest so required RTL,
netlist, timing-model, and corpus fixtures execute. `regression-smoke.yml`
corpus plus the NPU models, inputs, and oracle sources, installs the locked
third-party simulation models, and installs the locked Verilator, Icarus,
sv2v, and Yosys tools before Pytest so required RTL, netlist, timing-model, and
corpus fixtures execute. `regression-smoke.yml`
provides fast IHP130 feedback; the four PDK regression workflows remain
required PR coverage and do not repeat the format checks.
@@ -328,6 +329,10 @@ simulation, OpenSTA, and synthesis-recipe metrics. The locked tools are still
installed and checked. This temporary policy avoids the unresolved JPEG
synthesis-memory peak. Local full regressions retain those stages and remain the
source of synthesis and timing evidence until hosted coverage is restored.
`development-environment.yml` independently builds Docker and installs Nix,
checks each resulting tool environment, and runs the same hosted IHP130 PR
command set. It verifies environment reproducibility but does not expand the
behavioral-only evidence boundary.
`nightly.yml` repeats the fixed IHP130 architecture as two parallel behavioral
jobs: the PR IHP130 matrix and the extra CoreMark coverage. Source dependencies,
locked tool archives, and Verilator `ccache` use
+1
View File
@@ -64,6 +64,7 @@
mold
ncurses5
numactl
jdk17_headless
python3Full
unzip
which
+7
View File
@@ -69,6 +69,13 @@ def validate_lock(data: dict[str, Any]) -> None:
_validate_url(f"toolchain {platform}/{name}", tool["url"])
_validate_relative_path(f"toolchain {platform}/{name} archive", tool["archive"])
_validate_relative_path(f"toolchain {platform}/{name} path", tool["path"])
timeout = tool.get("download_timeout_seconds", 120)
if not isinstance(timeout, int) or isinstance(timeout, bool) or timeout <= 0:
raise LockError(
f"toolchain {platform}/{name} download_timeout_seconds must be positive"
)
if not isinstance(tool.get("resume", False), bool):
raise LockError(f"toolchain {platform}/{name} resume must be a boolean")
def _validate_archive(kind: str, name: str, value: dict[str, Any]) -> None:
+12 -1
View File
@@ -33,6 +33,7 @@ DEFAULT_TOOLS = (
"bitwuzla",
"opensta",
"openocd",
"sbt",
"riscv_gnu",
)
TOOL_EXECUTABLES = {
@@ -45,11 +46,21 @@ TOOL_EXECUTABLES = {
"bitwuzla": ("bitwuzla",),
"opensta": ("sta",),
"openocd": ("openocd",),
"sbt": ("sbt",),
"riscv_gnu": ("riscv32-unknown-elf-gcc", "riscv32-unknown-elf-gdb"),
}
PYTHON_IMPORTS = ("pytest", "ruff", "yamllint")
PYTHON_EXECUTABLES = ("mbake",)
SYSTEM_EXECUTABLES = ("cc", "ccache", "clang-format-14", "git", "make", "mold", "numactl")
SYSTEM_EXECUTABLES = (
"cc",
"ccache",
"clang-format-14",
"git",
"java",
"make",
"mold",
"numactl",
)
STAMP_NAME = "development-environment.json"
ACTIVATE_NAME = "activate.sh"
+10 -2
View File
@@ -10,6 +10,7 @@ import sys
import tarfile
import tempfile
from pathlib import Path, PurePosixPath
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
@@ -37,10 +38,17 @@ def safe_extract(archive: Path, destination: Path) -> None:
bundle.extractall(destination)
def install(name: str, spec: dict[str, str], cache: Path, update: bool) -> Path:
def install(name: str, spec: dict[str, Any], cache: Path, update: bool) -> Path:
downloads = cache / "downloads"
archive = downloads / spec["archive"]
download_file(spec["url"], archive, spec["sha256"], update=update, timeout=120)
download_file(
spec["url"],
archive,
spec["sha256"],
update=update,
timeout=spec.get("download_timeout_seconds", 120),
resume=spec.get("resume", False),
)
destination = cache / "toolchains" / f"{name}-{spec['version']}"
marker = destination / ".complete"
+46
View File
@@ -1098,6 +1098,15 @@ def test_dependency_lock_and_config_key_include_a_fixed_timestamp(tmp_path: Path
assert lock["sources"]["pdk_sky130"]["submodules"] == ["libraries/sky130_fd_sc_hd/latest"]
assert lock["container_images"]["ubuntu_22_04"]["image"] == "ubuntu"
assert lock["nix_inputs"]["nixpkgs"]["revision"] == "50ab793786d9de88ee30ec4e4c24fb4236fc2674"
assert lock["toolchains"]["ubuntu-22.04"]["sbt"] == {
"version": "1.10.0",
"url": "https://github.com/sbt/sbt/releases/download/v1.10.0/sbt-1.10.0.tgz",
"sha256": "154b7de6c19207c73d0a304f901c8c4b6ead9a9c3a99a98a9d72ac19419d2640",
"archive": "sbt-1.10.0.tgz",
"path": "sbt/bin",
"download_timeout_seconds": 600,
"resume": True,
}
validate_flake_lock(lock, ROOT / "flake.lock")
command = (
@@ -1146,6 +1155,16 @@ def test_dependency_lock_and_config_key_include_a_fixed_timestamp(tmp_path: Path
else:
raise AssertionError("invalid submodule path was accepted")
invalid_timeout = json.loads(json.dumps(lock))
invalid_timeout["toolchains"]["ubuntu-22.04"]["sbt"]["download_timeout_seconds"] = 0
broken.write_text(json.dumps(invalid_timeout), encoding="utf-8")
try:
load_lock(broken)
except LockError as error:
assert "download_timeout_seconds must be positive" in str(error)
else:
raise AssertionError("invalid toolchain download timeout was accepted")
def test_development_environment_contract_is_lock_pinned(tmp_path: Path) -> None:
lock_path = ROOT / "dependencies/dependencies.lock.json"
@@ -1155,6 +1174,7 @@ def test_development_environment_contract_is_lock_pinned(tmp_path: Path) -> None
assert stamp["tools"]["verilator"] == lock["toolchains"]["ubuntu-22.04"]["verilator"]["version"]
assert stamp["tools"]["openocd"] == lock["toolchains"]["ubuntu-22.04"]["openocd"]["version"]
assert stamp["tools"]["sbt"] == lock["toolchains"]["ubuntu-22.04"]["sbt"]["version"]
assert set(stamp["python_requirements"]) == {"requirements/build.txt", "requirements/ci.txt"}
activation = render_activation(cache, [cache / "venv/bin", cache / "toolchains/verilator/bin"])
assert "export RETROSOC_DEVELOPMENT_CACHE=" in activation
@@ -1168,14 +1188,40 @@ def test_container_and_nix_environment_files_use_locked_inputs() -> None:
sbom = make_sbom(lock)
assert f"ubuntu@{lock['container_images']['ubuntu_22_04']['digest']}" in dockerfile
assert "openjdk-17-jre-headless" in dockerfile
assert "scripts/development_environment.py" in dockerfile
assert "scripts/development_environment.py" in flake
assert "buildFHSEnv" in flake
assert "jdk17_headless" in flake
assert "retrosoc-development retrosoc-dev" in flake
assert any(component["name"] == "container/ubuntu_22_04" for component in sbom["components"])
assert any(component["name"] == "nix/nixpkgs" for component in sbom["components"])
def test_development_environment_workflow_runs_the_ihp130_hosted_contract() -> None:
workflow = (ROOT / ".github/workflows/development-environment.yml").read_text()
assert "environment: [docker, nix]" in workflow
assert "timeout-minutes: 360" in workflow
assert "docker build --tag retrosoc-dev-ci --file docker/Dockerfile ." in workflow
assert "cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24" in workflow
assert "nix flake check" in workflow
assert "development_environment.py" in workflow
assert '--cache "${RETROSOC_DEVELOPMENT_CACHE}" check' in workflow
for command in (
"make CONFIG=configs/ci/ihp130.mk setup",
"SIMU=IVERILOG SYNTH=NONE STA=NONE doctor",
"SIMU=VERILATOR SYNTH=NONE STA=NONE doctor",
"SIMU=IVERILOG SYNTH=YOSYS STA=NONE doctor",
"SIMU=IVERILOG SYNTH=NONE STA=OPENSTA doctor",
"--pdk IHP130 --behavioral-only",
):
assert command in workflow.replace("\\\n", "")
assert 'docker run --rm --init --platform linux/amd64' in workflow
assert 'nix run .#dev -- bash -c "${IHP130_REGRESSION_COMMAND}"' in workflow
assert "environment-${{ matrix.environment }}-ihp130-" in workflow
def test_regression_runner_uses_one_build_timestamp(monkeypatch) -> None:
monkeypatch.setenv("BUILD_TIMESTAMP", "2026-07-21-10-39")
assert regression_environment()["BUILD_TIMESTAMP"] == "2026-07-21-10-39"