From 57c9d334eaaf8d744120ec357d8c2059f203e082 Mon Sep 17 00:00:00 2001 From: maksyuki Date: Fri, 10 Jul 2026 17:48:11 +0800 Subject: [PATCH] feat: refactor MPW preparation and add regression and flow scripts - Updated to utilize a new dependency locking mechanism and improved repository management. - Introduced for running regression tests with configurable command sets for different profiles. - Added to execute commands with structured logging and result reporting. - Enhanced to support fetching specific revisions from git repositories. - Modified to use the new dependency source structure. - Updated OpenSTA makefile and TCL script to include metrics reporting. - Improved synthesis scripts to utilize the new flow execution model. - Added tests for dependency locking, flow execution, and warning analysis. - Ensured that all scripts and tools are consistent with the new structure and functionality. --- .github/CODEOWNERS | 7 + .github/actions/locked-tools/action.yml | 34 ++ .github/dependabot.yml | 13 + .github/workflows/_regression.yml | 103 +++++ .github/workflows/lint.yml | 42 -- .github/workflows/nightly.yml | 20 + .github/workflows/quality.yml | 59 +++ .github/workflows/regression.yml | 22 ++ .github/workflows/release.yml | 71 ++++ .github/workflows/style.yml | 42 -- .github/workflows/test.yml | 155 -------- .gitignore | 5 + .yamllint.yml | 7 + CONTRIBUTING.md | 32 +- Makefile | 113 +++++- README.md | 52 +-- app/asm/Makefile | 6 +- app/setup.py | 33 +- config/dependencies.lock.json | 124 ++++++ configs/ci/hazard3-rv32im-ihp130.mk | 12 + configs/ci/mdd-rv32im-ihp130.mk | 12 + configs/cluster/hazard3-ics55.mk | 12 + configs/nightly/picorv32-rv32im-ihp130.mk | 12 + docs/engineering.md | 102 +++++ pdk/setup.py | 30 +- pyproject.toml | 22 ++ quality/metrics/README.md | 5 + quality/metrics/policy.json | 12 + quality/warnings/README.md | 7 + .../hazard3-rv32im-ihp130/iverilog.json | 17 + .../hazard3-rv32im-ihp130/opensta.json | 6 + .../hazard3-rv32im-ihp130/verilator.json | 42 ++ .../warnings/hazard3-rv32im-ihp130/yosys.json | 42 ++ .../warnings/mdd-rv32im-ihp130/verilator.json | 87 ++++ .../picorv32-rv32im-ihp130/iverilog.json | 17 + .../picorv32-rv32im-ihp130/verilator.json | 42 ++ requirements/build.txt | 5 + requirements/ci.txt | 15 + rtl/clusterip/setup.py | 32 +- rtl/ip/setup.py | 11 +- rtl/mini/Makefile | 73 ++-- rtl/mini/mk/iverilog.mk | 100 +++-- rtl/mini/mk/software.mk | 54 ++- rtl/mini/mk/vcs.mk | 88 +++-- rtl/mini/mk/verilator.mk | 53 ++- rtl/mini/script/filelist_deps.py | 57 +++ rtl/mini/script/gen_iverilog_filelist.py | 2 +- scripts/analyze_warnings.py | 180 +++++++++ scripts/check_simulation.py | 56 +++ scripts/clean.py | 33 +- scripts/config_key.py | 55 +++ scripts/dependency_lock.py | 124 ++++++ scripts/doctor.py | 99 +++-- scripts/generate_mpw.py | 112 ++++++ scripts/install_toolchain.py | 93 +++++ scripts/manifest.py | 123 ++++++ scripts/metrics.py | 150 +++++++ scripts/package.py | 103 +++++ scripts/prepare_mpw.py | 46 ++- scripts/regress.py | 57 +++ scripts/run_flow.py | 128 ++++++ scripts/setup_helpers.py | 16 +- setup.py | 9 +- sta/opensta/opensta.mk | 23 +- sta/opensta/opensta.tcl | 29 +- syn/tools/export_soc_sources.py | 24 +- syn/yosys/script/synth.tcl | 1 + syn/yosys/yosys.mk | 38 +- tests/test_script_tools.py | 372 +++++++++++++++++- 69 files changed, 3200 insertions(+), 580 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/actions/locked-tools/action.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/_regression.yml delete mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/quality.yml create mode 100644 .github/workflows/regression.yml create mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/style.yml delete mode 100644 .github/workflows/test.yml create mode 100644 .yamllint.yml create mode 100644 config/dependencies.lock.json create mode 100644 configs/ci/hazard3-rv32im-ihp130.mk create mode 100644 configs/ci/mdd-rv32im-ihp130.mk create mode 100644 configs/cluster/hazard3-ics55.mk create mode 100644 configs/nightly/picorv32-rv32im-ihp130.mk create mode 100644 docs/engineering.md create mode 100644 pyproject.toml create mode 100644 quality/metrics/README.md create mode 100644 quality/metrics/policy.json create mode 100644 quality/warnings/README.md create mode 100644 quality/warnings/hazard3-rv32im-ihp130/iverilog.json create mode 100644 quality/warnings/hazard3-rv32im-ihp130/opensta.json create mode 100644 quality/warnings/hazard3-rv32im-ihp130/verilator.json create mode 100644 quality/warnings/hazard3-rv32im-ihp130/yosys.json create mode 100644 quality/warnings/mdd-rv32im-ihp130/verilator.json create mode 100644 quality/warnings/picorv32-rv32im-ihp130/iverilog.json create mode 100644 quality/warnings/picorv32-rv32im-ihp130/verilator.json create mode 100644 requirements/build.txt create mode 100644 requirements/ci.txt create mode 100644 rtl/mini/script/filelist_deps.py create mode 100644 scripts/analyze_warnings.py create mode 100644 scripts/check_simulation.py create mode 100644 scripts/config_key.py create mode 100644 scripts/dependency_lock.py create mode 100644 scripts/generate_mpw.py create mode 100644 scripts/install_toolchain.py create mode 100644 scripts/manifest.py create mode 100644 scripts/metrics.py create mode 100644 scripts/package.py create mode 100644 scripts/regress.py create mode 100644 scripts/run_flow.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..6ae694e9 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# Build and dependency controls require explicit maintainer review. +/Makefile @maksyuki +/config/ @maksyuki +/configs/ @maksyuki +/scripts/ @maksyuki +/.github/ @maksyuki +/quality/ @maksyuki diff --git a/.github/actions/locked-tools/action.yml b/.github/actions/locked-tools/action.yml new file mode 100644 index 00000000..0d782262 --- /dev/null +++ b/.github/actions/locked-tools/action.yml @@ -0,0 +1,34 @@ +name: Set up locked toolchains +description: Install checksum-verified retroSoC tool archives and their runtime libraries + +inputs: + tools: + description: Space-separated tool names from config/dependencies.lock.json + required: true + +runs: + using: composite + steps: + - name: Install runtime packages + shell: bash + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends \ + ccache g++ libfl2 libgoogle-perftools4 libunwind8 make mold numactl zlib1g + + - name: Install checksum-verified tools + shell: bash + env: + REQUESTED_TOOLS: ${{ inputs.tools }} + run: | + read -r -a tools <<< "${REQUESTED_TOOLS}" + args=() + for tool in "${tools[@]}"; do + args+=(--tool "${tool}") + done + python3 scripts/install_toolchain.py \ + --lock config/dependencies.lock.json \ + --platform ubuntu-22.04 \ + --cache .cache/retrosoc \ + --github-path "${GITHUB_PATH}" \ + "${args[@]}" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..fa51075e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + + - package-ecosystem: pip + directory: /requirements + schedule: + interval: monthly + open-pull-requests-limit: 5 diff --git a/.github/workflows/_regression.yml b/.github/workflows/_regression.yml new file mode 100644 index 00000000..f36a2303 --- /dev/null +++ b/.github/workflows/_regression.yml @@ -0,0 +1,103 @@ +name: reusable regression + +on: + workflow_call: + inputs: + suite: + description: Regression suite from scripts/regress.py + required: true + type: string + retention_days: + description: Report artifact retention + required: false + default: 30 + type: number + +permissions: + contents: read + +jobs: + regression: + runs-on: ubuntu-22.04 + timeout-minutes: 240 + env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Restore locked source dependencies + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: | + app/coremark/coremark-main + app/fatfs/ff16 + pdk/IHP-Open-PDK + rtl/clusterip/archinfo + rtl/clusterip/clint + rtl/clusterip/common + rtl/clusterip/crc + rtl/clusterip/i2c + rtl/clusterip/i2s + rtl/clusterip/plic + rtl/clusterip/ps2 + rtl/clusterip/pwm + rtl/clusterip/rng + rtl/clusterip/rtc + rtl/clusterip/spi + rtl/clusterip/timer + rtl/clusterip/uart + rtl/clusterip/wdg + rtl/ip/3rd-party + rtl/mini/mpw + .cache/retrosoc/stamps + key: sources-${{ runner.os }}-${{ hashFiles('config/dependencies.lock.json') }} + + - name: Restore locked toolchains + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: | + .cache/retrosoc/downloads + .cache/retrosoc/toolchains + key: tools-${{ runner.os }}-${{ hashFiles('config/dependencies.lock.json') }} + + - name: Restore Verilator compiler cache + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: .cache/retrosoc/ccache + key: ccache-${{ runner.os }}-${{ hashFiles('config/dependencies.lock.json') }}-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + ccache-${{ runner.os }}-${{ hashFiles('config/dependencies.lock.json') }}-${{ github.sha }}- + ccache-${{ runner.os }}-${{ hashFiles('config/dependencies.lock.json') }}- + + - name: Install build Python dependency + run: python3 -m pip install --requirement requirements/build.txt + + - name: Set up locked EDA and compiler tools + uses: ./.github/actions/locked-tools + with: + tools: verilator sv2v iverilog yosys opensta riscv_gnu + + - name: Set up locked source dependencies + run: make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk setup + + - name: Check environment + run: | + make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk \ + SIMU=IVERILOG SYNTH=YOSYS STA=OPENSTA doctor + make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk SIMU=VERILATOR doctor + + - name: Run ${{ inputs.suite }} regression + run: python3 scripts/regress.py --root . --suite "${{ inputs.suite }}" + + - name: Upload reports and structured results + if: always() + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: ${{ inputs.suite }}-regression-${{ github.run_id }}-${{ github.run_attempt }} + path: | + build/**/meta/*.json + build/**/*.log + build/**/*.rpt + if-no-files-found: warn + retention-days: ${{ inputs.retention_days }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index ac452966..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: lint - -on: - push: - branches: [ main, dev ] - pull_request: - branches: [ main, dev ] - -jobs: - build: - name: Build and Test - runs-on: ubuntu-22.04 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup environment - run: echo "No setup needed for this minimal workflow" - - - name: Run tests - run: echo "No tests defined yet" - - - name: Build project - run: echo "Build placeholder" - - - name: Success notification - run: echo "Workflow completed successfully" - -# jobs: -# lint-tgt: -# name: SystemVerilog Lint -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# - uses: chipsalliance/verible-linter-action@main -# with: -# paths: | -# ./rtl -# # extra_args: "--waiver_files lint/common_cells.style.waiver --rules=-interface-name-style --lint_fatal" -# # github_token: ${{ secrets.GITHUB_TOKEN }} -# # reviewdog_reporter: github-check diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..68051255 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,20 @@ +name: nightly + +on: + schedule: + - cron: "17 18 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-regression + cancel-in-progress: false + +jobs: + extended-open-source-flows: + uses: ./.github/workflows/_regression.yml + with: + suite: nightly + retention_days: 90 diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml new file mode 100644 index 00000000..dcaab810 --- /dev/null +++ b/.github/workflows/quality.yml @@ -0,0 +1,59 @@ +name: quality + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: quality-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + scripts: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Restore pip cache + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: ~/.cache/pip + key: pip-${{ runner.os }}-${{ hashFiles('requirements/ci.txt') }} + + - name: Install pinned quality tools + run: python3 -m pip install --requirement requirements/ci.txt + + - name: Validate dependency lock + run: python3 scripts/dependency_lock.py --lock config/dependencies.lock.json + + - name: Lint Python + run: ruff check . + + - name: Test scripts + run: python3 -m pytest -q + + - name: Lint YAML + run: yamllint .github .yamllint.yml + + - name: Install pinned actionlint + env: + GOBIN: ${{ runner.temp }}/actionlint-bin + run: | + go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 + echo "${GOBIN}" >> "${GITHUB_PATH}" + + - name: Lint GitHub Actions + run: actionlint + + - name: Validate regression definitions + run: | + python3 scripts/regress.py --root . --suite pr --dry-run + python3 scripts/regress.py --root . --suite nightly --dry-run diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml new file mode 100644 index 00000000..d6104831 --- /dev/null +++ b/.github/workflows/regression.yml @@ -0,0 +1,22 @@ +name: regression + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: regression-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verified-open-source-flows: + uses: ./.github/workflows/_regression.yml + with: + suite: pr + retention_days: 30 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..44917cc1 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,71 @@ +name: release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + source-package: + runs-on: ubuntu-22.04 + timeout-minutes: 90 + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Restore locked source dependencies + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: | + app/coremark/coremark-main + app/fatfs/ff16 + pdk/IHP-Open-PDK + rtl/clusterip/archinfo + rtl/clusterip/clint + rtl/clusterip/common + rtl/clusterip/crc + rtl/clusterip/i2c + rtl/clusterip/i2s + rtl/clusterip/plic + rtl/clusterip/ps2 + rtl/clusterip/pwm + rtl/clusterip/rng + rtl/clusterip/rtc + rtl/clusterip/spi + rtl/clusterip/timer + rtl/clusterip/uart + rtl/clusterip/wdg + rtl/ip/3rd-party + rtl/mini/mpw + .cache/retrosoc/stamps + key: sources-${{ runner.os }}-${{ hashFiles('config/dependencies.lock.json') }} + + - name: Install build Python dependency + run: python3 -m pip install --requirement requirements/build.txt + + - name: Set up locked source dependencies + run: make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk setup + + - name: Build checksummed source package and SBOM + run: make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk package + + - name: Upload workflow artifact + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: retrosoc-${{ github.ref_name }} + path: dist/** + if-no-files-found: error + retention-days: 90 + + - name: Publish GitHub release assets + env: + GH_TOKEN: ${{ github.token }} + run: | + if ! gh release view "${GITHUB_REF_NAME}" >/dev/null 2>&1; then + gh release create "${GITHUB_REF_NAME}" --generate-notes --verify-tag + fi + mapfile -d '' files < <(find dist -type f -print0) + gh release upload "${GITHUB_REF_NAME}" "${files[@]}" --clobber diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml deleted file mode 100644 index 914a18b6..00000000 --- a/.github/workflows/style.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: style - -on: - push: - branches: [ main, dev ] - pull_request: - branches: [ main, dev ] - -jobs: - build: - name: Build and Test - runs-on: ubuntu-22.04 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup environment - run: echo "No setup needed for this minimal workflow" - - - name: Run tests - run: echo "No tests defined yet" - - - name: Build project - run: echo "Build placeholder" - - - name: Success notification - run: echo "Workflow completed successfully" - -# jobs: -# lint-tgt: -# name: SystemVerilog Lint -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# - uses: chipsalliance/verible-linter-action@main -# with: -# paths: | -# ./rtl -# # extra_args: "--waiver_files lint/common_cells.style.waiver --rules=-interface-name-style --lint_fatal" -# # github_token: ${{ secrets.GITHUB_TOKEN }} -# # reviewdog_reporter: github-check diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 1ec4a71d..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,155 +0,0 @@ -name: test - -on: - push: - branches: [ main, dev ] - pull_request: - branches: [ main, dev ] - -jobs: - build: - runs-on: ubuntu-22.04 - - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Test build scripts - run: | - python3 -m pip install pytest - python3 -m pytest -q tests - - - name: Download and Extract Verilator Artifact - run: | - verilator_path="artifact-verilator" - mkdir $verilator_path - wget https://github.com/retroSoC/artifact/releases/download/verilator-v5.046/verilator-v5.046-ubuntu-22.04-gcc.tar.gz - tar -xvf verilator-v5.046-ubuntu-22.04-gcc.tar.gz -C $verilator_path - echo "$(pwd)/$verilator_path/bin" >> $GITHUB_PATH - sudo apt-get update - sudo apt-get install git help2man perl python3 make autoconf g++ flex bison ccache mold - sudo apt-get install libunwind-dev libgoogle-perftools-dev numactl perl-doc - sudo apt-get install libfl2 - sudo apt-get install libfl-dev - sudo apt-get install zlib1g zlib1g-dev - sudo apt-get install tree - - - name: Test Verilator Toolchain - run: | - verilator --version - - - name: Download and Extract Sv2v Artifact - run: | - sv2v_path="artifact-sv2v" - mkdir $sv2v_path - wget https://github.com/retroSoC/artifact/releases/download/sv2v-v0.0.13-9-gd381209/sv2v-v0.0.13-9-gd381209-ubuntu-22.04-gcc.tar.gz - tar -xvf sv2v-v0.0.13-9-gd381209-ubuntu-22.04-gcc.tar.gz -C $sv2v_path - echo "$(pwd)/$sv2v_path" >> $GITHUB_PATH - - - name: Test Sv2v Toolchain - run: | - sv2v --version - - - name: Download and Extract Iverilog Artifact - run: | - iverilog_path="artifact-iverilog" - mkdir $iverilog_path - wget https://github.com/retroSoC/artifact/releases/download/iverilog-v13_0/iverilog-v13_0-ubuntu-22.04-gcc.tar.gz - tar -xvf iverilog-v13_0-ubuntu-22.04-gcc.tar.gz -C $iverilog_path - echo "$(pwd)/$iverilog_path/bin" >> $GITHUB_PATH - - - name: Test Iverilog Toolchain - run: | - iverilog -V - - - name: Download and Extract Yosys Artifact - run: | - yosys_path="artifact-yosys" - mkdir $yosys_path - wget https://github.com/retroSoC/artifact/releases/download/yosys-v0.58/yosys-v0.58-ubuntu-22.04-gcc.tar.gz - tar -xvf yosys-v0.58-ubuntu-22.04-gcc.tar.gz -C $yosys_path - echo "$(pwd)/$yosys_path/bin" >> $GITHUB_PATH - - - name: Test Yosys Toolchain - run: | - yosys --version - - - name: Download and Extract OpenSTA Artifact - run: | - opensta_path="artifact-opensta" - mkdir $opensta_path - wget https://github.com/retroSoC/artifact/releases/download/opensta-v2.2.0-1767-ga93b51ec/opensta-v2.2.0-1767-ga93b51ec-ubuntu-22.04-gcc.tar.gz - tar -xvf opensta-v2.2.0-1767-ga93b51ec-ubuntu-22.04-gcc.tar.gz -C $opensta_path - echo "$(pwd)/$opensta_path/bin" >> $GITHUB_PATH - - - name: Test OpenSTA Toolchain - run: | - sta -version - - - name: Download and Extract RISC-V GNU Toolchain Artifact - run: | - riscv_toolchain_path="artifact-riscv-toolchain" - mkdir $riscv_toolchain_path - wget https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2025.05.01/riscv32-elf-ubuntu-22.04-gcc-nightly-2025.05.01-nightly.tar.xz - tar -xvf riscv32-elf-ubuntu-22.04-gcc-nightly-2025.05.01-nightly.tar.xz -C $riscv_toolchain_path - echo "$(pwd)/$riscv_toolchain_path/riscv/bin" >> $GITHUB_PATH - - - name: Test RISCV-V GNU Toolchain - run: | - riscv32-unknown-elf-gcc --version - - - name: Setup MPW Repository - run: | - pip install tomli - python3 setup.py - - - name: Generate modified core - run: python3 rtl/mini/core/setup.py - - - name: Download clusterIP - run: python3 rtl/clusterip/setup.py - - - name: Download 3rd-party IP and models - run: python3 rtl/ip/setup.py - - - name: Download Open PDK Repository - run: python3 pdk/setup.py - - - name: Config FatFs and Coremark Repository - run: python3 app/setup.py - - - name: Compile Single Core Firmware - run: make CORE=HAZARD3 IP=NONE ISA=RV32IM HAVE_CSR=NO PROG_TYPE=BASE LINK_TYPE=ld2_psram firmware - - - name: Run Verilator Behavioral Simulation(Single Core) - run: make CORE=HAZARD3 IP=NONE SIMU=VERILATOR PDK=IHP130 HAVE_SRAM_IF=NO HAVE_SRAM_MACRO=NO HAVE_PLL=NO sim - - - name: Compile MDD Firmware - run: make CORE=MDD IP=NONE ISA=RV32IM HAVE_CSR=NO PROG_TYPE=BASE LINK_TYPE=ld2_psram firmware - - - name: Run Verilator Behavioral Simulation(MDD) - run: make CORE=MDD IP=NONE SIMU=VERILATOR PDK=IHP130 HAVE_SRAM_IF=NO HAVE_SRAM_MACRO=NO HAVE_PLL=NO RTL_SIM_CORESEL=0 sim - - - name: Compile Assembly Firmware - run: make ISA=RV32IM HAVE_CSR=NO PROG_TYPE=BASE LINK_TYPE=ld2_psram asm - - - name: Run Iverilog Behavioral Simulation(Single Core) - run: make CORE=HAZARD3 IP=NONE SIMU=IVERILOG PDK=IHP130 HAVE_SRAM_IF=NO HAVE_SRAM_MACRO=NO HAVE_PLL=NO RTL_SIM_TIMEOUT=5200000 sim - - - name: Run synthesis - run: make CORE=HAZARD3 IP=NONE SYNTH=YOSYS PDK=IHP130 HAVE_SRAM_IF=NO HAVE_SRAM_MACRO=NO HAVE_PLL=NO synth - - - name: Run Iverilog Netlist Simulation(Single Core) - run: make CORE=HAZARD3 IP=NONE SIMU=IVERILOG PDK=IHP130 HAVE_SRAM_IF=NO HAVE_SRAM_MACRO=NO HAVE_PLL=NO RTL_SIM_TIMEOUT=5200000 netsim - - # - name: Upload to Actions Artifacts(Temporary storage) - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ env.PACKAGE_NAME }} - # path: | - # verilator/${{ env.PACKAGE_NAME }}.tar.gz - # verilator/${{ env.PACKAGE_NAME }}.tar.gz.sha256 - # retention-days: 90 # Retain for 90 days - - - name: Success Notification - run: echo "RTL simulation completed successfully" diff --git a/.gitignore b/.gitignore index 0d491051..05742221 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ *.ddb *.fst *.txt +!requirements/ +!requirements/*.txt *.gtkw *.vmf *.tar.bz2 @@ -39,6 +41,9 @@ audio_file.h .metals .mypy_cache .pytest_cache +/build/ +/.cache/ +/dist/ .vscode retrosoc_sections.lds retrosoc_syn* diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 00000000..ee49aa44 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,7 @@ +extends: default + +rules: + document-start: disable + line-length: + max: 160 + truthy: disable diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 444abb5a..275d79d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1,31 @@ -## CONTRIBUTE \ No newline at end of file +# Contributing + +Run the script quality gate before opening a pull request: + +```sh +python3 -m pip install --requirement requirements/ci.txt +ruff check . +python3 -m pytest -q +yamllint .github .yamllint.yml +python3 scripts/regress.py --root . --suite pr --dry-run +``` + +For build changes, use a committed profile and keep generated files below `build/` or `.cache/`. +Do not add new downloads directly to workflow YAML or setup scripts. Add the source or archive to +`config/dependencies.lock.json`, use the shared dependency helpers, and include a full Git commit or +SHA-256 checksum. + +The supported pull-request regression is: + +```sh +make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk setup +make regress-pr +``` + +Warning baseline changes must be reviewed separately from implementation changes. Regenerate only +the affected profile/tool file, inspect every normalized signature, and explain additions in the +pull request. Metrics remain observational until ten successful `main` runs have been collected and +promoted according to `docs/engineering.md`. + +Changes to the dependency lock, CI workflows, warning policy, metrics policy, or release packaging +require the owners listed in `.github/CODEOWNERS`. diff --git a/Makefile b/Makefile index 92bd3f01..ebeb8c91 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,22 @@ SHELL := /bin/bash .DEFAULT_GOAL := help .DELETE_ON_ERROR: +ROOT_PATH ?= $(abspath $(dir $(firstword $(MAKEFILE_LIST)))) +CONFIG ?= +LOCK_FILE ?= $(ROOT_PATH)/config/dependencies.lock.json + +ifneq ($(strip $(CONFIG)),) +CONFIG_PATH := $(if $(filter /%,$(CONFIG)),$(CONFIG),$(ROOT_PATH)/$(CONFIG)) +ifeq ($(wildcard $(CONFIG_PATH)),) +$(error Configuration profile not found: $(CONFIG_PATH)) +endif +include $(CONFIG_PATH) +PROFILE_NAME := $(basename $(notdir $(CONFIG_PATH))) +else +CONFIG_PATH := +PROFILE_NAME := manual +endif + SOC ?= MINI SIMU ?= VCS SYNTH ?= NONE @@ -19,8 +35,9 @@ RTL_SIM_PLLEN ?= NONE RTL_SIM_PLLCFG ?= NONE RTL_SIM_CORESEL ?= 0 RTL_SIM_TIMEOUT ?= -1 +SIM_FIRMWARE_NAME ?= $(FIRMWARE_NAME) +SIM_SUCCESS_MARKER ?= retroSoC: A Customized ASIC for Retro Stuff -ROOT_PATH ?= $(abspath $(dir $(firstword $(MAKEFILE_LIST)))) RTL_PATH := $(ROOT_PATH)/rtl/mini CORE ?= HAZARD3 @@ -34,6 +51,34 @@ FIRMWARE_NAME ?= retrosoc_fw PROG_TYPE ?= FULL LINK_TYPE ?= ld2_sram +BUILD_ROOT ?= $(ROOT_PATH)/build +CACHE_ROOT ?= $(ROOT_PATH)/.cache/retrosoc +MAX_JOBS ?= 16 +JOBS ?= $(shell count=$$(nproc 2>/dev/null || printf '1'); \ + if [ "$$count" -gt "$(MAX_JOBS)" ]; then printf '%s' '$(MAX_JOBS)'; \ + else printf '%s' "$$count"; fi) +CONFIG_KEY_VARS := SOC CORE IP PDK HAVE_PLL HAVE_SRAM_IF HAVE_SRAM_MACRO HAVE_SVA \ + ISA HAVE_CSR PROG_TYPE LINK_TYPE RTL_TOP FIRMWARE_NAME +VARIANT_ID := $(strip $(shell python3 $(ROOT_PATH)/scripts/config_key.py \ + --lock $(LOCK_FILE) --profile $(PROFILE_NAME) \ + $(foreach var,$(CONFIG_KEY_VARS),--value $(var)=$($(var))))) +ifeq ($(VARIANT_ID),) +$(error Failed to calculate build variant ID) +endif +LOCK_DIGEST := $(strip $(shell python3 $(ROOT_PATH)/scripts/dependency_lock.py --lock $(LOCK_FILE) --digest)) +VARIANT_ROOT := $(abspath $(BUILD_ROOT))/$(VARIANT_ID) +SW_BUILD_DIR := $(VARIANT_ROOT)/sw +SIM_TOOL_NAME := $(shell printf '%s' '$(SIMU)' | tr '[:upper:]' '[:lower:]') +SIM_BUILD_ROOT := $(VARIANT_ROOT)/sim/$(SIM_TOOL_NAME) +SYN_BUILD_ROOT := $(VARIANT_ROOT)/syn/yosys +STA_BUILD_ROOT := $(VARIANT_ROOT)/sta/opensta +META_DIR := $(VARIANT_ROOT)/meta +ifeq ($(SYNTH),YOSYS) +FLOW_FILELIST_DIR := $(SYN_BUILD_ROOT)/filelists +else +FLOW_FILELIST_DIR := $(SIM_BUILD_ROOT)/filelists +endif + VALID_SOC := MINI VALID_CORE := PICORV32 HAZARD3 MDD VALID_IP := NONE MDD @@ -114,7 +159,9 @@ ifeq ($(STA), OPENSTA) include sta/opensta/opensta.mk endif -.PHONY: help config doctor setup setup-mpw setup-core setup-clusterip setup-ip setup-pdk setup-app clean-all +.PHONY: help config doctor setup setup-mpw setup-core setup-clusterip setup-ip setup-pdk setup-app \ + clean-all purge-cache manifest check-warnings metrics check-metrics package \ + regress-pr regress-nightly sim-asm .NOTPARALLEL: setup help: @@ -122,19 +169,28 @@ help: 'retroSoC build targets:' \ ' firmware | asm build firmware' \ ' comp | sim behavioral simulation' \ + ' sim-asm build/run the assembly self-test' \ ' netcomp | netsim synthesized-netlist simulation' \ ' postcomp | postsim post-layout simulation' \ ' synth | sta synthesis and timing analysis' \ ' setup install pinned external dependencies' \ ' doctor check tools, paths, and selected configuration' \ - ' config print selected configuration' \ - ' clean | clean-all clean current backend or all generated output' \ + ' config | manifest print/write the effective configuration' \ + ' check-warnings | metrics analyze flow logs and reports' \ + ' check-metrics apply the committed metrics policy' \ + ' regress-pr | regress-nightly run supported regression suites' \ + ' package create checksummed source deliverables' \ + ' clean | clean-all clean current flow or all build output' \ + ' purge-cache remove dependency and compiler caches' \ '' \ - 'Key variables: SOC CORE IP SIMU SYNTH STA PDK ISA PROG_TYPE LINK_TYPE' + 'Usage: make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk SIMU=IVERILOG sim' config: @printf '%-18s %s\n' \ - ROOT_PATH '$(ROOT_PATH)' SOC '$(SOC)' CORE '$(CORE)' IP '$(IP)' \ + ROOT_PATH '$(ROOT_PATH)' CONFIG '$(or $(CONFIG_PATH),)' \ + VARIANT_ID '$(VARIANT_ID)' VARIANT_ROOT '$(VARIANT_ROOT)' \ + JOBS '$(JOBS)' \ + SOC '$(SOC)' CORE '$(CORE)' IP '$(IP)' \ SIMU '$(SIMU)' SYNTH '$(SYNTH)' STA '$(STA)' PDK '$(PDK)' \ HAVE_PLL '$(HAVE_PLL)' HAVE_SRAM_IF '$(HAVE_SRAM_IF)' \ HAVE_SRAM_MACRO '$(HAVE_SRAM_MACRO)' HAVE_SVA '$(HAVE_SVA)' \ @@ -144,13 +200,14 @@ config: doctor: @python3 $(ROOT_PATH)/scripts/doctor.py \ --root $(ROOT_PATH) --simu $(SIMU) --synth $(SYNTH) --sta $(STA) \ - --pdk $(PDK) --core $(CORE) --ip $(IP) + --pdk $(PDK) --core $(CORE) --ip $(IP) --lock $(LOCK_FILE) setup: setup-mpw setup-core setup-clusterip setup-ip setup-pdk setup-app setup-mpw: python3 $(ROOT_PATH)/setup.py - python3 $(ROOT_PATH)/scripts/prepare_mpw.py + python3 $(ROOT_PATH)/scripts/prepare_mpw.py \ + --lock-file $(CACHE_ROOT)/locks/mpw-prepare.lock @mkdir -p $(dir $(MPW_STAMP)) @touch $(MPW_STAMP) @@ -164,10 +221,46 @@ setup-ip: python3 $(ROOT_PATH)/rtl/ip/setup.py setup-pdk: - python3 $(ROOT_PATH)/pdk/setup.py + python3 $(ROOT_PATH)/pdk/setup.py --pdk $(PDK) setup-app: python3 $(ROOT_PATH)/app/setup.py clean-all: - python3 $(ROOT_PATH)/scripts/clean.py --root $(ROOT_PATH) + python3 $(ROOT_PATH)/scripts/clean.py --root $(ROOT_PATH) --path $(abspath $(BUILD_ROOT)) + +purge-cache: + python3 $(ROOT_PATH)/scripts/clean.py --root $(ROOT_PATH) --path $(abspath $(CACHE_ROOT)) + +manifest: + python3 $(ROOT_PATH)/scripts/manifest.py create --root $(ROOT_PATH) \ + --lock $(LOCK_FILE) --output $(META_DIR)/manifest.json --profile $(PROFILE_NAME) \ + $(foreach var,$(CONFIG_KEY_VARS),--config $(var)=$($(var))) \ + --config SIMU=$(SIMU) --config SYNTH=$(SYNTH) --config STA=$(STA) + +check-warnings: + python3 $(ROOT_PATH)/scripts/analyze_warnings.py check --root $(ROOT_PATH) \ + --profile $(PROFILE_NAME) --variant-root $(VARIANT_ROOT) --output $(META_DIR)/warnings.json + +metrics: + python3 $(ROOT_PATH)/scripts/metrics.py collect --variant-root $(VARIANT_ROOT) \ + --output $(META_DIR)/metrics.json + +check-metrics: metrics + python3 $(ROOT_PATH)/scripts/metrics.py check --metrics $(META_DIR)/metrics.json \ + --policy $(ROOT_PATH)/quality/metrics/policy.json \ + --baseline $(ROOT_PATH)/quality/metrics/baseline.json + +package: $(MPW_VARIANT_STAMP) $(FILELIST_STAMP) manifest + python3 $(ROOT_PATH)/scripts/package.py --root $(ROOT_PATH) --lock $(LOCK_FILE) \ + --variant-root $(VARIANT_ROOT) --output-dir $(ROOT_PATH)/dist/$(VARIANT_ID) + +regress-pr: + python3 $(ROOT_PATH)/scripts/regress.py --root $(ROOT_PATH) --suite pr + +regress-nightly: + python3 $(ROOT_PATH)/scripts/regress.py --root $(ROOT_PATH) --suite nightly + +sim-asm: asm + $(MAKE) SIM_FIRMWARE_NAME=$(ASM_FIRMWARE_NAME) \ + SIM_SUCCESS_MARKER='Mem wr/rd test success' sim diff --git a/README.md b/README.md index 3bd93266..7fbf2462 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,38 @@ # retroSoC -[![style](https://github.com/retroSoC/retroSoC/actions/workflows/style.yml/badge.svg)](https://github.com/retroSoC/retroSoC/actions/workflows/style.yml) -[![lint](https://github.com/retroSoC/retroSoC/actions/workflows/lint.yml/badge.svg)](https://github.com/retroSoC/retroSoC/actions/workflows/lint.yml) -[![test](https://github.com/retroSoC/retroSoC/actions/workflows/test.yml/badge.svg)](https://github.com/retroSoC/retroSoC/actions/workflows/test.yml) +[![quality](https://github.com/retroSoC/retroSoC/actions/workflows/quality.yml/badge.svg)](https://github.com/retroSoC/retroSoC/actions/workflows/quality.yml) +[![regression](https://github.com/retroSoC/retroSoC/actions/workflows/regression.yml/badge.svg)](https://github.com/retroSoC/retroSoC/actions/workflows/regression.yml) +[![nightly](https://github.com/retroSoC/retroSoC/actions/workflows/nightly.yml/badge.svg)](https://github.com/retroSoC/retroSoC/actions/workflows/nightly.yml) -## Build automation +## Build Automation -Run `make help` for the supported build targets and `make config` for the active -configuration. `make doctor` checks the selected simulator, synthesis/STA tools, -RISC-V compiler, PDK, and external source trees before a long build starts. - -External dependencies are pinned to known commits. Install them with `make setup`; -individual targets such as `setup-pdk`, `setup-ip`, and `setup-app` are also -available. Setup scripts are idempotent and accept `--update` when an existing -checkout must be moved to the pinned revision. - -Typical open-source flows are: +The build uses committed configuration profiles and a single dependency lock. Start with the +verified HAZARD3 profile: ```sh -make PROG_TYPE=BASE firmware -make SIMU=VERILATOR sim -make SIMU=IVERILOG sim -make SYNTH=YOSYS synth -make STA=OPENSTA sta +python3 -m pip install --requirement requirements/build.txt +make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk setup +make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk SIMU=IVERILOG doctor +make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk firmware +make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk SIMU=IVERILOG RTL_SIM_TIMEOUT=5200000 sim-asm ``` -Generated outputs are isolated under ignored build directories. `make clean` -cleans the selected simulator backend and `make clean-all` removes all generated -flow output while leaving dependency checkouts intact. +Other open-source flows use the same profile: + +```sh +make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk SIMU=VERILATOR sim +make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk SYNTH=YOSYS synth +make CONFIG=configs/ci/hazard3-rv32im-ihp130.mk STA=OPENSTA sta +``` + +`make help` lists targets, while `make config` prints the effective configuration and its variant +identifier. Outputs live below `build/-/`; firmware, simulator, synthesis, +STA, reports, and manifests are isolated inside that directory. `make clean` removes the selected +backend, `make clean-all` removes all build outputs, and `make purge-cache` explicitly removes +download and compiler caches. + +Dependency sources and binary tool archives are pinned in +[`config/dependencies.lock.json`](config/dependencies.lock.json). Downloads are checksum verified, +and every flow writes a machine-readable manifest and result records. See +[`docs/engineering.md`](docs/engineering.md) for CI coverage, warning baselines, metrics promotion, +release packaging, and cluster-only flows. diff --git a/app/asm/Makefile b/app/asm/Makefile index 4456c065..b24c45bd 100644 --- a/app/asm/Makefile +++ b/app/asm/Makefile @@ -9,9 +9,11 @@ OBJCOPY = $(GCC)objcopy AS_FLAGS = -fno-pic -march=rv32im -mabi=ilp32 LDFLAGS = -T flash.ld -melf32lriscv APP ?= hello -APP_ALL = $(APP)-asm +OUT_DIR ?= . +APP_ALL = $(OUT_DIR)/$(APP)-asm all: $(APP).s + @mkdir -p $(OUT_DIR) $(AS) $(AS_FLAGS) $(APP).s -o $(APP_ALL).o $(LD) $(LDFLAGS) $(APP_ALL).o -o $(APP_ALL).elf $(OBJDUMP) -d $(APP_ALL).elf > $(APP_ALL).txt @@ -19,4 +21,4 @@ all: $(APP).s $(OBJCOPY) -S -I binary -O verilog $(APP_ALL).bin $(APP_ALL).flash clean: - rm -rf $(APP_ALL).o $(APP_ALL).elf $(APP_ALL).txt $(APP_ALL).bin $(APP_ALL).flash \ No newline at end of file + rm -rf $(APP_ALL).o $(APP_ALL).elf $(APP_ALL).txt $(APP_ALL).bin $(APP_ALL).flash diff --git a/app/setup.py b/app/setup.py index 7e316773..ce499533 100644 --- a/app/setup.py +++ b/app/setup.py @@ -9,16 +9,13 @@ from zipfile import ZipFile ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import archive, source # noqa: E402 from scripts.setup_helpers import atomic_write, download_file, ensure_git_repo # noqa: E402 APP_DIR = Path(__file__).resolve().parent FATFS_DIR = APP_DIR / "fatfs" -FATFS_ARCHIVE = FATFS_DIR / "ff16/ff16.zip" -FATFS_URL = "https://elm-chan.org/fsw/ff/arc/ff16.zip" -FATFS_SHA256 = "41d98115f72b090c2d0c269a001c5c0216efd78fbd84bb6427be808a76315a5a" COREMARK_DIR = APP_DIR / "coremark/coremark-main" -COREMARK_REVISION = "1f483d5b8316753a742cbf5590caf5bd0a4e4777" def patch_coremark() -> None: @@ -47,26 +44,22 @@ def main() -> int: parser.add_argument("--update", action="store_true") args = parser.parse_args() - download_file( - FATFS_URL, - FATFS_ARCHIVE, - FATFS_SHA256, - update=args.update, - ) - fatfs_source = FATFS_ARCHIVE.parent / "source" + fatfs = archive("fatfs") + fatfs_archive = ROOT / fatfs["destination"] + download_file(fatfs["url"], fatfs_archive, fatfs["sha256"], update=args.update) + fatfs_source = fatfs_archive.parent / "source" if not fatfs_source.is_dir(): - with ZipFile(FATFS_ARCHIVE) as archive: - archive.extractall(FATFS_ARCHIVE.parent) + with ZipFile(fatfs_archive) as zip_archive: + zip_archive.extractall(fatfs_archive.parent) for name in ("ffconf.h", "diskio.c"): - source = FATFS_DIR / name - if not source.is_file(): - raise FileNotFoundError(f"FatFs configuration file missing: {source}") - shutil.copy2(source, fatfs_source / name) + config_source = FATFS_DIR / name + if not config_source.is_file(): + raise FileNotFoundError(f"FatFs configuration file missing: {config_source}") + shutil.copy2(config_source, fatfs_source / name) + coremark = source("coremark") ensure_git_repo( - "https://github.com/eembc/coremark.git", - COREMARK_DIR, - COREMARK_REVISION, + coremark["url"], ROOT / coremark["destination"], coremark["revision"], update=args.update, allow_dirty=True, ) diff --git a/config/dependencies.lock.json b/config/dependencies.lock.json new file mode 100644 index 00000000..c7a6420f --- /dev/null +++ b/config/dependencies.lock.json @@ -0,0 +1,124 @@ +{ + "schema_version": 1, + "sources": { + "mpw": { + "url": "https://github.com/retroSoC/mini-ver-mpw.git", + "revision": "30503eb8a7cf33a0c955913faba5014ebd00186f", + "destination": "rtl/mini/mpw", + "license": "NOASSERTION" + }, + "hazard3": { + "url": "https://github.com/Wren6991/Hazard3.git", + "revision": "5b3a34f6955e50e03bdcb964201c91462f7078c3", + "destination": "rtl/mini/mpw/Hazard3", + "license": "NOASSERTION" + }, + "ibex": { + "url": "https://github.com/lowRISC/ibex.git", + "revision": "0c233f54361d769f370889223acc456f2ac19d46", + "destination": "rtl/mini/mpw/ibex", + "license": "Apache-2.0" + }, + "coremark": { + "url": "https://github.com/eembc/coremark.git", + "revision": "1f483d5b8316753a742cbf5590caf5bd0a4e4777", + "destination": "app/coremark/coremark-main", + "license": "Apache-2.0" + }, + "third_party_ip": { + "url": "https://github.com/retroSoC/3rd-party.git", + "revision": "d991ec9f7bf696e7e5d840758487068b11bd4121", + "destination": "rtl/ip/3rd-party", + "license": "NOASSERTION" + }, + "pdk_ihp130": { + "url": "https://github.com/IHP-GmbH/IHP-Open-PDK.git", + "revision": "68eebafcd9b2f5e92c69d37a8d3d90eb266550f5", + "destination": "pdk/IHP-Open-PDK", + "recursive": true, + "license": "Apache-2.0" + }, + "pdk_ics55": { + "url": "https://github.com/openecos-projects/icsprout55-pdk.git", + "revision": "e696e093129ca2212487aa169af74d06ebd86eb6", + "destination": "pdk/icsprout55-pdk", + "recursive": true, + "license": "NOASSERTION" + }, + "cluster_archinfo": {"url": "https://github.com/retroSoC/archinfo.git", "revision": "d65dd13c83a1176668bf5178004814a1551bf6ba", "destination": "rtl/clusterip/archinfo", "license": "NOASSERTION"}, + "cluster_clint": {"url": "https://github.com/retroSoC/clint.git", "revision": "ce49f1a5501c0a9f18c0e3f358417648f3f2fb18", "destination": "rtl/clusterip/clint", "license": "NOASSERTION"}, + "cluster_common": {"url": "https://github.com/retroSoC/common.git", "revision": "d9921eb05160be873aebc46bb2f820bee2c7eebc", "destination": "rtl/clusterip/common", "license": "NOASSERTION"}, + "cluster_crc": {"url": "https://github.com/retroSoC/crc.git", "revision": "4654a63d06d487b5b9fe22af228c36c736b90bad", "destination": "rtl/clusterip/crc", "license": "NOASSERTION"}, + "cluster_i2c": {"url": "https://github.com/retroSoC/i2c.git", "revision": "73b024547c5cd70fb8bf801cc7757db2536c604f", "destination": "rtl/clusterip/i2c", "license": "NOASSERTION"}, + "cluster_i2s": {"url": "https://github.com/retroSoC/i2s.git", "revision": "bd0978e8b45e4cf4c1335a1fa7311385d2a4e805", "destination": "rtl/clusterip/i2s", "license": "NOASSERTION"}, + "cluster_plic": {"url": "https://github.com/retroSoC/plic.git", "revision": "c851d93829d2e75b224003a058e718925855c0cc", "destination": "rtl/clusterip/plic", "license": "NOASSERTION"}, + "cluster_ps2": {"url": "https://github.com/retroSoC/ps2.git", "revision": "60921e0f72d6113f8615d03ce54567388fc2410f", "destination": "rtl/clusterip/ps2", "license": "NOASSERTION"}, + "cluster_pwm": {"url": "https://github.com/retroSoC/pwm.git", "revision": "0777ef085c661d2b103f2fa0f8519f97eea3c950", "destination": "rtl/clusterip/pwm", "license": "NOASSERTION"}, + "cluster_rng": {"url": "https://github.com/retroSoC/rng.git", "revision": "7374e1e7f25bfd1ff9954ea99f19b77a583def29", "destination": "rtl/clusterip/rng", "license": "NOASSERTION"}, + "cluster_rtc": {"url": "https://github.com/retroSoC/rtc.git", "revision": "69963b8424cb00e04c8be70157b8ca738464e342", "destination": "rtl/clusterip/rtc", "license": "NOASSERTION"}, + "cluster_spi": {"url": "https://github.com/retroSoC/spi.git", "revision": "515cb408806584525605b4747db5e890dbdf1928", "destination": "rtl/clusterip/spi", "license": "NOASSERTION"}, + "cluster_timer": {"url": "https://github.com/retroSoC/timer.git", "revision": "0c44b0d8b742c6a477fda15a812cf22962b54dd8", "destination": "rtl/clusterip/timer", "license": "NOASSERTION"}, + "cluster_uart": {"url": "https://github.com/retroSoC/uart.git", "revision": "eca12213ba06f42e0a6dd0f27907fd4c447e2774", "destination": "rtl/clusterip/uart", "license": "NOASSERTION"}, + "cluster_wdg": {"url": "https://github.com/retroSoC/wdg.git", "revision": "b2603c1faeba96603392f0fe9e692e645d7e061f", "destination": "rtl/clusterip/wdg", "license": "NOASSERTION"} + }, + "archives": { + "fatfs": { + "url": "https://elm-chan.org/fsw/ff/arc/ff16.zip", + "sha256": "41d98115f72b090c2d0c269a001c5c0216efd78fbd84bb6427be808a76315a5a", + "destination": "app/fatfs/ff16/ff16.zip", + "license": "LicenseRef-FatFs" + }, + "serv": { + "url": "https://github.com/olofk/serv/archive/refs/tags/1.4.0.tar.gz", + "sha256": "f81c37b9f9d548c658e23bb7eb90fbed8b54e2e9e51fa42b8cd2f34cecc472ab", + "destination": "rtl/mini/mpw/serv-1.4.0.tar.gz", + "license": "ISC" + } + }, + "toolchains": { + "ubuntu-22.04": { + "verilator": { + "version": "5.046", + "url": "https://github.com/retroSoC/artifact/releases/download/verilator-v5.046/verilator-v5.046-ubuntu-22.04-gcc.tar.gz", + "sha256": "f91b8496a0aaf26bf72395e876e145ceb99c94ff724cc9a16bb81d243ce5d953", + "archive": "verilator-v5.046-ubuntu-22.04-gcc.tar.gz", + "path": "bin" + }, + "sv2v": { + "version": "0.0.13-9-gd381209", + "url": "https://github.com/retroSoC/artifact/releases/download/sv2v-v0.0.13-9-gd381209/sv2v-v0.0.13-9-gd381209-ubuntu-22.04-gcc.tar.gz", + "sha256": "af0f5438fc293d1469d4d3e01be442663c55f5db6ff28bbed40722eaf429c690", + "archive": "sv2v-v0.0.13-9-gd381209-ubuntu-22.04-gcc.tar.gz", + "path": "." + }, + "iverilog": { + "version": "13.0", + "url": "https://github.com/retroSoC/artifact/releases/download/iverilog-v13_0/iverilog-v13_0-ubuntu-22.04-gcc.tar.gz", + "sha256": "22b51e988a94abdd0cd94f31f804a9bd6183e98d9380799ad3441188abc12987", + "archive": "iverilog-v13_0-ubuntu-22.04-gcc.tar.gz", + "path": "bin" + }, + "yosys": { + "version": "0.58", + "url": "https://github.com/retroSoC/artifact/releases/download/yosys-v0.58/yosys-v0.58-ubuntu-22.04-gcc.tar.gz", + "sha256": "589447b6ed9aa20a1cc90155851f809490815089e5cc181ed4d23a5389b65adb", + "archive": "yosys-v0.58-ubuntu-22.04-gcc.tar.gz", + "path": "bin" + }, + "opensta": { + "version": "2.2.0-1767-ga93b51ec", + "url": "https://github.com/retroSoC/artifact/releases/download/opensta-v2.2.0-1767-ga93b51ec/opensta-v2.2.0-1767-ga93b51ec-ubuntu-22.04-gcc.tar.gz", + "sha256": "e8b17134cd0935d2f5d354e8233c27833834141f2aef44264a1ad5d2cca0623b", + "archive": "opensta-v2.2.0-1767-ga93b51ec-ubuntu-22.04-gcc.tar.gz", + "path": "bin" + }, + "riscv_gnu": { + "version": "2025.05.01", + "url": "https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2025.05.01/riscv32-elf-ubuntu-22.04-gcc-nightly-2025.05.01-nightly.tar.xz", + "sha256": "39dc7e8554d8d152258d2588cdb14c9dff5d2e8563325b78a923faa868a07191", + "archive": "riscv32-elf-ubuntu-22.04-gcc-nightly-2025.05.01-nightly.tar.xz", + "path": "riscv/bin" + } + } + } +} diff --git a/configs/ci/hazard3-rv32im-ihp130.mk b/configs/ci/hazard3-rv32im-ihp130.mk new file mode 100644 index 00000000..9b6317d3 --- /dev/null +++ b/configs/ci/hazard3-rv32im-ihp130.mk @@ -0,0 +1,12 @@ +SOC := MINI +CORE := HAZARD3 +IP := NONE +PDK := IHP130 +HAVE_PLL := NO +HAVE_SRAM_IF := NO +HAVE_SRAM_MACRO := NO +HAVE_SVA := NO +ISA := RV32IM +HAVE_CSR := NO +PROG_TYPE := BASE +LINK_TYPE := ld2_psram diff --git a/configs/ci/mdd-rv32im-ihp130.mk b/configs/ci/mdd-rv32im-ihp130.mk new file mode 100644 index 00000000..9b0b98dd --- /dev/null +++ b/configs/ci/mdd-rv32im-ihp130.mk @@ -0,0 +1,12 @@ +SOC := MINI +CORE := MDD +IP := NONE +PDK := IHP130 +HAVE_PLL := NO +HAVE_SRAM_IF := NO +HAVE_SRAM_MACRO := NO +HAVE_SVA := NO +ISA := RV32IM +HAVE_CSR := NO +PROG_TYPE := BASE +LINK_TYPE := ld2_psram diff --git a/configs/cluster/hazard3-ics55.mk b/configs/cluster/hazard3-ics55.mk new file mode 100644 index 00000000..950fe8ca --- /dev/null +++ b/configs/cluster/hazard3-ics55.mk @@ -0,0 +1,12 @@ +SOC := MINI +CORE := HAZARD3 +IP := NONE +PDK := ICS55 +HAVE_PLL := NO +HAVE_SRAM_IF := NO +HAVE_SRAM_MACRO := NO +HAVE_SVA := NO +ISA := RV32IM +HAVE_CSR := NO +PROG_TYPE := BASE +LINK_TYPE := ld2_psram diff --git a/configs/nightly/picorv32-rv32im-ihp130.mk b/configs/nightly/picorv32-rv32im-ihp130.mk new file mode 100644 index 00000000..81315a49 --- /dev/null +++ b/configs/nightly/picorv32-rv32im-ihp130.mk @@ -0,0 +1,12 @@ +SOC := MINI +CORE := PICORV32 +IP := NONE +PDK := IHP130 +HAVE_PLL := NO +HAVE_SRAM_IF := NO +HAVE_SRAM_MACRO := NO +HAVE_SVA := NO +ISA := RV32IM +HAVE_CSR := NO +PROG_TYPE := BASE +LINK_TYPE := ld2_psram diff --git a/docs/engineering.md b/docs/engineering.md new file mode 100644 index 00000000..bab01d86 --- /dev/null +++ b/docs/engineering.md @@ -0,0 +1,102 @@ +# Engineering Workflow + +## Supported Profiles + +| Tier | Profile | Automated coverage | +| --- | --- | --- | +| Pull request | `configs/ci/hazard3-rv32im-ihp130.mk` | firmware, Verilator, Icarus, Yosys, netlist Icarus, OpenSTA | +| Pull request | `configs/ci/mdd-rv32im-ihp130.mk` | firmware and Verilator | +| Nightly | `configs/nightly/picorv32-rv32im-ihp130.mk` | firmware, Verilator, and Icarus | +| Cluster | `configs/cluster/hazard3-ics55.mk` | configuration only; run with site PDK/tool access | + +VCS and ICS55 flows remain cluster-only because public runners do not have the licensed simulator or +the validated site environment. They use the same variant layout, manifests, result JSON, warning +scanner, and cleanup rules. + +## Reproducible Inputs + +`config/dependencies.lock.json` is the source of truth for external Git repositories, downloaded +archives, and Ubuntu 22.04 toolchain bundles. Git checkouts use full 40-character revisions. Archive +downloads are accepted only after SHA-256 verification. CI actions are pinned to commit IDs. + +To update a dependency: + +1. Update its revision, version, URL, and checksum in the lock. +2. Run `python3 scripts/dependency_lock.py`. +3. Run the affected setup target with `--update` directly, then run `make doctor`. +4. Run `python3 -m pytest -q` and the affected regression profile. +5. Review the generated manifest, warning delta, metrics, and SBOM before merging. + +Do not reuse an old checksum with a new URL or version. The lock digest is part of every variant ID, +so changing the lock creates a new output directory without contaminating prior results. + +## Build Layout + +The variant identifier is the profile name plus a hash over effective hardware/software build +settings and the dependency lock. Runtime-only settings such as wave capture and timeout do not +create a new variant. + +```text +build// + generated/mpw// + sw/ + sim// + syn/yosys/ + sta/opensta/ + meta/manifest.json + meta/warnings.json + meta/metrics.json +``` + +Generated filelists and MPW output are flow-local. Make depfiles track expanded RTL sources and +included headers. The shared MPW generator is protected by a file lock. Default tool parallelism is +capped at 16 and can be set with `JOBS=` or `MAX_JOBS=`. + +## Result Policy + +Every EDA and simulation flow command is run through `scripts/run_flow.py`, which streams a log and writes a +result JSON containing the command, timestamps, duration, exit code, and status. A simulation passes +only when the command succeeds, its log contains the firmware startup marker, and no fatal/failure +marker is present. Icarus regressions use the assembly self-test as `retrosoc_asm`, while the normal +`retrosoc_fw` image remains available for firmware size tracking and Verilator regressions. + +`make check-warnings` normalizes tool messages and compares them with +`quality/warnings//.json`. New signatures and increased counts fail; resolved warnings +are reported but do not fail. To refresh one baseline after review: + +```sh +python3 scripts/analyze_warnings.py baseline \ + --root . --profile hazard3-rv32im-ihp130 --tool verilator \ + --log build//sim/verilator/verilating.log \ + --output quality/warnings/hazard3-rv32im-ihp130/verilator.json +``` + +## Metrics Promotion + +`make check-metrics` currently uses `quality/metrics/policy.json` in `observe` mode. Regression +artifacts retain the metrics from each successful run. After ten successful `main` runs for the same +profile and locked tools, create the median baseline: + +```sh +python3 scripts/metrics.py baseline \ + --input run-01/metrics.json --input run-02/metrics.json \ + --input run-03/metrics.json --input run-04/metrics.json \ + --input run-05/metrics.json --input run-06/metrics.json \ + --input run-07/metrics.json --input run-08/metrics.json \ + --input run-09/metrics.json --input run-10/metrics.json \ + --output quality/metrics/baseline.json +``` + +Review the medians, commit the baseline, then change policy mode to `gate`. The committed future +limits are 5% firmware growth, 3% top area/cell growth, and 0.05 ns WNS regression. TNS is recorded +for diagnosis but is initially non-blocking. + +## CI And Releases + +`quality.yml` validates Python, YAML, GitHub Actions, the dependency lock, and script tests. +`regression.yml` runs verified configurations on pull requests and main branches. `nightly.yml` adds +PICORV32. Source dependencies, locked tool archives, and Verilator `ccache` use separate cache keys. + +Tags matching `v*` run `release.yml`. The release contains a flattened SystemVerilog export, a source +tarball, build manifest, dependency lock, CycloneDX SBOM, and `SHA256SUMS`. `make package` creates the +same files locally below `dist//`. diff --git a/pdk/setup.py b/pdk/setup.py index 8442806d..46b5013b 100644 --- a/pdk/setup.py +++ b/pdk/setup.py @@ -7,34 +7,24 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import source # noqa: E402 from scripts.setup_helpers import ensure_git_repo # noqa: E402 -DEPENDENCIES = ( - ( - "https://github.com/IHP-GmbH/IHP-Open-PDK.git", - "IHP-Open-PDK", - "68eebafcd9b2f5e92c69d37a8d3d90eb266550f5", - ), - ( - "https://github.com/openecos-projects/icsprout55-pdk.git", - "icsprout55-pdk", - "e696e093129ca2212487aa169af74d06ebd86eb6", - ), -) - - def main() -> int: parser = argparse.ArgumentParser(description="Install pinned open PDKs") parser.add_argument("--update", action="store_true") + parser.add_argument("--pdk", choices=("IHP130", "ICS55"), action="append") args = parser.parse_args() - pdk_dir = Path(__file__).resolve().parent - for url, name, revision in DEPENDENCIES: + selected = args.pdk or ("IHP130", "ICS55") + names = {"IHP130": "pdk_ihp130", "ICS55": "pdk_ics55"} + for name in (names[pdk] for pdk in selected): + dependency = source(name) ensure_git_repo( - url, - pdk_dir / name, - revision, - recursive=True, + dependency["url"], + ROOT / dependency["destination"], + dependency["revision"], + recursive=dependency.get("recursive", False), update=args.update, ) return 0 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..c6087594 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[tool.pytest.ini_options] +addopts = "-ra" +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 +target-version = "py310" +extend-exclude = [ + ".cache", + "build", + "dist", + "app/coremark/coremark-main", + "app/fatfs/ff16", + "pdk/IHP-Open-PDK", + "pdk/icsprout55-pdk", + "rtl/clusterip", + "rtl/ip/3rd-party", + "rtl/mini/mpw", +] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] diff --git a/quality/metrics/README.md b/quality/metrics/README.md new file mode 100644 index 00000000..8bf9bddb --- /dev/null +++ b/quality/metrics/README.md @@ -0,0 +1,5 @@ +# Metrics Policy + +The policy starts in `observe` mode. Exactly ten successful `main` run metric files are required to +create the median baseline. Promotion to `gate` is a reviewed commit containing both +`baseline.json` and the policy mode change. See `docs/engineering.md` for the command and thresholds. diff --git a/quality/metrics/policy.json b/quality/metrics/policy.json new file mode 100644 index 00000000..4dc0856a --- /dev/null +++ b/quality/metrics/policy.json @@ -0,0 +1,12 @@ +{ + "schema_version": 1, + "mode": "observe", + "required_baseline_samples": 10, + "maximum_growth": { + "firmware.retrosoc_fw.bin.bytes": 0.05, + "synthesis.top_area": 0.03, + "synthesis.top_cells": 0.03 + }, + "maximum_wns_regression": 0.05, + "tns_policy": "report_only" +} diff --git a/quality/warnings/README.md b/quality/warnings/README.md new file mode 100644 index 00000000..7b47ab94 --- /dev/null +++ b/quality/warnings/README.md @@ -0,0 +1,7 @@ +# Warning Baselines + +Baselines are scoped by committed profile and tool. Each entry stores a normalized warning signature, +its accepted count, and one example. Path roots and source line numbers are removed before hashing. + +Do not hand-edit signatures. Regenerate only from a successful flow using +`scripts/analyze_warnings.py baseline`, review the full diff, and run `make check-warnings`. diff --git a/quality/warnings/hazard3-rv32im-ihp130/iverilog.json b/quality/warnings/hazard3-rv32im-ihp130/iverilog.json new file mode 100644 index 00000000..26f02440 --- /dev/null +++ b/quality/warnings/hazard3-rv32im-ihp130/iverilog.json @@ -0,0 +1,17 @@ +{ + "entries": [ + { + "count": 2, + "example": "warning:Found both default and explicit timescale based delays. Use", + "signature": "aee22756a35889cc" + }, + { + "count": 4, + "example": "sorry:ifnone with an edge-sensitive path is not supported.", + "signature": "c4297f454f443fed" + } + ], + "profile": "hazard3-rv32im-ihp130", + "schema_version": 1, + "tool": "iverilog" +} diff --git a/quality/warnings/hazard3-rv32im-ihp130/opensta.json b/quality/warnings/hazard3-rv32im-ihp130/opensta.json new file mode 100644 index 00000000..12f5ea7d --- /dev/null +++ b/quality/warnings/hazard3-rv32im-ihp130/opensta.json @@ -0,0 +1,6 @@ +{ + "entries": [], + "profile": "hazard3-rv32im-ihp130", + "schema_version": 1, + "tool": "opensta" +} diff --git a/quality/warnings/hazard3-rv32im-ihp130/verilator.json b/quality/warnings/hazard3-rv32im-ihp130/verilator.json new file mode 100644 index 00000000..d43556d8 --- /dev/null +++ b/quality/warnings/hazard3-rv32im-ihp130/verilator.json @@ -0,0 +1,42 @@ +{ + "entries": [ + { + "count": 2, + "example": "CMPCONST:$ROOT/rtl/ip/native/spisd_write.sv:: Comparison is constant due to limited range", + "signature": "11b9d285d4747e4f" + }, + { + "count": 1, + "example": "SPECIFYIGN:$ROOT/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_sram/verilog/RM_IHPSG13_1P_1024x64_c2_bm_bist.v:: Ignoring unsupported: specify block construct", + "signature": "7a759d6e07651fd8" + }, + { + "count": 68, + "example": "SPECIFYIGN:$ROOT/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_stdcell/verilog/sg13g2_stdcell.v:: Ignoring unsupported: specify block construct", + "signature": "c54331448f63ac73" + }, + { + "count": 3, + "example": "WIDTHTRUNC:$ROOT/rtl/ip/native/spisd_write.sv:: Bit extraction of var[7:0] requires 3 bit index, not 4 bits.", + "signature": "cdd5c3ca84a609c6" + }, + { + "count": 1, + "example": "UNOPTFLAT:$ROOT/rtl/ip/native/xpi.sv:: Signal unoptimizable: Circular combinational logic: 'retrosoc_top.u_retrosoc_asic.u_xpi_if.io_di_i'", + "signature": "ee29e23bc895ee17" + }, + { + "count": 1, + "example": "WIDTHTRUNC:$ROOT/rtl/ip/native/spisd_read.sv:: Operator ASSIGNDLY expects 8 bits on the Assign RHS, but Assign RHS's REPLICATE generates 9 bits.", + "signature": "f8f8278c6ce4431d" + }, + { + "count": 11, + "example": "SPECIFYIGN:$ROOT/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_io/verilog/sg13g2_io.v:: Ignoring unsupported: specify block construct", + "signature": "faa191e2dbf4a242" + } + ], + "profile": "hazard3-rv32im-ihp130", + "schema_version": 1, + "tool": "verilator" +} diff --git a/quality/warnings/hazard3-rv32im-ihp130/yosys.json b/quality/warnings/hazard3-rv32im-ihp130/yosys.json new file mode 100644 index 00000000..7632dae6 --- /dev/null +++ b/quality/warnings/hazard3-rv32im-ihp130/yosys.json @@ -0,0 +1,42 @@ +{ + "entries": [ + { + "count": 1, + "example": "warning:Wire hazard3_csr$retrosoc_asic.u_retrosoc.u_core_wrapper.u_hazard3_cpu_1port.core.csr_u.\\debug_mode is used but has no driver.", + "signature": "1097d6b92f19e707" + }, + { + "count": 1, + "example": "warning:asynchronous load value missing for variable '\\update_irq_reg_arrays.i'", + "signature": "56c5f69398be8d7b" + }, + { + "count": 1, + "example": "warning:Wire hazard3_csr$retrosoc_asic.u_retrosoc.u_core_wrapper.u_hazard3_cpu_1port.core.csr_u.\\pending_dbg_resume_prev is used but has no driver.", + "signature": "5bbc6025e51705ec" + }, + { + "count": 9, + "example": "warning:The network is combinational (run \"fraig\" or \"fraig_sweep\").", + "signature": "62cc5ee554312205" + }, + { + "count": 2, + "example": "warning:asynchronous load value missing for variable '\\update_nonconst.prev'", + "signature": "a30df20b98d49196" + }, + { + "count": 2, + "example": "warning:asynchronous load value missing for variable '\\update_nonconst.nonconst'", + "signature": "d35463a1de40c7b2" + }, + { + "count": 1, + "example": "warning:Wire hazard3_csr$retrosoc_asic.u_retrosoc.u_core_wrapper.u_hazard3_cpu_1port.core.csr_u.\\dbg_req_resume_prev is used but has no driver.", + "signature": "ef03dcca25b39dcb" + } + ], + "profile": "hazard3-rv32im-ihp130", + "schema_version": 1, + "tool": "yosys" +} diff --git a/quality/warnings/mdd-rv32im-ihp130/verilator.json b/quality/warnings/mdd-rv32im-ihp130/verilator.json new file mode 100644 index 00000000..8443aa98 --- /dev/null +++ b/quality/warnings/mdd-rv32im-ihp130/verilator.json @@ -0,0 +1,87 @@ +{ + "entries": [ + { + "count": 2, + "example": "CMPCONST:$ROOT/rtl/ip/native/spisd_write.sv:: Comparison is constant due to limited range", + "signature": "11b9d285d4747e4f" + }, + { + "count": 1, + "example": "PINMISSING:$BUILD/generated/mpw/verilator/core/username4/./serv/serv_synth_wrapper_username4.v:: Instance has missing pin: 'o_ren'", + "signature": "173218df65cf7bcd" + }, + { + "count": 1, + "example": "UNOPTFLAT:$BUILD/generated/mpw/verilator/core/username1/./kianV/alu_username1.v:: Signal unoptimizable: Circular combinational logic: 'retrosoc_top.u_retrosoc_asic.u_retrosoc.u_core_wrapper.u_user_core_wrapper.u_user_core_top.u_user_core_design_username1.u_kianv_harris_mc_edition.datapath_unit_I.alu_I.is_sub'", + "signature": "180cdbb151faa02d" + }, + { + "count": 1, + "example": "UNOPTFLAT:$BUILD/generated/mpw/verilator/core/username1/./kianV/kianv_harris_mc_edition_username1.v:: Signal unoptimizable: Circular combinational logic: 'retrosoc_top.u_retrosoc_asic.u_retrosoc.u_core_wrapper.u_user_core_wrapper.u_user_core_top.u_user_core_design_username1.u_kianv_harris_mc_edition.alu_valid'", + "signature": "25979c35fe4eb73f" + }, + { + "count": 1, + "example": "WIDTHEXPAND:$BUILD/generated/mpw/verilator/core/username6/./darkriscv/darkriscv_username6.v:: Operator COND expects 32 bits on the Conditional True, but Conditional True's LT generates 1 bits.", + "signature": "259baf70503bfe0d" + }, + { + "count": 1, + "example": "WIDTHEXPAND:$BUILD/generated/mpw/verilator/core/username6/./darkriscv/darkriscv_username6.v:: Operator COND expects 32 bits on the Conditional True, but Conditional True's LTS generates 1 bits.", + "signature": "323be25f98c25432" + }, + { + "count": 14, + "example": "WIDTHTRUNC:$BUILD/generated/mpw/verilator/core/username6/./darkriscv/darkriscv_username6.v:: Logical operator COND expects 1 bit on the Conditional Test, but Conditional Test's VARREF 'FLUSH' generates 2 bits.", + "signature": "4a2065ce464b898c" + }, + { + "count": 1, + "example": "UNOPTFLAT:$BUILD/generated/mpw/verilator/core/username1/./kianV/kianv_harris_mc_edition_username1.v:: Signal unoptimizable: Circular combinational logic: 'retrosoc_top.u_retrosoc_asic.u_retrosoc.u_core_wrapper.u_user_core_wrapper.u_user_core_top.u_user_core_design_username1.u_kianv_harris_mc_edition.ALUSrcA'", + "signature": "4edf683bf8ef29d3" + }, + { + "count": 1, + "example": "UNOPTFLAT:$BUILD/generated/mpw/verilator/core/username1/./kianV/alu_decoder_username1.v:: Signal unoptimizable: Circular combinational logic: 'retrosoc_top.u_retrosoc_asic.u_retrosoc.u_core_wrapper.u_user_core_wrapper.u_user_core_top.u_user_core_design_username1.u_kianv_harris_mc_edition.control_unit_I.alu_decoder_I.ALUControl'", + "signature": "5dc2cc2658c8e2c1" + }, + { + "count": 1, + "example": "SPECIFYIGN:$ROOT/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_sram/verilog/RM_IHPSG13_1P_1024x64_c2_bm_bist.v:: Ignoring unsupported: specify block construct", + "signature": "7a759d6e07651fd8" + }, + { + "count": 68, + "example": "SPECIFYIGN:$ROOT/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_stdcell/verilog/sg13g2_stdcell.v:: Ignoring unsupported: specify block construct", + "signature": "c54331448f63ac73" + }, + { + "count": 3, + "example": "WIDTHTRUNC:$ROOT/rtl/ip/native/spisd_write.sv:: Bit extraction of var[7:0] requires 3 bit index, not 4 bits.", + "signature": "cdd5c3ca84a609c6" + }, + { + "count": 1, + "example": "UNOPTFLAT:$ROOT/rtl/ip/native/xpi.sv:: Signal unoptimizable: Circular combinational logic: 'retrosoc_top.u_retrosoc_asic.u_xpi_if.io_di_i'", + "signature": "ee29e23bc895ee17" + }, + { + "count": 1, + "example": "UNOPTFLAT:$BUILD/generated/mpw/verilator/core/username1/./kianV/kianv_harris_mc_edition_username1.v:: Signal unoptimizable: Circular combinational logic: 'retrosoc_top.u_retrosoc_asic.u_retrosoc.u_core_wrapper.u_user_core_wrapper.u_user_core_top.u_user_core_design_username1.u_kianv_harris_mc_edition.ALUSrcB'", + "signature": "f1f9cfed209f63b9" + }, + { + "count": 1, + "example": "WIDTHTRUNC:$ROOT/rtl/ip/native/spisd_read.sv:: Operator ASSIGNDLY expects 8 bits on the Assign RHS, but Assign RHS's REPLICATE generates 9 bits.", + "signature": "f8f8278c6ce4431d" + }, + { + "count": 11, + "example": "SPECIFYIGN:$ROOT/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_io/verilog/sg13g2_io.v:: Ignoring unsupported: specify block construct", + "signature": "faa191e2dbf4a242" + } + ], + "profile": "mdd-rv32im-ihp130", + "schema_version": 1, + "tool": "verilator" +} diff --git a/quality/warnings/picorv32-rv32im-ihp130/iverilog.json b/quality/warnings/picorv32-rv32im-ihp130/iverilog.json new file mode 100644 index 00000000..feefa2b7 --- /dev/null +++ b/quality/warnings/picorv32-rv32im-ihp130/iverilog.json @@ -0,0 +1,17 @@ +{ + "entries": [ + { + "count": 1, + "example": "warning:Found both default and explicit timescale based delays. Use", + "signature": "aee22756a35889cc" + }, + { + "count": 2, + "example": "sorry:ifnone with an edge-sensitive path is not supported.", + "signature": "c4297f454f443fed" + } + ], + "profile": "picorv32-rv32im-ihp130", + "schema_version": 1, + "tool": "iverilog" +} diff --git a/quality/warnings/picorv32-rv32im-ihp130/verilator.json b/quality/warnings/picorv32-rv32im-ihp130/verilator.json new file mode 100644 index 00000000..1ad680c2 --- /dev/null +++ b/quality/warnings/picorv32-rv32im-ihp130/verilator.json @@ -0,0 +1,42 @@ +{ + "entries": [ + { + "count": 2, + "example": "CMPCONST:$ROOT/rtl/ip/native/spisd_write.sv:: Comparison is constant due to limited range", + "signature": "11b9d285d4747e4f" + }, + { + "count": 1, + "example": "SPECIFYIGN:$ROOT/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_sram/verilog/RM_IHPSG13_1P_1024x64_c2_bm_bist.v:: Ignoring unsupported: specify block construct", + "signature": "7a759d6e07651fd8" + }, + { + "count": 68, + "example": "SPECIFYIGN:$ROOT/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_stdcell/verilog/sg13g2_stdcell.v:: Ignoring unsupported: specify block construct", + "signature": "c54331448f63ac73" + }, + { + "count": 3, + "example": "WIDTHTRUNC:$ROOT/rtl/ip/native/spisd_write.sv:: Bit extraction of var[7:0] requires 3 bit index, not 4 bits.", + "signature": "cdd5c3ca84a609c6" + }, + { + "count": 1, + "example": "UNOPTFLAT:$ROOT/rtl/ip/native/xpi.sv:: Signal unoptimizable: Circular combinational logic: 'retrosoc_top.u_retrosoc_asic.u_xpi_if.io_di_i'", + "signature": "ee29e23bc895ee17" + }, + { + "count": 1, + "example": "WIDTHTRUNC:$ROOT/rtl/ip/native/spisd_read.sv:: Operator ASSIGNDLY expects 8 bits on the Assign RHS, but Assign RHS's REPLICATE generates 9 bits.", + "signature": "f8f8278c6ce4431d" + }, + { + "count": 11, + "example": "SPECIFYIGN:$ROOT/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_io/verilog/sg13g2_io.v:: Ignoring unsupported: specify block construct", + "signature": "faa191e2dbf4a242" + } + ], + "profile": "picorv32-rv32im-ihp130", + "schema_version": 1, + "tool": "verilator" +} diff --git a/requirements/build.txt b/requirements/build.txt new file mode 100644 index 00000000..e6c6da45 --- /dev/null +++ b/requirements/build.txt @@ -0,0 +1,5 @@ +# Ubuntu 22.04/Python 3.10 build dependency lock. +--only-binary=:all: +--require-hashes + +tomli==2.4.1 --hash=sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe diff --git a/requirements/ci.txt b/requirements/ci.txt new file mode 100644 index 00000000..6b466585 --- /dev/null +++ b/requirements/ci.txt @@ -0,0 +1,15 @@ +# Ubuntu 22.04/Python 3.10 CI dependency lock, including transitive dependencies. +--only-binary=:all: +--require-hashes + +exceptiongroup==1.3.1 --hash=sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598 +iniconfig==2.3.0 --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 +packaging==26.2 --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e +pathspec==1.1.1 --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 +pluggy==1.6.0 --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 +pytest==8.3.5 --hash=sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820 +PyYAML==6.0.3 --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b +ruff==0.11.0 --hash=sha256:92c0c1ff014351c0b0cdfdb1e35fa83b780f1e065667167bb9502d47ca41e6db +tomli==2.4.1 --hash=sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe +typing_extensions==4.16.0 --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +yamllint==1.35.1 --hash=sha256:2e16e504bb129ff515b37823b472750b36b6de07963bd74b307341ef5ad8bdc3 diff --git a/rtl/clusterip/setup.py b/rtl/clusterip/setup.py index 87aa042e..239d82f9 100644 --- a/rtl/clusterip/setup.py +++ b/rtl/clusterip/setup.py @@ -7,38 +7,22 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import load_lock # noqa: E402 from scripts.setup_helpers import ensure_git_repo # noqa: E402 -REVISIONS = { - "archinfo": "d65dd13c83a1176668bf5178004814a1551bf6ba", - "clint": "ce49f1a5501c0a9f18c0e3f358417648f3f2fb18", - "common": "d9921eb05160be873aebc46bb2f820bee2c7eebc", - "crc": "4654a63d06d487b5b9fe22af228c36c736b90bad", - "i2c": "73b024547c5cd70fb8bf801cc7757db2536c604f", - "i2s": "bd0978e8b45e4cf4c1335a1fa7311385d2a4e805", - "plic": "c851d93829d2e75b224003a058e718925855c0cc", - "ps2": "60921e0f72d6113f8615d03ce54567388fc2410f", - "pwm": "0777ef085c661d2b103f2fa0f8519f97eea3c950", - "rng": "7374e1e7f25bfd1ff9954ea99f19b77a583def29", - "rtc": "69963b8424cb00e04c8be70157b8ca738464e342", - "spi": "515cb408806584525605b4747db5e890dbdf1928", - "timer": "0c44b0d8b742c6a477fda15a812cf22962b54dd8", - "uart": "eca12213ba06f42e0a6dd0f27907fd4c447e2774", - "wdg": "b2603c1faeba96603392f0fe9e692e645d7e061f", -} - - def main() -> int: parser = argparse.ArgumentParser(description="Install pinned cluster IP repositories") parser.add_argument("--update", action="store_true") args = parser.parse_args() - destination = Path(__file__).resolve().parent - for name, revision in REVISIONS.items(): + dependencies = load_lock()["sources"] + for name in sorted(item for item in dependencies if item.startswith("cluster_")): + dependency = dependencies[name] ensure_git_repo( - f"https://github.com/retroSoC/{name}.git", - destination / name, - revision, + dependency["url"], + ROOT / dependency["destination"], + dependency["revision"], + recursive=dependency.get("recursive", False), update=args.update, ) return 0 diff --git a/rtl/ip/setup.py b/rtl/ip/setup.py index 6c2756c9..ab611dc8 100644 --- a/rtl/ip/setup.py +++ b/rtl/ip/setup.py @@ -7,21 +7,18 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import source # noqa: E402 from scripts.setup_helpers import ensure_git_repo # noqa: E402 -REVISION = "d991ec9f7bf696e7e5d840758487068b11bd4121" - - def main() -> int: parser = argparse.ArgumentParser(description="Install pinned third-party IP") parser.add_argument("--update", action="store_true") args = parser.parse_args() + dependency = source("third_party_ip") ensure_git_repo( - "https://github.com/retroSoC/3rd-party.git", - Path(__file__).resolve().parent / "3rd-party", - REVISION, - update=args.update, + dependency["url"], ROOT / dependency["destination"], dependency["revision"], + recursive=dependency.get("recursive", False), update=args.update, ) return 0 diff --git a/rtl/mini/Makefile b/rtl/mini/Makefile index cb20bfa1..2d17d409 100644 --- a/rtl/mini/Makefile +++ b/rtl/mini/Makefile @@ -1,73 +1,76 @@ -GENERATED_FL_DIR := $(RTL_PATH)/.generated_fl -MPW_STAMP := $(RTL_PATH)/mpw/.build/.prepared +GENERATED_FL_DIR := $(FLOW_FILELIST_DIR) +MPW_STAMP := $(CACHE_ROOT)/stamps/mpw-$(LOCK_DIGEST).prepared +MPW_OUTPUT_DIR := $(VARIANT_ROOT)/generated/mpw/$(SIM_TOOL_NAME) +MPW_VARIANT_STAMP := $(MPW_OUTPUT_DIR)/.stamp +FILELIST_STAMP := $(GENERATED_FL_DIR)/.stamp +FILELIST_TEMPLATES := $(sort $(wildcard $(ROOT_PATH)/rtl/filelist/pdk_*.fl) \ + $(wildcard $(RTL_PATH)/filelist/*.fl) $(RTL_PATH)/lint.msg) $(MPW_STAMP): @test -d $(RTL_PATH)/mpw/.git || { \ echo "missing $(RTL_PATH)/mpw; run 'make setup-mpw' first" >&2; \ exit 1; \ } - python3 $(ROOT_PATH)/scripts/prepare_mpw.py + python3 $(ROOT_PATH)/scripts/prepare_mpw.py \ + --lock-file $(CACHE_ROOT)/locks/mpw-prepare.lock @mkdir -p $(@D) @touch $@ -gen_mpw_code: $(MPW_STAMP) -ifeq ($(IP), MDD) - @echo "[mpw] generating user ip ..." - python3 $(RTL_PATH)/mpw/ip.py - python3 $(RTL_PATH)/mpw/info.py IP -else - @echo "[mpw] skipping user ip generation" -endif -ifeq ($(CORE), MDD) - @echo "[mpw] generating core ..." - python3 $(RTL_PATH)/mpw/core.py $(SIMU) - python3 $(RTL_PATH)/mpw/info.py CORE -else - @echo "[mpw] skipping core generation" -endif +$(MPW_VARIANT_STAMP): $(MPW_STAMP) $(ROOT_PATH)/scripts/generate_mpw.py + python3 $(ROOT_PATH)/scripts/generate_mpw.py --root $(ROOT_PATH) \ + --output $(MPW_OUTPUT_DIR) --simu $(SIMU) --core $(CORE) --ip $(IP) \ + --lock-file $(CACHE_ROOT)/locks/mpw.lock + @mkdir -p $(@D) + @touch $@ -generate_filelist: +gen_mpw_code: $(MPW_VARIANT_STAMP) + +$(FILELIST_STAMP): $(FILELIST_TEMPLATES) $(RTL_PATH)/script/generate_filelist.py \ + $(RTL_PATH)/script/filelist.py python3 $(RTL_PATH)/script/generate_filelist.py \ --output-dir $(GENERATED_FL_DIR) \ $(foreach define,$(DEF_LIST),--define $(define)) + @touch $@ + +generate_filelist: $(FILELIST_STAMP) # software include $(RTL_PATH)/mk/software.mk #====================== filelist ====================== ifeq ($(PDK), IHP130) - NET_FLIST := -f $(RTL_PATH)/.generated_fl/pdk_ihp130.fl + NET_FLIST := -f $(GENERATED_FL_DIR)/pdk_ihp130.fl else ifeq ($(PDK), SKY130) - NET_FLIST := -f $(RTL_PATH)/.generated_fl/pdk_sky130.fl + NET_FLIST := -f $(GENERATED_FL_DIR)/pdk_sky130.fl else ifeq ($(PDK), ICS55) - NET_FLIST := -f $(RTL_PATH)/.generated_fl/pdk_ics55.fl + NET_FLIST := -f $(GENERATED_FL_DIR)/pdk_ics55.fl else ifeq ($(PDK), GF180) - NET_FLIST := -f $(RTL_PATH)/.generated_fl/pdk_gf180.fl + NET_FLIST := -f $(GENERATED_FL_DIR)/pdk_gf180.fl endif -NET_FLIST += -f $(RTL_PATH)/.generated_fl/def.fl -NET_FLIST += -f $(RTL_PATH)/.generated_fl/sys_def.fl -NET_FLIST += -f $(RTL_PATH)/.generated_fl/commonip.fl -NET_FLIST += -f $(RTL_PATH)/.generated_fl/inc.fl +NET_FLIST += -f $(GENERATED_FL_DIR)/def.fl +NET_FLIST += -f $(GENERATED_FL_DIR)/sys_def.fl +NET_FLIST += -f $(GENERATED_FL_DIR)/commonip.fl +NET_FLIST += -f $(GENERATED_FL_DIR)/inc.fl RTL_FLIST := $(NET_FLIST) -RTL_FLIST += -f $(RTL_PATH)/.generated_fl/clusterip.fl -RTL_FLIST += -f $(RTL_PATH)/.generated_fl/ip.fl -RTL_FLIST += -f $(RTL_PATH)/.generated_fl/tech.fl +RTL_FLIST += -f $(GENERATED_FL_DIR)/clusterip.fl +RTL_FLIST += -f $(GENERATED_FL_DIR)/ip.fl +RTL_FLIST += -f $(GENERATED_FL_DIR)/tech.fl ifeq ($(CORE), PICORV32) - RTL_FLIST += -f $(RTL_PATH)/.generated_fl/core_picorv32.fl + RTL_FLIST += -f $(GENERATED_FL_DIR)/core_picorv32.fl else ifeq ($(CORE), HAZARD3) - RTL_FLIST += -f $(RTL_PATH)/.generated_fl/core_hazard3.fl + RTL_FLIST += -f $(GENERATED_FL_DIR)/core_hazard3.fl else ifeq ($(CORE), MDD) - RTL_FLIST += -f $(RTL_PATH)/mpw/.build/core/core.fl + RTL_FLIST += -f $(MPW_OUTPUT_DIR)/core/core.fl endif ifeq ($(IP), MDD) - RTL_FLIST += -f $(RTL_PATH)/mpw/.build/ip/ip.fl + RTL_FLIST += -f $(MPW_OUTPUT_DIR)/ip/ip.fl endif -RTL_FLIST += -f $(RTL_PATH)/.generated_fl/top.fl +RTL_FLIST += -f $(GENERATED_FL_DIR)/top.fl #====================== end ====================== ifeq ($(SIMU), VCS) diff --git a/rtl/mini/mk/iverilog.mk b/rtl/mini/mk/iverilog.mk index bee09aae..4b744c2b 100644 --- a/rtl/mini/mk/iverilog.mk +++ b/rtl/mini/mk/iverilog.mk @@ -2,12 +2,12 @@ IVERILOG ?= iverilog VVP ?= vvp GTKWAVE ?= gtkwave -NETLIST_PATH ?= $(ROOT_PATH)/syn/yosys/.synth_build/out/retrosoc_asic_yosys.v +NETLIST_PATH ?= $(SYN_BUILD_ROOT)/out/retrosoc_asic_yosys.v POST_PATH ?= $(ROOT_PATH)/pd/sdf/retrosoc_asic.v SDF_PATH ?= $(ROOT_PATH)/pd/sdf/retrosoc_asic_CTS_MIN.sdf.gz SDF_SCOPE ?= $(RTL_TOP).u_retrosoc_asic -IVERILOG_ROOT := $(RTL_PATH)/.iverilog_build +IVERILOG_ROOT := $(SIM_BUILD_ROOT) IVERILOG_BEHV_DIR := $(IVERILOG_ROOT)/behv IVERILOG_NETL_DIR := $(IVERILOG_ROOT)/netl IVERILOG_POST_DIR := $(IVERILOG_ROOT)/post @@ -15,6 +15,15 @@ IVERILOG_BEHV_FLIST := $(IVERILOG_BEHV_DIR)/iverilog.fl IVERILOG_NETL_FLIST := $(IVERILOG_NETL_DIR)/iverilog.fl IVERILOG_POST_FLIST := $(IVERILOG_POST_DIR)/iverilog.fl CONVERTED_SOC := $(IVERILOG_BEHV_DIR)/converted_soc.v +CONVERTED_DEPFILE := $(IVERILOG_BEHV_DIR)/converted_soc.d +IVERILOG_BEHV_SIMV := $(IVERILOG_BEHV_DIR)/simv +IVERILOG_NETL_SIMV := $(IVERILOG_NETL_DIR)/simv +IVERILOG_POST_SIMV := $(IVERILOG_POST_DIR)/simv +IVERILOG_BEHV_DEPFILE := $(IVERILOG_BEHV_DIR)/simv.d +IVERILOG_NETL_DEPFILE := $(IVERILOG_NETL_DIR)/simv.d +IVERILOG_POST_DEPFILE := $(IVERILOG_POST_DIR)/simv.d + +-include $(CONVERTED_DEPFILE) $(IVERILOG_BEHV_DEPFILE) $(IVERILOG_NETL_DEPFILE) $(IVERILOG_POST_DEPFILE) IVERILOG_COMMON_OPTS := -g2012 IVERILOG_TIME_OPTS := -gno-specify @@ -22,64 +31,93 @@ IVERILOG_POST_OPTS := -gspecify -Tmin IVERILOG_SIM_OPTS := +$(RTL_SIM_PLLEN) +$(RTL_SIM_PLLCFG) \ +core_sel=$(RTL_SIM_CORESEL) +sim_timeout=$(RTL_SIM_TIMEOUT) +wave_$(WAVE) -$(CONVERTED_SOC): gen_mpw_code generate_filelist +$(CONVERTED_SOC): $(MPW_VARIANT_STAMP) $(FILELIST_STAMP) @mkdir -p $(@D) python3 $(RTL_PATH)/script/convt_sv2v.py $(RTL_FLIST) --output $@ + python3 $(RTL_PATH)/script/filelist_deps.py $(RTL_FLIST) --target $@ \ + --output $(CONVERTED_DEPFILE) -$(IVERILOG_BEHV_FLIST): generate_filelist $(CONVERTED_SOC) +$(IVERILOG_BEHV_FLIST): $(FILELIST_STAMP) $(CONVERTED_SOC) @mkdir -p $(@D) python3 $(RTL_PATH)/script/gen_iverilog_filelist.py \ - --mode behv --pdk $(PDK) --output $@ --converted $(CONVERTED_SOC) + --mode behv --pdk $(PDK) --generated-dir $(GENERATED_FL_DIR) \ + --output $@ --converted $(CONVERTED_SOC) -$(IVERILOG_NETL_FLIST): generate_filelist +$(IVERILOG_NETL_FLIST): $(FILELIST_STAMP) $(NETLIST_PATH) @mkdir -p $(@D) python3 $(RTL_PATH)/script/gen_iverilog_filelist.py \ - --mode netl --pdk $(PDK) --output $@ --netlist $(NETLIST_PATH) + --mode netl --pdk $(PDK) --generated-dir $(GENERATED_FL_DIR) \ + --output $@ --netlist $(NETLIST_PATH) -$(IVERILOG_POST_FLIST): generate_filelist +$(IVERILOG_POST_FLIST): $(FILELIST_STAMP) $(POST_PATH) $(SDF_PATH) @mkdir -p $(@D) python3 $(RTL_PATH)/script/gen_iverilog_filelist.py \ - --mode post --pdk $(PDK) --output $@ --netlist $(POST_PATH) --sdf $(SDF_PATH) \ + --mode post --pdk $(PDK) --generated-dir $(GENERATED_FL_DIR) \ + --output $@ --netlist $(POST_PATH) --sdf $(SDF_PATH) \ --sdf-scope $(SDF_SCOPE) convt_sv2v: $(CONVERTED_SOC) gen_iverilog_filelist: $(IVERILOG_BEHV_FLIST) -comp: $(IVERILOG_BEHV_FLIST) +$(IVERILOG_BEHV_SIMV): $(IVERILOG_BEHV_FLIST) @mkdir -p $(IVERILOG_BEHV_DIR) - @cd $(IVERILOG_BEHV_DIR) && set -o pipefail && \ - $(IVERILOG) $(IVERILOG_COMMON_OPTS) $(IVERILOG_TIME_OPTS) -f $< \ - -o simv -s $(RTL_TOP) 2>&1 | tee compile.log + python3 $(ROOT_PATH)/scripts/run_flow.py --tool iverilog \ + --log $(IVERILOG_BEHV_DIR)/compile.log --result $(IVERILOG_BEHV_DIR)/result-compile.json \ + --cwd $(IVERILOG_BEHV_DIR) -- $(IVERILOG) $(IVERILOG_COMMON_OPTS) \ + $(IVERILOG_TIME_OPTS) -f $< -o simv -s $(RTL_TOP) + python3 $(RTL_PATH)/script/filelist_deps.py -f $(IVERILOG_BEHV_FLIST) \ + --target $@ --output $(IVERILOG_BEHV_DEPFILE) -netcomp: $(IVERILOG_NETL_FLIST) +$(IVERILOG_NETL_SIMV): $(IVERILOG_NETL_FLIST) @mkdir -p $(IVERILOG_NETL_DIR) - @cd $(IVERILOG_NETL_DIR) && set -o pipefail && \ - $(IVERILOG) $(IVERILOG_COMMON_OPTS) $(IVERILOG_TIME_OPTS) -f $< \ - -o simv -s $(RTL_TOP) 2>&1 | tee compile.log + python3 $(ROOT_PATH)/scripts/run_flow.py --tool iverilog \ + --log $(IVERILOG_NETL_DIR)/compile.log --result $(IVERILOG_NETL_DIR)/result-compile.json \ + --cwd $(IVERILOG_NETL_DIR) -- $(IVERILOG) $(IVERILOG_COMMON_OPTS) \ + $(IVERILOG_TIME_OPTS) -f $< -o simv -s $(RTL_TOP) + python3 $(RTL_PATH)/script/filelist_deps.py -f $(IVERILOG_NETL_FLIST) \ + --target $@ --output $(IVERILOG_NETL_DEPFILE) -postcomp: $(IVERILOG_POST_FLIST) +$(IVERILOG_POST_SIMV): $(IVERILOG_POST_FLIST) @mkdir -p $(IVERILOG_POST_DIR) - @cd $(IVERILOG_POST_DIR) && set -o pipefail && \ - $(IVERILOG) $(IVERILOG_COMMON_OPTS) $(IVERILOG_POST_OPTS) -f $< \ - -o simv -s $(RTL_TOP) -s retrosoc_sdf_annotator 2>&1 | tee compile.log + python3 $(ROOT_PATH)/scripts/run_flow.py --tool iverilog \ + --log $(IVERILOG_POST_DIR)/compile.log --result $(IVERILOG_POST_DIR)/result-compile.json \ + --cwd $(IVERILOG_POST_DIR) -- $(IVERILOG) $(IVERILOG_COMMON_OPTS) \ + $(IVERILOG_POST_OPTS) -f $< -o simv -s $(RTL_TOP) -s retrosoc_sdf_annotator + python3 $(RTL_PATH)/script/filelist_deps.py -f $(IVERILOG_POST_FLIST) \ + --target $@ --output $(IVERILOG_POST_DEPFILE) + +comp: $(IVERILOG_BEHV_SIMV) +netcomp: $(IVERILOG_NETL_SIMV) +postcomp: $(IVERILOG_POST_SIMV) sim: comp python3 $(RTL_PATH)/script/prepare_norflash.py --sim-dir $(IVERILOG_BEHV_DIR) \ - --firmware $(ROOT_PATH)/.sw_build/$(FIRMWARE_NAME).hex - @cd $(IVERILOG_BEHV_DIR) && set -o pipefail && \ - stdbuf -oL -eL $(VVP) simv -fst $(IVERILOG_SIM_OPTS) 2>&1 | tee sim.log + --firmware $(SW_BUILD_DIR)/$(SIM_FIRMWARE_NAME).hex + python3 $(ROOT_PATH)/scripts/run_flow.py --tool iverilog-sim \ + --log $(IVERILOG_BEHV_DIR)/sim.log --result $(IVERILOG_BEHV_DIR)/result-sim.json \ + --cwd $(IVERILOG_BEHV_DIR) -- stdbuf -oL -eL $(VVP) simv -fst $(IVERILOG_SIM_OPTS) + python3 $(ROOT_PATH)/scripts/check_simulation.py --log $(IVERILOG_BEHV_DIR)/sim.log \ + --result $(IVERILOG_BEHV_DIR)/result-sim-check.json --require '$(SIM_SUCCESS_MARKER)' netsim: netcomp python3 $(RTL_PATH)/script/prepare_norflash.py --sim-dir $(IVERILOG_NETL_DIR) \ - --firmware $(ROOT_PATH)/.sw_build/$(FIRMWARE_NAME).hex - @cd $(IVERILOG_NETL_DIR) && set -o pipefail && \ - stdbuf -oL -eL $(VVP) simv -fst $(IVERILOG_SIM_OPTS) 2>&1 | tee sim.log + --firmware $(SW_BUILD_DIR)/$(SIM_FIRMWARE_NAME).hex + python3 $(ROOT_PATH)/scripts/run_flow.py --tool iverilog-sim \ + --log $(IVERILOG_NETL_DIR)/sim.log --result $(IVERILOG_NETL_DIR)/result-sim.json \ + --cwd $(IVERILOG_NETL_DIR) -- stdbuf -oL -eL $(VVP) simv -fst $(IVERILOG_SIM_OPTS) + python3 $(ROOT_PATH)/scripts/check_simulation.py --log $(IVERILOG_NETL_DIR)/sim.log \ + --result $(IVERILOG_NETL_DIR)/result-sim-check.json --require '$(SIM_SUCCESS_MARKER)' postsim: postcomp python3 $(RTL_PATH)/script/prepare_norflash.py --sim-dir $(IVERILOG_POST_DIR) \ - --firmware $(ROOT_PATH)/.sw_build/$(FIRMWARE_NAME).hex - @cd $(IVERILOG_POST_DIR) && set -o pipefail && \ - stdbuf -oL -eL $(VVP) simv -fst $(IVERILOG_SIM_OPTS) 2>&1 | tee sim.log + --firmware $(SW_BUILD_DIR)/$(SIM_FIRMWARE_NAME).hex + python3 $(ROOT_PATH)/scripts/run_flow.py --tool iverilog-sim \ + --log $(IVERILOG_POST_DIR)/sim.log --result $(IVERILOG_POST_DIR)/result-sim.json \ + --cwd $(IVERILOG_POST_DIR) -- stdbuf -oL -eL $(VVP) simv -fst $(IVERILOG_SIM_OPTS) + python3 $(ROOT_PATH)/scripts/check_simulation.py --log $(IVERILOG_POST_DIR)/sim.log \ + --result $(IVERILOG_POST_DIR)/result-sim-check.json --require '$(SIM_SUCCESS_MARKER)' + +comp netcomp postcomp sim netsim postsim: | manifest wave: cd $(IVERILOG_BEHV_DIR) && $(GTKWAVE) $(RTL_TOP).fst & @@ -89,6 +127,6 @@ postwave: cd $(IVERILOG_POST_DIR) && $(GTKWAVE) $(RTL_TOP).fst & clean: - rm -rf $(IVERILOG_ROOT) + python3 $(ROOT_PATH)/scripts/clean.py --root $(ROOT_PATH) --path $(IVERILOG_ROOT) .PHONY: convt_sv2v gen_iverilog_filelist comp netcomp postcomp sim netsim postsim wave netwave postwave clean diff --git a/rtl/mini/mk/software.mk b/rtl/mini/mk/software.mk index 938007dc..a01804d4 100644 --- a/rtl/mini/mk/software.mk +++ b/rtl/mini/mk/software.mk @@ -103,12 +103,13 @@ APP_PATH := $(ROOT_PATH)/app/base/src/at24cxx.c \ $(ROOT_PATH)/app/base/src/donut.c endif -INC_PATH := -I$(ROOT_PATH)/crt/inc \ +INC_PATH := -I$(SW_BUILD_DIR)/include \ + -I$(ROOT_PATH)/crt/inc \ -I$(ROOT_PATH)/app/base/inc ifneq ($(findstring MDD,$(CORE) $(IP)),) -INC_PATH += -I$(ROOT_PATH)/rtl/mini/mpw/.build +INC_PATH += -I$(MPW_OUTPUT_DIR) endif @@ -130,22 +131,39 @@ ifneq ($(filter RV32E RV32I,$(ISA)),) endif LDS_PATH := $(ROOT_PATH)/crt/lds/$(LINK_TYPE).lds +VERSION_HEADER := $(SW_BUILD_DIR)/include/socver.h +FIRMWARE_ELF := $(SW_BUILD_DIR)/firmware +ASM_FIRMWARE_NAME ?= retrosoc_asm +SW_HEADERS := $(shell find $(ROOT_PATH)/crt/inc $(ROOT_PATH)/app -type f \ + \( -name '*.h' -o -name '*.hpp' \) 2>/dev/null) -upd_ver_info: - python3 $(ROOT_PATH)/crt/ver.py --root $(ROOT_PATH) +$(VERSION_HEADER): FORCE_VERSION $(ROOT_PATH)/crt/ver.py $(ROOT_PATH)/crt/ver.tmpl + python3 $(ROOT_PATH)/crt/ver.py --root $(ROOT_PATH) \ + --output $@ -asm: gen_mpw_code - @mkdir -p $(ROOT_PATH)/.sw_build - $(MAKE) -C $(ROOT_PATH)/app/asm - cp $(ROOT_PATH)/app/asm/hello-asm.flash $(ROOT_PATH)/.sw_build/$(FIRMWARE_NAME).hex - cp $(ROOT_PATH)/app/asm/hello-asm.bin $(ROOT_PATH)/.sw_build/$(FIRMWARE_NAME).bin - cp $(ROOT_PATH)/app/asm/hello-asm.txt $(ROOT_PATH)/.sw_build/$(FIRMWARE_NAME)_all.txt +FORCE_VERSION: -firmware: gen_mpw_code upd_ver_info - @mkdir -p $(ROOT_PATH)/.sw_build - cd $(ROOT_PATH)/.sw_build && $(CP) -P -o $(LINK_TYPE).lds $(LDS_PATH) - cd $(ROOT_PATH)/.sw_build && $(CC) $(CFLAGS) $(INC_PATH) -o $@ $(SRC_PATH) - cd $(ROOT_PATH)/.sw_build && $(OBJC) -O verilog $@ $(FIRMWARE_NAME).hex - cd $(ROOT_PATH)/.sw_build && $(OBJC) -O binary $@ $(FIRMWARE_NAME).bin - cd $(ROOT_PATH)/.sw_build && $(DUMP) -d $@ > $(FIRMWARE_NAME).txt - cd $(ROOT_PATH)/.sw_build && $(DUMP) -D $@ > $(FIRMWARE_NAME)_all.txt +upd_ver_info: $(VERSION_HEADER) + +asm: $(MPW_VARIANT_STAMP) + @mkdir -p $(SW_BUILD_DIR)/asm + $(MAKE) -C $(ROOT_PATH)/app/asm OUT_DIR=$(SW_BUILD_DIR)/asm + cp $(SW_BUILD_DIR)/asm/hello-asm.flash $(SW_BUILD_DIR)/$(ASM_FIRMWARE_NAME).hex + cp $(SW_BUILD_DIR)/asm/hello-asm.bin $(SW_BUILD_DIR)/$(ASM_FIRMWARE_NAME).bin + cp $(SW_BUILD_DIR)/asm/hello-asm.txt $(SW_BUILD_DIR)/$(ASM_FIRMWARE_NAME)_all.txt + +$(FIRMWARE_ELF): $(MPW_VARIANT_STAMP) $(VERSION_HEADER) $(SRC_PATH) $(SW_HEADERS) $(LDS_PATH) \ + $(ROOT_PATH)/rtl/mini/mk/software.mk + @mkdir -p $(SW_BUILD_DIR) + cd $(SW_BUILD_DIR) && $(CP) -P -o $(LINK_TYPE).lds $(LDS_PATH) + cd $(SW_BUILD_DIR) && $(CC) $(CFLAGS) $(INC_PATH) -o $(@F) $(SRC_PATH) + cd $(SW_BUILD_DIR) && $(OBJC) -O verilog $(@F) $(FIRMWARE_NAME).hex + cd $(SW_BUILD_DIR) && $(OBJC) -O binary $(@F) $(FIRMWARE_NAME).bin + cd $(SW_BUILD_DIR) && $(DUMP) -d $(@F) > $(FIRMWARE_NAME).txt + cd $(SW_BUILD_DIR) && $(DUMP) -D $(@F) > $(FIRMWARE_NAME)_all.txt + +firmware: $(FIRMWARE_ELF) + +firmware asm: | manifest + +.PHONY: FORCE_VERSION diff --git a/rtl/mini/mk/vcs.mk b/rtl/mini/mk/vcs.mk index c66b466d..b505e5df 100644 --- a/rtl/mini/mk/vcs.mk +++ b/rtl/mini/mk/vcs.mk @@ -10,11 +10,11 @@ VERDI_TOOL := $(VCS_RUNNER) $(VERDI) COMP_LOG := -l compile.log SIM_LOG := -l sim.log # netlist file path -NETLIST_PATH := -v $(ROOT_PATH)/syn/yosys/.synth_build/out/retrosoc_asic_yosys.v +NETLIST_PATH := -v $(SYN_BUILD_ROOT)/out/retrosoc_asic_yosys.v POST_PATH := -v $(ROOT_PATH)/pd/sdf/retrosoc_asic.v SDF_PATH := "$(ROOT_PATH)/pd/sdf/retrosoc_asic_CTS_MIN.sdf.gz" # testbench filelist -TB_FLIST := -f $(RTL_PATH)/.generated_fl/tb.fl +TB_FLIST := -f $(GENERATED_FL_DIR)/tb.fl ## vcs option # -debug_region=cell+lib @@ -27,7 +27,7 @@ COMMON_OPTS := -full64 +v2k -sverilog -timescale=1ns/10ps \ $(EXTRA) \ -kdb \ -debug_access+all \ - -msg_config=$(RTL_PATH)/.generated_fl/lint.msg \ + -msg_config=$(GENERATED_FL_DIR)/lint.msg \ +error+500 \ +vcs+flush+all \ -xprop=$(RTL_PATH)/xprop_config \ @@ -49,45 +49,77 @@ SIM_OPTS := +vcs+loopreport+1000 -suppress=ASLR_DETECTED_INFO \ +$(RTL_SIM_PLLEN) +$(RTL_SIM_PLLCFG) +core_sel=$(RTL_SIM_CORESEL) +sim_timeout=$(RTL_SIM_TIMEOUT) \ +wave_$(WAVE) +VCS_BEHV_DIR := $(SIM_BUILD_ROOT)/behv +VCS_NETL_DIR := $(SIM_BUILD_ROOT)/netl +VCS_POST_DIR := $(SIM_BUILD_ROOT)/post +VCS_BEHV_SIMV := $(VCS_BEHV_DIR)/simv +VCS_NETL_SIMV := $(VCS_NETL_DIR)/simv +VCS_POST_SIMV := $(VCS_POST_DIR)/simv +VCS_BEHV_DEPFILE := $(VCS_BEHV_DIR)/simv.d +VCS_NETL_DEPFILE := $(VCS_NETL_DIR)/simv.d +VCS_POST_DEPFILE := $(VCS_POST_DIR)/simv.d -comp: DIR := .vcs_build/behv -comp: FLIST := $(RTL_FLIST) $(TB_FLIST) -comp: OPTS := $(TIME_OPTION) +-include $(VCS_BEHV_DEPFILE) $(VCS_NETL_DEPFILE) $(VCS_POST_DEPFILE) -netcomp: DIR := .vcs_build/netl -netcomp: FLIST := $(NETLIST_PATH) $(NET_FLIST) $(TB_FLIST) -netcomp: OPTS := $(TIME_OPTION) +$(VCS_BEHV_SIMV): DIR := $(VCS_BEHV_DIR) +$(VCS_BEHV_SIMV): FLIST := $(RTL_FLIST) $(TB_FLIST) +$(VCS_BEHV_SIMV): OPTS := $(TIME_OPTION) +$(VCS_BEHV_SIMV): DEPFILE := $(VCS_BEHV_DEPFILE) +$(VCS_BEHV_SIMV): DEPS_ARGS := $(RTL_FLIST) -f $(GENERATED_FL_DIR)/tb.fl -postcomp: DIR := .vcs_build/post -postcomp: FLIST := $(POST_PATH) $(NET_FLIST) $(TB_FLIST) -postcomp: OPTS := $(POST_OPTS) +$(VCS_NETL_SIMV): DIR := $(VCS_NETL_DIR) +$(VCS_NETL_SIMV): FLIST := $(NETLIST_PATH) $(NET_FLIST) $(TB_FLIST) +$(VCS_NETL_SIMV): OPTS := $(TIME_OPTION) +$(VCS_NETL_SIMV): DEPFILE := $(VCS_NETL_DEPFILE) +$(VCS_NETL_SIMV): DEPS_ARGS := $(NET_FLIST) -f $(GENERATED_FL_DIR)/tb.fl --extra $(SYN_BUILD_ROOT)/out/retrosoc_asic_yosys.v -sim: DIR := .vcs_build/behv -netsim: DIR := .vcs_build/netl -postsim: DIR := .vcs_build/post +$(VCS_POST_SIMV): DIR := $(VCS_POST_DIR) +$(VCS_POST_SIMV): FLIST := $(POST_PATH) $(NET_FLIST) $(TB_FLIST) +$(VCS_POST_SIMV): OPTS := $(POST_OPTS) +$(VCS_POST_SIMV): DEPFILE := $(VCS_POST_DEPFILE) +$(VCS_POST_SIMV): DEPS_ARGS := $(NET_FLIST) -f $(GENERATED_FL_DIR)/tb.fl --extra $(ROOT_PATH)/pd/sdf/retrosoc_asic.v --extra $(ROOT_PATH)/pd/sdf/retrosoc_asic_CTS_MIN.sdf.gz -wave: DIR := .vcs_build/behv -netwave: DIR := .vcs_build/netl -postwave: DIR := .vcs_build/post +sim: DIR := $(VCS_BEHV_DIR) +netsim: DIR := $(VCS_NETL_DIR) +postsim: DIR := $(VCS_POST_DIR) + +wave: DIR := $(SIM_BUILD_ROOT)/behv +netwave: DIR := $(SIM_BUILD_ROOT)/netl +postwave: DIR := $(SIM_BUILD_ROOT)/post -sim: comp -netsim: netcomp -postsim: postcomp +comp: $(VCS_BEHV_SIMV) +netcomp: $(VCS_NETL_SIMV) +postcomp: $(VCS_POST_SIMV) +sim: $(VCS_BEHV_SIMV) +netsim: $(VCS_NETL_SIMV) +postsim: $(VCS_POST_SIMV) -comp netcomp postcomp: gen_mpw_code generate_filelist +$(VCS_BEHV_SIMV) $(VCS_NETL_SIMV) $(VCS_POST_SIMV): $(MPW_VARIANT_STAMP) $(FILELIST_STAMP) + @mkdir -p $(DIR) + python3 $(ROOT_PATH)/scripts/run_flow.py --tool vcs --log $(DIR)/driver-compile.log \ + --result $(DIR)/result-compile.json --cwd $(DIR) -- \ + $(SIM_TOOL) $(COMMON_OPTS) $(OPTS) $(FLIST) -top $(RTL_TOP) $(COMP_LOG) + python3 $(RTL_PATH)/script/filelist_deps.py $(DEPS_ARGS) --target $@ --output $(DEPFILE) -comp netcomp postcomp: - @mkdir -p $(RTL_PATH)/$(DIR) - cd $(RTL_PATH)/$(DIR) && $(SIM_TOOL) $(COMMON_OPTS) $(OPTS) $(FLIST) -top $(RTL_TOP) $(COMP_LOG) +$(VCS_NETL_SIMV): $(SYN_BUILD_ROOT)/out/retrosoc_asic_yosys.v +$(VCS_POST_SIMV): $(ROOT_PATH)/pd/sdf/retrosoc_asic.v $(ROOT_PATH)/pd/sdf/retrosoc_asic_CTS_MIN.sdf.gz sim netsim postsim: - cd $(RTL_PATH)/$(DIR) && $(SIM_BINY) $(SIM_OPTS) $(if $(filter netsim postsim,$@),+bus_conflict_off) $(SIM_LOG) + python3 $(RTL_PATH)/script/prepare_norflash.py --sim-dir $(DIR) \ + --firmware $(SW_BUILD_DIR)/$(SIM_FIRMWARE_NAME).hex + python3 $(ROOT_PATH)/scripts/run_flow.py --tool vcs-sim --log $(DIR)/driver-sim.log \ + --result $(DIR)/result-sim.json --cwd $(DIR) -- \ + $(SIM_BINY) $(SIM_OPTS) $(if $(filter netsim postsim,$@),+bus_conflict_off) $(SIM_LOG) + python3 $(ROOT_PATH)/scripts/check_simulation.py --log $(DIR)/driver-sim.log \ + --result $(DIR)/result-sim-check.json --require '$(SIM_SUCCESS_MARKER)' + +comp netcomp postcomp sim netsim postsim: | manifest wave netwave postwave: - cd $(RTL_PATH)/$(DIR) && ($(VERDI_TOOL) -ssf $(RTL_TOP).fsdb -nologo &) + cd $(DIR) && ($(VERDI_TOOL) -ssf $(RTL_TOP).fsdb -nologo &) clean: - rm -rf $(RTL_PATH)/.vcs_build + python3 $(ROOT_PATH)/scripts/clean.py --root $(ROOT_PATH) --path $(SIM_BUILD_ROOT) .PHONY: comp netcomp postcomp sim netsim postsim wave clean diff --git a/rtl/mini/mk/verilator.mk b/rtl/mini/mk/verilator.mk index e115c8e4..7b574e3e 100644 --- a/rtl/mini/mk/verilator.mk +++ b/rtl/mini/mk/verilator.mk @@ -1,4 +1,4 @@ -BUILD_DIR := $(RTL_PATH)/.verilator_build +BUILD_DIR := $(SIM_BUILD_ROOT) SOC_CSRC_HOME += $(RTL_PATH)/csrc SOC_CSRC_LIB_HOME += $(RTL_PATH)/csrc SOC_CXXFILES += $(sort $(wildcard $(SOC_CSRC_HOME)/*.cpp)) @@ -19,6 +19,7 @@ SOC_VXXFILES += $(RTL_PATH)/vsrc/retrosoc_top.sv SOC_VSRC_INCLPATH += -I$(SOC_VSRC_HOME) VERILATOR ?= verilator +VERILATOR_JOBS ?= $(JOBS) VERILATOR_CXXFLAGS += -std=c++17 -Wall $(SOC_CSRC_INCLPATH) -DDUMP_WAVE_FST VERILATOR_FLAGS += --cc --exe --no-timing --top-module $(SOC_VSRC_TOP) VERILATOR_FLAGS += --x-assign unique -O3 -CFLAGS "$(VERILATOR_CXXFLAGS)" @@ -29,32 +30,60 @@ VERILATOR_FLAGS += -Mdir $(SOC_COMPILE_HOME) VERILATOR_FLAGS += $(SOC_VSRC_INCLPATH) $(SOC_CXXFILES) $(SOC_VXXFILES) SOC_SIM_TIME ?= 40 +VERILATOR_STAMP := $(BUILD_DIR)/verilate.stamp +VERILATOR_DEPFILE := $(BUILD_DIR)/verilate.d +VERILATOR_EMU := $(BUILD_DIR)/emu +VERILATOR_EXTRA_SOURCES := $(RTL_PATH)/../ip/tb/ESP_PSRAM64H.sv \ + $(RTL_PATH)/vsrc/flash_read_binder.sv \ + $(RTL_PATH)/vsrc/QSPIFlash.sv \ + $(RTL_PATH)/vsrc/retrosoc_top.sv \ + $(SOC_CXXFILES) $(wildcard $(SOC_CSRC_HOME)/*.h) \ + $(wildcard $(SOC_CSRC_HOME)/*.hpp) + +-include $(VERILATOR_DEPFILE) CCACHE := $(shell command -v ccache 2>/dev/null) ifneq ($(CCACHE),) export OBJCACHE = ccache -export CCACHE_DIR = $(BUILD_DIR)/ccache -export CCACHE_TEMPDIR = $(BUILD_DIR)/ccache/tmp +export CCACHE_DIR = $(CACHE_ROOT)/ccache/verilator +export CCACHE_TEMPDIR = $(CACHE_ROOT)/ccache/verilator/tmp endif -lint: gen_mpw_code generate_filelist +$(VERILATOR_STAMP): $(MPW_VARIANT_STAMP) $(FILELIST_STAMP) $(VERILATOR_EXTRA_SOURCES) @mkdir -p $(BUILD_DIR) - @mkdir -p $(BUILD_DIR)/ccache/tmp + @mkdir -p $(CCACHE_TEMPDIR) + python3 $(ROOT_PATH)/scripts/run_flow.py --tool verilator \ + --log $(BUILD_DIR)/verilating.log --result $(BUILD_DIR)/result-verilate.json \ + -- $(VERILATOR) $(VERILATOR_FLAGS) + python3 $(RTL_PATH)/script/filelist_deps.py $(RTL_FLIST) \ + $(foreach source,$(VERILATOR_EXTRA_SOURCES),--extra $(source)) \ + --target $@ --output $(VERILATOR_DEPFILE) + @touch $@ -comp: lint - $(VERILATOR) $(VERILATOR_FLAGS) > $(BUILD_DIR)/verilating.log 2>&1 - +$(MAKE) VM_PARALLEL_BUILDS=1 OPT_FAST="-O3" -C $(SOC_COMPILE_HOME) \ - -f V$(SOC_VSRC_TOP).mk > $(BUILD_DIR)/compile.log 2>&1 +$(VERILATOR_EMU): $(VERILATOR_STAMP) + python3 $(ROOT_PATH)/scripts/run_flow.py --tool verilator-compile \ + --log $(BUILD_DIR)/compile.log --result $(BUILD_DIR)/result-compile.json \ + -- make -j$(VERILATOR_JOBS) VM_PARALLEL_BUILDS=1 OPT_FAST=-O3 -C $(SOC_COMPILE_HOME) \ + -f V$(SOC_VSRC_TOP).mk + +lint: $(VERILATOR_STAMP) +comp: $(VERILATOR_EMU) sim: comp - @test -f $(ROOT_PATH)/.sw_build/$(FIRMWARE_NAME).bin || { \ + @test -f $(SW_BUILD_DIR)/$(SIM_FIRMWARE_NAME).bin || { \ echo "firmware image missing; run 'make firmware' first" >&2; exit 1; } - $(BUILD_DIR)/emu -i $(ROOT_PATH)/.sw_build/$(FIRMWARE_NAME).bin \ + python3 $(ROOT_PATH)/scripts/run_flow.py --tool verilator-sim \ + --log $(BUILD_DIR)/sim.log --result $(BUILD_DIR)/result-sim.json \ + --cwd $(BUILD_DIR) -- $(BUILD_DIR)/emu -i $(SW_BUILD_DIR)/$(SIM_FIRMWARE_NAME).bin \ -s $(RTL_SIM_CORESEL) -t $(SOC_SIM_TIME) + python3 $(ROOT_PATH)/scripts/check_simulation.py --log $(BUILD_DIR)/sim.log \ + --result $(BUILD_DIR)/result-sim-check.json --require '$(SIM_SUCCESS_MARKER)' + +comp sim: | manifest wave: clean: - rm -rf $(BUILD_DIR) + python3 $(ROOT_PATH)/scripts/clean.py --root $(ROOT_PATH) --path $(BUILD_DIR) .PHONY: comp sim clean diff --git a/rtl/mini/script/filelist_deps.py b/rtl/mini/script/filelist_deps.py new file mode 100644 index 00000000..dfb78d9a --- /dev/null +++ b/rtl/mini/script/filelist_deps.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +from pathlib import Path + +from filelist import atomic_write, parse_filelists + + +HEADER_SUFFIXES = {".h", ".vh", ".svh"} + + +def make_escape(path: Path) -> str: + return ( + str(path.resolve()) + .replace("\\", "\\\\") + .replace("$", "$$") + .replace(" ", "\\ ") + .replace("#", "\\#") + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate Make dependencies from RTL filelists") + parser.add_argument("-f", "--filelist", type=Path, action="append", default=[]) + parser.add_argument("--extra", type=Path, action="append", default=[]) + parser.add_argument("--target", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if not args.filelist: + parser.error("at least one -f/--filelist is required") + + parsed = parse_filelists(args.filelist) + dependencies = { + *(path.resolve() for path in args.filelist), + *(path.resolve() for path in parsed.files), + *(path.resolve() for path in parsed.library_files), + *(path.resolve() for path in args.extra), + } + for directory in parsed.incdirs: + if directory.is_dir(): + dependencies.update( + path.resolve() + for path in directory.rglob("*") + if path.is_file() and path.suffix.lower() in HEADER_SUFFIXES + ) + content = f"{make_escape(args.target)}:" + if dependencies: + content += " \\\n " + " \\\n ".join(make_escape(path) for path in sorted(dependencies)) + content += "\n" + atomic_write(args.output, content) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/rtl/mini/script/gen_iverilog_filelist.py b/rtl/mini/script/gen_iverilog_filelist.py index 2b50cd95..427516d6 100644 --- a/rtl/mini/script/gen_iverilog_filelist.py +++ b/rtl/mini/script/gen_iverilog_filelist.py @@ -6,7 +6,7 @@ import argparse import gzip from pathlib import Path -from filelist import FileList, atomic_write, parse_filelists, write_filelist +from filelist import atomic_write, parse_filelists, write_filelist SCRIPT_DIR = Path(__file__).resolve().parent diff --git a/scripts/analyze_warnings.py b/scripts/analyze_warnings.py new file mode 100644 index 00000000..673bd3ba --- /dev/null +++ b/scripts/analyze_warnings.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from collections import Counter +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.setup_helpers import atomic_write # noqa: E402 + + +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") +TIMESTAMP_RE = re.compile(r"^\[[^]]+\]\s*") +LOCATION_RE = re.compile( + r"(?P(?:[A-Za-z]:)?[^\s:]+?\." + r"(?:c|cc|cpp|cxx|h|hpp|py|sv|svh|tcl|v|vh)):\d+(?::\d+)?", + re.IGNORECASE, +) + + +def warning_line(tool: str, line: str) -> tuple[str, str] | None: + patterns = { + "verilator": re.compile(r"%Warning-([A-Z0-9_]+):\s*(.*)"), + "iverilog": re.compile(r"\b(warning|sorry):\s*(.*)", re.IGNORECASE), + "yosys": re.compile(r"\b(?:ABC:\s*)?Warning:\s*(.*)", re.IGNORECASE), + "opensta": re.compile(r"\bWarning\s+(\d+):\s*(.*)", re.IGNORECASE), + "vcs": re.compile(r"\bWarning(?:-\[([^]]+)\])?:\s*(.*)", re.IGNORECASE), + } + match = patterns[tool].search(line) + if not match: + return None + if tool in ("verilator", "opensta"): + return match.group(1), match.group(2) + if tool == "iverilog": + return match.group(1).lower(), match.group(2) + if tool == "vcs": + return match.group(1) or "warning", match.group(2) + return "warning", match.group(1) + + +def normalize(root: Path, identifier: str, message: str) -> str: + value = ANSI_RE.sub("", TIMESTAMP_RE.sub("", message.strip())) + resolved_root = str(root.resolve()) + value = re.sub( + re.escape(resolved_root) + r"/build/[^/\s:]+", + "$BUILD", + value, + ) + value = value.replace(resolved_root, "$ROOT") + value = LOCATION_RE.sub(lambda match: match.group("path") + ":", value) + value = re.sub(r"\s+", " ", value) + return f"{identifier}:{value}" + + +def scan(tool: str, logs: list[Path], root: Path) -> dict[str, object]: + counts: Counter[str] = Counter() + examples: dict[str, str] = {} + for log in logs: + if not log.is_file(): + continue + for line in log.read_text(encoding="utf-8", errors="replace").splitlines(): + parsed = warning_line(tool, line) + if parsed is None: + continue + normalized = normalize(root, *parsed) + signature = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16] + counts[signature] += 1 + examples.setdefault(signature, normalized) + entries = [ + {"signature": signature, "count": count, "example": examples[signature]} + for signature, count in sorted(counts.items()) + ] + return {"tool": tool, "logs": [str(path) for path in logs], "entries": entries} + + +def discover(variant_root: Path) -> dict[str, list[Path]]: + return { + "iverilog": sorted((variant_root / "sim/iverilog").glob("*/*.log")), + "verilator": sorted((variant_root / "sim/verilator").glob("*.log")), + "vcs": sorted((variant_root / "sim/vcs").glob("*/*.log")), + "yosys": sorted((variant_root / "syn/yosys").glob("*.log")), + "opensta": sorted((variant_root / "sta/opensta").glob("*.log")), + } + + +def write_baseline(args: argparse.Namespace) -> int: + result = scan(args.tool, [path.resolve() for path in args.log], args.root.resolve()) + baseline = { + "schema_version": 1, + "profile": args.profile, + "tool": args.tool, + "entries": result["entries"], + } + atomic_write(args.output, json.dumps(baseline, indent=2, sort_keys=True) + "\n") + print(f"warning baseline: {args.output.resolve()}") + return 0 + + +def check(args: argparse.Namespace) -> int: + root = args.root.resolve() + variant_root = args.variant_root.resolve() + tools = [args.tool] if args.tool else ["iverilog", "verilator", "vcs", "yosys", "opensta"] + logs = discover(variant_root) + reports: dict[str, object] = {} + failures: list[str] = [] + for tool in tools: + if not logs[tool]: + reports[tool] = {"status": "not_run", "entries": []} + continue + current = scan(tool, logs[tool], root) + baseline_path = root / "quality/warnings" / args.profile / f"{tool}.json" + baseline_entries: dict[str, int] = {} + if baseline_path.is_file(): + baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + baseline_entries = { + entry["signature"]: entry["count"] for entry in baseline["entries"] + } + current_entries = { + entry["signature"]: entry["count"] for entry in current["entries"] + } + new = sorted(set(current_entries) - set(baseline_entries)) + increased = sorted( + signature + for signature, count in current_entries.items() + if signature in baseline_entries and count > baseline_entries[signature] + ) + if new or increased: + failures.append(tool) + reports[tool] = { + **current, + "status": "failed" if new or increased else "passed", + "baseline": str(baseline_path), + "new_signatures": new, + "increased_signatures": increased, + "resolved_signatures": sorted(set(baseline_entries) - set(current_entries)), + } + output = { + "schema_version": 1, + "profile": args.profile, + "status": "failed" if failures else "passed", + "failed_tools": failures, + "tools": reports, + } + atomic_write(args.output, json.dumps(output, indent=2, sort_keys=True) + "\n") + print(f"warning check: {output['status']} ({args.output.resolve()})") + return int(bool(failures)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Manage normalized EDA warning baselines") + subparsers = parser.add_subparsers(dest="command", required=True) + baseline = subparsers.add_parser("baseline") + baseline.add_argument("--root", type=Path, default=ROOT) + baseline.add_argument("--profile", required=True) + baseline.add_argument("--tool", choices=("iverilog", "verilator", "vcs", "yosys", "opensta"), required=True) + baseline.add_argument("--log", type=Path, action="append", required=True) + baseline.add_argument("--output", type=Path, required=True) + check_parser = subparsers.add_parser("check") + check_parser.add_argument("--root", type=Path, default=ROOT) + check_parser.add_argument("--profile", required=True) + check_parser.add_argument("--variant-root", type=Path, required=True) + check_parser.add_argument("--tool", choices=("iverilog", "verilator", "vcs", "yosys", "opensta")) + check_parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + return write_baseline(args) if args.command == "baseline" else check(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_simulation.py b/scripts/check_simulation.py new file mode 100644 index 00000000..83e9178c --- /dev/null +++ b/scripts/check_simulation.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.setup_helpers import atomic_write # noqa: E402 + + +DEFAULT_SUCCESS = "retroSoC: A Customized ASIC for Retro Stuff" +DEFAULT_FAILURE = re.compile( + r"(?:\bFAILED?\b|\bFATAL\b|assertion failed|%Error)", re.IGNORECASE +) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Validate a retroSoC simulation log") + parser.add_argument("--log", type=Path, required=True) + parser.add_argument("--result", type=Path, required=True) + parser.add_argument("--require", action="append", default=[]) + parser.add_argument("--reject", action="append", default=[]) + args = parser.parse_args() + + if not args.log.is_file(): + raise SystemExit(f"simulation log not found: {args.log}") + content = args.log.read_text(encoding="utf-8", errors="replace") + required = args.require or [DEFAULT_SUCCESS] + missing = [marker for marker in required if marker not in content] + rejected = [pattern for pattern in args.reject if re.search(pattern, content)] + default_failure = DEFAULT_FAILURE.search(content) + if default_failure: + rejected.append(default_failure.group(0)) + + passed = not missing and not rejected + report = { + "schema_version": 1, + "status": "passed" if passed else "failed", + "log": str(args.log.resolve()), + "required_markers": required, + "missing_markers": missing, + "rejected_matches": sorted(set(rejected)), + } + atomic_write(args.result, json.dumps(report, indent=2, sort_keys=True) + "\n") + print(f"simulation check: {report['status']} ({args.result.resolve()})") + return int(not passed) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/clean.py b/scripts/clean.py index bada6dd7..6f6b032c 100644 --- a/scripts/clean.py +++ b/scripts/clean.py @@ -10,12 +10,21 @@ from pathlib import Path def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Remove retroSoC generated output") parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--path", type=Path, action="append", required=True) return parser.parse_args() -def remove(root: Path, relative: str) -> None: - target = (root / relative).resolve() +def remove(root: Path, requested: Path) -> None: + requested = requested.absolute() + requested.parent.resolve().relative_to(root) + if requested.is_symlink(): + requested.unlink() + print(f"cleaned {requested}") + return + target = requested.resolve() target.relative_to(root) + if target == root: + raise ValueError("refusing to remove the repository root") if target.is_dir() and not target.is_symlink(): shutil.rmtree(target) elif target.exists() or target.is_symlink(): @@ -24,21 +33,15 @@ def remove(root: Path, relative: str) -> None: def main() -> int: - root = parse_args().root.resolve() + args = parse_args() + root = args.root.resolve() if not (root / "Makefile").is_file() or not (root / ".git").exists(): raise SystemExit(f"not a retroSoC repository root: {root}") - generated = ( - ".sw_build", - "rtl/mini/.generated_fl", - "rtl/mini/.iverilog_build", - "rtl/mini/.verilator_build", - "rtl/mini/.vcs_build", - "syn/yosys/.synth_build", - "sta/opensta/retrosoc_sta.log", - "export/rtl", - ) - for relative in generated: - remove(root, relative) + try: + for requested in args.path: + remove(root, requested) + except ValueError as error: + raise SystemExit(f"refusing unsafe clean path: {error}") from error return 0 diff --git a/scripts/config_key.py b/scripts/config_key.py new file mode 100644 index 00000000..20e9a01a --- /dev/null +++ b/scripts/config_key.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import DEFAULT_LOCK, lock_digest, load_lock # noqa: E402 + + +def slug(value: str) -> str: + normalized = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + return normalized or "manual" + + +def variant_id(profile: str, values: list[str], lock: Path) -> str: + parsed: dict[str, str] = {} + for item in values: + name, separator, value = item.partition("=") + if not separator or not name or name in parsed: + raise ValueError(f"invalid or duplicate configuration value: {item}") + parsed[name] = value + payload = { + "schema_version": 1, + "configuration": dict(sorted(parsed.items())), + "dependency_lock_sha256": lock_digest(lock), + } + digest = hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest()[:12] + return f"{slug(profile)}-{digest}" + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate a deterministic build variant ID") + parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK) + parser.add_argument("--profile", default="manual") + parser.add_argument("--value", action="append", default=[]) + args = parser.parse_args() + load_lock(args.lock) + try: + print(variant_id(args.profile, args.value, args.lock)) + except ValueError as error: + parser.error(str(error)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dependency_lock.py b/scripts/dependency_lock.py new file mode 100644 index 00000000..abb3ae39 --- /dev/null +++ b/scripts/dependency_lock.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_LOCK = ROOT / "config/dependencies.lock.json" + + +class LockError(ValueError): + pass + + +def load_lock(path: Path = DEFAULT_LOCK) -> dict[str, Any]: + path = path.resolve() + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise LockError(f"cannot read dependency lock {path}: {error}") from error + validate_lock(data) + return data + + +def validate_lock(data: dict[str, Any]) -> None: + if data.get("schema_version") != 1: + raise LockError("dependency lock schema_version must be 1") + for section in ("sources", "archives", "toolchains"): + if not isinstance(data.get(section), dict) or not data[section]: + raise LockError(f"dependency lock section is missing or empty: {section}") + for name, source in data["sources"].items(): + _require_fields("source", name, source, ("url", "revision", "destination", "license")) + if not re.fullmatch(r"[0-9a-f]{40}", source["revision"]): + raise LockError(f"source {name} revision must be a full 40-character commit") + _validate_url(f"source {name}", source["url"]) + _validate_relative_path(f"source {name} destination", source["destination"]) + for name, archive in data["archives"].items(): + _validate_archive("archive", name, archive) + for platform, tools in data["toolchains"].items(): + if not isinstance(tools, dict) or not tools: + raise LockError(f"toolchain platform is empty: {platform}") + for name, tool in tools.items(): + _require_fields("toolchain", name, tool, ("version", "url", "sha256", "archive", "path")) + _validate_sha(f"toolchain {platform}/{name}", tool["sha256"]) + _validate_url(f"toolchain {platform}/{name}", tool["url"]) + _validate_relative_path(f"toolchain {platform}/{name} archive", tool["archive"]) + _validate_relative_path(f"toolchain {platform}/{name} path", tool["path"]) + + +def _validate_archive(kind: str, name: str, value: dict[str, Any]) -> None: + _require_fields(kind, name, value, ("url", "sha256", "destination", "license")) + _validate_sha(f"{kind} {name}", value["sha256"]) + _validate_url(f"{kind} {name}", value["url"]) + _validate_relative_path(f"{kind} {name} destination", value["destination"]) + + +def _require_fields(kind: str, name: str, value: Any, fields: tuple[str, ...]) -> None: + if not isinstance(value, dict): + raise LockError(f"{kind} {name} must be an object") + missing = [field for field in fields if not value.get(field)] + if missing: + raise LockError(f"{kind} {name} missing fields: {', '.join(missing)}") + + +def _validate_sha(label: str, value: str) -> None: + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise LockError(f"{label} sha256 must be 64 lowercase hexadecimal characters") + + +def _validate_url(label: str, value: str) -> None: + if not value.startswith("https://"): + raise LockError(f"{label} URL must use HTTPS") + + +def _validate_relative_path(label: str, value: str) -> None: + path = Path(value) + if path.is_absolute() or ".." in path.parts: + raise LockError(f"{label} must be a repository-relative path") + + +def lock_digest(path: Path = DEFAULT_LOCK) -> str: + return hashlib.sha256(path.resolve().read_bytes()).hexdigest() + + +def source(name: str, path: Path = DEFAULT_LOCK) -> dict[str, Any]: + data = load_lock(path) + try: + return data["sources"][name] + except KeyError as error: + raise LockError(f"unknown source dependency: {name}") from error + + +def archive(name: str, path: Path = DEFAULT_LOCK) -> dict[str, Any]: + data = load_lock(path) + try: + return data["archives"][name] + except KeyError as error: + raise LockError(f"unknown archive dependency: {name}") from error + + +def main() -> int: + parser = argparse.ArgumentParser(description="Validate the retroSoC dependency lock") + parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK) + parser.add_argument("--digest", action="store_true") + args = parser.parse_args() + data = load_lock(args.lock) + if args.digest: + print(lock_digest(args.lock)) + else: + print( + f"dependency lock valid: {len(data['sources'])} sources, " + f"{len(data['archives'])} archives" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/doctor.py b/scripts/doctor.py index dcb5be2b..88c7554f 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -3,10 +3,19 @@ from __future__ import annotations import argparse +import json import shutil +import subprocess +import sys from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import DEFAULT_LOCK, load_lock, lock_digest # noqa: E402 +from scripts.setup_helpers import atomic_write # noqa: E402 + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Check the retroSoC build environment") parser.add_argument("--root", type=Path, required=True) @@ -16,12 +25,35 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--pdk", required=True) parser.add_argument("--core", required=True) parser.add_argument("--ip", required=True) + parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK) + parser.add_argument("--format", choices=("text", "json"), default="text") + parser.add_argument("--output", type=Path) return parser.parse_args() +def version(executable: str) -> str | None: + path = shutil.which(executable) + if path is None: + return None + options = { + "iverilog": "-V", + "vvp": "-V", + "sta": "-version", + } + result = subprocess.run( + [executable, options.get(executable, "--version")], + text=True, + capture_output=True, + check=False, + ) + lines = (result.stdout or result.stderr).strip().splitlines() + return lines[0] if lines else str(path) + + def main() -> int: args = parse_args() root = args.root.resolve() + lock = load_lock(args.lock) tools = ["python3", "make", "git", "riscv32-unknown-elf-gcc"] if args.simu == "IVERILOG": tools.extend(("iverilog", "vvp", "sv2v")) @@ -30,42 +62,49 @@ def main() -> int: elif args.simu == "VCS": tools.extend(("bsub", "vcs")) if args.synth == "YOSYS": - tools.extend(("yosys", "gawk")) + tools.append("yosys") if args.sta == "OPENSTA": tools.append("sta") + source_names = ["mpw", "cluster_common", "third_party_ip"] + pdk_names = {"IHP130": "pdk_ihp130", "ICS55": "pdk_ics55"} + if args.pdk in pdk_names: + source_names.append(pdk_names[args.pdk]) paths = { - "MPW generator": root / "rtl/mini/mpw/.git", - "cluster IP": root / "rtl/clusterip/common", - "third-party models": root / "rtl/ip/3rd-party", + name: root / lock["sources"][name]["destination"] for name in source_names } - pdk_paths = { - "IHP130": root / "pdk/IHP-Open-PDK", - "ICS55": root / "pdk/icsprout55-pdk", - "SKY130": root / "pdk/skywater-sdk", - "GF180": root / "pdk/gf180mcu-pdk", + tool_results = { + tool: {"path": shutil.which(tool), "version": version(tool)} + for tool in dict.fromkeys(tools) } - paths[f"PDK {args.pdk}"] = pdk_paths[args.pdk] - - failures = 0 - print("Tools:") - for tool in dict.fromkeys(tools): - location = shutil.which(tool) - status = location or "MISSING" - print(f" {tool:<28} {status}") - failures += location is None - - print("Paths:") - for name, path in paths.items(): - exists = path.exists() - print(f" {name:<28} {path} {'[ok]' if exists else '[MISSING]'}") - failures += not exists - - if failures: - print(f"doctor found {failures} missing requirement(s)") - return 1 - print("doctor: all selected requirements are available") - return 0 + path_results = { + name: {"path": str(path), "exists": path.exists()} for name, path in paths.items() + } + failures = sum(value["path"] is None for value in tool_results.values()) + failures += sum(not value["exists"] for value in path_results.values()) + report = { + "schema_version": 1, + "ok": failures == 0, + "failures": failures, + "dependency_lock_sha256": lock_digest(args.lock), + "tools": tool_results, + "paths": path_results, + } + serialized = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + atomic_write(args.output, serialized) + if args.format == "json": + print(serialized, end="") + else: + print("Tools:") + for name, value in tool_results.items(): + print(f" {name:<28} {value['path'] or 'MISSING'}") + print("Paths:") + for name, value in path_results.items(): + suffix = "[ok]" if value["exists"] else "[MISSING]" + print(f" {name:<28} {value['path']} {suffix}") + print("doctor: all selected requirements are available" if not failures else f"doctor found {failures} missing requirement(s)") + return int(failures != 0) if __name__ == "__main__": diff --git a/scripts/generate_mpw.py b/scripts/generate_mpw.py new file mode 100644 index 00000000..e89958bd --- /dev/null +++ b/scripts/generate_mpw.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import TextIO + + +def tree_digest(root: Path) -> str: + digest = hashlib.sha256() + if not root.exists(): + return "" + for path in sorted( + item for item in root.rglob("*") if item.is_file() and item.name != ".stamp" + ): + digest.update(path.relative_to(root).as_posix().encode("utf-8")) + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run(root: Path, log: TextIO, script: Path, *args: str) -> None: + command = ["python3", str(script), *args] + log.write("+ " + " ".join(command) + "\n") + log.flush() + subprocess.run( + command, + cwd=root, + check=True, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + ) + + +def rewrite_filelists(root: Path, old_prefix: Path, new_prefix: Path) -> None: + for filelist in root.rglob("*.fl"): + content = filelist.read_text(encoding="utf-8") + content = content.replace(str(old_prefix.resolve()), str(new_prefix.resolve())) + filelist.write_text(content, encoding="utf-8") + + +def generate(args: argparse.Namespace) -> None: + root = args.root.resolve() + mpw = root / "rtl/mini/mpw" + shared = mpw / ".build" + output = args.output.resolve() + needs_core = args.core == "MDD" + needs_ip = args.ip == "MDD" + log_path = output.parent / f"{output.name}-generation.log" + + output.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("w", encoding="utf-8") as log, tempfile.TemporaryDirectory( + prefix=f".{output.name}.", dir=output.parent + ) as temp: + log.write(f"simulator={args.simu} core={args.core} ip={args.ip}\n") + candidate = Path(temp) / "output" + candidate.mkdir() + (shared / "user_design_info.h").unlink(missing_ok=True) + + if needs_ip: + run(root, log, mpw / "ip.py") + run(root, log, mpw / "info.py", "IP") + shutil.copytree(shared / "ip", candidate / "ip") + if needs_core: + run(root, log, mpw / "core.py", args.simu) + run(root, log, mpw / "info.py", "CORE") + shutil.copytree(shared / "core", candidate / "core") + info = shared / "user_design_info.h" + if info.is_file(): + shutil.copy2(info, candidate / info.name) + + rewrite_filelists(candidate, shared, output) + if tree_digest(candidate) == tree_digest(output): + print(f"[mpw] generated output unchanged: {output} (log: {log_path})") + return + if output.exists(): + shutil.rmtree(output) + os.replace(candidate, output) + print(f"[mpw] generated isolated output: {output} (log: {log_path})") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate isolated MPW build sources") + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--simu", choices=("VCS", "VERILATOR", "IVERILOG"), required=True) + parser.add_argument("--core", choices=("PICORV32", "HAZARD3", "MDD"), required=True) + parser.add_argument("--ip", choices=("NONE", "MDD"), required=True) + parser.add_argument("--lock-file", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + args.lock_file.parent.mkdir(parents=True, exist_ok=True) + with args.lock_file.open("a+", encoding="utf-8") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + generate(args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install_toolchain.py b/scripts/install_toolchain.py new file mode 100644 index 00000000..4df747b0 --- /dev/null +++ b/scripts/install_toolchain.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import os +import posixpath +import shutil +import sys +import tarfile +import tempfile +from pathlib import Path, PurePosixPath + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import DEFAULT_LOCK, load_lock # noqa: E402 +from scripts.setup_helpers import atomic_write, download_file # noqa: E402 + + +def safe_extract(archive: Path, destination: Path) -> None: + with tarfile.open(archive, "r:*") as bundle: + for member in bundle.getmembers(): + member_path = PurePosixPath(member.name) + if member_path.is_absolute() or ".." in member_path.parts: + raise ValueError(f"unsafe archive member path: {member.name}") + if member.ischr() or member.isblk() or member.isfifo(): + raise ValueError(f"unsupported archive member type: {member.name}") + if member.issym() or member.islnk(): + link = PurePosixPath(member.linkname) + combined = member_path.parent / link if member.issym() else link + normalized = PurePosixPath(posixpath.normpath(str(combined))) + escapes_root = normalized.parts[:1] == ("..",) + if link.is_absolute() or normalized.is_absolute() or escapes_root: + raise ValueError( + f"unsafe archive link: {member.name} -> {member.linkname}" + ) + bundle.extractall(destination) + + +def install(name: str, spec: dict[str, str], cache: Path, update: bool) -> Path: + downloads = cache / "downloads" + archive = downloads / spec["archive"] + download_file(spec["url"], archive, spec["sha256"], update=update, timeout=120) + + destination = cache / "toolchains" / f"{name}-{spec['version']}" + marker = destination / ".complete" + if marker.is_file() and marker.read_text(encoding="utf-8").strip() == spec["sha256"]: + return destination / spec["path"] + + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{destination.name}.", dir=destination.parent) as temp: + extracted = Path(temp) / "content" + extracted.mkdir() + safe_extract(archive, extracted) + atomic_write(extracted / ".complete", spec["sha256"] + "\n") + if destination.exists(): + shutil.rmtree(destination) + os.replace(extracted, destination) + return destination / spec["path"] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Install pinned retroSoC toolchains") + parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK) + parser.add_argument("--platform", default="ubuntu-22.04") + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--tool", action="append", required=True) + parser.add_argument("--github-path", type=Path) + parser.add_argument("--update", action="store_true") + args = parser.parse_args() + + lock = load_lock(args.lock) + try: + available = lock["toolchains"][args.platform] + except KeyError as error: + raise SystemExit(f"unsupported toolchain platform: {args.platform}") from error + paths: list[Path] = [] + for name in args.tool: + if name not in available: + raise SystemExit(f"toolchain is not locked for {args.platform}: {name}") + path = install(name, available[name], args.cache.resolve(), args.update) + paths.append(path) + print(f"{name}: {path}") + if args.github_path: + args.github_path.parent.mkdir(parents=True, exist_ok=True) + with args.github_path.open("a", encoding="utf-8") as stream: + for path in paths: + stream.write(str(path.resolve()) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/manifest.py b/scripts/manifest.py new file mode 100644 index 00000000..62ab98f9 --- /dev/null +++ b/scripts/manifest.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import DEFAULT_LOCK, lock_digest, load_lock # noqa: E402 +from scripts.setup_helpers import atomic_write # noqa: E402 + + +TOOLS = { + "python": ("python3", "--version"), + "make": ("make", "--version"), + "riscv_gcc": ("riscv32-unknown-elf-gcc", "--version"), + "iverilog": ("iverilog", "-V"), + "verilator": ("verilator", "--version"), + "yosys": ("yosys", "--version"), + "opensta": ("sta", "-version"), +} + + +def command_output(command: tuple[str, ...]) -> str | None: + if shutil.which(command[0]) is None: + return None + result = subprocess.run(command, text=True, capture_output=True, check=False) + output = (result.stdout or result.stderr).strip().splitlines() + return output[0] if output else None + + +def git_info(root: Path) -> dict[str, object]: + def git(*args: str) -> str: + return subprocess.check_output( + ["git", "-C", str(root), *args], text=True, stderr=subprocess.DEVNULL + ).strip() + + try: + return { + "commit": git("rev-parse", "HEAD"), + "branch": git("rev-parse", "--abbrev-ref", "HEAD"), + "dirty": bool(git("status", "--porcelain")), + } + except (subprocess.CalledProcessError, FileNotFoundError): + return {"commit": "unknown", "branch": "unknown", "dirty": None} + + +def parse_pairs(values: list[str]) -> dict[str, str]: + parsed: dict[str, str] = {} + for item in values: + key, separator, value = item.partition("=") + if not separator or not key or key in parsed: + raise ValueError(f"invalid or duplicate key=value item: {item}") + parsed[key] = value + return dict(sorted(parsed.items())) + + +def file_hashes(root: Path) -> dict[str, str]: + hashes: dict[str, str] = {} + if not root.exists(): + return hashes + for path in sorted(root.rglob("*.fl")): + hashes[str(path.relative_to(root))] = hashlib.sha256(path.read_bytes()).hexdigest() + return hashes + + +def create_manifest(args: argparse.Namespace) -> dict[str, object]: + root = args.root.resolve() + lock = args.lock.resolve() + load_lock(lock) + variant_root = args.output.resolve().parents[1] + return { + "schema_version": 1, + "generated_at": datetime.now(timezone.utc).isoformat(), + "repository": git_info(root), + "profile": args.profile, + "configuration": parse_pairs(args.config), + "dependency_lock": { + "path": str(lock.relative_to(root)), + "sha256": lock_digest(lock), + }, + "host": { + "platform": platform.platform(), + "machine": platform.machine(), + }, + "tools": {name: command_output(command) for name, command in TOOLS.items()}, + "filelists": file_hashes(variant_root), + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Create a retroSoC build manifest") + subparsers = parser.add_subparsers(dest="command", required=True) + create = subparsers.add_parser("create") + create.add_argument("--root", type=Path, required=True) + create.add_argument("--lock", type=Path, default=DEFAULT_LOCK) + create.add_argument("--output", type=Path, required=True) + create.add_argument("--profile", required=True) + create.add_argument("--config", action="append", default=[]) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + data = create_manifest(args) + except ValueError as error: + raise SystemExit(str(error)) from error + atomic_write(args.output, json.dumps(data, indent=2, sort_keys=True) + "\n") + print(f"manifest: {args.output.resolve()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/metrics.py b/scripts/metrics.py new file mode 100644 index 00000000..ac2fc49e --- /dev/null +++ b/scripts/metrics.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import re +import statistics +import sys +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.setup_helpers import atomic_write # noqa: E402 + + +AREA_RE = re.compile(r"Chip area for top module[^:]*:\s*([0-9.eE+-]+)") +HIERARCHY_RE = re.compile(r"^\s*(\d+)\s+[0-9.eE+-]+\s+retrosoc_asic\s*$", re.MULTILINE) + + +def collect(args: argparse.Namespace) -> int: + root = args.variant_root.resolve() + metrics: dict[str, Any] = { + "schema_version": 1, + "policy": "observe", + "firmware": {}, + "synthesis": {}, + "timing": {}, + "flows": {}, + } + for binary in sorted((root / "sw").glob("*.bin")): + metrics["firmware"][binary.name] = {"bytes": binary.stat().st_size} + + area_json = root / "syn/yosys/rpt/retrosoc_asic_area.json" + area_report = root / "syn/yosys/rpt/retrosoc_asic_area.rpt" + if area_json.is_file(): + design = json.loads(area_json.read_text(encoding="utf-8"))["design"] + metrics["synthesis"]["top_area"] = float(design["area"]) + metrics["synthesis"]["top_cells"] = int(design["num_cells"]) + elif area_report.is_file(): + text = area_report.read_text(encoding="utf-8", errors="replace") + areas = AREA_RE.findall(text) + hierarchy = HIERARCHY_RE.findall(text) + if areas: + metrics["synthesis"]["top_area"] = float(areas[-1]) + if hierarchy: + metrics["synthesis"]["top_cells"] = int(hierarchy[-1]) + + timing_report = root / "sta/opensta/timing_metrics.rpt" + if timing_report.is_file(): + labels = ("wns_min", "wns_max", "tns_min", "tns_max") + timing_text = timing_report.read_text(encoding="utf-8", errors="replace") + for line in timing_text.splitlines(): + label, separator, value = line.partition("=") + numbers = re.findall(r"[-+]?(?:\d+\.\d+|\d+)", value) + if separator and label in labels and numbers: + metrics["timing"][label] = float(numbers[-1]) + if not metrics["timing"]: + values = [ + float(value) + for value in re.findall(r"[-+]?(?:\d+\.\d+|\d+)", timing_text) + ] + metrics["timing"].update(dict(zip(labels, values[-4:]))) + + for result in sorted(root.glob("**/result*.json")): + try: + data = json.loads(result.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + metrics["flows"][str(result.relative_to(root))] = { + "status": data.get("status"), + "duration_seconds": data.get("duration_seconds"), + } + atomic_write(args.output, json.dumps(metrics, indent=2, sort_keys=True) + "\n") + print(f"metrics: {args.output.resolve()}") + return 0 + + +def numeric_values(value: Any, prefix: str = "") -> dict[str, float]: + output: dict[str, float] = {} + if isinstance(value, dict): + for key, child in value.items(): + path = f"{prefix}.{key}" if prefix else key + output.update(numeric_values(child, path)) + elif isinstance(value, (int, float)) and not isinstance(value, bool): + output[prefix] = float(value) + return output + + +def baseline(args: argparse.Namespace) -> int: + if len(args.input) != 10: + raise SystemExit("exactly 10 --input metrics files are required") + samples = [numeric_values(json.loads(path.read_text(encoding="utf-8"))) for path in args.input] + common = set.intersection(*(set(sample) for sample in samples)) + medians = {key: statistics.median(sample[key] for sample in samples) for key in sorted(common)} + data = {"schema_version": 1, "sample_count": 10, "values": medians} + atomic_write(args.output, json.dumps(data, indent=2, sort_keys=True) + "\n") + print(f"metrics baseline: {args.output.resolve()}") + return 0 + + +def check(args: argparse.Namespace) -> int: + policy = json.loads(args.policy.read_text(encoding="utf-8")) + current = numeric_values(json.loads(args.metrics.read_text(encoding="utf-8"))) + mode = policy.get("mode") + if mode == "observe": + print("metrics policy: observe") + return 0 + if mode != "gate": + raise SystemExit(f"unsupported metrics policy mode: {mode}") + if args.baseline is None or not args.baseline.is_file(): + raise SystemExit("a metrics baseline is required when policy mode is gate") + baseline_data = json.loads(args.baseline.read_text(encoding="utf-8"))["values"] + failures: list[str] = [] + for key, maximum_growth in policy["maximum_growth"].items(): + if key in current and key in baseline_data and current[key] > baseline_data[key] * (1 + maximum_growth): + failures.append(key) + wns_key = "timing.wns_max" + if wns_key in current and wns_key in baseline_data: + if current[wns_key] < baseline_data[wns_key] - policy["maximum_wns_regression"]: + failures.append(wns_key) + print("metrics check: " + ("failed: " + ", ".join(failures) if failures else "passed")) + return int(bool(failures)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Collect and compare retroSoC metrics") + subparsers = parser.add_subparsers(dest="command", required=True) + collect_parser = subparsers.add_parser("collect") + collect_parser.add_argument("--variant-root", type=Path, required=True) + collect_parser.add_argument("--output", type=Path, required=True) + baseline_parser = subparsers.add_parser("baseline") + baseline_parser.add_argument("--input", type=Path, action="append", required=True) + baseline_parser.add_argument("--output", type=Path, required=True) + check_parser = subparsers.add_parser("check") + check_parser.add_argument("--metrics", type=Path, required=True) + check_parser.add_argument("--baseline", type=Path) + check_parser.add_argument("--policy", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + return {"collect": collect, "baseline": baseline, "check": check}[args.command](args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/package.py b/scripts/package.py new file mode 100644 index 00000000..e9d99ffb --- /dev/null +++ b/scripts/package.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import DEFAULT_LOCK, load_lock # noqa: E402 + + +def component(name: str, kind: str, spec: dict[str, object]) -> dict[str, object]: + license_name = str(spec.get("license", "NOASSERTION")) + license_value = ( + {"name": license_name} + if license_name == "NOASSERTION" or license_name.startswith("LicenseRef-") + else {"id": license_name} + ) + item: dict[str, object] = { + "type": "library" if kind != "toolchain" else "application", + "name": name, + "licenses": [{"license": license_value}], + "externalReferences": [{"type": "distribution", "url": spec["url"]}], + } + if "revision" in spec: + item["version"] = spec["revision"] + item["properties"] = [{"name": "vcs:revision", "value": spec["revision"]}] + else: + item["version"] = spec.get("version", spec.get("sha256", "unknown")) + item["hashes"] = [{"alg": "SHA-256", "content": spec["sha256"]}] + return item + + +def make_sbom(lock: dict[str, object]) -> dict[str, object]: + components = [component(name, "source", spec) for name, spec in lock["sources"].items()] + components.extend(component(name, "archive", spec) for name, spec in lock["archives"].items()) + for platform, tools in lock["toolchains"].items(): + components.extend(component(f"{platform}/{name}", "toolchain", spec) for name, spec in tools.items()) + return { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "components": components, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Create retroSoC source deliverables") + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--lock", type=Path, default=DEFAULT_LOCK) + parser.add_argument("--variant-root", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + root = args.root.resolve() + manifest_path = args.variant_root.resolve() / "meta/manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + config = manifest["configuration"] + lock = load_lock(args.lock) + args.output_dir.parent.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory(prefix="retrosoc-package-", dir=args.output_dir.parent) as temp: + staging = Path(temp) / "package" + staging.mkdir() + command = [ + "python3", str(root / "syn/tools/export_soc_sources.py"), "sv", + "--output-dir", str(staging), "--pdk", config["PDK"], + "--core", config["CORE"], "--ip", config["IP"], "--simu", config["SIMU"], + "--dynamic-core-filelist", + str(args.variant_root / "generated/mpw" / config["SIMU"].lower() / "core/core.fl"), + "--dynamic-ip-filelist", + str(args.variant_root / "generated/mpw" / config["SIMU"].lower() / "ip/ip.fl"), + ] + for key, flag in (("HAVE_PLL", "--have-pll"), ("HAVE_SRAM_IF", "--have-sram-if"), ("HAVE_SRAM_MACRO", "--have-sram-macro"), ("HAVE_SVA", "--have-sva")): + if config.get(key) == "YES": + command.append(flag) + subprocess.run(command, cwd=root, check=True) + command[2] = "tar" + subprocess.run(command, cwd=root, check=True) + shutil.rmtree(staging / "rtl") + shutil.copy2(args.lock, staging / "dependencies.lock.json") + shutil.copy2(manifest_path, staging / "manifest.json") + (staging / "sbom.cdx.json").write_text(json.dumps(make_sbom(lock), indent=2, sort_keys=True) + "\n", encoding="utf-8") + checksums = [] + for path in sorted(item for item in staging.iterdir() if item.is_file()): + checksums.append(f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}") + (staging / "SHA256SUMS").write_text("\n".join(checksums) + "\n", encoding="utf-8") + if args.output_dir.exists(): + shutil.rmtree(args.output_dir) + shutil.copytree(staging, args.output_dir) + print(f"package: {args.output_dir.resolve()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prepare_mpw.py b/scripts/prepare_mpw.py index ec843ad3..202bf2f1 100644 --- a/scripts/prepare_mpw.py +++ b/scripts/prepare_mpw.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import fcntl import hashlib import shutil import sys @@ -11,15 +12,12 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) +from scripts.dependency_lock import archive, source # noqa: E402 from scripts.setup_helpers import download_file, ensure_git_repo # noqa: E402 MPW_DIR = ROOT / "rtl/mini/mpw" -HAZARD3_REVISION = "5b3a34f6955e50e03bdcb964201c91462f7078c3" -IBEX_REVISION = "0c233f54361d769f370889223acc456f2ac19d46" SERV_ARCHIVE = MPW_DIR / "serv-1.4.0.tar.gz" -SERV_URL = "https://github.com/olofk/serv/archive/refs/tags/1.4.0.tar.gz" -SERV_SHA256 = "f81c37b9f9d548c658e23bb7eb90fbed8b54e2e9e51fa42b8cd2f34cecc472ab" def tree_digest(root: Path) -> str: @@ -55,30 +53,24 @@ def extract_serv() -> Path: return extracted -def main() -> int: - parser = argparse.ArgumentParser(description="Prepare pinned MPW core sources") - parser.add_argument("--update", action="store_true") - args = parser.parse_args() +def prepare(update: bool) -> None: if not (MPW_DIR / ".git").is_dir(): raise SystemExit("MPW generator is missing; run 'python3 setup.py' first") + hazard3 = source("hazard3") ensure_git_repo( - "https://github.com/Wren6991/Hazard3.git", - MPW_DIR / "Hazard3", - HAZARD3_REVISION, - update=args.update, + hazard3["url"], ROOT / hazard3["destination"], hazard3["revision"], + update=update, ) + ibex = source("ibex") ensure_git_repo( - "https://github.com/lowRISC/ibex.git", - MPW_DIR / "ibex", - IBEX_REVISION, - update=args.update, + ibex["url"], ROOT / ibex["destination"], ibex["revision"], + update=update, ) + serv_archive = archive("serv") download_file( - SERV_URL, - SERV_ARCHIVE, - SERV_SHA256, - update=args.update, + serv_archive["url"], ROOT / serv_archive["destination"], + serv_archive["sha256"], update=update, ) serv = extract_serv() @@ -87,6 +79,20 @@ def main() -> int: sync_tree(MPW_DIR / "ibex/rtl", MPW_DIR / "core/username7/ibex") (MPW_DIR / ".build").mkdir(exist_ok=True) print("pinned MPW core sources are ready") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Prepare pinned MPW core sources") + parser.add_argument("--update", action="store_true") + parser.add_argument("--lock-file", type=Path) + args = parser.parse_args() + if args.lock_file is None: + prepare(args.update) + return 0 + args.lock_file.parent.mkdir(parents=True, exist_ok=True) + with args.lock_file.open("a+", encoding="utf-8") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + prepare(args.update) return 0 diff --git a/scripts/regress.py b/scripts/regress.py new file mode 100644 index 00000000..bc06b3be --- /dev/null +++ b/scripts/regress.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import subprocess +from pathlib import Path + + +PR_COMMANDS = ( + ("configs/ci/hazard3-rv32im-ihp130.mk", ("firmware",)), + ("configs/ci/mdd-rv32im-ihp130.mk", ("SIMU=VERILATOR", "firmware", "sim")), + ("configs/ci/hazard3-rv32im-ihp130.mk", ("SIMU=VERILATOR", "sim")), + ("configs/ci/hazard3-rv32im-ihp130.mk", ("SIMU=IVERILOG", "RTL_SIM_TIMEOUT=5200000", "sim-asm")), + ("configs/ci/hazard3-rv32im-ihp130.mk", ("SYNTH=YOSYS", "synth")), + ("configs/ci/hazard3-rv32im-ihp130.mk", ("SIMU=IVERILOG", "SIM_FIRMWARE_NAME=retrosoc_asm", "SIM_SUCCESS_MARKER=Mem wr/rd test success", "RTL_SIM_TIMEOUT=5200000", "netsim")), + ("configs/ci/hazard3-rv32im-ihp130.mk", ("STA=OPENSTA", "sta")), +) +NIGHTLY_COMMANDS = PR_COMMANDS + ( + ("configs/nightly/picorv32-rv32im-ihp130.mk", ("SIMU=VERILATOR", "firmware", "sim")), + ("configs/nightly/picorv32-rv32im-ihp130.mk", ("SIMU=IVERILOG", "RTL_SIM_TIMEOUT=5200000", "sim-asm")), +) +PR_PROFILES = ( + "configs/ci/hazard3-rv32im-ihp130.mk", + "configs/ci/mdd-rv32im-ihp130.mk", +) +NIGHTLY_PROFILES = PR_PROFILES + ("configs/nightly/picorv32-rv32im-ihp130.mk",) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run a supported retroSoC regression suite") + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--suite", choices=("pr", "nightly"), required=True) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + commands = PR_COMMANDS if args.suite == "pr" else NIGHTLY_COMMANDS + profiles = PR_PROFILES if args.suite == "pr" else NIGHTLY_PROFILES + for profile, values in commands: + command = ["make", f"CONFIG={profile}", *values] + print("+ " + " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, cwd=args.root, check=True) + for profile in profiles: + command = [ + "make", + f"CONFIG={profile}", + "check-warnings", + "check-metrics", + ] + print("+ " + " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, cwd=args.root, check=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_flow.py b/scripts/run_flow.py new file mode 100644 index 00000000..f2ba4ece --- /dev/null +++ b/scripts/run_flow.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.setup_helpers import atomic_write # noqa: E402 + + +def parse_env(values: list[str]) -> dict[str, str]: + environment = os.environ.copy() + for item in values: + name, separator, value = item.partition("=") + if not separator or not name: + raise ValueError(f"invalid environment assignment: {item}") + environment[name] = value + return environment + + +def should_display(tool: str, line: str) -> bool: + if tool != "yosys": + return True + keywords = ("Executing", "Warning", "Error", "Build succeeded", "End of script") + return any(keyword in line for keyword in keywords) + + +def jobserver_fds() -> tuple[int, ...]: + match = re.search(r"--jobserver-(?:auth|fds)=(\d+),(\d+)", os.environ.get("MAKEFLAGS", "")) + if not match: + return () + descriptors = tuple(int(value) for value in match.groups()) + try: + for descriptor in descriptors: + os.fstat(descriptor) + except OSError: + return () + return descriptors + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run a flow command with structured results") + parser.add_argument("--tool", required=True) + parser.add_argument("--log", type=Path, required=True) + parser.add_argument("--result", type=Path, required=True) + parser.add_argument("--cwd", type=Path) + parser.add_argument("--env", action="append", default=[]) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("a command is required after --") + try: + environment = parse_env(args.env) + except ValueError as error: + parser.error(str(error)) + + args.log.parent.mkdir(parents=True, exist_ok=True) + args.result.parent.mkdir(parents=True, exist_ok=True) + started = datetime.now(timezone.utc) + start_clock = time.monotonic() + returncode = 127 + error_message = None + process: subprocess.Popen[str] | None = None + try: + with args.log.open("w", encoding="utf-8", errors="replace") as log: + process = subprocess.Popen( + command, + cwd=args.cwd, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + bufsize=1, + pass_fds=jobserver_fds(), + ) + assert process.stdout is not None + for line in process.stdout: + log.write(line) + log.flush() + if should_display(args.tool, line): + sys.stdout.write(line) + sys.stdout.flush() + returncode = process.wait() + except KeyboardInterrupt: + error_message = "interrupted" + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + returncode = 130 + except (OSError, subprocess.SubprocessError) as error: + error_message = str(error) + + finished = datetime.now(timezone.utc) + result = { + "schema_version": 1, + "tool": args.tool, + "command": command, + "cwd": str(args.cwd.resolve()) if args.cwd else os.getcwd(), + "started_at": started.isoformat(), + "finished_at": finished.isoformat(), + "duration_seconds": round(time.monotonic() - start_clock, 3), + "exit_code": returncode, + "status": "passed" if returncode == 0 else "failed", + "log": str(args.log.resolve()), + } + if error_message: + result["error"] = error_message + atomic_write(args.result, json.dumps(result, indent=2, sort_keys=True) + "\n") + return returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/setup_helpers.py b/scripts/setup_helpers.py index c1471e1a..4e172107 100644 --- a/scripts/setup_helpers.py +++ b/scripts/setup_helpers.py @@ -41,11 +41,10 @@ def ensure_git_repo( created = False if not destination.exists(): destination.parent.mkdir(parents=True, exist_ok=True) - command = ["git", "clone"] - if recursive: - command.append("--recursive") - command.extend((url, str(destination))) - run(command) + run(("git", "init", str(destination))) + run(("git", "remote", "add", "origin", url), cwd=destination) + run(("git", "fetch", "--depth", "1", "origin", revision), cwd=destination) + run(("git", "checkout", "--detach", "FETCH_HEAD"), cwd=destination) created = True elif not (destination / ".git").exists(): raise RuntimeError(f"dependency path is not a Git repository: {destination}") @@ -62,13 +61,16 @@ def ensure_git_repo( try: git_output(destination, "cat-file", "-e", f"{revision}^{{commit}}") except subprocess.CalledProcessError: - run(("git", "fetch", "origin", revision), cwd=destination) + run(("git", "fetch", "--depth", "1", "origin", revision), cwd=destination) run(("git", "checkout", "--detach", revision), cwd=destination) elif dirty and not allow_dirty: raise RuntimeError(f"pinned dependency has local changes: {destination}") if recursive: - run(("git", "submodule", "update", "--init", "--recursive"), cwd=destination) + run( + ("git", "submodule", "update", "--init", "--recursive", "--depth", "1"), + cwd=destination, + ) print(f"[dependency] {destination.name}: {revision}") diff --git a/setup.py b/setup.py index 43e789c2..c63adf17 100644 --- a/setup.py +++ b/setup.py @@ -3,22 +3,21 @@ import argparse from pathlib import Path +from scripts.dependency_lock import source from scripts.setup_helpers import ensure_git_repo ROOT = Path(__file__).resolve().parent -MPW_REVISION = "30503eb8a7cf33a0c955913faba5014ebd00186f" def main() -> int: parser = argparse.ArgumentParser(description="Install the pinned MPW generator") parser.add_argument("--update", action="store_true") args = parser.parse_args() + dependency = source("mpw") ensure_git_repo( - "https://github.com/retroSoC/mini-ver-mpw.git", - ROOT / "rtl/mini/mpw", - MPW_REVISION, - update=args.update, + dependency["url"], ROOT / dependency["destination"], dependency["revision"], + recursive=dependency.get("recursive", False), update=args.update, ) return 0 diff --git a/sta/opensta/opensta.mk b/sta/opensta/opensta.mk index a432c3c5..030d2546 100644 --- a/sta/opensta/opensta.mk +++ b/sta/opensta/opensta.mk @@ -1,14 +1,18 @@ OPENSTA ?= sta -OPENSTA_NETLIST ?= $(ROOT_PATH)/syn/yosys/.synth_build/out/retrosoc_asic_yosys.v +OPENSTA_THREADS ?= $(JOBS) +OPENSTA_NETLIST ?= $(SYN_BUILD_ROOT)/out/retrosoc_asic_yosys.v OPENSTA_LIBERTY ?= $(ROOT_PATH)/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_stdcell/lib/sg13g2_stdcell_typ_1p20V_25C.lib OPENSTA_IO_LIB ?= $(ROOT_PATH)/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_io/lib/sg13g2_io_typ_1p2V_3p3V_25C.lib OPENSTA_SRAM_LIBS ?= $(wildcard $(ROOT_PATH)/pdk/IHP-Open-PDK/ihp-sg13g2/libs.ref/sg13g2_sram/lib/*_typ_1p20V_25C.lib) OPENSTA_SDC ?= $(ROOT_PATH)/sta/opensta/gen2.sdc -OPENSTA_REPORT ?= $(ROOT_PATH)/sta/opensta/retrosoc_sta.log -OPENSTA_CONFIG ?= $(ROOT_PATH)/syn/yosys/.synth_build/out/retrosoc_asic_yosys.config +OPENSTA_REPORT ?= $(STA_BUILD_ROOT)/retrosoc_sta.log +OPENSTA_LOG ?= $(STA_BUILD_ROOT)/opensta.log +OPENSTA_METRICS ?= $(STA_BUILD_ROOT)/timing_metrics.rpt +OPENSTA_CONFIG ?= $(SYN_BUILD_ROOT)/out/retrosoc_asic_yosys.config sta: + @mkdir -p $(STA_BUILD_ROOT) @for input in $(OPENSTA_NETLIST) $(OPENSTA_LIBERTY) $(OPENSTA_IO_LIB) $(OPENSTA_SRAM_LIBS) $(OPENSTA_SDC) $(OPENSTA_CONFIG); do \ test -f "$$input" || { echo "OpenSTA input missing: $$input" >&2; exit 1; }; \ done @@ -16,9 +20,14 @@ sta: echo "OpenSTA netlist configuration does not match PDK=$(PDK); rerun synthesis" >&2; \ exit 1; \ } - OPENSTA_NETLIST=$(OPENSTA_NETLIST) OPENSTA_LIBERTY=$(OPENSTA_LIBERTY) \ - OPENSTA_IO_LIB=$(OPENSTA_IO_LIB) OPENSTA_SRAM_LIBS="$(OPENSTA_SRAM_LIBS)" OPENSTA_SDC=$(OPENSTA_SDC) \ - OPENSTA_REPORT=$(OPENSTA_REPORT) \ - $(OPENSTA) $(ROOT_PATH)/sta/opensta/opensta.tcl -threads max + python3 $(ROOT_PATH)/scripts/run_flow.py --tool opensta --log $(OPENSTA_LOG) \ + --result $(STA_BUILD_ROOT)/result-sta.json \ + --env OPENSTA_NETLIST=$(OPENSTA_NETLIST) --env OPENSTA_LIBERTY=$(OPENSTA_LIBERTY) \ + --env OPENSTA_IO_LIB=$(OPENSTA_IO_LIB) --env 'OPENSTA_SRAM_LIBS=$(OPENSTA_SRAM_LIBS)' \ + --env OPENSTA_SDC=$(OPENSTA_SDC) --env OPENSTA_REPORT=$(OPENSTA_REPORT) \ + --env OPENSTA_METRICS=$(OPENSTA_METRICS) -- \ + $(OPENSTA) $(ROOT_PATH)/sta/opensta/opensta.tcl -threads $(OPENSTA_THREADS) + +sta: | manifest .PHONY: sta diff --git a/sta/opensta/opensta.tcl b/sta/opensta/opensta.tcl index 2f9921c0..fe41b978 100644 --- a/sta/opensta/opensta.tcl +++ b/sta/opensta/opensta.tcl @@ -1,4 +1,4 @@ -foreach variable {OPENSTA_NETLIST OPENSTA_LIBERTY OPENSTA_IO_LIB OPENSTA_SRAM_LIBS OPENSTA_SDC OPENSTA_REPORT} { +foreach variable {OPENSTA_NETLIST OPENSTA_LIBERTY OPENSTA_IO_LIB OPENSTA_SRAM_LIBS OPENSTA_SDC OPENSTA_REPORT OPENSTA_METRICS} { if {![info exists ::env($variable)] || $::env($variable) eq ""} { error "required environment variable is missing: $variable" } @@ -15,6 +15,33 @@ read_sdc $::env(OPENSTA_SDC) report_checks -path_delay min_max -path_group {clk_ext clk_aud } -sort_by_slack -slack_max 0.0 -group_path_count 1000 -endpoint_path_count 1000 > $::env(OPENSTA_REPORT) +set metrics_file $::env(OPENSTA_METRICS) +set metrics_tmp "${metrics_file}.tmp" +set metrics [open $metrics_file "w"] + +report_wns -min > $metrics_tmp +set input [open $metrics_tmp "r"] +puts $metrics "wns_min=[string trim [read $input]]" +close $input + +report_wns -max > $metrics_tmp +set input [open $metrics_tmp "r"] +puts $metrics "wns_max=[string trim [read $input]]" +close $input + +report_tns -min > $metrics_tmp +set input [open $metrics_tmp "r"] +puts $metrics "tns_min=[string trim [read $input]]" +close $input + +report_tns -max > $metrics_tmp +set input [open $metrics_tmp "r"] +puts $metrics "tns_max=[string trim [read $input]]" +close $input + +close $metrics +file delete -force $metrics_tmp + # set_propagated_clock clk # read_spef gcd_sky130hd.spef # set_power_activity -input -activity .1 diff --git a/syn/tools/export_soc_sources.py b/syn/tools/export_soc_sources.py index 99585e77..84377487 100644 --- a/syn/tools/export_soc_sources.py +++ b/syn/tools/export_soc_sources.py @@ -54,14 +54,20 @@ def configured_filelist(args: argparse.Namespace, generated_dir: Path) -> FileLi if args.core in ("HAZARD3", "PICORV32"): names.append(f"core_{args.core.lower()}.fl") else: - dynamic_core = REPO_ROOT / "rtl/mini/mpw/.build/core/core.fl" + dynamic_core = args.dynamic_core_filelist if not dynamic_core.is_file(): - raise FileNotFoundError("MDD core is not generated; run 'make CORE=MDD gen_mpw_code'") + raise FileNotFoundError( + f"MDD core filelist is not generated: {dynamic_core}; " + "run 'make CORE=MDD gen_mpw_code'" + ) names.append(str(dynamic_core)) if args.ip == "MDD": - dynamic_ip = REPO_ROOT / "rtl/mini/mpw/.build/ip/ip.fl" + dynamic_ip = args.dynamic_ip_filelist if not dynamic_ip.is_file(): - raise FileNotFoundError("MDD IP is not generated; run 'make IP=MDD gen_mpw_code'") + raise FileNotFoundError( + f"MDD IP filelist is not generated: {dynamic_ip}; " + "run 'make IP=MDD gen_mpw_code'" + ) names.append(str(dynamic_ip)) names.append("top.fl") paths = [Path(name) if Path(name).is_absolute() else generated_dir / name for name in names] @@ -181,6 +187,16 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--have-sram-if", action="store_true") parser.add_argument("--have-sram-macro", action="store_true") parser.add_argument("--have-sva", action="store_true") + parser.add_argument( + "--dynamic-core-filelist", + type=Path, + default=REPO_ROOT / "rtl/mini/mpw/.build/core/core.fl", + ) + parser.add_argument( + "--dynamic-ip-filelist", + type=Path, + default=REPO_ROOT / "rtl/mini/mpw/.build/ip/ip.fl", + ) parser.add_argument("--output-dir", type=Path, default=REPO_ROOT / "export") return parser.parse_args() diff --git a/syn/yosys/script/synth.tcl b/syn/yosys/script/synth.tcl index 47cf7928..18e7094c 100644 --- a/syn/yosys/script/synth.tcl +++ b/syn/yosys/script/synth.tcl @@ -171,6 +171,7 @@ yosys hilomap -singleton -hicell {*}$tech_cell_tiehi -locell {*}$tech_cell_tielo # final reports yosys tee -q -o "${report_dir}/${proj_name}_synth.rpt" check yosys tee -q -o "${report_dir}/${proj_name}_area.rpt" stat -top $top_design {*}$liberty_args +yosys tee -q -o "${report_dir}/${proj_name}_area.json" stat -json -top $top_design {*}$liberty_args yosys tee -q -o "${report_dir}/${proj_name}_area_logic.rpt" stat -top $top_design {*}$tech_cells_args # final netlist diff --git a/syn/yosys/yosys.mk b/syn/yosys/yosys.mk index 167d5cc8..2c6d0ded 100644 --- a/syn/yosys/yosys.mk +++ b/syn/yosys/yosys.mk @@ -20,7 +20,7 @@ # Directories # directory of the path to the last called Makefile (this one) YOSYS_DIR := $(realpath $(dir $(realpath $(lastword $(MAKEFILE_LIST))))) -YOSYS_BUILD := $(YOSYS_DIR)/.synth_build +YOSYS_BUILD := $(SYN_BUILD_ROOT) YOSYS_OUT := $(YOSYS_BUILD)/out YOSYS_TMP := $(YOSYS_BUILD)/tmp YOSYS_RPT := $(YOSYS_BUILD)/rpt @@ -29,38 +29,44 @@ include $(YOSYS_DIR)/synth_config.mk TOP_DESIGN ?= retrosoc_asic RTL_NAME ?= retrosoc_asic -SV_FLIST := $(RTL_PATH)/.generated_fl/yosys.fl +SV_FLIST := $(GENERATED_FL_DIR)/yosys.fl NETLIST := $(YOSYS_OUT)/$(RTL_NAME)_yosys.v NETLIST_DEBUG := $(YOSYS_OUT)/$(RTL_NAME)_yosys_debug.v NETLIST_CONFIG := $(YOSYS_OUT)/$(RTL_NAME)_yosys.config +YOSYS_DEPFILE := $(YOSYS_BUILD)/yosys.d +YOSYS_SCRIPTS := $(wildcard $(YOSYS_DIR)/script/*) $(YOSYS_DIR)/synth_config.mk $(YOSYS_DIR)/yosys.mk -gen_synth_filelist: gen_mpw_code generate_filelist +-include $(YOSYS_DEPFILE) + +$(SV_FLIST): $(MPW_VARIANT_STAMP) $(FILELIST_STAMP) @python3 $(RTL_PATH)/script/comb.py $(RTL_FLIST) --output $(SV_FLIST) + @python3 $(RTL_PATH)/script/filelist_deps.py $(RTL_FLIST) --target $(NETLIST) \ + --output $(YOSYS_DEPFILE) + +gen_synth_filelist: $(SV_FLIST) ## Synthesize netlist using Yosys synth: $(NETLIST) -$(NETLIST): gen_synth_filelist +$(NETLIST): $(SV_FLIST) $(YOSYS_SCRIPTS) @mkdir -p $(YOSYS_OUT) @mkdir -p $(YOSYS_TMP) @mkdir -p $(YOSYS_RPT) - export PDK="$(PDK)" SOC="$(SOC)" CORE="$(CORE)" IP="$(IP)" \ - SV_FLIST="$(SV_FLIST)" TOP_DESIGN="$(TOP_DESIGN)" CONFIG="$(NETLIST_CONFIG)" \ - PROJ_NAME="$(RTL_NAME)" WORK="$(YOSYS_TMP)" BUILD="$(YOSYS_OUT)" \ - REPORTS="$(YOSYS_RPT)" NETLIST="$(NETLIST)"; \ - set -o pipefail; yosys -c $(YOSYS_DIR)/script/synth.tcl \ - 2>&1 | TZ=UTC-8 gawk '{ print strftime("[%Y-%m-%d %H:%M %Z]"), $$0 }' \ - | tee "$(YOSYS_BUILD)/$(RTL_NAME).log" \ - | gawk -f $(YOSYS_DIR)/script/filter_output.awk + python3 $(ROOT_PATH)/scripts/run_flow.py --tool yosys \ + --log $(YOSYS_BUILD)/$(RTL_NAME).log --result $(YOSYS_BUILD)/result-synth.json \ + --env PDK=$(PDK) --env SOC=$(SOC) --env CORE=$(CORE) --env IP=$(IP) \ + --env SV_FLIST=$(SV_FLIST) --env TOP_DESIGN=$(TOP_DESIGN) --env CONFIG=$(NETLIST_CONFIG) \ + --env PROJ_NAME=$(RTL_NAME) --env WORK=$(YOSYS_TMP) --env BUILD=$(YOSYS_OUT) \ + --env REPORTS=$(YOSYS_RPT) --env NETLIST=$(NETLIST) -- \ + yosys -c $(YOSYS_DIR)/script/synth.tcl $(NETLIST_DEBUG): $(NETLIST) @test -f $@ synth_clean: - rm -rf $(YOSYS_OUT) - rm -rf $(YOSYS_TMP) - rm -rf $(YOSYS_RPT) - rm -f $(YOSYS_BUILD)/$(RTL_NAME).log + python3 $(ROOT_PATH)/scripts/clean.py --root $(ROOT_PATH) --path $(YOSYS_BUILD) .PHONY: gen_synth_filelist synth_clean synth + +synth: | manifest diff --git a/tests/test_script_tools.py b/tests/test_script_tools.py index 5e9caea0..6c14a252 100644 --- a/tests/test_script_tools.py +++ b/tests/test_script_tools.py @@ -1,8 +1,14 @@ from __future__ import annotations import hashlib +import io +import json +import re +import signal import subprocess import sys +import tarfile +import time from pathlib import Path @@ -11,8 +17,11 @@ FILELIST_SCRIPT_DIR = ROOT / "rtl/mini/script" sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(FILELIST_SCRIPT_DIR)) -from filelist import FileList, atomic_write, parse_filelists, write_filelist # noqa: E402 +from filelist import atomic_write, parse_filelists, write_filelist # noqa: E402 from generate_filelist import generate_all # noqa: E402 +from scripts.analyze_warnings import normalize # noqa: E402 +from scripts.dependency_lock import LockError, load_lock # noqa: E402 +from scripts.install_toolchain import safe_extract # noqa: E402 from scripts.setup_helpers import download_file, ensure_git_repo # noqa: E402 @@ -145,12 +154,28 @@ def test_dependency_helpers_are_idempotent(tmp_path: Path) -> None: def test_make_dry_run_and_validation_do_not_write_filelists() -> None: - generated_def = ROOT / "rtl/mini/.generated_fl/def.fl" - before = generated_def.stat().st_mtime_ns if generated_def.exists() else None + def build_state() -> dict[str, tuple[int, int]]: + build = ROOT / "build" + if not build.exists(): + return {} + return { + str(path.relative_to(build)): (path.stat().st_size, path.stat().st_mtime_ns) + for path in build.rglob("*") + if path.is_file() + } + + before = build_state() run("make", "-n", "help") run("make", "-n", "SIMU=IVERILOG", "comp") - after = generated_def.stat().st_mtime_ns if generated_def.exists() else None - assert after == before + run( + "make", + "-n", + "CONFIG=configs/ci/hazard3-rv32im-ihp130.mk", + "SIMU=IVERILOG", + "RTL_SIM_TIMEOUT=5200000", + "sim-asm", + ) + assert build_state() == before invalid = subprocess.run( ["make", "SIMU=UNKNOWN", "help"], @@ -162,6 +187,312 @@ def test_make_dry_run_and_validation_do_not_write_filelists() -> None: assert "Invalid SIMU='UNKNOWN'" in invalid.stderr +def test_dependency_lock_and_config_key_are_deterministic(tmp_path: Path) -> None: + lock = load_lock(ROOT / "config/dependencies.lock.json") + assert lock["schema_version"] == 1 + assert len(lock["sources"]["mpw"]["revision"]) == 40 + + command = ( + sys.executable, + str(ROOT / "scripts/config_key.py"), + "--lock", + str(ROOT / "config/dependencies.lock.json"), + "--profile", + "unit", + "--value", + "CORE=HAZARD3", + "--value", + "PDK=IHP130", + ) + first = run(*command).stdout.strip() + second = run(*command).stdout.strip() + assert first == second + assert first.startswith("unit-") + + broken = tmp_path / "broken.json" + broken.write_text('{"schema_version": 1}\n', encoding="utf-8") + try: + load_lock(broken) + except LockError as error: + assert "missing or empty" in str(error) + else: + raise AssertionError("invalid dependency lock was accepted") + + +def test_run_flow_writes_structured_result(tmp_path: Path) -> None: + log = tmp_path / "flow.log" + result = tmp_path / "result.json" + run( + sys.executable, + str(ROOT / "scripts/run_flow.py"), + "--tool", + "unit", + "--log", + str(log), + "--result", + str(result), + "--", + sys.executable, + "-c", + "print('flow output')", + ) + data = json.loads(result.read_text(encoding="utf-8")) + assert data["status"] == "passed" + assert data["exit_code"] == 0 + assert data["duration_seconds"] >= 0 + assert log.read_text(encoding="utf-8") == "flow output\n" + + +def test_run_flow_records_interruption(tmp_path: Path) -> None: + log = tmp_path / "interrupted.log" + result = tmp_path / "interrupted.json" + process = subprocess.Popen( + [ + sys.executable, + str(ROOT / "scripts/run_flow.py"), + "--tool", + "unit", + "--log", + str(log), + "--result", + str(result), + "--", + sys.executable, + "-c", + "import time; time.sleep(30)", + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + for _ in range(50): + if log.exists(): + break + time.sleep(0.02) + process.send_signal(signal.SIGINT) + process.communicate(timeout=10) + assert process.returncode == 130 + data = json.loads(result.read_text(encoding="utf-8")) + assert data["status"] == "failed" + assert data["exit_code"] == 130 + assert data["error"] == "interrupted" + + +def test_simulation_success_marker_and_failure_detection(tmp_path: Path) -> None: + log = tmp_path / "sim.log" + result = tmp_path / "result.json" + log.write_text( + "retroSoC: A Customized ASIC for Retro Stuff\nSimulation complete\n", + encoding="utf-8", + ) + run( + sys.executable, + str(ROOT / "scripts/check_simulation.py"), + "--log", + str(log), + "--result", + str(result), + ) + assert json.loads(result.read_text(encoding="utf-8"))["status"] == "passed" + + log.write_text( + "retroSoC: A Customized ASIC for Retro Stuff\nTEST FAILED\n", + encoding="utf-8", + ) + failed = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/check_simulation.py"), + "--log", + str(log), + "--result", + str(result), + ], + text=True, + capture_output=True, + ) + assert failed.returncode != 0 + assert json.loads(result.read_text(encoding="utf-8"))["status"] == "failed" + + +def test_warning_baseline_rejects_new_signature(tmp_path: Path) -> None: + profile = "unit-profile" + log = tmp_path / "variant/sim/verilator/verilating.log" + log.parent.mkdir(parents=True) + log.write_text( + f"%Warning-WIDTH: {tmp_path}/rtl/top.sv:12: width mismatch\n", + encoding="utf-8", + ) + baseline = tmp_path / f"quality/warnings/{profile}/verilator.json" + run( + sys.executable, + str(ROOT / "scripts/analyze_warnings.py"), + "baseline", + "--root", + str(tmp_path), + "--profile", + profile, + "--tool", + "verilator", + "--log", + str(log), + "--output", + str(baseline), + ) + report = tmp_path / "warnings.json" + run( + sys.executable, + str(ROOT / "scripts/analyze_warnings.py"), + "check", + "--root", + str(tmp_path), + "--profile", + profile, + "--variant-root", + str(tmp_path / "variant"), + "--tool", + "verilator", + "--output", + str(report), + ) + + log.write_text( + log.read_text(encoding="utf-8") + + f"%Warning-UNUSED: {tmp_path}/rtl/top.sv:20: unused signal\n", + encoding="utf-8", + ) + failed = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/analyze_warnings.py"), + "check", + "--root", + str(tmp_path), + "--profile", + profile, + "--variant-root", + str(tmp_path / "variant"), + "--tool", + "verilator", + "--output", + str(report), + ], + text=True, + capture_output=True, + ) + assert failed.returncode != 0 + assert json.loads(report.read_text(encoding="utf-8"))["failed_tools"] == ["verilator"] + + +def test_warning_normalization_keeps_ranges_and_removes_variant_hash(tmp_path: Path) -> None: + message = ( + f"{tmp_path}/build/profile-deadbeef/generated/core.sv:42: " + "Bit extraction of var[7:0] is too wide" + ) + normalized = normalize(tmp_path, "WIDTH", message) + assert normalized == ( + "WIDTH:$BUILD/generated/core.sv:: " + "Bit extraction of var[7:0] is too wide" + ) + + +def test_metrics_collection_and_observe_policy(tmp_path: Path) -> None: + variant = tmp_path / "variant" + (variant / "sw").mkdir(parents=True) + (variant / "sw/firmware.bin").write_bytes(b"1234") + report_dir = variant / "syn/yosys/rpt" + report_dir.mkdir(parents=True) + (report_dir / "retrosoc_asic_area.rpt").write_text( + " 42 1.23E+02 retrosoc_asic\n" + "Chip area for top module '\\retrosoc_asic': 123.0\n", + encoding="utf-8", + ) + (report_dir / "retrosoc_asic_area.json").write_text( + json.dumps({"design": {"area": 124.0, "num_cells": 43}}), + encoding="utf-8", + ) + timing_dir = variant / "sta/opensta" + timing_dir.mkdir(parents=True) + (timing_dir / "timing_metrics.rpt").write_text("-1.0\n-2.0\n-3.0\n-4.0\n", encoding="utf-8") + (variant / "result-unit.json").write_text( + json.dumps({"status": "passed", "duration_seconds": 1.25}), + encoding="utf-8", + ) + metrics = tmp_path / "metrics.json" + run( + sys.executable, + str(ROOT / "scripts/metrics.py"), + "collect", + "--variant-root", + str(variant), + "--output", + str(metrics), + ) + data = json.loads(metrics.read_text(encoding="utf-8")) + assert data["firmware"]["firmware.bin"]["bytes"] == 4 + assert data["synthesis"] == {"top_area": 124.0, "top_cells": 43} + assert data["timing"]["wns_min"] == -1.0 + + policy = tmp_path / "policy.json" + policy.write_text('{"mode": "observe"}\n', encoding="utf-8") + run( + sys.executable, + str(ROOT / "scripts/metrics.py"), + "check", + "--metrics", + str(metrics), + "--policy", + str(policy), + ) + + +def test_safe_extract_rejects_parent_traversal(tmp_path: Path) -> None: + archive = tmp_path / "unsafe.tar" + with tarfile.open(archive, "w") as bundle: + member = tarfile.TarInfo("../outside") + payload = b"unsafe" + member.size = len(payload) + bundle.addfile(member, io.BytesIO(payload)) + try: + safe_extract(archive, tmp_path / "output") + except ValueError as error: + assert "unsafe archive member" in str(error) + else: + raise AssertionError("unsafe archive was extracted") + assert not (tmp_path / "outside").exists() + + +def test_ci_actions_are_pinned_to_commits() -> None: + action_files = [ + *sorted((ROOT / ".github/workflows").glob("*.yml")), + *sorted((ROOT / ".github/actions").glob("*/action.yml")), + ] + remote_uses = [] + for path in action_files: + for line in path.read_text(encoding="utf-8").splitlines(): + match = re.match(r"\s*-?\s*uses:\s*([^\s#]+)", line) + if match and not match.group(1).startswith("./"): + remote_uses.append((path, match.group(1))) + assert remote_uses + for path, value in remote_uses: + assert re.fullmatch(r"[^@]+@[0-9a-f]{40}", value), f"unpinned action in {path}: {value}" + + +def test_python_requirements_are_hash_locked() -> None: + for path in sorted((ROOT / "requirements").glob("*.txt")): + content = path.read_text(encoding="utf-8") + assert "--require-hashes" in content + assert "--only-binary=:all:" in content + requirements = [ + line for line in content.splitlines() if line and not line.startswith(("#", "--")) + ] + assert requirements + for requirement in requirements: + assert re.search(r"==[^ ]+ --hash=sha256:[0-9a-f]{64}$", requirement), ( + f"unlocked requirement in {path}: {requirement}" + ) + + def test_clean_all_stays_within_repository(tmp_path: Path) -> None: fake_root = tmp_path / "repository" (fake_root / ".git").mkdir(parents=True) @@ -178,6 +509,37 @@ def test_clean_all_stays_within_repository(tmp_path: Path) -> None: str(ROOT / "scripts/clean.py"), "--root", str(fake_root), + "--path", + str(generated), ) assert not generated.exists() assert (dependency / "model.v").is_file() + + outside = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/clean.py"), + "--root", + str(fake_root), + "--path", + str(tmp_path / "outside"), + ], + text=True, + capture_output=True, + ) + assert outside.returncode != 0 + + external = tmp_path / "external" + external.mkdir() + link = fake_root / "build-link" + link.symlink_to(external, target_is_directory=True) + run( + sys.executable, + str(ROOT / "scripts/clean.py"), + "--root", + str(fake_root), + "--path", + str(link), + ) + assert not link.exists() + assert external.is_dir()