mirror of
https://github.com/retroSoC/retroSoC.git
synced 2026-11-04 22:32:36 +00:00
216 lines
7.7 KiB
Python
216 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
|
|
def run(command: Iterable[str], *, cwd: Path | None = None) -> None:
|
|
printable = " ".join(str(item) for item in command)
|
|
print(f"+ {printable}")
|
|
subprocess.run(
|
|
[str(item) for item in command],
|
|
cwd=cwd,
|
|
check=True,
|
|
)
|
|
|
|
|
|
def git_output(repo: Path, *args: str) -> str:
|
|
return subprocess.check_output(
|
|
["git", "-C", str(repo), *args], text=True, stderr=subprocess.STDOUT
|
|
).strip()
|
|
|
|
|
|
def ensure_git_commit(url: str, destination: Path, revision: str) -> None:
|
|
"""Fetch a locked historical object without switching the active checkout."""
|
|
if re.fullmatch(r"[0-9a-f]{40}", revision) is None:
|
|
raise ValueError("historical dependency requires a full Git commit revision")
|
|
destination = destination.resolve()
|
|
if not (destination / ".git").exists():
|
|
raise RuntimeError(f"prepare the current dependency checkout first: {destination}")
|
|
try:
|
|
git_output(destination, "cat-file", "-e", f"{revision}^{{commit}}")
|
|
except subprocess.CalledProcessError:
|
|
# No checkout/refspec: preserve HEAD, index, worktree and FETCH_HEAD.
|
|
run(("git", "fetch", "--no-tags", "--no-write-fetch-head", "--depth", "1",
|
|
url, revision), cwd=destination)
|
|
if git_output(destination, "rev-parse", f"{revision}^{{commit}}") != revision:
|
|
raise RuntimeError(f"historical dependency commit mismatch: {destination}: {revision}")
|
|
print(f"[dependency] historical object available: {destination.name}: {revision}")
|
|
|
|
|
|
def ensure_git_repo(
|
|
url: str,
|
|
destination: Path,
|
|
revision: str,
|
|
*,
|
|
recursive: bool = False,
|
|
submodules: tuple[str, ...] = (),
|
|
update: bool = False,
|
|
allow_dirty: bool = False,
|
|
) -> None:
|
|
destination = destination.resolve()
|
|
created = False
|
|
if not destination.exists():
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
run(("git", "init", str(destination)))
|
|
run(("git", "remote", "add", "origin", url), cwd=destination)
|
|
run(("git", "fetch", "--depth", "1", "origin", revision), cwd=destination)
|
|
run(("git", "checkout", "--detach", "FETCH_HEAD"), cwd=destination)
|
|
created = True
|
|
elif not (destination / ".git").exists():
|
|
raise RuntimeError(f"dependency path is not a Git repository: {destination}")
|
|
|
|
current = git_output(destination, "rev-parse", "HEAD")
|
|
dirty = git_output(destination, "status", "--porcelain")
|
|
if current != revision:
|
|
if not created and not update:
|
|
raise RuntimeError(
|
|
f"{destination} is at {current}, expected {revision}; rerun with --update"
|
|
)
|
|
if not created and dirty:
|
|
raise RuntimeError(f"refusing to update a dirty dependency: {destination}")
|
|
try:
|
|
git_output(destination, "cat-file", "-e", f"{revision}^{{commit}}")
|
|
except subprocess.CalledProcessError:
|
|
run(("git", "fetch", "--depth", "1", "origin", revision), cwd=destination)
|
|
run(("git", "checkout", "--detach", revision), cwd=destination)
|
|
elif dirty and not allow_dirty:
|
|
raise RuntimeError(f"pinned dependency has local changes: {destination}")
|
|
|
|
if recursive:
|
|
command: tuple[str, ...] = ("git", "submodule", "update", "--init", "--depth", "1")
|
|
if submodules:
|
|
command += ("--", *submodules)
|
|
else:
|
|
command += ("--recursive",)
|
|
run(command, cwd=destination)
|
|
print(f"[dependency] {destination.name}: {revision}")
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def download_file(
|
|
url: str,
|
|
destination: Path,
|
|
expected_sha256: str,
|
|
*,
|
|
update: bool = False,
|
|
retries: int = 3,
|
|
timeout: int = 30,
|
|
resume: bool = False,
|
|
) -> None:
|
|
destination = destination.resolve()
|
|
if destination.is_file():
|
|
actual = sha256(destination)
|
|
if actual == expected_sha256:
|
|
print(f"[dependency] archive verified: {destination}")
|
|
return
|
|
if not update:
|
|
raise RuntimeError(
|
|
f"checksum mismatch for {destination}: {actual}; rerun with --update"
|
|
)
|
|
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
last_error: Exception | None = None
|
|
partial = destination.with_name(f".{destination.name}.partial") if resume else None
|
|
for attempt in range(1, retries + 1):
|
|
if partial is None:
|
|
fd, temporary_name = tempfile.mkstemp(
|
|
prefix=f".{destination.name}.", dir=destination.parent
|
|
)
|
|
os.close(fd)
|
|
temporary = Path(temporary_name)
|
|
else:
|
|
temporary = partial
|
|
try:
|
|
received = temporary.stat().st_size if temporary.is_file() else 0
|
|
headers = {"User-Agent": "retroSoC-setup/1"}
|
|
if received:
|
|
headers["Range"] = f"bytes={received}-"
|
|
request = urllib.request.Request(url, headers=headers)
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
append = received != 0 and response.status == 206
|
|
with temporary.open("ab" if append else "wb") as output:
|
|
while chunk := response.read(1024 * 1024):
|
|
output.write(chunk)
|
|
actual = sha256(temporary)
|
|
if actual != expected_sha256:
|
|
if resume:
|
|
temporary.unlink(missing_ok=True)
|
|
raise RuntimeError(
|
|
f"checksum mismatch for {url}: {actual}, expected {expected_sha256}"
|
|
)
|
|
os.replace(temporary, destination)
|
|
print(f"[dependency] downloaded: {destination}")
|
|
return
|
|
except Exception as error:
|
|
last_error = error
|
|
if not resume:
|
|
temporary.unlink(missing_ok=True)
|
|
if attempt < retries:
|
|
time.sleep(attempt)
|
|
raise RuntimeError(f"failed to download {url}: {last_error}")
|
|
|
|
|
|
def download_file_from_urls(
|
|
urls: Iterable[str],
|
|
destination: Path,
|
|
expected_sha256: str,
|
|
*,
|
|
update: bool = False,
|
|
retries: int = 3,
|
|
timeout: int = 30,
|
|
resume: bool = False,
|
|
) -> None:
|
|
candidates = tuple(urls)
|
|
if not candidates:
|
|
raise ValueError("at least one download URL is required")
|
|
failures: list[str] = []
|
|
for url in candidates:
|
|
try:
|
|
download_file(
|
|
url,
|
|
destination,
|
|
expected_sha256,
|
|
update=update,
|
|
retries=retries,
|
|
timeout=timeout,
|
|
resume=resume,
|
|
)
|
|
return
|
|
except RuntimeError as error:
|
|
failures.append(str(error))
|
|
print(f"[dependency] mirror failed: {url}: {error}")
|
|
raise RuntimeError("all download mirrors failed: " + "; ".join(failures))
|
|
|
|
|
|
def atomic_write(path: Path, content: str) -> bool:
|
|
path = path.resolve()
|
|
if path.exists() and path.read_text(encoding="utf-8") == content:
|
|
return False
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
fd, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
|
stream.write(content)
|
|
os.replace(temporary_name, path)
|
|
except Exception:
|
|
Path(temporary_name).unlink(missing_ok=True)
|
|
raise
|
|
return True
|