diff --git a/profiles/base.py b/profiles/base.py new file mode 100644 index 0000000..b8a6bcc --- /dev/null +++ b/profiles/base.py @@ -0,0 +1,23 @@ +def profile(base): + # The base profile supplies defaults for every other profile, so it has no + # parent of its own (``base`` is None here). It deliberately omits the + # target-identifying core fields (arch/triple/libc); concrete profiles must + # provide those. + return Profile( + container_image="localhost/orchid-builder:latest", + options={ + # Install layout (where files land in the target system). + "prefix": "/usr", + "bindir": "/usr/bin", + "sbindir": "/usr/bin", + "libdir": "/usr/lib", + "libexecdir": "/usr/libexec", + "includedir": "/usr/include", + "sysconfdir": "/etc", + "localstatedir": "/var", + # Flags for tools built to run on the build machine (host). + "host_cflags": "-O2 -pipe", + "host_cxxflags": "-O2 -pipe", + "host_ldflags": "-Wl,-O1 -Wl,--sort-common -Wl,--as-needed", + }, + ) diff --git a/profiles/x86_64-glibc.py b/profiles/x86_64-glibc.py index bc2e0ef..ec91e3f 100644 --- a/profiles/x86_64-glibc.py +++ b/profiles/x86_64-glibc.py @@ -1,34 +1,17 @@ -def profile(): +def profile(base): arch = "x86_64" - libc = "glibc" - triple = f"{arch}-orchid-linux-gnu" - - host_cflags = "-O2 -pipe" - host_cxxflags = host_cflags - host_ldflags = "-Wl,-O1 -Wl,--sort-common -Wl,--as-needed" - - target_flags = " -march=x86-64-v3 -mtune=generic -fstack-clash-protection -fstack-protector-strong -fcf-protection" - cflags = host_cflags + target_flags - cxxflags = cflags - ldflags = host_ldflags + " -Wl,-z,now -Wl,-z,pack-relative-relocs" - - return { - "arch": arch, - "libc": libc, - "triple": triple, - "container_image": "localhost/orchid-builder:latest", - "host_cflags": host_cflags, - "host_cxxflags": host_cxxflags, - "host_ldflags": host_ldflags, - "cflags": cflags, - "cxxflags": cxxflags, - "ldflags": ldflags, - "prefix": "/usr", - "bindir": "/usr/bin", - "sbindir": "/usr/bin", - "libdir": "/usr/lib", - "libexecdir": "/usr/libexec", - "includedir": "/usr/include", - "sysconfdir": "/etc", - "localstatedir": "/var", - } + target_flags = ( + "-march=x86-64-v3 -mtune=generic -fstack-clash-protection " + "-fstack-protector-strong -fcf-protection" + ) + return Profile( + arch=arch, + triple=f"{arch}-orchid-linux-gnu", + options={ + "libc": "glibc", + # Target flags build on the base host flags. + "cflags": f"{base['host_cflags']} {target_flags}", + "cxxflags": f"{base['host_cflags']} {target_flags}", + "ldflags": f"{base['host_ldflags']} -Wl,-z,now -Wl,-z,pack-relative-relocs", + }, + ) diff --git a/profiles/x86_64-musl.py b/profiles/x86_64-musl.py index b05d344..d602851 100644 --- a/profiles/x86_64-musl.py +++ b/profiles/x86_64-musl.py @@ -1,34 +1,18 @@ -def profile(): +def profile(base): arch = "x86_64" libc = "musl" - triple = f"{arch}-orchid-linux-{libc}" - - host_cflags = "-O2 -pipe" - host_cxxflags = host_cflags - host_ldflags = "-Wl,-O1 -Wl,--sort-common -Wl,--as-needed" - - target_flags = " -march=x86-64-v3 -mtune=generic -fstack-clash-protection -fstack-protector-strong -fcf-protection" - cflags = host_cflags + target_flags - cxxflags = cflags - ldflags = host_ldflags + " -Wl,-z,now -Wl,-z,pack-relative-relocs" - - return { - "arch": arch, - "libc": libc, - "triple": triple, - "container_image": "localhost/orchid-builder:latest", - "host_cflags": host_cflags, - "host_cxxflags": host_cxxflags, - "host_ldflags": host_ldflags, - "cflags": cflags, - "cxxflags": cxxflags, - "ldflags": ldflags, - "prefix": "/usr", - "bindir": "/usr/bin", - "sbindir": "/usr/bin", - "libdir": "/usr/lib", - "libexecdir": "/usr/libexec", - "includedir": "/usr/include", - "sysconfdir": "/etc", - "localstatedir": "/var", - } + target_flags = ( + "-march=x86-64-v3 -mtune=generic -fstack-clash-protection " + "-fstack-protector-strong -fcf-protection" + ) + return Profile( + arch=arch, + triple=f"{arch}-orchid-linux-{libc}", + options={ + "libc": libc, + # Target flags build on the base host flags. + "cflags": f"{base['host_cflags']} {target_flags}", + "cxxflags": f"{base['host_cflags']} {target_flags}", + "ldflags": f"{base['host_ldflags']} -Wl,-z,now -Wl,-z,pack-relative-relocs", + }, + ) diff --git a/recipes/limine.py b/recipes/limine.py index 304504c..a905fc0 100644 --- a/recipes/limine.py +++ b/recipes/limine.py @@ -10,7 +10,7 @@ source = tarball( host_deps = ["autoconf", "automake", "binutils", "gcc"] deps = [profile["libc"]] -build_if = profile["arch"] in ("x86_64", "aarch64", "riscv64", "loongarch64") +build_if = profile.arch in ("x86_64", "aarch64", "riscv64", "loongarch64") arch_args = { "x86_64": [ @@ -25,8 +25,8 @@ arch_args = { } configure, build, install = autotools( - configure_args=["--enable-uefi-cd", *arch_args[profile["arch"]]], - configure_env={"TOOLCHAIN_FOR_TARGET": profile["triple"] + "-"}, + configure_args=["--enable-uefi-cd", *arch_args[profile.arch]], + configure_env={"TOOLCHAIN_FOR_TARGET": profile.triple + "-"}, ) subpackages = [ @@ -40,7 +40,7 @@ subpackages = [ ), ] -if profile["arch"] == "x86_64": +if profile.arch == "x86_64": subpackages.append( subpackage( "limine-bios", diff --git a/recipes/linux-headers.py b/recipes/linux-headers.py index 3121fae..64c8259 100644 --- a/recipes/linux-headers.py +++ b/recipes/linux-headers.py @@ -14,7 +14,7 @@ def build(self): # Stage the source into the (writable) build dir before invoking the kernel # build system, which is not happy with a read-only source tree. self.run("cp", "-rp", f"{self.source_dir}/.", self.build_dir) - arch = linux_archs.get(self.profile["arch"], self.profile["arch"]) + arch = linux_archs.get(self.arch, self.arch) self.run("make", "headers_install", f"ARCH={arch}") self.run( "find", diff --git a/recipes/linux/recipe.py b/recipes/linux/recipe.py index 9341ca3..dd8dd96 100644 --- a/recipes/linux/recipe.py +++ b/recipes/linux/recipe.py @@ -14,8 +14,8 @@ linux_subarchs = {"x86_64": "x86"} def _make(self, *extra): - arch = linux_archs.get(self.profile["arch"], self.profile["arch"]) - subarch = linux_subarchs.get(self.profile["arch"]) + arch = linux_archs.get(self.arch, self.arch) + subarch = linux_subarchs.get(self.arch) subarch_arg = (f"SUBARCH={subarch}",) if subarch else () return ( "make", @@ -30,7 +30,7 @@ def _make(self, *extra): def configure(self): self.run("cp", "-rp", f"{self.source_dir}/.", self.build_dir) self.run( - "cp", self.files / f"config.{self.profile['arch']}", self.build_dir / ".config" + "cp", self.files / f"config.{self.arch}", self.build_dir / ".config" ) self.run(*_make(self, "olddefconfig")) diff --git a/recipes/openssl.py b/recipes/openssl.py index dad9146..d2e9f3b 100644 --- a/recipes/openssl.py +++ b/recipes/openssl.py @@ -5,7 +5,7 @@ license = "Apache-2.0" url = "https://www.openssl.org/" source = tarball( url=f"https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz", - sha256="?", + sha256="002a2d6b30b58bf4bea46c43bdd96365aaf8daa6c428782aa4feee06da197df3", ) host_deps = ["binutils", "gcc"] deps = ["zlib"] @@ -18,9 +18,9 @@ ossl_targets = { def configure(self): - target = ossl_targets.get(self.profile["arch"]) + target = ossl_targets.get(self.arch) if target is None: - raise ValueError(f"openssl: unsupported arch {self.profile['arch']}") + raise ValueError(f"openssl: unsupported arch {self.arch}") self.run( self.source_dir / "Configure", @@ -48,6 +48,13 @@ def build(self): def install(self): + # OpenSSL's Makefile assigns `DESTDIR=` itself, so an environment DESTDIR is + # ignored; it must be passed as a make command-line variable to take effect. + # Without this, the (glibc) target libs install straight into the musl + # builder's /usr/lib and break it mid-install. self.run( - "make", "install_sw", "install_ssldirs", env={"DESTDIR": str(self.dest_dir)} + "make", + "install_sw", + "install_ssldirs", + f"DESTDIR={self.dest_dir}", ) diff --git a/recipes/pkgconf.py b/recipes/pkgconf.py index de7995c..1bd554a 100644 --- a/recipes/pkgconf.py +++ b/recipes/pkgconf.py @@ -1,11 +1,11 @@ -version = "3.3.3" +version = "2.5.1" revision = 1 description = "Lightweight pkg-config implementation" license = "ISC" url = "http://pkgconf.org/" source = tarball( url=f"https://distfiles.ariadne.space/pkgconf/pkgconf-{version}.tar.xz", - sha256="?", + sha256="cd05c9589b9f86ecf044c10a2269822bc9eb001eced2582cfffd658b0a50c243", ) host_deps = ["autoconf", "automake", "binutils", "gcc"] deps = [profile["libc"]] diff --git a/src/builder.py b/src/builder.py index 9dd685f..b0fb9fd 100644 --- a/src/builder.py +++ b/src/builder.py @@ -7,6 +7,7 @@ from src import apk, fetch, log from src.container import Container, Mount from src.context import RecipeContext from src.layout import Layout +from src.profile import Profile from src.plan import ( PHASE_STAMPS, Plan, @@ -65,7 +66,7 @@ def _build_path_env(host_deps_order: list[str], layout: Layout) -> str: def _container_for( - rs: RecipeSet, layout: Layout, profile: dict, r: Recipe + rs: RecipeSet, layout: Layout, profile: Profile, r: Recipe ) -> tuple[Container, list[str]]: host_order = transitive_host_deps(rs, r) name = _container_name("orchid", layout.build.name, r.kind, r.name) @@ -96,13 +97,13 @@ def _container_for( env = { "PATH": _build_path_env(host_order, layout), - "ORCHID_ARCH": profile["arch"], - "ORCHID_TRIPLE": profile["triple"], + "ORCHID_ARCH": profile.arch, + "ORCHID_TRIPLE": profile.triple, "ORCHID_JOBS": str(os.cpu_count() or 1), } c = Container( name=name, - image=profile["container_image"], + image=profile.container_image, mounts=mounts, tmpfs=tmpfs, network=False, @@ -212,7 +213,7 @@ def _sysroot_sync(c: Container, r: Recipe) -> None: def build_one( - rs: RecipeSet, layout: Layout, profile: dict, r: Recipe, *, forced: bool = False + rs: RecipeSet, layout: Layout, profile: Profile, r: Recipe, *, forced: bool = False ) -> None: log.info_field("recipe", _recipe_ref(r)) if forced: @@ -237,7 +238,7 @@ def build_one( if r.kind == "target": log.info_field("package", _recipe_ref(r)) _split_subpackages(c, r) - _package_target(c, r, profile["arch"]) + _package_target(c, r, profile.arch) else: log.info_field("finalize", _recipe_ref(r)) _finalize_host(c, layout, r) @@ -246,7 +247,7 @@ def build_one( log.ok_field("done", _recipe_ref(r)) -def execute(plan: Plan, rs: RecipeSet, layout: Layout, profile: dict) -> None: +def execute(plan: Plan, rs: RecipeSet, layout: Layout, profile: Profile) -> None: if not plan.order: log.info_field("plan", "nothing to do") return @@ -257,7 +258,7 @@ def execute(plan: Plan, rs: RecipeSet, layout: Layout, profile: dict) -> None: def install_to( layout: Layout, - profile: dict, + profile: Profile, dest: Path, pkgs: list[str], *, @@ -268,7 +269,7 @@ def install_to( return c = Container( name=_container_name("orchid", layout.build.name, "install"), - image=profile["container_image"], + image=profile.container_image, mounts=[ Mount(layout.pkgs_dir, "/pkgs", readonly=True), Mount(dest, "/sysroot", readonly=False), diff --git a/src/cli.py b/src/cli.py index 8e419ec..fa45573 100644 --- a/src/cli.py +++ b/src/cli.py @@ -50,7 +50,7 @@ def cmd_image(args) -> int: layout.ensure() prof = profile_mod.load_profile(layout) container.ensure_image( - layout.dockerfile, prof["container_image"], layout.image_hash_file + layout.dockerfile, prof.container_image, layout.image_hash_file ) return 0 @@ -115,7 +115,7 @@ def cmd_build(args) -> int: layout.ensure() prof, rs = _load(layout) container.ensure_image( - layout.dockerfile, prof["container_image"], layout.image_hash_file + layout.dockerfile, prof.container_image, layout.image_hash_file ) p = plan.build_plan(rs, layout, args.recipes or None, rebuild=args.rebuild) if args.dry_run: @@ -129,7 +129,7 @@ def cmd_install(args) -> int: layout.ensure() prof, rs = _load(layout) container.ensure_image( - layout.dockerfile, prof["container_image"], layout.image_hash_file + layout.dockerfile, prof.container_image, layout.image_hash_file ) dest = Path(args.dest).resolve() dest.mkdir(parents=True, exist_ok=True) diff --git a/src/context.py b/src/context.py index cf14d9e..55492fe 100644 --- a/src/context.py +++ b/src/context.py @@ -1,9 +1,11 @@ import os +from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import PurePosixPath from typing import Any from src.container import Container +from src.profile import Profile from src.recipe import Recipe @@ -12,7 +14,7 @@ class RecipeContext: """The `self` value passed to recipe phase functions.""" recipe: Recipe - profile: dict + profile: Profile container: Container jobs: int _dest_output: str | None = None @@ -73,13 +75,17 @@ class RecipeContext: def prefix(self) -> str: return f"/tools/{self.name}" if self.recipe.kind == "host" else "/usr" + @property + def options(self) -> Mapping[str, Any]: + return self.profile.options + @property def triple(self) -> str: - return self.profile["triple"] + return self.profile.triple @property def arch(self) -> str: - return self.profile["arch"] + return self.profile.arch def run( self, diff --git a/src/lib/autotools.py b/src/lib/autotools.py index 8f8c1f7..37a8172 100644 --- a/src/lib/autotools.py +++ b/src/lib/autotools.py @@ -1,5 +1,5 @@ def autotools_configure(self, extra_args=(), extra_env=None): - p = self.profile + p = self.options env = { "CFLAGS": p.get("cflags", ""), "CXXFLAGS": p.get("cxxflags", ""), @@ -9,7 +9,7 @@ def autotools_configure(self, extra_args=(), extra_env=None): env.update(extra_env) args = [ self.source_dir / "configure", - f"--host={p['triple']}", + f"--host={self.triple}", f"--with-sysroot={self.sysroot}", f"--prefix={p.get('prefix', '/usr')}", f"--sysconfdir={p.get('sysconfdir', '/etc')}", diff --git a/src/lib/cmake.py b/src/lib/cmake.py index b7d189c..6fd4af2 100644 --- a/src/lib/cmake.py +++ b/src/lib/cmake.py @@ -1,5 +1,5 @@ def cmake_configure(self, extra_args=(), extra_env=None, *, host=False): - p = self.profile + p = self.options if host: env = { "CFLAGS": p.get("host_cflags", ""), @@ -15,10 +15,10 @@ def cmake_configure(self, extra_args=(), extra_env=None, *, host=False): } toolchain = [ "-DCMAKE_SYSTEM_NAME=Linux", - f"-DCMAKE_SYSTEM_PROCESSOR={p['arch']}", + f"-DCMAKE_SYSTEM_PROCESSOR={self.arch}", f"-DCMAKE_SYSROOT={self.sysroot}", - f"-DCMAKE_C_COMPILER={p['triple']}-gcc", - f"-DCMAKE_CXX_COMPILER={p['triple']}-g++", + f"-DCMAKE_C_COMPILER={self.triple}-gcc", + f"-DCMAKE_CXX_COMPILER={self.triple}-g++", "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER", "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY", "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY", diff --git a/src/profile.py b/src/profile.py index 15bb636..2ae890e 100644 --- a/src/profile.py +++ b/src/profile.py @@ -1,42 +1,123 @@ import importlib.util +from collections.abc import Mapping +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any from src.layout import Layout -REQUIRED_KEYS = ("arch", "triple", "container_image") +# Core fields identify the build target. Every resolved profile must define +# them, and they are read as attributes (profile.arch) rather than options. +CORE_FIELDS = ("arch", "triple", "container_image") + +# Name of the profile that supplies defaults for every other profile. +BASE_PROFILE = "base" + +_MISSING = object() -def _load_module(path: Path, name: str): +@dataclass(frozen=True) +class Profile: + """A resolved build profile. + + The *core* fields (see ``CORE_FIELDS``) identify the build target and must + be present. Everything that tunes how packages are built from source -- + install layout, compiler flags, and free-form feature switches -- lives in + ``options`` and is looked up by key. + """ + + name: str = "" + arch: str = "" + triple: str = "" + container_image: str = "" + options: Mapping[str, Any] = field(default_factory=dict) + + def option(self, key: str, default: Any = _MISSING) -> Any: + """Return option ``key``; fall back to ``default`` or raise KeyError.""" + if key in self.options: + return self.options[key] + if default is not _MISSING: + return default + if key in CORE_FIELDS: + raise KeyError( + f"profile {self.name!r}: {key!r} is a core field; " + f"access it as profile.{key}, not as an option" + ) + raise KeyError(f"profile {self.name!r}: option {key!r} is not defined") + + def __getitem__(self, key: str) -> Any: + return self.option(key) + + def get(self, key: str, default: Any = None) -> Any: + return self.option(key, default) + + def overlay(self, child: "Profile") -> "Profile": + """Layer ``child`` over ``self`` (the base) and return the result. + + Core fields fall back to the base wherever the child leaves them empty; + options are merged key-by-key, with the child winning. + """ + return Profile( + name=child.name or self.name, + arch=child.arch or self.arch, + triple=child.triple or self.triple, + container_image=child.container_image or self.container_image, + options={**self.options, **child.options}, + ) + + +def _load_module(path: Path, name: str, ns: dict): spec = importlib.util.spec_from_file_location(name, path) if spec is None or spec.loader is None: raise RuntimeError(f"cannot load profile module {path}") mod = importlib.util.module_from_spec(spec) + mod.__dict__.update(ns) spec.loader.exec_module(mod) return mod -def load_profile(layout: Layout) -> dict[str, Any]: +def _eval_profile(path: Path, name: str, base: "Profile | None") -> Profile: + mod = _load_module(path, f"orchid_profile_{name}", {"Profile": Profile}) + if not hasattr(mod, "profile"): + raise AttributeError(f"profile {name!r}: must define profile(base)") + spec = mod.profile(base) + if not isinstance(spec, Profile): + raise TypeError(f"profile {name!r}: profile(base) must return a Profile(...)") + if not isinstance(spec.options, Mapping) or any( + not isinstance(k, str) for k in spec.options + ): + raise TypeError(f"profile {name!r}: options must be a dict with string keys") + return replace(spec, name=name) + + +def load_profile(layout: Layout) -> Profile: link = layout.profile_link if not link.exists(): raise FileNotFoundError(f"{link} missing a profile") target = link.resolve() if not target.is_file(): - raise FileNotFoundError(f"profile {target.name}: missing config.py") - mod = _load_module(target, f"orchid_profile_{target.name}") - if not hasattr(mod, "profile"): - raise AttributeError(f"profile {target.name}: config.py must define profile()") - data = mod.profile() - if not isinstance(data, dict): - raise TypeError(f"profile {target.name}: profile() must return a dict") - missing = [k for k in REQUIRED_KEYS if k not in data] + raise FileNotFoundError(f"profile {target.name}: missing config") + + # The base profile is optional, but supplies defaults when present. + base = Profile(name=BASE_PROFILE) + base_file = layout.profiles_dir / f"{BASE_PROFILE}.py" + if base_file.is_file(): + base = _eval_profile(base_file, BASE_PROFILE, None) + + child = _eval_profile(target, target.stem, base) + resolved = base.overlay(child) + + missing = [f for f in CORE_FIELDS if not getattr(resolved, f)] if missing: - raise ValueError(f"profile {target.name}: missing keys {missing}") - data["__name__"] = target.name - return data + raise ValueError(f"profile {resolved.name!r}: missing core fields {missing}") + return resolved def init_build_dir(build: Path, repo: Path, profile_name: str) -> None: + if profile_name == BASE_PROFILE: + raise ValueError( + f"{BASE_PROFILE!r} only supplies defaults and cannot be built directly" + ) profile_src = repo / "profiles" / (profile_name + ".py") if not (profile_src).is_file(): raise FileNotFoundError(f"profile {profile_name!r} not found at {profile_src}") diff --git a/src/recipe.py b/src/recipe.py index f781721..4de125f 100644 --- a/src/recipe.py +++ b/src/recipe.py @@ -8,6 +8,7 @@ from typing import Any, Callable from src import source as src_mod from src.layout import Layout +from src.profile import Profile from src.source import Subpackage, Tarball, subpackage, tarball, git PHASE_NAMES = ("prepare", "configure", "build", "install") @@ -79,7 +80,7 @@ def _lib_symbols() -> dict: # Symbols which are injected into the recipe -def _builtins(profile: dict) -> dict: +def _builtins(profile: Profile) -> dict: return { **_lib_symbols(), "tarball": tarball, @@ -112,7 +113,12 @@ def _plain_field(mod, recipe_name: str, field_name: str) -> str: def _load_one( - name: str, kind: str, recipe_file: Path, recipe_dir: Path, pure: bool, profile: dict + name: str, + kind: str, + recipe_file: Path, + recipe_dir: Path, + pure: bool, + profile: Profile, ) -> Recipe | None: mod = _load_module(recipe_file, _builtins(profile)) @@ -208,7 +214,7 @@ def _load_one( ) -def _discover(root: Path, kind: str, profile: dict) -> dict[str, Recipe]: +def _discover(root: Path, kind: str, profile: Profile) -> dict[str, Recipe]: out: dict[str, Recipe] = {} if not root.is_dir(): return out @@ -252,7 +258,7 @@ class RecipeSet: return [*self.target.values(), *self.host.values()] -def load_recipes(layout: Layout, profile: dict) -> RecipeSet: +def load_recipes(layout: Layout, profile: Profile) -> RecipeSet: target = _discover(layout.recipes_dir, "target", profile) host = _discover(layout.host_recipes_dir, "host", profile) return RecipeSet(target=target, host=host)