More stuff

This commit is contained in:
2026-05-18 00:49:24 +02:00
parent 695f30d678
commit 98f3fee099
15 changed files with 333 additions and 132 deletions
+3 -2
View File
@@ -6,6 +6,7 @@ signing_key = "build/keys/distro.rsa"
signing_pubkey = "build/keys/distro.rsa.pub" signing_pubkey = "build/keys/distro.rsa.pub"
target_arch = "x86_64" target_arch = "x86_64"
libc = "musl"
host_cflags = "-O2 -pipe" host_cflags = "-O2 -pipe"
host_cxxflags = "" host_cxxflags = ""
@@ -22,8 +23,8 @@ if target_arch == "x86_64":
target_ldflags += " -Wl,-z,pack-relative-relocs" target_ldflags += " -Wl,-z,pack-relative-relocs"
options = { options = {
"libc": "musl", "libc": libc,
"target_triple": "x86_64-linux-musl", "target_triple": target_arch + "-linux-" + libc,
"host_cflags": host_cflags, "host_cflags": host_cflags,
"host_cxxflags": host_cxxflags, "host_cxxflags": host_cxxflags,
"host_ldflags": host_ldflags, "host_ldflags": host_ldflags,
+5 -6
View File
@@ -13,12 +13,11 @@ source = {
host_deps = [] host_deps = []
def configure(ctx): def configure(ctx):
triple = ctx.options["target_triple"]
ctx.run([ ctx.run([
ctx.source_dir + "/configure", ctx.source_dir + "/configure",
"--prefix=" + ctx.prefix, "--prefix=" + ctx.prefix,
"--target=" + triple, "--target=" + OPTIONS.target_triple,
"--with-sysroot=" + ctx.prefix + "/" + triple, "--with-sysroot=" + ctx.prefix + "/" + OPTIONS.target_triple,
"--disable-nls", "--disable-nls",
"--disable-werror", "--disable-werror",
"--enable-deterministic-archives", "--enable-deterministic-archives",
@@ -29,9 +28,9 @@ def configure(ctx):
# gprofng's libcollector does not build against musl/recent gcc. # gprofng's libcollector does not build against musl/recent gcc.
"--disable-gprofng", "--disable-gprofng",
], env = { ], env = {
"CFLAGS": ctx.options["host_cflags"], "CFLAGS": OPTIONS.host_cflags,
"CXXFLAGS": ctx.options["host_cxxflags"], "CXXFLAGS": OPTIONS.host_cxxflags,
"LDFLAGS": ctx.options["host_ldflags"], "LDFLAGS": OPTIONS.host_ldflags,
}) })
def build(ctx): def build(ctx):
+5 -6
View File
@@ -13,12 +13,11 @@ source = {
host_deps = ["binutils"] host_deps = ["binutils"]
def configure(ctx): def configure(ctx):
triple = ctx.options["target_triple"]
ctx.run([ ctx.run([
ctx.source_dir + "/configure", ctx.source_dir + "/configure",
"--prefix=" + ctx.prefix, "--prefix=" + ctx.prefix,
"--target=" + triple, "--target=" + OPTIONS.target_triple,
"--with-sysroot=" + ctx.prefix + "/" + triple, "--with-sysroot=" + ctx.prefix + "/" + OPTIONS.target_triple,
"--without-headers", "--without-headers",
"--with-newlib", "--with-newlib",
"--enable-languages=c,c++", "--enable-languages=c,c++",
@@ -34,9 +33,9 @@ def configure(ctx):
"--disable-libvtv", "--disable-libvtv",
"--disable-multilib", "--disable-multilib",
], env = { ], env = {
"CFLAGS": ctx.options["host_cflags"], "CFLAGS": OPTIONS.host_cflags,
"CXXFLAGS": ctx.options["host_cxxflags"], "CXXFLAGS": OPTIONS.host_cxxflags,
"LDFLAGS": ctx.options["host_ldflags"], "LDFLAGS": OPTIONS.host_ldflags,
}) })
def build(ctx): def build(ctx):
+29 -12
View File
@@ -1,14 +1,35 @@
# Commonly used helpers, auto-loaded into every recipe. # Commonly used helpers, auto-loaded into every recipe.
def autotools_configure(ctx, extra_args = []): def _toolchain_env(ctx):
sysroot_flag = " --sysroot=" + ctx.sysroot
return {
"CFLAGS": OPTIONS.cflags + sysroot_flag,
"CXXFLAGS": OPTIONS.cxxflags + sysroot_flag,
"LDFLAGS": OPTIONS.ldflags + sysroot_flag,
}
# Autotools
def autotools_configure(ctx, extra_args = [], extra_env = {}):
args = [ args = [
ctx.source_dir + "/configure", ctx.source_dir + "/configure",
"--prefix=" + ctx.prefix, "--prefix=" + ctx.prefix,
"--sysconfdir=/etc", "--sysconfdir=/etc",
"--localstatedir=/var", "--localstatedir=/var",
"--bindir=" + ctx.prefix + "/bin",
"--sbindir=" + ctx.prefix + "/bin",
"--libdir=" + ctx.prefix + "/lib",
"--with-sysroot=" + ctx.sysroot,
"--disable-static",
"--enable-shared",
] ]
args.append("--host=" + OPTIONS.target_triple)
args.extend(extra_args) args.extend(extra_args)
ctx.run(args, env = _toolchain_env(ctx))
envs = _toolchain_env(ctx)
envs.update(extra_env)
ctx.run(args, env = envs)
def autotools_build(ctx, extra_args = []): def autotools_build(ctx, extra_args = []):
args = ["make", "-j" + str(ctx.jobs)] args = ["make", "-j" + str(ctx.jobs)]
@@ -25,15 +46,17 @@ def autotools_install(ctx, pkg, extra_args = []):
args.extend(extra_args) args.extend(extra_args)
ctx.run(args) ctx.run(args)
def autotools(configure_args = [], build_args = [], install_args = []): def autotools(configure_args = [], configure_env = [], build_args = [], install_args = []):
def _configure(ctx): def _configure(ctx):
autotools_configure(ctx, extra_args = configure_args) autotools_configure(ctx, extra_args = configure_args, extra_env = configure_env)
def _build(ctx): def _build(ctx):
autotools_build(ctx, extra_args = build_args) autotools_build(ctx, extra_args = build_args)
def _install(ctx, pkg): def _install(ctx, pkg):
autotools_install(ctx, pkg, extra_args = install_args) autotools_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install return _configure, _build, _install
# Meson
def meson_configure(ctx, extra_args = []): def meson_configure(ctx, extra_args = []):
args = [ args = [
"meson", "meson",
@@ -51,7 +74,6 @@ def meson_build(ctx):
def meson_install(ctx, pkg): def meson_install(ctx, pkg):
ctx.run(["meson", "install", "-C", ctx.build_dir, "--destdir", pkg.destdir]) ctx.run(["meson", "install", "-C", ctx.build_dir, "--destdir", pkg.destdir])
def meson(configure_args = [], build_args = [], install_args = []): def meson(configure_args = [], build_args = [], install_args = []):
def _configure(ctx): def _configure(ctx):
meson_configure(ctx, extra_args = configure_args) meson_configure(ctx, extra_args = configure_args)
@@ -61,6 +83,8 @@ def meson(configure_args = [], build_args = [], install_args = []):
meson_install(ctx, pkg, extra_args = install_args) meson_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install return _configure, _build, _install
# Make
def make(ctx, target = None, extra_args = []): def make(ctx, target = None, extra_args = []):
args = ["make", "-C", ctx.source_dir, "O=" + ctx.build_dir, args = ["make", "-C", ctx.source_dir, "O=" + ctx.build_dir,
"-j" + str(ctx.jobs)] "-j" + str(ctx.jobs)]
@@ -73,10 +97,3 @@ def make_install(ctx, pkg, extra_args = []):
args = ["make", "-C", ctx.build_dir, "DESTDIR=" + pkg.destdir, "install"] args = ["make", "-C", ctx.build_dir, "DESTDIR=" + pkg.destdir, "install"]
args.extend(extra_args) args.extend(extra_args)
ctx.run(args) ctx.run(args)
def _toolchain_env(ctx):
env = {}
for key, var in [("cflags", "CFLAGS"), ("cxxflags", "CXXFLAGS"), ("ldflags", "LDFLAGS")]:
if key in ctx.options:
env[var] = ctx.options[key]
return env
+25
View File
@@ -0,0 +1,25 @@
name = "limine"
version = "12.2.0"
revision = 1
description = "Modern, secure, portable, multiprotocol bootloader and boot manager"
license = "BSD-2-Clause"
source = {
"url": f"https://github.com/Limine-Bootloader/Limine/releases/download/v{version}/limine-{version}.tar.gz",
"sha256": "db8a119878cfeead63c0a78236c577c40539c5759496950ea0ed32a6cf567865",
"strip_components": 1,
}
host_deps = ["binutils", "gcc"]
deps = [OPTIONS.libc]
def configure(ctx):
toolchain = OPTIONS.target_triple + "-"
autotools_configure(ctx, extra_env = {
"TOOLCHAIN_FOR_TARGET": toolchain,
"LD_FOR_TARGET": toolchain + "ld",
"OBJCOPY_FOR_TARGET": toolchain + "objcopy",
"OBJDUMP_FOR_TARGET": toolchain + "objdump",
})
_, build, install = autotools()
+1 -2
View File
@@ -13,13 +13,12 @@ source = {
host_deps = ["binutils", "gcc"] host_deps = ["binutils", "gcc"]
def _make_args(ctx, *args): def _make_args(ctx, *args):
triple = ctx.options["target_triple"]
result = [ result = [
"make", "make",
"-C", ctx.source_dir, "-C", ctx.source_dir,
"O=" + ctx.build_dir, "O=" + ctx.build_dir,
"ARCH=x86_64", "ARCH=x86_64",
f"CROSS_COMPILE={triple}-", "CROSS_COMPILE=" + OPTIONS.target_triple + "-",
"-j" + str(ctx.jobs), "-j" + str(ctx.jobs),
] ]
result.extend(args) result.extend(args)
+4 -5
View File
@@ -13,18 +13,17 @@ source = {
host_deps = ["binutils", "gcc"] host_deps = ["binutils", "gcc"]
def configure(ctx): def configure(ctx):
triple = ctx.options["target_triple"]
ctx.run( ctx.run(
[ [
ctx.source_dir + "/configure", ctx.source_dir + "/configure",
"--prefix=/usr", "--prefix=/usr",
"--syslibdir=/lib", "--syslibdir=/lib",
"--target=" + triple, "--target=" + OPTIONS.target_triple,
], ],
env = { env = {
"CC": triple + "-gcc", "CC": OPTIONS.target_triple + "-gcc",
"CFLAGS": ctx.options["cflags"], "CFLAGS": OPTIONS.cflags,
"LDFLAGS": ctx.options["ldflags"], "LDFLAGS": OPTIONS.ldflags,
}, },
) )
+1 -1
View File
@@ -38,7 +38,7 @@ pub fn mkpkg_plan(
"--info".to_owned(), "--info".to_owned(),
format!("origin:{}", package.recipe), format!("origin:{}", package.recipe),
]; ];
for dep in &package.run_deps { for dep in &package.deps {
args.push("--info".to_owned()); args.push("--info".to_owned());
args.push(format!("depends:{dep}")); args.push(format!("depends:{dep}"));
} }
+44 -16
View File
@@ -112,6 +112,16 @@ impl Builder {
self.preflight_container()?; self.preflight_container()?;
let repo = self.pkgs_dir(); let repo = self.pkgs_dir();
fs::create_dir_all(&repo)?; fs::create_dir_all(&repo)?;
// Nothing to index yet — leave it alone so callers don't trip on a
// failing `*.apk` glob.
let has_apks = fs::read_dir(&repo)?.any(|e| {
e.ok()
.and_then(|e| e.path().extension().map(|x| x == "apk"))
.unwrap_or(false)
});
if !has_apks {
return Ok(());
}
log::step( log::step(
"index", "index",
&format!("signing repository at {}", repo.display()), &format!("signing repository at {}", repo.display()),
@@ -121,14 +131,14 @@ impl Builder {
if !key.exists() || !pubkey.exists() { if !key.exists() || !pubkey.exists() {
bail!("signing key is not configured or missing; run `distro init-key` first"); bail!("signing key is not configured or missing; run `distro init-key` first");
} }
let index_name = "APKINDEX.adb"; let index_name = "APKINDEX.tar.gz";
let status = Command::new(&self.config.container_runtime) let status = Command::new(&self.config.container_runtime)
.arg("run") .arg("run")
.arg("--rm") .arg("--rm")
.arg("-v") .arg("-v")
.arg(format!("{}:/repo", repo.display())) .arg(format!("{}:/repo", repo.display()))
.arg("-v") .arg("-v")
.arg(format!("{}:/keys/private.rsa:ro", key.display())) .arg(format!("{}:/keys/distro.rsa:ro", key.display()))
.arg("-v") .arg("-v")
.arg(format!( .arg(format!(
"{}:/etc/apk/keys/distro.rsa.pub:ro", "{}:/etc/apk/keys/distro.rsa.pub:ro",
@@ -138,7 +148,7 @@ impl Builder {
.arg("/bin/sh") .arg("/bin/sh")
.arg("-lc") .arg("-lc")
.arg(format!( .arg(format!(
"cd /repo && apk --sign-key /keys/private.rsa mkndx -o {index_name} *.apk" "cd /repo && apk --sign-key /keys/distro.rsa mkndx -o {index_name} *.apk"
)) ))
.status() .status()
.context("failed to run repository index command")?; .context("failed to run repository index command")?;
@@ -207,7 +217,7 @@ impl Builder {
.arg("-v") .arg("-v")
.arg(format!("{}:/rootfs", root.display())) .arg(format!("{}:/rootfs", root.display()))
.arg("-v") .arg("-v")
.arg(format!("{}:/repo:ro", self.pkgs_dir().display())) .arg(format!("{}:/repo:ro", self.pkgs_root().display()))
.arg("-v") .arg("-v")
.arg(format!( .arg(format!(
"{}:/etc/apk/keys/distro.rsa.pub:ro", "{}:/etc/apk/keys/distro.rsa.pub:ro",
@@ -217,8 +227,11 @@ impl Builder {
.arg("apk") .arg("apk")
.arg("--root") .arg("--root")
.arg("/rootfs") .arg("/rootfs")
.arg("--keys-dir")
.arg("/etc/apk/keys")
.arg("--initdb")
.arg("--repository") .arg("--repository")
.arg("/repo/APKINDEX.adb") .arg("/repo")
.arg("add") .arg("add")
.args(packages) .args(packages)
.status() .status()
@@ -315,7 +328,7 @@ impl Builder {
&source_dir, &source_dir,
&build_dir, &build_dir,
dest_dir, dest_dir,
sysroot.as_deref(), sysroot.as_ref().map(|s| s.path()),
)?; )?;
self.apk_mkpkg(output, dest_dir)?; self.apk_mkpkg(output, dest_dir)?;
@@ -466,11 +479,11 @@ impl Builder {
.arg("-v") .arg("-v")
.arg(format!("{}:/out", repo.display())) .arg(format!("{}:/out", repo.display()))
.arg("-v") .arg("-v")
.arg(format!("{}:/keys/private.rsa:ro", signing_key.display())) .arg(format!("{}:/keys/distro.rsa:ro", signing_key.display()))
.arg(&self.config.container_image) .arg(&self.config.container_image)
.arg("apk") .arg("apk")
.arg("--sign-key") .arg("--sign-key")
.arg("/keys/private.rsa") .arg("/keys/distro.rsa")
.args(plan.args) .args(plan.args)
.status() .status()
.context("failed to run apk mkpkg command")?; .context("failed to run apk mkpkg command")?;
@@ -627,11 +640,11 @@ impl Builder {
Ok(Some(sandbox)) Ok(Some(sandbox))
} }
fn materialize_sysroot(&self, recipe: &Recipe) -> Result<Option<PathBuf>> { fn materialize_sysroot(&self, recipe: &Recipe) -> Result<Option<tempfile::TempDir>> {
let mut deps: Vec<String> = recipe let mut deps: Vec<String> = recipe
.build_deps .build_deps
.iter() .iter()
.chain(recipe.run_deps.iter()) .chain(recipe.deps.iter())
.cloned() .cloned()
.collect(); .collect();
deps.sort(); deps.sort();
@@ -639,23 +652,29 @@ impl Builder {
if deps.is_empty() { if deps.is_empty() {
return Ok(None); return Ok(None);
} }
// The local repo index must be present and current so apk can resolve
// and verify the just-built target packages.
self.repo_index()?;
let pubkey = self.abs_config_path(&self.config.signing_pubkey); let pubkey = self.abs_config_path(&self.config.signing_pubkey);
if !pubkey.exists() { if !pubkey.exists() {
bail!("target dependency sysroot requires a configured public signing key"); bail!("target dependency sysroot requires a configured public signing key");
} }
let sysroot = self.repo.join("build/sysroots").join(&recipe.id); fs::create_dir_all(self.repo.join("build"))?;
let sysroot = tempfile::Builder::new()
.prefix(&format!("sysroot-{}-", recipe.id))
.tempdir_in(self.repo.join("build"))
.context("failed to create sysroot tempdir")?;
log::info( log::info(
"sysroot", "sysroot",
&format!("{} <- [{}]", recipe.id, deps.join(", ")), &format!("{} <- [{}]", recipe.id, deps.join(", ")),
); );
Self::recreate(&sysroot)?;
let status = Command::new(&self.config.container_runtime) let status = Command::new(&self.config.container_runtime)
.arg("run") .arg("run")
.arg("--rm") .arg("--rm")
.arg("-v") .arg("-v")
.arg(format!("{}:/sysroot", sysroot.display())) .arg(format!("{}:/sysroot", sysroot.path().display()))
.arg("-v") .arg("-v")
.arg(format!("{}:/repo:ro", self.pkgs_dir().display())) .arg(format!("{}:/repo:ro", self.pkgs_root().display()))
.arg("-v") .arg("-v")
.arg(format!( .arg(format!(
"{}:/etc/apk/keys/distro.rsa.pub:ro", "{}:/etc/apk/keys/distro.rsa.pub:ro",
@@ -665,8 +684,11 @@ impl Builder {
.arg("apk") .arg("apk")
.arg("--root") .arg("--root")
.arg("/sysroot") .arg("/sysroot")
.arg("--keys-dir")
.arg("/etc/apk/keys")
.arg("--initdb")
.arg("--repository") .arg("--repository")
.arg("/repo/APKINDEX.adb") .arg("/repo")
.arg("add") .arg("add")
.args(&deps) .args(&deps)
.status() .status()
@@ -817,9 +839,15 @@ impl Builder {
fn host_pkg_dir_by_id(&self, host_recipe_id: &str) -> PathBuf { fn host_pkg_dir_by_id(&self, host_recipe_id: &str) -> PathBuf {
self.repo.join("build/host-pkgs").join(host_recipe_id) self.repo.join("build/host-pkgs").join(host_recipe_id)
} }
fn pkgs_dir(&self) -> PathBuf { /// Root of the target package repo. apk treats this as the repo root and
/// expects `<root>/<arch>/APKINDEX.tar.gz` underneath.
fn pkgs_root(&self) -> PathBuf {
self.repo.join("build/pkgs") self.repo.join("build/pkgs")
} }
/// Arch-specific package directory: where .apk files and the index live.
fn pkgs_dir(&self) -> PathBuf {
self.pkgs_root().join(&self.config.target_arch)
}
fn manifest_path(&self, output_key: &str) -> PathBuf { fn manifest_path(&self, output_key: &str) -> PathBuf {
// Output keys may contain `:` (e.g. `host:gcc`); the manifest file // Output keys may contain `:` (e.g. `host:gcc`); the manifest file
// name uses the filesystem-safe slug form instead. // name uses the filesystem-safe slug form instead.
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::starlark_eval::{eval_file, get_json_map, get_string, get_string_default}; use crate::starlark::{eval_file, get_json_map, get_string, get_string_default};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue; use serde_json::Value as JsonValue;
+1 -1
View File
@@ -29,7 +29,7 @@ impl PackageGraph {
let mut edges = output.all_target_deps(); let mut edges = output.all_target_deps();
if output.name == recipe.name { if output.name == recipe.name {
edges.extend(recipe.build_deps.iter().cloned()); edges.extend(recipe.build_deps.iter().cloned());
edges.extend(recipe.run_deps.iter().cloned()); edges.extend(recipe.deps.iter().cloned());
} }
match output.kind { match output.kind {
PackageKind::Host => { PackageKind::Host => {
+1 -1
View File
@@ -9,7 +9,7 @@ mod phase;
mod recipe; mod recipe;
mod rewrite; mod rewrite;
mod source; mod source;
mod starlark_eval; mod starlark;
mod update; mod update;
use anyhow::Result; use anyhow::Result;
+3 -4
View File
@@ -1,5 +1,5 @@
use crate::config::Config; use crate::config::Config;
use crate::starlark_eval::{eval_content_with_extra, options_literal, prepend_common_lib_load}; use crate::starlark::{eval_content_with_extra, prepend_common_lib_load};
use allocative::Allocative; use allocative::Allocative;
use anyhow::{Result, anyhow, bail}; use anyhow::{Result, anyhow, bail};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@@ -135,14 +135,13 @@ pub fn collect_phase_commands(
let raw = std::fs::read_to_string(recipe_path)?; let raw = std::fs::read_to_string(recipe_path)?;
// Auto-load helpers from `lib/common.star` so recipes never need an // Auto-load helpers from `lib/common.star` so recipes never need an
// explicit `load()` for the canonical helpers. // explicit `load()` for the canonical helpers.
let mut content = prepend_common_lib_load(Some(repo_root), &raw)?; let mut content = prepend_common_lib_load(Some(repo_root), Some(config), &raw)?;
let jobs = std::thread::available_parallelism() let jobs = std::thread::available_parallelism()
.map(|j| j.get()) .map(|j| j.get())
.unwrap_or(1); .unwrap_or(1);
let options = options_literal(config)?;
let source_dir_expr = source_dir_literal(&env.source_dir)?; let source_dir_expr = source_dir_literal(&env.source_dir)?;
let ctx_literal = format!( let ctx_literal = format!(
"struct(run = ctx_run, install = ctx_install, jobs = {jobs}, options = {options}, \ "struct(run = ctx_run, install = ctx_install, jobs = {jobs}, \
source_dir = {sd}, build_dir = {bd}, dest_dir = {dd}, prefix = {pf}, sysroot = {sr})", source_dir = {sd}, build_dir = {bd}, dest_dir = {dd}, prefix = {pf}, sysroot = {sr})",
sd = source_dir_expr, sd = source_dir_expr,
bd = serde_json::to_string(env.build_dir)?, bd = serde_json::to_string(env.build_dir)?,
+12 -25
View File
@@ -1,5 +1,5 @@
use crate::config::Config; use crate::config::Config;
use crate::starlark_eval::{ use crate::starlark::{
eval_content, get_i32_default, get_json, get_string, get_string_default, get_string_vec, eval_content, get_i32_default, get_json, get_string, get_string_default, get_string_vec,
has_name, prepend_common_lib_load, has_name, prepend_common_lib_load,
}; };
@@ -63,7 +63,7 @@ pub struct OutputPackage {
pub build_deps: Vec<String>, pub build_deps: Vec<String>,
/// Target packages declared as runtime dependencies (apk `depends:`). /// Target packages declared as runtime dependencies (apk `depends:`).
/// Also installed into the sysroot so the recipe can link against them. /// Also installed into the sysroot so the recipe can link against them.
pub run_deps: Vec<String>, pub deps: Vec<String>,
pub install_fn: String, pub install_fn: String,
} }
@@ -76,7 +76,7 @@ impl OutputPackage {
/// and to compute the build graph). /// and to compute the build graph).
pub fn all_target_deps(&self) -> Vec<String> { pub fn all_target_deps(&self) -> Vec<String> {
let mut out = self.build_deps.clone(); let mut out = self.build_deps.clone();
out.extend(self.run_deps.iter().cloned()); out.extend(self.deps.iter().cloned());
out out
} }
} }
@@ -95,7 +95,7 @@ pub struct Recipe {
pub sources: Vec<Source>, pub sources: Vec<Source>,
pub host_deps: Vec<String>, pub host_deps: Vec<String>,
pub build_deps: Vec<String>, pub build_deps: Vec<String>,
pub run_deps: Vec<String>, pub deps: Vec<String>,
pub outputs: Vec<OutputPackage>, pub outputs: Vec<OutputPackage>,
pub configure_fn: Option<String>, pub configure_fn: Option<String>,
pub build_fn: Option<String>, pub build_fn: Option<String>,
@@ -201,7 +201,7 @@ impl Recipe {
// Auto-load helpers from `lib/common.star` so recipes never need an // Auto-load helpers from `lib/common.star` so recipes never need an
// explicit `load()` for the canonical helpers. // explicit `load()` for the canonical helpers.
let raw = std::fs::read_to_string(path)?; let raw = std::fs::read_to_string(path)?;
let content = prepend_common_lib_load(Some(repo_root), &raw)?; let content = prepend_common_lib_load(Some(repo_root), Some(config), &raw)?;
let module = eval_content( let module = eval_content(
path, path,
content, content,
@@ -224,13 +224,7 @@ impl Recipe {
let description = get_string_default(&module, "description", "???")?; let description = get_string_default(&module, "description", "???")?;
let license = get_string_default(&module, "license", "???")?; let license = get_string_default(&module, "license", "???")?;
let build_deps = get_string_vec(&module, "build_deps")?; let build_deps = get_string_vec(&module, "build_deps")?;
let run_deps = get_string_vec(&module, "run_deps")?; let deps = get_string_vec(&module, "deps")?;
if has_name(&module, "deps") {
bail!(
"recipe `{id}` uses removed `deps`; split it into `build_deps` \
(sysroot only) and `run_deps` (apk depends + sysroot)"
);
}
let host_deps = get_string_vec(&module, "host_deps")?; let host_deps = get_string_vec(&module, "host_deps")?;
let sources = parse_sources(get_json(&module, "sources")?, get_json(&module, "source")?)?; let sources = parse_sources(get_json(&module, "sources")?, get_json(&module, "source")?)?;
let subpackages = parse_subpackages(get_json(&module, "subpackages")?)?; let subpackages = parse_subpackages(get_json(&module, "subpackages")?)?;
@@ -245,7 +239,7 @@ impl Recipe {
description: description.clone(), description: description.clone(),
license: license.clone(), license: license.clone(),
build_deps: build_deps.clone(), build_deps: build_deps.clone(),
run_deps: run_deps.clone(), deps: deps.clone(),
install_fn: "install".to_owned(), install_fn: "install".to_owned(),
}); });
for subpkg in subpackages { for subpkg in subpackages {
@@ -254,12 +248,6 @@ impl Recipe {
.and_then(JsonValue::as_str) .and_then(JsonValue::as_str)
.ok_or_else(|| anyhow!("subpackage in `{name}` is missing string `name`"))? .ok_or_else(|| anyhow!("subpackage in `{name}` is missing string `name`"))?
.to_owned(); .to_owned();
if subpkg.contains_key("deps") {
bail!(
"subpackage `{sub_name}` in `{id}` uses removed `deps`; use \
`build_deps` and/or `run_deps`"
);
}
outputs.push(OutputPackage { outputs.push(OutputPackage {
name: sub_name, name: sub_name,
recipe: recipe_key.clone(), recipe: recipe_key.clone(),
@@ -278,8 +266,7 @@ impl Recipe {
.to_owned(), .to_owned(),
build_deps: json_string_list(subpkg.get("build_deps"), "subpackage build_deps")? build_deps: json_string_list(subpkg.get("build_deps"), "subpackage build_deps")?
.unwrap_or_default(), .unwrap_or_default(),
run_deps: json_string_list(subpkg.get("run_deps"), "subpackage run_deps")? deps: json_string_list(subpkg.get("deps"), "subpackage deps")?.unwrap_or_default(),
.unwrap_or_default(),
install_fn: subpkg install_fn: subpkg
.get("install") .get("install")
.and_then(JsonValue::as_str) .and_then(JsonValue::as_str)
@@ -301,7 +288,7 @@ impl Recipe {
sources, sources,
host_deps, host_deps,
build_deps, build_deps,
run_deps, deps,
outputs, outputs,
configure_fn: has_name(&module, "configure").then_some("configure".to_owned()), configure_fn: has_name(&module, "configure").then_some("configure".to_owned()),
build_fn: has_name(&module, "build").then_some("build".to_owned()), build_fn: has_name(&module, "build").then_some("build".to_owned()),
@@ -428,20 +415,20 @@ pub fn unresolved_deps(recipes: &RecipeSet) -> Vec<String> {
let mut missing = Vec::new(); let mut missing = Vec::new();
for recipe in recipes.recipes.values() { for recipe in recipes.recipes.values() {
// host_deps always refer to host outputs (canonical `host:<name>`); // host_deps always refer to host outputs (canonical `host:<name>`);
// build_deps / run_deps refer to target outputs (bare names). // build_deps / deps refer to target outputs (bare names).
for dep in &recipe.host_deps { for dep in &recipe.host_deps {
let key = PackageKind::Host.key(dep); let key = PackageKind::Host.key(dep);
if !names.contains(&key) { if !names.contains(&key) {
missing.push(format!("{} -> {key}", recipe.key())); missing.push(format!("{} -> {key}", recipe.key()));
} }
} }
for dep in recipe.build_deps.iter().chain(recipe.run_deps.iter()) { for dep in recipe.build_deps.iter().chain(recipe.deps.iter()) {
if !names.contains(dep) { if !names.contains(dep) {
missing.push(format!("{} -> {dep}", recipe.key())); missing.push(format!("{} -> {dep}", recipe.key()));
} }
} }
for output in &recipe.outputs { for output in &recipe.outputs {
for dep in output.build_deps.iter().chain(output.run_deps.iter()) { for dep in output.build_deps.iter().chain(output.deps.iter()) {
if !names.contains(dep) { if !names.contains(dep) {
missing.push(format!("{} -> {dep}", output.key())); missing.push(format!("{} -> {dep}", output.key()));
} }
+198 -50
View File
@@ -1,5 +1,5 @@
use crate::config::Config; use crate::config::Config;
use allocative::Allocative; use allocative::{Allocative, Visitor, ident_key};
use anyhow::{Result, anyhow, bail}; use anyhow::{Result, anyhow, bail};
use serde_json::Value as JsonValue; use serde_json::Value as JsonValue;
use starlark::environment::{FrozenModule, Globals, Module}; use starlark::environment::{FrozenModule, Globals, Module};
@@ -11,8 +11,49 @@ use starlark::values::{AnyLifetime, Heap, NoSerialize, ProvidesStaticType, Starl
use starlark_derive::starlark_value; use starlark_derive::starlark_value;
use std::collections::{BTreeMap, HashMap}; use std::collections::{BTreeMap, HashMap};
use std::fmt::{self, Display}; use std::fmt::{self, Display};
use std::mem;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
#[derive(Debug, Clone, ProvidesStaticType, NoSerialize)]
pub struct OptionsValue {
values: BTreeMap<String, JsonValue>,
}
impl OptionsValue {
fn new(values: BTreeMap<String, JsonValue>) -> Result<Self> {
for key in values.keys() {
validate_starlark_identifier(key)?;
}
Ok(Self { values })
}
}
impl Display for OptionsValue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "options")
}
}
starlark_simple_value!(OptionsValue);
impl Allocative for OptionsValue {
fn visit<'a, 'b: 'a>(&self, visitor: &'a mut Visitor<'b>) {
let mut visitor = visitor.enter_self(self);
visitor.visit_simple(
ident_key!(values),
mem::size_of::<(String, JsonValue)>() * self.values.len(),
);
visitor.exit();
}
}
#[starlark_value(type = "options")]
impl<'v> StarlarkValue<'v> for OptionsValue {
fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
self.values.get(attr).map(|value| heap.alloc(value))
}
}
#[derive(Debug, Clone, ProvidesStaticType, NoSerialize, Allocative)] #[derive(Debug, Clone, ProvidesStaticType, NoSerialize, Allocative)]
pub struct SettingsValue { pub struct SettingsValue {
target_arch: String, target_arch: String,
@@ -72,29 +113,34 @@ pub fn eval_file(
/// Path of the implicit helper library auto-loaded into every recipe. /// Path of the implicit helper library auto-loaded into every recipe.
pub const COMMON_LIB_MODULE: &str = "//lib:common.star"; pub const COMMON_LIB_MODULE: &str = "//lib:common.star";
const COMMON_LIB_RELATIVE: &str = "lib/common.star"; const COMMON_LIB_RELATIVE: &str = "lib/common.star";
const OPTIONS_NAME: &str = "OPTIONS";
/// Names exported by `lib/common.star`, if the file exists. Empty otherwise. /// Names exported by `lib/common.star`, if the file exists. Empty otherwise.
pub fn common_lib_names(repo_root: &Path) -> Result<Vec<String>> { pub fn common_lib_names(repo_root: &Path, settings: Option<&Config>) -> Result<Vec<String>> {
let path = repo_root.join(COMMON_LIB_RELATIVE); let path = repo_root.join(COMMON_LIB_RELATIVE);
if !path.exists() { if !path.exists() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
let module = eval_file(&path, None, Some(repo_root))?; let module = eval_file(&path, settings, Some(repo_root))?;
Ok(module Ok(module
.names() .names()
.map(|n| n.as_str().to_owned()) .map(|n| n.as_str().to_owned())
.filter(|n| !n.starts_with('_') && n != "settings") .filter(|n| !n.starts_with('_') && n != "settings" && n != OPTIONS_NAME)
.collect()) .collect())
} }
/// Prepend an implicit `load("//lib:common.star", ...)` so every recipe sees /// Prepend an implicit `load("//lib:common.star", ...)` so every recipe sees
/// the shared helpers without an explicit import. Does nothing if there's no /// the shared helpers without an explicit import. Does nothing if there's no
/// `lib/common.star` or no repo root. /// `lib/common.star` or no repo root.
pub fn prepend_common_lib_load(repo_root: Option<&Path>, content: &str) -> Result<String> { pub fn prepend_common_lib_load(
repo_root: Option<&Path>,
settings: Option<&Config>,
content: &str,
) -> Result<String> {
let Some(root) = repo_root else { let Some(root) = repo_root else {
return Ok(content.to_owned()); return Ok(content.to_owned());
}; };
let names = common_lib_names(root)?; let names = common_lib_names(root, settings)?;
if names.is_empty() { if names.is_empty() {
return Ok(content.to_owned()); return Ok(content.to_owned());
} }
@@ -127,6 +173,7 @@ pub fn eval_content_with_extra<'a>(
extra: Option<&'a dyn AnyLifetime<'a>>, extra: Option<&'a dyn AnyLifetime<'a>>,
) -> Result<Module> { ) -> Result<Module> {
let filename = path.display().to_string(); let filename = path.display().to_string();
validate_options_source(path, &content)?;
let ast = AstModule::parse( let ast = AstModule::parse(
&filename, &filename,
content, content,
@@ -148,9 +195,11 @@ pub fn eval_content_with_extra<'a>(
None => None, None => None,
}; };
let module = Module::new(); let module = Module::new();
if let Some(config) = settings { let protected_options = if let Some(config) = settings {
module.set("settings", module.heap().alloc(SettingsValue::from(config))); Some(set_config_values(&module, config)?)
} } else {
None
};
{ {
let mut eval = Evaluator::new(&module); let mut eval = Evaluator::new(&module);
if let Some(loader) = &loader { if let Some(loader) = &loader {
@@ -160,9 +209,85 @@ pub fn eval_content_with_extra<'a>(
eval.eval_module(ast, &globals) eval.eval_module(ast, &globals)
.map_err(|err| anyhow!("{err}"))?; .map_err(|err| anyhow!("{err}"))?;
} }
validate_options_binding(path, &module, protected_options)?;
Ok(module) Ok(module)
} }
fn validate_options_source(path: &Path, content: &str) -> Result<()> {
let line_offset = implicit_common_load_line_offset(content);
for (index, line) in content.lines().enumerate() {
let code = line.split('#').next().unwrap_or_default().trim_start();
if defines_or_reassigns_options(code) {
let line = (index + 1).saturating_sub(line_offset).max(1);
bail!(
"{}:{} must not define or reassign `{OPTIONS_NAME}`",
path.display(),
line
);
}
}
Ok(())
}
fn implicit_common_load_line_offset(content: &str) -> usize {
content
.lines()
.next()
.is_some_and(|line| line.starts_with(&format!("load(\"{COMMON_LIB_MODULE}\",")))
as usize
}
fn defines_or_reassigns_options(code: &str) -> bool {
let Some(rest) = code.strip_prefix(OPTIONS_NAME) else {
return defines_options_function(code) || binds_options_loop_variable(code);
};
let rest = rest.trim_start();
rest.starts_with('=')
|| rest.starts_with("+=")
|| rest.starts_with("-=")
|| rest.starts_with("*=")
|| rest.starts_with("/=")
|| rest.starts_with("%=")
|| rest.starts_with("&=")
|| rest.starts_with("|=")
|| rest.starts_with("^=")
}
fn defines_options_function(code: &str) -> bool {
code.strip_prefix("def ")
.and_then(|rest| rest.trim_start().strip_prefix(OPTIONS_NAME))
.is_some_and(|rest| rest.trim_start().starts_with('('))
}
fn binds_options_loop_variable(code: &str) -> bool {
code.strip_prefix("for ")
.and_then(|rest| rest.trim_start().strip_prefix(OPTIONS_NAME))
.is_some_and(|rest| rest.trim_start().starts_with("in "))
}
fn validate_options_binding(
path: &Path,
module: &Module,
expected: Option<Value<'_>>,
) -> Result<()> {
let Some(expected) = expected else {
return Ok(());
};
let actual = module.get(OPTIONS_NAME).ok_or_else(|| {
anyhow!(
"{} removed the protected `{OPTIONS_NAME}` binding",
path.display()
)
})?;
if !actual.ptr_eq(expected) {
bail!(
"{} must not define or reassign `{OPTIONS_NAME}`",
path.display()
);
}
Ok(())
}
#[derive(Clone)] #[derive(Clone)]
struct RepoFileLoader { struct RepoFileLoader {
modules: HashMap<String, FrozenModule>, modules: HashMap<String, FrozenModule>,
@@ -211,6 +336,7 @@ fn load_module(
let path = resolve_load_path(repo_root, module_id)?; let path = resolve_load_path(repo_root, module_id)?;
let content = std::fs::read_to_string(&path)?; let content = std::fs::read_to_string(&path)?;
let filename = path.display().to_string(); let filename = path.display().to_string();
validate_options_source(&path, &content)?;
let ast = let ast =
AstModule::parse(&filename, content, &Dialect::Standard).map_err(|err| anyhow!("{err}"))?; AstModule::parse(&filename, content, &Dialect::Standard).map_err(|err| anyhow!("{err}"))?;
for load in ast.loads() { for load in ast.loads() {
@@ -226,20 +352,80 @@ fn load_module(
modules: modules.clone(), modules: modules.clone(),
}; };
let module = Module::new(); let module = Module::new();
if let Some(config) = settings { let protected_options = if let Some(config) = settings {
module.set("settings", module.heap().alloc(SettingsValue::from(config))); Some(set_config_values(&module, config)?)
} } else {
None
};
{ {
let mut eval = Evaluator::new(&module); let mut eval = Evaluator::new(&module);
eval.set_loader(&nested_loader); eval.set_loader(&nested_loader);
eval.eval_module(ast, &globals) eval.eval_module(ast, &globals)
.map_err(|err| anyhow!("{err}"))?; .map_err(|err| anyhow!("{err}"))?;
} }
validate_options_binding(&path, &module, protected_options)?;
let frozen = module.freeze().map_err(|err| anyhow!("{err:?}"))?; let frozen = module.freeze().map_err(|err| anyhow!("{err:?}"))?;
modules.insert(module_id.to_owned(), frozen); modules.insert(module_id.to_owned(), frozen);
Ok(()) Ok(())
} }
fn set_config_values<'v>(module: &'v Module, config: &Config) -> Result<Value<'v>> {
module.set("settings", module.heap().alloc(SettingsValue::from(config)));
let options = module
.heap()
.alloc(OptionsValue::new(config.options.clone())?);
module.set(OPTIONS_NAME, options);
Ok(options)
}
fn validate_starlark_identifier(name: &str) -> Result<()> {
let mut chars = name.chars();
let Some(first) = chars.next() else {
bail!("config option name cannot be empty");
};
if !(first == '_' || first.is_ascii_alphabetic()) {
bail!("config option `{name}` is not a valid Starlark identifier");
}
if chars.any(|ch| !(ch == '_' || ch.is_ascii_alphanumeric())) {
bail!("config option `{name}` is not a valid Starlark identifier");
}
if matches!(
name,
"and"
| "as"
| "assert"
| "break"
| "class"
| "continue"
| "def"
| "del"
| "elif"
| "else"
| "except"
| "finally"
| "for"
| "from"
| "global"
| "if"
| "import"
| "in"
| "is"
| "lambda"
| "load"
| "not"
| "or"
| "pass"
| "return"
| "try"
| "while"
| "with"
| "yield"
) {
bail!("config option `{name}` is a reserved Starlark keyword");
}
Ok(())
}
fn resolve_load_path(repo_root: &Path, module_id: &str) -> Result<PathBuf> { fn resolve_load_path(repo_root: &Path, module_id: &str) -> Result<PathBuf> {
let relative = if let Some(stripped) = module_id.strip_prefix("//") { let relative = if let Some(stripped) = module_id.strip_prefix("//") {
stripped.replace(':', "/") stripped.replace(':', "/")
@@ -257,12 +443,6 @@ fn resolve_load_path(repo_root: &Path, module_id: &str) -> Result<PathBuf> {
Ok(canonical_path) Ok(canonical_path)
} }
pub fn options_literal(config: &Config) -> Result<String> {
json_to_starlark_literal(&JsonValue::Object(
config.options.clone().into_iter().collect(),
))
}
fn option_value_to_string(value: &JsonValue) -> String { fn option_value_to_string(value: &JsonValue) -> String {
match value { match value {
JsonValue::String(value) => value.clone(), JsonValue::String(value) => value.clone(),
@@ -273,38 +453,6 @@ fn option_value_to_string(value: &JsonValue) -> String {
} }
} }
fn json_to_starlark_literal(value: &JsonValue) -> Result<String> {
Ok(match value {
JsonValue::Null => "None".to_owned(),
JsonValue::Bool(true) => "True".to_owned(),
JsonValue::Bool(false) => "False".to_owned(),
JsonValue::Number(value) => value.to_string(),
JsonValue::String(value) => serde_json::to_string(value)?,
JsonValue::Array(values) => format!(
"[{}]",
values
.iter()
.map(json_to_starlark_literal)
.collect::<Result<Vec<_>>>()?
.join(", ")
),
JsonValue::Object(values) => format!(
"{{{}}}",
values
.iter()
.map(|(key, value)| {
Ok(format!(
"{}: {}",
serde_json::to_string(key)?,
json_to_starlark_literal(value)?
))
})
.collect::<Result<Vec<_>>>()?
.join(", ")
),
})
}
pub fn get_string(module: &Module, name: &str) -> Result<String> { pub fn get_string(module: &Module, name: &str) -> Result<String> {
module module
.get(name) .get(name)