This commit is contained in:
2026-05-17 23:23:13 +02:00
parent a23e2c83d1
commit 695f30d678
25 changed files with 6000 additions and 0 deletions
+4
View File
@@ -0,0 +1,4 @@
target
build
.git
*.bak
+4
View File
@@ -0,0 +1,4 @@
/target
/build
*.bak
*.lock
Generated
+2900
View File
File diff suppressed because it is too large Load Diff
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "distro"
version = "0.1.0"
edition = "2024"
license = "MIT"
[dependencies]
anyhow = "1.0"
clap = { version = "4.5", features = ["derive"] }
hex = "0.4"
reqwest = { version = "0.12", default-features = false, features = [
"blocking",
"rustls-tls",
] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
sha2 = "0.10"
shell-escape = "0.1"
starlark = "0.13"
starlark_derive = "0.13"
allocative = "0.3"
tempfile = "3.10"
walkdir = "2.5"
+58
View File
@@ -0,0 +1,58 @@
FROM docker.io/library/alpine:edge
RUN apk upgrade --no-cache && \
apk add --no-cache \
alpine-sdk \
apk-tools \
autoconf \
automake \
bash \
bc \
bison \
bzip2 \
ca-certificates \
cmake \
coreutils \
curl \
file \
findutils \
flex \
gettext-dev \
git \
gzip \
elfutils-dev \
gmp-dev \
mpfr-dev \
mpc1-dev \
libtool \
linux-headers \
meson \
ninja \
openssl \
openssl-dev \
patch \
pkgconf \
python3 \
tar \
texinfo \
xz \
zstd
RUN rm -rf /tmp/mkpkg-root /tmp/distro-preflight.apk /tmp/APKINDEX.adb /tmp/distro-preflight.rsa && \
openssl genrsa -out /tmp/distro-preflight.rsa 2048 >/dev/null 2>&1 && \
openssl rsa -in /tmp/distro-preflight.rsa -pubout -out /etc/apk/keys/distro-preflight.rsa.pub >/dev/null 2>&1 && \
mkdir -p /tmp/mkpkg-root/usr/share/distro && \
printf ok > /tmp/mkpkg-root/usr/share/distro/preflight && \
apk --sign-key /tmp/distro-preflight.rsa mkpkg \
--files /tmp/mkpkg-root \
--output /tmp/distro-preflight.apk \
--info name:distro-preflight \
--info version:0-r0 \
--info arch:noarch \
--info description:preflight \
--info license:MIT >/dev/null && \
apk --sign-key /tmp/distro-preflight.rsa mkndx -o /tmp/APKINDEX.adb /tmp/distro-preflight.apk >/dev/null && \
test -s /tmp/distro-preflight.apk && \
test -s /tmp/APKINDEX.adb
WORKDIR /work
+35
View File
@@ -0,0 +1,35 @@
container_runtime = "podman"
container_image = "localhost/distro-builder:latest"
container_dockerfile = "Dockerfile"
signing_key = "build/keys/distro.rsa"
signing_pubkey = "build/keys/distro.rsa.pub"
target_arch = "x86_64"
host_cflags = "-O2 -pipe"
host_cxxflags = ""
host_ldflags = "-Wl,-O1 -Wl,--sort-common -Wl,--as-needed"
target_cflags = host_cflags
target_cxxflags = host_cxxflags
target_ldflags = host_ldflags + " -Wl,-z,now"
if target_arch == "x86_64":
flags = " -march=x86-64-v3 -mtune=generic -fstack-clash-protection -fstack-protector-strong -fcf-protection"
target_cflags += flags
target_cxxflags += flags
target_ldflags += " -Wl,-z,pack-relative-relocs"
options = {
"libc": "musl",
"target_triple": "x86_64-linux-musl",
"host_cflags": host_cflags,
"host_cxxflags": host_cxxflags,
"host_ldflags": host_ldflags,
"cflags": target_cflags,
"cxxflags": target_cxxflags,
"ldflags": target_ldflags,
"wayland": True,
"x11": True,
}
+43
View File
@@ -0,0 +1,43 @@
name = "binutils"
version = "2.46.0"
revision = 1
description = "GNU binutils cross-compiled for the target triple"
license = "GPL-3.0-or-later"
source = {
"url": "https://ftp.gnu.org/gnu/binutils/binutils-" + version + ".tar.xz",
"sha256": "d75a94f4d73e7a4086f7513e67e439e8fcdcbb726ffe63f4661744e6256b2cf2",
"strip_components": 1,
}
host_deps = []
def configure(ctx):
triple = ctx.options["target_triple"]
ctx.run([
ctx.source_dir + "/configure",
"--prefix=" + ctx.prefix,
"--target=" + triple,
"--with-sysroot=" + ctx.prefix + "/" + triple,
"--disable-nls",
"--disable-werror",
"--enable-deterministic-archives",
"--enable-ld=default",
"--enable-plugins",
"--enable-threads",
"--with-system-zlib",
# gprofng's libcollector does not build against musl/recent gcc.
"--disable-gprofng",
], env = {
"CFLAGS": ctx.options["host_cflags"],
"CXXFLAGS": ctx.options["host_cxxflags"],
"LDFLAGS": ctx.options["host_ldflags"],
})
def build(ctx):
ctx.run(["make", "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["make", "DESTDIR=" + pkg.destdir, "install"])
# Drop static archives we don't need on the cross side.
ctx.run(["sh", "-c", "rm -f " + pkg.destdir + ctx.prefix + "/lib/*.a"])
+49
View File
@@ -0,0 +1,49 @@
name = "gcc"
version = "16.1.0"
revision = 1
description = "GNU GCC cross-compiler (bootstrap stage, C/C++ only)"
license = "GPL-3.0-or-later"
source = {
"url": f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz",
"sha256": "50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79",
"strip_components": 1,
}
host_deps = ["binutils"]
def configure(ctx):
triple = ctx.options["target_triple"]
ctx.run([
ctx.source_dir + "/configure",
"--prefix=" + ctx.prefix,
"--target=" + triple,
"--with-sysroot=" + ctx.prefix + "/" + triple,
"--without-headers",
"--with-newlib",
"--enable-languages=c,c++",
"--enable-default-pie",
"--enable-default-ssp",
"--disable-nls",
"--disable-shared",
"--disable-threads",
"--disable-libssp",
"--disable-libgomp",
"--disable-libquadmath",
"--disable-libatomic",
"--disable-libvtv",
"--disable-multilib",
], env = {
"CFLAGS": ctx.options["host_cflags"],
"CXXFLAGS": ctx.options["host_cxxflags"],
"LDFLAGS": ctx.options["host_ldflags"],
})
def build(ctx):
jobs = "-j" + str(ctx.jobs)
ctx.run(["make", jobs, "all-gcc"])
ctx.run(["make", jobs, "all-target-libgcc"])
def install(ctx, pkg):
ctx.run(["make", "DESTDIR=" + pkg.destdir, "install-gcc"])
ctx.run(["make", "DESTDIR=" + pkg.destdir, "install-target-libgcc"])
+82
View File
@@ -0,0 +1,82 @@
# Commonly used helpers, auto-loaded into every recipe.
def autotools_configure(ctx, extra_args = []):
args = [
ctx.source_dir + "/configure",
"--prefix=" + ctx.prefix,
"--sysconfdir=/etc",
"--localstatedir=/var",
]
args.extend(extra_args)
ctx.run(args, env = _toolchain_env(ctx))
def autotools_build(ctx, extra_args = []):
args = ["make", "-j" + str(ctx.jobs)]
args.extend(extra_args)
ctx.run(args)
def autotools_check(ctx, extra_args = []):
args = ["make", "check", "-j" + str(ctx.jobs)]
args.extend(extra_args)
ctx.run(args)
def autotools_install(ctx, pkg, extra_args = []):
args = ["make", "install", "DESTDIR=" + pkg.destdir]
args.extend(extra_args)
ctx.run(args)
def autotools(configure_args = [], build_args = [], install_args = []):
def _configure(ctx):
autotools_configure(ctx, extra_args = configure_args)
def _build(ctx):
autotools_build(ctx, extra_args = build_args)
def _install(ctx, pkg):
autotools_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install
def meson_configure(ctx, extra_args = []):
args = [
"meson",
"setup",
ctx.build_dir,
ctx.source_dir,
"--prefix=" + ctx.prefix,
]
args.extend(extra_args)
ctx.run(args, env = _toolchain_env(ctx))
def meson_build(ctx):
ctx.run(["meson", "compile", "-C", ctx.build_dir, "-j" + str(ctx.jobs)])
def meson_install(ctx, pkg):
ctx.run(["meson", "install", "-C", ctx.build_dir, "--destdir", pkg.destdir])
def meson(configure_args = [], build_args = [], install_args = []):
def _configure(ctx):
meson_configure(ctx, extra_args = configure_args)
def _build(ctx):
meson_build(ctx, extra_args = build_args)
def _install(ctx, pkg):
meson_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install
def make(ctx, target = None, extra_args = []):
args = ["make", "-C", ctx.source_dir, "O=" + ctx.build_dir,
"-j" + str(ctx.jobs)]
args.extend(extra_args)
if target:
args.append(target)
ctx.run(args)
def make_install(ctx, pkg, extra_args = []):
args = ["make", "-C", ctx.build_dir, "DESTDIR=" + pkg.destdir, "install"]
args.extend(extra_args)
ctx.run(args)
def _toolchain_env(ctx):
env = {}
for key, var in [("cflags", "CFLAGS"), ("cxxflags", "CXXFLAGS"), ("ldflags", "LDFLAGS")]:
if key in ctx.options:
env[var] = ctx.options[key]
return env
+38
View File
@@ -0,0 +1,38 @@
name = "linux"
version = "7.0.9"
revision = 1
description = "Linux kernel"
license = "GPL-2.0-only"
source = {
"url": f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz",
"sha256": "ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4",
"strip_components": 1,
}
host_deps = ["binutils", "gcc"]
def _make_args(ctx, *args):
triple = ctx.options["target_triple"]
result = [
"make",
"-C", ctx.source_dir,
"O=" + ctx.build_dir,
"ARCH=x86_64",
f"CROSS_COMPILE={triple}-",
"-j" + str(ctx.jobs),
]
result.extend(args)
return result
def configure(ctx):
ctx.run(_make_args(ctx, "defconfig"))
def build(ctx):
ctx.run(_make_args(ctx, "bzImage"))
def install(ctx, pkg):
ctx.install(
ctx.build_dir + "/arch/x86/boot/bzImage",
pkg.destdir + "/boot/vmlinuz-" + version,
)
+32
View File
@@ -0,0 +1,32 @@
name = "musl"
version = "1.2.6"
revision = 1
description = "Musl C library"
license = "MIT"
source = {
"url": f"https://musl.libc.org/releases/musl-{version}.tar.gz",
"sha256": "d585fd3b613c66151fc3249e8ed44f77020cb5e6c1e635a616d3f9f82460512a",
"strip_components": 1,
}
host_deps = ["binutils", "gcc"]
def configure(ctx):
triple = ctx.options["target_triple"]
ctx.run(
[
ctx.source_dir + "/configure",
"--prefix=/usr",
"--syslibdir=/lib",
"--target=" + triple,
],
env = {
"CC": triple + "-gcc",
"CFLAGS": ctx.options["cflags"],
"LDFLAGS": ctx.options["ldflags"],
},
)
build = autotools_build
install = autotools_install
+46
View File
@@ -0,0 +1,46 @@
use crate::config::Config;
use crate::recipe::OutputPackage;
use std::path::Path;
#[derive(Debug, Clone)]
pub struct ApkPackagePlan {
pub args: Vec<String>,
}
pub fn mkpkg_plan(
config: &Config,
package: &OutputPackage,
files_root: &Path,
output_dir: &Path,
) -> ApkPackagePlan {
let file_name = format!(
"{}-{}-r{}.apk",
package.name, package.version, package.revision
);
let output = output_dir.join(file_name);
let version = format!("{}-r{}", package.version, package.revision);
let mut args = vec![
"mkpkg".to_owned(),
"--files".to_owned(),
files_root.display().to_string(),
"--output".to_owned(),
output.display().to_string(),
"--info".to_owned(),
format!("name:{}", package.name),
"--info".to_owned(),
format!("version:{version}"),
"--info".to_owned(),
format!("arch:{}", config.target_arch),
"--info".to_owned(),
format!("description:{}", package.description),
"--info".to_owned(),
format!("license:{}", package.license),
"--info".to_owned(),
format!("origin:{}", package.recipe),
];
for dep in &package.run_deps {
args.push("--info".to_owned());
args.push(format!("depends:{dep}"));
}
ApkPackagePlan { args }
}
+921
View File
@@ -0,0 +1,921 @@
use crate::apk;
use crate::config::Config;
use crate::graph::PackageGraph;
use crate::log;
use crate::patches;
use crate::phase::{PhaseCommand, PhaseEnv, SourceDir, collect_phase_commands};
use crate::recipe::{OutputPackage, PackageKind, Recipe, RecipeSet};
use crate::source;
use anyhow::{Context, Result, anyhow, bail};
use sha2::{Digest, Sha256};
use std::cell::{Cell, RefCell};
use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
const C_SOURCE: &str = "/source";
const C_BUILD: &str = "/builddir";
const C_DEST: &str = "/dest";
const C_SYSROOT: &str = "/sysroot";
const HOST_PREFIX: &str = "/usr/local";
const TARGET_PREFIX: &str = "/usr";
#[derive(Debug)]
pub struct Builder {
repo: PathBuf,
config: Config,
skip_checks: bool,
preflight_done: Cell<bool>,
built_recipes: RefCell<HashSet<String>>,
}
impl Builder {
pub fn new(repo: PathBuf, config: Config, skip_checks: bool) -> Self {
Self {
repo,
config,
skip_checks,
preflight_done: Cell::new(false),
built_recipes: RefCell::default(),
}
}
pub fn build(
&self,
recipes: &RecipeSet,
graph: &PackageGraph,
package: Option<&str>,
rebuild: bool,
) -> Result<()> {
self.preflight_container()?;
let (label, order) = match package {
Some(p) => (p.to_owned(), graph.build_order(p)?),
None => ("<all>".to_owned(), graph.build_order_all()?),
};
log::step(
"plan",
&format!("{label}: {} package(s) [{}]", order.len(), order.join(", ")),
);
for output_name in order {
let output = graph.output(&output_name)?;
let recipe = recipes.recipe_for_package(&output_name)?;
self.ensure_source(recipe)?;
// Only force a rebuild of the explicitly requested package;
// dependencies stay cached when their manifest is up to date.
// With no package (build all), every package honours --rebuild.
let force = rebuild && package.map_or(true, |p| p == output_name);
match output.kind {
PackageKind::Host => self.build_host(recipe, output, force)?,
PackageKind::Target => self.build_target(recipe, output, force)?,
}
}
self.repo_index()?;
Ok(())
}
pub fn fetch(&self, recipes: &RecipeSet, package: &str) -> Result<()> {
let recipe = recipes.recipe_by_user_ref(package)?;
log::step("fetch", &recipe.key());
source::fetch_sources(recipe, &self.source_cache_dir())?;
Ok(())
}
pub fn shell(&self, recipes: &RecipeSet, graph: &PackageGraph, package: &str) -> Result<()> {
graph.output(package)?;
let recipe = recipes.recipe_for_package(package)?;
self.preflight_container()?;
let source = self.unpacked_source_dir(recipe);
let build = self.build_dir(recipe);
fs::create_dir_all(&source)?;
fs::create_dir_all(&build)?;
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-it")
.arg("-v")
.arg(format!("{}:{C_SOURCE}:ro", source.display()))
.arg("-v")
.arg(format!("{}:{C_BUILD}", build.display()))
.arg("-w")
.arg(C_BUILD)
.arg(&self.config.container_image)
.arg("/bin/sh")
.status()?;
if !status.success() {
bail!("container shell exited with {status}");
}
Ok(())
}
pub fn repo_index(&self) -> Result<()> {
self.preflight_container()?;
let repo = self.pkgs_dir();
fs::create_dir_all(&repo)?;
log::step(
"index",
&format!("signing repository at {}", repo.display()),
);
let key = self.abs_config_path(&self.config.signing_key);
let pubkey = self.abs_config_path(&self.config.signing_pubkey);
if !key.exists() || !pubkey.exists() {
bail!("signing key is not configured or missing; run `distro init-key` first");
}
let index_name = "APKINDEX.adb";
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/repo", repo.display()))
.arg("-v")
.arg(format!("{}:/keys/private.rsa:ro", key.display()))
.arg("-v")
.arg(format!(
"{}:/etc/apk/keys/distro.rsa.pub:ro",
pubkey.display()
))
.arg(&self.config.container_image)
.arg("/bin/sh")
.arg("-lc")
.arg(format!(
"cd /repo && apk --sign-key /keys/private.rsa mkndx -o {index_name} *.apk"
))
.status()
.context("failed to run repository index command")?;
if !status.success() {
bail!("repository index command failed with {status}");
}
Ok(())
}
pub fn init_key(&self) -> Result<()> {
let key = self.abs_config_path(&self.config.signing_key);
let pubkey = self.abs_config_path(&self.config.signing_pubkey);
if key.exists() && pubkey.exists() {
log::skip("init-key", "signing key already present");
return Ok(());
}
log::step("init-key", &format!("generating {}", key.display()));
let key_dir = key
.parent()
.ok_or_else(|| anyhow!("invalid signing key path"))?;
let key_name = key
.file_name()
.ok_or_else(|| anyhow!("invalid signing key path"))?
.to_string_lossy();
fs::create_dir_all(key_dir)?;
self.preflight_container()?;
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/keys", key_dir.display()))
.arg(&self.config.container_image)
.arg("/bin/sh")
.arg("-lc")
.arg(format!(
"openssl genrsa -out /keys/{key_name} 4096 && \
openssl rsa -in /keys/{key_name} -pubout -out /keys/{key_name}.pub"
))
.status()
.context("failed to run key generation command")?;
if !status.success() {
bail!("key generation failed with {status}");
}
Ok(())
}
pub fn rootfs(&self, root: &Path, packages: &[String]) -> Result<()> {
self.preflight_container()?;
let pubkey = self.abs_config_path(&self.config.signing_pubkey);
if !pubkey.exists() {
bail!("rootfs requires a configured public signing key; run `distro init-key` first");
}
fs::create_dir_all(root)?;
log::step(
"rootfs",
&format!(
"{} -> {} [{}]",
packages.join(", "),
root.display(),
root.display()
),
);
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/rootfs", root.display()))
.arg("-v")
.arg(format!("{}:/repo:ro", self.pkgs_dir().display()))
.arg("-v")
.arg(format!(
"{}:/etc/apk/keys/distro.rsa.pub:ro",
pubkey.display()
))
.arg(&self.config.container_image)
.arg("apk")
.arg("--root")
.arg("/rootfs")
.arg("--repository")
.arg("/repo/APKINDEX.adb")
.arg("add")
.args(packages)
.status()
.context("failed to run rootfs apk command")?;
if !status.success() {
bail!("rootfs creation failed with {status}");
}
Ok(())
}
// --- per-recipe build steps -------------------------------------------
fn build_host(&self, recipe: &Recipe, output: &OutputPackage, rebuild: bool) -> Result<()> {
let manifest = self.manifest_path(&output.key());
let hash = self.manifest_hash(recipe)?;
if !rebuild && fs::read_to_string(&manifest).ok().as_deref() == Some(hash.as_str()) {
log::skip("up-to-date", &output.key());
return Ok(());
}
log::step(
"build",
&format!("{} {}-r{}", output.key(), recipe.version, recipe.revision),
);
let build_dir = self.host_build_dir(recipe);
let dest_dir = self.host_pkg_dir(recipe);
let source_dir = self.unpacked_source_dir(recipe);
// A rebuild starts from a clean build dir; sources stay shared.
if rebuild {
Self::recreate(&build_dir)?;
} else {
fs::create_dir_all(&build_dir)?;
}
Self::recreate(&dest_dir)?;
fs::create_dir_all(&build_dir)?;
let env = PhaseEnv {
source_dir: source_dir_env(recipe),
build_dir: C_BUILD,
dest_dir: C_DEST,
prefix: HOST_PREFIX,
sysroot: C_SYSROOT,
};
self.run_recipe_phases(
recipe,
output,
&env,
&source_dir,
&build_dir,
&dest_dir,
None,
)?;
fs::create_dir_all(manifest.parent().unwrap())?;
fs::write(manifest, hash)?;
Ok(())
}
fn build_target(&self, recipe: &Recipe, output: &OutputPackage, rebuild: bool) -> Result<()> {
let manifest = self.manifest_path(&output.key());
let hash = self.manifest_hash(recipe)?;
if !rebuild && fs::read_to_string(&manifest).ok().as_deref() == Some(hash.as_str()) {
log::skip("up-to-date", &output.key());
return Ok(());
}
log::step(
"build",
&format!("{} {}-r{}", output.key(), recipe.version, recipe.revision),
);
let build_dir = self.build_dir(recipe);
let source_dir = self.unpacked_source_dir(recipe);
// A rebuild starts from a clean build dir; sources stay shared.
if rebuild {
Self::recreate(&build_dir)?;
} else {
fs::create_dir_all(&build_dir)?;
}
let dest_tmp =
tempfile::tempdir_in(&build_dir).context("failed to create ephemeral destdir")?;
let dest_dir = dest_tmp.path();
let sysroot = self.materialize_sysroot(recipe)?;
let env = PhaseEnv {
source_dir: source_dir_env(recipe),
build_dir: C_BUILD,
dest_dir: C_DEST,
prefix: TARGET_PREFIX,
sysroot: C_SYSROOT,
};
self.run_recipe_phases(
recipe,
output,
&env,
&source_dir,
&build_dir,
dest_dir,
sysroot.as_deref(),
)?;
self.apk_mkpkg(output, dest_dir)?;
fs::create_dir_all(manifest.parent().unwrap())?;
fs::write(manifest, hash)?;
Ok(())
}
fn run_recipe_phases(
&self,
recipe: &Recipe,
output: &OutputPackage,
env: &PhaseEnv<'_>,
source_dir: &Path,
build_dir: &Path,
dest_dir: &Path,
target_sysroot: Option<&Path>,
) -> Result<()> {
let host_sandbox = self.materialize_host_sandbox(recipe)?;
let recipe_already_built = self.built_recipes.borrow().contains(&recipe.key());
// configure/build/check run at most once per recipe per invocation.
let mut shared_phases: Vec<(&str, bool)> = Vec::new();
if !recipe_already_built {
if let Some(p) = recipe.configure_fn.as_deref() {
shared_phases.push((p, false));
}
if let Some(p) = recipe.build_fn.as_deref() {
shared_phases.push((p, false));
}
if !self.skip_checks {
if let Some(p) = recipe.check_fn.as_deref() {
shared_phases.push((p, false));
}
}
}
// install runs per output (each output has its own destdir/install_fn).
let install_phase = (output.install_fn.as_str(), true);
let mut commands: Vec<PhaseCommand> = Vec::new();
for (phase, takes_pkg) in shared_phases.iter().chain(std::iter::once(&install_phase)) {
let pkg = if *takes_pkg {
Some((output.name.as_str(), C_DEST))
} else {
None
};
let owner = if *takes_pkg {
output.key()
} else {
recipe.key()
};
log::info("phase", &format!("{phase} {owner}"));
commands.extend(collect_phase_commands(
&recipe.path,
&self.repo,
&self.config,
phase,
env,
pkg,
)?);
}
if commands.is_empty() {
self.built_recipes.borrow_mut().insert(recipe.key());
return Ok(());
}
self.run_in_container(
source_dir,
build_dir,
dest_dir,
host_sandbox.as_ref().map(|s| s.path()),
target_sysroot,
&commands,
)?;
self.built_recipes.borrow_mut().insert(recipe.key());
Ok(())
}
fn run_in_container(
&self,
source_dir: &Path,
build_dir: &Path,
dest_dir: &Path,
host_sandbox: Option<&Path>,
target_sysroot: Option<&Path>,
commands: &[PhaseCommand],
) -> Result<()> {
fs::create_dir_all(dest_dir)?;
let mut process = Command::new(&self.config.container_runtime);
process
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:{C_SOURCE}:ro", source_dir.display()))
.arg("-v")
.arg(format!("{}:{C_BUILD}", build_dir.display()))
.arg("-v")
.arg(format!("{}:{C_DEST}", dest_dir.display()))
.arg("-w")
.arg(C_BUILD);
if let Some(sandbox) = host_sandbox {
// Host packages are configured with --prefix=/usr/local, so we
// bind the assembled tree exactly there (matching Jinx). This
// means rpaths, --print-search-dirs, pkg-config lookups, etc.
// all keep working with no extra environment fiddling.
process
.arg("-v")
.arg(format!("{}:{HOST_PREFIX}:ro", sandbox.display()));
}
if let Some(sysroot) = target_sysroot {
process
.arg("-v")
.arg(format!("{}:{C_SYSROOT}:ro", sysroot.display()))
.arg("-e")
.arg(format!("PKG_CONFIG_SYSROOT_DIR={C_SYSROOT}"))
.arg("-e")
.arg(format!("SYSROOT={C_SYSROOT}"));
}
process
.arg(&self.config.container_image)
.arg("/bin/sh")
.arg("-c")
.arg(build_phase_script(commands));
let status = process
.status()
.context("failed to start container for phase commands")?;
if !status.success() {
bail!("phase commands failed with {status}");
}
Ok(())
}
fn apk_mkpkg(&self, output: &OutputPackage, dest_dir: &Path) -> Result<()> {
let repo = self.pkgs_dir();
fs::create_dir_all(&repo)?;
let signing_key = self.abs_config_path(&self.config.signing_key);
if !signing_key.exists() {
bail!("package signing key is missing; run `distro init-key` first");
}
log::step("package", &format!("{} -> {}", output.name, repo.display()));
let plan = apk::mkpkg_plan(&self.config, output, Path::new(C_DEST), Path::new("/out"));
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:{C_DEST}:ro", dest_dir.display()))
.arg("-v")
.arg(format!("{}:/out", repo.display()))
.arg("-v")
.arg(format!("{}:/keys/private.rsa:ro", signing_key.display()))
.arg(&self.config.container_image)
.arg("apk")
.arg("--sign-key")
.arg("/keys/private.rsa")
.args(plan.args)
.status()
.context("failed to run apk mkpkg command")?;
if !status.success() {
bail!("apk mkpkg failed with {status}");
}
Ok(())
}
// --- source preparation (Jinx-style persistent sources/<recipe>/) ------
fn ensure_source(&self, recipe: &Recipe) -> Result<()> {
let cached = source::fetch_sources(recipe, &self.source_cache_dir())?;
let unpacked = self.unpacked_source_dir(recipe);
let version_stamp = self.source_stamp(recipe, "version");
let want_version = format!("{}-r{}", recipe.version, recipe.revision);
let need_extract =
fs::read_to_string(&version_stamp).ok().as_deref() != Some(&want_version);
if need_extract {
log::info("unpack", &recipe.key());
if unpacked.exists() {
fs::remove_dir_all(&unpacked)?;
}
fs::create_dir_all(&unpacked)?;
for (src, tarball) in recipe.sources.iter().zip(cached.iter()) {
let dest = if src.name.is_empty() {
unpacked.clone()
} else {
unpacked.join(&src.name)
};
self.extract_tarball(tarball, &dest, src.strip_components)?;
}
fs::write(&version_stamp, &want_version)?;
let patched = self.source_stamp(recipe, "patched");
let _ = fs::remove_file(&patched);
}
let patched_stamp = self.source_stamp(recipe, "patched");
if !patched_stamp.exists() {
self.apply_patches(recipe, &unpacked)?;
fs::write(&patched_stamp, "")?;
}
Ok(())
}
fn extract_tarball(&self, tarball: &Path, dest: &Path, strip_components: u32) -> Result<()> {
fs::create_dir_all(dest)?;
log::info(
"extract",
&format!("{} -> {}", tarball.display(), dest.display()),
);
let strip = if strip_components > 0 {
format!("--strip-components={strip_components}")
} else {
String::new()
};
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/in.tar:ro", tarball.display()))
.arg("-v")
.arg(format!("{}:/out", dest.display()))
.arg(&self.config.container_image)
.arg("/bin/sh")
.arg("-c")
.arg(format!("tar -xf /in.tar -C /out {strip}"))
.status()
.context("failed to unpack source archive")?;
if !status.success() {
bail!("source unpack failed with {status}");
}
Ok(())
}
fn apply_patches(&self, recipe: &Recipe, source_dir: &Path) -> Result<()> {
let patches = patches::discover(&recipe.dir)?;
if patches.is_empty() {
return Ok(());
}
if recipe.sources.iter().any(|s| !s.name.is_empty()) {
bail!(
"recipe `{}` has patches/ but uses multi-source layout; \
apply patches yourself via ctx.run in the configure phase",
recipe.id
);
}
for patch in patches {
log::info(
"patch",
&format!(
"{} <- {}",
recipe.id,
patch
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default(),
),
);
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/source", source_dir.display()))
.arg("-v")
.arg(format!("{}:/patch.diff:ro", patch.display()))
.arg(&self.config.container_image)
.arg("patch")
.arg("-d")
.arg("/source")
.arg("-p1")
.arg("-i")
.arg("/patch.diff")
.status()
.context("failed to apply patch")?;
if !status.success() {
bail!("patch {} failed with {status}", patch.display());
}
}
Ok(())
}
// --- host sandbox / target sysroot materialization --------------------
/// Assemble all `host_deps` into a single ephemeral tree that we can mount
/// at `/usr/local` inside the container. Following Jinx, we hard-link
/// rather than byte-copy so this is essentially free. The returned
/// `TempDir` cleans up when dropped.
fn materialize_host_sandbox(&self, recipe: &Recipe) -> Result<Option<tempfile::TempDir>> {
if recipe.host_deps.is_empty() {
return Ok(None);
}
fs::create_dir_all(self.repo.join("build"))?;
let sandbox = tempfile::Builder::new()
.prefix(&format!("host-sandbox-{}-", recipe.id))
.tempdir_in(self.repo.join("build"))
.context("failed to create host sandbox tempdir")?;
log::info(
"host-sandbox",
&format!("{} <- [{}]", recipe.id, recipe.host_deps.join(", ")),
);
for dep in &recipe.host_deps {
let source = self.host_pkg_dir_by_id(dep).join("usr/local");
if !source.exists() {
bail!(
"host dependency `{dep}` has not been built at {}",
source.display()
);
}
hardlink_tree(&source, sandbox.path())?;
}
Ok(Some(sandbox))
}
fn materialize_sysroot(&self, recipe: &Recipe) -> Result<Option<PathBuf>> {
let mut deps: Vec<String> = recipe
.build_deps
.iter()
.chain(recipe.run_deps.iter())
.cloned()
.collect();
deps.sort();
deps.dedup();
if deps.is_empty() {
return Ok(None);
}
let pubkey = self.abs_config_path(&self.config.signing_pubkey);
if !pubkey.exists() {
bail!("target dependency sysroot requires a configured public signing key");
}
let sysroot = self.repo.join("build/sysroots").join(&recipe.id);
log::info(
"sysroot",
&format!("{} <- [{}]", recipe.id, deps.join(", ")),
);
Self::recreate(&sysroot)?;
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/sysroot", sysroot.display()))
.arg("-v")
.arg(format!("{}:/repo:ro", self.pkgs_dir().display()))
.arg("-v")
.arg(format!(
"{}:/etc/apk/keys/distro.rsa.pub:ro",
pubkey.display()
))
.arg(&self.config.container_image)
.arg("apk")
.arg("--root")
.arg("/sysroot")
.arg("--repository")
.arg("/repo/APKINDEX.adb")
.arg("add")
.args(&deps)
.status()
.context("failed to install target dependency sysroot")?;
if !status.success() {
bail!("target dependency sysroot install failed with {status}");
}
Ok(Some(sysroot))
}
// --- preflight & container image --------------------------------------
fn preflight_container(&self) -> Result<()> {
if self.preflight_done.get() {
return Ok(());
}
let status = Command::new(&self.config.container_runtime)
.arg("--version")
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.with_context(|| {
format!(
"{} is required but was not found",
self.config.container_runtime
)
})?;
if !status.success() {
bail!(
"{} preflight failed with {status}",
self.config.container_runtime
);
}
let dockerfile = self.abs_config_path(&self.config.container_dockerfile);
self.ensure_container_image(&dockerfile)?;
self.preflight_done.set(true);
Ok(())
}
fn ensure_container_image(&self, dockerfile: &Path) -> Result<()> {
if !dockerfile.exists() {
bail!(
"configured container Dockerfile does not exist: {}",
dockerfile.display()
);
}
let hash = self.container_build_hash(dockerfile)?;
let stamp = self.repo.join("build/container-image.hash");
if fs::read_to_string(&stamp).ok().as_deref() == Some(hash.as_str())
&& self.container_image_exists()?
{
return Ok(());
}
log::step(
"image",
&format!(
"building {} from {}",
self.config.container_image,
dockerfile.display()
),
);
let status = Command::new(&self.config.container_runtime)
.arg("build")
.arg("-f")
.arg(dockerfile)
.arg("-t")
.arg(&self.config.container_image)
.arg(&self.repo)
.status()
.with_context(|| {
format!(
"failed to build container image `{}` from {}",
self.config.container_image,
dockerfile.display()
)
})?;
if !status.success() {
bail!(
"container image build failed for `{}` with {status}",
self.config.container_image
);
}
fs::create_dir_all(stamp.parent().unwrap())?;
fs::write(stamp, hash)?;
Ok(())
}
fn container_image_exists(&self) -> Result<bool> {
let status = Command::new(&self.config.container_runtime)
.arg("image")
.arg("exists")
.arg(&self.config.container_image)
.status()
.with_context(|| {
format!(
"failed to inspect container image `{}`",
self.config.container_image
)
})?;
Ok(status.success())
}
fn container_build_hash(&self, dockerfile: &Path) -> Result<String> {
let mut hasher = Sha256::new();
hasher.update(fs::read(dockerfile)?);
hasher.update(self.config.container_image.as_bytes());
Ok(hex::encode(hasher.finalize()))
}
fn manifest_hash(&self, recipe: &Recipe) -> Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.config.target_arch.as_bytes());
hasher.update(recipe.version.as_bytes());
hasher.update(recipe.revision.to_le_bytes());
for source in &recipe.sources {
hasher.update(source.url.as_bytes());
hasher.update(source.sha256.as_bytes());
}
for patch in patches::discover(&recipe.dir)? {
hasher.update(patch.display().to_string().as_bytes());
hasher.update(fs::read(patch)?);
}
Ok(hex::encode(hasher.finalize()))
}
// --- path helpers -----------------------------------------------------
fn source_cache_dir(&self) -> PathBuf {
self.repo.join("build/cache/sources")
}
fn unpacked_source_dir(&self, recipe: &Recipe) -> PathBuf {
self.repo.join("build/sources").join(recipe.slug())
}
fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.repo
.join("build/sources")
.join(format!("{}.{kind}", recipe.slug()))
}
fn build_dir(&self, recipe: &Recipe) -> PathBuf {
self.repo.join("build/builds").join(&recipe.id)
}
fn host_build_dir(&self, recipe: &Recipe) -> PathBuf {
self.repo.join("build/host-builds").join(&recipe.id)
}
fn host_pkg_dir(&self, recipe: &Recipe) -> PathBuf {
self.repo.join("build/host-pkgs").join(&recipe.id)
}
fn host_pkg_dir_by_id(&self, host_recipe_id: &str) -> PathBuf {
self.repo.join("build/host-pkgs").join(host_recipe_id)
}
fn pkgs_dir(&self) -> PathBuf {
self.repo.join("build/pkgs")
}
fn manifest_path(&self, output_key: &str) -> PathBuf {
// Output keys may contain `:` (e.g. `host:gcc`); the manifest file
// name uses the filesystem-safe slug form instead.
let safe = output_key.replace(':', "-");
self.repo
.join("build/manifests")
.join(format!("{safe}.hash"))
}
fn abs_config_path(&self, path: &Path) -> PathBuf {
if path.is_absolute() {
path.to_path_buf()
} else {
self.repo.join(path)
}
}
fn recreate(path: &Path) -> Result<()> {
if path.exists() {
fs::remove_dir_all(path)?;
}
fs::create_dir_all(path)?;
Ok(())
}
}
fn build_phase_script(commands: &[PhaseCommand]) -> String {
let mut parts: Vec<String> = Vec::with_capacity(commands.len() + 1);
parts.push("set -e".to_owned());
for cmd in commands {
let mut tokens: Vec<String> = Vec::with_capacity(cmd.env.len() + cmd.argv.len());
for (k, v) in &cmd.env {
let value = shell_escape::unix::escape(v.as_str().into()).into_owned();
tokens.push(format!("{k}={value}"));
}
for arg in &cmd.argv {
tokens.push(shell_escape::unix::escape(arg.as_str().into()).into_owned());
}
parts.push(tokens.join(" "));
}
parts.join("\n")
}
/// Expose `/source` as a string for single-source recipes, or a struct of
/// `/source/<name>` paths for multi-source recipes.
fn source_dir_env(recipe: &Recipe) -> SourceDir {
if recipe.sources.iter().all(|s| s.name.is_empty()) {
SourceDir::Single(C_SOURCE.to_owned())
} else {
let map = recipe
.sources
.iter()
.map(|s| (s.name.clone(), format!("{C_SOURCE}/{}", s.name)))
.collect();
SourceDir::Many(map)
}
}
/// Mirror `src` into `dst` using hard links for regular files (and preserving
/// symlinks). This matches Jinx's `cp -Pplr`: no bytes are copied, just inode
/// references, so assembling a host-deps sandbox is essentially free.
fn hardlink_tree(src: &Path, dst: &Path) -> Result<()> {
fs::create_dir_all(dst)?;
for entry in walkdir::WalkDir::new(src) {
let entry = entry?;
let relative = entry.path().strip_prefix(src)?;
if relative.as_os_str().is_empty() {
continue;
}
let target = dst.join(relative);
let file_type = entry.file_type();
if file_type.is_dir() {
fs::create_dir_all(&target)?;
} else if file_type.is_symlink() {
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)?;
}
let link_target = fs::read_link(entry.path())?;
// Replace any pre-existing entry (multiple host deps may ship the
// same symlink path).
let _ = fs::remove_file(&target);
std::os::unix::fs::symlink(&link_target, &target)?;
} else if file_type.is_file() {
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)?;
}
if target.exists() {
continue;
}
fs::hard_link(entry.path(), &target).with_context(|| {
format!(
"failed to hard-link {} -> {}",
entry.path().display(),
target.display()
)
})?;
}
}
Ok(())
}
+119
View File
@@ -0,0 +1,119 @@
use crate::build::Builder;
use crate::config::Config;
use crate::graph::PackageGraph;
use crate::recipe::RecipeSet;
use anyhow::{Context, Result, bail};
use clap::{Parser, Subcommand};
use std::fs;
use std::path::PathBuf;
#[derive(Debug, Parser)]
#[command(name = "distro", version)]
pub struct Cli {
#[arg(long, default_value = ".")]
pub repo: PathBuf,
#[arg(long)]
pub skip_checks: bool,
#[command(subcommand)]
pub command: Command,
}
#[derive(Debug, Subcommand)]
pub enum Command {
Build {
package: Option<String>,
},
Rebuild {
package: Option<String>,
},
Fetch {
package: String,
},
Graph {
package: Option<String>,
},
List,
Info {
package: String,
},
Clean,
Shell {
package: String,
},
RepoIndex,
InitKey,
Rootfs {
root: PathBuf,
packages: Vec<String>,
},
Update {
#[arg(long)]
bump: bool,
packages: Vec<String>,
},
}
pub fn run(cli: Cli) -> Result<()> {
let repo = cli.repo.canonicalize().unwrap_or(cli.repo);
match &cli.command {
Command::Clean => {
let build = repo.join("build");
if build.exists() {
fs::remove_dir_all(&build)
.with_context(|| format!("failed to remove {}", build.display()))?;
}
Ok(())
}
Command::InitKey => {
let config = Config::load(&repo.join("config.star"))?;
Builder::new(repo, config, cli.skip_checks).init_key()
}
_ => {
let config = Config::load(&repo.join("config.star"))?;
let recipes = RecipeSet::load(&repo, &config)?;
if let Command::Update { bump, packages } = &cli.command {
return crate::update::run(&recipes, packages, *bump);
}
let graph = PackageGraph::new(&recipes)?;
let builder = Builder::new(repo, config, cli.skip_checks);
match cli.command {
Command::Build { package } => {
builder.build(&recipes, &graph, package.as_deref(), false)
}
Command::Rebuild { package } => {
builder.build(&recipes, &graph, package.as_deref(), true)
}
Command::Fetch { package } => builder.fetch(&recipes, &package),
Command::Graph { package } => {
for line in graph.render(package.as_deref())? {
println!("{line}");
}
Ok(())
}
Command::List => {
for output in graph.outputs() {
println!("{output}");
}
Ok(())
}
Command::Info { package } => {
let output = graph.output(&package)?;
println!("{}", serde_json::to_string_pretty(output)?);
Ok(())
}
Command::Shell { package } => builder.shell(&recipes, &graph, &package),
Command::RepoIndex => builder.repo_index(),
Command::Rootfs { root, packages } => {
if packages.is_empty() {
bail!("rootfs requires at least one package");
}
builder.rootfs(&root, &packages)
}
Command::Update { .. } => unreachable!(),
Command::Clean | Command::InitKey => unreachable!(),
}
}
}
}
+37
View File
@@ -0,0 +1,37 @@
use crate::starlark_eval::{eval_file, get_json_map, get_string, get_string_default};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Config {
pub target_arch: String,
pub options: BTreeMap<String, JsonValue>,
pub container_runtime: String,
pub container_image: String,
pub container_dockerfile: PathBuf,
pub signing_key: PathBuf,
pub signing_pubkey: PathBuf,
}
impl Config {
pub fn load(path: &Path) -> Result<Self> {
let module = eval_file(path, None, None)
.with_context(|| format!("failed to evaluate {}", path.display()))?;
Ok(Self {
target_arch: get_string(&module, "target_arch")?,
options: get_json_map(&module, "options")?,
container_runtime: get_string_default(&module, "container_runtime", "podman")?,
container_image: get_string(&module, "container_image")?,
container_dockerfile: PathBuf::from(get_string_default(
&module,
"container_dockerfile",
"Dockerfile",
)?),
signing_key: PathBuf::from(get_string(&module, "signing_key")?),
signing_pubkey: PathBuf::from(get_string(&module, "signing_pubkey")?),
})
}
}
+135
View File
@@ -0,0 +1,135 @@
use crate::recipe::{OutputPackage, PackageKind, RecipeSet, unresolved_deps};
use anyhow::{Result, anyhow, bail};
use std::collections::{BTreeMap, BTreeSet};
#[derive(Debug, Clone)]
pub struct PackageGraph {
outputs: BTreeMap<String, OutputPackage>,
target_edges: BTreeMap<String, Vec<String>>,
host_edges: BTreeMap<String, Vec<String>>,
}
impl PackageGraph {
pub fn new(recipes: &RecipeSet) -> Result<Self> {
let missing = unresolved_deps(recipes);
if !missing.is_empty() {
bail!("unresolved local dependencies:\n{}", missing.join("\n"));
}
let mut target_edges = BTreeMap::new();
let mut host_edges = BTreeMap::new();
for recipe in recipes.recipes.values() {
// host_deps always resolve into the host namespace.
let host_dep_keys: Vec<String> = recipe
.host_deps
.iter()
.map(|d| PackageKind::Host.key(d))
.collect();
for output in &recipe.outputs {
let key = output.key();
let mut edges = output.all_target_deps();
if output.name == recipe.name {
edges.extend(recipe.build_deps.iter().cloned());
edges.extend(recipe.run_deps.iter().cloned());
}
match output.kind {
PackageKind::Host => {
host_edges.insert(key, host_dep_keys.clone());
}
PackageKind::Target => {
let mut deps = host_dep_keys.clone();
deps.extend(edges);
target_edges.insert(key, deps);
}
}
}
}
Ok(Self {
outputs: recipes.outputs.clone(),
target_edges,
host_edges,
})
}
pub fn output(&self, package: &str) -> Result<&OutputPackage> {
self.outputs
.get(package)
.ok_or_else(|| anyhow!("unknown package `{package}`"))
}
pub fn outputs(&self) -> impl Iterator<Item = &str> {
self.outputs.keys().map(String::as_str)
}
pub fn build_order(&self, package: &str) -> Result<Vec<String>> {
self.output(package)?;
let mut visiting = BTreeSet::new();
let mut visited = BTreeSet::new();
let mut order = Vec::new();
self.visit(package, &mut visiting, &mut visited, &mut order)?;
Ok(order)
}
/// Topologically-ordered list of every output in the graph (host + target).
pub fn build_order_all(&self) -> Result<Vec<String>> {
let mut visiting = BTreeSet::new();
let mut visited = BTreeSet::new();
let mut order = Vec::new();
for package in self.outputs.keys() {
self.visit(package, &mut visiting, &mut visited, &mut order)?;
}
Ok(order)
}
fn visit(
&self,
package: &str,
visiting: &mut BTreeSet<String>,
visited: &mut BTreeSet<String>,
order: &mut Vec<String>,
) -> Result<()> {
if visited.contains(package) {
return Ok(());
}
if !visiting.insert(package.to_owned()) {
bail!("dependency cycle involving `{package}`");
}
let deps = self
.target_edges
.get(package)
.or_else(|| self.host_edges.get(package))
.cloned()
.unwrap_or_default();
for dep in deps {
self.visit(&dep, visiting, visited, order)?;
}
visiting.remove(package);
visited.insert(package.to_owned());
order.push(package.to_owned());
Ok(())
}
pub fn render(&self, package: Option<&str>) -> Result<Vec<String>> {
match package {
Some(package) => {
let order = self.build_order(package)?;
Ok(order
.into_iter()
.map(|pkg| format!("{pkg}: {:?}", self.edges(&pkg)))
.collect())
}
None => Ok(self
.outputs
.keys()
.map(|pkg| format!("{pkg}: {:?}", self.edges(pkg)))
.collect()),
}
}
fn edges(&self, package: &str) -> Vec<String> {
self.target_edges
.get(package)
.or_else(|| self.host_edges.get(package))
.cloned()
.unwrap_or_default()
}
}
+35
View File
@@ -0,0 +1,35 @@
//! Tiny stderr logger. We use a consistent `==> <action>: <details>` prefix
//! so progress messages are easy to scan during long builds.
use std::io::{IsTerminal, Write};
const ARROW: &str = "==>";
fn paint(color: &str, text: &str) -> String {
if std::io::stderr().is_terminal() {
format!("\x1b[{color}m{text}\x1b[0m")
} else {
text.to_owned()
}
}
fn emit(color: &str, action: &str, details: &str) {
let arrow = paint(color, ARROW);
let action = paint("1", action);
let _ = writeln!(std::io::stderr(), "{arrow} {action} {details}");
}
/// Major step, e.g. starting a build or packaging an output.
pub fn step(action: &str, details: &str) {
emit("1;34", action, details); // bold blue
}
/// Cache hit / skipped work.
pub fn skip(action: &str, details: &str) {
emit("1;33", action, details); // bold yellow
}
/// Sub-step inside a larger action.
pub fn info(action: &str, details: &str) {
emit("1;32", action, details); // bold green
}
+21
View File
@@ -0,0 +1,21 @@
mod apk;
mod build;
mod cli;
mod config;
mod graph;
mod log;
mod patches;
mod phase;
mod recipe;
mod rewrite;
mod source;
mod starlark_eval;
mod update;
use anyhow::Result;
use clap::Parser;
fn main() -> Result<()> {
let cli = cli::Cli::parse();
cli::run(cli)
}
+19
View File
@@ -0,0 +1,19 @@
use anyhow::Result;
use std::path::{Path, PathBuf};
pub fn discover(recipe_dir: &Path) -> Result<Vec<PathBuf>> {
let patch_dir = recipe_dir.join("patches");
if !patch_dir.exists() {
return Ok(Vec::new());
}
let mut patches = Vec::new();
for entry in std::fs::read_dir(&patch_dir)? {
let entry = entry?;
let path = entry.path();
if path.extension().and_then(|ext| ext.to_str()) == Some("patch") {
patches.push(path);
}
}
patches.sort();
Ok(patches)
}
+214
View File
@@ -0,0 +1,214 @@
use crate::config::Config;
use crate::starlark_eval::{eval_content_with_extra, options_literal, prepend_common_lib_load};
use allocative::Allocative;
use anyhow::{Result, anyhow, bail};
use serde::{Deserialize, Serialize};
use starlark::environment::{GlobalsBuilder, LibraryExtension};
use starlark::eval::Evaluator;
use starlark::starlark_module;
use starlark::values::none::NoneType;
use starlark::values::{ProvidesStaticType, Value};
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::path::Path;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Allocative)]
pub struct PhaseCommand {
pub argv: Vec<String>,
/// Extra environment variables exported just for this command, in the
/// order the recipe supplied them. Empty means "inherit only".
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub env: Vec<(String, String)>,
}
/// How `ctx.source_dir` is exposed to Starlark.
///
/// * `Single` is a single container path (e.g. `/source`), used when the
/// recipe declared a `source = {...}` form.
/// * `Many` is a map of source name to container path, exposed as a
/// `struct(...)` (e.g. `ctx.source_dir.linux`), used when the recipe
/// declared `sources = {"linux": {...}, ...}`.
#[derive(Debug, Clone)]
pub enum SourceDir {
Single(String),
Many(BTreeMap<String, String>),
}
/// Per-phase paths exposed to Starlark as `ctx.source_dir`, `ctx.build_dir`,
/// `ctx.dest_dir`, `ctx.prefix`, `ctx.sysroot` (Jinx-inspired).
#[derive(Debug, Clone)]
pub struct PhaseEnv<'a> {
pub source_dir: SourceDir,
pub build_dir: &'a str,
pub dest_dir: &'a str,
pub prefix: &'a str,
pub sysroot: &'a str,
}
#[derive(Debug, Default, ProvidesStaticType, Allocative)]
struct CommandStore {
commands: RefCell<Vec<PhaseCommand>>,
}
impl CommandStore {
fn push(&self, command: PhaseCommand) {
self.commands.borrow_mut().push(command);
}
}
#[starlark_module]
fn phase_globals(builder: &mut GlobalsBuilder) {
fn ctx_run<'v>(
argv: Value<'v>,
#[starlark(require = named, default = NoneType)] env: Value<'v>,
eval: &mut Evaluator,
) -> anyhow::Result<NoneType> {
let json = argv.to_json()?;
let values: Vec<String> = serde_json::from_str(&json)
.map_err(|err| anyhow!("ctx.run expects a list of strings: {err}"))?;
if values.is_empty() {
bail!("ctx.run argv cannot be empty");
}
let env_vars = parse_env(env)?;
store(eval)?.push(PhaseCommand {
argv: values,
env: env_vars,
});
Ok(NoneType)
}
fn ctx_install(
src: &str,
dst: &str,
#[starlark(require = named, default = "644")] mode: &str,
eval: &mut Evaluator,
) -> anyhow::Result<NoneType> {
store(eval)?.push(PhaseCommand {
argv: vec![
"install".to_owned(),
format!("-Dm{mode}"),
src.to_owned(),
dst.to_owned(),
],
env: Vec::new(),
});
Ok(NoneType)
}
}
fn parse_env(value: Value<'_>) -> anyhow::Result<Vec<(String, String)>> {
if value.is_none() {
return Ok(Vec::new());
}
let json = value.to_json()?;
let map: serde_json::Map<String, serde_json::Value> = serde_json::from_str(&json)
.map_err(|err| anyhow!("ctx.run env must be a dict of string -> string: {err}"))?;
let mut out = Vec::with_capacity(map.len());
for (key, val) in map {
let serde_json::Value::String(val) = val else {
bail!("ctx.run env value for `{key}` must be a string");
};
if key.is_empty() || key.contains('=') {
bail!("ctx.run env key `{key}` is not a valid variable name");
}
out.push((key, val));
}
Ok(out)
}
fn store<'a, 'b, 'c, 'd>(eval: &'a Evaluator<'b, 'c, 'd>) -> anyhow::Result<&'a CommandStore> {
eval.extra
.ok_or_else(|| anyhow!("ctx command used without command store"))?
.downcast_ref::<CommandStore>()
.ok_or_else(|| anyhow!("command store has the wrong type"))
}
pub fn collect_phase_commands(
recipe_path: &Path,
repo_root: &Path,
config: &Config,
phase: &str,
env: &PhaseEnv<'_>,
package: Option<(&str, &str)>,
) -> Result<Vec<PhaseCommand>> {
validate_identifier(phase)?;
let raw = std::fs::read_to_string(recipe_path)?;
// Auto-load helpers from `lib/common.star` so recipes never need an
// explicit `load()` for the canonical helpers.
let mut content = prepend_common_lib_load(Some(repo_root), &raw)?;
let jobs = std::thread::available_parallelism()
.map(|j| j.get())
.unwrap_or(1);
let options = options_literal(config)?;
let source_dir_expr = source_dir_literal(&env.source_dir)?;
let ctx_literal = format!(
"struct(run = ctx_run, install = ctx_install, jobs = {jobs}, options = {options}, \
source_dir = {sd}, build_dir = {bd}, dest_dir = {dd}, prefix = {pf}, sysroot = {sr})",
sd = source_dir_expr,
bd = serde_json::to_string(env.build_dir)?,
dd = serde_json::to_string(env.dest_dir)?,
pf = serde_json::to_string(env.prefix)?,
sr = serde_json::to_string(env.sysroot)?,
);
let call = match package {
Some((name, destdir)) => format!(
"\n__ctx = {ctx_literal}\n__pkg = struct(name = {n}, destdir = {d})\n{phase}(__ctx, __pkg)\n",
n = serde_json::to_string(name)?,
d = serde_json::to_string(destdir)?,
),
None => format!("\n__ctx = {ctx_literal}\n{phase}(__ctx)\n"),
};
content.push_str(&call);
let globals = GlobalsBuilder::extended_by(&[LibraryExtension::StructType])
.with(phase_globals)
.build();
let cmd_store = CommandStore::default();
eval_content_with_extra(
recipe_path,
content,
Some(config),
Some(repo_root),
globals,
Some(&cmd_store),
)?;
Ok(cmd_store.commands.into_inner())
}
fn source_dir_literal(source_dir: &SourceDir) -> Result<String> {
match source_dir {
SourceDir::Single(path) => Ok(serde_json::to_string(path)?),
SourceDir::Many(map) => {
let mut fields = Vec::with_capacity(map.len());
for (name, path) in map {
if !is_valid_field_name(name) {
bail!("source name `{name}` is not a valid Starlark identifier");
}
fields.push(format!("{name} = {}", serde_json::to_string(path)?));
}
Ok(format!("struct({})", fields.join(", ")))
}
}
}
fn is_valid_field_name(name: &str) -> bool {
let mut chars = name.chars();
match chars.next() {
Some(c) if c == '_' || c.is_ascii_alphabetic() => {}
_ => return false,
}
chars.all(|c| c == '_' || c.is_ascii_alphanumeric())
}
fn validate_identifier(name: &str) -> Result<()> {
let mut chars = name.chars();
let Some(first) = chars.next() else {
bail!("phase function name cannot be empty");
};
if !(first == '_' || first.is_ascii_alphabetic()) {
bail!("invalid phase function name `{name}`");
}
if chars.any(|ch| !(ch == '_' || ch.is_ascii_alphanumeric())) {
bail!("invalid phase function name `{name}`");
}
Ok(())
}
+452
View File
@@ -0,0 +1,452 @@
use crate::config::Config;
use crate::starlark_eval::{
eval_content, get_i32_default, get_json, get_string, get_string_default, get_string_vec,
has_name, prepend_common_lib_load,
};
use anyhow::{Result, anyhow, bail};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum PackageKind {
Host,
Target,
}
impl PackageKind {
/// Canonical `host:`/bare key form used across the graph, CLI and
/// manifest layer. Host and target trees are completely separate
/// namespaces — they may share names.
pub fn key(&self, name: &str) -> String {
match self {
PackageKind::Host => format!("host:{name}"),
PackageKind::Target => name.to_owned(),
}
}
/// Filesystem-safe variant of [`PackageKind::key`] (no `:`), used to
/// derive build/source/manifest directory names.
pub fn slug(&self, name: &str) -> String {
match self {
PackageKind::Host => format!("host-{name}"),
PackageKind::Target => name.to_owned(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Source {
/// Empty for the single-`source` form, otherwise the dict key from `sources`.
pub name: String,
pub url: String,
pub sha256: String,
/// Number of leading path components to strip when extracting (tar's
/// `--strip-components`). `0` means strip nothing.
pub strip_components: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct OutputPackage {
pub name: String,
/// Canonical key of the owning recipe (see [`PackageKind::key`]).
pub recipe: String,
pub kind: PackageKind,
pub version: String,
pub revision: i32,
pub description: String,
pub license: String,
/// Target packages installed into the build sysroot. Not propagated as
/// apk `depends:` metadata.
pub build_deps: Vec<String>,
/// Target packages declared as runtime dependencies (apk `depends:`).
/// Also installed into the sysroot so the recipe can link against them.
pub run_deps: Vec<String>,
pub install_fn: String,
}
impl OutputPackage {
pub fn key(&self) -> String {
self.kind.key(&self.name)
}
/// Union of build- and run-dependencies (used to materialize the sysroot
/// and to compute the build graph).
pub fn all_target_deps(&self) -> Vec<String> {
let mut out = self.build_deps.clone();
out.extend(self.run_deps.iter().cloned());
out
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Recipe {
pub id: String,
pub path: PathBuf,
pub dir: PathBuf,
pub name: String,
pub kind: PackageKind,
pub version: String,
pub revision: i32,
pub description: String,
pub license: String,
pub sources: Vec<Source>,
pub host_deps: Vec<String>,
pub build_deps: Vec<String>,
pub run_deps: Vec<String>,
pub outputs: Vec<OutputPackage>,
pub configure_fn: Option<String>,
pub build_fn: Option<String>,
pub check_fn: Option<String>,
}
impl Recipe {
/// Canonical key (`host:<id>` or `<id>`), used as the recipe-level
/// identifier in graphs, manifests and CLI references.
pub fn key(&self) -> String {
self.kind.key(&self.id)
}
/// Filesystem-safe variant of [`Recipe::key`].
pub fn slug(&self) -> String {
self.kind.slug(&self.id)
}
}
#[derive(Debug, Clone)]
pub struct RecipeSet {
pub recipes: BTreeMap<String, Recipe>,
pub outputs: BTreeMap<String, OutputPackage>,
}
impl RecipeSet {
/// Discover recipes under `repo_root`:
/// * `recipes/<name>/recipe.star` → target packages
/// * `host-recipes/<name>/recipe.star` → host packages
pub fn load(repo_root: &Path, config: &Config) -> Result<Self> {
let mut recipes = BTreeMap::new();
for (subdir, kind) in [
("recipes", PackageKind::Target),
("host-recipes", PackageKind::Host),
] {
let root = repo_root.join(subdir);
if !root.exists() {
continue;
}
for entry in WalkDir::new(&root).follow_links(false) {
let entry = entry?;
if entry.file_type().is_file() && entry.file_name() == "recipe.star" {
let recipe = Recipe::load(entry.path(), config, repo_root, kind.clone())?;
let key = recipe.key();
if recipes.insert(key.clone(), recipe).is_some() {
bail!("duplicate recipe `{key}` below {}", root.display());
}
}
}
}
let mut outputs = BTreeMap::new();
for recipe in recipes.values() {
for output in &recipe.outputs {
let key = output.key();
if outputs.insert(key.clone(), output.clone()).is_some() {
bail!("duplicate package output `{key}`");
}
}
}
Ok(Self { recipes, outputs })
}
/// Look up a recipe by the package key produced by an output.
pub fn recipe_for_package(&self, package: &str) -> Result<&Recipe> {
let output = self
.outputs
.get(package)
.ok_or_else(|| anyhow!("unknown package `{package}`"))?;
self.recipes.get(&output.recipe).ok_or_else(|| {
anyhow!(
"package `{package}` references missing recipe `{}`",
output.recipe
)
})
}
/// Resolve a user-supplied reference (recipe key, output key, or bare
/// id — provided it isn't ambiguous between the host and target trees).
pub fn recipe_by_user_ref(&self, name: &str) -> Result<&Recipe> {
if let Some(recipe) = self.recipes.get(name) {
return Ok(recipe);
}
if self.outputs.contains_key(name) {
return self.recipe_for_package(name);
}
// Bare id: search both trees, error on ambiguity.
let host_key = PackageKind::Host.key(name);
let target_key = PackageKind::Target.key(name);
match (self.recipes.get(&host_key), self.recipes.get(&target_key)) {
(Some(_), Some(_)) => bail!(
"`{name}` is ambiguous: matches both `{host_key}` and `{target_key}`; \
use the explicit form"
),
(Some(r), None) | (None, Some(r)) => Ok(r),
(None, None) => bail!("unknown recipe `{name}`"),
}
}
}
impl Recipe {
pub fn load(path: &Path, config: &Config, repo_root: &Path, kind: PackageKind) -> Result<Self> {
// Auto-load helpers from `lib/common.star` so recipes never need an
// explicit `load()` for the canonical helpers.
let raw = std::fs::read_to_string(path)?;
let content = prepend_common_lib_load(Some(repo_root), &raw)?;
let module = eval_content(
path,
content,
Some(config),
Some(repo_root),
starlark::environment::Globals::standard(),
)?;
let dir = path
.parent()
.unwrap_or_else(|| Path::new("."))
.to_path_buf();
let id = dir
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| anyhow!("recipe path has no package directory: {}", path.display()))?
.to_owned();
let name = get_string(&module, "name")?;
let version = get_string(&module, "version")?;
let revision = get_i32_default(&module, "revision", 0)?;
let description = get_string_default(&module, "description", "???")?;
let license = get_string_default(&module, "license", "???")?;
let build_deps = get_string_vec(&module, "build_deps")?;
let run_deps = get_string_vec(&module, "run_deps")?;
if has_name(&module, "deps") {
bail!(
"recipe `{id}` uses removed `deps`; split it into `build_deps` \
(sysroot only) and `run_deps` (apk depends + sysroot)"
);
}
let host_deps = get_string_vec(&module, "host_deps")?;
let sources = parse_sources(get_json(&module, "sources")?, get_json(&module, "source")?)?;
let subpackages = parse_subpackages(get_json(&module, "subpackages")?)?;
let mut outputs = Vec::new();
let recipe_key = kind.key(&id);
outputs.push(OutputPackage {
name: name.clone(),
recipe: recipe_key.clone(),
kind: kind.clone(),
version: version.clone(),
revision,
description: description.clone(),
license: license.clone(),
build_deps: build_deps.clone(),
run_deps: run_deps.clone(),
install_fn: "install".to_owned(),
});
for subpkg in subpackages {
let sub_name = subpkg
.get("name")
.and_then(JsonValue::as_str)
.ok_or_else(|| anyhow!("subpackage in `{name}` is missing string `name`"))?
.to_owned();
if subpkg.contains_key("deps") {
bail!(
"subpackage `{sub_name}` in `{id}` uses removed `deps`; use \
`build_deps` and/or `run_deps`"
);
}
outputs.push(OutputPackage {
name: sub_name,
recipe: recipe_key.clone(),
kind: kind.clone(),
version: version.clone(),
revision,
description: subpkg
.get("description")
.and_then(JsonValue::as_str)
.unwrap_or(&description)
.to_owned(),
license: subpkg
.get("license")
.and_then(JsonValue::as_str)
.unwrap_or(&license)
.to_owned(),
build_deps: json_string_list(subpkg.get("build_deps"), "subpackage build_deps")?
.unwrap_or_default(),
run_deps: json_string_list(subpkg.get("run_deps"), "subpackage run_deps")?
.unwrap_or_default(),
install_fn: subpkg
.get("install")
.and_then(JsonValue::as_str)
.unwrap_or("install")
.to_owned(),
});
}
validate_required(&outputs)?;
Ok(Self {
id,
path: path.to_path_buf(),
dir,
name,
kind,
version,
revision,
description,
license,
sources,
host_deps,
build_deps,
run_deps,
outputs,
configure_fn: has_name(&module, "configure").then_some("configure".to_owned()),
build_fn: has_name(&module, "build").then_some("build".to_owned()),
check_fn: has_name(&module, "check").then_some("check".to_owned()),
})
}
}
fn validate_required(outputs: &[OutputPackage]) -> Result<()> {
for output in outputs {
if output.name.trim().is_empty() {
bail!("package output name cannot be empty");
}
for (field, value) in [
("version", &output.version),
("description", &output.description),
("license", &output.license),
] {
if value.trim().is_empty() {
bail!(
"package `{}` has empty required field `{field}`",
output.name
);
}
}
}
Ok(())
}
fn parse_sources(
sources: Option<JsonValue>,
legacy_source: Option<JsonValue>,
) -> Result<Vec<Source>> {
match (sources, legacy_source) {
(Some(_), Some(_)) => bail!("recipe defines both `sources` and `source`; use only one"),
(None, None) => Ok(Vec::new()),
(None, Some(single)) => {
let obj = single
.as_object()
.ok_or_else(|| anyhow!("`source` must be a dict"))?;
Ok(vec![parse_source_entry(String::new(), obj)?])
}
(Some(multi), None) => {
let obj = multi
.as_object()
.ok_or_else(|| anyhow!("`sources` must be a dict of {{name: source}}"))?;
obj.iter()
.map(|(name, value)| {
if name.is_empty() {
bail!("source name in `sources` cannot be empty");
}
let entry = value
.as_object()
.ok_or_else(|| anyhow!("source `{name}` must be a dict"))?;
parse_source_entry(name.clone(), entry)
})
.collect()
}
}
}
fn parse_source_entry(name: String, obj: &serde_json::Map<String, JsonValue>) -> Result<Source> {
let url = obj
.get("url")
.and_then(JsonValue::as_str)
.ok_or_else(|| anyhow!("source entry missing string `url`"))?
.to_owned();
let sha256 = obj
.get("sha256")
.and_then(JsonValue::as_str)
.unwrap_or("???")
.to_owned();
let strip_components = match obj.get("strip_components") {
None => 0,
Some(JsonValue::Number(n)) => n
.as_u64()
.and_then(|v| u32::try_from(v).ok())
.ok_or_else(|| anyhow!("source `strip_components` must be a non-negative integer"))?,
Some(_) => bail!("source `strip_components` must be an integer"),
};
Ok(Source {
name,
url,
sha256,
strip_components,
})
}
fn parse_subpackages(value: Option<JsonValue>) -> Result<Vec<serde_json::Map<String, JsonValue>>> {
match value {
Some(JsonValue::Array(values)) => values
.into_iter()
.map(|value| {
value
.as_object()
.cloned()
.ok_or_else(|| anyhow!("subpackages entries must be objects"))
})
.collect(),
Some(_) => bail!("subpackages must be a list of objects"),
None => Ok(Vec::new()),
}
}
fn json_string_list(value: Option<&JsonValue>, label: &str) -> Result<Option<Vec<String>>> {
match value {
Some(JsonValue::Array(values)) => values
.iter()
.map(|value| {
value
.as_str()
.map(ToOwned::to_owned)
.ok_or_else(|| anyhow!("{label} must contain only strings"))
})
.collect::<Result<Vec<_>>>()
.map(Some),
Some(_) => bail!("{label} must be a string list"),
None => Ok(None),
}
}
pub fn unresolved_deps(recipes: &RecipeSet) -> Vec<String> {
let names: BTreeSet<_> = recipes.outputs.keys().cloned().collect();
let mut missing = Vec::new();
for recipe in recipes.recipes.values() {
// host_deps always refer to host outputs (canonical `host:<name>`);
// build_deps / run_deps refer to target outputs (bare names).
for dep in &recipe.host_deps {
let key = PackageKind::Host.key(dep);
if !names.contains(&key) {
missing.push(format!("{} -> {key}", recipe.key()));
}
}
for dep in recipe.build_deps.iter().chain(recipe.run_deps.iter()) {
if !names.contains(dep) {
missing.push(format!("{} -> {dep}", recipe.key()));
}
}
for output in &recipe.outputs {
for dep in output.build_deps.iter().chain(output.run_deps.iter()) {
if !names.contains(dep) {
missing.push(format!("{} -> {dep}", output.key()));
}
}
}
}
missing
}
+39
View File
@@ -0,0 +1,39 @@
use anyhow::{Context, Result};
use std::fs;
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Rewrite {
pub field: String,
pub old: String,
pub new: String,
}
pub fn backup_path(path: &Path) -> PathBuf {
let mut backup = path.as_os_str().to_os_string();
backup.push(".bak");
PathBuf::from(backup)
}
pub fn rewrite_placeholders(path: &Path, rewrites: &[Rewrite]) -> Result<bool> {
if rewrites.is_empty() {
return Ok(false);
}
let original =
fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?;
let mut updated = original.clone();
for rewrite in rewrites {
let quoted_old = format!("{} = \"{}\"", rewrite.field, rewrite.old);
let quoted_new = format!("{} = \"{}\"", rewrite.field, rewrite.new);
updated = updated.replacen(&quoted_old, &quoted_new, 1);
let dict_old = format!("\"{}\": \"{}\"", rewrite.field, rewrite.old);
let dict_new = format!("\"{}\": \"{}\"", rewrite.field, rewrite.new);
updated = updated.replacen(&dict_old, &dict_new, 1);
}
if updated == original {
return Ok(false);
}
fs::write(backup_path(path), original)?;
fs::write(path, updated)?;
Ok(true)
}
+73
View File
@@ -0,0 +1,73 @@
use crate::log;
use crate::recipe::Recipe;
use crate::rewrite::{Rewrite, rewrite_placeholders};
use anyhow::{Context, Result, bail};
use sha2::{Digest, Sha256};
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
pub fn fetch_sources(recipe: &Recipe, cache_dir: &Path) -> Result<Vec<PathBuf>> {
fs::create_dir_all(cache_dir)?;
let mut rewrites = Vec::new();
let mut paths = Vec::new();
for source in &recipe.sources {
let label = if source.name.is_empty() {
recipe.key()
} else {
format!("{}:{}", recipe.key(), source.name)
};
let cached = source.sha256 != "???" && cache_dir.join(&source.sha256).exists();
if cached {
log::skip("cached", &format!("{label} ({})", source.url));
paths.push(cache_dir.join(&source.sha256));
continue;
}
log::step("fetch", &format!("{label} <- {}", source.url));
let bytes = download(&source.url)?;
let actual = sha256_hex(&bytes);
if source.sha256 == "???" {
log::info("sha256", &format!("{label} = {actual}"));
rewrites.push(Rewrite {
field: "sha256".into(),
old: "???".into(),
new: actual.clone(),
});
} else if source.sha256 != actual {
bail!(
"checksum mismatch for {}: expected {}, got {}",
source.url,
source.sha256,
actual
);
}
let path = cache_dir.join(&actual);
if !path.exists() {
fs::write(&path, &bytes)
.with_context(|| format!("failed to write {}", path.display()))?;
}
paths.push(path);
}
rewrite_placeholders(&recipe.path, &rewrites)?;
Ok(paths)
}
fn download(url: &str) -> Result<Vec<u8>> {
if let Some(path) = url.strip_prefix("file://") {
return Ok(fs::read(path)?);
}
// Some mirrors reject requests without a User-Agent with HTTP 403, so set an explicit one.
let client = reqwest::blocking::Client::builder()
.user_agent(concat!("distro/", env!("CARGO_PKG_VERSION")))
.build()?;
let mut response = client.get(url).send()?.error_for_status()?;
let mut bytes = Vec::new();
response.read_to_end(&mut bytes)?;
Ok(bytes)
}
pub fn sha256_hex(bytes: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(bytes);
hex::encode(hasher.finalize())
}
+365
View File
@@ -0,0 +1,365 @@
use crate::config::Config;
use allocative::Allocative;
use anyhow::{Result, anyhow, bail};
use serde_json::Value as JsonValue;
use starlark::environment::{FrozenModule, Globals, Module};
use starlark::eval::{Evaluator, FileLoader};
use starlark::starlark_simple_value;
use starlark::syntax::{AstModule, Dialect};
use starlark::values::dict::AllocDict;
use starlark::values::{AnyLifetime, Heap, NoSerialize, ProvidesStaticType, StarlarkValue, Value};
use starlark_derive::starlark_value;
use std::collections::{BTreeMap, HashMap};
use std::fmt::{self, Display};
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, ProvidesStaticType, NoSerialize, Allocative)]
pub struct SettingsValue {
target_arch: String,
container_runtime: String,
container_image: String,
container_dockerfile: String,
options: BTreeMap<String, String>,
}
impl From<&Config> for SettingsValue {
fn from(config: &Config) -> Self {
Self {
target_arch: config.target_arch.clone(),
container_runtime: config.container_runtime.clone(),
container_image: config.container_image.clone(),
container_dockerfile: config.container_dockerfile.display().to_string(),
options: config
.options
.iter()
.map(|(key, value)| (key.clone(), option_value_to_string(value)))
.collect(),
}
}
}
impl Display for SettingsValue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "settings")
}
}
starlark_simple_value!(SettingsValue);
#[starlark_value(type = "settings")]
impl<'v> StarlarkValue<'v> for SettingsValue {
fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
match attr {
"target_arch" => Some(heap.alloc(self.target_arch.as_str())),
"container_runtime" => Some(heap.alloc(self.container_runtime.as_str())),
"container_image" => Some(heap.alloc(self.container_image.as_str())),
"container_dockerfile" => Some(heap.alloc(self.container_dockerfile.as_str())),
"options" => Some(heap.alloc(AllocDict(self.options.clone()))),
_ => None,
}
}
}
pub fn eval_file(
path: &Path,
settings: Option<&Config>,
repo_root: Option<&Path>,
) -> Result<Module> {
let content = std::fs::read_to_string(path)?;
eval_content(path, content, settings, repo_root, Globals::standard())
}
/// Path of the implicit helper library auto-loaded into every recipe.
pub const COMMON_LIB_MODULE: &str = "//lib:common.star";
const COMMON_LIB_RELATIVE: &str = "lib/common.star";
/// Names exported by `lib/common.star`, if the file exists. Empty otherwise.
pub fn common_lib_names(repo_root: &Path) -> Result<Vec<String>> {
let path = repo_root.join(COMMON_LIB_RELATIVE);
if !path.exists() {
return Ok(Vec::new());
}
let module = eval_file(&path, None, Some(repo_root))?;
Ok(module
.names()
.map(|n| n.as_str().to_owned())
.filter(|n| !n.starts_with('_') && n != "settings")
.collect())
}
/// Prepend an implicit `load("//lib:common.star", ...)` so every recipe sees
/// the shared helpers without an explicit import. Does nothing if there's no
/// `lib/common.star` or no repo root.
pub fn prepend_common_lib_load(repo_root: Option<&Path>, content: &str) -> Result<String> {
let Some(root) = repo_root else {
return Ok(content.to_owned());
};
let names = common_lib_names(root)?;
if names.is_empty() {
return Ok(content.to_owned());
}
let names_lit = names
.iter()
.map(|n| format!("{:?}", n))
.collect::<Vec<_>>()
.join(", ");
Ok(format!(
"load(\"{COMMON_LIB_MODULE}\", {names_lit})\n{content}"
))
}
pub fn eval_content(
path: &Path,
content: String,
settings: Option<&Config>,
repo_root: Option<&Path>,
globals: Globals,
) -> Result<Module> {
eval_content_with_extra(path, content, settings, repo_root, globals, None)
}
pub fn eval_content_with_extra<'a>(
path: &Path,
content: String,
settings: Option<&Config>,
repo_root: Option<&Path>,
globals: Globals,
extra: Option<&'a dyn AnyLifetime<'a>>,
) -> Result<Module> {
let filename = path.display().to_string();
let ast = AstModule::parse(
&filename,
content,
&Dialect {
enable_f_strings: true,
enable_top_level_stmt: true,
..Dialect::Standard
},
)
.map_err(|err| anyhow!("{err}"))?;
let loader = match repo_root {
Some(root) => Some(RepoFileLoader::new(root, settings, globals.clone(), &ast)?),
None if !ast.loads().is_empty() => {
bail!(
"load() requires a repo root while evaluating {}",
path.display()
)
}
None => None,
};
let module = Module::new();
if let Some(config) = settings {
module.set("settings", module.heap().alloc(SettingsValue::from(config)));
}
{
let mut eval = Evaluator::new(&module);
if let Some(loader) = &loader {
eval.set_loader(loader);
}
eval.extra = extra;
eval.eval_module(ast, &globals)
.map_err(|err| anyhow!("{err}"))?;
}
Ok(module)
}
#[derive(Clone)]
struct RepoFileLoader {
modules: HashMap<String, FrozenModule>,
}
impl RepoFileLoader {
fn new(
repo_root: &Path,
settings: Option<&Config>,
globals: Globals,
ast: &AstModule,
) -> Result<Self> {
let mut modules = HashMap::new();
for load in ast.loads() {
load_module(
repo_root,
settings,
globals.clone(),
load.module_id,
&mut modules,
)?;
}
Ok(Self { modules })
}
}
impl FileLoader for RepoFileLoader {
fn load(&self, path: &str) -> starlark::Result<FrozenModule> {
self.modules
.get(path)
.cloned()
.ok_or_else(|| starlark::Error::new_other(anyhow!("unknown Starlark module `{path}`")))
}
}
fn load_module(
repo_root: &Path,
settings: Option<&Config>,
globals: Globals,
module_id: &str,
modules: &mut HashMap<String, FrozenModule>,
) -> Result<()> {
if modules.contains_key(module_id) {
return Ok(());
}
let path = resolve_load_path(repo_root, module_id)?;
let content = std::fs::read_to_string(&path)?;
let filename = path.display().to_string();
let ast =
AstModule::parse(&filename, content, &Dialect::Standard).map_err(|err| anyhow!("{err}"))?;
for load in ast.loads() {
load_module(
repo_root,
settings,
globals.clone(),
load.module_id,
modules,
)?;
}
let nested_loader = RepoFileLoader {
modules: modules.clone(),
};
let module = Module::new();
if let Some(config) = settings {
module.set("settings", module.heap().alloc(SettingsValue::from(config)));
}
{
let mut eval = Evaluator::new(&module);
eval.set_loader(&nested_loader);
eval.eval_module(ast, &globals)
.map_err(|err| anyhow!("{err}"))?;
}
let frozen = module.freeze().map_err(|err| anyhow!("{err:?}"))?;
modules.insert(module_id.to_owned(), frozen);
Ok(())
}
fn resolve_load_path(repo_root: &Path, module_id: &str) -> Result<PathBuf> {
let relative = if let Some(stripped) = module_id.strip_prefix("//") {
stripped.replace(':', "/")
} else {
module_id.to_owned()
};
let path = repo_root.join(relative);
let canonical_root = repo_root
.canonicalize()
.unwrap_or_else(|_| repo_root.to_path_buf());
let canonical_path = path.canonicalize().unwrap_or(path);
if !canonical_path.starts_with(&canonical_root) {
bail!("Starlark load escapes repo root: {module_id}");
}
Ok(canonical_path)
}
pub fn options_literal(config: &Config) -> Result<String> {
json_to_starlark_literal(&JsonValue::Object(
config.options.clone().into_iter().collect(),
))
}
fn option_value_to_string(value: &JsonValue) -> String {
match value {
JsonValue::String(value) => value.clone(),
JsonValue::Bool(value) => value.to_string(),
JsonValue::Number(value) => value.to_string(),
JsonValue::Null => "null".to_owned(),
JsonValue::Array(_) | JsonValue::Object(_) => value.to_string(),
}
}
fn json_to_starlark_literal(value: &JsonValue) -> Result<String> {
Ok(match value {
JsonValue::Null => "None".to_owned(),
JsonValue::Bool(true) => "True".to_owned(),
JsonValue::Bool(false) => "False".to_owned(),
JsonValue::Number(value) => value.to_string(),
JsonValue::String(value) => serde_json::to_string(value)?,
JsonValue::Array(values) => format!(
"[{}]",
values
.iter()
.map(json_to_starlark_literal)
.collect::<Result<Vec<_>>>()?
.join(", ")
),
JsonValue::Object(values) => format!(
"{{{}}}",
values
.iter()
.map(|(key, value)| {
Ok(format!(
"{}: {}",
serde_json::to_string(key)?,
json_to_starlark_literal(value)?
))
})
.collect::<Result<Vec<_>>>()?
.join(", ")
),
})
}
pub fn get_string(module: &Module, name: &str) -> Result<String> {
module
.get(name)
.and_then(|v| v.unpack_str().map(ToOwned::to_owned))
.ok_or_else(|| anyhow!("missing or non-string Starlark variable `{name}`"))
}
pub fn get_string_default(module: &Module, name: &str, default: &str) -> Result<String> {
Ok(match module.get(name) {
Some(value) => value
.unpack_str()
.ok_or_else(|| anyhow!("non-string Starlark variable `{name}`"))?
.to_owned(),
None => default.to_owned(),
})
}
pub fn get_i32_default(module: &Module, name: &str, default: i32) -> Result<i32> {
Ok(match module.get(name) {
Some(value) => value
.unpack_i32()
.ok_or_else(|| anyhow!("non-integer Starlark variable `{name}`"))?,
None => default,
})
}
pub fn has_name(module: &Module, name: &str) -> bool {
module.get(name).is_some()
}
pub fn get_json(module: &Module, name: &str) -> Result<Option<JsonValue>> {
match module.get(name) {
Some(value) => Ok(Some(serde_json::from_str(&value.to_json()?)?)),
None => Ok(None),
}
}
pub fn get_string_vec(module: &Module, name: &str) -> Result<Vec<String>> {
match get_json(module, name)? {
Some(JsonValue::Array(values)) => values
.into_iter()
.map(|value| match value {
JsonValue::String(s) => Ok(s),
_ => bail!("`{name}` must be a list of strings"),
})
.collect(),
Some(_) => bail!("`{name}` must be a list of strings"),
None => Ok(Vec::new()),
}
}
pub fn get_json_map(module: &Module, name: &str) -> Result<BTreeMap<String, JsonValue>> {
match get_json(module, name)? {
Some(JsonValue::Object(values)) => Ok(values.into_iter().collect()),
Some(_) => bail!("`{name}` must be a dict"),
None => Ok(BTreeMap::new()),
}
}
+256
View File
@@ -0,0 +1,256 @@
//! `distro update` — check repology for newer upstream versions and bump
//! recipes in-place.
//!
//! For each requested recipe we query
//! `https://repology.org/api/v1/project/<name>` and pick the highest version
//! among entries whose `status` is `"newest"` or `"unique"` (i.e. not
//! outdated, not rolling/devel, not ignored). If that version is strictly
//! greater than the recipe's current `version`, we rewrite:
//!
//! * `version = "..."` → the new upstream version
//! * `revision = N` → `revision = 1`
//! * every `"sha256": "..."` → `"sha256": "???"` so the next fetch fills it
//!
//! Repology asks API clients to set a descriptive User-Agent.
use crate::log;
use crate::recipe::{Recipe, RecipeSet};
use anyhow::{Context, Result, bail};
use serde::Deserialize;
use std::cmp::Ordering;
use std::fs;
use std::path::Path;
const REPOLOGY_API: &str = "https://repology.org/api/v1/project";
#[derive(Debug, Deserialize)]
struct RepologyEntry {
#[serde(default)]
version: String,
#[serde(default)]
status: String,
}
pub fn run(recipes: &RecipeSet, names: &[String], bump: bool) -> Result<()> {
let targets: Vec<&Recipe> = if names.is_empty() {
recipes.recipes.values().collect()
} else {
let mut out = Vec::with_capacity(names.len());
for name in names {
out.push(recipes.recipe_by_user_ref(name)?);
}
out
};
let client = reqwest::blocking::Client::builder()
.user_agent(concat!(
"distro/",
env!("CARGO_PKG_VERSION"),
" (+repology version checker)"
))
.build()?;
let mut outdated = 0usize;
let mut bumped = 0usize;
let mut up_to_date = 0usize;
let mut errored = 0usize;
for recipe in targets {
match check_one(&client, recipe) {
Ok(Some(new_version)) => {
if bump {
bump_recipe(&recipe.path, &new_version)?;
log::step(
"bump",
&format!("{}: {} -> {}", recipe.key(), recipe.version, new_version),
);
bumped += 1;
} else {
log::step(
"outdated",
&format!("{}: {} -> {}", recipe.key(), recipe.version, new_version),
);
outdated += 1;
}
}
Ok(None) => {
log::skip(
"up-to-date",
&format!("{} {}", recipe.key(), recipe.version),
);
up_to_date += 1;
}
Err(e) => {
log::info("error", &format!("{}: {e}", recipe.key()));
errored += 1;
}
}
}
if bump {
log::step(
"summary",
&format!("{bumped} bumped, {up_to_date} up-to-date, {errored} errored"),
);
} else {
log::step(
"summary",
&format!(
"{outdated} outdated, {up_to_date} up-to-date, {errored} errored (re-run with --bump to rewrite recipes)"
),
);
}
Ok(())
}
fn check_one(client: &reqwest::blocking::Client, recipe: &Recipe) -> Result<Option<String>> {
let url = format!("{REPOLOGY_API}/{}", recipe.name);
let resp = client
.get(&url)
.send()
.with_context(|| format!("GET {url}"))?;
if resp.status() == reqwest::StatusCode::NOT_FOUND {
bail!("repology has no project named `{}`", recipe.name);
}
let entries: Vec<RepologyEntry> = serde_json::from_slice(&resp.error_for_status()?.bytes()?)
.context("failed to parse repology response")?;
let latest = entries
.iter()
.filter(|e| matches!(e.status.as_str(), "newest" | "unique"))
.map(|e| e.version.as_str())
.max_by(|a, b| natural_cmp(a, b));
match latest {
Some(v) if natural_cmp(v, &recipe.version) == Ordering::Greater => Ok(Some(v.to_owned())),
_ => Ok(None),
}
}
fn bump_recipe(path: &Path, new_version: &str) -> Result<()> {
let original =
fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?;
let mut out = String::with_capacity(original.len());
let mut version_replaced = false;
let mut revision_replaced = false;
for line in original.split_inclusive('\n') {
let trimmed = line.trim_start();
if !version_replaced && trimmed.starts_with("version") {
if let Some(replaced) = replace_string_assignment(line, "version", new_version) {
out.push_str(&replaced);
version_replaced = true;
continue;
}
}
if !revision_replaced && trimmed.starts_with("revision") {
if let Some(replaced) = replace_int_assignment(line, "revision", 1) {
out.push_str(&replaced);
revision_replaced = true;
continue;
}
}
out.push_str(line);
}
if !version_replaced {
bail!("could not find `version = \"...\"` in {}", path.display());
}
// Reset every sha256 placeholder so the next fetch re-derives it.
out = out.replace("\"sha256\": \"", "\x00sha256_marker\x00\"");
out = regex_lite_replace_sha(&out);
out = out.replace("\x00sha256_marker\x00\"", "\"sha256\": \"");
fs::write(path, out).with_context(|| format!("failed to write {}", path.display()))?;
Ok(())
}
/// Replace every quoted sha256 value with `"???"`, leaving keys/whitespace
/// alone. Avoids a full regex dep by walking the string by hand.
fn regex_lite_replace_sha(input: &str) -> String {
let mut out = String::with_capacity(input.len());
let bytes = input.as_bytes();
let needle = b"\x00sha256_marker\x00\"";
let mut i = 0;
while i < bytes.len() {
if bytes[i..].starts_with(needle) {
out.push_str("\x00sha256_marker\x00\"???\"");
i += needle.len();
// skip the original value up to the closing quote
while i < bytes.len() && bytes[i] != b'"' {
i += 1;
}
if i < bytes.len() {
i += 1; // consume the closing quote we replaced
}
} else {
out.push(bytes[i] as char);
i += 1;
}
}
out
}
fn replace_string_assignment(line: &str, key: &str, new_value: &str) -> Option<String> {
// Match `<key><space>=<space>"value"` while preserving surrounding text
// (indentation, trailing newline).
let stripped = line.strip_prefix(key)?;
let rest = stripped.trim_start_matches(|c: char| c == ' ' || c == '\t');
let after_eq = rest.strip_prefix('=')?.trim_start();
let after_quote = after_eq.strip_prefix('"')?;
let end = after_quote.find('"')?;
let trailing = &after_quote[end + 1..];
Some(format!("{key} = \"{new_value}\"{trailing}"))
}
fn replace_int_assignment(line: &str, key: &str, new_value: i32) -> Option<String> {
let stripped = line.strip_prefix(key)?;
let rest = stripped.trim_start_matches(|c: char| c == ' ' || c == '\t');
let after_eq = rest.strip_prefix('=')?.trim_start();
let end = after_eq.find(|c: char| !c.is_ascii_digit())?;
let trailing = &after_eq[end..];
Some(format!("{key} = {new_value}{trailing}"))
}
/// dpkg-ish natural comparison: split into runs of digits and non-digits and
/// compare numerically where both sides are digits, lexicographically
/// otherwise. Good enough for upstream tarball versions.
fn natural_cmp(a: &str, b: &str) -> Ordering {
let mut ai = a.chars().peekable();
let mut bi = b.chars().peekable();
loop {
match (ai.peek().copied(), bi.peek().copied()) {
(None, None) => return Ordering::Equal,
(None, _) => return Ordering::Less,
(_, None) => return Ordering::Greater,
(Some(x), Some(y)) if x.is_ascii_digit() && y.is_ascii_digit() => {
let mut na = String::new();
while let Some(&c) = ai.peek() {
if c.is_ascii_digit() {
na.push(c);
ai.next();
} else {
break;
}
}
let mut nb = String::new();
while let Some(&c) = bi.peek() {
if c.is_ascii_digit() {
nb.push(c);
bi.next();
} else {
break;
}
}
let xa: u64 = na.parse().unwrap_or(0);
let xb: u64 = nb.parse().unwrap_or(0);
match xa.cmp(&xb) {
Ordering::Equal => continue,
other => return other,
}
}
(Some(x), Some(y)) => match x.cmp(&y) {
Ordering::Equal => {
ai.next();
bi.next();
}
other => return other,
},
}
}
}