marv7000/distro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: marv7000/distro:main
Branches Tags
marv7000/distro:main marv7000/distro:python marv7000/distro:rewrite2 marv7000/distro:rewrite
..
compare: marv7000/distro:0c9a3fde9452a8f4a16a45779b3428715acc7480
Branches Tags
marv7000/distro:python marv7000/distro:rewrite2 marv7000/distro:rewrite marv7000/distro:main

2 Commits
main .. 0c9a3fde94

Author SHA1 Message Date
iretq 0c9a3fde94 wip 2 2026-05-18 20:39:21 +02:00
iretq 0d610fd2de wip 2026-05-18 19:32:33 +02:00
34 changed files with 1941 additions and 3050 deletions
Expand all files Collapse all files
Dockerfile
+2 -20
View File
@@ -1,7 +1,6 @@
FROM docker.io/library/alpine:edge
FROM alpine:3.22.4
RUN apk upgrade --no-cache && \
apk add --no-cache \
RUN apk upgrade --no-cache && apk add --no-cache \
alpine-sdk \
apk-tools \
autoconf \
@@ -38,21 +37,4 @@ RUN apk upgrade --no-cache && \
xz \
zstd
RUN rm -rf /tmp/mkpkg-root /tmp/distro-preflight.apk /tmp/APKINDEX.adb /tmp/distro-preflight.rsa && \
openssl genrsa -out /tmp/distro-preflight.rsa 2048 >/dev/null 2>&1 && \
openssl rsa -in /tmp/distro-preflight.rsa -pubout -out /etc/apk/keys/distro-preflight.rsa.pub >/dev/null 2>&1 && \
mkdir -p /tmp/mkpkg-root/usr/share/distro && \
printf ok > /tmp/mkpkg-root/usr/share/distro/preflight && \
apk --sign-key /tmp/distro-preflight.rsa mkpkg \
--files /tmp/mkpkg-root \
--output /tmp/distro-preflight.apk \
--info name:distro-preflight \
--info version:0-r0 \
--info arch:noarch \
--info description:preflight \
--info license:MIT >/dev/null && \
apk --sign-key /tmp/distro-preflight.rsa mkndx -o /tmp/APKINDEX.adb /tmp/distro-preflight.apk >/dev/null && \
test -s /tmp/distro-preflight.apk && \
test -s /tmp/APKINDEX.adb
WORKDIR /work
README.md
-2
View File
@@ -1,2 +0,0 @@
# distro
config.star
+18 -18
View File
@@ -1,11 +1,8 @@
container_runtime = "podman"
container_image = "localhost/distro-builder:latest"
container_image = "local/distro-builder:latest"
container_dockerfile = "Dockerfile"
signing_key = "build/keys/distro.rsa"
signing_pubkey = "build/keys/distro.rsa.pub"
target_arch = "x86_64"
arch = "x86_64"
libc = "musl"
host_cflags = "-O2 -pipe"
@@ -16,21 +13,24 @@ target_cflags = host_cflags
target_cxxflags = host_cxxflags
target_ldflags = host_ldflags + " -Wl,-z,now"
if target_arch == "x86_64":
if arch == "x86_64":
flags = " -march=x86-64-v3 -mtune=generic -fstack-clash-protection -fstack-protector-strong -fcf-protection"
target_cflags += flags
target_cxxflags += flags
target_ldflags += " -Wl,-z,pack-relative-relocs"
options = {
"libc": libc,
"target_triple": target_arch + "-linux-" + libc,
"host_cflags": host_cflags,
"host_cxxflags": host_cxxflags,
"host_ldflags": host_ldflags,
"cflags": target_cflags,
"cxxflags": target_cxxflags,
"ldflags": target_ldflags,
"wayland": True,
"x11": True,
}
options = dict(
target_arch = arch,
target_triple = f"{arch}-linux-{libc}",
host_cflags = host_cflags,
host_cxxflags = host_cxxflags,
host_ldflags = host_ldflags,
cflags = target_cflags,
cxxflags = target_cxxflags,
ldflags = target_ldflags,
libc = libc,
)
host-recipes/binutils.star
+39
View File
@@ -0,0 +1,39 @@
version = "2.46.0"
revision = 1
metadata = meta(
description = "GNU binutils cross-compiled for the target triple",
license = "GPL-3.0-or-later",
)
source = tarball_source(
url = "https://ftp.gnu.org/gnu/binutils/binutils-" + version + ".tar.xz",
sha256 = "?",
strip_components = 1,
)
def configure(ctx):
ctx.run([
ctx.source_dir / "configure",
"--prefix=" + ctx.prefix,
"--target=" + options.target_triple,
"--with-sysroot=" + ctx.sysroot,
"--with-pic",
"--enable-cet",
"--enable-default-execstack=no",
"--enable-deterministic-archives",
"--enable-ld=default",
"--enable-new-dtags",
"--enable-plugins",
"--enable-relro",
"--enable-separate-code",
"--enable-threads",
# gprofng's libcollector does not build against musl.
"--disable-gprofng",
"--disable-nls",
"--disable-werror",
], env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
})
_, build, install = autotools()
host-recipes/binutils/recipe.star
-42
View File
@@ -1,42 +0,0 @@
name = "binutils"
version = "2.46.0"
revision = 1
description = "GNU binutils cross-compiled for the target triple"
license = "GPL-3.0-or-later"
source = {
"url": "https://ftp.gnu.org/gnu/binutils/binutils-" + version + ".tar.xz",
"sha256": "d75a94f4d73e7a4086f7513e67e439e8fcdcbb726ffe63f4661744e6256b2cf2",
"strip_components": 1,
}
host_deps = []
def configure(ctx):
ctx.run([
ctx.source_dir + "/configure",
"--prefix=" + ctx.prefix,
"--target=" + OPTIONS.target_triple,
"--with-sysroot=" + ctx.prefix + "/" + OPTIONS.target_triple,
"--disable-nls",
"--disable-werror",
"--enable-deterministic-archives",
"--enable-ld=default",
"--enable-plugins",
"--enable-threads",
"--with-system-zlib",
# gprofng's libcollector does not build against musl/recent gcc.
"--disable-gprofng",
], env = {
"CFLAGS": OPTIONS.host_cflags,
"CXXFLAGS": OPTIONS.host_cxxflags,
"LDFLAGS": OPTIONS.host_ldflags,
})
def build(ctx):
ctx.run(["make", "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["make", "DESTDIR=" + pkg.destdir, "install"])
# Drop static archives we don't need on the cross side.
ctx.run(["sh", "-c", "rm -f " + pkg.destdir + ctx.prefix + "/lib/*.a"])
host-recipes/gcc/recipe.star → host-recipes/gcc-bootstrap.star
+16 -17
View File
@@ -1,23 +1,22 @@
name = "gcc"
version = "16.1.0"
revision = 1
description = "GNU GCC cross-compiler (bootstrap stage, C/C++ only)"
license = "GPL-3.0-or-later"
source = {
"url": f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz",
"sha256": "50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79",
"strip_components": 1,
}
metadata = meta(
description = "GNU GCC cross-compiler (bootstrap stage, C/C++ only)",
license = "GPL-3.0-or-later",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz",
sha256 = "50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79",
strip_components = 1,
)
host_deps = ["binutils"]
def configure(ctx):
ctx.run([
ctx.source_dir + "/configure",
"--target=" + options.target_triple,
"--prefix=" + ctx.prefix,
"--target=" + OPTIONS.target_triple,
"--with-sysroot=" + ctx.prefix + "/" + OPTIONS.target_triple,
"--with-sysroot=" + ctx.sysroot,
"--without-headers",
"--with-newlib",
"--enable-languages=c,c++",
@@ -33,9 +32,9 @@ def configure(ctx):
"--disable-libvtv",
"--disable-multilib",
], env = {
"CFLAGS": OPTIONS.host_cflags,
"CXXFLAGS": OPTIONS.host_cxxflags,
"LDFLAGS": OPTIONS.host_ldflags,
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
})
def build(ctx):
@@ -44,5 +43,5 @@ def build(ctx):
ctx.run(["make", jobs, "all-target-libgcc"])
def install(ctx, pkg):
ctx.run(["make", "DESTDIR=" + pkg.destdir, "install-gcc"])
ctx.run(["make", "DESTDIR=" + pkg.destdir, "install-target-libgcc"])
ctx.run(["make", "install-gcc"], env = {"DESTDIR": pkg.dest_dir})
ctx.run(["make", "install-target-libgcc"], env = {"DESTDIR": pkg.dest_dir})
lib/common.star
+14 -76
View File
@@ -1,50 +1,31 @@
# Commonly used helpers, auto-loaded into every recipe.
def _toolchain_env(ctx):
sysroot_flag = " --sysroot=" + ctx.sysroot
return {
"CFLAGS": OPTIONS.cflags + sysroot_flag,
"CXXFLAGS": OPTIONS.cxxflags + sysroot_flag,
"LDFLAGS": OPTIONS.ldflags + sysroot_flag,
}
# Autotools
# Commonly used helpers.
def autotools_configure(ctx, extra_args = [], extra_env = {}):
args = [
ctx.source_dir + "/configure",
env = {
"CFLAGS": options.cflags,
"CXXFLAGS": options.cxxflags,
"LDFLAGS": options.ldflags,
}
env.update(extra_env)
ctx.run([
ctx.source_dir / "configure",
"--host=" + options.target_triple,
"--with-sysroot=" + ctx.sysroot,
"--prefix=" + ctx.prefix,
"--sysconfdir=/etc",
"--localstatedir=/var",
"--bindir=" + ctx.prefix + "/bin",
"--sbindir=" + ctx.prefix + "/bin",
"--libdir=" + ctx.prefix + "/lib",
"--with-sysroot=" + ctx.sysroot,
"--disable-static",
"--enable-shared",
]
args.append("--host=" + OPTIONS.target_triple)
args.extend(extra_args)
envs = _toolchain_env(ctx)
envs.update(extra_env)
ctx.run(args, env = envs)
] + extra_args, env = env)
def autotools_build(ctx, extra_args = []):
args = ["make", "-j" + str(ctx.jobs)]
args.extend(extra_args)
ctx.run(args)
def autotools_check(ctx, extra_args = []):
args = ["make", "check", "-j" + str(ctx.jobs)]
args.extend(extra_args)
ctx.run(args)
ctx.run(["make", "-j" + str(ctx.jobs)] + extra_args)
def autotools_install(ctx, pkg, extra_args = []):
args = ["make", "install", "DESTDIR=" + pkg.destdir]
args.extend(extra_args)
ctx.run(args)
ctx.run(["make", "install"] + extra_args, env = {"DESTDIR": pkg.destdir})
def autotools(configure_args = [], configure_env = [], build_args = [], install_args = []):
def _configure(ctx):
@@ -54,46 +35,3 @@ def autotools(configure_args = [], configure_env = [], build_args = [], install_
def _install(ctx, pkg):
autotools_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install
# Meson
def meson_configure(ctx, extra_args = []):
args = [
"meson",
"setup",
ctx.build_dir,
ctx.source_dir,
"--prefix=" + ctx.prefix,
]
args.extend(extra_args)
ctx.run(args, env = _toolchain_env(ctx))
def meson_build(ctx):
ctx.run(["meson", "compile", "-C", ctx.build_dir, "-j" + str(ctx.jobs)])
def meson_install(ctx, pkg):
ctx.run(["meson", "install", "-C", ctx.build_dir, "--destdir", pkg.destdir])
def meson(configure_args = [], build_args = [], install_args = []):
def _configure(ctx):
meson_configure(ctx, extra_args = configure_args)
def _build(ctx):
meson_build(ctx, extra_args = build_args)
def _install(ctx, pkg):
meson_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install
# Make
def make(ctx, target = None, extra_args = []):
args = ["make", "-C", ctx.source_dir, "O=" + ctx.build_dir,
"-j" + str(ctx.jobs)]
args.extend(extra_args)
if target:
args.append(target)
ctx.run(args)
def make_install(ctx, pkg, extra_args = []):
args = ["make", "-C", ctx.build_dir, "DESTDIR=" + pkg.destdir, "install"]
args.extend(extra_args)
ctx.run(args)
recipes/limine.star
+25
View File
@@ -0,0 +1,25 @@
version = "12.2.0"
revision = 1
metadata = meta(
description = "Modern, secure, portable, multiprotocol bootloader and boot manager",
license = "BSD-2-Clause",
)
source = tarball_source(
url = f"https://github.com/Limine-Bootloader/Limine/releases/download/v{version}/limine-{version}.tar.gz",
sha256 = "db8a119878cfeead63c0a78236c577c40539c5759496950ea0ed32a6cf567865",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
subpackages = [
subpackage(
name = "limine-bios",
),
]
configure, build, install = autotools(configure_env = {
"TOOLCHAIN_FOR_TARGET": options.target_triple + "-",
"LD_FOR_TARGET": options.target_triple + "-" + "ld",
"OBJCOPY_FOR_TARGET": options.target_triple + "-" + "objcopy",
"OBJDUMP_FOR_TARGET": options.target_triple + "-" + "objdump",
})
recipes/limine/recipe.star
-25
View File
@@ -1,25 +0,0 @@
name = "limine"
version = "12.2.0"
revision = 1
description = "Modern, secure, portable, multiprotocol bootloader and boot manager"
license = "BSD-2-Clause"
source = {
"url": f"https://github.com/Limine-Bootloader/Limine/releases/download/v{version}/limine-{version}.tar.gz",
"sha256": "db8a119878cfeead63c0a78236c577c40539c5759496950ea0ed32a6cf567865",
"strip_components": 1,
}
host_deps = ["binutils", "gcc"]
deps = [OPTIONS.libc]
def configure(ctx):
toolchain = OPTIONS.target_triple + "-"
autotools_configure(ctx, extra_env = {
"TOOLCHAIN_FOR_TARGET": toolchain,
"LD_FOR_TARGET": toolchain + "ld",
"OBJCOPY_FOR_TARGET": toolchain + "objcopy",
"OBJDUMP_FOR_TARGET": toolchain + "objdump",
})
_, build, install = autotools()
recipes/linux-headers.star
+20
View File
@@ -0,0 +1,20 @@
version = "7.0.9"
revision = 1
metadata = meta(
description = "Linux kernel headers for userspace development",
license = "GPL-2.0-only",
)
source = tarball_source(
url = f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz",
sha256 = "ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4",
strip_components = 1,
)
def build(ctx):
ctx.run(["cp", "-rp", ctx.source_dir / ".", ctx.build_dir])
ctx.run(["make", "headers_install", "ARCH=" + options.target_arch])
ctx.run(["find", ctx.build_dir / "usr/include", "-type", "f", "!", "-name", "*.h", "-delete"])
def install(ctx, pkg):
ctx.run(["mkdir", "-p", pkg.dest_dir / ctx.prefix])
ctx.run(["cp", "-rp", ctx.build_dir / "usr/include", pkg.dest_dir / ctx.prefix])
recipes/linux/recipe.star → recipes/linux.star.old
View File
recipes/musl.star
+26
View File
@@ -0,0 +1,26 @@
version = "1.2.6"
revision = 1
metadata = meta(
description = "Small, standards-conformant implementation of libc",
license = "MIT",
)
source = tarball_source(
url = f"https://musl.libc.org/releases/musl-{version}.tar.gz",
sha256 = "?",
strip_components = 1,
)
host_deps = ["binutils", "gcc-bootstrap"]
def configure(ctx):
ctx.run([
ctx.source_dir / "configure",
"--target=" + options.target_triple,
"--prefix=" + ctx.prefix,
"--syslibdir=/lib",
], env = {
"CC": options.target_triple + "-gcc",
"CFLAGS": options.cflags,
"LDFLAGS": options.ldflags,
})
_, build, install = autotools()
recipes/musl/recipe.star
-31
View File
@@ -1,31 +0,0 @@
name = "musl"
version = "1.2.6"
revision = 1
description = "Musl C library"
license = "MIT"
source = {
"url": f"https://musl.libc.org/releases/musl-{version}.tar.gz",
"sha256": "d585fd3b613c66151fc3249e8ed44f77020cb5e6c1e635a616d3f9f82460512a",
"strip_components": 1,
}
host_deps = ["binutils", "gcc"]
def configure(ctx):
ctx.run(
[
ctx.source_dir + "/configure",
"--prefix=/usr",
"--syslibdir=/lib",
"--target=" + OPTIONS.target_triple,
],
env = {
"CC": OPTIONS.target_triple + "-gcc",
"CFLAGS": OPTIONS.cflags,
"LDFLAGS": OPTIONS.ldflags,
},
)
build = autotools_build
install = autotools_install
src/apk.rs
-46
View File
@@ -1,46 +0,0 @@
use crate::config::Config;
use crate::recipe::OutputPackage;
use std::path::Path;
#[derive(Debug, Clone)]
pub struct ApkPackagePlan {
pub args: Vec<String>,
}
pub fn mkpkg_plan(
config: &Config,
package: &OutputPackage,
files_root: &Path,
output_dir: &Path,
) -> ApkPackagePlan {
let file_name = format!(
"{}-{}-r{}.apk",
package.name, package.version, package.revision
);
let output = output_dir.join(file_name);
let version = format!("{}-r{}", package.version, package.revision);
let mut args = vec![
"mkpkg".to_owned(),
"--files".to_owned(),
files_root.display().to_string(),
"--output".to_owned(),
output.display().to_string(),
"--info".to_owned(),
format!("name:{}", package.name),
"--info".to_owned(),
format!("version:{version}"),
"--info".to_owned(),
format!("arch:{}", config.target_arch),
"--info".to_owned(),
format!("description:{}", package.description),
"--info".to_owned(),
format!("license:{}", package.license),
"--info".to_owned(),
format!("origin:{}", package.recipe),
];
for dep in &package.deps {
args.push("--info".to_owned());
args.push(format!("depends:{dep}"));
}
ApkPackagePlan { args }
}
src/build.rs
-949
View File
@@ -1,949 +0,0 @@
use crate::apk;
use crate::config::Config;
use crate::graph::PackageGraph;
use crate::log;
use crate::patches;
use crate::phase::{PhaseCommand, PhaseEnv, SourceDir, collect_phase_commands};
use crate::recipe::{OutputPackage, PackageKind, Recipe, RecipeSet};
use crate::source;
use anyhow::{Context, Result, anyhow, bail};
use sha2::{Digest, Sha256};
use std::cell::{Cell, RefCell};
use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
const C_SOURCE: &str = "/source";
const C_BUILD: &str = "/builddir";
const C_DEST: &str = "/dest";
const C_SYSROOT: &str = "/sysroot";
const HOST_PREFIX: &str = "/usr/local";
const TARGET_PREFIX: &str = "/usr";
#[derive(Debug)]
pub struct Builder {
repo: PathBuf,
config: Config,
skip_checks: bool,
preflight_done: Cell<bool>,
built_recipes: RefCell<HashSet<String>>,
}
impl Builder {
pub fn new(repo: PathBuf, config: Config, skip_checks: bool) -> Self {
Self {
repo,
config,
skip_checks,
preflight_done: Cell::new(false),
built_recipes: RefCell::default(),
}
}
pub fn build(
&self,
recipes: &RecipeSet,
graph: &PackageGraph,
package: Option<&str>,
rebuild: bool,
) -> Result<()> {
self.preflight_container()?;
let (label, order) = match package {
Some(p) => (p.to_owned(), graph.build_order(p)?),
None => ("<all>".to_owned(), graph.build_order_all()?),
};
log::step(
"plan",
&format!("{label}: {} package(s) [{}]", order.len(), order.join(", ")),
);
for output_name in order {
let output = graph.output(&output_name)?;
let recipe = recipes.recipe_for_package(&output_name)?;
self.ensure_source(recipe)?;
// Only force a rebuild of the explicitly requested package;
// dependencies stay cached when their manifest is up to date.
// With no package (build all), every package honours --rebuild.
let force = rebuild && package.map_or(true, |p| p == output_name);
match output.kind {
PackageKind::Host => self.build_host(recipe, output, force)?,
PackageKind::Target => self.build_target(recipe, output, force)?,
}
}
self.repo_index()?;
Ok(())
}
pub fn fetch(&self, recipes: &RecipeSet, package: &str) -> Result<()> {
let recipe = recipes.recipe_by_user_ref(package)?;
log::step("fetch", &recipe.key());
source::fetch_sources(recipe, &self.source_cache_dir())?;
Ok(())
}
pub fn shell(&self, recipes: &RecipeSet, graph: &PackageGraph, package: &str) -> Result<()> {
graph.output(package)?;
let recipe = recipes.recipe_for_package(package)?;
self.preflight_container()?;
let source = self.unpacked_source_dir(recipe);
let build = self.build_dir(recipe);
fs::create_dir_all(&source)?;
fs::create_dir_all(&build)?;
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-it")
.arg("-v")
.arg(format!("{}:{C_SOURCE}:ro", source.display()))
.arg("-v")
.arg(format!("{}:{C_BUILD}", build.display()))
.arg("-w")
.arg(C_BUILD)
.arg(&self.config.container_image)
.arg("/bin/sh")
.status()?;
if !status.success() {
bail!("container shell exited with {status}");
}
Ok(())
}
pub fn repo_index(&self) -> Result<()> {
self.preflight_container()?;
let repo = self.pkgs_dir();
fs::create_dir_all(&repo)?;
// Nothing to index yet — leave it alone so callers don't trip on a
// failing `*.apk` glob.
let has_apks = fs::read_dir(&repo)?.any(|e| {
e.ok()
.and_then(|e| e.path().extension().map(|x| x == "apk"))
.unwrap_or(false)
});
if !has_apks {
return Ok(());
}
log::step(
"index",
&format!("signing repository at {}", repo.display()),
);
let key = self.abs_config_path(&self.config.signing_key);
let pubkey = self.abs_config_path(&self.config.signing_pubkey);
if !key.exists() || !pubkey.exists() {
bail!("signing key is not configured or missing; run `distro init-key` first");
}
let index_name = "APKINDEX.tar.gz";
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/repo", repo.display()))
.arg("-v")
.arg(format!("{}:/keys/distro.rsa:ro", key.display()))
.arg("-v")
.arg(format!(
"{}:/etc/apk/keys/distro.rsa.pub:ro",
pubkey.display()
))
.arg(&self.config.container_image)
.arg("/bin/sh")
.arg("-lc")
.arg(format!(
"cd /repo && apk --sign-key /keys/distro.rsa mkndx -o {index_name} *.apk"
))
.status()
.context("failed to run repository index command")?;
if !status.success() {
bail!("repository index command failed with {status}");
}
Ok(())
}
pub fn init_key(&self) -> Result<()> {
let key = self.abs_config_path(&self.config.signing_key);
let pubkey = self.abs_config_path(&self.config.signing_pubkey);
if key.exists() && pubkey.exists() {
log::skip("init-key", "signing key already present");
return Ok(());
}
log::step("init-key", &format!("generating {}", key.display()));
let key_dir = key
.parent()
.ok_or_else(|| anyhow!("invalid signing key path"))?;
let key_name = key
.file_name()
.ok_or_else(|| anyhow!("invalid signing key path"))?
.to_string_lossy();
fs::create_dir_all(key_dir)?;
self.preflight_container()?;
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/keys", key_dir.display()))
.arg(&self.config.container_image)
.arg("/bin/sh")
.arg("-lc")
.arg(format!(
"openssl genrsa -out /keys/{key_name} 4096 && \
openssl rsa -in /keys/{key_name} -pubout -out /keys/{key_name}.pub"
))
.status()
.context("failed to run key generation command")?;
if !status.success() {
bail!("key generation failed with {status}");
}
Ok(())
}
pub fn rootfs(&self, root: &Path, packages: &[String]) -> Result<()> {
self.preflight_container()?;
let pubkey = self.abs_config_path(&self.config.signing_pubkey);
if !pubkey.exists() {
bail!("rootfs requires a configured public signing key; run `distro init-key` first");
}
fs::create_dir_all(root)?;
log::step(
"rootfs",
&format!(
"{} -> {} [{}]",
packages.join(", "),
root.display(),
root.display()
),
);
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/rootfs", root.display()))
.arg("-v")
.arg(format!("{}:/repo:ro", self.pkgs_root().display()))
.arg("-v")
.arg(format!(
"{}:/etc/apk/keys/distro.rsa.pub:ro",
pubkey.display()
))
.arg(&self.config.container_image)
.arg("apk")
.arg("--root")
.arg("/rootfs")
.arg("--keys-dir")
.arg("/etc/apk/keys")
.arg("--initdb")
.arg("--repository")
.arg("/repo")
.arg("add")
.args(packages)
.status()
.context("failed to run rootfs apk command")?;
if !status.success() {
bail!("rootfs creation failed with {status}");
}
Ok(())
}
// --- per-recipe build steps -------------------------------------------
fn build_host(&self, recipe: &Recipe, output: &OutputPackage, rebuild: bool) -> Result<()> {
let manifest = self.manifest_path(&output.key());
let hash = self.manifest_hash(recipe)?;
if !rebuild && fs::read_to_string(&manifest).ok().as_deref() == Some(hash.as_str()) {
log::skip("up-to-date", &output.key());
return Ok(());
}
log::step(
"build",
&format!("{} {}-r{}", output.key(), recipe.version, recipe.revision),
);
let build_dir = self.host_build_dir(recipe);
let dest_dir = self.host_pkg_dir(recipe);
let source_dir = self.unpacked_source_dir(recipe);
// A rebuild starts from a clean build dir; sources stay shared.
if rebuild {
Self::recreate(&build_dir)?;
} else {
fs::create_dir_all(&build_dir)?;
}
Self::recreate(&dest_dir)?;
fs::create_dir_all(&build_dir)?;
let env = PhaseEnv {
source_dir: source_dir_env(recipe),
build_dir: C_BUILD,
dest_dir: C_DEST,
prefix: HOST_PREFIX,
sysroot: C_SYSROOT,
};
self.run_recipe_phases(
recipe,
output,
&env,
&source_dir,
&build_dir,
&dest_dir,
None,
)?;
fs::create_dir_all(manifest.parent().unwrap())?;
fs::write(manifest, hash)?;
Ok(())
}
fn build_target(&self, recipe: &Recipe, output: &OutputPackage, rebuild: bool) -> Result<()> {
let manifest = self.manifest_path(&output.key());
let hash = self.manifest_hash(recipe)?;
if !rebuild && fs::read_to_string(&manifest).ok().as_deref() == Some(hash.as_str()) {
log::skip("up-to-date", &output.key());
return Ok(());
}
log::step(
"build",
&format!("{} {}-r{}", output.key(), recipe.version, recipe.revision),
);
let build_dir = self.build_dir(recipe);
let source_dir = self.unpacked_source_dir(recipe);
// A rebuild starts from a clean build dir; sources stay shared.
if rebuild {
Self::recreate(&build_dir)?;
} else {
fs::create_dir_all(&build_dir)?;
}
let dest_tmp =
tempfile::tempdir_in(&build_dir).context("failed to create ephemeral destdir")?;
let dest_dir = dest_tmp.path();
let sysroot = self.materialize_sysroot(recipe)?;
let env = PhaseEnv {
source_dir: source_dir_env(recipe),
build_dir: C_BUILD,
dest_dir: C_DEST,
prefix: TARGET_PREFIX,
sysroot: C_SYSROOT,
};
self.run_recipe_phases(
recipe,
output,
&env,
&source_dir,
&build_dir,
dest_dir,
sysroot.as_ref().map(|s| s.path()),
)?;
self.apk_mkpkg(output, dest_dir)?;
fs::create_dir_all(manifest.parent().unwrap())?;
fs::write(manifest, hash)?;
Ok(())
}
fn run_recipe_phases(
&self,
recipe: &Recipe,
output: &OutputPackage,
env: &PhaseEnv<'_>,
source_dir: &Path,
build_dir: &Path,
dest_dir: &Path,
target_sysroot: Option<&Path>,
) -> Result<()> {
let host_sandbox = self.materialize_host_sandbox(recipe)?;
let recipe_already_built = self.built_recipes.borrow().contains(&recipe.key());
// configure/build/check run at most once per recipe per invocation.
let mut shared_phases: Vec<(&str, bool)> = Vec::new();
if !recipe_already_built {
if let Some(p) = recipe.configure_fn.as_deref() {
shared_phases.push((p, false));
}
if let Some(p) = recipe.build_fn.as_deref() {
shared_phases.push((p, false));
}
if !self.skip_checks {
if let Some(p) = recipe.check_fn.as_deref() {
shared_phases.push((p, false));
}
}
}
// install runs per output (each output has its own destdir/install_fn).
let install_phase = (output.install_fn.as_str(), true);
let mut commands: Vec<PhaseCommand> = Vec::new();
for (phase, takes_pkg) in shared_phases.iter().chain(std::iter::once(&install_phase)) {
let pkg = if *takes_pkg {
Some((output.name.as_str(), C_DEST))
} else {
None
};
let owner = if *takes_pkg {
output.key()
} else {
recipe.key()
};
log::info("phase", &format!("{phase} {owner}"));
commands.extend(collect_phase_commands(
&recipe.path,
&self.repo,
&self.config,
phase,
env,
pkg,
)?);
}
if commands.is_empty() {
self.built_recipes.borrow_mut().insert(recipe.key());
return Ok(());
}
self.run_in_container(
source_dir,
build_dir,
dest_dir,
host_sandbox.as_ref().map(|s| s.path()),
target_sysroot,
&commands,
)?;
self.built_recipes.borrow_mut().insert(recipe.key());
Ok(())
}
fn run_in_container(
&self,
source_dir: &Path,
build_dir: &Path,
dest_dir: &Path,
host_sandbox: Option<&Path>,
target_sysroot: Option<&Path>,
commands: &[PhaseCommand],
) -> Result<()> {
fs::create_dir_all(dest_dir)?;
let mut process = Command::new(&self.config.container_runtime);
process
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:{C_SOURCE}:ro", source_dir.display()))
.arg("-v")
.arg(format!("{}:{C_BUILD}", build_dir.display()))
.arg("-v")
.arg(format!("{}:{C_DEST}", dest_dir.display()))
.arg("-w")
.arg(C_BUILD);
if let Some(sandbox) = host_sandbox {
// Host packages are configured with --prefix=/usr/local, so we
// bind the assembled tree exactly there (matching Jinx). This
// means rpaths, --print-search-dirs, pkg-config lookups, etc.
// all keep working with no extra environment fiddling.
process
.arg("-v")
.arg(format!("{}:{HOST_PREFIX}:ro", sandbox.display()));
}
if let Some(sysroot) = target_sysroot {
process
.arg("-v")
.arg(format!("{}:{C_SYSROOT}:ro", sysroot.display()))
.arg("-e")
.arg(format!("PKG_CONFIG_SYSROOT_DIR={C_SYSROOT}"))
.arg("-e")
.arg(format!("SYSROOT={C_SYSROOT}"));
}
process
.arg(&self.config.container_image)
.arg("/bin/sh")
.arg("-c")
.arg(build_phase_script(commands));
let status = process
.status()
.context("failed to start container for phase commands")?;
if !status.success() {
bail!("phase commands failed with {status}");
}
Ok(())
}
fn apk_mkpkg(&self, output: &OutputPackage, dest_dir: &Path) -> Result<()> {
let repo = self.pkgs_dir();
fs::create_dir_all(&repo)?;
let signing_key = self.abs_config_path(&self.config.signing_key);
if !signing_key.exists() {
bail!("package signing key is missing; run `distro init-key` first");
}
log::step("package", &format!("{} -> {}", output.name, repo.display()));
let plan = apk::mkpkg_plan(&self.config, output, Path::new(C_DEST), Path::new("/out"));
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:{C_DEST}:ro", dest_dir.display()))
.arg("-v")
.arg(format!("{}:/out", repo.display()))
.arg("-v")
.arg(format!("{}:/keys/distro.rsa:ro", signing_key.display()))
.arg(&self.config.container_image)
.arg("apk")
.arg("--sign-key")
.arg("/keys/distro.rsa")
.args(plan.args)
.status()
.context("failed to run apk mkpkg command")?;
if !status.success() {
bail!("apk mkpkg failed with {status}");
}
Ok(())
}
// --- source preparation (Jinx-style persistent sources/<recipe>/) ------
fn ensure_source(&self, recipe: &Recipe) -> Result<()> {
let cached = source::fetch_sources(recipe, &self.source_cache_dir())?;
let unpacked = self.unpacked_source_dir(recipe);
let version_stamp = self.source_stamp(recipe, "version");
let want_version = format!("{}-r{}", recipe.version, recipe.revision);
let need_extract =
fs::read_to_string(&version_stamp).ok().as_deref() != Some(&want_version);
if need_extract {
log::info("unpack", &recipe.key());
if unpacked.exists() {
fs::remove_dir_all(&unpacked)?;
}
fs::create_dir_all(&unpacked)?;
for (src, tarball) in recipe.sources.iter().zip(cached.iter()) {
let dest = if src.name.is_empty() {
unpacked.clone()
} else {
unpacked.join(&src.name)
};
self.extract_tarball(tarball, &dest, src.strip_components)?;
}
fs::write(&version_stamp, &want_version)?;
let patched = self.source_stamp(recipe, "patched");
let _ = fs::remove_file(&patched);
}
let patched_stamp = self.source_stamp(recipe, "patched");
if !patched_stamp.exists() {
self.apply_patches(recipe, &unpacked)?;
fs::write(&patched_stamp, "")?;
}
Ok(())
}
fn extract_tarball(&self, tarball: &Path, dest: &Path, strip_components: u32) -> Result<()> {
fs::create_dir_all(dest)?;
log::info(
"extract",
&format!("{} -> {}", tarball.display(), dest.display()),
);
let strip = if strip_components > 0 {
format!("--strip-components={strip_components}")
} else {
String::new()
};
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/in.tar:ro", tarball.display()))
.arg("-v")
.arg(format!("{}:/out", dest.display()))
.arg(&self.config.container_image)
.arg("/bin/sh")
.arg("-c")
.arg(format!("tar -xf /in.tar -C /out {strip}"))
.status()
.context("failed to unpack source archive")?;
if !status.success() {
bail!("source unpack failed with {status}");
}
Ok(())
}
fn apply_patches(&self, recipe: &Recipe, source_dir: &Path) -> Result<()> {
let patches = patches::discover(&recipe.dir)?;
if patches.is_empty() {
return Ok(());
}
if recipe.sources.iter().any(|s| !s.name.is_empty()) {
bail!(
"recipe `{}` has patches/ but uses multi-source layout; \
apply patches yourself via ctx.run in the configure phase",
recipe.id
);
}
for patch in patches {
log::info(
"patch",
&format!(
"{} <- {}",
recipe.id,
patch
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default(),
),
);
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/source", source_dir.display()))
.arg("-v")
.arg(format!("{}:/patch.diff:ro", patch.display()))
.arg(&self.config.container_image)
.arg("patch")
.arg("-d")
.arg("/source")
.arg("-p1")
.arg("-i")
.arg("/patch.diff")
.status()
.context("failed to apply patch")?;
if !status.success() {
bail!("patch {} failed with {status}", patch.display());
}
}
Ok(())
}
// --- host sandbox / target sysroot materialization --------------------
/// Assemble all `host_deps` into a single ephemeral tree that we can mount
/// at `/usr/local` inside the container. Following Jinx, we hard-link
/// rather than byte-copy so this is essentially free. The returned
/// `TempDir` cleans up when dropped.
fn materialize_host_sandbox(&self, recipe: &Recipe) -> Result<Option<tempfile::TempDir>> {
if recipe.host_deps.is_empty() {
return Ok(None);
}
fs::create_dir_all(self.repo.join("build"))?;
let sandbox = tempfile::Builder::new()
.prefix(&format!("host-sandbox-{}-", recipe.id))
.tempdir_in(self.repo.join("build"))
.context("failed to create host sandbox tempdir")?;
log::info(
"host-sandbox",
&format!("{} <- [{}]", recipe.id, recipe.host_deps.join(", ")),
);
for dep in &recipe.host_deps {
let source = self.host_pkg_dir_by_id(dep).join("usr/local");
if !source.exists() {
bail!(
"host dependency `{dep}` has not been built at {}",
source.display()
);
}
hardlink_tree(&source, sandbox.path())?;
}
Ok(Some(sandbox))
}
fn materialize_sysroot(&self, recipe: &Recipe) -> Result<Option<tempfile::TempDir>> {
let mut deps: Vec<String> = recipe
.build_deps
.iter()
.chain(recipe.deps.iter())
.cloned()
.collect();
deps.sort();
deps.dedup();
if deps.is_empty() {
return Ok(None);
}
// The local repo index must be present and current so apk can resolve
// and verify the just-built target packages.
self.repo_index()?;
let pubkey = self.abs_config_path(&self.config.signing_pubkey);
if !pubkey.exists() {
bail!("target dependency sysroot requires a configured public signing key");
}
fs::create_dir_all(self.repo.join("build"))?;
let sysroot = tempfile::Builder::new()
.prefix(&format!("sysroot-{}-", recipe.id))
.tempdir_in(self.repo.join("build"))
.context("failed to create sysroot tempdir")?;
log::info(
"sysroot",
&format!("{} <- [{}]", recipe.id, deps.join(", ")),
);
let status = Command::new(&self.config.container_runtime)
.arg("run")
.arg("--rm")
.arg("-v")
.arg(format!("{}:/sysroot", sysroot.path().display()))
.arg("-v")
.arg(format!("{}:/repo:ro", self.pkgs_root().display()))
.arg("-v")
.arg(format!(
"{}:/etc/apk/keys/distro.rsa.pub:ro",
pubkey.display()
))
.arg(&self.config.container_image)
.arg("apk")
.arg("--root")
.arg("/sysroot")
.arg("--keys-dir")
.arg("/etc/apk/keys")
.arg("--initdb")
.arg("--repository")
.arg("/repo")
.arg("add")
.args(&deps)
.status()
.context("failed to install target dependency sysroot")?;
if !status.success() {
bail!("target dependency sysroot install failed with {status}");
}
Ok(Some(sysroot))
}
// --- preflight & container image --------------------------------------
fn preflight_container(&self) -> Result<()> {
if self.preflight_done.get() {
return Ok(());
}
let status = Command::new(&self.config.container_runtime)
.arg("--version")
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.with_context(|| {
format!(
"{} is required but was not found",
self.config.container_runtime
)
})?;
if !status.success() {
bail!(
"{} preflight failed with {status}",
self.config.container_runtime
);
}
let dockerfile = self.abs_config_path(&self.config.container_dockerfile);
self.ensure_container_image(&dockerfile)?;
self.preflight_done.set(true);
Ok(())
}
fn ensure_container_image(&self, dockerfile: &Path) -> Result<()> {
if !dockerfile.exists() {
bail!(
"configured container Dockerfile does not exist: {}",
dockerfile.display()
);
}
let hash = self.container_build_hash(dockerfile)?;
let stamp = self.repo.join("build/container-image.hash");
if fs::read_to_string(&stamp).ok().as_deref() == Some(hash.as_str())
&& self.container_image_exists()?
{
return Ok(());
}
log::step(
"image",
&format!(
"building {} from {}",
self.config.container_image,
dockerfile.display()
),
);
let status = Command::new(&self.config.container_runtime)
.arg("build")
.arg("-f")
.arg(dockerfile)
.arg("-t")
.arg(&self.config.container_image)
.arg(&self.repo)
.status()
.with_context(|| {
format!(
"failed to build container image `{}` from {}",
self.config.container_image,
dockerfile.display()
)
})?;
if !status.success() {
bail!(
"container image build failed for `{}` with {status}",
self.config.container_image
);
}
fs::create_dir_all(stamp.parent().unwrap())?;
fs::write(stamp, hash)?;
Ok(())
}
fn container_image_exists(&self) -> Result<bool> {
let status = Command::new(&self.config.container_runtime)
.arg("image")
.arg("exists")
.arg(&self.config.container_image)
.status()
.with_context(|| {
format!(
"failed to inspect container image `{}`",
self.config.container_image
)
})?;
Ok(status.success())
}
fn container_build_hash(&self, dockerfile: &Path) -> Result<String> {
let mut hasher = Sha256::new();
hasher.update(fs::read(dockerfile)?);
hasher.update(self.config.container_image.as_bytes());
Ok(hex::encode(hasher.finalize()))
}
fn manifest_hash(&self, recipe: &Recipe) -> Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.config.target_arch.as_bytes());
hasher.update(recipe.version.as_bytes());
hasher.update(recipe.revision.to_le_bytes());
for source in &recipe.sources {
hasher.update(source.url.as_bytes());
hasher.update(source.sha256.as_bytes());
}
for patch in patches::discover(&recipe.dir)? {
hasher.update(patch.display().to_string().as_bytes());
hasher.update(fs::read(patch)?);
}
Ok(hex::encode(hasher.finalize()))
}
// --- path helpers -----------------------------------------------------
fn source_cache_dir(&self) -> PathBuf {
self.repo.join("build/cache/sources")
}
fn unpacked_source_dir(&self, recipe: &Recipe) -> PathBuf {
self.repo.join("build/sources").join(recipe.slug())
}
fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.repo
.join("build/sources")
.join(format!("{}.{kind}", recipe.slug()))
}
fn build_dir(&self, recipe: &Recipe) -> PathBuf {
self.repo.join("build/builds").join(&recipe.id)
}
fn host_build_dir(&self, recipe: &Recipe) -> PathBuf {
self.repo.join("build/host-builds").join(&recipe.id)
}
fn host_pkg_dir(&self, recipe: &Recipe) -> PathBuf {
self.repo.join("build/host-pkgs").join(&recipe.id)
}
fn host_pkg_dir_by_id(&self, host_recipe_id: &str) -> PathBuf {
self.repo.join("build/host-pkgs").join(host_recipe_id)
}
/// Root of the target package repo. apk treats this as the repo root and
/// expects `<root>/<arch>/APKINDEX.tar.gz` underneath.
fn pkgs_root(&self) -> PathBuf {
self.repo.join("build/pkgs")
}
/// Arch-specific package directory: where .apk files and the index live.
fn pkgs_dir(&self) -> PathBuf {
self.pkgs_root().join(&self.config.target_arch)
}
fn manifest_path(&self, output_key: &str) -> PathBuf {
// Output keys may contain `:` (e.g. `host:gcc`); the manifest file
// name uses the filesystem-safe slug form instead.
let safe = output_key.replace(':', "-");
self.repo
.join("build/manifests")
.join(format!("{safe}.hash"))
}
fn abs_config_path(&self, path: &Path) -> PathBuf {
if path.is_absolute() {
path.to_path_buf()
} else {
self.repo.join(path)
}
}
fn recreate(path: &Path) -> Result<()> {
if path.exists() {
fs::remove_dir_all(path)?;
}
fs::create_dir_all(path)?;
Ok(())
}
}
fn build_phase_script(commands: &[PhaseCommand]) -> String {
let mut parts: Vec<String> = Vec::with_capacity(commands.len() + 1);
parts.push("set -e".to_owned());
for cmd in commands {
let mut tokens: Vec<String> = Vec::with_capacity(cmd.env.len() + cmd.argv.len());
for (k, v) in &cmd.env {
let value = shell_escape::unix::escape(v.as_str().into()).into_owned();
tokens.push(format!("{k}={value}"));
}
for arg in &cmd.argv {
tokens.push(shell_escape::unix::escape(arg.as_str().into()).into_owned());
}
parts.push(tokens.join(" "));
}
parts.join("\n")
}
/// Expose `/source` as a string for single-source recipes, or a struct of
/// `/source/<name>` paths for multi-source recipes.
fn source_dir_env(recipe: &Recipe) -> SourceDir {
if recipe.sources.iter().all(|s| s.name.is_empty()) {
SourceDir::Single(C_SOURCE.to_owned())
} else {
let map = recipe
.sources
.iter()
.map(|s| (s.name.clone(), format!("{C_SOURCE}/{}", s.name)))
.collect();
SourceDir::Many(map)
}
}
/// Mirror `src` into `dst` using hard links for regular files (and preserving
/// symlinks). This matches Jinx's `cp -Pplr`: no bytes are copied, just inode
/// references, so assembling a host-deps sandbox is essentially free.
fn hardlink_tree(src: &Path, dst: &Path) -> Result<()> {
fs::create_dir_all(dst)?;
for entry in walkdir::WalkDir::new(src) {
let entry = entry?;
let relative = entry.path().strip_prefix(src)?;
if relative.as_os_str().is_empty() {
continue;
}
let target = dst.join(relative);
let file_type = entry.file_type();
if file_type.is_dir() {
fs::create_dir_all(&target)?;
} else if file_type.is_symlink() {
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)?;
}
let link_target = fs::read_link(entry.path())?;
// Replace any pre-existing entry (multiple host deps may ship the
// same symlink path).
let _ = fs::remove_file(&target);
std::os::unix::fs::symlink(&link_target, &target)?;
} else if file_type.is_file() {
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)?;
}
if target.exists() {
continue;
}
fs::hard_link(entry.path(), &target).with_context(|| {
format!(
"failed to hard-link {} -> {}",
entry.path().display(),
target.display()
)
})?;
}
}
Ok(())
}
src/builder.rs
+178
View File
@@ -0,0 +1,178 @@
use std::{
fs,
path::{Path, PathBuf},
process::Command,
};
use anyhow::{Context, bail};
use sha2::{Digest, Sha256};
use crate::{
config::Config,
graph::{TaskPlan, TaskPlanner},
log,
recipe::RecipeSet,
};
#[derive(Debug)]
pub struct Builder {
root: PathBuf,
config: Config,
container_ready: bool,
}
impl Builder {
pub fn new(root: PathBuf, config: Config) -> Self {
Self {
root,
config,
container_ready: false,
}
}
pub fn build(
&mut self,
recipes: &RecipeSet,
requested: &[String],
rebuild: bool,
dry_run: bool,
) -> anyhow::Result<()> {
let plan = TaskPlanner::new(&self.root, &self.config.arch, recipes)
.build_plan(requested, rebuild)?;
self.print_plan(&plan);
if !dry_run {
bail!("task execution is not implemented yet");
}
Ok(())
}
pub fn fetch(
&mut self,
recipes: &RecipeSet,
requested: &[String],
dry_run: bool,
) -> anyhow::Result<()> {
let plan =
TaskPlanner::new(&self.root, &self.config.arch, recipes).fetch_plan(requested)?;
self.print_plan(&plan);
if !dry_run {
bail!("task execution is not implemented yet");
}
Ok(())
}
pub fn ensure_container_ready(&mut self) -> anyhow::Result<()> {
if !self.container_ready {
self.ensure_container_image(&self.abs_config_path(&self.config.container_dockerfile))?;
self.container_ready = true;
}
Ok(())
}
fn ensure_container_image(&self, dockerfile: &Path) -> anyhow::Result<()> {
if !dockerfile.exists() {
bail!(
"configured container Dockerfile does not exist: {}",
dockerfile.display()
);
}
let hash = self.container_build_hash(dockerfile)?;
let stamp = self.root.join("build/container-image.hash");
if fs::read_to_string(&stamp).ok().as_deref() == Some(hash.as_str())
&& self.container_image_exists()?
{
log::skip(
"image",
&format!("using cached {}", self.config.container_image),
);
return Ok(());
}
log::step(
"image",
&format!(
"building {} from {}",
self.config.container_image,
dockerfile.display()
),
);
let runtime = self.config.container_runtime.as_str();
let status = Command::new(runtime)
.arg("build")
.arg("-f")
.arg(dockerfile)
.arg("-t")
.arg(&self.config.container_image)
.arg(&self.root)
.status()
.with_context(|| {
format!(
"failed to build container image `{}` from {}",
self.config.container_image,
dockerfile.display()
)
})?;
if !status.success() {
bail!(
"container image build failed for `{}` with {status}",
self.config.container_image
);
}
fs::create_dir_all(stamp.parent().unwrap())?;
fs::write(stamp, hash)?;
Ok(())
}
fn container_image_exists(&self) -> anyhow::Result<bool> {
let runtime = self.config.container_runtime.as_str();
let status = Command::new(runtime)
.arg("image")
.arg("exists")
.arg(&self.config.container_image)
.status()
.with_context(|| {
format!(
"failed to inspect container image `{}`",
self.config.container_image
)
})?;
Ok(status.success())
}
fn container_build_hash(&self, dockerfile: &Path) -> anyhow::Result<String> {
let mut hasher = Sha256::new();
hasher.update(fs::read(dockerfile)?);
hasher.update(self.config.container_image.as_bytes());
Ok(hex::encode(hasher.finalize()))
}
fn print_plan(&self, plan: &TaskPlan) {
if plan.is_empty() {
log::skip("plan", "nothing to do");
return;
}
log::step(
"plan",
&format!(
"{} active task(s), {} edge(s)",
plan.order().len(),
plan.dependency_count()
),
);
for task in plan.order() {
println!("{task}");
}
}
fn abs_config_path(&self, path: &Path) -> PathBuf {
if path.is_absolute() {
path.to_path_buf()
} else {
self.root.join(path)
}
}
}
src/cli.rs
+61 -103
View File
@@ -1,119 +1,77 @@
use crate::build::Builder;
use crate::config::Config;
use crate::graph::PackageGraph;
use crate::recipe::RecipeSet;
use anyhow::{Context, Result, bail};
use clap::{Parser, Subcommand};
use std::fs;
use std::path::PathBuf;
use crate::{builder::Builder, config::Config, eval, recipe::RecipeSet};
#[derive(Debug, Parser)]
#[command(name = "distro", version)]
pub struct Cli {
#[arg(long, default_value = ".")]
pub repo: PathBuf,
#[arg(long)]
pub skip_checks: bool,
struct Cli {
#[arg(
long,
short,
default_value = ".",
help = "Directory containing the configuration and recipe files"
)]
root: PathBuf,
#[command(subcommand)]
pub command: Command,
command: Command,
}
#[derive(Debug, Parser)]
#[command(about = "Fetch sources for the given recipes")]
struct FetchCommand {
#[arg(
required = true,
help = "List of recipes to fetch, host recipes should be prefixed with `host:`"
)]
recipes: Vec<String>,
#[arg(long, short = 'n', help = "Print what will be done and exit")]
dry_run: bool,
}
#[derive(Debug, Parser)]
#[command(about = "Build the given recipes")]
struct BuildCommand {
#[arg(
required = true,
help = "List of recipes to build, host recipes should be prefixed with `host:`"
)]
recipes: Vec<String>,
#[arg(long, short, help = "Perform a full rebuild of the given recipes")]
rebuild: bool,
#[arg(long, short = 'n', help = "Print what will be done and exit")]
dry_run: bool,
}
#[derive(Debug, Subcommand)]
pub enum Command {
Build {
package: Option<String>,
},
Rebuild {
package: Option<String>,
},
Fetch {
package: String,
},
Graph {
package: Option<String>,
},
List,
Info {
package: String,
},
Clean,
Shell {
package: String,
},
RepoIndex,
InitKey,
Rootfs {
root: PathBuf,
packages: Vec<String>,
},
Update {
#[arg(long)]
bump: bool,
packages: Vec<String>,
},
enum Command {
Fetch(FetchCommand),
Build(BuildCommand),
#[command(about = "Create or refresh the configured build container image")]
Image,
}
pub fn run(cli: Cli) -> Result<()> {
let repo = cli.repo.canonicalize().unwrap_or(cli.repo);
match &cli.command {
Command::Clean => {
let build = repo.join("build");
if build.exists() {
fs::remove_dir_all(&build)
.with_context(|| format!("failed to remove {}", build.display()))?;
}
Ok(())
}
Command::InitKey => {
let config = Config::load(&repo.join("config.star"))?;
Builder::new(repo, config, cli.skip_checks).init_key()
}
_ => {
let config = Config::load(&repo.join("config.star"))?;
let recipes = RecipeSet::load(&repo, &config)?;
if let Command::Update { bump, packages } = &cli.command {
return crate::update::run(&recipes, packages, *bump);
}
let graph = PackageGraph::new(&recipes)?;
let builder = Builder::new(repo, config, cli.skip_checks);
pub fn run() -> anyhow::Result<()> {
let cli = Cli::parse();
let root_path = cli.root.canonicalize().unwrap_or(cli.root);
let config = Config::load(&root_path.join("config.star"))?;
match cli.command {
Command::Build { package } => {
builder.build(&recipes, &graph, package.as_deref(), false)
Command::Fetch(command) => {
let lib = eval::eval_lib(&root_path.join("lib"), Some(&config.options))?;
let recipes = RecipeSet::load(&root_path, &config.options, lib.as_ref())?;
let mut builder = Builder::new(root_path, config);
builder.fetch(&recipes, &command.recipes, command.dry_run)
}
Command::Rebuild { package } => {
builder.build(&recipes, &graph, package.as_deref(), true)
}
Command::Fetch { package } => builder.fetch(&recipes, &package),
Command::Graph { package } => {
for line in graph.render(package.as_deref())? {
println!("{line}");
}
Ok(())
}
Command::List => {
for output in graph.outputs() {
println!("{output}");
}
Ok(())
}
Command::Info { package } => {
let output = graph.output(&package)?;
println!("{}", serde_json::to_string_pretty(output)?);
Ok(())
}
Command::Shell { package } => builder.shell(&recipes, &graph, &package),
Command::RepoIndex => builder.repo_index(),
Command::Rootfs { root, packages } => {
if packages.is_empty() {
bail!("rootfs requires at least one package");
}
builder.rootfs(&root, &packages)
}
Command::Update { .. } => unreachable!(),
Command::Clean | Command::InitKey => unreachable!(),
Command::Build(command) => {
let lib = eval::eval_lib(&root_path.join("lib"), Some(&config.options))?;
let recipes = RecipeSet::load(&root_path, &config.options, lib.as_ref())?;
let mut builder = Builder::new(root_path, config);
builder.build(&recipes, &command.recipes, command.rebuild, command.dry_run)
}
Command::Image => {
let mut builder = Builder::new(root_path, config);
builder.ensure_container_ready()
}
}
}
src/config.rs
+106 -26
View File
@@ -1,37 +1,117 @@
use crate::starlark::{eval_file, get_json_map, get_string, get_string_default};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::{
collections::HashMap,
path::{Path, PathBuf},
};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
use starlark::values::dict::FrozenDictRef;
use crate::{
eval::{ExtractError, eval_file, extract_string},
options::Options,
};
#[derive(Debug)]
pub enum ContainerRuntime {
Docker,
Podman,
}
impl ContainerRuntime {
pub fn as_str(&self) -> &'static str {
match self {
Self::Docker => "docker",
Self::Podman => "podman",
}
}
}
impl std::fmt::Display for ContainerRuntime {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
impl TryFrom<&str> for ContainerRuntime {
type Error = anyhow::Error;
fn try_from(value: &str) -> anyhow::Result<Self> {
match value {
"docker" => Ok(Self::Docker),
"podman" => Ok(Self::Podman),
_ => anyhow::bail!("invalid runtime: {value}"),
}
}
}
#[derive(Debug)]
pub struct Config {
pub target_arch: String,
pub options: BTreeMap<String, JsonValue>,
pub container_runtime: String,
pub container_runtime: ContainerRuntime,
pub container_image: String,
pub container_dockerfile: PathBuf,
pub signing_key: PathBuf,
pub signing_pubkey: PathBuf,
pub arch: String,
pub options: Options,
}
impl Config {
pub fn load(path: &Path) -> Result<Self> {
let module = eval_file(path, None, None)
.with_context(|| format!("failed to evaluate {}", path.display()))?;
pub fn load(path: &Path) -> anyhow::Result<Self> {
let module = eval_file(path, None, None)?;
let container_runtime = match extract_string(&module, "container_runtime") {
Ok(v) => ContainerRuntime::try_from(v.as_str())?,
Err(ExtractError::NotFound) => ContainerRuntime::Podman,
Err(ExtractError::TypeMismatch) => anyhow::bail!("`container_runtime` is not a string"),
};
let container_image = match extract_string(&module, "container_image") {
Ok(container_image) => container_image,
Err(ExtractError::NotFound) => {
anyhow::bail!("`container_image` config variable not set")
}
Err(ExtractError::TypeMismatch) => anyhow::bail!("`container_image` is not a string"),
};
let container_dockerfile = match extract_string(&module, "container_dockerfile") {
Ok(container_dockerfile) => PathBuf::from(container_dockerfile),
Err(ExtractError::NotFound) => PathBuf::from("Dockerfile"),
Err(ExtractError::TypeMismatch) => {
anyhow::bail!("`container_dockerfile` is not a string")
}
};
let arch = match extract_string(&module, "arch") {
Ok(arch) => arch,
Err(ExtractError::NotFound) => anyhow::bail!("`arch` config variable not set"),
Err(ExtractError::TypeMismatch) => anyhow::bail!("`arch` is not a string"),
};
let frozen_module = module.freeze()?;
let options_value = frozen_module
.get_option("options")?
.ok_or_else(|| anyhow::anyhow!("`options` config variable not set"))?;
let entries = {
// SAFETY: the FrozenValue is only used to construct a FrozenDictRef whose
// lifetime is bounded by `options_value`, which keeps the frozen heap alive.
let dict =
FrozenDictRef::from_frozen_value(unsafe { options_value.unchecked_frozen_value() })
.ok_or_else(|| anyhow::anyhow!("`options` is not a dict"))?;
dict.iter()
.map(|(k, v)| {
let key = k
.to_value()
.unpack_str()
.ok_or_else(|| anyhow::anyhow!("non-string key in `options`"))?
.to_owned();
Ok((key, options_value.map(|_| v)))
})
.collect::<anyhow::Result<HashMap<_, _>>>()?
};
let options = Options::new(entries);
Ok(Self {
target_arch: get_string(&module, "target_arch")?,
options: get_json_map(&module, "options")?,
container_runtime: get_string_default(&module, "container_runtime", "podman")?,
container_image: get_string(&module, "container_image")?,
container_dockerfile: PathBuf::from(get_string_default(
&module,
"container_dockerfile",
"Dockerfile",
)?),
signing_key: PathBuf::from(get_string(&module, "signing_key")?),
signing_pubkey: PathBuf::from(get_string(&module, "signing_pubkey")?),
container_runtime,
container_image,
container_dockerfile,
arch,
options,
})
}
}
src/eval.rs
+177
View File
@@ -0,0 +1,177 @@
use anyhow::Context;
use starlark::{
environment::{FrozenModule, Globals, GlobalsBuilder, Module},
eval::Evaluator,
syntax::{AstModule, Dialect},
values::list::ListRef,
};
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
use crate::{
options::Options,
recipe::{GitSource, Metadata, Source, Subpackage, TarballSource},
};
#[derive(Debug)]
pub enum ExtractError {
NotFound,
TypeMismatch,
}
impl std::fmt::Display for ExtractError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ExtractError::NotFound => write!(f, "missing"),
ExtractError::TypeMismatch => write!(f, "wrong type"),
}
}
}
impl std::error::Error for ExtractError {}
#[starlark::starlark_module]
fn builder_globals(builder: &mut GlobalsBuilder) {
fn meta(
maintainer: Option<String>,
description: Option<String>,
license: Option<String>,
website: Option<String>,
) -> anyhow::Result<Metadata> {
Ok(Metadata::new(maintainer, description, license, website))
}
fn tarball_source(
url: String,
sha256: String,
strip_components: Option<u32>,
) -> anyhow::Result<Source> {
Ok(Source::Tarball(TarballSource::new(
url,
sha256,
strip_components.unwrap_or(0),
)))
}
fn git_source(url: String, commit: String) -> anyhow::Result<Source> {
Ok(Source::Git(GitSource::new(url, commit)))
}
fn subpackage(name: String, metadata: Option<&Metadata>) -> anyhow::Result<Subpackage> {
let metadata = metadata
.cloned()
.unwrap_or_else(|| Metadata::new(None, None, None, None));
Ok(Subpackage::new(name, metadata))
}
}
pub fn eval_file(
path: &Path,
options: Option<&Options>,
lib: Option<&FrozenModule>,
) -> anyhow::Result<Module> {
let module = Module::new();
if let Some(lib) = lib {
module.import_public_symbols(lib);
}
if let Some(options) = options {
inject_options(&module, options);
}
let ast = AstModule::parse_file(path, &dialect()).map_err(|err| anyhow::anyhow!("{err}"))?;
let globals = globals();
let mut eval = Evaluator::new(&module);
eval.eval_module(ast, &globals)
.map_err(|err| anyhow::anyhow!("{err}"))?;
drop(eval);
Ok(module)
}
/// Parse and evaluate every `.star` file under `dir` into a single frozen
/// module whose public bindings can be imported into recipe modules. Returns
/// `Ok(None)` if `dir` doesn't exist or contains no `.star` files.
pub fn eval_lib(dir: &Path, options: Option<&Options>) -> anyhow::Result<Option<FrozenModule>> {
if !dir.exists() {
return Ok(None);
}
let mut files: Vec<PathBuf> = Vec::new();
for entry in WalkDir::new(dir) {
let entry = entry.with_context(|| format!("walking lib directory {}", dir.display()))?;
let path = entry.path();
if entry.file_type().is_file() && path.extension().is_some_and(|ext| ext == "star") {
files.push(path.to_path_buf());
}
}
if files.is_empty() {
return Ok(None);
}
// Sorted for deterministic ordering when later definitions shadow earlier ones.
files.sort();
let module = Module::new();
if let Some(options) = options {
inject_options(&module, options);
}
let dialect = dialect();
let globals = globals();
for file in &files {
let ast = AstModule::parse_file(file, &dialect)
.map_err(|err| anyhow::anyhow!("parsing {}: {err}", file.display()))?;
let mut eval = Evaluator::new(&module);
eval.eval_module(ast, &globals)
.map_err(|err| anyhow::anyhow!("evaluating {}: {err}", file.display()))?;
}
Ok(Some(module.freeze()?))
}
fn dialect() -> Dialect {
Dialect {
enable_top_level_stmt: true,
enable_f_strings: true,
..Dialect::Standard
}
}
fn globals() -> Globals {
GlobalsBuilder::standard().with(builder_globals).build()
}
fn inject_options(module: &Module, options: &Options) {
let value = module.heap().alloc(options.clone());
module.set("options", value);
}
pub fn extract_string(module: &Module, key: &str) -> Result<String, ExtractError> {
module
.get(key)
.ok_or_else(|| ExtractError::NotFound)
.and_then(|v| {
v.unpack_str()
.map(|v| v.to_string())
.ok_or_else(|| ExtractError::TypeMismatch)
})
}
pub fn extract_i32(module: &Module, key: &str) -> Result<i32, ExtractError> {
module
.get(key)
.ok_or(ExtractError::NotFound)
.and_then(|v| v.unpack_i32().ok_or(ExtractError::TypeMismatch))
}
pub fn extract_string_list(module: &Module, key: &str) -> Result<Vec<String>, ExtractError> {
let value = module.get(key).ok_or(ExtractError::NotFound)?;
let list = ListRef::from_value(value).ok_or(ExtractError::TypeMismatch)?;
list.iter()
.map(|v| {
v.unpack_str()
.map(|s| s.to_string())
.ok_or(ExtractError::TypeMismatch)
})
.collect()
}
src/graph.rs
+524 -107
View File
@@ -1,135 +1,552 @@
use crate::recipe::{OutputPackage, PackageKind, RecipeSet, unresolved_deps};
use anyhow::{Result, anyhow, bail};
use std::collections::{BTreeMap, BTreeSet};
use std::{
collections::{BTreeMap, BTreeSet},
fmt, fs,
path::{Path, PathBuf},
};
#[derive(Debug, Clone)]
pub struct PackageGraph {
outputs: BTreeMap<String, OutputPackage>,
target_edges: BTreeMap<String, Vec<String>>,
host_edges: BTreeMap<String, Vec<String>>,
use anyhow::{Context, bail};
use sha2::{Digest, Sha256};
use crate::recipe::{OutputPackage, Recipe, RecipeKind, RecipeSet};
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub enum TaskId {
FetchSources(String),
PrepareSources(String),
ConfigureRecipe(String),
BuildRecipe(String),
InstallPackageFiles(String),
ProduceApk(String),
InstallHostRecipe(String),
}
impl PackageGraph {
pub fn new(recipes: &RecipeSet) -> Result<Self> {
let missing = unresolved_deps(recipes);
if !missing.is_empty() {
bail!("unresolved local dependencies:\n{}", missing.join("\n"));
}
let mut target_edges = BTreeMap::new();
let mut host_edges = BTreeMap::new();
for recipe in recipes.recipes.values() {
// host_deps always resolve into the host namespace.
let host_dep_keys: Vec<String> = recipe
.host_deps
.iter()
.map(|d| PackageKind::Host.key(d))
.collect();
for output in &recipe.outputs {
let key = output.key();
let mut edges = output.all_target_deps();
if output.name == recipe.name {
edges.extend(recipe.build_deps.iter().cloned());
edges.extend(recipe.deps.iter().cloned());
}
match output.kind {
PackageKind::Host => {
host_edges.insert(key, host_dep_keys.clone());
}
PackageKind::Target => {
let mut deps = host_dep_keys.clone();
deps.extend(edges);
target_edges.insert(key, deps);
impl fmt::Display for TaskId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::FetchSources(recipe) => write!(f, "fetch sources {recipe}"),
Self::PrepareSources(recipe) => write!(f, "prepare sources {recipe}"),
Self::ConfigureRecipe(recipe) => write!(f, "configure {recipe}"),
Self::BuildRecipe(recipe) => write!(f, "build {recipe}"),
Self::InstallPackageFiles(output) => write!(f, "install package files {output}"),
Self::ProduceApk(output) => write!(f, "produce apk {output}"),
Self::InstallHostRecipe(recipe) => write!(f, "install host recipe {recipe}"),
}
}
}
}
Ok(Self {
outputs: recipes.outputs.clone(),
target_edges,
host_edges,
})
}
pub fn output(&self, package: &str) -> Result<&OutputPackage> {
self.outputs
.get(package)
.ok_or_else(|| anyhow!("unknown package `{package}`"))
#[derive(Debug)]
pub struct TaskPlan {
dependencies: BTreeMap<TaskId, Vec<TaskId>>,
order: Vec<TaskId>,
}
pub fn outputs(&self) -> impl Iterator<Item = &str> {
self.outputs.keys().map(String::as_str)
impl TaskPlan {
pub fn order(&self) -> &[TaskId] {
&self.order
}
pub fn build_order(&self, package: &str) -> Result<Vec<String>> {
self.output(package)?;
let mut visiting = BTreeSet::new();
let mut visited = BTreeSet::new();
let mut order = Vec::new();
self.visit(package, &mut visiting, &mut visited, &mut order)?;
Ok(order)
pub fn is_empty(&self) -> bool {
self.order.is_empty()
}
/// Topologically-ordered list of every output in the graph (host + target).
pub fn build_order_all(&self) -> Result<Vec<String>> {
let mut visiting = BTreeSet::new();
let mut visited = BTreeSet::new();
let mut order = Vec::new();
for package in self.outputs.keys() {
self.visit(package, &mut visiting, &mut visited, &mut order)?;
}
Ok(order)
pub fn dependency_count(&self) -> usize {
self.dependencies.values().map(Vec::len).sum()
}
fn visit(
&self,
package: &str,
visiting: &mut BTreeSet<String>,
visited: &mut BTreeSet<String>,
order: &mut Vec<String>,
) -> Result<()> {
if visited.contains(package) {
#[cfg(test)]
pub fn dependencies(&self, task: &TaskId) -> Option<&[TaskId]> {
self.dependencies.get(task).map(Vec::as_slice)
}
}
pub struct TaskPlanner<'a> {
root: &'a Path,
arch: &'a str,
recipes: &'a RecipeSet,
force: bool,
dependencies: BTreeMap<TaskId, Vec<TaskId>>,
inactive: BTreeSet<TaskId>,
visiting: BTreeSet<TaskId>,
visited: BTreeSet<TaskId>,
}
impl<'a> TaskPlanner<'a> {
pub fn new(root: &'a Path, arch: &'a str, recipes: &'a RecipeSet) -> Self {
Self {
root,
arch,
recipes,
force: false,
dependencies: BTreeMap::new(),
inactive: BTreeSet::new(),
visiting: BTreeSet::new(),
visited: BTreeSet::new(),
}
}
pub fn build_plan(mut self, requests: &[String], force: bool) -> anyhow::Result<TaskPlan> {
self.force = force;
for request in requests {
let recipe = self.recipes.recipe(request)?;
match recipe.kind() {
RecipeKind::Package => {
for output in recipe.outputs() {
self.visit(TaskId::ProduceApk(output.key()))?;
}
}
RecipeKind::HostPackage => {
self.visit(TaskId::InstallHostRecipe(recipe.key()))?;
}
}
}
self.into_plan()
}
pub fn fetch_plan(mut self, requests: &[String]) -> anyhow::Result<TaskPlan> {
for request in requests {
let recipe = self.recipes.recipe(request)?;
self.visit(TaskId::FetchSources(recipe.key()))?;
}
self.into_plan()
}
fn visit(&mut self, task: TaskId) -> anyhow::Result<()> {
if self.visited.contains(&task) || self.inactive.contains(&task) {
return Ok(());
}
if !visiting.insert(package.to_owned()) {
bail!("dependency cycle involving `{package}`");
if !self.is_active(&task)? {
self.inactive.insert(task);
return Ok(());
}
let deps = self
.target_edges
.get(package)
.or_else(|| self.host_edges.get(package))
.cloned()
.unwrap_or_default();
for dep in deps {
self.visit(&dep, visiting, visited, order)?;
if !self.visiting.insert(task.clone()) {
bail!("task dependency cycle involving `{task}`");
}
visiting.remove(package);
visited.insert(package.to_owned());
order.push(package.to_owned());
let dependencies = self.dependencies(&task)?;
let mut active_dependencies = Vec::new();
for dependency in dependencies {
self.visit(dependency.clone())?;
if self.dependencies.contains_key(&dependency) {
active_dependencies.push(dependency);
}
}
self.visiting.remove(&task);
self.visited.insert(task.clone());
self.dependencies.insert(task, active_dependencies);
Ok(())
}
pub fn render(&self, package: Option<&str>) -> Result<Vec<String>> {
match package {
Some(package) => {
let order = self.build_order(package)?;
Ok(order
.into_iter()
.map(|pkg| format!("{pkg}: {:?}", self.edges(&pkg)))
.collect())
fn into_plan(self) -> anyhow::Result<TaskPlan> {
let mut order = Vec::new();
let mut visiting = BTreeSet::new();
let mut visited = BTreeSet::new();
for task in self.dependencies.keys() {
topo_visit(
task,
&self.dependencies,
&mut visiting,
&mut visited,
&mut order,
)?;
}
Ok(TaskPlan {
dependencies: self.dependencies,
order,
})
}
fn dependencies(&self, task: &TaskId) -> anyhow::Result<Vec<TaskId>> {
match task {
TaskId::FetchSources(_) => Ok(Vec::new()),
TaskId::PrepareSources(recipe) => Ok(vec![TaskId::FetchSources(recipe.clone())]),
TaskId::ConfigureRecipe(recipe) => {
let recipe = self.recipes.recipe(recipe)?;
let mut deps = vec![TaskId::PrepareSources(recipe.key())];
deps.extend(
recipe
.host_deps()
.iter()
.map(|dep| TaskId::InstallHostRecipe(RecipeKind::HostPackage.key(dep))),
);
deps.extend(
recipe
.build_deps()
.iter()
.chain(recipe.deps().iter())
.map(|dep| TaskId::ProduceApk(dep.clone())),
);
Ok(deps)
}
TaskId::BuildRecipe(recipe) => Ok(vec![TaskId::ConfigureRecipe(recipe.clone())]),
TaskId::InstallPackageFiles(output) => {
let output = self.recipes.output(output)?;
Ok(vec![TaskId::BuildRecipe(output.recipe().to_owned())])
}
TaskId::ProduceApk(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
let mut deps = vec![TaskId::InstallPackageFiles(output.key())];
deps.extend(
recipe
.deps()
.iter()
.chain(recipe.run_deps().iter())
.map(|dep| TaskId::ProduceApk(dep.clone())),
);
Ok(deps)
}
TaskId::InstallHostRecipe(recipe) => {
self.recipes.recipe(recipe)?;
Ok(vec![TaskId::BuildRecipe(recipe.clone())])
}
None => Ok(self
.outputs
.keys()
.map(|pkg| format!("{pkg}: {:?}", self.edges(pkg)))
.collect()),
}
}
fn edges(&self, package: &str) -> Vec<String> {
self.target_edges
.get(package)
.or_else(|| self.host_edges.get(package))
.cloned()
.unwrap_or_default()
fn is_active(&self, task: &TaskId) -> anyhow::Result<bool> {
match task {
TaskId::FetchSources(recipe) => self.fetch_sources_active(self.recipes.recipe(recipe)?),
TaskId::PrepareSources(recipe) => {
self.prepare_sources_active(self.recipes.recipe(recipe)?)
}
TaskId::ConfigureRecipe(recipe) => {
self.recipe_task_active(self.recipes.recipe(recipe)?, "configure")
}
TaskId::BuildRecipe(recipe) => {
self.recipe_task_active(self.recipes.recipe(recipe)?, "build")
}
TaskId::InstallPackageFiles(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
self.output_task_active(recipe, output, "install")
}
TaskId::ProduceApk(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
self.produce_apk_active(recipe, output)
}
TaskId::InstallHostRecipe(recipe) => {
let recipe = self.recipes.recipe(recipe)?;
self.install_host_recipe_active(recipe)
}
}
}
fn fetch_sources_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
Ok(recipe.sources().entries().iter().any(|(_, source)| {
source.is_unknown_cache_key() || !self.source_cache_path(source.cache_key()).exists()
}))
}
fn prepare_sources_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
if self.force {
return Ok(true);
}
let want_version = format!("{}-r{}", recipe.version(), recipe.revision());
if fs::read_to_string(self.source_stamp(recipe, "version"))
.ok()
.as_deref()
!= Some(want_version.as_str())
{
return Ok(true);
}
if self.recipe_has_patches(recipe)? && !self.source_stamp(recipe, "patched").exists() {
return Ok(true);
}
Ok(false)
}
fn recipe_task_active(&self, recipe: &Recipe, kind: &str) -> anyhow::Result<bool> {
if self.force {
return Ok(true);
}
Ok(fs::read_to_string(self.recipe_task_stamp(recipe, kind))
.ok()
.as_deref()
!= Some(self.recipe_fingerprint(recipe)?.as_str()))
}
fn output_task_active(
&self,
recipe: &Recipe,
output: &OutputPackage,
kind: &str,
) -> anyhow::Result<bool> {
if self.force {
return Ok(true);
}
Ok(fs::read_to_string(self.output_task_stamp(output, kind))
.ok()
.as_deref()
!= Some(self.output_fingerprint(recipe, output)?.as_str()))
}
fn produce_apk_active(&self, recipe: &Recipe, output: &OutputPackage) -> anyhow::Result<bool> {
if self.force {
return Ok(true);
}
if !self.apk_path(recipe, output).exists() {
return Ok(true);
}
Ok(fs::read_to_string(self.output_task_stamp(output, "apk"))
.ok()
.as_deref()
!= Some(self.output_fingerprint(recipe, output)?.as_str()))
}
fn install_host_recipe_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
if self.force {
return Ok(true);
}
if !self.host_install_dir(recipe).exists() {
return Ok(true);
}
Ok(
fs::read_to_string(self.recipe_task_stamp(recipe, "host-install"))
.ok()
.as_deref()
!= Some(self.recipe_fingerprint(recipe)?.as_str()),
)
}
fn recipe_fingerprint(&self, recipe: &Recipe) -> anyhow::Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.arch.as_bytes());
hasher.update(recipe.key().as_bytes());
hasher.update(recipe.version().as_bytes());
hasher.update(recipe.revision().to_le_bytes());
hasher.update(
fs::read(recipe.path())
.with_context(|| format!("reading recipe {}", recipe.path().display()))?,
);
for (name, source) in recipe.sources().entries() {
hasher.update(name.unwrap_or("").as_bytes());
hasher.update(source.url().as_bytes());
hasher.update(source.cache_key().as_bytes());
}
for patch in self.recipe_patches(recipe)? {
hasher.update(patch.display().to_string().as_bytes());
hasher
.update(fs::read(&patch).with_context(|| format!("reading {}", patch.display()))?);
}
Ok(hex::encode(hasher.finalize()))
}
fn output_fingerprint(
&self,
recipe: &Recipe,
output: &OutputPackage,
) -> anyhow::Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.recipe_fingerprint(recipe)?.as_bytes());
hasher.update(output.key().as_bytes());
Ok(hex::encode(hasher.finalize()))
}
fn recipe_has_patches(&self, recipe: &Recipe) -> anyhow::Result<bool> {
Ok(!self.recipe_patches(recipe)?.is_empty())
}
fn recipe_patches(&self, recipe: &Recipe) -> anyhow::Result<Vec<PathBuf>> {
let patches_dir = recipe.dir().join("patches");
if !patches_dir.exists() {
return Ok(Vec::new());
}
let mut patches = Vec::new();
for entry in fs::read_dir(&patches_dir)
.with_context(|| format!("reading patches directory {}", patches_dir.display()))?
{
let entry = entry?;
let path = entry.path();
if path.is_file() {
patches.push(path);
}
}
patches.sort();
Ok(patches)
}
fn source_cache_path(&self, key: &str) -> PathBuf {
self.root.join("build/cache/sources").join(key)
}
fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.root
.join("build/sources")
.join(format!("{}.{kind}", recipe.slug()))
}
fn recipe_task_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.root
.join("build/tasks")
.join(format!("{}.{kind}", recipe.slug()))
}
fn output_task_stamp(&self, output: &OutputPackage, kind: &str) -> PathBuf {
self.root
.join("build/tasks")
.join(format!("{}.{kind}", output.key().replace(':', "-")))
}
fn apk_path(&self, recipe: &Recipe, output: &OutputPackage) -> PathBuf {
self.root.join("build/pkgs").join(self.arch).join(format!(
"{}-{}-r{}.apk",
output.name(),
recipe.version(),
recipe.revision()
))
}
fn host_install_dir(&self, recipe: &Recipe) -> PathBuf {
self.root
.join("build/host-pkgs")
.join(recipe.slug())
.join("usr/local")
}
}
fn topo_visit(
task: &TaskId,
dependencies: &BTreeMap<TaskId, Vec<TaskId>>,
visiting: &mut BTreeSet<TaskId>,
visited: &mut BTreeSet<TaskId>,
order: &mut Vec<TaskId>,
) -> anyhow::Result<()> {
if visited.contains(task) {
return Ok(());
}
if !visiting.insert(task.clone()) {
bail!("task dependency cycle involving `{task}`");
}
for dependency in dependencies.get(task).into_iter().flatten() {
topo_visit(dependency, dependencies, visiting, visited, order)?;
}
visiting.remove(task);
visited.insert(task.clone());
order.push(task.clone());
Ok(())
}
#[cfg(test)]
mod tests {
use std::fs;
use tempfile::TempDir;
use crate::{config::Config, eval, recipe::RecipeSet};
use super::{TaskId, TaskPlanner};
fn write_config(root: &TempDir) {
fs::write(
root.path().join("config.star"),
r#"
container_runtime = "podman"
container_image = "local/test:latest"
container_dockerfile = "Dockerfile"
arch = "x86_64"
options = dict(target_arch = arch)
"#,
)
.unwrap();
fs::write(root.path().join("Dockerfile"), "FROM scratch\n").unwrap();
}
fn write_recipe(root: &TempDir, dir: &str, name: &str, extra: &str) {
let recipe_dir = root.path().join(dir);
fs::create_dir_all(&recipe_dir).unwrap();
fs::write(
recipe_dir.join(format!("{name}.star")),
format!(
r#"
version = "1.0"
revision = 1
source = tarball_source(url = "file:///tmp/{name}.tar", sha256 = "hash-{name}")
{extra}
def build(ctx):
ctx.run(["true"])
def install(ctx, pkg):
ctx.run(["true"])
"#
),
)
.unwrap();
}
fn load(root: &TempDir) -> (Config, RecipeSet) {
let config = Config::load(&root.path().join("config.star")).unwrap();
let lib = eval::eval_lib(&root.path().join("lib"), Some(&config.options)).unwrap();
let recipes = RecipeSet::load(root.path(), &config.options, lib.as_ref()).unwrap();
(config, recipes)
}
#[test]
fn inactive_seed_does_not_pull_dependencies() {
let root = TempDir::new().unwrap();
write_config(&root);
write_recipe(&root, "recipes", "dep", "");
write_recipe(&root, "recipes", "app", r#"deps = ["dep"]"#);
let (config, recipes) = load(&root);
let planner = TaskPlanner::new(root.path(), &config.arch, &recipes);
let output = recipes.output("app").unwrap();
let recipe = recipes.recipe(output.recipe()).unwrap();
fs::create_dir_all(root.path().join("build/pkgs/x86_64")).unwrap();
fs::write(root.path().join("build/pkgs/x86_64/app-1.0-r1.apk"), "").unwrap();
fs::create_dir_all(root.path().join("build/tasks")).unwrap();
fs::write(
root.path().join("build/tasks/app.apk"),
planner.output_fingerprint(recipe, output).unwrap(),
)
.unwrap();
let plan = TaskPlanner::new(root.path(), &config.arch, &recipes)
.build_plan(&["app".to_owned()], false)
.unwrap();
assert!(plan.is_empty());
}
#[test]
fn active_target_recipe_keeps_edges_and_topo_order() {
let root = TempDir::new().unwrap();
write_config(&root);
write_recipe(&root, "recipes", "app", "");
let (config, recipes) = load(&root);
let plan = TaskPlanner::new(root.path(), &config.arch, &recipes)
.build_plan(&["app".to_owned()], false)
.unwrap();
assert_eq!(
plan.order(),
&[
TaskId::FetchSources("app".to_owned()),
TaskId::PrepareSources("app".to_owned()),
TaskId::ConfigureRecipe("app".to_owned()),
TaskId::BuildRecipe("app".to_owned()),
TaskId::InstallPackageFiles("app".to_owned()),
TaskId::ProduceApk("app".to_owned()),
]
);
assert_eq!(
plan.dependencies(&TaskId::ProduceApk("app".to_owned())),
Some([TaskId::InstallPackageFiles("app".to_owned())].as_slice())
);
}
#[test]
fn host_dependencies_are_installed_as_host_recipes() {
let root = TempDir::new().unwrap();
write_config(&root);
write_recipe(&root, "host-recipes", "binutils", "");
write_recipe(&root, "recipes", "app", r#"host_deps = ["binutils"]"#);
let (config, recipes) = load(&root);
let plan = TaskPlanner::new(root.path(), &config.arch, &recipes)
.build_plan(&["app".to_owned()], false)
.unwrap();
assert!(
plan.order()
.contains(&TaskId::InstallHostRecipe("host:binutils".to_owned()))
);
}
}
src/log.rs
+11 -20
View File
@@ -1,35 +1,26 @@
//! Tiny stderr logger. We use a consistent `==> <action>: <details>` prefix
//! so progress messages are easy to scan during long builds.
use std::{io::IsTerminal, sync::LazyLock};
use std::io::{IsTerminal, Write};
static IS_STDERR_TERMINAL: LazyLock<bool> = LazyLock::new(|| std::io::stderr().is_terminal());
const ARROW: &str = "==>";
fn paint(color: &str, text: &str) -> String {
if std::io::stderr().is_terminal() {
format!("\x1b[{color}m{text}\x1b[0m")
} else {
text.to_owned()
}
}
fn emit(color: &str, action: &str, details: &str) {
let arrow = paint(color, ARROW);
let action = paint("1", action);
let _ = writeln!(std::io::stderr(), "{arrow} {action} {details}");
if *IS_STDERR_TERMINAL {
eprintln!("\x1b[{color}m{ARROW} \x1b[1m{action} \x1b[0m{details}");
} else {
eprintln!("{ARROW} {action} {details}");
}
}
/// Major step, e.g. starting a build or packaging an output.
pub fn step(action: &str, details: &str) {
emit("1;34", action, details); // bold blue
emit("1;34", action, details);
}
/// Cache hit / skipped work.
pub fn skip(action: &str, details: &str) {
emit("1;33", action, details); // bold yellow
emit("1;33", action, details);
}
/// Sub-step inside a larger action.
#[allow(dead_code)]
pub fn info(action: &str, details: &str) {
emit("1;32", action, details); // bold green
emit("1;32", action, details);
}
src/main.rs
+5 -14
View File
@@ -1,21 +1,12 @@
mod apk;
mod build;
mod builder;
mod cli;
mod config;
mod eval;
mod graph;
mod log;
mod patches;
mod phase;
mod options;
mod recipe;
mod rewrite;
mod source;
mod starlark;
mod update;
use anyhow::Result;
use clap::Parser;
fn main() -> Result<()> {
let cli = cli::Cli::parse();
cli::run(cli)
fn main() -> anyhow::Result<()> {
cli::run()
}
src/options.rs
+35
View File
@@ -0,0 +1,35 @@
use allocative::Allocative;
use starlark::values::{Heap, OwnedFrozenValue, StarlarkValue, Value};
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value};
use std::collections::HashMap;
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct Options {
entries: HashMap<String, OwnedFrozenValue>,
}
impl Options {
pub fn new(entries: HashMap<String, OwnedFrozenValue>) -> Self {
Self { entries }
}
}
impl std::fmt::Display for Options {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "options")
}
}
starlark::starlark_simple_value!(Options);
#[starlark_value(type = "options")]
impl<'v> StarlarkValue<'v> for Options {
fn get_attr(&self, attribute: &str, _heap: &'v Heap) -> Option<Value<'v>> {
let owned = self.entries.get(attribute)?;
// SAFETY: `self` is kept alive by the module heap into which it was
// allocated, and `owned` holds an Arc to its source frozen heap. The
// returned Value therefore remains valid for as long as the receiving
// module is alive.
Some(unsafe { owned.unchecked_frozen_value() }.to_value())
}
}
src/patches.rs
-19
View File
@@ -1,19 +0,0 @@
use anyhow::Result;
use std::path::{Path, PathBuf};
pub fn discover(recipe_dir: &Path) -> Result<Vec<PathBuf>> {
let patch_dir = recipe_dir.join("patches");
if !patch_dir.exists() {
return Ok(Vec::new());
}
let mut patches = Vec::new();
for entry in std::fs::read_dir(&patch_dir)? {
let entry = entry?;
let path = entry.path();
if path.extension().and_then(|ext| ext.to_str()) == Some("patch") {
patches.push(path);
}
}
patches.sort();
Ok(patches)
}
src/phase.rs
-213
View File
@@ -1,213 +0,0 @@
use crate::config::Config;
use crate::starlark::{eval_content_with_extra, prepend_common_lib_load};
use allocative::Allocative;
use anyhow::{Result, anyhow, bail};
use serde::{Deserialize, Serialize};
use starlark::environment::{GlobalsBuilder, LibraryExtension};
use starlark::eval::Evaluator;
use starlark::starlark_module;
use starlark::values::none::NoneType;
use starlark::values::{ProvidesStaticType, Value};
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::path::Path;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Allocative)]
pub struct PhaseCommand {
pub argv: Vec<String>,
/// Extra environment variables exported just for this command, in the
/// order the recipe supplied them. Empty means "inherit only".
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub env: Vec<(String, String)>,
}
/// How `ctx.source_dir` is exposed to Starlark.
///
/// * `Single` is a single container path (e.g. `/source`), used when the
/// recipe declared a `source = {...}` form.
/// * `Many` is a map of source name to container path, exposed as a
/// `struct(...)` (e.g. `ctx.source_dir.linux`), used when the recipe
/// declared `sources = {"linux": {...}, ...}`.
#[derive(Debug, Clone)]
pub enum SourceDir {
Single(String),
Many(BTreeMap<String, String>),
}
/// Per-phase paths exposed to Starlark as `ctx.source_dir`, `ctx.build_dir`,
/// `ctx.dest_dir`, `ctx.prefix`, `ctx.sysroot` (Jinx-inspired).
#[derive(Debug, Clone)]
pub struct PhaseEnv<'a> {
pub source_dir: SourceDir,
pub build_dir: &'a str,
pub dest_dir: &'a str,
pub prefix: &'a str,
pub sysroot: &'a str,
}
#[derive(Debug, Default, ProvidesStaticType, Allocative)]
struct CommandStore {
commands: RefCell<Vec<PhaseCommand>>,
}
impl CommandStore {
fn push(&self, command: PhaseCommand) {
self.commands.borrow_mut().push(command);
}
}
#[starlark_module]
fn phase_globals(builder: &mut GlobalsBuilder) {
fn ctx_run<'v>(
argv: Value<'v>,
#[starlark(require = named, default = NoneType)] env: Value<'v>,
eval: &mut Evaluator,
) -> anyhow::Result<NoneType> {
let json = argv.to_json()?;
let values: Vec<String> = serde_json::from_str(&json)
.map_err(|err| anyhow!("ctx.run expects a list of strings: {err}"))?;
if values.is_empty() {
bail!("ctx.run argv cannot be empty");
}
let env_vars = parse_env(env)?;
store(eval)?.push(PhaseCommand {
argv: values,
env: env_vars,
});
Ok(NoneType)
}
fn ctx_install(
src: &str,
dst: &str,
#[starlark(require = named, default = "644")] mode: &str,
eval: &mut Evaluator,
) -> anyhow::Result<NoneType> {
store(eval)?.push(PhaseCommand {
argv: vec![
"install".to_owned(),
format!("-Dm{mode}"),
src.to_owned(),
dst.to_owned(),
],
env: Vec::new(),
});
Ok(NoneType)
}
}
fn parse_env(value: Value<'_>) -> anyhow::Result<Vec<(String, String)>> {
if value.is_none() {
return Ok(Vec::new());
}
let json = value.to_json()?;
let map: serde_json::Map<String, serde_json::Value> = serde_json::from_str(&json)
.map_err(|err| anyhow!("ctx.run env must be a dict of string -> string: {err}"))?;
let mut out = Vec::with_capacity(map.len());
for (key, val) in map {
let serde_json::Value::String(val) = val else {
bail!("ctx.run env value for `{key}` must be a string");
};
if key.is_empty() || key.contains('=') {
bail!("ctx.run env key `{key}` is not a valid variable name");
}
out.push((key, val));
}
Ok(out)
}
fn store<'a, 'b, 'c, 'd>(eval: &'a Evaluator<'b, 'c, 'd>) -> anyhow::Result<&'a CommandStore> {
eval.extra
.ok_or_else(|| anyhow!("ctx command used without command store"))?
.downcast_ref::<CommandStore>()
.ok_or_else(|| anyhow!("command store has the wrong type"))
}
pub fn collect_phase_commands(
recipe_path: &Path,
repo_root: &Path,
config: &Config,
phase: &str,
env: &PhaseEnv<'_>,
package: Option<(&str, &str)>,
) -> Result<Vec<PhaseCommand>> {
validate_identifier(phase)?;
let raw = std::fs::read_to_string(recipe_path)?;
// Auto-load helpers from `lib/common.star` so recipes never need an
// explicit `load()` for the canonical helpers.
let mut content = prepend_common_lib_load(Some(repo_root), Some(config), &raw)?;
let jobs = std::thread::available_parallelism()
.map(|j| j.get())
.unwrap_or(1);
let source_dir_expr = source_dir_literal(&env.source_dir)?;
let ctx_literal = format!(
"struct(run = ctx_run, install = ctx_install, jobs = {jobs}, \
source_dir = {sd}, build_dir = {bd}, dest_dir = {dd}, prefix = {pf}, sysroot = {sr})",
sd = source_dir_expr,
bd = serde_json::to_string(env.build_dir)?,
dd = serde_json::to_string(env.dest_dir)?,
pf = serde_json::to_string(env.prefix)?,
sr = serde_json::to_string(env.sysroot)?,
);
let call = match package {
Some((name, destdir)) => format!(
"\n__ctx = {ctx_literal}\n__pkg = struct(name = {n}, destdir = {d})\n{phase}(__ctx, __pkg)\n",
n = serde_json::to_string(name)?,
d = serde_json::to_string(destdir)?,
),
None => format!("\n__ctx = {ctx_literal}\n{phase}(__ctx)\n"),
};
content.push_str(&call);
let globals = GlobalsBuilder::extended_by(&[LibraryExtension::StructType])
.with(phase_globals)
.build();
let cmd_store = CommandStore::default();
eval_content_with_extra(
recipe_path,
content,
Some(config),
Some(repo_root),
globals,
Some(&cmd_store),
)?;
Ok(cmd_store.commands.into_inner())
}
fn source_dir_literal(source_dir: &SourceDir) -> Result<String> {
match source_dir {
SourceDir::Single(path) => Ok(serde_json::to_string(path)?),
SourceDir::Many(map) => {
let mut fields = Vec::with_capacity(map.len());
for (name, path) in map {
if !is_valid_field_name(name) {
bail!("source name `{name}` is not a valid Starlark identifier");
}
fields.push(format!("{name} = {}", serde_json::to_string(path)?));
}
Ok(format!("struct({})", fields.join(", ")))
}
}
}
fn is_valid_field_name(name: &str) -> bool {
let mut chars = name.chars();
match chars.next() {
Some(c) if c == '_' || c.is_ascii_alphabetic() => {}
_ => return false,
}
chars.all(|c| c == '_' || c.is_ascii_alphanumeric())
}
fn validate_identifier(name: &str) -> Result<()> {
let mut chars = name.chars();
let Some(first) = chars.next() else {
bail!("phase function name cannot be empty");
};
if !(first == '_' || first.is_ascii_alphabetic()) {
bail!("invalid phase function name `{name}`");
}
if chars.any(|ch| !(ch == '_' || ch.is_ascii_alphanumeric())) {
bail!("invalid phase function name `{name}`");
}
Ok(())
}
src/recipe.rs
-439
View File
@@ -1,439 +0,0 @@
use crate::config::Config;
use crate::starlark::{
eval_content, get_i32_default, get_json, get_string, get_string_default, get_string_vec,
has_name, prepend_common_lib_load,
};
use anyhow::{Result, anyhow, bail};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum PackageKind {
Host,
Target,
}
impl PackageKind {
/// Canonical `host:`/bare key form used across the graph, CLI and
/// manifest layer. Host and target trees are completely separate
/// namespaces — they may share names.
pub fn key(&self, name: &str) -> String {
match self {
PackageKind::Host => format!("host:{name}"),
PackageKind::Target => name.to_owned(),
}
}
/// Filesystem-safe variant of [`PackageKind::key`] (no `:`), used to
/// derive build/source/manifest directory names.
pub fn slug(&self, name: &str) -> String {
match self {
PackageKind::Host => format!("host-{name}"),
PackageKind::Target => name.to_owned(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Source {
/// Empty for the single-`source` form, otherwise the dict key from `sources`.
pub name: String,
pub url: String,
pub sha256: String,
/// Number of leading path components to strip when extracting (tar's
/// `--strip-components`). `0` means strip nothing.
pub strip_components: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct OutputPackage {
pub name: String,
/// Canonical key of the owning recipe (see [`PackageKind::key`]).
pub recipe: String,
pub kind: PackageKind,
pub version: String,
pub revision: i32,
pub description: String,
pub license: String,
/// Target packages installed into the build sysroot. Not propagated as
/// apk `depends:` metadata.
pub build_deps: Vec<String>,
/// Target packages declared as runtime dependencies (apk `depends:`).
/// Also installed into the sysroot so the recipe can link against them.
pub deps: Vec<String>,
pub install_fn: String,
}
impl OutputPackage {
pub fn key(&self) -> String {
self.kind.key(&self.name)
}
/// Union of build- and run-dependencies (used to materialize the sysroot
/// and to compute the build graph).
pub fn all_target_deps(&self) -> Vec<String> {
let mut out = self.build_deps.clone();
out.extend(self.deps.iter().cloned());
out
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Recipe {
pub id: String,
pub path: PathBuf,
pub dir: PathBuf,
pub name: String,
pub kind: PackageKind,
pub version: String,
pub revision: i32,
pub description: String,
pub license: String,
pub sources: Vec<Source>,
pub host_deps: Vec<String>,
pub build_deps: Vec<String>,
pub deps: Vec<String>,
pub outputs: Vec<OutputPackage>,
pub configure_fn: Option<String>,
pub build_fn: Option<String>,
pub check_fn: Option<String>,
}
impl Recipe {
/// Canonical key (`host:<id>` or `<id>`), used as the recipe-level
/// identifier in graphs, manifests and CLI references.
pub fn key(&self) -> String {
self.kind.key(&self.id)
}
/// Filesystem-safe variant of [`Recipe::key`].
pub fn slug(&self) -> String {
self.kind.slug(&self.id)
}
}
#[derive(Debug, Clone)]
pub struct RecipeSet {
pub recipes: BTreeMap<String, Recipe>,
pub outputs: BTreeMap<String, OutputPackage>,
}
impl RecipeSet {
/// Discover recipes under `repo_root`:
/// * `recipes/<name>/recipe.star` → target packages
/// * `host-recipes/<name>/recipe.star` → host packages
pub fn load(repo_root: &Path, config: &Config) -> Result<Self> {
let mut recipes = BTreeMap::new();
for (subdir, kind) in [
("recipes", PackageKind::Target),
("host-recipes", PackageKind::Host),
] {
let root = repo_root.join(subdir);
if !root.exists() {
continue;
}
for entry in WalkDir::new(&root).follow_links(false) {
let entry = entry?;
if entry.file_type().is_file() && entry.file_name() == "recipe.star" {
let recipe = Recipe::load(entry.path(), config, repo_root, kind.clone())?;
let key = recipe.key();
if recipes.insert(key.clone(), recipe).is_some() {
bail!("duplicate recipe `{key}` below {}", root.display());
}
}
}
}
let mut outputs = BTreeMap::new();
for recipe in recipes.values() {
for output in &recipe.outputs {
let key = output.key();
if outputs.insert(key.clone(), output.clone()).is_some() {
bail!("duplicate package output `{key}`");
}
}
}
Ok(Self { recipes, outputs })
}
/// Look up a recipe by the package key produced by an output.
pub fn recipe_for_package(&self, package: &str) -> Result<&Recipe> {
let output = self
.outputs
.get(package)
.ok_or_else(|| anyhow!("unknown package `{package}`"))?;
self.recipes.get(&output.recipe).ok_or_else(|| {
anyhow!(
"package `{package}` references missing recipe `{}`",
output.recipe
)
})
}
/// Resolve a user-supplied reference (recipe key, output key, or bare
/// id — provided it isn't ambiguous between the host and target trees).
pub fn recipe_by_user_ref(&self, name: &str) -> Result<&Recipe> {
if let Some(recipe) = self.recipes.get(name) {
return Ok(recipe);
}
if self.outputs.contains_key(name) {
return self.recipe_for_package(name);
}
// Bare id: search both trees, error on ambiguity.
let host_key = PackageKind::Host.key(name);
let target_key = PackageKind::Target.key(name);
match (self.recipes.get(&host_key), self.recipes.get(&target_key)) {
(Some(_), Some(_)) => bail!(
"`{name}` is ambiguous: matches both `{host_key}` and `{target_key}`; \
use the explicit form"
),
(Some(r), None) | (None, Some(r)) => Ok(r),
(None, None) => bail!("unknown recipe `{name}`"),
}
}
}
impl Recipe {
pub fn load(path: &Path, config: &Config, repo_root: &Path, kind: PackageKind) -> Result<Self> {
// Auto-load helpers from `lib/common.star` so recipes never need an
// explicit `load()` for the canonical helpers.
let raw = std::fs::read_to_string(path)?;
let content = prepend_common_lib_load(Some(repo_root), Some(config), &raw)?;
let module = eval_content(
path,
content,
Some(config),
Some(repo_root),
starlark::environment::Globals::standard(),
)?;
let dir = path
.parent()
.unwrap_or_else(|| Path::new("."))
.to_path_buf();
let id = dir
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| anyhow!("recipe path has no package directory: {}", path.display()))?
.to_owned();
let name = get_string(&module, "name")?;
let version = get_string(&module, "version")?;
let revision = get_i32_default(&module, "revision", 0)?;
let description = get_string_default(&module, "description", "???")?;
let license = get_string_default(&module, "license", "???")?;
let build_deps = get_string_vec(&module, "build_deps")?;
let deps = get_string_vec(&module, "deps")?;
let host_deps = get_string_vec(&module, "host_deps")?;
let sources = parse_sources(get_json(&module, "sources")?, get_json(&module, "source")?)?;
let subpackages = parse_subpackages(get_json(&module, "subpackages")?)?;
let mut outputs = Vec::new();
let recipe_key = kind.key(&id);
outputs.push(OutputPackage {
name: name.clone(),
recipe: recipe_key.clone(),
kind: kind.clone(),
version: version.clone(),
revision,
description: description.clone(),
license: license.clone(),
build_deps: build_deps.clone(),
deps: deps.clone(),
install_fn: "install".to_owned(),
});
for subpkg in subpackages {
let sub_name = subpkg
.get("name")
.and_then(JsonValue::as_str)
.ok_or_else(|| anyhow!("subpackage in `{name}` is missing string `name`"))?
.to_owned();
outputs.push(OutputPackage {
name: sub_name,
recipe: recipe_key.clone(),
kind: kind.clone(),
version: version.clone(),
revision,
description: subpkg
.get("description")
.and_then(JsonValue::as_str)
.unwrap_or(&description)
.to_owned(),
license: subpkg
.get("license")
.and_then(JsonValue::as_str)
.unwrap_or(&license)
.to_owned(),
build_deps: json_string_list(subpkg.get("build_deps"), "subpackage build_deps")?
.unwrap_or_default(),
deps: json_string_list(subpkg.get("deps"), "subpackage deps")?.unwrap_or_default(),
install_fn: subpkg
.get("install")
.and_then(JsonValue::as_str)
.unwrap_or("install")
.to_owned(),
});
}
validate_required(&outputs)?;
Ok(Self {
id,
path: path.to_path_buf(),
dir,
name,
kind,
version,
revision,
description,
license,
sources,
host_deps,
build_deps,
deps,
outputs,
configure_fn: has_name(&module, "configure").then_some("configure".to_owned()),
build_fn: has_name(&module, "build").then_some("build".to_owned()),
check_fn: has_name(&module, "check").then_some("check".to_owned()),
})
}
}
fn validate_required(outputs: &[OutputPackage]) -> Result<()> {
for output in outputs {
if output.name.trim().is_empty() {
bail!("package output name cannot be empty");
}
for (field, value) in [
("version", &output.version),
("description", &output.description),
("license", &output.license),
] {
if value.trim().is_empty() {
bail!(
"package `{}` has empty required field `{field}`",
output.name
);
}
}
}
Ok(())
}
fn parse_sources(
sources: Option<JsonValue>,
legacy_source: Option<JsonValue>,
) -> Result<Vec<Source>> {
match (sources, legacy_source) {
(Some(_), Some(_)) => bail!("recipe defines both `sources` and `source`; use only one"),
(None, None) => Ok(Vec::new()),
(None, Some(single)) => {
let obj = single
.as_object()
.ok_or_else(|| anyhow!("`source` must be a dict"))?;
Ok(vec![parse_source_entry(String::new(), obj)?])
}
(Some(multi), None) => {
let obj = multi
.as_object()
.ok_or_else(|| anyhow!("`sources` must be a dict of {{name: source}}"))?;
obj.iter()
.map(|(name, value)| {
if name.is_empty() {
bail!("source name in `sources` cannot be empty");
}
let entry = value
.as_object()
.ok_or_else(|| anyhow!("source `{name}` must be a dict"))?;
parse_source_entry(name.clone(), entry)
})
.collect()
}
}
}
fn parse_source_entry(name: String, obj: &serde_json::Map<String, JsonValue>) -> Result<Source> {
let url = obj
.get("url")
.and_then(JsonValue::as_str)
.ok_or_else(|| anyhow!("source entry missing string `url`"))?
.to_owned();
let sha256 = obj
.get("sha256")
.and_then(JsonValue::as_str)
.unwrap_or("???")
.to_owned();
let strip_components = match obj.get("strip_components") {
None => 0,
Some(JsonValue::Number(n)) => n
.as_u64()
.and_then(|v| u32::try_from(v).ok())
.ok_or_else(|| anyhow!("source `strip_components` must be a non-negative integer"))?,
Some(_) => bail!("source `strip_components` must be an integer"),
};
Ok(Source {
name,
url,
sha256,
strip_components,
})
}
fn parse_subpackages(value: Option<JsonValue>) -> Result<Vec<serde_json::Map<String, JsonValue>>> {
match value {
Some(JsonValue::Array(values)) => values
.into_iter()
.map(|value| {
value
.as_object()
.cloned()
.ok_or_else(|| anyhow!("subpackages entries must be objects"))
})
.collect(),
Some(_) => bail!("subpackages must be a list of objects"),
None => Ok(Vec::new()),
}
}
fn json_string_list(value: Option<&JsonValue>, label: &str) -> Result<Option<Vec<String>>> {
match value {
Some(JsonValue::Array(values)) => values
.iter()
.map(|value| {
value
.as_str()
.map(ToOwned::to_owned)
.ok_or_else(|| anyhow!("{label} must contain only strings"))
})
.collect::<Result<Vec<_>>>()
.map(Some),
Some(_) => bail!("{label} must be a string list"),
None => Ok(None),
}
}
pub fn unresolved_deps(recipes: &RecipeSet) -> Vec<String> {
let names: BTreeSet<_> = recipes.outputs.keys().cloned().collect();
let mut missing = Vec::new();
for recipe in recipes.recipes.values() {
// host_deps always refer to host outputs (canonical `host:<name>`);
// build_deps / deps refer to target outputs (bare names).
for dep in &recipe.host_deps {
let key = PackageKind::Host.key(dep);
if !names.contains(&key) {
missing.push(format!("{} -> {key}", recipe.key()));
}
}
for dep in recipe.build_deps.iter().chain(recipe.deps.iter()) {
if !names.contains(dep) {
missing.push(format!("{} -> {dep}", recipe.key()));
}
}
for output in &recipe.outputs {
for dep in output.build_deps.iter().chain(output.deps.iter()) {
if !names.contains(dep) {
missing.push(format!("{} -> {dep}", output.key()));
}
}
}
}
missing
}
src/recipe/metadata.rs
+54
View File
@@ -0,0 +1,54 @@
use allocative::Allocative;
use starlark::values::StarlarkValue;
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value};
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct Metadata {
maintainer: Option<String>,
description: Option<String>,
license: Option<String>,
website: Option<String>,
}
impl std::fmt::Display for Metadata {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "metadata")
}
}
starlark::starlark_simple_value!(Metadata);
#[starlark_value(type = "metadata")]
impl<'v> StarlarkValue<'v> for Metadata {}
impl Metadata {
pub fn new(
maintainer: Option<String>,
description: Option<String>,
license: Option<String>,
website: Option<String>,
) -> Self {
Self {
maintainer,
description,
license,
website,
}
}
pub fn maintainer(&self) -> Option<&str> {
self.maintainer.as_deref()
}
pub fn description(&self) -> Option<&str> {
self.description.as_deref()
}
pub fn license(&self) -> Option<&str> {
self.license.as_deref()
}
pub fn website(&self) -> Option<&str> {
self.website.as_deref()
}
}
src/recipe/mod.rs
+481
View File
@@ -0,0 +1,481 @@
mod metadata;
mod source;
mod subpackage;
use anyhow::{Context, bail};
use starlark::{
environment::{FrozenModule, Module},
values::{OwnedFrozenValue, UnpackValue, ValueLike, list::ListRef, typing::StarlarkCallable},
};
use std::{
collections::HashMap,
fmt,
path::{Path, PathBuf},
};
use walkdir::WalkDir;
use crate::{
eval::{self, ExtractError},
options::Options,
};
pub use metadata::Metadata;
pub use source::{GitSource, Source, TarballSource};
pub use subpackage::Subpackage;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RecipeKind {
Package,
HostPackage,
}
impl RecipeKind {
pub fn key(self, name: &str) -> String {
match self {
Self::Package => name.to_string(),
Self::HostPackage => format!("host:{name}"),
}
}
pub fn slug(self, name: &str) -> String {
self.key(name).replace(':', "-")
}
}
#[derive(Debug)]
pub enum Sources {
Single(source::Source),
Multiple(HashMap<String, source::Source>),
}
impl Sources {
pub fn entries(&self) -> Vec<(Option<&str>, &source::Source)> {
match self {
Self::Single(source) => vec![(None, source)],
Self::Multiple(sources) => {
let mut entries = sources
.iter()
.map(|(name, source)| (Some(name.as_str()), source))
.collect::<Vec<_>>();
entries.sort_by_key(|(name, _)| *name);
entries
}
}
}
}
pub struct Recipe {
/// Recipe name without namespace prefix.
name: String,
/// Path to the recipe's .star file.
path: PathBuf,
/// What kind of a recipe is that?
kind: RecipeKind,
/// Version shared by every package output of this recipe.
version: String,
/// Revision shared by every package output of this recipe.
revision: i32,
/// List of sources required to build this recipe.
sources: Sources,
/// All packages produced by this recipe.
/// This is empty for host recipes.
outputs: Vec<OutputPackage>,
/// Host packages requires for this recipe.
host_deps: Vec<String>,
/// Packages installed to the system root during build of the recipe, but
/// not listed as part of the `apk` dependencies.
build_deps: Vec<String>,
/// Packages installed to the system root during build of the recipe AND
/// listed as part of the `apk` dependencies.
deps: Vec<String>,
/// Packages NOT installed to the system root during build, only listed
/// as part of the `apk` dependencies.
run_deps: Vec<String>,
/// Starlark phase functions defined by the recipe.
phases: RecipePhases,
}
impl fmt::Debug for Recipe {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Recipe")
.field("path", &self.path)
.field("kind", &self.kind)
.field("version", &self.version)
.field("revision", &self.revision)
.field("sources", &self.sources)
.field("outputs", &self.outputs)
.field("host_deps", &self.host_deps)
.field("build_deps", &self.build_deps)
.field("deps", &self.deps)
.field("run_deps", &self.run_deps)
.field("phases", &self.phases)
.finish()
}
}
impl Recipe {
pub fn load(
path: &Path,
name: &str,
kind: RecipeKind,
options: &Options,
lib: Option<&FrozenModule>,
) -> anyhow::Result<Self> {
let module = eval::eval_file(path, Some(options), lib)
.with_context(|| format!("evaluating recipe {}", path.display()))?;
let version = eval::extract_string(&module, "version")
.map_err(|e| anyhow::anyhow!("field `version`: {e}"))?;
let revision = match eval::extract_i32(&module, "revision") {
Ok(v) => v,
Err(ExtractError::NotFound) => 1,
Err(e) => bail!("field `revision`: {e}"),
};
let metadata = match module.get("metadata") {
None => Metadata::new(None, None, None, None),
Some(value) => value
.downcast_ref::<Metadata>()
.ok_or_else(|| anyhow::anyhow!("field `metadata`: expected a metadata value"))?
.clone(),
};
let source_value = module
.get("source")
.ok_or_else(|| anyhow::anyhow!("field `source`: missing"))?;
let source = source_value
.downcast_ref::<Source>()
.ok_or_else(|| anyhow::anyhow!("field `source`: expected a source value"))?
.clone();
let sources = Sources::Single(source);
let host_deps = optional_string_list(&module, "host_deps")?;
let build_deps = optional_string_list(&module, "build_deps")?;
let deps = optional_string_list(&module, "deps")?;
let run_deps = optional_string_list(&module, "run_deps")?;
let recipe_key = kind.key(name);
let outputs = match kind {
RecipeKind::Package => {
let mut outputs = vec![OutputPackage {
recipe: recipe_key.clone(),
name: name.to_owned(),
metadata: metadata.clone(),
}];
if let Some(value) = module.get("subpackages") {
let list = ListRef::from_value(value)
.ok_or_else(|| anyhow::anyhow!("field `subpackages`: expected a list"))?;
for item in list.iter() {
let sub = item.downcast_ref::<Subpackage>().ok_or_else(|| {
anyhow::anyhow!(
"field `subpackages`: each entry must be a subpackage value"
)
})?;
outputs.push(OutputPackage {
recipe: recipe_key.clone(),
name: sub.name().to_owned(),
metadata: sub.metadata().clone(),
});
}
}
outputs
}
RecipeKind::HostPackage => {
if module.get("subpackages").is_some() {
bail!("host recipes cannot declare `subpackages`");
}
Vec::new()
}
};
let module = module
.freeze()
.map_err(|err| anyhow::anyhow!("freezing recipe module {}: {err:?}", path.display()))?;
let phases = RecipePhases::load(&module)?;
Ok(Recipe {
name: name.to_owned(),
path: path.to_path_buf(),
kind,
version,
revision,
sources,
outputs,
host_deps,
build_deps,
deps,
run_deps,
phases,
})
}
pub fn phases(&self) -> &RecipePhases {
&self.phases
}
pub fn key(&self) -> String {
self.kind.key(&self.name)
}
pub fn slug(&self) -> String {
self.kind.slug(&self.name)
}
pub fn dir(&self) -> &Path {
self.path.parent().unwrap_or_else(|| Path::new("."))
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn kind(&self) -> RecipeKind {
self.kind
}
pub fn version(&self) -> &str {
&self.version
}
pub fn revision(&self) -> i32 {
self.revision
}
pub fn sources(&self) -> &Sources {
&self.sources
}
pub fn outputs(&self) -> &[OutputPackage] {
&self.outputs
}
pub fn host_deps(&self) -> &[String] {
&self.host_deps
}
pub fn build_deps(&self) -> &[String] {
&self.build_deps
}
pub fn deps(&self) -> &[String] {
&self.deps
}
pub fn run_deps(&self) -> &[String] {
&self.run_deps
}
}
fn optional_string_list(module: &Module, key: &str) -> anyhow::Result<Vec<String>> {
match eval::extract_string_list(module, key) {
Ok(v) => Ok(v),
Err(ExtractError::NotFound) => Ok(Vec::new()),
Err(e) => Err(anyhow::anyhow!("field `{key}`: {e}")),
}
}
pub struct RecipePhases {
configure: Option<OwnedFrozenValue>,
build: OwnedFrozenValue,
install: OwnedFrozenValue,
}
impl fmt::Debug for RecipePhases {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RecipePhases")
.field("configure", &self.configure.is_some())
.field("build", &true)
.field("install", &true)
.finish()
}
}
impl RecipePhases {
fn load(module: &FrozenModule) -> anyhow::Result<Self> {
Ok(Self {
configure: optional_phase_function(module, "configure")?,
build: required_phase_function(module, "build")?,
install: required_phase_function(module, "install")?,
})
}
pub fn configure(&self) -> Option<&OwnedFrozenValue> {
self.configure.as_ref()
}
pub fn build(&self) -> &OwnedFrozenValue {
&self.build
}
pub fn install(&self) -> &OwnedFrozenValue {
&self.install
}
}
fn optional_phase_function(
module: &FrozenModule,
name: &str,
) -> anyhow::Result<Option<OwnedFrozenValue>> {
let Some(value) = module
.get_option(name)
.with_context(|| format!("field `{name}`"))?
else {
return Ok(None);
};
validate_phase_function(name, &value)?;
Ok(Some(value))
}
fn required_phase_function(module: &FrozenModule, name: &str) -> anyhow::Result<OwnedFrozenValue> {
let value = module
.get_option(name)
.with_context(|| format!("field `{name}`"))?
.ok_or_else(|| anyhow::anyhow!("field `{name}`: missing"))?;
validate_phase_function(name, &value)?;
Ok(value)
}
fn validate_phase_function(name: &str, value: &OwnedFrozenValue) -> anyhow::Result<()> {
let callable: Option<StarlarkCallable<'_>> = StarlarkCallable::unpack_value_opt(value.value());
if callable.is_none() {
bail!("field `{name}`: expected a callable value");
}
Ok(())
}
#[derive(Clone, Debug)]
pub struct OutputPackage {
/// Canonical key of the owning recipe.
recipe: String,
/// Name of the output package.
name: String,
/// Metadata attached to the output package.
metadata: Metadata,
}
impl OutputPackage {
pub fn key(&self) -> String {
self.name.clone()
}
pub fn recipe(&self) -> &str {
&self.recipe
}
pub fn name(&self) -> &str {
&self.name
}
}
#[derive(Debug)]
pub struct RecipeSet {
recipes: HashMap<String, Recipe>,
outputs: HashMap<String, OutputPackage>,
}
impl RecipeSet {
pub fn load(
root_path: &Path,
options: &Options,
lib: Option<&FrozenModule>,
) -> anyhow::Result<Self> {
let mut recipes = HashMap::new();
let mut outputs = HashMap::new();
for (path, kind) in [
("recipes", RecipeKind::Package),
("host-recipes", RecipeKind::HostPackage),
] {
let recipes_dir = root_path.join(path);
if !recipes_dir.exists() {
continue;
}
for (name, path) in discover_recipes(&recipes_dir)? {
let recipe = Recipe::load(&path, &name, kind, options, lib)
.with_context(|| format!("loading recipe `{name}`"))?;
let key = kind.key(&name);
if recipes.insert(key.clone(), recipe).is_some() {
bail!("duplicate recipe `{key}`");
}
}
}
for recipe in recipes.values() {
for output in &recipe.outputs {
let key = recipe.kind.key(&output.name);
if outputs.insert(key.clone(), output.clone()).is_some() {
bail!("duplicate package output `{key}`");
}
}
}
Ok(Self { recipes, outputs })
}
pub fn recipe(&self, key: &str) -> anyhow::Result<&Recipe> {
self.recipes
.get(key)
.ok_or_else(|| anyhow::anyhow!("unknown recipe `{key}`"))
}
pub fn output(&self, key: &str) -> anyhow::Result<&OutputPackage> {
self.outputs
.get(key)
.ok_or_else(|| anyhow::anyhow!("unknown output package `{key}`"))
}
}
/// Find all recipe `.star` files under `dir`, returning a map of recipe name
/// to its `.star` file. Intermediate directories act as categories and are
/// not themselves recipes. Within any subtree, a recipe takes either form:
/// - `.../<name>.star` — name is the file stem
/// - `.../<name>/recipe.star` — name is the parent directory
fn discover_recipes(dir: &Path) -> anyhow::Result<HashMap<String, PathBuf>> {
let mut recipes: HashMap<String, PathBuf> = HashMap::new();
let walker = WalkDir::new(dir).follow_links(false);
for entry in walker {
let entry =
entry.with_context(|| format!("walking recipes directory {}", dir.display()))?;
if !entry.file_type().is_file() {
continue;
}
let path = entry.path();
let Some(file_name) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
let name = if file_name == "recipe.star" {
let Some(parent_name) = path
.parent()
.and_then(|p| p.file_name())
.and_then(|n| n.to_str())
else {
continue;
};
parent_name.to_owned()
} else if let Some(stem) = file_name.strip_suffix(".star") {
stem.to_owned()
} else {
continue;
};
if let Some(existing) = recipes.insert(name.clone(), path.to_path_buf()) {
bail!(
"recipe `{name}` is defined twice: {} and {}",
existing.display(),
recipes[&name].display(),
);
}
}
Ok(recipes)
}
src/recipe/source.rs
+111
View File
@@ -0,0 +1,111 @@
use allocative::Allocative;
use starlark::values::StarlarkValue;
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value};
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct TarballSource {
url: String,
sha256: String,
strip_components: u32,
}
impl std::fmt::Display for TarballSource {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "tarball_source")
}
}
starlark::starlark_simple_value!(TarballSource);
#[starlark_value(type = "tarball_source")]
impl<'v> StarlarkValue<'v> for TarballSource {}
impl TarballSource {
pub fn new(url: String, sha256: String, strip_components: u32) -> Self {
Self {
url,
sha256,
strip_components,
}
}
pub fn url(&self) -> &str {
&self.url
}
pub fn sha256(&self) -> &str {
&self.sha256
}
pub fn strip_components(&self) -> u32 {
self.strip_components
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct GitSource {
url: String,
commit: String,
}
impl std::fmt::Display for GitSource {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "git_source")
}
}
starlark::starlark_simple_value!(GitSource);
#[starlark_value(type = "git_source")]
impl<'v> StarlarkValue<'v> for GitSource {}
impl GitSource {
pub fn new(url: String, commit: String) -> Self {
Self { url, commit }
}
pub fn url(&self) -> &str {
&self.url
}
pub fn commit(&self) -> &str {
&self.commit
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub enum Source {
Tarball(TarballSource),
Git(GitSource),
}
impl std::fmt::Display for Source {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "source")
}
}
starlark::starlark_simple_value!(Source);
#[starlark_value(type = "source")]
impl<'v> StarlarkValue<'v> for Source {}
impl Source {
pub fn url(&self) -> &str {
match self {
Self::Tarball(source) => source.url(),
Self::Git(source) => source.url(),
}
}
pub fn cache_key(&self) -> &str {
match self {
Self::Tarball(source) => source.sha256(),
Self::Git(source) => source.commit(),
}
}
pub fn is_unknown_cache_key(&self) -> bool {
matches!(self.cache_key(), "?" | "???")
}
}
src/recipe/subpackage.rs
+36
View File
@@ -0,0 +1,36 @@
use allocative::Allocative;
use starlark::values::StarlarkValue;
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value};
use crate::recipe::Metadata;
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct Subpackage {
name: String,
metadata: Metadata,
}
impl Subpackage {
pub fn new(name: String, metadata: Metadata) -> Self {
Self { name, metadata }
}
pub fn name(&self) -> &str {
&self.name
}
pub fn metadata(&self) -> &Metadata {
&self.metadata
}
}
impl std::fmt::Display for Subpackage {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "subpackage")
}
}
starlark::starlark_simple_value!(Subpackage);
#[starlark_value(type = "subpackage")]
impl<'v> StarlarkValue<'v> for Subpackage {}
src/rewrite.rs
-39
View File
@@ -1,39 +0,0 @@
use anyhow::{Context, Result};
use std::fs;
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Rewrite {
pub field: String,
pub old: String,
pub new: String,
}
pub fn backup_path(path: &Path) -> PathBuf {
let mut backup = path.as_os_str().to_os_string();
backup.push(".bak");
PathBuf::from(backup)
}
pub fn rewrite_placeholders(path: &Path, rewrites: &[Rewrite]) -> Result<bool> {
if rewrites.is_empty() {
return Ok(false);
}
let original =
fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?;
let mut updated = original.clone();
for rewrite in rewrites {
let quoted_old = format!("{} = \"{}\"", rewrite.field, rewrite.old);
let quoted_new = format!("{} = \"{}\"", rewrite.field, rewrite.new);
updated = updated.replacen(&quoted_old, &quoted_new, 1);
let dict_old = format!("\"{}\": \"{}\"", rewrite.field, rewrite.old);
let dict_new = format!("\"{}\": \"{}\"", rewrite.field, rewrite.new);
updated = updated.replacen(&dict_old, &dict_new, 1);
}
if updated == original {
return Ok(false);
}
fs::write(backup_path(path), original)?;
fs::write(path, updated)?;
Ok(true)
}
src/source.rs
-73
View File
@@ -1,73 +0,0 @@
use crate::log;
use crate::recipe::Recipe;
use crate::rewrite::{Rewrite, rewrite_placeholders};
use anyhow::{Context, Result, bail};
use sha2::{Digest, Sha256};
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
pub fn fetch_sources(recipe: &Recipe, cache_dir: &Path) -> Result<Vec<PathBuf>> {
fs::create_dir_all(cache_dir)?;
let mut rewrites = Vec::new();
let mut paths = Vec::new();
for source in &recipe.sources {
let label = if source.name.is_empty() {
recipe.key()
} else {
format!("{}:{}", recipe.key(), source.name)
};
let cached = source.sha256 != "???" && cache_dir.join(&source.sha256).exists();
if cached {
log::skip("cached", &format!("{label} ({})", source.url));
paths.push(cache_dir.join(&source.sha256));
continue;
}
log::step("fetch", &format!("{label} <- {}", source.url));
let bytes = download(&source.url)?;
let actual = sha256_hex(&bytes);
if source.sha256 == "???" {
log::info("sha256", &format!("{label} = {actual}"));
rewrites.push(Rewrite {
field: "sha256".into(),
old: "???".into(),
new: actual.clone(),
});
} else if source.sha256 != actual {
bail!(
"checksum mismatch for {}: expected {}, got {}",
source.url,
source.sha256,
actual
);
}
let path = cache_dir.join(&actual);
if !path.exists() {
fs::write(&path, &bytes)
.with_context(|| format!("failed to write {}", path.display()))?;
}
paths.push(path);
}
rewrite_placeholders(&recipe.path, &rewrites)?;
Ok(paths)
}
fn download(url: &str) -> Result<Vec<u8>> {
if let Some(path) = url.strip_prefix("file://") {
return Ok(fs::read(path)?);
}
// Some mirrors reject requests without a User-Agent with HTTP 403, so set an explicit one.
let client = reqwest::blocking::Client::builder()
.user_agent(concat!("distro/", env!("CARGO_PKG_VERSION")))
.build()?;
let mut response = client.get(url).send()?.error_for_status()?;
let mut bytes = Vec::new();
response.read_to_end(&mut bytes)?;
Ok(bytes)
}
pub fn sha256_hex(bytes: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(bytes);
hex::encode(hasher.finalize())
}
src/starlark.rs
-513
View File
@@ -1,513 +0,0 @@
use crate::config::Config;
use allocative::{Allocative, Visitor, ident_key};
use anyhow::{Result, anyhow, bail};
use serde_json::Value as JsonValue;
use starlark::environment::{FrozenModule, Globals, Module};
use starlark::eval::{Evaluator, FileLoader};
use starlark::starlark_simple_value;
use starlark::syntax::{AstModule, Dialect};
use starlark::values::dict::AllocDict;
use starlark::values::{AnyLifetime, Heap, NoSerialize, ProvidesStaticType, StarlarkValue, Value};
use starlark_derive::starlark_value;
use std::collections::{BTreeMap, HashMap};
use std::fmt::{self, Display};
use std::mem;
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, ProvidesStaticType, NoSerialize)]
pub struct OptionsValue {
values: BTreeMap<String, JsonValue>,
}
impl OptionsValue {
fn new(values: BTreeMap<String, JsonValue>) -> Result<Self> {
for key in values.keys() {
validate_starlark_identifier(key)?;
}
Ok(Self { values })
}
}
impl Display for OptionsValue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "options")
}
}
starlark_simple_value!(OptionsValue);
impl Allocative for OptionsValue {
fn visit<'a, 'b: 'a>(&self, visitor: &'a mut Visitor<'b>) {
let mut visitor = visitor.enter_self(self);
visitor.visit_simple(
ident_key!(values),
mem::size_of::<(String, JsonValue)>() * self.values.len(),
);
visitor.exit();
}
}
#[starlark_value(type = "options")]
impl<'v> StarlarkValue<'v> for OptionsValue {
fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
self.values.get(attr).map(|value| heap.alloc(value))
}
}
#[derive(Debug, Clone, ProvidesStaticType, NoSerialize, Allocative)]
pub struct SettingsValue {
target_arch: String,
container_runtime: String,
container_image: String,
container_dockerfile: String,
options: BTreeMap<String, String>,
}
impl From<&Config> for SettingsValue {
fn from(config: &Config) -> Self {
Self {
target_arch: config.target_arch.clone(),
container_runtime: config.container_runtime.clone(),
container_image: config.container_image.clone(),
container_dockerfile: config.container_dockerfile.display().to_string(),
options: config
.options
.iter()
.map(|(key, value)| (key.clone(), option_value_to_string(value)))
.collect(),
}
}
}
impl Display for SettingsValue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "settings")
}
}
starlark_simple_value!(SettingsValue);
#[starlark_value(type = "settings")]
impl<'v> StarlarkValue<'v> for SettingsValue {
fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
match attr {
"target_arch" => Some(heap.alloc(self.target_arch.as_str())),
"container_runtime" => Some(heap.alloc(self.container_runtime.as_str())),
"container_image" => Some(heap.alloc(self.container_image.as_str())),
"container_dockerfile" => Some(heap.alloc(self.container_dockerfile.as_str())),
"options" => Some(heap.alloc(AllocDict(self.options.clone()))),
_ => None,
}
}
}
pub fn eval_file(
path: &Path,
settings: Option<&Config>,
repo_root: Option<&Path>,
) -> Result<Module> {
let content = std::fs::read_to_string(path)?;
eval_content(path, content, settings, repo_root, Globals::standard())
}
/// Path of the implicit helper library auto-loaded into every recipe.
pub const COMMON_LIB_MODULE: &str = "//lib:common.star";
const COMMON_LIB_RELATIVE: &str = "lib/common.star";
const OPTIONS_NAME: &str = "OPTIONS";
/// Names exported by `lib/common.star`, if the file exists. Empty otherwise.
pub fn common_lib_names(repo_root: &Path, settings: Option<&Config>) -> Result<Vec<String>> {
let path = repo_root.join(COMMON_LIB_RELATIVE);
if !path.exists() {
return Ok(Vec::new());
}
let module = eval_file(&path, settings, Some(repo_root))?;
Ok(module
.names()
.map(|n| n.as_str().to_owned())
.filter(|n| !n.starts_with('_') && n != "settings" && n != OPTIONS_NAME)
.collect())
}
/// Prepend an implicit `load("//lib:common.star", ...)` so every recipe sees
/// the shared helpers without an explicit import. Does nothing if there's no
/// `lib/common.star` or no repo root.
pub fn prepend_common_lib_load(
repo_root: Option<&Path>,
settings: Option<&Config>,
content: &str,
) -> Result<String> {
let Some(root) = repo_root else {
return Ok(content.to_owned());
};
let names = common_lib_names(root, settings)?;
if names.is_empty() {
return Ok(content.to_owned());
}
let names_lit = names
.iter()
.map(|n| format!("{:?}", n))
.collect::<Vec<_>>()
.join(", ");
Ok(format!(
"load(\"{COMMON_LIB_MODULE}\", {names_lit})\n{content}"
))
}
pub fn eval_content(
path: &Path,
content: String,
settings: Option<&Config>,
repo_root: Option<&Path>,
globals: Globals,
) -> Result<Module> {
eval_content_with_extra(path, content, settings, repo_root, globals, None)
}
pub fn eval_content_with_extra<'a>(
path: &Path,
content: String,
settings: Option<&Config>,
repo_root: Option<&Path>,
globals: Globals,
extra: Option<&'a dyn AnyLifetime<'a>>,
) -> Result<Module> {
let filename = path.display().to_string();
validate_options_source(path, &content)?;
let ast = AstModule::parse(
&filename,
content,
&Dialect {
enable_f_strings: true,
enable_top_level_stmt: true,
..Dialect::Standard
},
)
.map_err(|err| anyhow!("{err}"))?;
let loader = match repo_root {
Some(root) => Some(RepoFileLoader::new(root, settings, globals.clone(), &ast)?),
None if !ast.loads().is_empty() => {
bail!(
"load() requires a repo root while evaluating {}",
path.display()
)
}
None => None,
};
let module = Module::new();
let protected_options = if let Some(config) = settings {
Some(set_config_values(&module, config)?)
} else {
None
};
{
let mut eval = Evaluator::new(&module);
if let Some(loader) = &loader {
eval.set_loader(loader);
}
eval.extra = extra;
eval.eval_module(ast, &globals)
.map_err(|err| anyhow!("{err}"))?;
}
validate_options_binding(path, &module, protected_options)?;
Ok(module)
}
fn validate_options_source(path: &Path, content: &str) -> Result<()> {
let line_offset = implicit_common_load_line_offset(content);
for (index, line) in content.lines().enumerate() {
let code = line.split('#').next().unwrap_or_default().trim_start();
if defines_or_reassigns_options(code) {
let line = (index + 1).saturating_sub(line_offset).max(1);
bail!(
"{}:{} must not define or reassign `{OPTIONS_NAME}`",
path.display(),
line
);
}
}
Ok(())
}
fn implicit_common_load_line_offset(content: &str) -> usize {
content
.lines()
.next()
.is_some_and(|line| line.starts_with(&format!("load(\"{COMMON_LIB_MODULE}\",")))
as usize
}
fn defines_or_reassigns_options(code: &str) -> bool {
let Some(rest) = code.strip_prefix(OPTIONS_NAME) else {
return defines_options_function(code) || binds_options_loop_variable(code);
};
let rest = rest.trim_start();
rest.starts_with('=')
|| rest.starts_with("+=")
|| rest.starts_with("-=")
|| rest.starts_with("*=")
|| rest.starts_with("/=")
|| rest.starts_with("%=")
|| rest.starts_with("&=")
|| rest.starts_with("|=")
|| rest.starts_with("^=")
}
fn defines_options_function(code: &str) -> bool {
code.strip_prefix("def ")
.and_then(|rest| rest.trim_start().strip_prefix(OPTIONS_NAME))
.is_some_and(|rest| rest.trim_start().starts_with('('))
}
fn binds_options_loop_variable(code: &str) -> bool {
code.strip_prefix("for ")
.and_then(|rest| rest.trim_start().strip_prefix(OPTIONS_NAME))
.is_some_and(|rest| rest.trim_start().starts_with("in "))
}
fn validate_options_binding(
path: &Path,
module: &Module,
expected: Option<Value<'_>>,
) -> Result<()> {
let Some(expected) = expected else {
return Ok(());
};
let actual = module.get(OPTIONS_NAME).ok_or_else(|| {
anyhow!(
"{} removed the protected `{OPTIONS_NAME}` binding",
path.display()
)
})?;
if !actual.ptr_eq(expected) {
bail!(
"{} must not define or reassign `{OPTIONS_NAME}`",
path.display()
);
}
Ok(())
}
#[derive(Clone)]
struct RepoFileLoader {
modules: HashMap<String, FrozenModule>,
}
impl RepoFileLoader {
fn new(
repo_root: &Path,
settings: Option<&Config>,
globals: Globals,
ast: &AstModule,
) -> Result<Self> {
let mut modules = HashMap::new();
for load in ast.loads() {
load_module(
repo_root,
settings,
globals.clone(),
load.module_id,
&mut modules,
)?;
}
Ok(Self { modules })
}
}
impl FileLoader for RepoFileLoader {
fn load(&self, path: &str) -> starlark::Result<FrozenModule> {
self.modules
.get(path)
.cloned()
.ok_or_else(|| starlark::Error::new_other(anyhow!("unknown Starlark module `{path}`")))
}
}
fn load_module(
repo_root: &Path,
settings: Option<&Config>,
globals: Globals,
module_id: &str,
modules: &mut HashMap<String, FrozenModule>,
) -> Result<()> {
if modules.contains_key(module_id) {
return Ok(());
}
let path = resolve_load_path(repo_root, module_id)?;
let content = std::fs::read_to_string(&path)?;
let filename = path.display().to_string();
validate_options_source(&path, &content)?;
let ast =
AstModule::parse(&filename, content, &Dialect::Standard).map_err(|err| anyhow!("{err}"))?;
for load in ast.loads() {
load_module(
repo_root,
settings,
globals.clone(),
load.module_id,
modules,
)?;
}
let nested_loader = RepoFileLoader {
modules: modules.clone(),
};
let module = Module::new();
let protected_options = if let Some(config) = settings {
Some(set_config_values(&module, config)?)
} else {
None
};
{
let mut eval = Evaluator::new(&module);
eval.set_loader(&nested_loader);
eval.eval_module(ast, &globals)
.map_err(|err| anyhow!("{err}"))?;
}
validate_options_binding(&path, &module, protected_options)?;
let frozen = module.freeze().map_err(|err| anyhow!("{err:?}"))?;
modules.insert(module_id.to_owned(), frozen);
Ok(())
}
fn set_config_values<'v>(module: &'v Module, config: &Config) -> Result<Value<'v>> {
module.set("settings", module.heap().alloc(SettingsValue::from(config)));
let options = module
.heap()
.alloc(OptionsValue::new(config.options.clone())?);
module.set(OPTIONS_NAME, options);
Ok(options)
}
fn validate_starlark_identifier(name: &str) -> Result<()> {
let mut chars = name.chars();
let Some(first) = chars.next() else {
bail!("config option name cannot be empty");
};
if !(first == '_' || first.is_ascii_alphabetic()) {
bail!("config option `{name}` is not a valid Starlark identifier");
}
if chars.any(|ch| !(ch == '_' || ch.is_ascii_alphanumeric())) {
bail!("config option `{name}` is not a valid Starlark identifier");
}
if matches!(
name,
"and"
| "as"
| "assert"
| "break"
| "class"
| "continue"
| "def"
| "del"
| "elif"
| "else"
| "except"
| "finally"
| "for"
| "from"
| "global"
| "if"
| "import"
| "in"
| "is"
| "lambda"
| "load"
| "not"
| "or"
| "pass"
| "return"
| "try"
| "while"
| "with"
| "yield"
) {
bail!("config option `{name}` is a reserved Starlark keyword");
}
Ok(())
}
fn resolve_load_path(repo_root: &Path, module_id: &str) -> Result<PathBuf> {
let relative = if let Some(stripped) = module_id.strip_prefix("//") {
stripped.replace(':', "/")
} else {
module_id.to_owned()
};
let path = repo_root.join(relative);
let canonical_root = repo_root
.canonicalize()
.unwrap_or_else(|_| repo_root.to_path_buf());
let canonical_path = path.canonicalize().unwrap_or(path);
if !canonical_path.starts_with(&canonical_root) {
bail!("Starlark load escapes repo root: {module_id}");
}
Ok(canonical_path)
}
fn option_value_to_string(value: &JsonValue) -> String {
match value {
JsonValue::String(value) => value.clone(),
JsonValue::Bool(value) => value.to_string(),
JsonValue::Number(value) => value.to_string(),
JsonValue::Null => "null".to_owned(),
JsonValue::Array(_) | JsonValue::Object(_) => value.to_string(),
}
}
pub fn get_string(module: &Module, name: &str) -> Result<String> {
module
.get(name)
.and_then(|v| v.unpack_str().map(ToOwned::to_owned))
.ok_or_else(|| anyhow!("missing or non-string Starlark variable `{name}`"))
}
pub fn get_string_default(module: &Module, name: &str, default: &str) -> Result<String> {
Ok(match module.get(name) {
Some(value) => value
.unpack_str()
.ok_or_else(|| anyhow!("non-string Starlark variable `{name}`"))?
.to_owned(),
None => default.to_owned(),
})
}
pub fn get_i32_default(module: &Module, name: &str, default: i32) -> Result<i32> {
Ok(match module.get(name) {
Some(value) => value
.unpack_i32()
.ok_or_else(|| anyhow!("non-integer Starlark variable `{name}`"))?,
None => default,
})
}
pub fn has_name(module: &Module, name: &str) -> bool {
module.get(name).is_some()
}
pub fn get_json(module: &Module, name: &str) -> Result<Option<JsonValue>> {
match module.get(name) {
Some(value) => Ok(Some(serde_json::from_str(&value.to_json()?)?)),
None => Ok(None),
}
}
pub fn get_string_vec(module: &Module, name: &str) -> Result<Vec<String>> {
match get_json(module, name)? {
Some(JsonValue::Array(values)) => values
.into_iter()
.map(|value| match value {
JsonValue::String(s) => Ok(s),
_ => bail!("`{name}` must be a list of strings"),
})
.collect(),
Some(_) => bail!("`{name}` must be a list of strings"),
None => Ok(Vec::new()),
}
}
pub fn get_json_map(module: &Module, name: &str) -> Result<BTreeMap<String, JsonValue>> {
match get_json(module, name)? {
Some(JsonValue::Object(values)) => Ok(values.into_iter().collect()),
Some(_) => bail!("`{name}` must be a dict"),
None => Ok(BTreeMap::new()),
}
}
src/update.rs
-256
View File
@@ -1,256 +0,0 @@
//! `distro update` — check repology for newer upstream versions and bump
//! recipes in-place.
//!
//! For each requested recipe we query
//! `https://repology.org/api/v1/project/<name>` and pick the highest version
//! among entries whose `status` is `"newest"` or `"unique"` (i.e. not
//! outdated, not rolling/devel, not ignored). If that version is strictly
//! greater than the recipe's current `version`, we rewrite:
//!
//! * `version = "..."` → the new upstream version
//! * `revision = N` → `revision = 1`
//! * every `"sha256": "..."` → `"sha256": "???"` so the next fetch fills it
//!
//! Repology asks API clients to set a descriptive User-Agent.
use crate::log;
use crate::recipe::{Recipe, RecipeSet};
use anyhow::{Context, Result, bail};
use serde::Deserialize;
use std::cmp::Ordering;
use std::fs;
use std::path::Path;
const REPOLOGY_API: &str = "https://repology.org/api/v1/project";
#[derive(Debug, Deserialize)]
struct RepologyEntry {
#[serde(default)]
version: String,
#[serde(default)]
status: String,
}
pub fn run(recipes: &RecipeSet, names: &[String], bump: bool) -> Result<()> {
let targets: Vec<&Recipe> = if names.is_empty() {
recipes.recipes.values().collect()
} else {
let mut out = Vec::with_capacity(names.len());
for name in names {
out.push(recipes.recipe_by_user_ref(name)?);
}
out
};
let client = reqwest::blocking::Client::builder()
.user_agent(concat!(
"distro/",
env!("CARGO_PKG_VERSION"),
" (+repology version checker)"
))
.build()?;
let mut outdated = 0usize;
let mut bumped = 0usize;
let mut up_to_date = 0usize;
let mut errored = 0usize;
for recipe in targets {
match check_one(&client, recipe) {
Ok(Some(new_version)) => {
if bump {
bump_recipe(&recipe.path, &new_version)?;
log::step(
"bump",
&format!("{}: {} -> {}", recipe.key(), recipe.version, new_version),
);
bumped += 1;
} else {
log::step(
"outdated",
&format!("{}: {} -> {}", recipe.key(), recipe.version, new_version),
);
outdated += 1;
}
}
Ok(None) => {
log::skip(
"up-to-date",
&format!("{} {}", recipe.key(), recipe.version),
);
up_to_date += 1;
}
Err(e) => {
log::info("error", &format!("{}: {e}", recipe.key()));
errored += 1;
}
}
}
if bump {
log::step(
"summary",
&format!("{bumped} bumped, {up_to_date} up-to-date, {errored} errored"),
);
} else {
log::step(
"summary",
&format!(
"{outdated} outdated, {up_to_date} up-to-date, {errored} errored (re-run with --bump to rewrite recipes)"
),
);
}
Ok(())
}
fn check_one(client: &reqwest::blocking::Client, recipe: &Recipe) -> Result<Option<String>> {
let url = format!("{REPOLOGY_API}/{}", recipe.name);
let resp = client
.get(&url)
.send()
.with_context(|| format!("GET {url}"))?;
if resp.status() == reqwest::StatusCode::NOT_FOUND {
bail!("repology has no project named `{}`", recipe.name);
}
let entries: Vec<RepologyEntry> = serde_json::from_slice(&resp.error_for_status()?.bytes()?)
.context("failed to parse repology response")?;
let latest = entries
.iter()
.filter(|e| matches!(e.status.as_str(), "newest" | "unique"))
.map(|e| e.version.as_str())
.max_by(|a, b| natural_cmp(a, b));
match latest {
Some(v) if natural_cmp(v, &recipe.version) == Ordering::Greater => Ok(Some(v.to_owned())),
_ => Ok(None),
}
}
fn bump_recipe(path: &Path, new_version: &str) -> Result<()> {
let original =
fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?;
let mut out = String::with_capacity(original.len());
let mut version_replaced = false;
let mut revision_replaced = false;
for line in original.split_inclusive('\n') {
let trimmed = line.trim_start();
if !version_replaced && trimmed.starts_with("version") {
if let Some(replaced) = replace_string_assignment(line, "version", new_version) {
out.push_str(&replaced);
version_replaced = true;
continue;
}
}
if !revision_replaced && trimmed.starts_with("revision") {
if let Some(replaced) = replace_int_assignment(line, "revision", 1) {
out.push_str(&replaced);
revision_replaced = true;
continue;
}
}
out.push_str(line);
}
if !version_replaced {
bail!("could not find `version = \"...\"` in {}", path.display());
}
// Reset every sha256 placeholder so the next fetch re-derives it.
out = out.replace("\"sha256\": \"", "\x00sha256_marker\x00\"");
out = regex_lite_replace_sha(&out);
out = out.replace("\x00sha256_marker\x00\"", "\"sha256\": \"");
fs::write(path, out).with_context(|| format!("failed to write {}", path.display()))?;
Ok(())
}
/// Replace every quoted sha256 value with `"???"`, leaving keys/whitespace
/// alone. Avoids a full regex dep by walking the string by hand.
fn regex_lite_replace_sha(input: &str) -> String {
let mut out = String::with_capacity(input.len());
let bytes = input.as_bytes();
let needle = b"\x00sha256_marker\x00\"";
let mut i = 0;
while i < bytes.len() {
if bytes[i..].starts_with(needle) {
out.push_str("\x00sha256_marker\x00\"???\"");
i += needle.len();
// skip the original value up to the closing quote
while i < bytes.len() && bytes[i] != b'"' {
i += 1;
}
if i < bytes.len() {
i += 1; // consume the closing quote we replaced
}
} else {
out.push(bytes[i] as char);
i += 1;
}
}
out
}
fn replace_string_assignment(line: &str, key: &str, new_value: &str) -> Option<String> {
// Match `<key><space>=<space>"value"` while preserving surrounding text
// (indentation, trailing newline).
let stripped = line.strip_prefix(key)?;
let rest = stripped.trim_start_matches(|c: char| c == ' ' || c == '\t');
let after_eq = rest.strip_prefix('=')?.trim_start();
let after_quote = after_eq.strip_prefix('"')?;
let end = after_quote.find('"')?;
let trailing = &after_quote[end + 1..];
Some(format!("{key} = \"{new_value}\"{trailing}"))
}
fn replace_int_assignment(line: &str, key: &str, new_value: i32) -> Option<String> {
let stripped = line.strip_prefix(key)?;
let rest = stripped.trim_start_matches(|c: char| c == ' ' || c == '\t');
let after_eq = rest.strip_prefix('=')?.trim_start();
let end = after_eq.find(|c: char| !c.is_ascii_digit())?;
let trailing = &after_eq[end..];
Some(format!("{key} = {new_value}{trailing}"))
}
/// dpkg-ish natural comparison: split into runs of digits and non-digits and
/// compare numerically where both sides are digits, lexicographically
/// otherwise. Good enough for upstream tarball versions.
fn natural_cmp(a: &str, b: &str) -> Ordering {
let mut ai = a.chars().peekable();
let mut bi = b.chars().peekable();
loop {
match (ai.peek().copied(), bi.peek().copied()) {
(None, None) => return Ordering::Equal,
(None, _) => return Ordering::Less,
(_, None) => return Ordering::Greater,
(Some(x), Some(y)) if x.is_ascii_digit() && y.is_ascii_digit() => {
let mut na = String::new();
while let Some(&c) = ai.peek() {
if c.is_ascii_digit() {
na.push(c);
ai.next();
} else {
break;
}
}
let mut nb = String::new();
while let Some(&c) = bi.peek() {
if c.is_ascii_digit() {
nb.push(c);
bi.next();
} else {
break;
}
}
let xa: u64 = na.parse().unwrap_or(0);
let xb: u64 = nb.parse().unwrap_or(0);
match xa.cmp(&xb) {
Ordering::Equal => continue,
other => return other,
}
}
(Some(x), Some(y)) => match x.cmp(&y) {
Ordering::Equal => {
ai.next();
bi.next();
}
other => return other,
},
}
}
}