diff --git a/Dockerfile b/Dockerfile index e0eaf5c..762d2c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ -FROM docker.io/library/alpine:edge +FROM alpine:3.22.4 -RUN apk upgrade --no-cache && \ - apk add --no-cache \ +RUN apk upgrade --no-cache && apk add --no-cache \ alpine-sdk \ apk-tools \ autoconf \ @@ -38,21 +37,4 @@ RUN apk upgrade --no-cache && \ xz \ zstd -RUN rm -rf /tmp/mkpkg-root /tmp/distro-preflight.apk /tmp/APKINDEX.adb /tmp/distro-preflight.rsa && \ - openssl genrsa -out /tmp/distro-preflight.rsa 2048 >/dev/null 2>&1 && \ - openssl rsa -in /tmp/distro-preflight.rsa -pubout -out /etc/apk/keys/distro-preflight.rsa.pub >/dev/null 2>&1 && \ - mkdir -p /tmp/mkpkg-root/usr/share/distro && \ - printf ok > /tmp/mkpkg-root/usr/share/distro/preflight && \ - apk --sign-key /tmp/distro-preflight.rsa mkpkg \ - --files /tmp/mkpkg-root \ - --output /tmp/distro-preflight.apk \ - --info name:distro-preflight \ - --info version:0-r0 \ - --info arch:noarch \ - --info description:preflight \ - --info license:MIT >/dev/null && \ - apk --sign-key /tmp/distro-preflight.rsa mkndx -o /tmp/APKINDEX.adb /tmp/distro-preflight.apk >/dev/null && \ - test -s /tmp/distro-preflight.apk && \ - test -s /tmp/APKINDEX.adb - WORKDIR /work diff --git a/README.md b/README.md deleted file mode 100644 index 2607570..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# distro - diff --git a/config.star b/config.star index b7594f0..de8492a 100644 --- a/config.star +++ b/config.star @@ -1,11 +1,8 @@ container_runtime = "podman" -container_image = "localhost/distro-builder:latest" +container_image = "local/distro-builder:latest" container_dockerfile = "Dockerfile" -signing_key = "build/keys/distro.rsa" -signing_pubkey = "build/keys/distro.rsa.pub" - -target_arch = "x86_64" +arch = "x86_64" libc = "musl" host_cflags = "-O2 -pipe" @@ -16,21 +13,24 @@ target_cflags = host_cflags target_cxxflags = host_cxxflags target_ldflags = host_ldflags + " -Wl,-z,now" -if target_arch == "x86_64": +if arch == "x86_64": flags = " -march=x86-64-v3 -mtune=generic -fstack-clash-protection -fstack-protector-strong -fcf-protection" + target_cflags += flags target_cxxflags += flags target_ldflags += " -Wl,-z,pack-relative-relocs" -options = { - "libc": libc, - "target_triple": target_arch + "-linux-" + libc, - "host_cflags": host_cflags, - "host_cxxflags": host_cxxflags, - "host_ldflags": host_ldflags, - "cflags": target_cflags, - "cxxflags": target_cxxflags, - "ldflags": target_ldflags, - "wayland": True, - "x11": True, -} +options = dict( + target_arch = arch, + target_triple = f"{arch}-linux-{libc}", + + host_cflags = host_cflags, + host_cxxflags = host_cxxflags, + host_ldflags = host_ldflags, + + cflags = target_cflags, + cxxflags = target_cxxflags, + ldflags = target_ldflags, + + libc = libc, +) diff --git a/host-recipes/binutils.star b/host-recipes/binutils.star new file mode 100644 index 0000000..299ad87 --- /dev/null +++ b/host-recipes/binutils.star @@ -0,0 +1,39 @@ +version = "2.46.0" +revision = 1 +metadata = meta( + description = "GNU binutils cross-compiled for the target triple", + license = "GPL-3.0-or-later", +) +source = tarball_source( + url = "https://ftp.gnu.org/gnu/binutils/binutils-" + version + ".tar.xz", + sha256 = "?", + strip_components = 1, +) + +def configure(ctx): + ctx.run([ + ctx.source_dir / "configure", + "--prefix=" + ctx.prefix, + "--target=" + options.target_triple, + "--with-sysroot=" + ctx.sysroot, + "--with-pic", + "--enable-cet", + "--enable-default-execstack=no", + "--enable-deterministic-archives", + "--enable-ld=default", + "--enable-new-dtags", + "--enable-plugins", + "--enable-relro", + "--enable-separate-code", + "--enable-threads", + # gprofng's libcollector does not build against musl. + "--disable-gprofng", + "--disable-nls", + "--disable-werror", + ], env = { + "CFLAGS": options.host_cflags, + "CXXFLAGS": options.host_cxxflags, + "LDFLAGS": options.host_ldflags, + }) + +_, build, install = autotools() diff --git a/host-recipes/binutils/recipe.star b/host-recipes/binutils/recipe.star deleted file mode 100644 index 4775011..0000000 --- a/host-recipes/binutils/recipe.star +++ /dev/null @@ -1,42 +0,0 @@ -name = "binutils" -version = "2.46.0" -revision = 1 -description = "GNU binutils cross-compiled for the target triple" -license = "GPL-3.0-or-later" - -source = { - "url": "https://ftp.gnu.org/gnu/binutils/binutils-" + version + ".tar.xz", - "sha256": "d75a94f4d73e7a4086f7513e67e439e8fcdcbb726ffe63f4661744e6256b2cf2", - "strip_components": 1, -} - -host_deps = [] - -def configure(ctx): - ctx.run([ - ctx.source_dir + "/configure", - "--prefix=" + ctx.prefix, - "--target=" + OPTIONS.target_triple, - "--with-sysroot=" + ctx.prefix + "/" + OPTIONS.target_triple, - "--disable-nls", - "--disable-werror", - "--enable-deterministic-archives", - "--enable-ld=default", - "--enable-plugins", - "--enable-threads", - "--with-system-zlib", - # gprofng's libcollector does not build against musl/recent gcc. - "--disable-gprofng", - ], env = { - "CFLAGS": OPTIONS.host_cflags, - "CXXFLAGS": OPTIONS.host_cxxflags, - "LDFLAGS": OPTIONS.host_ldflags, - }) - -def build(ctx): - ctx.run(["make", "-j" + str(ctx.jobs)]) - -def install(ctx, pkg): - ctx.run(["make", "DESTDIR=" + pkg.destdir, "install"]) - # Drop static archives we don't need on the cross side. - ctx.run(["sh", "-c", "rm -f " + pkg.destdir + ctx.prefix + "/lib/*.a"]) diff --git a/host-recipes/gcc/recipe.star b/host-recipes/gcc.star.old similarity index 100% rename from host-recipes/gcc/recipe.star rename to host-recipes/gcc.star.old diff --git a/lib/common.star b/lib/common.star index 86b94d4..b7f42bb 100644 --- a/lib/common.star +++ b/lib/common.star @@ -1,50 +1,31 @@ -# Commonly used helpers, auto-loaded into every recipe. - -def _toolchain_env(ctx): - sysroot_flag = " --sysroot=" + ctx.sysroot - return { - "CFLAGS": OPTIONS.cflags + sysroot_flag, - "CXXFLAGS": OPTIONS.cxxflags + sysroot_flag, - "LDFLAGS": OPTIONS.ldflags + sysroot_flag, - } - -# Autotools +# Commonly used helpers. def autotools_configure(ctx, extra_args = [], extra_env = {}): - args = [ - ctx.source_dir + "/configure", + env = { + "CFLAGS": options.cflags, + "CXXFLAGS": options.cxxflags, + "LDFLAGS": options.ldflags, + } + env.update(extra_env) + ctx.run([ + ctx.source_dir / "configure", + "--host=" + options.target_triple, + "--with-sysroot=" + ctx.sysroot, "--prefix=" + ctx.prefix, "--sysconfdir=/etc", "--localstatedir=/var", "--bindir=" + ctx.prefix + "/bin", "--sbindir=" + ctx.prefix + "/bin", "--libdir=" + ctx.prefix + "/lib", - "--with-sysroot=" + ctx.sysroot, "--disable-static", "--enable-shared", - ] - args.append("--host=" + OPTIONS.target_triple) - args.extend(extra_args) - - envs = _toolchain_env(ctx) - envs.update(extra_env) - - ctx.run(args, env = envs) + ] + extra_args, env = env) def autotools_build(ctx, extra_args = []): - args = ["make", "-j" + str(ctx.jobs)] - args.extend(extra_args) - ctx.run(args) - -def autotools_check(ctx, extra_args = []): - args = ["make", "check", "-j" + str(ctx.jobs)] - args.extend(extra_args) - ctx.run(args) + ctx.run(["make", "-j" + str(ctx.jobs)] + extra_args) def autotools_install(ctx, pkg, extra_args = []): - args = ["make", "install", "DESTDIR=" + pkg.destdir] - args.extend(extra_args) - ctx.run(args) + ctx.run(["make", "install"] + extra_args, env = {"DESTDIR": pkg.destdir}) def autotools(configure_args = [], configure_env = [], build_args = [], install_args = []): def _configure(ctx): @@ -54,46 +35,3 @@ def autotools(configure_args = [], configure_env = [], build_args = [], install_ def _install(ctx, pkg): autotools_install(ctx, pkg, extra_args = install_args) return _configure, _build, _install - -# Meson - -def meson_configure(ctx, extra_args = []): - args = [ - "meson", - "setup", - ctx.build_dir, - ctx.source_dir, - "--prefix=" + ctx.prefix, - ] - args.extend(extra_args) - ctx.run(args, env = _toolchain_env(ctx)) - -def meson_build(ctx): - ctx.run(["meson", "compile", "-C", ctx.build_dir, "-j" + str(ctx.jobs)]) - -def meson_install(ctx, pkg): - ctx.run(["meson", "install", "-C", ctx.build_dir, "--destdir", pkg.destdir]) - -def meson(configure_args = [], build_args = [], install_args = []): - def _configure(ctx): - meson_configure(ctx, extra_args = configure_args) - def _build(ctx): - meson_build(ctx, extra_args = build_args) - def _install(ctx, pkg): - meson_install(ctx, pkg, extra_args = install_args) - return _configure, _build, _install - -# Make - -def make(ctx, target = None, extra_args = []): - args = ["make", "-C", ctx.source_dir, "O=" + ctx.build_dir, - "-j" + str(ctx.jobs)] - args.extend(extra_args) - if target: - args.append(target) - ctx.run(args) - -def make_install(ctx, pkg, extra_args = []): - args = ["make", "-C", ctx.build_dir, "DESTDIR=" + pkg.destdir, "install"] - args.extend(extra_args) - ctx.run(args) diff --git a/recipes/limine.star b/recipes/limine.star new file mode 100644 index 0000000..d769e0a --- /dev/null +++ b/recipes/limine.star @@ -0,0 +1,25 @@ +version = "12.2.0" +revision = 1 +metadata = meta( + description = "Modern, secure, portable, multiprotocol bootloader and boot manager", + license = "BSD-2-Clause", +) +source = tarball_source( + url = f"https://github.com/Limine-Bootloader/Limine/releases/download/v{version}/limine-{version}.tar.gz", + sha256 = "db8a119878cfeead63c0a78236c577c40539c5759496950ea0ed32a6cf567865", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] +deps = [options.libc] +subpackages = [ + subpackage( + name = "limine-bios", + ), +] + +configure, build, install = autotools(configure_env = { + "TOOLCHAIN_FOR_TARGET": options.target_triple + "-", + "LD_FOR_TARGET": options.target_triple + "-" + "ld", + "OBJCOPY_FOR_TARGET": options.target_triple + "-" + "objcopy", + "OBJDUMP_FOR_TARGET": options.target_triple + "-" + "objdump", +}) diff --git a/recipes/limine/recipe.star b/recipes/limine/recipe.star deleted file mode 100644 index 41c40e7..0000000 --- a/recipes/limine/recipe.star +++ /dev/null @@ -1,25 +0,0 @@ -name = "limine" -version = "12.2.0" -revision = 1 -description = "Modern, secure, portable, multiprotocol bootloader and boot manager" -license = "BSD-2-Clause" - -source = { - "url": f"https://github.com/Limine-Bootloader/Limine/releases/download/v{version}/limine-{version}.tar.gz", - "sha256": "db8a119878cfeead63c0a78236c577c40539c5759496950ea0ed32a6cf567865", - "strip_components": 1, -} - -host_deps = ["binutils", "gcc"] -deps = [OPTIONS.libc] - -def configure(ctx): - toolchain = OPTIONS.target_triple + "-" - autotools_configure(ctx, extra_env = { - "TOOLCHAIN_FOR_TARGET": toolchain, - "LD_FOR_TARGET": toolchain + "ld", - "OBJCOPY_FOR_TARGET": toolchain + "objcopy", - "OBJDUMP_FOR_TARGET": toolchain + "objdump", - }) - -_, build, install = autotools() diff --git a/recipes/linux-headers.star b/recipes/linux-headers.star new file mode 100644 index 0000000..c09610d --- /dev/null +++ b/recipes/linux-headers.star @@ -0,0 +1,20 @@ +version = "7.0.9" +revision = 1 +metadata = meta( + description = "Linux kernel headers for userspace development", + license = "GPL-2.0-only", +) +source = tarball_source( + url = f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz", + sha256 = "ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4", + strip_components = 1, +) + +def build(ctx): + ctx.run(["cp", "-rp", ctx.source_dir / ".", ctx.build_dir]) + ctx.run(["make", "headers_install", "ARCH=" + options.target_arch]) + ctx.run(["find", ctx.build_dir / "usr/include", "-type", "f", "!", "-name", "*.h", "-delete"]) + +def install(ctx, pkg): + ctx.run(["mkdir", "-p", pkg.dest_dir / ctx.prefix]) + ctx.run(["cp", "-rp", ctx.build_dir / "usr/include", pkg.dest_dir / ctx.prefix]) diff --git a/recipes/linux/recipe.star b/recipes/linux.star.old similarity index 100% rename from recipes/linux/recipe.star rename to recipes/linux.star.old diff --git a/recipes/musl.star b/recipes/musl.star new file mode 100644 index 0000000..8a5d0b0 --- /dev/null +++ b/recipes/musl.star @@ -0,0 +1,26 @@ +version = "1.2.6" +revision = 1 +metadata = meta( + description = "Small, standards-conformant implementation of libc", + license = "MIT", +) +source = tarball_source( + url = f"https://musl.libc.org/releases/musl-{version}.tar.gz", + sha256 = "?", + strip_components = 1, +) +host_deps = ["binutils", "gcc-bootstrap"] + +def configure(ctx): + ctx.run([ + ctx.source_dir / "configure", + "--target=" + options.target_triple, + "--prefix=" + ctx.prefix, + "--syslibdir=/lib", + ], env = { + "CC": options.target_triple + "-gcc", + "CFLAGS": options.cflags, + "LDFLAGS": options.ldflags, + }) + +_, build, install = autotools() diff --git a/recipes/musl/recipe.star b/recipes/musl/recipe.star deleted file mode 100644 index e013188..0000000 --- a/recipes/musl/recipe.star +++ /dev/null @@ -1,31 +0,0 @@ -name = "musl" -version = "1.2.6" -revision = 1 -description = "Musl C library" -license = "MIT" - -source = { - "url": f"https://musl.libc.org/releases/musl-{version}.tar.gz", - "sha256": "d585fd3b613c66151fc3249e8ed44f77020cb5e6c1e635a616d3f9f82460512a", - "strip_components": 1, -} - -host_deps = ["binutils", "gcc"] - -def configure(ctx): - ctx.run( - [ - ctx.source_dir + "/configure", - "--prefix=/usr", - "--syslibdir=/lib", - "--target=" + OPTIONS.target_triple, - ], - env = { - "CC": OPTIONS.target_triple + "-gcc", - "CFLAGS": OPTIONS.cflags, - "LDFLAGS": OPTIONS.ldflags, - }, - ) - -build = autotools_build -install = autotools_install diff --git a/src/apk.rs b/src/apk.rs deleted file mode 100644 index 07d8144..0000000 --- a/src/apk.rs +++ /dev/null @@ -1,46 +0,0 @@ -use crate::config::Config; -use crate::recipe::OutputPackage; -use std::path::Path; - -#[derive(Debug, Clone)] -pub struct ApkPackagePlan { - pub args: Vec, -} - -pub fn mkpkg_plan( - config: &Config, - package: &OutputPackage, - files_root: &Path, - output_dir: &Path, -) -> ApkPackagePlan { - let file_name = format!( - "{}-{}-r{}.apk", - package.name, package.version, package.revision - ); - let output = output_dir.join(file_name); - let version = format!("{}-r{}", package.version, package.revision); - let mut args = vec![ - "mkpkg".to_owned(), - "--files".to_owned(), - files_root.display().to_string(), - "--output".to_owned(), - output.display().to_string(), - "--info".to_owned(), - format!("name:{}", package.name), - "--info".to_owned(), - format!("version:{version}"), - "--info".to_owned(), - format!("arch:{}", config.target_arch), - "--info".to_owned(), - format!("description:{}", package.description), - "--info".to_owned(), - format!("license:{}", package.license), - "--info".to_owned(), - format!("origin:{}", package.recipe), - ]; - for dep in &package.deps { - args.push("--info".to_owned()); - args.push(format!("depends:{dep}")); - } - ApkPackagePlan { args } -} diff --git a/src/build.rs b/src/build.rs deleted file mode 100644 index b31254d..0000000 --- a/src/build.rs +++ /dev/null @@ -1,949 +0,0 @@ -use crate::apk; -use crate::config::Config; -use crate::graph::PackageGraph; -use crate::log; -use crate::patches; -use crate::phase::{PhaseCommand, PhaseEnv, SourceDir, collect_phase_commands}; -use crate::recipe::{OutputPackage, PackageKind, Recipe, RecipeSet}; -use crate::source; -use anyhow::{Context, Result, anyhow, bail}; -use sha2::{Digest, Sha256}; -use std::cell::{Cell, RefCell}; -use std::collections::HashSet; -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; - -const C_SOURCE: &str = "/source"; -const C_BUILD: &str = "/builddir"; -const C_DEST: &str = "/dest"; -const C_SYSROOT: &str = "/sysroot"; -const HOST_PREFIX: &str = "/usr/local"; -const TARGET_PREFIX: &str = "/usr"; - -#[derive(Debug)] -pub struct Builder { - repo: PathBuf, - config: Config, - skip_checks: bool, - preflight_done: Cell, - built_recipes: RefCell>, -} - -impl Builder { - pub fn new(repo: PathBuf, config: Config, skip_checks: bool) -> Self { - Self { - repo, - config, - skip_checks, - preflight_done: Cell::new(false), - built_recipes: RefCell::default(), - } - } - - pub fn build( - &self, - recipes: &RecipeSet, - graph: &PackageGraph, - package: Option<&str>, - rebuild: bool, - ) -> Result<()> { - self.preflight_container()?; - let (label, order) = match package { - Some(p) => (p.to_owned(), graph.build_order(p)?), - None => ("".to_owned(), graph.build_order_all()?), - }; - log::step( - "plan", - &format!("{label}: {} package(s) [{}]", order.len(), order.join(", ")), - ); - for output_name in order { - let output = graph.output(&output_name)?; - let recipe = recipes.recipe_for_package(&output_name)?; - self.ensure_source(recipe)?; - // Only force a rebuild of the explicitly requested package; - // dependencies stay cached when their manifest is up to date. - // With no package (build all), every package honours --rebuild. - let force = rebuild && package.map_or(true, |p| p == output_name); - match output.kind { - PackageKind::Host => self.build_host(recipe, output, force)?, - PackageKind::Target => self.build_target(recipe, output, force)?, - } - } - self.repo_index()?; - Ok(()) - } - - pub fn fetch(&self, recipes: &RecipeSet, package: &str) -> Result<()> { - let recipe = recipes.recipe_by_user_ref(package)?; - log::step("fetch", &recipe.key()); - source::fetch_sources(recipe, &self.source_cache_dir())?; - Ok(()) - } - - pub fn shell(&self, recipes: &RecipeSet, graph: &PackageGraph, package: &str) -> Result<()> { - graph.output(package)?; - let recipe = recipes.recipe_for_package(package)?; - self.preflight_container()?; - let source = self.unpacked_source_dir(recipe); - let build = self.build_dir(recipe); - fs::create_dir_all(&source)?; - fs::create_dir_all(&build)?; - let status = Command::new(&self.config.container_runtime) - .arg("run") - .arg("--rm") - .arg("-it") - .arg("-v") - .arg(format!("{}:{C_SOURCE}:ro", source.display())) - .arg("-v") - .arg(format!("{}:{C_BUILD}", build.display())) - .arg("-w") - .arg(C_BUILD) - .arg(&self.config.container_image) - .arg("/bin/sh") - .status()?; - if !status.success() { - bail!("container shell exited with {status}"); - } - Ok(()) - } - - pub fn repo_index(&self) -> Result<()> { - self.preflight_container()?; - let repo = self.pkgs_dir(); - fs::create_dir_all(&repo)?; - // Nothing to index yet — leave it alone so callers don't trip on a - // failing `*.apk` glob. - let has_apks = fs::read_dir(&repo)?.any(|e| { - e.ok() - .and_then(|e| e.path().extension().map(|x| x == "apk")) - .unwrap_or(false) - }); - if !has_apks { - return Ok(()); - } - log::step( - "index", - &format!("signing repository at {}", repo.display()), - ); - let key = self.abs_config_path(&self.config.signing_key); - let pubkey = self.abs_config_path(&self.config.signing_pubkey); - if !key.exists() || !pubkey.exists() { - bail!("signing key is not configured or missing; run `distro init-key` first"); - } - let index_name = "APKINDEX.tar.gz"; - let status = Command::new(&self.config.container_runtime) - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{}:/repo", repo.display())) - .arg("-v") - .arg(format!("{}:/keys/distro.rsa:ro", key.display())) - .arg("-v") - .arg(format!( - "{}:/etc/apk/keys/distro.rsa.pub:ro", - pubkey.display() - )) - .arg(&self.config.container_image) - .arg("/bin/sh") - .arg("-lc") - .arg(format!( - "cd /repo && apk --sign-key /keys/distro.rsa mkndx -o {index_name} *.apk" - )) - .status() - .context("failed to run repository index command")?; - if !status.success() { - bail!("repository index command failed with {status}"); - } - Ok(()) - } - - pub fn init_key(&self) -> Result<()> { - let key = self.abs_config_path(&self.config.signing_key); - let pubkey = self.abs_config_path(&self.config.signing_pubkey); - if key.exists() && pubkey.exists() { - log::skip("init-key", "signing key already present"); - return Ok(()); - } - log::step("init-key", &format!("generating {}", key.display())); - let key_dir = key - .parent() - .ok_or_else(|| anyhow!("invalid signing key path"))?; - let key_name = key - .file_name() - .ok_or_else(|| anyhow!("invalid signing key path"))? - .to_string_lossy(); - fs::create_dir_all(key_dir)?; - self.preflight_container()?; - let status = Command::new(&self.config.container_runtime) - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{}:/keys", key_dir.display())) - .arg(&self.config.container_image) - .arg("/bin/sh") - .arg("-lc") - .arg(format!( - "openssl genrsa -out /keys/{key_name} 4096 && \ - openssl rsa -in /keys/{key_name} -pubout -out /keys/{key_name}.pub" - )) - .status() - .context("failed to run key generation command")?; - if !status.success() { - bail!("key generation failed with {status}"); - } - Ok(()) - } - - pub fn rootfs(&self, root: &Path, packages: &[String]) -> Result<()> { - self.preflight_container()?; - let pubkey = self.abs_config_path(&self.config.signing_pubkey); - if !pubkey.exists() { - bail!("rootfs requires a configured public signing key; run `distro init-key` first"); - } - fs::create_dir_all(root)?; - log::step( - "rootfs", - &format!( - "{} -> {} [{}]", - packages.join(", "), - root.display(), - root.display() - ), - ); - let status = Command::new(&self.config.container_runtime) - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{}:/rootfs", root.display())) - .arg("-v") - .arg(format!("{}:/repo:ro", self.pkgs_root().display())) - .arg("-v") - .arg(format!( - "{}:/etc/apk/keys/distro.rsa.pub:ro", - pubkey.display() - )) - .arg(&self.config.container_image) - .arg("apk") - .arg("--root") - .arg("/rootfs") - .arg("--keys-dir") - .arg("/etc/apk/keys") - .arg("--initdb") - .arg("--repository") - .arg("/repo") - .arg("add") - .args(packages) - .status() - .context("failed to run rootfs apk command")?; - if !status.success() { - bail!("rootfs creation failed with {status}"); - } - Ok(()) - } - - // --- per-recipe build steps ------------------------------------------- - - fn build_host(&self, recipe: &Recipe, output: &OutputPackage, rebuild: bool) -> Result<()> { - let manifest = self.manifest_path(&output.key()); - let hash = self.manifest_hash(recipe)?; - if !rebuild && fs::read_to_string(&manifest).ok().as_deref() == Some(hash.as_str()) { - log::skip("up-to-date", &output.key()); - return Ok(()); - } - log::step( - "build", - &format!("{} {}-r{}", output.key(), recipe.version, recipe.revision), - ); - let build_dir = self.host_build_dir(recipe); - let dest_dir = self.host_pkg_dir(recipe); - let source_dir = self.unpacked_source_dir(recipe); - // A rebuild starts from a clean build dir; sources stay shared. - if rebuild { - Self::recreate(&build_dir)?; - } else { - fs::create_dir_all(&build_dir)?; - } - Self::recreate(&dest_dir)?; - fs::create_dir_all(&build_dir)?; - - let env = PhaseEnv { - source_dir: source_dir_env(recipe), - build_dir: C_BUILD, - dest_dir: C_DEST, - prefix: HOST_PREFIX, - sysroot: C_SYSROOT, - }; - self.run_recipe_phases( - recipe, - output, - &env, - &source_dir, - &build_dir, - &dest_dir, - None, - )?; - - fs::create_dir_all(manifest.parent().unwrap())?; - fs::write(manifest, hash)?; - Ok(()) - } - - fn build_target(&self, recipe: &Recipe, output: &OutputPackage, rebuild: bool) -> Result<()> { - let manifest = self.manifest_path(&output.key()); - let hash = self.manifest_hash(recipe)?; - if !rebuild && fs::read_to_string(&manifest).ok().as_deref() == Some(hash.as_str()) { - log::skip("up-to-date", &output.key()); - return Ok(()); - } - log::step( - "build", - &format!("{} {}-r{}", output.key(), recipe.version, recipe.revision), - ); - let build_dir = self.build_dir(recipe); - let source_dir = self.unpacked_source_dir(recipe); - // A rebuild starts from a clean build dir; sources stay shared. - if rebuild { - Self::recreate(&build_dir)?; - } else { - fs::create_dir_all(&build_dir)?; - } - - let dest_tmp = - tempfile::tempdir_in(&build_dir).context("failed to create ephemeral destdir")?; - let dest_dir = dest_tmp.path(); - - let sysroot = self.materialize_sysroot(recipe)?; - let env = PhaseEnv { - source_dir: source_dir_env(recipe), - build_dir: C_BUILD, - dest_dir: C_DEST, - prefix: TARGET_PREFIX, - sysroot: C_SYSROOT, - }; - self.run_recipe_phases( - recipe, - output, - &env, - &source_dir, - &build_dir, - dest_dir, - sysroot.as_ref().map(|s| s.path()), - )?; - - self.apk_mkpkg(output, dest_dir)?; - - fs::create_dir_all(manifest.parent().unwrap())?; - fs::write(manifest, hash)?; - Ok(()) - } - - fn run_recipe_phases( - &self, - recipe: &Recipe, - output: &OutputPackage, - env: &PhaseEnv<'_>, - source_dir: &Path, - build_dir: &Path, - dest_dir: &Path, - target_sysroot: Option<&Path>, - ) -> Result<()> { - let host_sandbox = self.materialize_host_sandbox(recipe)?; - let recipe_already_built = self.built_recipes.borrow().contains(&recipe.key()); - - // configure/build/check run at most once per recipe per invocation. - let mut shared_phases: Vec<(&str, bool)> = Vec::new(); - if !recipe_already_built { - if let Some(p) = recipe.configure_fn.as_deref() { - shared_phases.push((p, false)); - } - if let Some(p) = recipe.build_fn.as_deref() { - shared_phases.push((p, false)); - } - if !self.skip_checks { - if let Some(p) = recipe.check_fn.as_deref() { - shared_phases.push((p, false)); - } - } - } - // install runs per output (each output has its own destdir/install_fn). - let install_phase = (output.install_fn.as_str(), true); - - let mut commands: Vec = Vec::new(); - for (phase, takes_pkg) in shared_phases.iter().chain(std::iter::once(&install_phase)) { - let pkg = if *takes_pkg { - Some((output.name.as_str(), C_DEST)) - } else { - None - }; - let owner = if *takes_pkg { - output.key() - } else { - recipe.key() - }; - log::info("phase", &format!("{phase} {owner}")); - commands.extend(collect_phase_commands( - &recipe.path, - &self.repo, - &self.config, - phase, - env, - pkg, - )?); - } - if commands.is_empty() { - self.built_recipes.borrow_mut().insert(recipe.key()); - return Ok(()); - } - - self.run_in_container( - source_dir, - build_dir, - dest_dir, - host_sandbox.as_ref().map(|s| s.path()), - target_sysroot, - &commands, - )?; - self.built_recipes.borrow_mut().insert(recipe.key()); - Ok(()) - } - - fn run_in_container( - &self, - source_dir: &Path, - build_dir: &Path, - dest_dir: &Path, - host_sandbox: Option<&Path>, - target_sysroot: Option<&Path>, - commands: &[PhaseCommand], - ) -> Result<()> { - fs::create_dir_all(dest_dir)?; - let mut process = Command::new(&self.config.container_runtime); - process - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{}:{C_SOURCE}:ro", source_dir.display())) - .arg("-v") - .arg(format!("{}:{C_BUILD}", build_dir.display())) - .arg("-v") - .arg(format!("{}:{C_DEST}", dest_dir.display())) - .arg("-w") - .arg(C_BUILD); - if let Some(sandbox) = host_sandbox { - // Host packages are configured with --prefix=/usr/local, so we - // bind the assembled tree exactly there (matching Jinx). This - // means rpaths, --print-search-dirs, pkg-config lookups, etc. - // all keep working with no extra environment fiddling. - process - .arg("-v") - .arg(format!("{}:{HOST_PREFIX}:ro", sandbox.display())); - } - if let Some(sysroot) = target_sysroot { - process - .arg("-v") - .arg(format!("{}:{C_SYSROOT}:ro", sysroot.display())) - .arg("-e") - .arg(format!("PKG_CONFIG_SYSROOT_DIR={C_SYSROOT}")) - .arg("-e") - .arg(format!("SYSROOT={C_SYSROOT}")); - } - process - .arg(&self.config.container_image) - .arg("/bin/sh") - .arg("-c") - .arg(build_phase_script(commands)); - let status = process - .status() - .context("failed to start container for phase commands")?; - if !status.success() { - bail!("phase commands failed with {status}"); - } - Ok(()) - } - - fn apk_mkpkg(&self, output: &OutputPackage, dest_dir: &Path) -> Result<()> { - let repo = self.pkgs_dir(); - fs::create_dir_all(&repo)?; - let signing_key = self.abs_config_path(&self.config.signing_key); - if !signing_key.exists() { - bail!("package signing key is missing; run `distro init-key` first"); - } - log::step("package", &format!("{} -> {}", output.name, repo.display())); - let plan = apk::mkpkg_plan(&self.config, output, Path::new(C_DEST), Path::new("/out")); - let status = Command::new(&self.config.container_runtime) - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{}:{C_DEST}:ro", dest_dir.display())) - .arg("-v") - .arg(format!("{}:/out", repo.display())) - .arg("-v") - .arg(format!("{}:/keys/distro.rsa:ro", signing_key.display())) - .arg(&self.config.container_image) - .arg("apk") - .arg("--sign-key") - .arg("/keys/distro.rsa") - .args(plan.args) - .status() - .context("failed to run apk mkpkg command")?; - if !status.success() { - bail!("apk mkpkg failed with {status}"); - } - Ok(()) - } - - // --- source preparation (Jinx-style persistent sources//) ------ - - fn ensure_source(&self, recipe: &Recipe) -> Result<()> { - let cached = source::fetch_sources(recipe, &self.source_cache_dir())?; - - let unpacked = self.unpacked_source_dir(recipe); - let version_stamp = self.source_stamp(recipe, "version"); - let want_version = format!("{}-r{}", recipe.version, recipe.revision); - let need_extract = - fs::read_to_string(&version_stamp).ok().as_deref() != Some(&want_version); - - if need_extract { - log::info("unpack", &recipe.key()); - if unpacked.exists() { - fs::remove_dir_all(&unpacked)?; - } - fs::create_dir_all(&unpacked)?; - for (src, tarball) in recipe.sources.iter().zip(cached.iter()) { - let dest = if src.name.is_empty() { - unpacked.clone() - } else { - unpacked.join(&src.name) - }; - self.extract_tarball(tarball, &dest, src.strip_components)?; - } - fs::write(&version_stamp, &want_version)?; - let patched = self.source_stamp(recipe, "patched"); - let _ = fs::remove_file(&patched); - } - - let patched_stamp = self.source_stamp(recipe, "patched"); - if !patched_stamp.exists() { - self.apply_patches(recipe, &unpacked)?; - fs::write(&patched_stamp, "")?; - } - Ok(()) - } - - fn extract_tarball(&self, tarball: &Path, dest: &Path, strip_components: u32) -> Result<()> { - fs::create_dir_all(dest)?; - log::info( - "extract", - &format!("{} -> {}", tarball.display(), dest.display()), - ); - let strip = if strip_components > 0 { - format!("--strip-components={strip_components}") - } else { - String::new() - }; - let status = Command::new(&self.config.container_runtime) - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{}:/in.tar:ro", tarball.display())) - .arg("-v") - .arg(format!("{}:/out", dest.display())) - .arg(&self.config.container_image) - .arg("/bin/sh") - .arg("-c") - .arg(format!("tar -xf /in.tar -C /out {strip}")) - .status() - .context("failed to unpack source archive")?; - if !status.success() { - bail!("source unpack failed with {status}"); - } - Ok(()) - } - - fn apply_patches(&self, recipe: &Recipe, source_dir: &Path) -> Result<()> { - let patches = patches::discover(&recipe.dir)?; - if patches.is_empty() { - return Ok(()); - } - if recipe.sources.iter().any(|s| !s.name.is_empty()) { - bail!( - "recipe `{}` has patches/ but uses multi-source layout; \ - apply patches yourself via ctx.run in the configure phase", - recipe.id - ); - } - for patch in patches { - log::info( - "patch", - &format!( - "{} <- {}", - recipe.id, - patch - .file_name() - .map(|n| n.to_string_lossy().into_owned()) - .unwrap_or_default(), - ), - ); - let status = Command::new(&self.config.container_runtime) - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{}:/source", source_dir.display())) - .arg("-v") - .arg(format!("{}:/patch.diff:ro", patch.display())) - .arg(&self.config.container_image) - .arg("patch") - .arg("-d") - .arg("/source") - .arg("-p1") - .arg("-i") - .arg("/patch.diff") - .status() - .context("failed to apply patch")?; - if !status.success() { - bail!("patch {} failed with {status}", patch.display()); - } - } - Ok(()) - } - - // --- host sandbox / target sysroot materialization -------------------- - - /// Assemble all `host_deps` into a single ephemeral tree that we can mount - /// at `/usr/local` inside the container. Following Jinx, we hard-link - /// rather than byte-copy so this is essentially free. The returned - /// `TempDir` cleans up when dropped. - fn materialize_host_sandbox(&self, recipe: &Recipe) -> Result> { - if recipe.host_deps.is_empty() { - return Ok(None); - } - fs::create_dir_all(self.repo.join("build"))?; - let sandbox = tempfile::Builder::new() - .prefix(&format!("host-sandbox-{}-", recipe.id)) - .tempdir_in(self.repo.join("build")) - .context("failed to create host sandbox tempdir")?; - log::info( - "host-sandbox", - &format!("{} <- [{}]", recipe.id, recipe.host_deps.join(", ")), - ); - for dep in &recipe.host_deps { - let source = self.host_pkg_dir_by_id(dep).join("usr/local"); - if !source.exists() { - bail!( - "host dependency `{dep}` has not been built at {}", - source.display() - ); - } - hardlink_tree(&source, sandbox.path())?; - } - Ok(Some(sandbox)) - } - - fn materialize_sysroot(&self, recipe: &Recipe) -> Result> { - let mut deps: Vec = recipe - .build_deps - .iter() - .chain(recipe.deps.iter()) - .cloned() - .collect(); - deps.sort(); - deps.dedup(); - if deps.is_empty() { - return Ok(None); - } - // The local repo index must be present and current so apk can resolve - // and verify the just-built target packages. - self.repo_index()?; - let pubkey = self.abs_config_path(&self.config.signing_pubkey); - if !pubkey.exists() { - bail!("target dependency sysroot requires a configured public signing key"); - } - fs::create_dir_all(self.repo.join("build"))?; - let sysroot = tempfile::Builder::new() - .prefix(&format!("sysroot-{}-", recipe.id)) - .tempdir_in(self.repo.join("build")) - .context("failed to create sysroot tempdir")?; - log::info( - "sysroot", - &format!("{} <- [{}]", recipe.id, deps.join(", ")), - ); - let status = Command::new(&self.config.container_runtime) - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{}:/sysroot", sysroot.path().display())) - .arg("-v") - .arg(format!("{}:/repo:ro", self.pkgs_root().display())) - .arg("-v") - .arg(format!( - "{}:/etc/apk/keys/distro.rsa.pub:ro", - pubkey.display() - )) - .arg(&self.config.container_image) - .arg("apk") - .arg("--root") - .arg("/sysroot") - .arg("--keys-dir") - .arg("/etc/apk/keys") - .arg("--initdb") - .arg("--repository") - .arg("/repo") - .arg("add") - .args(&deps) - .status() - .context("failed to install target dependency sysroot")?; - if !status.success() { - bail!("target dependency sysroot install failed with {status}"); - } - Ok(Some(sysroot)) - } - - // --- preflight & container image -------------------------------------- - - fn preflight_container(&self) -> Result<()> { - if self.preflight_done.get() { - return Ok(()); - } - let status = Command::new(&self.config.container_runtime) - .arg("--version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .with_context(|| { - format!( - "{} is required but was not found", - self.config.container_runtime - ) - })?; - if !status.success() { - bail!( - "{} preflight failed with {status}", - self.config.container_runtime - ); - } - let dockerfile = self.abs_config_path(&self.config.container_dockerfile); - self.ensure_container_image(&dockerfile)?; - self.preflight_done.set(true); - Ok(()) - } - - fn ensure_container_image(&self, dockerfile: &Path) -> Result<()> { - if !dockerfile.exists() { - bail!( - "configured container Dockerfile does not exist: {}", - dockerfile.display() - ); - } - let hash = self.container_build_hash(dockerfile)?; - let stamp = self.repo.join("build/container-image.hash"); - if fs::read_to_string(&stamp).ok().as_deref() == Some(hash.as_str()) - && self.container_image_exists()? - { - return Ok(()); - } - log::step( - "image", - &format!( - "building {} from {}", - self.config.container_image, - dockerfile.display() - ), - ); - let status = Command::new(&self.config.container_runtime) - .arg("build") - .arg("-f") - .arg(dockerfile) - .arg("-t") - .arg(&self.config.container_image) - .arg(&self.repo) - .status() - .with_context(|| { - format!( - "failed to build container image `{}` from {}", - self.config.container_image, - dockerfile.display() - ) - })?; - if !status.success() { - bail!( - "container image build failed for `{}` with {status}", - self.config.container_image - ); - } - fs::create_dir_all(stamp.parent().unwrap())?; - fs::write(stamp, hash)?; - Ok(()) - } - - fn container_image_exists(&self) -> Result { - let status = Command::new(&self.config.container_runtime) - .arg("image") - .arg("exists") - .arg(&self.config.container_image) - .status() - .with_context(|| { - format!( - "failed to inspect container image `{}`", - self.config.container_image - ) - })?; - Ok(status.success()) - } - - fn container_build_hash(&self, dockerfile: &Path) -> Result { - let mut hasher = Sha256::new(); - hasher.update(fs::read(dockerfile)?); - hasher.update(self.config.container_image.as_bytes()); - Ok(hex::encode(hasher.finalize())) - } - - fn manifest_hash(&self, recipe: &Recipe) -> Result { - let mut hasher = Sha256::new(); - hasher.update(self.config.target_arch.as_bytes()); - hasher.update(recipe.version.as_bytes()); - hasher.update(recipe.revision.to_le_bytes()); - for source in &recipe.sources { - hasher.update(source.url.as_bytes()); - hasher.update(source.sha256.as_bytes()); - } - for patch in patches::discover(&recipe.dir)? { - hasher.update(patch.display().to_string().as_bytes()); - hasher.update(fs::read(patch)?); - } - Ok(hex::encode(hasher.finalize())) - } - - // --- path helpers ----------------------------------------------------- - - fn source_cache_dir(&self) -> PathBuf { - self.repo.join("build/cache/sources") - } - fn unpacked_source_dir(&self, recipe: &Recipe) -> PathBuf { - self.repo.join("build/sources").join(recipe.slug()) - } - fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf { - self.repo - .join("build/sources") - .join(format!("{}.{kind}", recipe.slug())) - } - fn build_dir(&self, recipe: &Recipe) -> PathBuf { - self.repo.join("build/builds").join(&recipe.id) - } - fn host_build_dir(&self, recipe: &Recipe) -> PathBuf { - self.repo.join("build/host-builds").join(&recipe.id) - } - fn host_pkg_dir(&self, recipe: &Recipe) -> PathBuf { - self.repo.join("build/host-pkgs").join(&recipe.id) - } - fn host_pkg_dir_by_id(&self, host_recipe_id: &str) -> PathBuf { - self.repo.join("build/host-pkgs").join(host_recipe_id) - } - /// Root of the target package repo. apk treats this as the repo root and - /// expects `//APKINDEX.tar.gz` underneath. - fn pkgs_root(&self) -> PathBuf { - self.repo.join("build/pkgs") - } - /// Arch-specific package directory: where .apk files and the index live. - fn pkgs_dir(&self) -> PathBuf { - self.pkgs_root().join(&self.config.target_arch) - } - fn manifest_path(&self, output_key: &str) -> PathBuf { - // Output keys may contain `:` (e.g. `host:gcc`); the manifest file - // name uses the filesystem-safe slug form instead. - let safe = output_key.replace(':', "-"); - self.repo - .join("build/manifests") - .join(format!("{safe}.hash")) - } - fn abs_config_path(&self, path: &Path) -> PathBuf { - if path.is_absolute() { - path.to_path_buf() - } else { - self.repo.join(path) - } - } - - fn recreate(path: &Path) -> Result<()> { - if path.exists() { - fs::remove_dir_all(path)?; - } - fs::create_dir_all(path)?; - Ok(()) - } -} - -fn build_phase_script(commands: &[PhaseCommand]) -> String { - let mut parts: Vec = Vec::with_capacity(commands.len() + 1); - parts.push("set -e".to_owned()); - for cmd in commands { - let mut tokens: Vec = Vec::with_capacity(cmd.env.len() + cmd.argv.len()); - for (k, v) in &cmd.env { - let value = shell_escape::unix::escape(v.as_str().into()).into_owned(); - tokens.push(format!("{k}={value}")); - } - for arg in &cmd.argv { - tokens.push(shell_escape::unix::escape(arg.as_str().into()).into_owned()); - } - parts.push(tokens.join(" ")); - } - parts.join("\n") -} - -/// Expose `/source` as a string for single-source recipes, or a struct of -/// `/source/` paths for multi-source recipes. -fn source_dir_env(recipe: &Recipe) -> SourceDir { - if recipe.sources.iter().all(|s| s.name.is_empty()) { - SourceDir::Single(C_SOURCE.to_owned()) - } else { - let map = recipe - .sources - .iter() - .map(|s| (s.name.clone(), format!("{C_SOURCE}/{}", s.name))) - .collect(); - SourceDir::Many(map) - } -} - -/// Mirror `src` into `dst` using hard links for regular files (and preserving -/// symlinks). This matches Jinx's `cp -Pplr`: no bytes are copied, just inode -/// references, so assembling a host-deps sandbox is essentially free. -fn hardlink_tree(src: &Path, dst: &Path) -> Result<()> { - fs::create_dir_all(dst)?; - for entry in walkdir::WalkDir::new(src) { - let entry = entry?; - let relative = entry.path().strip_prefix(src)?; - if relative.as_os_str().is_empty() { - continue; - } - let target = dst.join(relative); - let file_type = entry.file_type(); - if file_type.is_dir() { - fs::create_dir_all(&target)?; - } else if file_type.is_symlink() { - if let Some(parent) = target.parent() { - fs::create_dir_all(parent)?; - } - let link_target = fs::read_link(entry.path())?; - // Replace any pre-existing entry (multiple host deps may ship the - // same symlink path). - let _ = fs::remove_file(&target); - std::os::unix::fs::symlink(&link_target, &target)?; - } else if file_type.is_file() { - if let Some(parent) = target.parent() { - fs::create_dir_all(parent)?; - } - if target.exists() { - continue; - } - fs::hard_link(entry.path(), &target).with_context(|| { - format!( - "failed to hard-link {} -> {}", - entry.path().display(), - target.display() - ) - })?; - } - } - Ok(()) -} diff --git a/src/builder.rs b/src/builder.rs new file mode 100644 index 0000000..0784f2f --- /dev/null +++ b/src/builder.rs @@ -0,0 +1 @@ +pub struct Builder; diff --git a/src/cli.rs b/src/cli.rs index b43ee75..f855070 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,119 +1,63 @@ -use crate::build::Builder; -use crate::config::Config; -use crate::graph::PackageGraph; -use crate::recipe::RecipeSet; -use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; -use std::fs; use std::path::PathBuf; +use crate::{builder::Builder, config::Config, eval, recipe::RecipeSet}; + #[derive(Debug, Parser)] -#[command(name = "distro", version)] -pub struct Cli { - #[arg(long, default_value = ".")] - pub repo: PathBuf, - - #[arg(long)] - pub skip_checks: bool, - +struct Cli { + #[arg( + long, + short, + default_value = ".", + help = "Directory containing the configuration and recipe files" + )] + root: PathBuf, #[command(subcommand)] - pub command: Command, + command: Command, +} + +#[derive(Debug, Parser)] +#[command(about = "Fetch sources for the given recipes")] +struct FetchCommand { + #[arg( + required = true, + help = "List of recipes to fetch, host recipes should be prefixed with `host:`" + )] + recipes: Vec, + #[arg(long, short = 'n', help = "Print what will be done and exit")] + dry_run: bool, +} + +#[derive(Debug, Parser)] +#[command(about = "Build the given recipes")] +struct BuildCommand { + #[arg( + required = true, + help = "List of recipes to build, host recipes should be prefixed with `host:`" + )] + recipes: Vec, + #[arg(long, short, help = "Perform a full rebuild of the given recipes")] + rebuild: bool, + #[arg(long, short = 'n', help = "Print what will be done and exit")] + dry_run: bool, } #[derive(Debug, Subcommand)] -pub enum Command { - Build { - package: Option, - }, - Rebuild { - package: Option, - }, - Fetch { - package: String, - }, - Graph { - package: Option, - }, - List, - Info { - package: String, - }, - Clean, - Shell { - package: String, - }, - RepoIndex, - InitKey, - Rootfs { - root: PathBuf, - packages: Vec, - }, - Update { - #[arg(long)] - bump: bool, - packages: Vec, - }, +enum Command { + Fetch(FetchCommand), + Build(BuildCommand), } -pub fn run(cli: Cli) -> Result<()> { - let repo = cli.repo.canonicalize().unwrap_or(cli.repo); - match &cli.command { - Command::Clean => { - let build = repo.join("build"); - if build.exists() { - fs::remove_dir_all(&build) - .with_context(|| format!("failed to remove {}", build.display()))?; - } - Ok(()) - } - Command::InitKey => { - let config = Config::load(&repo.join("config.star"))?; - Builder::new(repo, config, cli.skip_checks).init_key() - } - _ => { - let config = Config::load(&repo.join("config.star"))?; - let recipes = RecipeSet::load(&repo, &config)?; - if let Command::Update { bump, packages } = &cli.command { - return crate::update::run(&recipes, packages, *bump); - } - let graph = PackageGraph::new(&recipes)?; - let builder = Builder::new(repo, config, cli.skip_checks); - match cli.command { - Command::Build { package } => { - builder.build(&recipes, &graph, package.as_deref(), false) - } - Command::Rebuild { package } => { - builder.build(&recipes, &graph, package.as_deref(), true) - } - Command::Fetch { package } => builder.fetch(&recipes, &package), - Command::Graph { package } => { - for line in graph.render(package.as_deref())? { - println!("{line}"); - } - Ok(()) - } - Command::List => { - for output in graph.outputs() { - println!("{output}"); - } - Ok(()) - } - Command::Info { package } => { - let output = graph.output(&package)?; - println!("{}", serde_json::to_string_pretty(output)?); - Ok(()) - } - Command::Shell { package } => builder.shell(&recipes, &graph, &package), - Command::RepoIndex => builder.repo_index(), - Command::Rootfs { root, packages } => { - if packages.is_empty() { - bail!("rootfs requires at least one package"); - } - builder.rootfs(&root, &packages) - } - Command::Update { .. } => unreachable!(), - Command::Clean | Command::InitKey => unreachable!(), - } - } +pub fn run() -> anyhow::Result<()> { + let cli = Cli::parse(); + + let root_path = cli.root.canonicalize().unwrap_or(cli.root); + let config = Config::load(&root_path.join("config.star"))?; + let lib = eval::eval_lib(&root_path.join("lib"), Some(&config.options))?; + let recipes = RecipeSet::load(&root_path, &config.options, lib.as_ref())?; + + match cli.command { + Command::Fetch { .. } => todo!(), + Command::Build { .. } => todo!(), } } diff --git a/src/config.rs b/src/config.rs index 04f3f1e..b330a1c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,37 +1,100 @@ -use crate::starlark::{eval_file, get_json_map, get_string, get_string_default}; -use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; -use serde_json::Value as JsonValue; -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +use starlark::values::dict::FrozenDictRef; + +use crate::{ + eval::{ExtractError, eval_file, extract_string}, + options::Options, +}; + +#[derive(Debug)] +pub enum ContainerRuntime { + Podman, +} + +impl TryFrom<&str> for ContainerRuntime { + type Error = anyhow::Error; + + fn try_from(value: &str) -> anyhow::Result { + match value { + "podman" => Ok(Self::Podman), + _ => anyhow::bail!("invalid runtime: {value}"), + } + } +} + +#[derive(Debug)] pub struct Config { - pub target_arch: String, - pub options: BTreeMap, - pub container_runtime: String, + pub container_runtime: ContainerRuntime, pub container_image: String, pub container_dockerfile: PathBuf, - pub signing_key: PathBuf, - pub signing_pubkey: PathBuf, + + pub arch: String, + pub options: Options, } impl Config { - pub fn load(path: &Path) -> Result { - let module = eval_file(path, None, None) - .with_context(|| format!("failed to evaluate {}", path.display()))?; + pub fn load(path: &Path) -> anyhow::Result { + let module = eval_file(path, None, None)?; + + let container_runtime = match extract_string(&module, "container_runtime") { + Ok(v) => ContainerRuntime::try_from(v.as_str())?, + Err(ExtractError::NotFound) => ContainerRuntime::Podman, + Err(ExtractError::TypeMismatch) => anyhow::bail!("`container_runtime` is not a string"), + }; + let container_image = match extract_string(&module, "container_image") { + Ok(container_image) => container_image, + Err(ExtractError::NotFound) => { + anyhow::bail!("`container_image` config variable not set") + } + Err(ExtractError::TypeMismatch) => anyhow::bail!("`container_image` is not a string"), + }; + let container_dockerfile = match extract_string(&module, "container_dockerfile") { + Ok(container_dockerfile) => PathBuf::from(container_dockerfile), + Err(ExtractError::NotFound) => PathBuf::from("Dockerfile"), + Err(ExtractError::TypeMismatch) => { + anyhow::bail!("`container_dockerfile` is not a string") + } + }; + let arch = match extract_string(&module, "arch") { + Ok(arch) => arch, + Err(ExtractError::NotFound) => anyhow::bail!("`arch` config variable not set"), + Err(ExtractError::TypeMismatch) => anyhow::bail!("`arch` is not a string"), + }; + + let frozen_module = module.freeze()?; + let options_value = frozen_module + .get_option("options")? + .ok_or_else(|| anyhow::anyhow!("`options` config variable not set"))?; + let entries = { + // SAFETY: the FrozenValue is only used to construct a FrozenDictRef whose + // lifetime is bounded by `options_value`, which keeps the frozen heap alive. + let dict = + FrozenDictRef::from_frozen_value(unsafe { options_value.unchecked_frozen_value() }) + .ok_or_else(|| anyhow::anyhow!("`options` is not a dict"))?; + dict.iter() + .map(|(k, v)| { + let key = k + .to_value() + .unpack_str() + .ok_or_else(|| anyhow::anyhow!("non-string key in `options`"))? + .to_owned(); + Ok((key, options_value.map(|_| v))) + }) + .collect::>>()? + }; + let options = Options::new(entries); + Ok(Self { - target_arch: get_string(&module, "target_arch")?, - options: get_json_map(&module, "options")?, - container_runtime: get_string_default(&module, "container_runtime", "podman")?, - container_image: get_string(&module, "container_image")?, - container_dockerfile: PathBuf::from(get_string_default( - &module, - "container_dockerfile", - "Dockerfile", - )?), - signing_key: PathBuf::from(get_string(&module, "signing_key")?), - signing_pubkey: PathBuf::from(get_string(&module, "signing_pubkey")?), + container_runtime, + container_image, + container_dockerfile, + + arch, + options, }) } } diff --git a/src/eval.rs b/src/eval.rs new file mode 100644 index 0000000..0e10148 --- /dev/null +++ b/src/eval.rs @@ -0,0 +1,177 @@ +use anyhow::Context; +use starlark::{ + environment::{FrozenModule, Globals, GlobalsBuilder, Module}, + eval::Evaluator, + syntax::{AstModule, Dialect}, + values::list::ListRef, +}; +use std::path::{Path, PathBuf}; +use walkdir::WalkDir; + +use crate::{ + options::Options, + recipe::{GitSource, Metadata, Source, Subpackage, TarballSource}, +}; + +#[derive(Debug)] +pub enum ExtractError { + NotFound, + TypeMismatch, +} + +impl std::fmt::Display for ExtractError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ExtractError::NotFound => write!(f, "missing"), + ExtractError::TypeMismatch => write!(f, "wrong type"), + } + } +} + +impl std::error::Error for ExtractError {} + +#[starlark::starlark_module] +fn builder_globals(builder: &mut GlobalsBuilder) { + fn meta( + maintainer: Option, + description: Option, + license: Option, + website: Option, + ) -> anyhow::Result { + Ok(Metadata::new(maintainer, description, license, website)) + } + + fn tarball_source( + url: String, + sha256: String, + strip_components: Option, + ) -> anyhow::Result { + Ok(Source::Tarball(TarballSource::new( + url, + sha256, + strip_components.unwrap_or(0), + ))) + } + + fn git_source(url: String, commit: String) -> anyhow::Result { + Ok(Source::Git(GitSource::new(url, commit))) + } + + fn subpackage(name: String, metadata: Option<&Metadata>) -> anyhow::Result { + let metadata = metadata + .cloned() + .unwrap_or_else(|| Metadata::new(None, None, None, None)); + Ok(Subpackage::new(name, metadata)) + } +} + +pub fn eval_file( + path: &Path, + options: Option<&Options>, + lib: Option<&FrozenModule>, +) -> anyhow::Result { + let module = Module::new(); + if let Some(lib) = lib { + module.import_public_symbols(lib); + } + if let Some(options) = options { + inject_options(&module, options); + } + + let ast = AstModule::parse_file(path, &dialect()).map_err(|err| anyhow::anyhow!("{err}"))?; + let globals = globals(); + + let mut eval = Evaluator::new(&module); + eval.eval_module(ast, &globals) + .map_err(|err| anyhow::anyhow!("{err}"))?; + drop(eval); + + Ok(module) +} + +/// Parse and evaluate every `.star` file under `dir` into a single frozen +/// module whose public bindings can be imported into recipe modules. Returns +/// `Ok(None)` if `dir` doesn't exist or contains no `.star` files. +pub fn eval_lib(dir: &Path, options: Option<&Options>) -> anyhow::Result> { + if !dir.exists() { + return Ok(None); + } + + let mut files: Vec = Vec::new(); + for entry in WalkDir::new(dir) { + let entry = entry.with_context(|| format!("walking lib directory {}", dir.display()))?; + let path = entry.path(); + if entry.file_type().is_file() && path.extension().is_some_and(|ext| ext == "star") { + files.push(path.to_path_buf()); + } + } + if files.is_empty() { + return Ok(None); + } + // Sorted for deterministic ordering when later definitions shadow earlier ones. + files.sort(); + + let module = Module::new(); + if let Some(options) = options { + inject_options(&module, options); + } + let dialect = dialect(); + let globals = globals(); + + for file in &files { + let ast = AstModule::parse_file(file, &dialect) + .map_err(|err| anyhow::anyhow!("parsing {}: {err}", file.display()))?; + let mut eval = Evaluator::new(&module); + eval.eval_module(ast, &globals) + .map_err(|err| anyhow::anyhow!("evaluating {}: {err}", file.display()))?; + } + + Ok(Some(module.freeze()?)) +} + +fn dialect() -> Dialect { + Dialect { + enable_top_level_stmt: true, + enable_f_strings: true, + ..Dialect::Standard + } +} + +fn globals() -> Globals { + GlobalsBuilder::standard().with(builder_globals).build() +} + +fn inject_options(module: &Module, options: &Options) { + let value = module.heap().alloc(options.clone()); + module.set("options", value); +} + +pub fn extract_string(module: &Module, key: &str) -> Result { + module + .get(key) + .ok_or_else(|| ExtractError::NotFound) + .and_then(|v| { + v.unpack_str() + .map(|v| v.to_string()) + .ok_or_else(|| ExtractError::TypeMismatch) + }) +} + +pub fn extract_i32(module: &Module, key: &str) -> Result { + module + .get(key) + .ok_or(ExtractError::NotFound) + .and_then(|v| v.unpack_i32().ok_or(ExtractError::TypeMismatch)) +} + +pub fn extract_string_list(module: &Module, key: &str) -> Result, ExtractError> { + let value = module.get(key).ok_or(ExtractError::NotFound)?; + let list = ListRef::from_value(value).ok_or(ExtractError::TypeMismatch)?; + list.iter() + .map(|v| { + v.unpack_str() + .map(|s| s.to_string()) + .ok_or(ExtractError::TypeMismatch) + }) + .collect() +} diff --git a/src/graph.rs b/src/graph.rs deleted file mode 100644 index 42f16d5..0000000 --- a/src/graph.rs +++ /dev/null @@ -1,135 +0,0 @@ -use crate::recipe::{OutputPackage, PackageKind, RecipeSet, unresolved_deps}; -use anyhow::{Result, anyhow, bail}; -use std::collections::{BTreeMap, BTreeSet}; - -#[derive(Debug, Clone)] -pub struct PackageGraph { - outputs: BTreeMap, - target_edges: BTreeMap>, - host_edges: BTreeMap>, -} - -impl PackageGraph { - pub fn new(recipes: &RecipeSet) -> Result { - let missing = unresolved_deps(recipes); - if !missing.is_empty() { - bail!("unresolved local dependencies:\n{}", missing.join("\n")); - } - let mut target_edges = BTreeMap::new(); - let mut host_edges = BTreeMap::new(); - for recipe in recipes.recipes.values() { - // host_deps always resolve into the host namespace. - let host_dep_keys: Vec = recipe - .host_deps - .iter() - .map(|d| PackageKind::Host.key(d)) - .collect(); - for output in &recipe.outputs { - let key = output.key(); - let mut edges = output.all_target_deps(); - if output.name == recipe.name { - edges.extend(recipe.build_deps.iter().cloned()); - edges.extend(recipe.deps.iter().cloned()); - } - match output.kind { - PackageKind::Host => { - host_edges.insert(key, host_dep_keys.clone()); - } - PackageKind::Target => { - let mut deps = host_dep_keys.clone(); - deps.extend(edges); - target_edges.insert(key, deps); - } - } - } - } - Ok(Self { - outputs: recipes.outputs.clone(), - target_edges, - host_edges, - }) - } - - pub fn output(&self, package: &str) -> Result<&OutputPackage> { - self.outputs - .get(package) - .ok_or_else(|| anyhow!("unknown package `{package}`")) - } - - pub fn outputs(&self) -> impl Iterator { - self.outputs.keys().map(String::as_str) - } - - pub fn build_order(&self, package: &str) -> Result> { - self.output(package)?; - let mut visiting = BTreeSet::new(); - let mut visited = BTreeSet::new(); - let mut order = Vec::new(); - self.visit(package, &mut visiting, &mut visited, &mut order)?; - Ok(order) - } - - /// Topologically-ordered list of every output in the graph (host + target). - pub fn build_order_all(&self) -> Result> { - let mut visiting = BTreeSet::new(); - let mut visited = BTreeSet::new(); - let mut order = Vec::new(); - for package in self.outputs.keys() { - self.visit(package, &mut visiting, &mut visited, &mut order)?; - } - Ok(order) - } - - fn visit( - &self, - package: &str, - visiting: &mut BTreeSet, - visited: &mut BTreeSet, - order: &mut Vec, - ) -> Result<()> { - if visited.contains(package) { - return Ok(()); - } - if !visiting.insert(package.to_owned()) { - bail!("dependency cycle involving `{package}`"); - } - let deps = self - .target_edges - .get(package) - .or_else(|| self.host_edges.get(package)) - .cloned() - .unwrap_or_default(); - for dep in deps { - self.visit(&dep, visiting, visited, order)?; - } - visiting.remove(package); - visited.insert(package.to_owned()); - order.push(package.to_owned()); - Ok(()) - } - - pub fn render(&self, package: Option<&str>) -> Result> { - match package { - Some(package) => { - let order = self.build_order(package)?; - Ok(order - .into_iter() - .map(|pkg| format!("{pkg}: {:?}", self.edges(&pkg))) - .collect()) - } - None => Ok(self - .outputs - .keys() - .map(|pkg| format!("{pkg}: {:?}", self.edges(pkg))) - .collect()), - } - } - - fn edges(&self, package: &str) -> Vec { - self.target_edges - .get(package) - .or_else(|| self.host_edges.get(package)) - .cloned() - .unwrap_or_default() - } -} diff --git a/src/log.rs b/src/log.rs deleted file mode 100644 index 37d6e43..0000000 --- a/src/log.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Tiny stderr logger. We use a consistent `==> :
` prefix -//! so progress messages are easy to scan during long builds. - -use std::io::{IsTerminal, Write}; - -const ARROW: &str = "==>"; - -fn paint(color: &str, text: &str) -> String { - if std::io::stderr().is_terminal() { - format!("\x1b[{color}m{text}\x1b[0m") - } else { - text.to_owned() - } -} - -fn emit(color: &str, action: &str, details: &str) { - let arrow = paint(color, ARROW); - let action = paint("1", action); - let _ = writeln!(std::io::stderr(), "{arrow} {action} {details}"); -} - -/// Major step, e.g. starting a build or packaging an output. -pub fn step(action: &str, details: &str) { - emit("1;34", action, details); // bold blue -} - -/// Cache hit / skipped work. -pub fn skip(action: &str, details: &str) { - emit("1;33", action, details); // bold yellow -} - -/// Sub-step inside a larger action. -pub fn info(action: &str, details: &str) { - emit("1;32", action, details); // bold green -} diff --git a/src/main.rs b/src/main.rs index 7719956..6d9edcd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,21 +1,10 @@ -mod apk; -mod build; +mod builder; mod cli; mod config; -mod graph; -mod log; -mod patches; -mod phase; +mod eval; +mod options; mod recipe; -mod rewrite; -mod source; -mod starlark; -mod update; -use anyhow::Result; -use clap::Parser; - -fn main() -> Result<()> { - let cli = cli::Cli::parse(); - cli::run(cli) +fn main() -> anyhow::Result<()> { + cli::run() } diff --git a/src/options.rs b/src/options.rs new file mode 100644 index 0000000..fd064d3 --- /dev/null +++ b/src/options.rs @@ -0,0 +1,35 @@ +use allocative::Allocative; +use starlark::values::{Heap, OwnedFrozenValue, StarlarkValue, Value}; +use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub struct Options { + entries: HashMap, +} + +impl Options { + pub fn new(entries: HashMap) -> Self { + Self { entries } + } +} + +impl std::fmt::Display for Options { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "options") + } +} + +starlark::starlark_simple_value!(Options); + +#[starlark_value(type = "options")] +impl<'v> StarlarkValue<'v> for Options { + fn get_attr(&self, attribute: &str, _heap: &'v Heap) -> Option> { + let owned = self.entries.get(attribute)?; + // SAFETY: `self` is kept alive by the module heap into which it was + // allocated, and `owned` holds an Arc to its source frozen heap. The + // returned Value therefore remains valid for as long as the receiving + // module is alive. + Some(unsafe { owned.unchecked_frozen_value() }.to_value()) + } +} diff --git a/src/patches.rs b/src/patches.rs deleted file mode 100644 index cb7c512..0000000 --- a/src/patches.rs +++ /dev/null @@ -1,19 +0,0 @@ -use anyhow::Result; -use std::path::{Path, PathBuf}; - -pub fn discover(recipe_dir: &Path) -> Result> { - let patch_dir = recipe_dir.join("patches"); - if !patch_dir.exists() { - return Ok(Vec::new()); - } - let mut patches = Vec::new(); - for entry in std::fs::read_dir(&patch_dir)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|ext| ext.to_str()) == Some("patch") { - patches.push(path); - } - } - patches.sort(); - Ok(patches) -} diff --git a/src/phase.rs b/src/phase.rs deleted file mode 100644 index 5c57d6c..0000000 --- a/src/phase.rs +++ /dev/null @@ -1,213 +0,0 @@ -use crate::config::Config; -use crate::starlark::{eval_content_with_extra, prepend_common_lib_load}; -use allocative::Allocative; -use anyhow::{Result, anyhow, bail}; -use serde::{Deserialize, Serialize}; -use starlark::environment::{GlobalsBuilder, LibraryExtension}; -use starlark::eval::Evaluator; -use starlark::starlark_module; -use starlark::values::none::NoneType; -use starlark::values::{ProvidesStaticType, Value}; -use std::cell::RefCell; -use std::collections::BTreeMap; -use std::path::Path; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Allocative)] -pub struct PhaseCommand { - pub argv: Vec, - /// Extra environment variables exported just for this command, in the - /// order the recipe supplied them. Empty means "inherit only". - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub env: Vec<(String, String)>, -} - -/// How `ctx.source_dir` is exposed to Starlark. -/// -/// * `Single` is a single container path (e.g. `/source`), used when the -/// recipe declared a `source = {...}` form. -/// * `Many` is a map of source name to container path, exposed as a -/// `struct(...)` (e.g. `ctx.source_dir.linux`), used when the recipe -/// declared `sources = {"linux": {...}, ...}`. -#[derive(Debug, Clone)] -pub enum SourceDir { - Single(String), - Many(BTreeMap), -} - -/// Per-phase paths exposed to Starlark as `ctx.source_dir`, `ctx.build_dir`, -/// `ctx.dest_dir`, `ctx.prefix`, `ctx.sysroot` (Jinx-inspired). -#[derive(Debug, Clone)] -pub struct PhaseEnv<'a> { - pub source_dir: SourceDir, - pub build_dir: &'a str, - pub dest_dir: &'a str, - pub prefix: &'a str, - pub sysroot: &'a str, -} - -#[derive(Debug, Default, ProvidesStaticType, Allocative)] -struct CommandStore { - commands: RefCell>, -} - -impl CommandStore { - fn push(&self, command: PhaseCommand) { - self.commands.borrow_mut().push(command); - } -} - -#[starlark_module] -fn phase_globals(builder: &mut GlobalsBuilder) { - fn ctx_run<'v>( - argv: Value<'v>, - #[starlark(require = named, default = NoneType)] env: Value<'v>, - eval: &mut Evaluator, - ) -> anyhow::Result { - let json = argv.to_json()?; - let values: Vec = serde_json::from_str(&json) - .map_err(|err| anyhow!("ctx.run expects a list of strings: {err}"))?; - if values.is_empty() { - bail!("ctx.run argv cannot be empty"); - } - let env_vars = parse_env(env)?; - store(eval)?.push(PhaseCommand { - argv: values, - env: env_vars, - }); - Ok(NoneType) - } - - fn ctx_install( - src: &str, - dst: &str, - #[starlark(require = named, default = "644")] mode: &str, - eval: &mut Evaluator, - ) -> anyhow::Result { - store(eval)?.push(PhaseCommand { - argv: vec![ - "install".to_owned(), - format!("-Dm{mode}"), - src.to_owned(), - dst.to_owned(), - ], - env: Vec::new(), - }); - Ok(NoneType) - } -} - -fn parse_env(value: Value<'_>) -> anyhow::Result> { - if value.is_none() { - return Ok(Vec::new()); - } - let json = value.to_json()?; - let map: serde_json::Map = serde_json::from_str(&json) - .map_err(|err| anyhow!("ctx.run env must be a dict of string -> string: {err}"))?; - let mut out = Vec::with_capacity(map.len()); - for (key, val) in map { - let serde_json::Value::String(val) = val else { - bail!("ctx.run env value for `{key}` must be a string"); - }; - if key.is_empty() || key.contains('=') { - bail!("ctx.run env key `{key}` is not a valid variable name"); - } - out.push((key, val)); - } - Ok(out) -} - -fn store<'a, 'b, 'c, 'd>(eval: &'a Evaluator<'b, 'c, 'd>) -> anyhow::Result<&'a CommandStore> { - eval.extra - .ok_or_else(|| anyhow!("ctx command used without command store"))? - .downcast_ref::() - .ok_or_else(|| anyhow!("command store has the wrong type")) -} - -pub fn collect_phase_commands( - recipe_path: &Path, - repo_root: &Path, - config: &Config, - phase: &str, - env: &PhaseEnv<'_>, - package: Option<(&str, &str)>, -) -> Result> { - validate_identifier(phase)?; - let raw = std::fs::read_to_string(recipe_path)?; - // Auto-load helpers from `lib/common.star` so recipes never need an - // explicit `load()` for the canonical helpers. - let mut content = prepend_common_lib_load(Some(repo_root), Some(config), &raw)?; - let jobs = std::thread::available_parallelism() - .map(|j| j.get()) - .unwrap_or(1); - let source_dir_expr = source_dir_literal(&env.source_dir)?; - let ctx_literal = format!( - "struct(run = ctx_run, install = ctx_install, jobs = {jobs}, \ - source_dir = {sd}, build_dir = {bd}, dest_dir = {dd}, prefix = {pf}, sysroot = {sr})", - sd = source_dir_expr, - bd = serde_json::to_string(env.build_dir)?, - dd = serde_json::to_string(env.dest_dir)?, - pf = serde_json::to_string(env.prefix)?, - sr = serde_json::to_string(env.sysroot)?, - ); - let call = match package { - Some((name, destdir)) => format!( - "\n__ctx = {ctx_literal}\n__pkg = struct(name = {n}, destdir = {d})\n{phase}(__ctx, __pkg)\n", - n = serde_json::to_string(name)?, - d = serde_json::to_string(destdir)?, - ), - None => format!("\n__ctx = {ctx_literal}\n{phase}(__ctx)\n"), - }; - content.push_str(&call); - let globals = GlobalsBuilder::extended_by(&[LibraryExtension::StructType]) - .with(phase_globals) - .build(); - let cmd_store = CommandStore::default(); - eval_content_with_extra( - recipe_path, - content, - Some(config), - Some(repo_root), - globals, - Some(&cmd_store), - )?; - Ok(cmd_store.commands.into_inner()) -} - -fn source_dir_literal(source_dir: &SourceDir) -> Result { - match source_dir { - SourceDir::Single(path) => Ok(serde_json::to_string(path)?), - SourceDir::Many(map) => { - let mut fields = Vec::with_capacity(map.len()); - for (name, path) in map { - if !is_valid_field_name(name) { - bail!("source name `{name}` is not a valid Starlark identifier"); - } - fields.push(format!("{name} = {}", serde_json::to_string(path)?)); - } - Ok(format!("struct({})", fields.join(", "))) - } - } -} - -fn is_valid_field_name(name: &str) -> bool { - let mut chars = name.chars(); - match chars.next() { - Some(c) if c == '_' || c.is_ascii_alphabetic() => {} - _ => return false, - } - chars.all(|c| c == '_' || c.is_ascii_alphanumeric()) -} - -fn validate_identifier(name: &str) -> Result<()> { - let mut chars = name.chars(); - let Some(first) = chars.next() else { - bail!("phase function name cannot be empty"); - }; - if !(first == '_' || first.is_ascii_alphabetic()) { - bail!("invalid phase function name `{name}`"); - } - if chars.any(|ch| !(ch == '_' || ch.is_ascii_alphanumeric())) { - bail!("invalid phase function name `{name}`"); - } - Ok(()) -} diff --git a/src/recipe.rs b/src/recipe.rs deleted file mode 100644 index fd77bd4..0000000 --- a/src/recipe.rs +++ /dev/null @@ -1,439 +0,0 @@ -use crate::config::Config; -use crate::starlark::{ - eval_content, get_i32_default, get_json, get_string, get_string_default, get_string_vec, - has_name, prepend_common_lib_load, -}; -use anyhow::{Result, anyhow, bail}; -use serde::{Deserialize, Serialize}; -use serde_json::Value as JsonValue; -use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Path, PathBuf}; -use walkdir::WalkDir; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum PackageKind { - Host, - Target, -} - -impl PackageKind { - /// Canonical `host:`/bare key form used across the graph, CLI and - /// manifest layer. Host and target trees are completely separate - /// namespaces — they may share names. - pub fn key(&self, name: &str) -> String { - match self { - PackageKind::Host => format!("host:{name}"), - PackageKind::Target => name.to_owned(), - } - } - - /// Filesystem-safe variant of [`PackageKind::key`] (no `:`), used to - /// derive build/source/manifest directory names. - pub fn slug(&self, name: &str) -> String { - match self { - PackageKind::Host => format!("host-{name}"), - PackageKind::Target => name.to_owned(), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Source { - /// Empty for the single-`source` form, otherwise the dict key from `sources`. - pub name: String, - pub url: String, - pub sha256: String, - /// Number of leading path components to strip when extracting (tar's - /// `--strip-components`). `0` means strip nothing. - pub strip_components: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct OutputPackage { - pub name: String, - /// Canonical key of the owning recipe (see [`PackageKind::key`]). - pub recipe: String, - pub kind: PackageKind, - pub version: String, - pub revision: i32, - pub description: String, - pub license: String, - /// Target packages installed into the build sysroot. Not propagated as - /// apk `depends:` metadata. - pub build_deps: Vec, - /// Target packages declared as runtime dependencies (apk `depends:`). - /// Also installed into the sysroot so the recipe can link against them. - pub deps: Vec, - pub install_fn: String, -} - -impl OutputPackage { - pub fn key(&self) -> String { - self.kind.key(&self.name) - } - - /// Union of build- and run-dependencies (used to materialize the sysroot - /// and to compute the build graph). - pub fn all_target_deps(&self) -> Vec { - let mut out = self.build_deps.clone(); - out.extend(self.deps.iter().cloned()); - out - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Recipe { - pub id: String, - pub path: PathBuf, - pub dir: PathBuf, - pub name: String, - pub kind: PackageKind, - pub version: String, - pub revision: i32, - pub description: String, - pub license: String, - pub sources: Vec, - pub host_deps: Vec, - pub build_deps: Vec, - pub deps: Vec, - pub outputs: Vec, - pub configure_fn: Option, - pub build_fn: Option, - pub check_fn: Option, -} - -impl Recipe { - /// Canonical key (`host:` or ``), used as the recipe-level - /// identifier in graphs, manifests and CLI references. - pub fn key(&self) -> String { - self.kind.key(&self.id) - } - - /// Filesystem-safe variant of [`Recipe::key`]. - pub fn slug(&self) -> String { - self.kind.slug(&self.id) - } -} - -#[derive(Debug, Clone)] -pub struct RecipeSet { - pub recipes: BTreeMap, - pub outputs: BTreeMap, -} - -impl RecipeSet { - /// Discover recipes under `repo_root`: - /// * `recipes//recipe.star` → target packages - /// * `host-recipes//recipe.star` → host packages - pub fn load(repo_root: &Path, config: &Config) -> Result { - let mut recipes = BTreeMap::new(); - for (subdir, kind) in [ - ("recipes", PackageKind::Target), - ("host-recipes", PackageKind::Host), - ] { - let root = repo_root.join(subdir); - if !root.exists() { - continue; - } - for entry in WalkDir::new(&root).follow_links(false) { - let entry = entry?; - if entry.file_type().is_file() && entry.file_name() == "recipe.star" { - let recipe = Recipe::load(entry.path(), config, repo_root, kind.clone())?; - let key = recipe.key(); - if recipes.insert(key.clone(), recipe).is_some() { - bail!("duplicate recipe `{key}` below {}", root.display()); - } - } - } - } - - let mut outputs = BTreeMap::new(); - for recipe in recipes.values() { - for output in &recipe.outputs { - let key = output.key(); - if outputs.insert(key.clone(), output.clone()).is_some() { - bail!("duplicate package output `{key}`"); - } - } - } - Ok(Self { recipes, outputs }) - } - - /// Look up a recipe by the package key produced by an output. - pub fn recipe_for_package(&self, package: &str) -> Result<&Recipe> { - let output = self - .outputs - .get(package) - .ok_or_else(|| anyhow!("unknown package `{package}`"))?; - self.recipes.get(&output.recipe).ok_or_else(|| { - anyhow!( - "package `{package}` references missing recipe `{}`", - output.recipe - ) - }) - } - - /// Resolve a user-supplied reference (recipe key, output key, or bare - /// id — provided it isn't ambiguous between the host and target trees). - pub fn recipe_by_user_ref(&self, name: &str) -> Result<&Recipe> { - if let Some(recipe) = self.recipes.get(name) { - return Ok(recipe); - } - if self.outputs.contains_key(name) { - return self.recipe_for_package(name); - } - // Bare id: search both trees, error on ambiguity. - let host_key = PackageKind::Host.key(name); - let target_key = PackageKind::Target.key(name); - match (self.recipes.get(&host_key), self.recipes.get(&target_key)) { - (Some(_), Some(_)) => bail!( - "`{name}` is ambiguous: matches both `{host_key}` and `{target_key}`; \ - use the explicit form" - ), - (Some(r), None) | (None, Some(r)) => Ok(r), - (None, None) => bail!("unknown recipe `{name}`"), - } - } -} - -impl Recipe { - pub fn load(path: &Path, config: &Config, repo_root: &Path, kind: PackageKind) -> Result { - // Auto-load helpers from `lib/common.star` so recipes never need an - // explicit `load()` for the canonical helpers. - let raw = std::fs::read_to_string(path)?; - let content = prepend_common_lib_load(Some(repo_root), Some(config), &raw)?; - let module = eval_content( - path, - content, - Some(config), - Some(repo_root), - starlark::environment::Globals::standard(), - )?; - let dir = path - .parent() - .unwrap_or_else(|| Path::new(".")) - .to_path_buf(); - let id = dir - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| anyhow!("recipe path has no package directory: {}", path.display()))? - .to_owned(); - let name = get_string(&module, "name")?; - let version = get_string(&module, "version")?; - let revision = get_i32_default(&module, "revision", 0)?; - let description = get_string_default(&module, "description", "???")?; - let license = get_string_default(&module, "license", "???")?; - let build_deps = get_string_vec(&module, "build_deps")?; - let deps = get_string_vec(&module, "deps")?; - let host_deps = get_string_vec(&module, "host_deps")?; - let sources = parse_sources(get_json(&module, "sources")?, get_json(&module, "source")?)?; - let subpackages = parse_subpackages(get_json(&module, "subpackages")?)?; - let mut outputs = Vec::new(); - let recipe_key = kind.key(&id); - outputs.push(OutputPackage { - name: name.clone(), - recipe: recipe_key.clone(), - kind: kind.clone(), - version: version.clone(), - revision, - description: description.clone(), - license: license.clone(), - build_deps: build_deps.clone(), - deps: deps.clone(), - install_fn: "install".to_owned(), - }); - for subpkg in subpackages { - let sub_name = subpkg - .get("name") - .and_then(JsonValue::as_str) - .ok_or_else(|| anyhow!("subpackage in `{name}` is missing string `name`"))? - .to_owned(); - outputs.push(OutputPackage { - name: sub_name, - recipe: recipe_key.clone(), - kind: kind.clone(), - version: version.clone(), - revision, - description: subpkg - .get("description") - .and_then(JsonValue::as_str) - .unwrap_or(&description) - .to_owned(), - license: subpkg - .get("license") - .and_then(JsonValue::as_str) - .unwrap_or(&license) - .to_owned(), - build_deps: json_string_list(subpkg.get("build_deps"), "subpackage build_deps")? - .unwrap_or_default(), - deps: json_string_list(subpkg.get("deps"), "subpackage deps")?.unwrap_or_default(), - install_fn: subpkg - .get("install") - .and_then(JsonValue::as_str) - .unwrap_or("install") - .to_owned(), - }); - } - validate_required(&outputs)?; - Ok(Self { - id, - path: path.to_path_buf(), - dir, - name, - kind, - version, - revision, - description, - license, - sources, - host_deps, - build_deps, - deps, - outputs, - configure_fn: has_name(&module, "configure").then_some("configure".to_owned()), - build_fn: has_name(&module, "build").then_some("build".to_owned()), - check_fn: has_name(&module, "check").then_some("check".to_owned()), - }) - } -} - -fn validate_required(outputs: &[OutputPackage]) -> Result<()> { - for output in outputs { - if output.name.trim().is_empty() { - bail!("package output name cannot be empty"); - } - for (field, value) in [ - ("version", &output.version), - ("description", &output.description), - ("license", &output.license), - ] { - if value.trim().is_empty() { - bail!( - "package `{}` has empty required field `{field}`", - output.name - ); - } - } - } - Ok(()) -} - -fn parse_sources( - sources: Option, - legacy_source: Option, -) -> Result> { - match (sources, legacy_source) { - (Some(_), Some(_)) => bail!("recipe defines both `sources` and `source`; use only one"), - (None, None) => Ok(Vec::new()), - (None, Some(single)) => { - let obj = single - .as_object() - .ok_or_else(|| anyhow!("`source` must be a dict"))?; - Ok(vec![parse_source_entry(String::new(), obj)?]) - } - (Some(multi), None) => { - let obj = multi - .as_object() - .ok_or_else(|| anyhow!("`sources` must be a dict of {{name: source}}"))?; - obj.iter() - .map(|(name, value)| { - if name.is_empty() { - bail!("source name in `sources` cannot be empty"); - } - let entry = value - .as_object() - .ok_or_else(|| anyhow!("source `{name}` must be a dict"))?; - parse_source_entry(name.clone(), entry) - }) - .collect() - } - } -} - -fn parse_source_entry(name: String, obj: &serde_json::Map) -> Result { - let url = obj - .get("url") - .and_then(JsonValue::as_str) - .ok_or_else(|| anyhow!("source entry missing string `url`"))? - .to_owned(); - let sha256 = obj - .get("sha256") - .and_then(JsonValue::as_str) - .unwrap_or("???") - .to_owned(); - let strip_components = match obj.get("strip_components") { - None => 0, - Some(JsonValue::Number(n)) => n - .as_u64() - .and_then(|v| u32::try_from(v).ok()) - .ok_or_else(|| anyhow!("source `strip_components` must be a non-negative integer"))?, - Some(_) => bail!("source `strip_components` must be an integer"), - }; - Ok(Source { - name, - url, - sha256, - strip_components, - }) -} - -fn parse_subpackages(value: Option) -> Result>> { - match value { - Some(JsonValue::Array(values)) => values - .into_iter() - .map(|value| { - value - .as_object() - .cloned() - .ok_or_else(|| anyhow!("subpackages entries must be objects")) - }) - .collect(), - Some(_) => bail!("subpackages must be a list of objects"), - None => Ok(Vec::new()), - } -} - -fn json_string_list(value: Option<&JsonValue>, label: &str) -> Result>> { - match value { - Some(JsonValue::Array(values)) => values - .iter() - .map(|value| { - value - .as_str() - .map(ToOwned::to_owned) - .ok_or_else(|| anyhow!("{label} must contain only strings")) - }) - .collect::>>() - .map(Some), - Some(_) => bail!("{label} must be a string list"), - None => Ok(None), - } -} - -pub fn unresolved_deps(recipes: &RecipeSet) -> Vec { - let names: BTreeSet<_> = recipes.outputs.keys().cloned().collect(); - let mut missing = Vec::new(); - for recipe in recipes.recipes.values() { - // host_deps always refer to host outputs (canonical `host:`); - // build_deps / deps refer to target outputs (bare names). - for dep in &recipe.host_deps { - let key = PackageKind::Host.key(dep); - if !names.contains(&key) { - missing.push(format!("{} -> {key}", recipe.key())); - } - } - for dep in recipe.build_deps.iter().chain(recipe.deps.iter()) { - if !names.contains(dep) { - missing.push(format!("{} -> {dep}", recipe.key())); - } - } - for output in &recipe.outputs { - for dep in output.build_deps.iter().chain(output.deps.iter()) { - if !names.contains(dep) { - missing.push(format!("{} -> {dep}", output.key())); - } - } - } - } - missing -} diff --git a/src/recipe/metadata.rs b/src/recipe/metadata.rs new file mode 100644 index 0000000..30ee479 --- /dev/null +++ b/src/recipe/metadata.rs @@ -0,0 +1,54 @@ +use allocative::Allocative; +use starlark::values::StarlarkValue; +use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value}; + +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub struct Metadata { + maintainer: Option, + description: Option, + license: Option, + website: Option, +} + +impl std::fmt::Display for Metadata { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "metadata") + } +} + +starlark::starlark_simple_value!(Metadata); + +#[starlark_value(type = "metadata")] +impl<'v> StarlarkValue<'v> for Metadata {} + +impl Metadata { + pub fn new( + maintainer: Option, + description: Option, + license: Option, + website: Option, + ) -> Self { + Self { + maintainer, + description, + license, + website, + } + } + + pub fn maintainer(&self) -> Option<&str> { + self.maintainer.as_deref() + } + + pub fn description(&self) -> Option<&str> { + self.description.as_deref() + } + + pub fn license(&self) -> Option<&str> { + self.license.as_deref() + } + + pub fn website(&self) -> Option<&str> { + self.website.as_deref() + } +} diff --git a/src/recipe/mod.rs b/src/recipe/mod.rs new file mode 100644 index 0000000..2165ced --- /dev/null +++ b/src/recipe/mod.rs @@ -0,0 +1,380 @@ +mod metadata; +mod source; +mod subpackage; + +use anyhow::{Context, bail}; +use starlark::{ + environment::{FrozenModule, Module}, + values::{OwnedFrozenValue, UnpackValue, ValueLike, list::ListRef, typing::StarlarkCallable}, +}; +use std::{ + collections::HashMap, + fmt, + path::{Path, PathBuf}, +}; +use walkdir::WalkDir; + +use crate::{ + eval::{self, ExtractError}, + options::Options, +}; + +pub use metadata::Metadata; +pub use source::{GitSource, Source, TarballSource}; +pub use subpackage::Subpackage; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecipeKind { + Package, + HostPackage, +} + +impl RecipeKind { + pub fn key(self, name: &str) -> String { + match self { + Self::Package => name.to_string(), + Self::HostPackage => format!("host:{name}"), + } + } +} + +#[derive(Debug)] +pub enum Sources { + Single(source::Source), + Multiple(HashMap), +} + +pub struct Recipe { + /// Path to the recipe's .star file. + path: PathBuf, + /// What kind of a recipe is that? + kind: RecipeKind, + /// Version shared by every package output of this recipe. + version: String, + /// Revision shared by every package output of this recipe. + revision: i32, + /// List of sources required to build this recipe. + sources: Sources, + /// All packages produced by this recipe. + /// This is empty for host recipes. + outputs: Vec, + /// Host packages requires for this recipe. + host_deps: Vec, + /// Packages installed to the system root during build of the recipe, but + /// not listed as part of the `apk` dependencies. + build_deps: Vec, + /// Packages installed to the system root during build of the recipe AND + /// listed as part of the `apk` dependencies. + deps: Vec, + /// Packages NOT installed to the system root during build, only listed + /// as part of the `apk` dependencies. + run_deps: Vec, + /// Starlark phase functions defined by the recipe. + phases: RecipePhases, +} + +impl fmt::Debug for Recipe { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Recipe") + .field("path", &self.path) + .field("kind", &self.kind) + .field("version", &self.version) + .field("revision", &self.revision) + .field("sources", &self.sources) + .field("outputs", &self.outputs) + .field("host_deps", &self.host_deps) + .field("build_deps", &self.build_deps) + .field("deps", &self.deps) + .field("run_deps", &self.run_deps) + .field("phases", &self.phases) + .finish() + } +} + +impl Recipe { + pub fn load( + path: &Path, + name: &str, + kind: RecipeKind, + options: &Options, + lib: Option<&FrozenModule>, + ) -> anyhow::Result { + let module = eval::eval_file(path, Some(options), lib) + .with_context(|| format!("evaluating recipe {}", path.display()))?; + + let version = eval::extract_string(&module, "version") + .map_err(|e| anyhow::anyhow!("field `version`: {e}"))?; + + let revision = match eval::extract_i32(&module, "revision") { + Ok(v) => v, + Err(ExtractError::NotFound) => 1, + Err(e) => bail!("field `revision`: {e}"), + }; + + let metadata = match module.get("metadata") { + None => Metadata::new(None, None, None, None), + Some(value) => value + .downcast_ref::() + .ok_or_else(|| anyhow::anyhow!("field `metadata`: expected a metadata value"))? + .clone(), + }; + + let source_value = module + .get("source") + .ok_or_else(|| anyhow::anyhow!("field `source`: missing"))?; + let source = source_value + .downcast_ref::() + .ok_or_else(|| anyhow::anyhow!("field `source`: expected a source value"))? + .clone(); + let sources = Sources::Single(source); + + let host_deps = optional_string_list(&module, "host_deps")?; + let build_deps = optional_string_list(&module, "build_deps")?; + let deps = optional_string_list(&module, "deps")?; + let run_deps = optional_string_list(&module, "run_deps")?; + + let recipe_key = kind.key(name); + let outputs = match kind { + RecipeKind::Package => { + let mut outputs = vec![OutputPackage { + recipe: recipe_key.clone(), + name: name.to_owned(), + metadata: metadata.clone(), + }]; + if let Some(value) = module.get("subpackages") { + let list = ListRef::from_value(value) + .ok_or_else(|| anyhow::anyhow!("field `subpackages`: expected a list"))?; + for item in list.iter() { + let sub = item.downcast_ref::().ok_or_else(|| { + anyhow::anyhow!( + "field `subpackages`: each entry must be a subpackage value" + ) + })?; + outputs.push(OutputPackage { + recipe: recipe_key.clone(), + name: sub.name().to_owned(), + metadata: sub.metadata().clone(), + }); + } + } + outputs + } + RecipeKind::HostPackage => { + if module.get("subpackages").is_some() { + bail!("host recipes cannot declare `subpackages`"); + } + Vec::new() + } + }; + + let module = module + .freeze() + .map_err(|err| anyhow::anyhow!("freezing recipe module {}: {err:?}", path.display()))?; + let phases = RecipePhases::load(&module)?; + + Ok(Recipe { + path: path.to_path_buf(), + kind, + version, + revision, + sources, + outputs, + host_deps, + build_deps, + deps, + run_deps, + phases, + }) + } + + pub fn phases(&self) -> &RecipePhases { + &self.phases + } +} + +fn optional_string_list(module: &Module, key: &str) -> anyhow::Result> { + match eval::extract_string_list(module, key) { + Ok(v) => Ok(v), + Err(ExtractError::NotFound) => Ok(Vec::new()), + Err(e) => Err(anyhow::anyhow!("field `{key}`: {e}")), + } +} + +pub struct RecipePhases { + configure: Option, + build: OwnedFrozenValue, + install: OwnedFrozenValue, +} + +impl fmt::Debug for RecipePhases { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RecipePhases") + .field("configure", &self.configure.is_some()) + .field("build", &true) + .field("install", &true) + .finish() + } +} + +impl RecipePhases { + fn load(module: &FrozenModule) -> anyhow::Result { + Ok(Self { + configure: optional_phase_function(module, "configure")?, + build: required_phase_function(module, "build")?, + install: required_phase_function(module, "install")?, + }) + } + + pub fn configure(&self) -> Option<&OwnedFrozenValue> { + self.configure.as_ref() + } + + pub fn build(&self) -> &OwnedFrozenValue { + &self.build + } + + pub fn install(&self) -> &OwnedFrozenValue { + &self.install + } +} + +fn optional_phase_function( + module: &FrozenModule, + name: &str, +) -> anyhow::Result> { + let Some(value) = module + .get_option(name) + .with_context(|| format!("field `{name}`"))? + else { + return Ok(None); + }; + + validate_phase_function(name, &value)?; + Ok(Some(value)) +} + +fn required_phase_function(module: &FrozenModule, name: &str) -> anyhow::Result { + let value = module + .get_option(name) + .with_context(|| format!("field `{name}`"))? + .ok_or_else(|| anyhow::anyhow!("field `{name}`: missing"))?; + + validate_phase_function(name, &value)?; + Ok(value) +} + +fn validate_phase_function(name: &str, value: &OwnedFrozenValue) -> anyhow::Result<()> { + let callable: Option> = StarlarkCallable::unpack_value_opt(value.value()); + if callable.is_none() { + bail!("field `{name}`: expected a callable value"); + } + Ok(()) +} + +#[derive(Clone, Debug)] +pub struct OutputPackage { + /// Canonical key of the owning recipe. + recipe: String, + /// Name of the output package. + name: String, + /// Metadata attached to the output package. + metadata: Metadata, +} + +#[derive(Debug)] +pub struct RecipeSet { + recipes: HashMap, + outputs: HashMap, +} + +impl RecipeSet { + pub fn load( + root_path: &Path, + options: &Options, + lib: Option<&FrozenModule>, + ) -> anyhow::Result { + let mut recipes = HashMap::new(); + let mut outputs = HashMap::new(); + + for (path, kind) in [ + ("recipes", RecipeKind::Package), + ("host-recipes", RecipeKind::HostPackage), + ] { + let recipes_dir = root_path.join(path); + + if !recipes_dir.exists() { + continue; + } + + for (name, path) in discover_recipes(&recipes_dir)? { + let recipe = Recipe::load(&path, &name, kind, options, lib) + .with_context(|| format!("loading recipe `{name}`"))?; + let key = kind.key(&name); + + if recipes.insert(key.clone(), recipe).is_some() { + bail!("duplicate recipe `{key}`"); + } + } + } + + for recipe in recipes.values() { + for output in &recipe.outputs { + let key = recipe.kind.key(&output.name); + + if outputs.insert(key.clone(), output.clone()).is_some() { + bail!("duplicate package output `{key}`"); + } + } + } + + Ok(Self { recipes, outputs }) + } +} + +/// Find all recipe `.star` files under `dir`, returning a map of recipe name +/// to its `.star` file. Intermediate directories act as categories and are +/// not themselves recipes. Within any subtree, a recipe takes either form: +/// - `.../.star` — name is the file stem +/// - `...//recipe.star` — name is the parent directory +fn discover_recipes(dir: &Path) -> anyhow::Result> { + let mut recipes: HashMap = HashMap::new(); + + let walker = WalkDir::new(dir).follow_links(false); + for entry in walker { + let entry = + entry.with_context(|| format!("walking recipes directory {}", dir.display()))?; + if !entry.file_type().is_file() { + continue; + } + + let path = entry.path(); + let Some(file_name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + + let name = if file_name == "recipe.star" { + let Some(parent_name) = path + .parent() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + else { + continue; + }; + parent_name.to_owned() + } else if let Some(stem) = file_name.strip_suffix(".star") { + stem.to_owned() + } else { + continue; + }; + + if let Some(existing) = recipes.insert(name.clone(), path.to_path_buf()) { + bail!( + "recipe `{name}` is defined twice: {} and {}", + existing.display(), + recipes[&name].display(), + ); + } + } + + Ok(recipes) +} diff --git a/src/recipe/source.rs b/src/recipe/source.rs new file mode 100644 index 0000000..97f4af8 --- /dev/null +++ b/src/recipe/source.rs @@ -0,0 +1,91 @@ +use allocative::Allocative; +use starlark::values::StarlarkValue; +use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value}; + +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub struct TarballSource { + url: String, + sha256: String, + strip_components: u32, +} + +impl std::fmt::Display for TarballSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "tarball_source") + } +} + +starlark::starlark_simple_value!(TarballSource); + +#[starlark_value(type = "tarball_source")] +impl<'v> StarlarkValue<'v> for TarballSource {} + +impl TarballSource { + pub fn new(url: String, sha256: String, strip_components: u32) -> Self { + Self { + url, + sha256, + strip_components, + } + } + + pub fn url(&self) -> &str { + &self.url + } + + pub fn sha256(&self) -> &str { + &self.sha256 + } + + pub fn strip_components(&self) -> u32 { + self.strip_components + } +} + +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub struct GitSource { + url: String, + commit: String, +} + +impl std::fmt::Display for GitSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "git_source") + } +} + +starlark::starlark_simple_value!(GitSource); + +#[starlark_value(type = "git_source")] +impl<'v> StarlarkValue<'v> for GitSource {} + +impl GitSource { + pub fn new(url: String, commit: String) -> Self { + Self { url, commit } + } + + pub fn url(&self) -> &str { + &self.url + } + + pub fn commit(&self) -> &str { + &self.commit + } +} + +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub enum Source { + Tarball(TarballSource), + Git(GitSource), +} + +impl std::fmt::Display for Source { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "source") + } +} + +starlark::starlark_simple_value!(Source); + +#[starlark_value(type = "source")] +impl<'v> StarlarkValue<'v> for Source {} diff --git a/src/recipe/subpackage.rs b/src/recipe/subpackage.rs new file mode 100644 index 0000000..b1d9c9f --- /dev/null +++ b/src/recipe/subpackage.rs @@ -0,0 +1,36 @@ +use allocative::Allocative; +use starlark::values::StarlarkValue; +use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value}; + +use crate::recipe::Metadata; + +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub struct Subpackage { + name: String, + metadata: Metadata, +} + +impl Subpackage { + pub fn new(name: String, metadata: Metadata) -> Self { + Self { name, metadata } + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn metadata(&self) -> &Metadata { + &self.metadata + } +} + +impl std::fmt::Display for Subpackage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "subpackage") + } +} + +starlark::starlark_simple_value!(Subpackage); + +#[starlark_value(type = "subpackage")] +impl<'v> StarlarkValue<'v> for Subpackage {} diff --git a/src/rewrite.rs b/src/rewrite.rs deleted file mode 100644 index 0ee478a..0000000 --- a/src/rewrite.rs +++ /dev/null @@ -1,39 +0,0 @@ -use anyhow::{Context, Result}; -use std::fs; -use std::path::{Path, PathBuf}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Rewrite { - pub field: String, - pub old: String, - pub new: String, -} - -pub fn backup_path(path: &Path) -> PathBuf { - let mut backup = path.as_os_str().to_os_string(); - backup.push(".bak"); - PathBuf::from(backup) -} - -pub fn rewrite_placeholders(path: &Path, rewrites: &[Rewrite]) -> Result { - if rewrites.is_empty() { - return Ok(false); - } - let original = - fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; - let mut updated = original.clone(); - for rewrite in rewrites { - let quoted_old = format!("{} = \"{}\"", rewrite.field, rewrite.old); - let quoted_new = format!("{} = \"{}\"", rewrite.field, rewrite.new); - updated = updated.replacen("ed_old, "ed_new, 1); - let dict_old = format!("\"{}\": \"{}\"", rewrite.field, rewrite.old); - let dict_new = format!("\"{}\": \"{}\"", rewrite.field, rewrite.new); - updated = updated.replacen(&dict_old, &dict_new, 1); - } - if updated == original { - return Ok(false); - } - fs::write(backup_path(path), original)?; - fs::write(path, updated)?; - Ok(true) -} diff --git a/src/source.rs b/src/source.rs deleted file mode 100644 index 899d431..0000000 --- a/src/source.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::log; -use crate::recipe::Recipe; -use crate::rewrite::{Rewrite, rewrite_placeholders}; -use anyhow::{Context, Result, bail}; -use sha2::{Digest, Sha256}; -use std::fs; -use std::io::Read; -use std::path::{Path, PathBuf}; - -pub fn fetch_sources(recipe: &Recipe, cache_dir: &Path) -> Result> { - fs::create_dir_all(cache_dir)?; - let mut rewrites = Vec::new(); - let mut paths = Vec::new(); - for source in &recipe.sources { - let label = if source.name.is_empty() { - recipe.key() - } else { - format!("{}:{}", recipe.key(), source.name) - }; - let cached = source.sha256 != "???" && cache_dir.join(&source.sha256).exists(); - if cached { - log::skip("cached", &format!("{label} ({})", source.url)); - paths.push(cache_dir.join(&source.sha256)); - continue; - } - log::step("fetch", &format!("{label} <- {}", source.url)); - let bytes = download(&source.url)?; - let actual = sha256_hex(&bytes); - if source.sha256 == "???" { - log::info("sha256", &format!("{label} = {actual}")); - rewrites.push(Rewrite { - field: "sha256".into(), - old: "???".into(), - new: actual.clone(), - }); - } else if source.sha256 != actual { - bail!( - "checksum mismatch for {}: expected {}, got {}", - source.url, - source.sha256, - actual - ); - } - let path = cache_dir.join(&actual); - if !path.exists() { - fs::write(&path, &bytes) - .with_context(|| format!("failed to write {}", path.display()))?; - } - paths.push(path); - } - rewrite_placeholders(&recipe.path, &rewrites)?; - Ok(paths) -} - -fn download(url: &str) -> Result> { - if let Some(path) = url.strip_prefix("file://") { - return Ok(fs::read(path)?); - } - // Some mirrors reject requests without a User-Agent with HTTP 403, so set an explicit one. - let client = reqwest::blocking::Client::builder() - .user_agent(concat!("distro/", env!("CARGO_PKG_VERSION"))) - .build()?; - let mut response = client.get(url).send()?.error_for_status()?; - let mut bytes = Vec::new(); - response.read_to_end(&mut bytes)?; - Ok(bytes) -} - -pub fn sha256_hex(bytes: &[u8]) -> String { - let mut hasher = Sha256::new(); - hasher.update(bytes); - hex::encode(hasher.finalize()) -} diff --git a/src/starlark.rs b/src/starlark.rs deleted file mode 100644 index d0bc23e..0000000 --- a/src/starlark.rs +++ /dev/null @@ -1,513 +0,0 @@ -use crate::config::Config; -use allocative::{Allocative, Visitor, ident_key}; -use anyhow::{Result, anyhow, bail}; -use serde_json::Value as JsonValue; -use starlark::environment::{FrozenModule, Globals, Module}; -use starlark::eval::{Evaluator, FileLoader}; -use starlark::starlark_simple_value; -use starlark::syntax::{AstModule, Dialect}; -use starlark::values::dict::AllocDict; -use starlark::values::{AnyLifetime, Heap, NoSerialize, ProvidesStaticType, StarlarkValue, Value}; -use starlark_derive::starlark_value; -use std::collections::{BTreeMap, HashMap}; -use std::fmt::{self, Display}; -use std::mem; -use std::path::{Path, PathBuf}; - -#[derive(Debug, Clone, ProvidesStaticType, NoSerialize)] -pub struct OptionsValue { - values: BTreeMap, -} - -impl OptionsValue { - fn new(values: BTreeMap) -> Result { - for key in values.keys() { - validate_starlark_identifier(key)?; - } - Ok(Self { values }) - } -} - -impl Display for OptionsValue { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "options") - } -} - -starlark_simple_value!(OptionsValue); - -impl Allocative for OptionsValue { - fn visit<'a, 'b: 'a>(&self, visitor: &'a mut Visitor<'b>) { - let mut visitor = visitor.enter_self(self); - visitor.visit_simple( - ident_key!(values), - mem::size_of::<(String, JsonValue)>() * self.values.len(), - ); - visitor.exit(); - } -} - -#[starlark_value(type = "options")] -impl<'v> StarlarkValue<'v> for OptionsValue { - fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option> { - self.values.get(attr).map(|value| heap.alloc(value)) - } -} - -#[derive(Debug, Clone, ProvidesStaticType, NoSerialize, Allocative)] -pub struct SettingsValue { - target_arch: String, - container_runtime: String, - container_image: String, - container_dockerfile: String, - options: BTreeMap, -} - -impl From<&Config> for SettingsValue { - fn from(config: &Config) -> Self { - Self { - target_arch: config.target_arch.clone(), - container_runtime: config.container_runtime.clone(), - container_image: config.container_image.clone(), - container_dockerfile: config.container_dockerfile.display().to_string(), - options: config - .options - .iter() - .map(|(key, value)| (key.clone(), option_value_to_string(value))) - .collect(), - } - } -} - -impl Display for SettingsValue { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "settings") - } -} - -starlark_simple_value!(SettingsValue); - -#[starlark_value(type = "settings")] -impl<'v> StarlarkValue<'v> for SettingsValue { - fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option> { - match attr { - "target_arch" => Some(heap.alloc(self.target_arch.as_str())), - "container_runtime" => Some(heap.alloc(self.container_runtime.as_str())), - "container_image" => Some(heap.alloc(self.container_image.as_str())), - "container_dockerfile" => Some(heap.alloc(self.container_dockerfile.as_str())), - "options" => Some(heap.alloc(AllocDict(self.options.clone()))), - _ => None, - } - } -} - -pub fn eval_file( - path: &Path, - settings: Option<&Config>, - repo_root: Option<&Path>, -) -> Result { - let content = std::fs::read_to_string(path)?; - eval_content(path, content, settings, repo_root, Globals::standard()) -} - -/// Path of the implicit helper library auto-loaded into every recipe. -pub const COMMON_LIB_MODULE: &str = "//lib:common.star"; -const COMMON_LIB_RELATIVE: &str = "lib/common.star"; -const OPTIONS_NAME: &str = "OPTIONS"; - -/// Names exported by `lib/common.star`, if the file exists. Empty otherwise. -pub fn common_lib_names(repo_root: &Path, settings: Option<&Config>) -> Result> { - let path = repo_root.join(COMMON_LIB_RELATIVE); - if !path.exists() { - return Ok(Vec::new()); - } - let module = eval_file(&path, settings, Some(repo_root))?; - Ok(module - .names() - .map(|n| n.as_str().to_owned()) - .filter(|n| !n.starts_with('_') && n != "settings" && n != OPTIONS_NAME) - .collect()) -} - -/// Prepend an implicit `load("//lib:common.star", ...)` so every recipe sees -/// the shared helpers without an explicit import. Does nothing if there's no -/// `lib/common.star` or no repo root. -pub fn prepend_common_lib_load( - repo_root: Option<&Path>, - settings: Option<&Config>, - content: &str, -) -> Result { - let Some(root) = repo_root else { - return Ok(content.to_owned()); - }; - let names = common_lib_names(root, settings)?; - if names.is_empty() { - return Ok(content.to_owned()); - } - let names_lit = names - .iter() - .map(|n| format!("{:?}", n)) - .collect::>() - .join(", "); - Ok(format!( - "load(\"{COMMON_LIB_MODULE}\", {names_lit})\n{content}" - )) -} - -pub fn eval_content( - path: &Path, - content: String, - settings: Option<&Config>, - repo_root: Option<&Path>, - globals: Globals, -) -> Result { - eval_content_with_extra(path, content, settings, repo_root, globals, None) -} - -pub fn eval_content_with_extra<'a>( - path: &Path, - content: String, - settings: Option<&Config>, - repo_root: Option<&Path>, - globals: Globals, - extra: Option<&'a dyn AnyLifetime<'a>>, -) -> Result { - let filename = path.display().to_string(); - validate_options_source(path, &content)?; - let ast = AstModule::parse( - &filename, - content, - &Dialect { - enable_f_strings: true, - enable_top_level_stmt: true, - ..Dialect::Standard - }, - ) - .map_err(|err| anyhow!("{err}"))?; - let loader = match repo_root { - Some(root) => Some(RepoFileLoader::new(root, settings, globals.clone(), &ast)?), - None if !ast.loads().is_empty() => { - bail!( - "load() requires a repo root while evaluating {}", - path.display() - ) - } - None => None, - }; - let module = Module::new(); - let protected_options = if let Some(config) = settings { - Some(set_config_values(&module, config)?) - } else { - None - }; - { - let mut eval = Evaluator::new(&module); - if let Some(loader) = &loader { - eval.set_loader(loader); - } - eval.extra = extra; - eval.eval_module(ast, &globals) - .map_err(|err| anyhow!("{err}"))?; - } - validate_options_binding(path, &module, protected_options)?; - Ok(module) -} - -fn validate_options_source(path: &Path, content: &str) -> Result<()> { - let line_offset = implicit_common_load_line_offset(content); - for (index, line) in content.lines().enumerate() { - let code = line.split('#').next().unwrap_or_default().trim_start(); - if defines_or_reassigns_options(code) { - let line = (index + 1).saturating_sub(line_offset).max(1); - bail!( - "{}:{} must not define or reassign `{OPTIONS_NAME}`", - path.display(), - line - ); - } - } - Ok(()) -} - -fn implicit_common_load_line_offset(content: &str) -> usize { - content - .lines() - .next() - .is_some_and(|line| line.starts_with(&format!("load(\"{COMMON_LIB_MODULE}\","))) - as usize -} - -fn defines_or_reassigns_options(code: &str) -> bool { - let Some(rest) = code.strip_prefix(OPTIONS_NAME) else { - return defines_options_function(code) || binds_options_loop_variable(code); - }; - let rest = rest.trim_start(); - rest.starts_with('=') - || rest.starts_with("+=") - || rest.starts_with("-=") - || rest.starts_with("*=") - || rest.starts_with("/=") - || rest.starts_with("%=") - || rest.starts_with("&=") - || rest.starts_with("|=") - || rest.starts_with("^=") -} - -fn defines_options_function(code: &str) -> bool { - code.strip_prefix("def ") - .and_then(|rest| rest.trim_start().strip_prefix(OPTIONS_NAME)) - .is_some_and(|rest| rest.trim_start().starts_with('(')) -} - -fn binds_options_loop_variable(code: &str) -> bool { - code.strip_prefix("for ") - .and_then(|rest| rest.trim_start().strip_prefix(OPTIONS_NAME)) - .is_some_and(|rest| rest.trim_start().starts_with("in ")) -} - -fn validate_options_binding( - path: &Path, - module: &Module, - expected: Option>, -) -> Result<()> { - let Some(expected) = expected else { - return Ok(()); - }; - let actual = module.get(OPTIONS_NAME).ok_or_else(|| { - anyhow!( - "{} removed the protected `{OPTIONS_NAME}` binding", - path.display() - ) - })?; - if !actual.ptr_eq(expected) { - bail!( - "{} must not define or reassign `{OPTIONS_NAME}`", - path.display() - ); - } - Ok(()) -} - -#[derive(Clone)] -struct RepoFileLoader { - modules: HashMap, -} - -impl RepoFileLoader { - fn new( - repo_root: &Path, - settings: Option<&Config>, - globals: Globals, - ast: &AstModule, - ) -> Result { - let mut modules = HashMap::new(); - for load in ast.loads() { - load_module( - repo_root, - settings, - globals.clone(), - load.module_id, - &mut modules, - )?; - } - Ok(Self { modules }) - } -} - -impl FileLoader for RepoFileLoader { - fn load(&self, path: &str) -> starlark::Result { - self.modules - .get(path) - .cloned() - .ok_or_else(|| starlark::Error::new_other(anyhow!("unknown Starlark module `{path}`"))) - } -} - -fn load_module( - repo_root: &Path, - settings: Option<&Config>, - globals: Globals, - module_id: &str, - modules: &mut HashMap, -) -> Result<()> { - if modules.contains_key(module_id) { - return Ok(()); - } - let path = resolve_load_path(repo_root, module_id)?; - let content = std::fs::read_to_string(&path)?; - let filename = path.display().to_string(); - validate_options_source(&path, &content)?; - let ast = - AstModule::parse(&filename, content, &Dialect::Standard).map_err(|err| anyhow!("{err}"))?; - for load in ast.loads() { - load_module( - repo_root, - settings, - globals.clone(), - load.module_id, - modules, - )?; - } - let nested_loader = RepoFileLoader { - modules: modules.clone(), - }; - let module = Module::new(); - let protected_options = if let Some(config) = settings { - Some(set_config_values(&module, config)?) - } else { - None - }; - { - let mut eval = Evaluator::new(&module); - eval.set_loader(&nested_loader); - eval.eval_module(ast, &globals) - .map_err(|err| anyhow!("{err}"))?; - } - validate_options_binding(&path, &module, protected_options)?; - let frozen = module.freeze().map_err(|err| anyhow!("{err:?}"))?; - modules.insert(module_id.to_owned(), frozen); - Ok(()) -} - -fn set_config_values<'v>(module: &'v Module, config: &Config) -> Result> { - module.set("settings", module.heap().alloc(SettingsValue::from(config))); - let options = module - .heap() - .alloc(OptionsValue::new(config.options.clone())?); - module.set(OPTIONS_NAME, options); - Ok(options) -} - -fn validate_starlark_identifier(name: &str) -> Result<()> { - let mut chars = name.chars(); - let Some(first) = chars.next() else { - bail!("config option name cannot be empty"); - }; - if !(first == '_' || first.is_ascii_alphabetic()) { - bail!("config option `{name}` is not a valid Starlark identifier"); - } - if chars.any(|ch| !(ch == '_' || ch.is_ascii_alphanumeric())) { - bail!("config option `{name}` is not a valid Starlark identifier"); - } - if matches!( - name, - "and" - | "as" - | "assert" - | "break" - | "class" - | "continue" - | "def" - | "del" - | "elif" - | "else" - | "except" - | "finally" - | "for" - | "from" - | "global" - | "if" - | "import" - | "in" - | "is" - | "lambda" - | "load" - | "not" - | "or" - | "pass" - | "return" - | "try" - | "while" - | "with" - | "yield" - ) { - bail!("config option `{name}` is a reserved Starlark keyword"); - } - Ok(()) -} - -fn resolve_load_path(repo_root: &Path, module_id: &str) -> Result { - let relative = if let Some(stripped) = module_id.strip_prefix("//") { - stripped.replace(':', "/") - } else { - module_id.to_owned() - }; - let path = repo_root.join(relative); - let canonical_root = repo_root - .canonicalize() - .unwrap_or_else(|_| repo_root.to_path_buf()); - let canonical_path = path.canonicalize().unwrap_or(path); - if !canonical_path.starts_with(&canonical_root) { - bail!("Starlark load escapes repo root: {module_id}"); - } - Ok(canonical_path) -} - -fn option_value_to_string(value: &JsonValue) -> String { - match value { - JsonValue::String(value) => value.clone(), - JsonValue::Bool(value) => value.to_string(), - JsonValue::Number(value) => value.to_string(), - JsonValue::Null => "null".to_owned(), - JsonValue::Array(_) | JsonValue::Object(_) => value.to_string(), - } -} - -pub fn get_string(module: &Module, name: &str) -> Result { - module - .get(name) - .and_then(|v| v.unpack_str().map(ToOwned::to_owned)) - .ok_or_else(|| anyhow!("missing or non-string Starlark variable `{name}`")) -} - -pub fn get_string_default(module: &Module, name: &str, default: &str) -> Result { - Ok(match module.get(name) { - Some(value) => value - .unpack_str() - .ok_or_else(|| anyhow!("non-string Starlark variable `{name}`"))? - .to_owned(), - None => default.to_owned(), - }) -} - -pub fn get_i32_default(module: &Module, name: &str, default: i32) -> Result { - Ok(match module.get(name) { - Some(value) => value - .unpack_i32() - .ok_or_else(|| anyhow!("non-integer Starlark variable `{name}`"))?, - None => default, - }) -} - -pub fn has_name(module: &Module, name: &str) -> bool { - module.get(name).is_some() -} - -pub fn get_json(module: &Module, name: &str) -> Result> { - match module.get(name) { - Some(value) => Ok(Some(serde_json::from_str(&value.to_json()?)?)), - None => Ok(None), - } -} - -pub fn get_string_vec(module: &Module, name: &str) -> Result> { - match get_json(module, name)? { - Some(JsonValue::Array(values)) => values - .into_iter() - .map(|value| match value { - JsonValue::String(s) => Ok(s), - _ => bail!("`{name}` must be a list of strings"), - }) - .collect(), - Some(_) => bail!("`{name}` must be a list of strings"), - None => Ok(Vec::new()), - } -} - -pub fn get_json_map(module: &Module, name: &str) -> Result> { - match get_json(module, name)? { - Some(JsonValue::Object(values)) => Ok(values.into_iter().collect()), - Some(_) => bail!("`{name}` must be a dict"), - None => Ok(BTreeMap::new()), - } -} diff --git a/src/update.rs b/src/update.rs deleted file mode 100644 index 3c7232c..0000000 --- a/src/update.rs +++ /dev/null @@ -1,256 +0,0 @@ -//! `distro update` — check repology for newer upstream versions and bump -//! recipes in-place. -//! -//! For each requested recipe we query -//! `https://repology.org/api/v1/project/` and pick the highest version -//! among entries whose `status` is `"newest"` or `"unique"` (i.e. not -//! outdated, not rolling/devel, not ignored). If that version is strictly -//! greater than the recipe's current `version`, we rewrite: -//! -//! * `version = "..."` → the new upstream version -//! * `revision = N` → `revision = 1` -//! * every `"sha256": "..."` → `"sha256": "???"` so the next fetch fills it -//! -//! Repology asks API clients to set a descriptive User-Agent. - -use crate::log; -use crate::recipe::{Recipe, RecipeSet}; -use anyhow::{Context, Result, bail}; -use serde::Deserialize; -use std::cmp::Ordering; -use std::fs; -use std::path::Path; - -const REPOLOGY_API: &str = "https://repology.org/api/v1/project"; - -#[derive(Debug, Deserialize)] -struct RepologyEntry { - #[serde(default)] - version: String, - #[serde(default)] - status: String, -} - -pub fn run(recipes: &RecipeSet, names: &[String], bump: bool) -> Result<()> { - let targets: Vec<&Recipe> = if names.is_empty() { - recipes.recipes.values().collect() - } else { - let mut out = Vec::with_capacity(names.len()); - for name in names { - out.push(recipes.recipe_by_user_ref(name)?); - } - out - }; - - let client = reqwest::blocking::Client::builder() - .user_agent(concat!( - "distro/", - env!("CARGO_PKG_VERSION"), - " (+repology version checker)" - )) - .build()?; - - let mut outdated = 0usize; - let mut bumped = 0usize; - let mut up_to_date = 0usize; - let mut errored = 0usize; - - for recipe in targets { - match check_one(&client, recipe) { - Ok(Some(new_version)) => { - if bump { - bump_recipe(&recipe.path, &new_version)?; - log::step( - "bump", - &format!("{}: {} -> {}", recipe.key(), recipe.version, new_version), - ); - bumped += 1; - } else { - log::step( - "outdated", - &format!("{}: {} -> {}", recipe.key(), recipe.version, new_version), - ); - outdated += 1; - } - } - Ok(None) => { - log::skip( - "up-to-date", - &format!("{} {}", recipe.key(), recipe.version), - ); - up_to_date += 1; - } - Err(e) => { - log::info("error", &format!("{}: {e}", recipe.key())); - errored += 1; - } - } - } - - if bump { - log::step( - "summary", - &format!("{bumped} bumped, {up_to_date} up-to-date, {errored} errored"), - ); - } else { - log::step( - "summary", - &format!( - "{outdated} outdated, {up_to_date} up-to-date, {errored} errored (re-run with --bump to rewrite recipes)" - ), - ); - } - Ok(()) -} - -fn check_one(client: &reqwest::blocking::Client, recipe: &Recipe) -> Result> { - let url = format!("{REPOLOGY_API}/{}", recipe.name); - let resp = client - .get(&url) - .send() - .with_context(|| format!("GET {url}"))?; - if resp.status() == reqwest::StatusCode::NOT_FOUND { - bail!("repology has no project named `{}`", recipe.name); - } - let entries: Vec = serde_json::from_slice(&resp.error_for_status()?.bytes()?) - .context("failed to parse repology response")?; - let latest = entries - .iter() - .filter(|e| matches!(e.status.as_str(), "newest" | "unique")) - .map(|e| e.version.as_str()) - .max_by(|a, b| natural_cmp(a, b)); - match latest { - Some(v) if natural_cmp(v, &recipe.version) == Ordering::Greater => Ok(Some(v.to_owned())), - _ => Ok(None), - } -} - -fn bump_recipe(path: &Path, new_version: &str) -> Result<()> { - let original = - fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; - let mut out = String::with_capacity(original.len()); - let mut version_replaced = false; - let mut revision_replaced = false; - for line in original.split_inclusive('\n') { - let trimmed = line.trim_start(); - if !version_replaced && trimmed.starts_with("version") { - if let Some(replaced) = replace_string_assignment(line, "version", new_version) { - out.push_str(&replaced); - version_replaced = true; - continue; - } - } - if !revision_replaced && trimmed.starts_with("revision") { - if let Some(replaced) = replace_int_assignment(line, "revision", 1) { - out.push_str(&replaced); - revision_replaced = true; - continue; - } - } - out.push_str(line); - } - if !version_replaced { - bail!("could not find `version = \"...\"` in {}", path.display()); - } - // Reset every sha256 placeholder so the next fetch re-derives it. - out = out.replace("\"sha256\": \"", "\x00sha256_marker\x00\""); - out = regex_lite_replace_sha(&out); - out = out.replace("\x00sha256_marker\x00\"", "\"sha256\": \""); - fs::write(path, out).with_context(|| format!("failed to write {}", path.display()))?; - Ok(()) -} - -/// Replace every quoted sha256 value with `"???"`, leaving keys/whitespace -/// alone. Avoids a full regex dep by walking the string by hand. -fn regex_lite_replace_sha(input: &str) -> String { - let mut out = String::with_capacity(input.len()); - let bytes = input.as_bytes(); - let needle = b"\x00sha256_marker\x00\""; - let mut i = 0; - while i < bytes.len() { - if bytes[i..].starts_with(needle) { - out.push_str("\x00sha256_marker\x00\"???\""); - i += needle.len(); - // skip the original value up to the closing quote - while i < bytes.len() && bytes[i] != b'"' { - i += 1; - } - if i < bytes.len() { - i += 1; // consume the closing quote we replaced - } - } else { - out.push(bytes[i] as char); - i += 1; - } - } - out -} - -fn replace_string_assignment(line: &str, key: &str, new_value: &str) -> Option { - // Match `="value"` while preserving surrounding text - // (indentation, trailing newline). - let stripped = line.strip_prefix(key)?; - let rest = stripped.trim_start_matches(|c: char| c == ' ' || c == '\t'); - let after_eq = rest.strip_prefix('=')?.trim_start(); - let after_quote = after_eq.strip_prefix('"')?; - let end = after_quote.find('"')?; - let trailing = &after_quote[end + 1..]; - Some(format!("{key} = \"{new_value}\"{trailing}")) -} - -fn replace_int_assignment(line: &str, key: &str, new_value: i32) -> Option { - let stripped = line.strip_prefix(key)?; - let rest = stripped.trim_start_matches(|c: char| c == ' ' || c == '\t'); - let after_eq = rest.strip_prefix('=')?.trim_start(); - let end = after_eq.find(|c: char| !c.is_ascii_digit())?; - let trailing = &after_eq[end..]; - Some(format!("{key} = {new_value}{trailing}")) -} - -/// dpkg-ish natural comparison: split into runs of digits and non-digits and -/// compare numerically where both sides are digits, lexicographically -/// otherwise. Good enough for upstream tarball versions. -fn natural_cmp(a: &str, b: &str) -> Ordering { - let mut ai = a.chars().peekable(); - let mut bi = b.chars().peekable(); - loop { - match (ai.peek().copied(), bi.peek().copied()) { - (None, None) => return Ordering::Equal, - (None, _) => return Ordering::Less, - (_, None) => return Ordering::Greater, - (Some(x), Some(y)) if x.is_ascii_digit() && y.is_ascii_digit() => { - let mut na = String::new(); - while let Some(&c) = ai.peek() { - if c.is_ascii_digit() { - na.push(c); - ai.next(); - } else { - break; - } - } - let mut nb = String::new(); - while let Some(&c) = bi.peek() { - if c.is_ascii_digit() { - nb.push(c); - bi.next(); - } else { - break; - } - } - let xa: u64 = na.parse().unwrap_or(0); - let xb: u64 = nb.parse().unwrap_or(0); - match xa.cmp(&xb) { - Ordering::Equal => continue, - other => return other, - } - } - (Some(x), Some(y)) => match x.cmp(&y) { - Ordering::Equal => { - ai.next(); - bi.next(); - } - other => return other, - }, - } - } -}