From 45d47e8d843bd7258a52a86a5e5b1dd79f1254bd Mon Sep 17 00:00:00 2001 From: Marvin Friedrich Date: Tue, 19 May 2026 03:24:10 +0200 Subject: [PATCH] wip3 --- Dockerfile | 2 +- host-recipes/binutils.star | 4 +- lib/common.star | 6 +- recipes/linux-headers.star | 8 +- recipes/musl.star | 4 +- src/builder.rs | 723 ++++++++++++++++++++++++++++++++++++- src/container.rs | 163 +++++++++ src/graph.rs | 371 ++++++------------- src/layout.rs | 131 +++++++ src/main.rs | 3 + src/phase.rs | 296 +++++++++++++++ src/recipe/mod.rs | 69 +++- 12 files changed, 1481 insertions(+), 299 deletions(-) create mode 100644 src/container.rs create mode 100644 src/layout.rs create mode 100644 src/phase.rs diff --git a/Dockerfile b/Dockerfile index 762d2c8..a6d50a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.22.4 +FROM alpine:edge RUN apk upgrade --no-cache && apk add --no-cache \ alpine-sdk \ diff --git a/host-recipes/binutils.star b/host-recipes/binutils.star index 299ad87..3f574b4 100644 --- a/host-recipes/binutils.star +++ b/host-recipes/binutils.star @@ -6,13 +6,13 @@ metadata = meta( ) source = tarball_source( url = "https://ftp.gnu.org/gnu/binutils/binutils-" + version + ".tar.xz", - sha256 = "?", + sha256 = "d75a94f4d73e7a4086f7513e67e439e8fcdcbb726ffe63f4661744e6256b2cf2", strip_components = 1, ) def configure(ctx): ctx.run([ - ctx.source_dir / "configure", + ctx.source_dir + "/configure", "--prefix=" + ctx.prefix, "--target=" + options.target_triple, "--with-sysroot=" + ctx.sysroot, diff --git a/lib/common.star b/lib/common.star index b7f42bb..779181c 100644 --- a/lib/common.star +++ b/lib/common.star @@ -8,7 +8,7 @@ def autotools_configure(ctx, extra_args = [], extra_env = {}): } env.update(extra_env) ctx.run([ - ctx.source_dir / "configure", + ctx.source_dir + "/configure", "--host=" + options.target_triple, "--with-sysroot=" + ctx.sysroot, "--prefix=" + ctx.prefix, @@ -25,9 +25,9 @@ def autotools_build(ctx, extra_args = []): ctx.run(["make", "-j" + str(ctx.jobs)] + extra_args) def autotools_install(ctx, pkg, extra_args = []): - ctx.run(["make", "install"] + extra_args, env = {"DESTDIR": pkg.destdir}) + ctx.run(["make", "install"] + extra_args, env = {"DESTDIR": pkg.dest_dir}) -def autotools(configure_args = [], configure_env = [], build_args = [], install_args = []): +def autotools(configure_args = [], configure_env = {}, build_args = [], install_args = []): def _configure(ctx): autotools_configure(ctx, extra_args = configure_args, extra_env = configure_env) def _build(ctx): diff --git a/recipes/linux-headers.star b/recipes/linux-headers.star index c09610d..04d83db 100644 --- a/recipes/linux-headers.star +++ b/recipes/linux-headers.star @@ -11,10 +11,10 @@ source = tarball_source( ) def build(ctx): - ctx.run(["cp", "-rp", ctx.source_dir / ".", ctx.build_dir]) + ctx.run(["cp", "-rp", ctx.source_dir + "/.", ctx.build_dir]) ctx.run(["make", "headers_install", "ARCH=" + options.target_arch]) - ctx.run(["find", ctx.build_dir / "usr/include", "-type", "f", "!", "-name", "*.h", "-delete"]) + ctx.run(["find", ctx.build_dir + "/usr/include", "-type", "f", "!", "-name", "*.h", "-delete"]) def install(ctx, pkg): - ctx.run(["mkdir", "-p", pkg.dest_dir / ctx.prefix]) - ctx.run(["cp", "-rp", ctx.build_dir / "usr/include", pkg.dest_dir / ctx.prefix]) + ctx.run(["mkdir", "-p", pkg.dest_dir + ctx.prefix]) + ctx.run(["cp", "-rp", ctx.build_dir + "/usr/include", pkg.dest_dir + ctx.prefix]) diff --git a/recipes/musl.star b/recipes/musl.star index 8a5d0b0..c796e82 100644 --- a/recipes/musl.star +++ b/recipes/musl.star @@ -6,14 +6,14 @@ metadata = meta( ) source = tarball_source( url = f"https://musl.libc.org/releases/musl-{version}.tar.gz", - sha256 = "?", + sha256 = "d585fd3b613c66151fc3249e8ed44f77020cb5e6c1e635a616d3f9f82460512a", strip_components = 1, ) host_deps = ["binutils", "gcc-bootstrap"] def configure(ctx): ctx.run([ - ctx.source_dir / "configure", + ctx.source_dir + "/configure", "--target=" + options.target_triple, "--prefix=" + ctx.prefix, "--syslibdir=/lib", diff --git a/src/builder.rs b/src/builder.rs index 1e8955a..c633f14 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1,17 +1,27 @@ use std::{ + cell::RefCell, + collections::{BTreeSet, VecDeque}, fs, + io::Write, path::{Path, PathBuf}, - process::Command, + process::{Command, Stdio}, + rc::Rc, + thread, + time::{SystemTime, UNIX_EPOCH}, }; use anyhow::{Context, bail}; use sha2::{Digest, Sha256}; +use starlark::values::OwnedFrozenValue; use crate::{ config::Config, - graph::{TaskPlan, TaskPlanner}, + container::{Container, Mount}, + graph::{TaskId, TaskPlan, TaskPlanner, task_recipe_slug}, + layout::Layout, log, - recipe::RecipeSet, + phase::{self, PackageContext, PhaseArg, PhaseContext, PhaseRuntime, PhaseRuntimeGuard, SourceDir}, + recipe::{GitSource, OutputPackage, Recipe, RecipeKind, RecipeSet, Source}, }; #[derive(Debug)] @@ -40,10 +50,10 @@ impl Builder { let plan = TaskPlanner::new(&self.root, &self.config.arch, recipes) .build_plan(requested, rebuild)?; self.print_plan(&plan); - if !dry_run { - bail!("task execution is not implemented yet"); + if dry_run { + return Ok(()); } - Ok(()) + self.execute_plan(recipes, &plan) } pub fn fetch( @@ -55,10 +65,10 @@ impl Builder { let plan = TaskPlanner::new(&self.root, &self.config.arch, recipes).fetch_plan(requested)?; self.print_plan(&plan); - if !dry_run { - bail!("task execution is not implemented yet"); + if dry_run { + return Ok(()); } - Ok(()) + self.execute_plan(recipes, &plan) } pub fn ensure_container_ready(&mut self) -> anyhow::Result<()> { @@ -70,6 +80,610 @@ impl Builder { Ok(()) } + fn execute_plan(&mut self, recipes: &RecipeSet, plan: &TaskPlan) -> anyhow::Result<()> { + if plan.is_empty() { + return Ok(()); + } + + let mut active: Option = None; + + for task in plan.order() { + let recipe_slug = task_recipe_slug(task, recipes)?; + + if let Some(current) = active.as_ref() { + if current.recipe_key != recipe_slug { + active = None; + } + } + + if task_needs_container(task) && active.is_none() { + self.ensure_container_ready()?; + active = Some(self.start_recipe_container(recipes, &recipe_slug)?); + } + + let layout = Layout::new(&self.root, &self.config.arch); + self.run_task(&layout, recipes, task, active.as_ref())?; + } + + Ok(()) + } + + fn run_task( + &self, + layout: &Layout<'_>, + recipes: &RecipeSet, + task: &TaskId, + active: Option<&ActiveContainer>, + ) -> anyhow::Result<()> { + match task { + TaskId::FetchSources(key) => { + let recipe = recipes.recipe(key)?; + self.task_fetch_sources(layout, recipe) + } + TaskId::PrepareSources(key) => { + let recipe = recipes.recipe(key)?; + self.task_prepare_sources(layout, recipe) + } + TaskId::ConfigureRecipe(key) => { + let recipe = recipes.recipe(key)?; + let active = active.expect("configure task requires an active container"); + self.task_configure(layout, recipe, active) + } + TaskId::BuildRecipe(key) => { + let recipe = recipes.recipe(key)?; + let active = active.expect("build task requires an active container"); + self.task_build(layout, recipe, active) + } + TaskId::InstallPackageFiles(output_key) => { + let output = recipes.output(output_key)?; + let recipe = recipes.recipe(output.recipe())?; + let active = active.expect("install task requires an active container"); + self.task_install_package(layout, recipe, output, active) + } + TaskId::ProduceApk(output_key) => { + let output = recipes.output(output_key)?; + let recipe = recipes.recipe(output.recipe())?; + let active = active.expect("apk task requires an active container"); + self.task_produce_apk(layout, recipe, output, active) + } + TaskId::InstallHostRecipe(key) => { + let recipe = recipes.recipe(key)?; + let active = active.expect("host install task requires an active container"); + self.task_install_host(layout, recipe, active) + } + } + } + + fn task_fetch_sources(&self, layout: &Layout<'_>, recipe: &Recipe) -> anyhow::Result<()> { + fs::create_dir_all(layout.source_cache_dir())?; + for (name, source) in recipe.sources().entries() { + let label = name.unwrap_or("source"); + if !source.is_unknown_cache_key() + && layout.source_cache_path(source.cache_key()).exists() + { + log::skip("fetch", &format!("{} ({})", recipe.key(), label)); + continue; + } + log::step( + "fetch", + &format!("{} ({}) from {}", recipe.key(), label, source.url()), + ); + match source { + Source::Tarball(_) => self.fetch_tarball(layout, recipe, source)?, + Source::Git(git) => self.fetch_git(layout, recipe, git)?, + } + } + Ok(()) + } + + fn fetch_tarball( + &self, + layout: &Layout<'_>, + recipe: &Recipe, + source: &Source, + ) -> anyhow::Result<()> { + let url = source.url(); + let expected = source.cache_key().to_owned(); + let unknown = source.is_unknown_cache_key(); + + let client = reqwest::blocking::Client::builder() + .user_agent("distro-builder") + .build() + .context("building HTTP client")?; + let mut response = client + .get(url) + .send() + .with_context(|| format!("downloading {url}"))? + .error_for_status() + .with_context(|| format!("downloading {url}"))?; + + let cache_dir = layout.source_cache_dir(); + fs::create_dir_all(&cache_dir)?; + let mut tmp = tempfile::NamedTempFile::new_in(&cache_dir) + .with_context(|| format!("creating temp file in {}", cache_dir.display()))?; + let mut hasher = Sha256::new(); + let mut buf = [0u8; 64 * 1024]; + loop { + let n = std::io::Read::read(&mut response, &mut buf) + .with_context(|| format!("reading response body for {url}"))?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + tmp.write_all(&buf[..n])?; + } + tmp.flush()?; + let hash = hex::encode(hasher.finalize()); + + if unknown { + log::info( + "fetch", + &format!("{}: computed sha256 = {hash}", recipe.key()), + ); + } else if hash != expected { + bail!("sha256 mismatch for {url}: expected {expected}, got {hash}"); + } + + let final_path = layout.source_cache_path(&hash); + if !final_path.exists() { + tmp.persist(&final_path).with_context(|| { + format!("renaming downloaded archive to {}", final_path.display()) + })?; + } + + if unknown { + bail!( + "{}: source sha256 is unknown; update the recipe with sha256 = \"{hash}\"", + recipe.key() + ); + } + Ok(()) + } + + fn fetch_git( + &self, + layout: &Layout<'_>, + recipe: &Recipe, + source: &GitSource, + ) -> anyhow::Result<()> { + let url = source.url(); + let commit = source.commit(); + let unknown = matches!(commit, "?" | "???"); + + let cache_dir = layout.source_cache_dir(); + fs::create_dir_all(&cache_dir)?; + let tmp = tempfile::tempdir_in(&cache_dir)?; + let work = tmp.path().join("repo"); + + let status = Command::new("git") + .arg("clone") + .arg("--bare") + .arg(url) + .arg(&work) + .status() + .with_context(|| format!("spawning git clone {url}"))?; + if !status.success() { + bail!("git clone failed for {url} with {status}"); + } + + let rev = if unknown { + let out = Command::new("git") + .arg("-C") + .arg(&work) + .arg("rev-parse") + .arg("HEAD") + .output() + .context("spawning git rev-parse HEAD")?; + if !out.status.success() { + bail!("git rev-parse HEAD failed with {}", out.status); + } + let rev = String::from_utf8(out.stdout)?.trim().to_owned(); + log::info( + "fetch", + &format!("{}: resolved git HEAD = {rev}", recipe.key()), + ); + rev + } else { + let status = Command::new("git") + .arg("-C") + .arg(&work) + .arg("cat-file") + .arg("-e") + .arg(commit) + .status() + .context("spawning git cat-file")?; + if !status.success() { + bail!("commit {commit} not found in {url}"); + } + commit.to_owned() + }; + + let final_path = layout.source_cache_path(&rev); + if !final_path.exists() { + fs::rename(&work, &final_path).with_context(|| { + format!( + "moving git clone to {} from {}", + final_path.display(), + work.display() + ) + })?; + } + + if unknown { + bail!( + "{}: source commit is unknown; update the recipe with commit = \"{rev}\"", + recipe.key() + ); + } + Ok(()) + } + + fn task_prepare_sources(&self, layout: &Layout<'_>, recipe: &Recipe) -> anyhow::Result<()> { + log::step("prepare", &recipe.key()); + let src = layout.source_workdir(recipe); + if src.exists() { + fs::remove_dir_all(&src).with_context(|| format!("clearing {}", src.display()))?; + } + fs::create_dir_all(&src)?; + let build = layout.build_workdir(recipe); + if build.exists() { + fs::remove_dir_all(&build).with_context(|| format!("clearing {}", build.display()))?; + } + fs::create_dir_all(&build)?; + + for (name, source) in recipe.sources().entries() { + if source.is_unknown_cache_key() { + bail!( + "source for {} has an unknown cache key; run `distro fetch` first", + recipe.key() + ); + } + let cache_path = layout.source_cache_path(source.cache_key()); + if !cache_path.exists() { + bail!( + "missing cached source for {} at {}", + recipe.key(), + cache_path.display() + ); + } + let dst = match name { + None => src.clone(), + Some(named) => { + let dst = src.join(named); + fs::create_dir_all(&dst)?; + dst + } + }; + match source { + Source::Tarball(tar) => { + let mut cmd = Command::new("tar"); + cmd.arg("-xf").arg(&cache_path); + if tar.strip_components() > 0 { + cmd.arg(format!("--strip-components={}", tar.strip_components())); + } + cmd.arg("-C").arg(&dst); + let status = cmd + .status() + .with_context(|| format!("spawning tar -xf {}", cache_path.display()))?; + if !status.success() { + bail!( + "tar extraction of {} failed with {status}", + cache_path.display() + ); + } + } + Source::Git(git) => { + let status = Command::new("git") + .arg("clone") + .arg(&cache_path) + .arg(&dst) + .status() + .context("spawning git clone from cache")?; + if !status.success() { + bail!("git clone from cache failed with {status}"); + } + let status = Command::new("git") + .arg("-C") + .arg(&dst) + .arg("checkout") + .arg(git.commit()) + .status() + .context("spawning git checkout")?; + if !status.success() { + bail!("git checkout {} failed with {status}", git.commit()); + } + } + } + } + + let version_stamp = layout.source_stamp(recipe, "version"); + fs::create_dir_all(version_stamp.parent().unwrap())?; + fs::write( + &version_stamp, + format!("{}-r{}", recipe.version(), recipe.revision()), + )?; + + if layout.recipe_has_patches(recipe)? { + log::skip( + "patches", + &format!("{}: patch application not implemented", recipe.key()), + ); + fs::write(layout.source_stamp(recipe, "patched"), b"skipped")?; + } + + Ok(()) + } + + fn task_configure( + &self, + layout: &Layout<'_>, + recipe: &Recipe, + active: &ActiveContainer, + ) -> anyhow::Result<()> { + log::step("configure", &recipe.key()); + if let Some(func) = recipe.phases().configure() { + let ctx = PhaseContext::new(source_dir_for(recipe), prefix_for(recipe.kind()), default_jobs()); + self.invoke_with_runtime(active, &[PhaseArg::Ctx(ctx)], func)?; + } + self.write_recipe_stamp(layout, recipe, "configure") + } + + fn task_build( + &self, + layout: &Layout<'_>, + recipe: &Recipe, + active: &ActiveContainer, + ) -> anyhow::Result<()> { + log::step("build", &recipe.key()); + let ctx = PhaseContext::new(source_dir_for(recipe), prefix_for(recipe.kind()), default_jobs()); + self.invoke_with_runtime(active, &[PhaseArg::Ctx(ctx)], recipe.phases().build())?; + self.write_recipe_stamp(layout, recipe, "build") + } + + fn task_install_package( + &self, + layout: &Layout<'_>, + recipe: &Recipe, + output: &OutputPackage, + active: &ActiveContainer, + ) -> anyhow::Result<()> { + log::step("install", &output.key()); + let dest = format!("/dest/{}", output.name()); + active.container.borrow().exec( + &["mkdir".to_owned(), "-p".to_owned(), dest.clone()], + &base_env(&active.base_path), + "/", + )?; + let ctx = PhaseContext::new(source_dir_for(recipe), prefix_for(recipe.kind()), default_jobs()); + let pkg = PackageContext::new(dest); + self.invoke_with_runtime( + active, + &[PhaseArg::Ctx(ctx), PhaseArg::Pkg(pkg)], + recipe.phases().install(), + )?; + self.write_output_stamp(layout, recipe, output, "install") + } + + fn task_install_host( + &self, + layout: &Layout<'_>, + recipe: &Recipe, + active: &ActiveContainer, + ) -> anyhow::Result<()> { + log::step("host-install", &recipe.key()); + let dest = format!("/dest/{}", recipe.slug()); + active.container.borrow().exec( + &["mkdir".to_owned(), "-p".to_owned(), dest.clone()], + &base_env(&active.base_path), + "/", + )?; + let ctx = PhaseContext::new(source_dir_for(recipe), prefix_for(recipe.kind()), default_jobs()); + let pkg = PackageContext::new(dest.clone()); + self.invoke_with_runtime( + active, + &[PhaseArg::Ctx(ctx), PhaseArg::Pkg(pkg)], + recipe.phases().install(), + )?; + + let host_root = layout.host_install_root(recipe); + if host_root.exists() { + fs::remove_dir_all(&host_root) + .with_context(|| format!("clearing {}", host_root.display()))?; + } + fs::create_dir_all(&host_root)?; + active + .container + .borrow() + .cp_out(&format!("{dest}/."), &host_root)?; + + self.write_recipe_stamp(layout, recipe, "host-install") + } + + fn task_produce_apk( + &self, + layout: &Layout<'_>, + recipe: &Recipe, + output: &OutputPackage, + active: &ActiveContainer, + ) -> anyhow::Result<()> { + log::step("apk", &output.key()); + let files_dir = format!("/dest/{}", output.name()); + let file_name = format!( + "{}-{}-r{}.apk", + output.name(), + recipe.version(), + recipe.revision() + ); + let out_in_container = format!("/pkgs/{file_name}"); + let version = format!("{}-r{}", recipe.version(), recipe.revision()); + + let mut argv: Vec = vec![ + "apk".to_owned(), + "mkpkg".to_owned(), + "--files".to_owned(), + files_dir, + "--output".to_owned(), + out_in_container, + "--info".to_owned(), + format!("name:{}", output.name()), + "--info".to_owned(), + format!("version:{version}"), + "--info".to_owned(), + format!("arch:{}", self.config.arch), + "--info".to_owned(), + format!("origin:{}", recipe.key()), + ]; + if let Some(desc) = output.metadata().description() { + argv.push("--info".to_owned()); + argv.push(format!("description:{desc}")); + } + if let Some(license) = output.metadata().license() { + argv.push("--info".to_owned()); + argv.push(format!("license:{license}")); + } + if let Some(url) = output.metadata().website() { + argv.push("--info".to_owned()); + argv.push(format!("url:{url}")); + } + if let Some(packager) = output.metadata().maintainer() { + argv.push("--info".to_owned()); + argv.push(format!("packager:{packager}")); + } + for dep in recipe.deps().iter().chain(recipe.run_deps().iter()) { + argv.push("--info".to_owned()); + argv.push(format!("depends:{dep}")); + } + + active + .container + .borrow() + .exec(&argv, &base_env(&active.base_path), "/")?; + + self.write_output_stamp(layout, recipe, output, "apk") + } + + fn invoke_with_runtime( + &self, + active: &ActiveContainer, + args: &[PhaseArg], + func: &OwnedFrozenValue, + ) -> anyhow::Result<()> { + let runtime = PhaseRuntime { + container: active.container.clone(), + base_path: active.base_path.clone(), + base_env: bare_env(), + }; + let _guard = PhaseRuntimeGuard::enter(runtime); + phase::invoke_phase(func, args) + } + + fn start_recipe_container( + &self, + recipes: &RecipeSet, + recipe_key: &str, + ) -> anyhow::Result { + let recipe = recipes.recipe(recipe_key)?; + let layout = Layout::new(&self.root, &self.config.arch); + + let source_dir = layout.source_workdir(recipe); + let build_dir = layout.build_workdir(recipe); + fs::create_dir_all(&source_dir)?; + fs::create_dir_all(&build_dir)?; + + let host_deps = transitive_host_deps(recipes, recipe)?; + let pkgs_dir = self.root.join("build/pkgs").join(&self.config.arch); + fs::create_dir_all(&pkgs_dir)?; + let mut mounts = vec![ + Mount { + host: source_dir, + container: "/sources".to_owned(), + read_only: false, + }, + Mount { + host: build_dir, + container: "/build".to_owned(), + read_only: false, + }, + Mount { + host: pkgs_dir, + container: "/pkgs".to_owned(), + read_only: false, + }, + ]; + let mut tools_bins: Vec = Vec::new(); + for dep_key in &host_deps { + let dep_recipe = recipes.recipe(dep_key)?; + let install = layout.host_install_dir(dep_recipe); + if !install.exists() { + bail!( + "missing host install for {dep_key} at {}; build it first", + install.display() + ); + } + let bare = dep_key.strip_prefix("host:").unwrap_or(dep_key); + mounts.push(Mount { + host: install, + container: format!("/tools/{bare}/usr/local"), + read_only: true, + }); + tools_bins.push(format!("/tools/{bare}/usr/local/bin")); + } + + let name = format!( + "distro-builder-{}-{:x}", + std::process::id(), + random_suffix() + ); + let container = Container::start( + &self.config.container_runtime, + &self.config.container_image, + &name, + &mounts, + )?; + + let mut path_segments = tools_bins; + path_segments.push("/usr/local/sbin".to_owned()); + path_segments.push("/usr/local/bin".to_owned()); + path_segments.push("/usr/sbin".to_owned()); + path_segments.push("/usr/bin".to_owned()); + path_segments.push("/sbin".to_owned()); + path_segments.push("/bin".to_owned()); + let base_path = path_segments.join(":"); + + let active = ActiveContainer { + recipe_key: recipe_key.to_owned(), + container: Rc::new(RefCell::new(container)), + base_path, + }; + + Ok(active) + } + + fn write_recipe_stamp( + &self, + layout: &Layout<'_>, + recipe: &Recipe, + kind: &str, + ) -> anyhow::Result<()> { + let path = layout.recipe_task_stamp(recipe, kind); + fs::create_dir_all(path.parent().unwrap())?; + fs::write(path, layout.recipe_fingerprint(recipe)?)?; + Ok(()) + } + + fn write_output_stamp( + &self, + layout: &Layout<'_>, + recipe: &Recipe, + output: &OutputPackage, + kind: &str, + ) -> anyhow::Result<()> { + let path = layout.output_task_stamp(output, kind); + fs::create_dir_all(path.parent().unwrap())?; + fs::write(path, layout.output_fingerprint(recipe, output)?)?; + Ok(()) + } + fn ensure_container_image(&self, dockerfile: &Path) -> anyhow::Result<()> { if !dockerfile.exists() { bail!( @@ -132,6 +746,8 @@ impl Builder { .arg("image") .arg("exists") .arg(&self.config.container_image) + .stdout(Stdio::null()) + .stderr(Stdio::null()) .status() .with_context(|| { format!( @@ -176,3 +792,92 @@ impl Builder { } } } + +struct ActiveContainer { + recipe_key: String, + container: Rc>, + base_path: String, +} + +fn task_needs_container(task: &TaskId) -> bool { + matches!( + task, + TaskId::ConfigureRecipe(_) + | TaskId::BuildRecipe(_) + | TaskId::InstallPackageFiles(_) + | TaskId::ProduceApk(_) + | TaskId::InstallHostRecipe(_) + ) +} + +fn prefix_for(kind: RecipeKind) -> &'static str { + match kind { + RecipeKind::Package => "/usr", + RecipeKind::HostPackage => "/usr/local", + } +} + +fn source_dir_for(recipe: &Recipe) -> SourceDir { + let entries = recipe.sources().entries(); + let named: Vec<(&str, &crate::recipe::Source)> = entries + .iter() + .filter_map(|(name, src)| name.map(|n| (n, *src))) + .collect(); + if named.is_empty() { + SourceDir::single("/sources") + } else { + SourceDir::named(named.into_iter().map(|(n, _)| (n.to_owned(), format!("/sources/{n}")))) + } +} + +fn default_jobs() -> i32 { + thread::available_parallelism() + .map(|n| n.get() as i32) + .unwrap_or(1) +} + +fn bare_env() -> Vec<(String, String)> { + vec![ + ("PATH".to_owned(), String::new()), + ("HOME".to_owned(), "/tmp".to_owned()), + ("TERM".to_owned(), "dumb".to_owned()), + ("LC_ALL".to_owned(), "C".to_owned()), + ] +} + +fn base_env(path: &str) -> Vec<(String, String)> { + let mut env = bare_env(); + if let Some(slot) = env.iter_mut().find(|(k, _)| k == "PATH") { + slot.1 = path.to_owned(); + } + env +} + +fn transitive_host_deps(recipes: &RecipeSet, recipe: &Recipe) -> anyhow::Result> { + let mut order: Vec = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + let mut queue: VecDeque = recipe + .host_deps() + .iter() + .map(|d| RecipeKind::HostPackage.key(d)) + .collect(); + while let Some(key) = queue.pop_front() { + if !seen.insert(key.clone()) { + continue; + } + let dep = recipes.recipe(&key)?; + for sub in dep.host_deps() { + queue.push_back(RecipeKind::HostPackage.key(sub)); + } + order.push(key); + } + Ok(order) +} + +fn random_suffix() -> u64 { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.subsec_nanos() as u64) + .unwrap_or(0); + nanos.wrapping_mul(0x9E3779B97F4A7C15) +} diff --git a/src/container.rs b/src/container.rs new file mode 100644 index 0000000..6af66b4 --- /dev/null +++ b/src/container.rs @@ -0,0 +1,163 @@ +use std::{ + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use anyhow::{Context, bail}; + +use crate::config::ContainerRuntime; + +#[derive(Debug, Clone)] +pub struct Mount { + pub host: PathBuf, + pub container: String, + pub read_only: bool, +} + +#[derive(Debug)] +pub struct Container { + runtime: &'static str, + id: String, + stopped: bool, +} + +impl Container { + pub fn start( + runtime: &ContainerRuntime, + image: &str, + name: &str, + mounts: &[Mount], + ) -> anyhow::Result { + let runtime_str = runtime.as_str(); + let mut cmd = Command::new(runtime_str); + cmd.arg("run") + .arg("-d") + .arg("--rm") + .arg("--name") + .arg(name) + .arg("--read-only") + .arg("--tmpfs") + .arg("/tmp") + .arg("--tmpfs") + .arg("/dest") + .arg("--tmpfs") + .arg("/sysroot") + .arg("--network=none"); + + if matches!(runtime, ContainerRuntime::Podman) { + cmd.arg("--userns=keep-id"); + } + + for mount in mounts { + let mut spec = format!("{}:{}", mount.host.display(), mount.container); + if mount.read_only { + spec.push_str(":ro"); + } + cmd.arg("-v").arg(spec); + } + + cmd.arg(image).arg("sleep").arg("infinity"); + cmd.stdout(Stdio::piped()).stderr(Stdio::inherit()); + + let output = cmd + .output() + .with_context(|| format!("spawning `{runtime_str} run` for image `{image}`"))?; + if !output.status.success() { + bail!( + "`{runtime_str} run` failed with {} for image `{image}`", + output.status + ); + } + + let id = String::from_utf8(output.stdout) + .context("container id is not valid UTF-8")? + .trim() + .to_owned(); + if id.is_empty() { + bail!("`{runtime_str} run` returned an empty container id"); + } + + Ok(Self { + runtime: runtime_str, + id, + stopped: false, + }) + } + + pub fn exec(&self, argv: &[String], env: &[(String, String)], cwd: &str) -> anyhow::Result<()> { + if argv.is_empty() { + bail!("ctx.run called with an empty argv"); + } + + let mut cmd = Command::new(self.runtime); + cmd.arg("exec").arg("-w").arg(cwd); + for (k, v) in env { + cmd.arg("-e").arg(format!("{k}={v}")); + } + cmd.arg(&self.id); + cmd.args(argv); + + let status = cmd.status().with_context(|| { + format!("spawning `{} exec` in container {}", self.runtime, self.id) + })?; + if !status.success() { + bail!("command {argv:?} failed with {status}"); + } + Ok(()) + } + + pub fn cp_out(&self, src_in_container: &str, host_dst: &Path) -> anyhow::Result<()> { + let spec = format!("{}:{}", self.id, src_in_container); + let status = Command::new(self.runtime) + .arg("cp") + .arg(spec) + .arg(host_dst) + .status() + .with_context(|| format!("spawning `{} cp`", self.runtime))?; + if !status.success() { + bail!( + "`{} cp` failed with {status} for {src_in_container} -> {}", + self.runtime, + host_dst.display() + ); + } + Ok(()) + } + + pub fn stop(mut self) -> anyhow::Result<()> { + self.stop_inner() + } + + fn stop_inner(&mut self) -> anyhow::Result<()> { + if self.stopped { + return Ok(()); + } + self.stopped = true; + let status = Command::new(self.runtime) + .arg("rm") + .arg("-f") + .arg(&self.id) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .with_context(|| format!("spawning `{} rm`", self.runtime))?; + if !status.success() { + bail!("`{} rm -f {}` failed with {status}", self.runtime, self.id); + } + Ok(()) + } +} + +impl Drop for Container { + fn drop(&mut self) { + if !self.stopped { + let _ = Command::new(self.runtime) + .arg("rm") + .arg("-f") + .arg(&self.id) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + } +} diff --git a/src/graph.rs b/src/graph.rs index efbb6f7..5f03611 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1,13 +1,15 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt, fs, - path::{Path, PathBuf}, + path::Path, }; -use anyhow::{Context, bail}; -use sha2::{Digest, Sha256}; +use anyhow::bail; -use crate::recipe::{OutputPackage, Recipe, RecipeKind, RecipeSet}; +use crate::{ + layout::Layout, + recipe::{OutputPackage, Recipe, RecipeKind, RecipeSet}, +}; #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] pub enum TaskId { @@ -60,8 +62,7 @@ impl TaskPlan { } pub struct TaskPlanner<'a> { - root: &'a Path, - arch: &'a str, + layout: Layout<'a>, recipes: &'a RecipeSet, force: bool, dependencies: BTreeMap>, @@ -73,8 +74,7 @@ pub struct TaskPlanner<'a> { impl<'a> TaskPlanner<'a> { pub fn new(root: &'a Path, arch: &'a str, recipes: &'a RecipeSet) -> Self { Self { - root, - arch, + layout: Layout::new(root, arch), recipes, force: false, dependencies: BTreeMap::new(), @@ -138,18 +138,7 @@ impl<'a> TaskPlanner<'a> { } fn into_plan(self) -> anyhow::Result { - let mut order = Vec::new(); - let mut visiting = BTreeSet::new(); - let mut visited = BTreeSet::new(); - for task in self.dependencies.keys() { - topo_visit( - task, - &self.dependencies, - &mut visiting, - &mut visited, - &mut order, - )?; - } + let order = recipe_contiguous_order(&self.dependencies, self.recipes)?; Ok(TaskPlan { dependencies: self.dependencies, order, @@ -218,7 +207,8 @@ impl<'a> TaskPlanner<'a> { TaskId::InstallPackageFiles(output) => { let output = self.recipes.output(output)?; let recipe = self.recipes.recipe(output.recipe())?; - self.output_task_active(recipe, output, "install") + Ok(self.output_task_active(recipe, output, "install")? + || self.produce_apk_active(recipe, output)?) } TaskId::ProduceApk(output) => { let output = self.recipes.output(output)?; @@ -234,7 +224,8 @@ impl<'a> TaskPlanner<'a> { fn fetch_sources_active(&self, recipe: &Recipe) -> anyhow::Result { Ok(recipe.sources().entries().iter().any(|(_, source)| { - source.is_unknown_cache_key() || !self.source_cache_path(source.cache_key()).exists() + source.is_unknown_cache_key() + || !self.layout.source_cache_path(source.cache_key()).exists() })) } @@ -243,14 +234,16 @@ impl<'a> TaskPlanner<'a> { return Ok(true); } let want_version = format!("{}-r{}", recipe.version(), recipe.revision()); - if fs::read_to_string(self.source_stamp(recipe, "version")) + if fs::read_to_string(self.layout.source_stamp(recipe, "version")) .ok() .as_deref() != Some(want_version.as_str()) { return Ok(true); } - if self.recipe_has_patches(recipe)? && !self.source_stamp(recipe, "patched").exists() { + if self.layout.recipe_has_patches(recipe)? + && !self.layout.source_stamp(recipe, "patched").exists() + { return Ok(true); } Ok(false) @@ -260,10 +253,12 @@ impl<'a> TaskPlanner<'a> { if self.force { return Ok(true); } - Ok(fs::read_to_string(self.recipe_task_stamp(recipe, kind)) - .ok() - .as_deref() - != Some(self.recipe_fingerprint(recipe)?.as_str())) + Ok( + fs::read_to_string(self.layout.recipe_task_stamp(recipe, kind)) + .ok() + .as_deref() + != Some(self.layout.recipe_fingerprint(recipe)?.as_str()), + ) } fn output_task_active( @@ -275,278 +270,116 @@ impl<'a> TaskPlanner<'a> { if self.force { return Ok(true); } - Ok(fs::read_to_string(self.output_task_stamp(output, kind)) - .ok() - .as_deref() - != Some(self.output_fingerprint(recipe, output)?.as_str())) + Ok( + fs::read_to_string(self.layout.output_task_stamp(output, kind)) + .ok() + .as_deref() + != Some(self.layout.output_fingerprint(recipe, output)?.as_str()), + ) } fn produce_apk_active(&self, recipe: &Recipe, output: &OutputPackage) -> anyhow::Result { if self.force { return Ok(true); } - if !self.apk_path(recipe, output).exists() { + if !self.layout.apk_path(recipe, output).exists() { return Ok(true); } - Ok(fs::read_to_string(self.output_task_stamp(output, "apk")) - .ok() - .as_deref() - != Some(self.output_fingerprint(recipe, output)?.as_str())) + Ok( + fs::read_to_string(self.layout.output_task_stamp(output, "apk")) + .ok() + .as_deref() + != Some(self.layout.output_fingerprint(recipe, output)?.as_str()), + ) } fn install_host_recipe_active(&self, recipe: &Recipe) -> anyhow::Result { if self.force { return Ok(true); } - if !self.host_install_dir(recipe).exists() { + if !self.layout.host_install_dir(recipe).exists() { return Ok(true); } Ok( - fs::read_to_string(self.recipe_task_stamp(recipe, "host-install")) + fs::read_to_string(self.layout.recipe_task_stamp(recipe, "host-install")) .ok() .as_deref() - != Some(self.recipe_fingerprint(recipe)?.as_str()), + != Some(self.layout.recipe_fingerprint(recipe)?.as_str()), ) } +} - fn recipe_fingerprint(&self, recipe: &Recipe) -> anyhow::Result { - let mut hasher = Sha256::new(); - hasher.update(self.arch.as_bytes()); - hasher.update(recipe.key().as_bytes()); - hasher.update(recipe.version().as_bytes()); - hasher.update(recipe.revision().to_le_bytes()); - hasher.update( - fs::read(recipe.path()) - .with_context(|| format!("reading recipe {}", recipe.path().display()))?, - ); - for (name, source) in recipe.sources().entries() { - hasher.update(name.unwrap_or("").as_bytes()); - hasher.update(source.url().as_bytes()); - hasher.update(source.cache_key().as_bytes()); +fn recipe_contiguous_order( + dependencies: &BTreeMap>, + recipes: &RecipeSet, +) -> anyhow::Result> { + let total = dependencies.len(); + let mut remaining: BTreeMap> = dependencies + .iter() + .map(|(task, deps)| (task.clone(), deps.iter().cloned().collect())) + .collect(); + + let mut dependents: BTreeMap> = BTreeMap::new(); + for (task, deps) in dependencies { + for dep in deps { + dependents + .entry(dep.clone()) + .or_default() + .push(task.clone()); } - for patch in self.recipe_patches(recipe)? { - hasher.update(patch.display().to_string().as_bytes()); - hasher - .update(fs::read(&patch).with_context(|| format!("reading {}", patch.display()))?); - } - Ok(hex::encode(hasher.finalize())) } - fn output_fingerprint( - &self, - recipe: &Recipe, - output: &OutputPackage, - ) -> anyhow::Result { - let mut hasher = Sha256::new(); - hasher.update(self.recipe_fingerprint(recipe)?.as_bytes()); - hasher.update(output.key().as_bytes()); - Ok(hex::encode(hasher.finalize())) - } - - fn recipe_has_patches(&self, recipe: &Recipe) -> anyhow::Result { - Ok(!self.recipe_patches(recipe)?.is_empty()) - } - - fn recipe_patches(&self, recipe: &Recipe) -> anyhow::Result> { - let patches_dir = recipe.dir().join("patches"); - if !patches_dir.exists() { - return Ok(Vec::new()); - } - let mut patches = Vec::new(); - for entry in fs::read_dir(&patches_dir) - .with_context(|| format!("reading patches directory {}", patches_dir.display()))? - { - let entry = entry?; - let path = entry.path(); - if path.is_file() { - patches.push(path); + let mut order = Vec::with_capacity(total); + let mut current_recipe: Option = None; + while order.len() < total { + let next = pick_next(&remaining, current_recipe.as_deref(), recipes)?; + let slug = task_recipe_slug(&next, recipes)?; + current_recipe = Some(slug); + remaining.remove(&next); + if let Some(children) = dependents.get(&next) { + for child in children { + if let Some(deps) = remaining.get_mut(child) { + deps.remove(&next); + } } } - patches.sort(); - Ok(patches) - } - - fn source_cache_path(&self, key: &str) -> PathBuf { - self.root.join("build/cache/sources").join(key) - } - - fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf { - self.root - .join("build/sources") - .join(format!("{}.{kind}", recipe.slug())) - } - - fn recipe_task_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf { - self.root - .join("build/tasks") - .join(format!("{}.{kind}", recipe.slug())) - } - - fn output_task_stamp(&self, output: &OutputPackage, kind: &str) -> PathBuf { - self.root - .join("build/tasks") - .join(format!("{}.{kind}", output.key().replace(':', "-"))) - } - - fn apk_path(&self, recipe: &Recipe, output: &OutputPackage) -> PathBuf { - self.root.join("build/pkgs").join(self.arch).join(format!( - "{}-{}-r{}.apk", - output.name(), - recipe.version(), - recipe.revision() - )) - } - - fn host_install_dir(&self, recipe: &Recipe) -> PathBuf { - self.root - .join("build/host-pkgs") - .join(recipe.slug()) - .join("usr/local") + order.push(next); } + Ok(order) } -fn topo_visit( - task: &TaskId, - dependencies: &BTreeMap>, - visiting: &mut BTreeSet, - visited: &mut BTreeSet, - order: &mut Vec, -) -> anyhow::Result<()> { - if visited.contains(task) { - return Ok(()); +fn pick_next( + remaining: &BTreeMap>, + current_recipe: Option<&str>, + recipes: &RecipeSet, +) -> anyhow::Result { + let mut fallback: Option = None; + for (task, deps) in remaining { + if !deps.is_empty() { + continue; + } + if let Some(active) = current_recipe { + let slug = task_recipe_slug(task, recipes)?; + if slug == active { + return Ok(task.clone()); + } + } + if fallback.is_none() { + fallback = Some(task.clone()); + } } - if !visiting.insert(task.clone()) { - bail!("task dependency cycle involving `{task}`"); - } - for dependency in dependencies.get(task).into_iter().flatten() { - topo_visit(dependency, dependencies, visiting, visited, order)?; - } - visiting.remove(task); - visited.insert(task.clone()); - order.push(task.clone()); - Ok(()) + fallback.ok_or_else(|| anyhow::anyhow!("task dependency cycle detected")) } -#[cfg(test)] -mod tests { - use std::fs; - - use tempfile::TempDir; - - use crate::{config::Config, eval, recipe::RecipeSet}; - - use super::{TaskId, TaskPlanner}; - - fn write_config(root: &TempDir) { - fs::write( - root.path().join("config.star"), - r#" -container_runtime = "podman" -container_image = "local/test:latest" -container_dockerfile = "Dockerfile" -arch = "x86_64" -options = dict(target_arch = arch) -"#, - ) - .unwrap(); - fs::write(root.path().join("Dockerfile"), "FROM scratch\n").unwrap(); - } - - fn write_recipe(root: &TempDir, dir: &str, name: &str, extra: &str) { - let recipe_dir = root.path().join(dir); - fs::create_dir_all(&recipe_dir).unwrap(); - fs::write( - recipe_dir.join(format!("{name}.star")), - format!( - r#" -version = "1.0" -revision = 1 -source = tarball_source(url = "file:///tmp/{name}.tar", sha256 = "hash-{name}") -{extra} -def build(ctx): - ctx.run(["true"]) -def install(ctx, pkg): - ctx.run(["true"]) -"# - ), - ) - .unwrap(); - } - - fn load(root: &TempDir) -> (Config, RecipeSet) { - let config = Config::load(&root.path().join("config.star")).unwrap(); - let lib = eval::eval_lib(&root.path().join("lib"), Some(&config.options)).unwrap(); - let recipes = RecipeSet::load(root.path(), &config.options, lib.as_ref()).unwrap(); - (config, recipes) - } - - #[test] - fn inactive_seed_does_not_pull_dependencies() { - let root = TempDir::new().unwrap(); - write_config(&root); - write_recipe(&root, "recipes", "dep", ""); - write_recipe(&root, "recipes", "app", r#"deps = ["dep"]"#); - let (config, recipes) = load(&root); - - let planner = TaskPlanner::new(root.path(), &config.arch, &recipes); - let output = recipes.output("app").unwrap(); - let recipe = recipes.recipe(output.recipe()).unwrap(); - fs::create_dir_all(root.path().join("build/pkgs/x86_64")).unwrap(); - fs::write(root.path().join("build/pkgs/x86_64/app-1.0-r1.apk"), "").unwrap(); - fs::create_dir_all(root.path().join("build/tasks")).unwrap(); - fs::write( - root.path().join("build/tasks/app.apk"), - planner.output_fingerprint(recipe, output).unwrap(), - ) - .unwrap(); - - let plan = TaskPlanner::new(root.path(), &config.arch, &recipes) - .build_plan(&["app".to_owned()], false) - .unwrap(); - assert!(plan.is_empty()); - } - - #[test] - fn active_target_recipe_keeps_edges_and_topo_order() { - let root = TempDir::new().unwrap(); - write_config(&root); - write_recipe(&root, "recipes", "app", ""); - let (config, recipes) = load(&root); - - let plan = TaskPlanner::new(root.path(), &config.arch, &recipes) - .build_plan(&["app".to_owned()], false) - .unwrap(); - assert_eq!( - plan.order(), - &[ - TaskId::FetchSources("app".to_owned()), - TaskId::PrepareSources("app".to_owned()), - TaskId::ConfigureRecipe("app".to_owned()), - TaskId::BuildRecipe("app".to_owned()), - TaskId::InstallPackageFiles("app".to_owned()), - TaskId::ProduceApk("app".to_owned()), - ] - ); - assert_eq!( - plan.dependencies(&TaskId::ProduceApk("app".to_owned())), - Some([TaskId::InstallPackageFiles("app".to_owned())].as_slice()) - ); - } - - #[test] - fn host_dependencies_are_installed_as_host_recipes() { - let root = TempDir::new().unwrap(); - write_config(&root); - write_recipe(&root, "host-recipes", "binutils", ""); - write_recipe(&root, "recipes", "app", r#"host_deps = ["binutils"]"#); - let (config, recipes) = load(&root); - - let plan = TaskPlanner::new(root.path(), &config.arch, &recipes) - .build_plan(&["app".to_owned()], false) - .unwrap(); - assert!( - plan.order() - .contains(&TaskId::InstallHostRecipe("host:binutils".to_owned())) - ); - } +pub fn task_recipe_slug(task: &TaskId, recipes: &RecipeSet) -> anyhow::Result { + Ok(match task { + TaskId::FetchSources(recipe) + | TaskId::PrepareSources(recipe) + | TaskId::ConfigureRecipe(recipe) + | TaskId::BuildRecipe(recipe) + | TaskId::InstallHostRecipe(recipe) => recipe.clone(), + TaskId::InstallPackageFiles(output) | TaskId::ProduceApk(output) => { + recipes.output(output)?.recipe().to_owned() + } + }) } diff --git a/src/layout.rs b/src/layout.rs new file mode 100644 index 0000000..35524b8 --- /dev/null +++ b/src/layout.rs @@ -0,0 +1,131 @@ +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use anyhow::Context; +use sha2::{Digest, Sha256}; + +use crate::recipe::{OutputPackage, Recipe}; + +pub struct Layout<'a> { + pub root: &'a Path, + pub arch: &'a str, +} + +impl<'a> Layout<'a> { + pub fn new(root: &'a Path, arch: &'a str) -> Self { + Self { root, arch } + } + + pub fn source_cache_dir(&self) -> PathBuf { + self.root.join("build/cache/sources") + } + + pub fn source_cache_path(&self, key: &str) -> PathBuf { + self.source_cache_dir().join(key) + } + + pub fn source_workdir(&self, recipe: &Recipe) -> PathBuf { + self.root.join("build/sources").join(recipe.slug()) + } + + pub fn build_workdir(&self, recipe: &Recipe) -> PathBuf { + self.root.join("build/builds").join(recipe.slug()) + } + + pub fn host_install_dir(&self, recipe: &Recipe) -> PathBuf { + self.root + .join("build/host-pkgs") + .join(recipe.slug()) + .join("usr/local") + } + + pub fn host_install_root(&self, recipe: &Recipe) -> PathBuf { + self.root.join("build/host-pkgs").join(recipe.slug()) + } + + pub fn apk_path(&self, recipe: &Recipe, output: &OutputPackage) -> PathBuf { + self.root.join("build/pkgs").join(self.arch).join(format!( + "{}-{}-r{}.apk", + output.name(), + recipe.version(), + recipe.revision() + )) + } + + pub fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf { + self.root + .join("build/sources") + .join(format!("{}.{kind}", recipe.slug())) + } + + pub fn recipe_task_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf { + self.root + .join("build/tasks") + .join(format!("{}.{kind}", recipe.slug())) + } + + pub fn output_task_stamp(&self, output: &OutputPackage, kind: &str) -> PathBuf { + self.root + .join("build/tasks") + .join(format!("{}.{kind}", output.key().replace(':', "-"))) + } + + pub fn recipe_fingerprint(&self, recipe: &Recipe) -> anyhow::Result { + let mut hasher = Sha256::new(); + hasher.update(self.arch.as_bytes()); + hasher.update(recipe.key().as_bytes()); + hasher.update(recipe.version().as_bytes()); + hasher.update(recipe.revision().to_le_bytes()); + hasher.update( + fs::read(recipe.path()) + .with_context(|| format!("reading recipe {}", recipe.path().display()))?, + ); + for (name, source) in recipe.sources().entries() { + hasher.update(name.unwrap_or("").as_bytes()); + hasher.update(source.url().as_bytes()); + hasher.update(source.cache_key().as_bytes()); + } + for patch in self.recipe_patches(recipe)? { + hasher.update(patch.display().to_string().as_bytes()); + hasher + .update(fs::read(&patch).with_context(|| format!("reading {}", patch.display()))?); + } + Ok(hex::encode(hasher.finalize())) + } + + pub fn output_fingerprint( + &self, + recipe: &Recipe, + output: &OutputPackage, + ) -> anyhow::Result { + let mut hasher = Sha256::new(); + hasher.update(self.recipe_fingerprint(recipe)?.as_bytes()); + hasher.update(output.key().as_bytes()); + Ok(hex::encode(hasher.finalize())) + } + + pub fn recipe_has_patches(&self, recipe: &Recipe) -> anyhow::Result { + Ok(!self.recipe_patches(recipe)?.is_empty()) + } + + pub fn recipe_patches(&self, recipe: &Recipe) -> anyhow::Result> { + let patches_dir = recipe.dir().join("patches"); + if !patches_dir.exists() { + return Ok(Vec::new()); + } + let mut patches = Vec::new(); + for entry in fs::read_dir(&patches_dir) + .with_context(|| format!("reading patches directory {}", patches_dir.display()))? + { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + patches.push(path); + } + } + patches.sort(); + Ok(patches) + } +} diff --git a/src/main.rs b/src/main.rs index 6fa9cdb..e8833eb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,13 @@ mod builder; mod cli; mod config; +mod container; mod eval; mod graph; +mod layout; mod log; mod options; +mod phase; mod recipe; fn main() -> anyhow::Result<()> { diff --git a/src/phase.rs b/src/phase.rs new file mode 100644 index 0000000..1544c1a --- /dev/null +++ b/src/phase.rs @@ -0,0 +1,296 @@ +use std::{cell::RefCell, collections::BTreeMap, rc::Rc}; + +use allocative::Allocative; +use anyhow::anyhow; +use starlark::{ + collections::SmallMap, + environment::{Methods, MethodsBuilder, MethodsStatic, Module}, + eval::Evaluator, + values::{Heap, OwnedFrozenValue, StarlarkValue, Value, list::UnpackList, none::NoneType}, +}; +use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_module, starlark_value}; + +use crate::container::Container; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[derive(Clone)] +pub struct PhaseRuntime { + pub container: Rc>, + pub base_path: String, + pub base_env: Vec<(String, String)>, +} + +pub struct PhaseRuntimeGuard; + +impl PhaseRuntimeGuard { + pub fn enter(runtime: PhaseRuntime) -> Self { + CURRENT.with(|cell| { + let prev = cell.borrow_mut().replace(runtime); + assert!(prev.is_none(), "phase runtime already set"); + }); + Self + } +} + +impl Drop for PhaseRuntimeGuard { + fn drop(&mut self) { + CURRENT.with(|cell| { + cell.borrow_mut().take(); + }); + } +} + +fn with_current(f: impl FnOnce(&PhaseRuntime) -> R) -> anyhow::Result { + CURRENT.with(|cell| { + let borrow = cell.borrow(); + let runtime = borrow + .as_ref() + .ok_or_else(|| anyhow!("ctx.run called outside of a phase invocation"))?; + Ok(f(runtime)) + }) +} + +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub struct SourceDir { + default: String, + entries: BTreeMap, +} + +impl SourceDir { + pub fn single(path: impl Into) -> Self { + Self { + default: path.into(), + entries: BTreeMap::new(), + } + } + + pub fn named(entries: I) -> Self + where + I: IntoIterator, + K: Into, + V: Into, + { + let entries: BTreeMap = entries + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(); + let default = entries + .values() + .next() + .cloned() + .unwrap_or_else(|| "/sources".to_owned()); + Self { default, entries } + } +} + +impl std::fmt::Display for SourceDir { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.default) + } +} + +starlark::starlark_simple_value!(SourceDir); + +#[starlark_value(type = "source_dir")] +impl<'v> StarlarkValue<'v> for SourceDir { + fn at(&self, index: Value<'v>, heap: &'v Heap) -> starlark::Result> { + let key = index.unpack_str().ok_or_else(|| { + starlark::Error::new_other(anyhow!("source_dir index must be a string")) + })?; + let path = self.entries.get(key).ok_or_else(|| { + starlark::Error::new_other(anyhow!( + "no source named `{key}` (available: {})", + if self.entries.is_empty() { + "".to_owned() + } else { + self.entries + .keys() + .map(String::as_str) + .collect::>() + .join(", ") + } + )) + })?; + Ok(heap.alloc(path.as_str())) + } + + fn add(&self, rhs: Value<'v>, heap: &'v Heap) -> Option>> { + let suffix = rhs.unpack_str()?; + Some(Ok(heap.alloc(format!("{}{}", self.default, suffix)))) + } + + fn radd(&self, lhs: Value<'v>, heap: &'v Heap) -> Option>> { + let prefix = lhs.unpack_str()?; + Some(Ok(heap.alloc(format!("{}{}", prefix, self.default)))) + } +} + +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub struct PhaseContext { + source_dir: SourceDir, + build_dir: String, + prefix: String, + sysroot: String, + jobs: i32, +} + +impl PhaseContext { + pub fn new(source_dir: SourceDir, prefix: &str, jobs: i32) -> Self { + Self { + source_dir, + build_dir: "/build".to_owned(), + prefix: prefix.to_owned(), + sysroot: "/sysroot".to_owned(), + jobs, + } + } +} + +impl std::fmt::Display for PhaseContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ctx") + } +} + +starlark::starlark_simple_value!(PhaseContext); + +#[starlark_value(type = "phase_context")] +impl<'v> StarlarkValue<'v> for PhaseContext { + fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option> { + Some(match attr { + "source_dir" => heap.alloc(self.source_dir.clone()), + "build_dir" => heap.alloc(self.build_dir.as_str()), + "prefix" => heap.alloc(self.prefix.as_str()), + "sysroot" => heap.alloc(self.sysroot.as_str()), + "jobs" => heap.alloc(self.jobs), + _ => return None, + }) + } + + fn has_attr(&self, attr: &str, _heap: &'v Heap) -> bool { + matches!( + attr, + "source_dir" | "build_dir" | "prefix" | "sysroot" | "jobs" | "run" + ) + } + + fn dir_attr(&self) -> Vec { + [ + "source_dir", + "build_dir", + "prefix", + "sysroot", + "jobs", + "run", + ] + .into_iter() + .map(String::from) + .collect() + } + + fn get_methods() -> Option<&'static Methods> { + static RES: MethodsStatic = MethodsStatic::new(); + RES.methods(phase_context_methods) + } +} + +#[starlark_module] +fn phase_context_methods(builder: &mut MethodsBuilder) { + fn run<'v>( + #[starlark(this)] _this: Value<'v>, + #[starlark(require = pos)] argv: UnpackList, + #[starlark(require = named)] env: Option>, + ) -> anyhow::Result { + run_in_container(&argv.items, env.unwrap_or_default())?; + Ok(NoneType) + } +} + +fn run_in_container( + argv: &[String], + env_overrides: SmallMap, +) -> anyhow::Result<()> { + let (container, env) = with_current(|runtime| { + let mut env: Vec<(String, String)> = runtime + .base_env + .iter() + .cloned() + .map(|(k, v)| { + if k == "PATH" { + (k, runtime.base_path.clone()) + } else { + (k, v) + } + }) + .collect(); + for (k, v) in env_overrides { + if let Some(slot) = env.iter_mut().find(|(existing, _)| existing == &k) { + slot.1 = v; + } else { + env.push((k, v)); + } + } + (runtime.container.clone(), env) + })?; + container.borrow().exec(argv, &env, "/build") +} + +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub struct PackageContext { + dest_dir: String, +} + +impl PackageContext { + pub fn new(dest_dir: String) -> Self { + Self { dest_dir } + } +} + +impl std::fmt::Display for PackageContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "pkg") + } +} + +starlark::starlark_simple_value!(PackageContext); + +#[starlark_value(type = "package_context")] +impl<'v> StarlarkValue<'v> for PackageContext { + fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option> { + match attr { + "dest_dir" => Some(heap.alloc(self.dest_dir.as_str())), + _ => None, + } + } + + fn has_attr(&self, attr: &str, _heap: &'v Heap) -> bool { + attr == "dest_dir" + } + + fn dir_attr(&self) -> Vec { + vec!["dest_dir".to_owned()] + } +} + +pub fn invoke_phase(func: &OwnedFrozenValue, args: &[PhaseArg]) -> anyhow::Result<()> { + let module = Module::new(); + let mut eval = Evaluator::new(&module); + let allocated: Vec> = args + .iter() + .map(|arg| match arg { + PhaseArg::Ctx(ctx) => module.heap().alloc(ctx.clone()), + PhaseArg::Pkg(pkg) => module.heap().alloc(pkg.clone()), + }) + .collect(); + eval.eval_function(func.value(), &allocated, &[]) + .map_err(|err| anyhow!("{err}"))?; + Ok(()) +} + +pub enum PhaseArg { + Ctx(PhaseContext), + Pkg(PackageContext), +} diff --git a/src/recipe/mod.rs b/src/recipe/mod.rs index 6b8f57a..df20b98 100644 --- a/src/recipe/mod.rs +++ b/src/recipe/mod.rs @@ -5,7 +5,10 @@ mod subpackage; use anyhow::{Context, bail}; use starlark::{ environment::{FrozenModule, Module}, - values::{OwnedFrozenValue, UnpackValue, ValueLike, list::ListRef, typing::StarlarkCallable}, + values::{ + OwnedFrozenValue, UnpackValue, ValueLike, dict::DictRef, list::ListRef, + typing::StarlarkCallable, + }, }; use std::{ collections::HashMap, @@ -141,14 +144,47 @@ impl Recipe { .clone(), }; - let source_value = module - .get("source") - .ok_or_else(|| anyhow::anyhow!("field `source`: missing"))?; - let source = source_value - .downcast_ref::() - .ok_or_else(|| anyhow::anyhow!("field `source`: expected a source value"))? - .clone(); - let sources = Sources::Single(source); + let source_value = module.get("source"); + let sources_value = module.get("sources"); + let sources = match (source_value, sources_value) { + (None, None) => bail!("recipe must define either `source` or `sources`"), + (Some(_), Some(_)) => { + bail!("recipe must define exactly one of `source` or `sources`, not both") + } + (Some(value), None) => { + let source = value + .downcast_ref::() + .ok_or_else(|| anyhow::anyhow!("field `source`: expected a source value"))? + .clone(); + Sources::Single(source) + } + (None, Some(value)) => { + let dict = DictRef::from_value(value).ok_or_else(|| { + anyhow::anyhow!("field `sources`: expected a dict of name -> source") + })?; + if dict.iter().len() == 0 { + bail!("field `sources`: must contain at least one entry"); + } + let mut map: HashMap = HashMap::new(); + for (key, value) in dict.iter() { + let key_str = key + .unpack_str() + .ok_or_else(|| anyhow::anyhow!("field `sources`: keys must be strings"))?; + if !is_valid_source_name(key_str) { + bail!( + "field `sources`: invalid source name `{key_str}` (allowed: letters, digits, `-`, `_`; must start with a letter or digit)" + ); + } + let source = value.downcast_ref::().ok_or_else(|| { + anyhow::anyhow!("field `sources`: entry `{key_str}` is not a source value") + })?; + if map.insert(key_str.to_owned(), source.clone()).is_some() { + bail!("field `sources`: duplicate key `{key_str}`"); + } + } + Sources::Multiple(map) + } + }; let host_deps = optional_string_list(&module, "host_deps")?; let build_deps = optional_string_list(&module, "build_deps")?; @@ -275,6 +311,17 @@ fn optional_string_list(module: &Module, key: &str) -> anyhow::Result bool { + let mut chars = name.chars(); + let Some(first) = chars.next() else { + return false; + }; + if !first.is_ascii_alphanumeric() { + return false; + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') +} + pub struct RecipePhases { configure: Option, build: OwnedFrozenValue, @@ -368,6 +415,10 @@ impl OutputPackage { pub fn name(&self) -> &str { &self.name } + + pub fn metadata(&self) -> &Metadata { + &self.metadata + } } #[derive(Debug)]