idk

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.22.4
+FROM alpine:edge
 
 RUN apk upgrade --no-cache && apk add --no-cache \
     alpine-sdk \

host-recipes/binutils.star

@@ -6,13 +6,13 @@ metadata = meta(
 )
 source = tarball_source(
     url = "https://ftp.gnu.org/gnu/binutils/binutils-" + version + ".tar.xz",
-    sha256 = "?",
+    sha256 = "d75a94f4d73e7a4086f7513e67e439e8fcdcbb726ffe63f4661744e6256b2cf2",
     strip_components = 1,
 )
 
 def configure(ctx):
     ctx.run([
-        ctx.source_dir / "configure",
+        ctx.source_dir + "/configure",
         "--prefix=" + ctx.prefix,
         "--target=" + options.target_triple,
         "--with-sysroot=" + ctx.sysroot,

lib/common.star

@@ -8,7 +8,7 @@ def autotools_configure(ctx, extra_args = [], extra_env = {}):
     }
     env.update(extra_env)
     ctx.run([
-        ctx.source_dir / "configure",
+        ctx.source_dir + "/configure",
         "--host=" + options.target_triple,
         "--with-sysroot=" + ctx.sysroot,
         "--prefix=" + ctx.prefix,
@@ -25,9 +25,9 @@ def autotools_build(ctx, extra_args = []):
     ctx.run(["make", "-j" + str(ctx.jobs)] + extra_args)
 
 def autotools_install(ctx, pkg, extra_args = []):
-    ctx.run(["make", "install"] + extra_args, env = {"DESTDIR": pkg.destdir})
+    ctx.run(["make", "install"] + extra_args, env = {"DESTDIR": pkg.dest_dir})
 
-def autotools(configure_args = [], configure_env = [], build_args = [], install_args = []):
+def autotools(configure_args = [], configure_env = {}, build_args = [], install_args = []):
     def _configure(ctx):
         autotools_configure(ctx, extra_args = configure_args, extra_env = configure_env)
     def _build(ctx):

recipes/linux-headers.star

@@ -11,10 +11,10 @@ source = tarball_source(
 )
 
 def build(ctx):
-    ctx.run(["cp", "-rp", ctx.source_dir / ".", ctx.build_dir])
+    ctx.run(["cp", "-rp", ctx.source_dir + "/.", ctx.build_dir])
     ctx.run(["make", "headers_install", "ARCH=" + options.target_arch])
-    ctx.run(["find", ctx.build_dir / "usr/include", "-type", "f", "!", "-name", "*.h", "-delete"])
+    ctx.run(["find", ctx.build_dir + "/usr/include", "-type", "f", "!", "-name", "*.h", "-delete"])
 
 def install(ctx, pkg):
-    ctx.run(["mkdir", "-p", pkg.dest_dir / ctx.prefix])
+    ctx.run(["mkdir", "-p", pkg.dest_dir + ctx.prefix])
-    ctx.run(["cp", "-rp", ctx.build_dir / "usr/include", pkg.dest_dir / ctx.prefix])
+    ctx.run(["cp", "-rp", ctx.build_dir + "/usr/include", pkg.dest_dir + ctx.prefix])

recipes/musl.star

@@ -6,14 +6,14 @@ metadata = meta(
 )
 source = tarball_source(
     url = f"https://musl.libc.org/releases/musl-{version}.tar.gz",
-    sha256 = "?",
+    sha256 = "d585fd3b613c66151fc3249e8ed44f77020cb5e6c1e635a616d3f9f82460512a",
     strip_components = 1,
 )
 host_deps = ["binutils", "gcc-bootstrap"]
 
 def configure(ctx):
     ctx.run([
-        ctx.source_dir / "configure",
+        ctx.source_dir + "/configure",
         "--target=" + options.target_triple,
         "--prefix=" + ctx.prefix,
         "--syslibdir=/lib",

src/builder.rs

@@ -1,17 +1,29 @@
 use std::{
+    cell::RefCell,
+    collections::{BTreeSet, VecDeque},
     fs,
+    io::Write,
     path::{Path, PathBuf},
-    process::Command,
+    process::{Command, Stdio},
+    rc::Rc,
+    thread,
+    time::{SystemTime, UNIX_EPOCH},
 };
 
 use anyhow::{Context, bail};
 use sha2::{Digest, Sha256};
+use starlark::values::OwnedFrozenValue;
 
 use crate::{
     config::Config,
-    graph::{TaskPlan, TaskPlanner},
+    container::{Container, Mount},
+    graph::{TaskId, TaskPlan, TaskPlanner, task_recipe_slug},
+    layout::Layout,
     log,
-    recipe::RecipeSet,
+    phase::{
+        self, PackageContext, PhaseArg, PhaseContext, PhaseRuntime, PhaseRuntimeGuard, SourceDir,
+    },
+    recipe::{GitSource, OutputPackage, Recipe, RecipeKind, RecipeSet, Source},
 };
 
 #[derive(Debug)]
@@ -40,10 +52,10 @@ impl Builder {
         let plan = TaskPlanner::new(&self.root, &self.config.arch, recipes)
             .build_plan(requested, rebuild)?;
         self.print_plan(&plan);
-        if !dry_run {
+        if dry_run {
-            bail!("task execution is not implemented yet");
+            return Ok(());
         }
-        Ok(())
+        self.execute_plan(recipes, &plan)
     }
 
     pub fn fetch(
@@ -55,10 +67,10 @@ impl Builder {
         let plan =
             TaskPlanner::new(&self.root, &self.config.arch, recipes).fetch_plan(requested)?;
         self.print_plan(&plan);
-        if !dry_run {
+        if dry_run {
-            bail!("task execution is not implemented yet");
+            return Ok(());
         }
-        Ok(())
+        self.execute_plan(recipes, &plan)
     }
 
     pub fn ensure_container_ready(&mut self) -> anyhow::Result<()> {
@@ -70,6 +82,626 @@ impl Builder {
         Ok(())
     }
 
+    fn execute_plan(&mut self, recipes: &RecipeSet, plan: &TaskPlan) -> anyhow::Result<()> {
+        if plan.is_empty() {
+            return Ok(());
+        }
+
+        let mut active: Option<ActiveContainer> = None;
+
+        for task in plan.order() {
+            let recipe_slug = task_recipe_slug(task, recipes)?;
+
+            if let Some(current) = active.as_ref() {
+                if current.recipe_key != recipe_slug {
+                    active = None;
+                }
+            }
+
+            if task_needs_container(task) && active.is_none() {
+                self.ensure_container_ready()?;
+                active = Some(self.start_recipe_container(recipes, &recipe_slug)?);
+            }
+
+            let layout = Layout::new(&self.root, &self.config.arch);
+            self.run_task(&layout, recipes, task, active.as_ref())?;
+        }
+
+        Ok(())
+    }
+
+    fn run_task(
+        &self,
+        layout: &Layout<'_>,
+        recipes: &RecipeSet,
+        task: &TaskId,
+        active: Option<&ActiveContainer>,
+    ) -> anyhow::Result<()> {
+        match task {
+            TaskId::FetchSources(key) => {
+                let recipe = recipes.recipe(key)?;
+                self.task_fetch_sources(layout, recipe)
+            }
+            TaskId::PrepareSources(key) => {
+                let recipe = recipes.recipe(key)?;
+                self.task_prepare_sources(layout, recipe)
+            }
+            TaskId::ConfigureRecipe(key) => {
+                let recipe = recipes.recipe(key)?;
+                let active = active.expect("configure task requires an active container");
+                self.task_configure(layout, recipe, active)
+            }
+            TaskId::BuildRecipe(key) => {
+                let recipe = recipes.recipe(key)?;
+                let active = active.expect("build task requires an active container");
+                self.task_build(layout, recipe, active)
+            }
+            TaskId::InstallPackageFiles(output_key) => {
+                let output = recipes.output(output_key)?;
+                let recipe = recipes.recipe(output.recipe())?;
+                let active = active.expect("install task requires an active container");
+                self.task_install_package(layout, recipe, output, active)
+            }
+            TaskId::ProduceApk(output_key) => {
+                let output = recipes.output(output_key)?;
+                let recipe = recipes.recipe(output.recipe())?;
+                let active = active.expect("apk task requires an active container");
+                self.task_produce_apk(layout, recipe, output, active)
+            }
+            TaskId::InstallHostRecipe(key) => {
+                let recipe = recipes.recipe(key)?;
+                let active = active.expect("host install task requires an active container");
+                self.task_install_host(layout, recipe, active)
+            }
+        }
+    }
+
+    fn task_fetch_sources(&self, layout: &Layout<'_>, recipe: &Recipe) -> anyhow::Result<()> {
+        fs::create_dir_all(layout.source_cache_dir())?;
+        for (name, source) in recipe.sources().entries() {
+            let label = name.unwrap_or("source");
+            if !source.is_unknown_cache_key()
+                && layout.source_cache_path(source.cache_key()).exists()
+            {
+                log::skip("fetch", &format!("{} ({})", recipe.key(), label));
+                continue;
+            }
+            log::step(
+                "fetch",
+                &format!("{} ({}) from {}", recipe.key(), label, source.url()),
+            );
+            match source {
+                Source::Tarball(_) => self.fetch_tarball(layout, recipe, source)?,
+                Source::Git(git) => self.fetch_git(layout, recipe, git)?,
+            }
+        }
+        Ok(())
+    }
+
+    fn fetch_tarball(
+        &self,
+        layout: &Layout<'_>,
+        recipe: &Recipe,
+        source: &Source,
+    ) -> anyhow::Result<()> {
+        let url = source.url();
+        let expected = source.cache_key().to_owned();
+        let unknown = source.is_unknown_cache_key();
+
+        let client = reqwest::blocking::Client::builder()
+            .user_agent("distro-builder")
+            .build()
+            .context("building HTTP client")?;
+        let mut response = client
+            .get(url)
+            .send()
+            .with_context(|| format!("downloading {url}"))?
+            .error_for_status()
+            .with_context(|| format!("downloading {url}"))?;
+
+        let cache_dir = layout.source_cache_dir();
+        fs::create_dir_all(&cache_dir)?;
+        let mut tmp = tempfile::NamedTempFile::new_in(&cache_dir)
+            .with_context(|| format!("creating temp file in {}", cache_dir.display()))?;
+        let mut hasher = Sha256::new();
+        let mut buf = [0u8; 64 * 1024];
+        loop {
+            let n = std::io::Read::read(&mut response, &mut buf)
+                .with_context(|| format!("reading response body for {url}"))?;
+            if n == 0 {
+                break;
+            }
+            hasher.update(&buf[..n]);
+            tmp.write_all(&buf[..n])?;
+        }
+        tmp.flush()?;
+        let hash = hex::encode(hasher.finalize());
+
+        if unknown {
+            log::info(
+                "fetch",
+                &format!("{}: computed sha256 = {hash}", recipe.key()),
+            );
+        } else if hash != expected {
+            bail!("sha256 mismatch for {url}: expected {expected}, got {hash}");
+        }
+
+        let final_path = layout.source_cache_path(&hash);
+        if !final_path.exists() {
+            tmp.persist(&final_path).with_context(|| {
+                format!("renaming downloaded archive to {}", final_path.display())
+            })?;
+        }
+
+        if unknown {
+            bail!(
+                "{}: source sha256 is unknown; update the recipe with sha256 = \"{hash}\"",
+                recipe.key()
+            );
+        }
+        Ok(())
+    }
+
+    fn fetch_git(
+        &self,
+        layout: &Layout<'_>,
+        recipe: &Recipe,
+        source: &GitSource,
+    ) -> anyhow::Result<()> {
+        let url = source.url();
+        let commit = source.commit();
+        let unknown = matches!(commit, "?" | "???");
+
+        let cache_dir = layout.source_cache_dir();
+        fs::create_dir_all(&cache_dir)?;
+        let tmp = tempfile::tempdir_in(&cache_dir)?;
+        let work = tmp.path().join("repo");
+
+        let status = Command::new("git")
+            .arg("clone")
+            .arg("--bare")
+            .arg(url)
+            .arg(&work)
+            .status()
+            .with_context(|| format!("spawning git clone {url}"))?;
+        if !status.success() {
+            bail!("git clone failed for {url} with {status}");
+        }
+
+        let rev = if unknown {
+            let out = Command::new("git")
+                .arg("-C")
+                .arg(&work)
+                .arg("rev-parse")
+                .arg("HEAD")
+                .output()
+                .context("spawning git rev-parse HEAD")?;
+            if !out.status.success() {
+                bail!("git rev-parse HEAD failed with {}", out.status);
+            }
+            let rev = String::from_utf8(out.stdout)?.trim().to_owned();
+            log::info(
+                "fetch",
+                &format!("{}: resolved git HEAD = {rev}", recipe.key()),
+            );
+            rev
+        } else {
+            let status = Command::new("git")
+                .arg("-C")
+                .arg(&work)
+                .arg("cat-file")
+                .arg("-e")
+                .arg(commit)
+                .status()
+                .context("spawning git cat-file")?;
+            if !status.success() {
+                bail!("commit {commit} not found in {url}");
+            }
+            commit.to_owned()
+        };
+
+        let final_path = layout.source_cache_path(&rev);
+        if !final_path.exists() {
+            fs::rename(&work, &final_path).with_context(|| {
+                format!(
+                    "moving git clone to {} from {}",
+                    final_path.display(),
+                    work.display()
+                )
+            })?;
+        }
+
+        if unknown {
+            bail!(
+                "{}: source commit is unknown; update the recipe with commit = \"{rev}\"",
+                recipe.key()
+            );
+        }
+        Ok(())
+    }
+
+    fn task_prepare_sources(&self, layout: &Layout<'_>, recipe: &Recipe) -> anyhow::Result<()> {
+        log::step("prepare", &recipe.key());
+        let src = layout.source_workdir(recipe);
+        if src.exists() {
+            fs::remove_dir_all(&src).with_context(|| format!("clearing {}", src.display()))?;
+        }
+        fs::create_dir_all(&src)?;
+        let build = layout.build_workdir(recipe);
+        if build.exists() {
+            fs::remove_dir_all(&build).with_context(|| format!("clearing {}", build.display()))?;
+        }
+        fs::create_dir_all(&build)?;
+
+        for (name, source) in recipe.sources().entries() {
+            if source.is_unknown_cache_key() {
+                bail!(
+                    "source for {} has an unknown cache key; run `distro fetch` first",
+                    recipe.key()
+                );
+            }
+            let cache_path = layout.source_cache_path(source.cache_key());
+            if !cache_path.exists() {
+                bail!(
+                    "missing cached source for {} at {}",
+                    recipe.key(),
+                    cache_path.display()
+                );
+            }
+            let dst = match name {
+                None => src.clone(),
+                Some(named) => {
+                    let dst = src.join(named);
+                    fs::create_dir_all(&dst)?;
+                    dst
+                }
+            };
+            match source {
+                Source::Tarball(tar) => {
+                    let mut cmd = Command::new("tar");
+                    cmd.arg("-xf").arg(&cache_path);
+                    if tar.strip_components() > 0 {
+                        cmd.arg(format!("--strip-components={}", tar.strip_components()));
+                    }
+                    cmd.arg("-C").arg(&dst);
+                    let status = cmd
+                        .status()
+                        .with_context(|| format!("spawning tar -xf {}", cache_path.display()))?;
+                    if !status.success() {
+                        bail!(
+                            "tar extraction of {} failed with {status}",
+                            cache_path.display()
+                        );
+                    }
+                }
+                Source::Git(git) => {
+                    let status = Command::new("git")
+                        .arg("clone")
+                        .arg(&cache_path)
+                        .arg(&dst)
+                        .status()
+                        .context("spawning git clone from cache")?;
+                    if !status.success() {
+                        bail!("git clone from cache failed with {status}");
+                    }
+                    let status = Command::new("git")
+                        .arg("-C")
+                        .arg(&dst)
+                        .arg("checkout")
+                        .arg(git.commit())
+                        .status()
+                        .context("spawning git checkout")?;
+                    if !status.success() {
+                        bail!("git checkout {} failed with {status}", git.commit());
+                    }
+                }
+            }
+        }
+
+        let version_stamp = layout.source_stamp(recipe, "version");
+        fs::create_dir_all(version_stamp.parent().unwrap())?;
+        fs::write(
+            &version_stamp,
+            format!("{}-r{}", recipe.version(), recipe.revision()),
+        )?;
+
+        if layout.recipe_has_patches(recipe)? {
+            log::skip(
+                "patches",
+                &format!("{}: patch application not implemented", recipe.key()),
+            );
+            fs::write(layout.source_stamp(recipe, "patched"), b"skipped")?;
+        }
+
+        Ok(())
+    }
+
+    fn task_configure(
+        &self,
+        layout: &Layout<'_>,
+        recipe: &Recipe,
+        active: &ActiveContainer,
+    ) -> anyhow::Result<()> {
+        log::step("configure", &recipe.key());
+        if let Some(func) = recipe.phases().configure() {
+            let ctx = PhaseContext::new(
+                source_dir_for(recipe),
+                prefix_for(recipe.kind()),
+                default_jobs(),
+            );
+            self.invoke_with_runtime(active, &[PhaseArg::Ctx(ctx)], func)?;
+        }
+        self.write_recipe_stamp(layout, recipe, "configure")
+    }
+
+    fn task_build(
+        &self,
+        layout: &Layout<'_>,
+        recipe: &Recipe,
+        active: &ActiveContainer,
+    ) -> anyhow::Result<()> {
+        log::step("build", &recipe.key());
+        let ctx = PhaseContext::new(
+            source_dir_for(recipe),
+            prefix_for(recipe.kind()),
+            default_jobs(),
+        );
+        self.invoke_with_runtime(active, &[PhaseArg::Ctx(ctx)], recipe.phases().build())?;
+        self.write_recipe_stamp(layout, recipe, "build")
+    }
+
+    fn task_install_package(
+        &self,
+        layout: &Layout<'_>,
+        recipe: &Recipe,
+        output: &OutputPackage,
+        active: &ActiveContainer,
+    ) -> anyhow::Result<()> {
+        log::step("install", &output.key());
+        let dest = format!("/dest/{}", output.name());
+        active.container.borrow().exec(
+            &["mkdir".to_owned(), "-p".to_owned(), dest.clone()],
+            &base_env(&active.base_path),
+            "/",
+        )?;
+        let ctx = PhaseContext::new(
+            source_dir_for(recipe),
+            prefix_for(recipe.kind()),
+            default_jobs(),
+        );
+        let pkg = PackageContext::new(dest);
+        self.invoke_with_runtime(
+            active,
+            &[PhaseArg::Ctx(ctx), PhaseArg::Pkg(pkg)],
+            recipe.phases().install(),
+        )?;
+        self.write_output_stamp(layout, recipe, output, "install")
+    }
+
+    fn task_install_host(
+        &self,
+        layout: &Layout<'_>,
+        recipe: &Recipe,
+        active: &ActiveContainer,
+    ) -> anyhow::Result<()> {
+        log::step("host-install", &recipe.key());
+        let dest = format!("/dest/{}", recipe.slug());
+        active.container.borrow().exec(
+            &["mkdir".to_owned(), "-p".to_owned(), dest.clone()],
+            &base_env(&active.base_path),
+            "/",
+        )?;
+        let ctx = PhaseContext::new(
+            source_dir_for(recipe),
+            prefix_for(recipe.kind()),
+            default_jobs(),
+        );
+        let pkg = PackageContext::new(dest.clone());
+        self.invoke_with_runtime(
+            active,
+            &[PhaseArg::Ctx(ctx), PhaseArg::Pkg(pkg)],
+            recipe.phases().install(),
+        )?;
+
+        let host_root = layout.host_install_root(recipe);
+        if host_root.exists() {
+            fs::remove_dir_all(&host_root)
+                .with_context(|| format!("clearing {}", host_root.display()))?;
+        }
+        fs::create_dir_all(&host_root)?;
+        active
+            .container
+            .borrow()
+            .cp_out(&format!("{dest}/."), &host_root)?;
+
+        self.write_recipe_stamp(layout, recipe, "host-install")
+    }
+
+    fn task_produce_apk(
+        &self,
+        layout: &Layout<'_>,
+        recipe: &Recipe,
+        output: &OutputPackage,
+        active: &ActiveContainer,
+    ) -> anyhow::Result<()> {
+        log::step("apk", &output.key());
+        let files_dir = format!("/dest/{}", output.name());
+        let file_name = format!(
+            "{}-{}-r{}.apk",
+            output.name(),
+            recipe.version(),
+            recipe.revision()
+        );
+        let out_in_container = format!("/pkgs/{file_name}");
+        let version = format!("{}-r{}", recipe.version(), recipe.revision());
+
+        let mut argv: Vec<String> = vec![
+            "apk".to_owned(),
+            "mkpkg".to_owned(),
+            "--files".to_owned(),
+            files_dir,
+            "--output".to_owned(),
+            out_in_container,
+            "--info".to_owned(),
+            format!("name:{}", output.name()),
+            "--info".to_owned(),
+            format!("version:{version}"),
+            "--info".to_owned(),
+            format!("arch:{}", self.config.arch),
+            "--info".to_owned(),
+            format!("origin:{}", recipe.key()),
+        ];
+        if let Some(desc) = output.metadata().description() {
+            argv.push("--info".to_owned());
+            argv.push(format!("description:{desc}"));
+        }
+        if let Some(license) = output.metadata().license() {
+            argv.push("--info".to_owned());
+            argv.push(format!("license:{license}"));
+        }
+        if let Some(url) = output.metadata().website() {
+            argv.push("--info".to_owned());
+            argv.push(format!("url:{url}"));
+        }
+        if let Some(packager) = output.metadata().maintainer() {
+            argv.push("--info".to_owned());
+            argv.push(format!("packager:{packager}"));
+        }
+        for dep in recipe.deps().iter().chain(recipe.run_deps().iter()) {
+            argv.push("--info".to_owned());
+            argv.push(format!("depends:{dep}"));
+        }
+
+        active
+            .container
+            .borrow()
+            .exec(&argv, &base_env(&active.base_path), "/")?;
+
+        self.write_output_stamp(layout, recipe, output, "apk")
+    }
+
+    fn invoke_with_runtime(
+        &self,
+        active: &ActiveContainer,
+        args: &[PhaseArg],
+        func: &OwnedFrozenValue,
+    ) -> anyhow::Result<()> {
+        let runtime = PhaseRuntime {
+            container: active.container.clone(),
+            base_path: active.base_path.clone(),
+            base_env: bare_env(),
+        };
+        let _guard = PhaseRuntimeGuard::enter(runtime);
+        phase::invoke_phase(func, args)
+    }
+
+    fn start_recipe_container(
+        &self,
+        recipes: &RecipeSet,
+        recipe_key: &str,
+    ) -> anyhow::Result<ActiveContainer> {
+        let recipe = recipes.recipe(recipe_key)?;
+        let layout = Layout::new(&self.root, &self.config.arch);
+
+        let source_dir = layout.source_workdir(recipe);
+        let build_dir = layout.build_workdir(recipe);
+        fs::create_dir_all(&source_dir)?;
+        fs::create_dir_all(&build_dir)?;
+
+        let host_deps = transitive_host_deps(recipes, recipe)?;
+        let pkgs_dir = self.root.join("build/pkgs").join(&self.config.arch);
+        fs::create_dir_all(&pkgs_dir)?;
+        let mut mounts = vec![
+            Mount {
+                host: source_dir,
+                container: "/sources".to_owned(),
+                read_only: false,
+            },
+            Mount {
+                host: build_dir,
+                container: "/build".to_owned(),
+                read_only: false,
+            },
+            Mount {
+                host: pkgs_dir,
+                container: "/pkgs".to_owned(),
+                read_only: false,
+            },
+        ];
+        let mut tools_bins: Vec<String> = Vec::new();
+        for dep_key in &host_deps {
+            let dep_recipe = recipes.recipe(dep_key)?;
+            let install = layout.host_install_dir(dep_recipe);
+            if !install.exists() {
+                bail!(
+                    "missing host install for {dep_key} at {}; build it first",
+                    install.display()
+                );
+            }
+            let bare = dep_key.strip_prefix("host:").unwrap_or(dep_key);
+            mounts.push(Mount {
+                host: install,
+                container: format!("/tools/{bare}/usr/local"),
+                read_only: true,
+            });
+            tools_bins.push(format!("/tools/{bare}/usr/local/bin"));
+        }
+
+        let name = format!(
+            "distro-builder-{}-{:x}",
+            std::process::id(),
+            random_suffix()
+        );
+        let container = Container::start(
+            &self.config.container_runtime,
+            &self.config.container_image,
+            &name,
+            &mounts,
+        )?;
+
+        let mut path_segments = tools_bins;
+        path_segments.push("/usr/local/sbin".to_owned());
+        path_segments.push("/usr/local/bin".to_owned());
+        path_segments.push("/usr/sbin".to_owned());
+        path_segments.push("/usr/bin".to_owned());
+        path_segments.push("/sbin".to_owned());
+        path_segments.push("/bin".to_owned());
+        let base_path = path_segments.join(":");
+
+        let active = ActiveContainer {
+            recipe_key: recipe_key.to_owned(),
+            container: Rc::new(RefCell::new(container)),
+            base_path,
+        };
+
+        Ok(active)
+    }
+
+    fn write_recipe_stamp(
+        &self,
+        layout: &Layout<'_>,
+        recipe: &Recipe,
+        kind: &str,
+    ) -> anyhow::Result<()> {
+        let path = layout.recipe_task_stamp(recipe, kind);
+        fs::create_dir_all(path.parent().unwrap())?;
+        fs::write(path, layout.recipe_fingerprint(recipe)?)?;
+        Ok(())
+    }
+
+    fn write_output_stamp(
+        &self,
+        layout: &Layout<'_>,
+        recipe: &Recipe,
+        output: &OutputPackage,
+        kind: &str,
+    ) -> anyhow::Result<()> {
+        let path = layout.output_task_stamp(output, kind);
+        fs::create_dir_all(path.parent().unwrap())?;
+        fs::write(path, layout.output_fingerprint(recipe, output)?)?;
+        Ok(())
+    }
+
     fn ensure_container_image(&self, dockerfile: &Path) -> anyhow::Result<()> {
         if !dockerfile.exists() {
             bail!(
@@ -132,6 +764,8 @@ impl Builder {
             .arg("image")
             .arg("exists")
             .arg(&self.config.container_image)
+            .stdout(Stdio::null())
+            .stderr(Stdio::null())
             .status()
             .with_context(|| {
                 format!(
@@ -176,3 +810,96 @@ impl Builder {
         }
     }
 }
+
+struct ActiveContainer {
+    recipe_key: String,
+    container: Rc<RefCell<Container>>,
+    base_path: String,
+}
+
+fn task_needs_container(task: &TaskId) -> bool {
+    matches!(
+        task,
+        TaskId::ConfigureRecipe(_)
+            | TaskId::BuildRecipe(_)
+            | TaskId::InstallPackageFiles(_)
+            | TaskId::ProduceApk(_)
+            | TaskId::InstallHostRecipe(_)
+    )
+}
+
+fn prefix_for(kind: RecipeKind) -> &'static str {
+    match kind {
+        RecipeKind::Package => "/usr",
+        RecipeKind::HostPackage => "/usr/local",
+    }
+}
+
+fn source_dir_for(recipe: &Recipe) -> SourceDir {
+    let entries = recipe.sources().entries();
+    let named: Vec<(&str, &crate::recipe::Source)> = entries
+        .iter()
+        .filter_map(|(name, src)| name.map(|n| (n, *src)))
+        .collect();
+    if named.is_empty() {
+        SourceDir::single("/sources")
+    } else {
+        SourceDir::named(
+            named
+                .into_iter()
+                .map(|(n, _)| (n.to_owned(), format!("/sources/{n}"))),
+        )
+    }
+}
+
+fn default_jobs() -> i32 {
+    thread::available_parallelism()
+        .map(|n| n.get() as i32)
+        .unwrap_or(1)
+}
+
+fn bare_env() -> Vec<(String, String)> {
+    vec![
+        ("PATH".to_owned(), String::new()),
+        ("HOME".to_owned(), "/tmp".to_owned()),
+        ("TERM".to_owned(), "dumb".to_owned()),
+        ("LC_ALL".to_owned(), "C".to_owned()),
+    ]
+}
+
+fn base_env(path: &str) -> Vec<(String, String)> {
+    let mut env = bare_env();
+    if let Some(slot) = env.iter_mut().find(|(k, _)| k == "PATH") {
+        slot.1 = path.to_owned();
+    }
+    env
+}
+
+fn transitive_host_deps(recipes: &RecipeSet, recipe: &Recipe) -> anyhow::Result<Vec<String>> {
+    let mut order: Vec<String> = Vec::new();
+    let mut seen: BTreeSet<String> = BTreeSet::new();
+    let mut queue: VecDeque<String> = recipe
+        .host_deps()
+        .iter()
+        .map(|d| RecipeKind::HostPackage.key(d))
+        .collect();
+    while let Some(key) = queue.pop_front() {
+        if !seen.insert(key.clone()) {
+            continue;
+        }
+        let dep = recipes.recipe(&key)?;
+        for sub in dep.host_deps() {
+            queue.push_back(RecipeKind::HostPackage.key(sub));
+        }
+        order.push(key);
+    }
+    Ok(order)
+}
+
+fn random_suffix() -> u64 {
+    let nanos = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.subsec_nanos() as u64)
+        .unwrap_or(0);
+    nanos.wrapping_mul(0x9E3779B97F4A7C15)
+}

src/container.rs

@@ -0,0 +1,163 @@
+use std::{
+    path::{Path, PathBuf},
+    process::{Command, Stdio},
+};
+
+use anyhow::{Context, bail};
+
+use crate::config::ContainerRuntime;
+
+#[derive(Debug, Clone)]
+pub struct Mount {
+    pub host: PathBuf,
+    pub container: String,
+    pub read_only: bool,
+}
+
+#[derive(Debug)]
+pub struct Container {
+    runtime: &'static str,
+    id: String,
+    stopped: bool,
+}
+
+impl Container {
+    pub fn start(
+        runtime: &ContainerRuntime,
+        image: &str,
+        name: &str,
+        mounts: &[Mount],
+    ) -> anyhow::Result<Self> {
+        let runtime_str = runtime.as_str();
+        let mut cmd = Command::new(runtime_str);
+        cmd.arg("run")
+            .arg("-d")
+            .arg("--rm")
+            .arg("--name")
+            .arg(name)
+            .arg("--read-only")
+            .arg("--tmpfs")
+            .arg("/tmp")
+            .arg("--tmpfs")
+            .arg("/dest")
+            .arg("--tmpfs")
+            .arg("/sysroot")
+            .arg("--network=none");
+
+        if matches!(runtime, ContainerRuntime::Podman) {
+            cmd.arg("--userns=keep-id");
+        }
+
+        for mount in mounts {
+            let mut spec = format!("{}:{}", mount.host.display(), mount.container);
+            if mount.read_only {
+                spec.push_str(":ro");
+            }
+            cmd.arg("-v").arg(spec);
+        }
+
+        cmd.arg(image).arg("sleep").arg("infinity");
+        cmd.stdout(Stdio::piped()).stderr(Stdio::inherit());
+
+        let output = cmd
+            .output()
+            .with_context(|| format!("spawning `{runtime_str} run` for image `{image}`"))?;
+        if !output.status.success() {
+            bail!(
+                "`{runtime_str} run` failed with {} for image `{image}`",
+                output.status
+            );
+        }
+
+        let id = String::from_utf8(output.stdout)
+            .context("container id is not valid UTF-8")?
+            .trim()
+            .to_owned();
+        if id.is_empty() {
+            bail!("`{runtime_str} run` returned an empty container id");
+        }
+
+        Ok(Self {
+            runtime: runtime_str,
+            id,
+            stopped: false,
+        })
+    }
+
+    pub fn exec(&self, argv: &[String], env: &[(String, String)], cwd: &str) -> anyhow::Result<()> {
+        if argv.is_empty() {
+            bail!("ctx.run called with an empty argv");
+        }
+
+        let mut cmd = Command::new(self.runtime);
+        cmd.arg("exec").arg("-w").arg(cwd);
+        for (k, v) in env {
+            cmd.arg("-e").arg(format!("{k}={v}"));
+        }
+        cmd.arg(&self.id);
+        cmd.args(argv);
+
+        let status = cmd.status().with_context(|| {
+            format!("spawning `{} exec` in container {}", self.runtime, self.id)
+        })?;
+        if !status.success() {
+            bail!("command {argv:?} failed with {status}");
+        }
+        Ok(())
+    }
+
+    pub fn cp_out(&self, src_in_container: &str, host_dst: &Path) -> anyhow::Result<()> {
+        let spec = format!("{}:{}", self.id, src_in_container);
+        let status = Command::new(self.runtime)
+            .arg("cp")
+            .arg(spec)
+            .arg(host_dst)
+            .status()
+            .with_context(|| format!("spawning `{} cp`", self.runtime))?;
+        if !status.success() {
+            bail!(
+                "`{} cp` failed with {status} for {src_in_container} -> {}",
+                self.runtime,
+                host_dst.display()
+            );
+        }
+        Ok(())
+    }
+
+    pub fn stop(mut self) -> anyhow::Result<()> {
+        self.stop_inner()
+    }
+
+    fn stop_inner(&mut self) -> anyhow::Result<()> {
+        if self.stopped {
+            return Ok(());
+        }
+        self.stopped = true;
+        let status = Command::new(self.runtime)
+            .arg("rm")
+            .arg("-f")
+            .arg(&self.id)
+            .stdout(Stdio::null())
+            .stderr(Stdio::null())
+            .status()
+            .with_context(|| format!("spawning `{} rm`", self.runtime))?;
+        if !status.success() {
+            bail!("`{} rm -f {}` failed with {status}", self.runtime, self.id);
+        }
+        Ok(())
+    }
+}
+
+impl Drop for Container {
+    fn drop(&mut self) {
+        if !self.stopped {
+            let _ = Command::new(self.runtime)
+                .arg("rm")
+                .arg("-f")
+                .arg(&self.id)
+                .stdout(Stdio::null())
+                .stderr(Stdio::null())
+                .status();
+        }
+    }
+}

src/graph.rs

@@ -1,13 +1,15 @@
 use std::{
     collections::{BTreeMap, BTreeSet},
     fmt, fs,
-    path::{Path, PathBuf},
+    path::Path,
 };
 
-use anyhow::{Context, bail};
+use anyhow::bail;
-use sha2::{Digest, Sha256};
 
-use crate::recipe::{OutputPackage, Recipe, RecipeKind, RecipeSet};
+use crate::{
+    layout::Layout,
+    recipe::{OutputPackage, Recipe, RecipeKind, RecipeSet},
+};
 
 #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
 pub enum TaskId {
@@ -60,8 +62,7 @@ impl TaskPlan {
 }
 
 pub struct TaskPlanner<'a> {
-    root: &'a Path,
+    layout: Layout<'a>,
-    arch: &'a str,
     recipes: &'a RecipeSet,
     force: bool,
     dependencies: BTreeMap<TaskId, Vec<TaskId>>,
@@ -73,8 +74,7 @@ pub struct TaskPlanner<'a> {
 impl<'a> TaskPlanner<'a> {
     pub fn new(root: &'a Path, arch: &'a str, recipes: &'a RecipeSet) -> Self {
         Self {
-            root,
+            layout: Layout::new(root, arch),
-            arch,
             recipes,
             force: false,
             dependencies: BTreeMap::new(),
@@ -138,18 +138,7 @@ impl<'a> TaskPlanner<'a> {
     }
 
     fn into_plan(self) -> anyhow::Result<TaskPlan> {
-        let mut order = Vec::new();
+        let order = recipe_contiguous_order(&self.dependencies, self.recipes)?;
-        let mut visiting = BTreeSet::new();
-        let mut visited = BTreeSet::new();
-        for task in self.dependencies.keys() {
-            topo_visit(
-                task,
-                &self.dependencies,
-                &mut visiting,
-                &mut visited,
-                &mut order,
-            )?;
-        }
         Ok(TaskPlan {
             dependencies: self.dependencies,
             order,
@@ -218,7 +207,8 @@ impl<'a> TaskPlanner<'a> {
             TaskId::InstallPackageFiles(output) => {
                 let output = self.recipes.output(output)?;
                 let recipe = self.recipes.recipe(output.recipe())?;
-                self.output_task_active(recipe, output, "install")
+                Ok(self.output_task_active(recipe, output, "install")?
+                    || self.produce_apk_active(recipe, output)?)
             }
             TaskId::ProduceApk(output) => {
                 let output = self.recipes.output(output)?;
@@ -234,7 +224,8 @@ impl<'a> TaskPlanner<'a> {
 
     fn fetch_sources_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
         Ok(recipe.sources().entries().iter().any(|(_, source)| {
-            source.is_unknown_cache_key() || !self.source_cache_path(source.cache_key()).exists()
+            source.is_unknown_cache_key()
+                || !self.layout.source_cache_path(source.cache_key()).exists()
         }))
     }
 
@@ -243,14 +234,16 @@ impl<'a> TaskPlanner<'a> {
             return Ok(true);
         }
         let want_version = format!("{}-r{}", recipe.version(), recipe.revision());
-        if fs::read_to_string(self.source_stamp(recipe, "version"))
+        if fs::read_to_string(self.layout.source_stamp(recipe, "version"))
             .ok()
             .as_deref()
             != Some(want_version.as_str())
         {
             return Ok(true);
         }
-        if self.recipe_has_patches(recipe)? && !self.source_stamp(recipe, "patched").exists() {
+        if self.layout.recipe_has_patches(recipe)?
+            && !self.layout.source_stamp(recipe, "patched").exists()
+        {
             return Ok(true);
         }
         Ok(false)
@@ -260,10 +253,12 @@ impl<'a> TaskPlanner<'a> {
         if self.force {
             return Ok(true);
         }
-        Ok(fs::read_to_string(self.recipe_task_stamp(recipe, kind))
+        Ok(
-            .ok()
+            fs::read_to_string(self.layout.recipe_task_stamp(recipe, kind))
-            .as_deref()
+                .ok()
-            != Some(self.recipe_fingerprint(recipe)?.as_str()))
+                .as_deref()
+                != Some(self.layout.recipe_fingerprint(recipe)?.as_str()),
+        )
     }
 
     fn output_task_active(
@@ -275,278 +270,116 @@ impl<'a> TaskPlanner<'a> {
         if self.force {
             return Ok(true);
         }
-        Ok(fs::read_to_string(self.output_task_stamp(output, kind))
+        Ok(
-            .ok()
+            fs::read_to_string(self.layout.output_task_stamp(output, kind))
-            .as_deref()
+                .ok()
-            != Some(self.output_fingerprint(recipe, output)?.as_str()))
+                .as_deref()
+                != Some(self.layout.output_fingerprint(recipe, output)?.as_str()),
+        )
     }
 
     fn produce_apk_active(&self, recipe: &Recipe, output: &OutputPackage) -> anyhow::Result<bool> {
         if self.force {
             return Ok(true);
         }
-        if !self.apk_path(recipe, output).exists() {
+        if !self.layout.apk_path(recipe, output).exists() {
             return Ok(true);
         }
-        Ok(fs::read_to_string(self.output_task_stamp(output, "apk"))
+        Ok(
-            .ok()
+            fs::read_to_string(self.layout.output_task_stamp(output, "apk"))
-            .as_deref()
+                .ok()
-            != Some(self.output_fingerprint(recipe, output)?.as_str()))
+                .as_deref()
+                != Some(self.layout.output_fingerprint(recipe, output)?.as_str()),
+        )
     }
 
     fn install_host_recipe_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
         if self.force {
             return Ok(true);
         }
-        if !self.host_install_dir(recipe).exists() {
+        if !self.layout.host_install_dir(recipe).exists() {
             return Ok(true);
         }
         Ok(
-            fs::read_to_string(self.recipe_task_stamp(recipe, "host-install"))
+            fs::read_to_string(self.layout.recipe_task_stamp(recipe, "host-install"))
                 .ok()
                 .as_deref()
-                != Some(self.recipe_fingerprint(recipe)?.as_str()),
+                != Some(self.layout.recipe_fingerprint(recipe)?.as_str()),
         )
     }
+}
 
-    fn recipe_fingerprint(&self, recipe: &Recipe) -> anyhow::Result<String> {
+fn recipe_contiguous_order(
-        let mut hasher = Sha256::new();
+    dependencies: &BTreeMap<TaskId, Vec<TaskId>>,
-        hasher.update(self.arch.as_bytes());
+    recipes: &RecipeSet,
-        hasher.update(recipe.key().as_bytes());
+) -> anyhow::Result<Vec<TaskId>> {
-        hasher.update(recipe.version().as_bytes());
+    let total = dependencies.len();
-        hasher.update(recipe.revision().to_le_bytes());
+    let mut remaining: BTreeMap<TaskId, BTreeSet<TaskId>> = dependencies
-        hasher.update(
+        .iter()
-            fs::read(recipe.path())
+        .map(|(task, deps)| (task.clone(), deps.iter().cloned().collect()))
-                .with_context(|| format!("reading recipe {}", recipe.path().display()))?,
+        .collect();
-        );
+
-        for (name, source) in recipe.sources().entries() {
+    let mut dependents: BTreeMap<TaskId, Vec<TaskId>> = BTreeMap::new();
-            hasher.update(name.unwrap_or("").as_bytes());
+    for (task, deps) in dependencies {
-            hasher.update(source.url().as_bytes());
+        for dep in deps {
-            hasher.update(source.cache_key().as_bytes());
+            dependents
+                .entry(dep.clone())
+                .or_default()
+                .push(task.clone());
         }
-        for patch in self.recipe_patches(recipe)? {
-            hasher.update(patch.display().to_string().as_bytes());
-            hasher
-                .update(fs::read(&patch).with_context(|| format!("reading {}", patch.display()))?);
-        }
-        Ok(hex::encode(hasher.finalize()))
     }
 
-    fn output_fingerprint(
+    let mut order = Vec::with_capacity(total);
-        &self,
+    let mut current_recipe: Option<String> = None;
-        recipe: &Recipe,
+    while order.len() < total {
-        output: &OutputPackage,
+        let next = pick_next(&remaining, current_recipe.as_deref(), recipes)?;
-    ) -> anyhow::Result<String> {
+        let slug = task_recipe_slug(&next, recipes)?;
-        let mut hasher = Sha256::new();
+        current_recipe = Some(slug);
-        hasher.update(self.recipe_fingerprint(recipe)?.as_bytes());
+        remaining.remove(&next);
-        hasher.update(output.key().as_bytes());
+        if let Some(children) = dependents.get(&next) {
-        Ok(hex::encode(hasher.finalize()))
+            for child in children {
-    }
+                if let Some(deps) = remaining.get_mut(child) {
-
+                    deps.remove(&next);
-    fn recipe_has_patches(&self, recipe: &Recipe) -> anyhow::Result<bool> {
+                }
-        Ok(!self.recipe_patches(recipe)?.is_empty())
-    }
-
-    fn recipe_patches(&self, recipe: &Recipe) -> anyhow::Result<Vec<PathBuf>> {
-        let patches_dir = recipe.dir().join("patches");
-        if !patches_dir.exists() {
-            return Ok(Vec::new());
-        }
-        let mut patches = Vec::new();
-        for entry in fs::read_dir(&patches_dir)
-            .with_context(|| format!("reading patches directory {}", patches_dir.display()))?
-        {
-            let entry = entry?;
-            let path = entry.path();
-            if path.is_file() {
-                patches.push(path);
             }
         }
-        patches.sort();
+        order.push(next);
-        Ok(patches)
-    }
-
-    fn source_cache_path(&self, key: &str) -> PathBuf {
-        self.root.join("build/cache/sources").join(key)
-    }
-
-    fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
-        self.root
-            .join("build/sources")
-            .join(format!("{}.{kind}", recipe.slug()))
-    }
-
-    fn recipe_task_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
-        self.root
-            .join("build/tasks")
-            .join(format!("{}.{kind}", recipe.slug()))
-    }
-
-    fn output_task_stamp(&self, output: &OutputPackage, kind: &str) -> PathBuf {
-        self.root
-            .join("build/tasks")
-            .join(format!("{}.{kind}", output.key().replace(':', "-")))
-    }
-
-    fn apk_path(&self, recipe: &Recipe, output: &OutputPackage) -> PathBuf {
-        self.root.join("build/pkgs").join(self.arch).join(format!(
-            "{}-{}-r{}.apk",
-            output.name(),
-            recipe.version(),
-            recipe.revision()
-        ))
-    }
-
-    fn host_install_dir(&self, recipe: &Recipe) -> PathBuf {
-        self.root
-            .join("build/host-pkgs")
-            .join(recipe.slug())
-            .join("usr/local")
     }
+    Ok(order)
 }
 
-fn topo_visit(
+fn pick_next(
-    task: &TaskId,
+    remaining: &BTreeMap<TaskId, BTreeSet<TaskId>>,
-    dependencies: &BTreeMap<TaskId, Vec<TaskId>>,
+    current_recipe: Option<&str>,
-    visiting: &mut BTreeSet<TaskId>,
+    recipes: &RecipeSet,
-    visited: &mut BTreeSet<TaskId>,
+) -> anyhow::Result<TaskId> {
-    order: &mut Vec<TaskId>,
+    let mut fallback: Option<TaskId> = None;
-) -> anyhow::Result<()> {
+    for (task, deps) in remaining {
-    if visited.contains(task) {
+        if !deps.is_empty() {
-        return Ok(());
+            continue;
+        }
+        if let Some(active) = current_recipe {
+            let slug = task_recipe_slug(task, recipes)?;
+            if slug == active {
+                return Ok(task.clone());
+            }
+        }
+        if fallback.is_none() {
+            fallback = Some(task.clone());
+        }
     }
-    if !visiting.insert(task.clone()) {
+    fallback.ok_or_else(|| anyhow::anyhow!("task dependency cycle detected"))
-        bail!("task dependency cycle involving `{task}`");
-    }
-    for dependency in dependencies.get(task).into_iter().flatten() {
-        topo_visit(dependency, dependencies, visiting, visited, order)?;
-    }
-    visiting.remove(task);
-    visited.insert(task.clone());
-    order.push(task.clone());
-    Ok(())
 }
 
-#[cfg(test)]
+pub fn task_recipe_slug(task: &TaskId, recipes: &RecipeSet) -> anyhow::Result<String> {
-mod tests {
+    Ok(match task {
-    use std::fs;
+        TaskId::FetchSources(recipe)
-
+        | TaskId::PrepareSources(recipe)
-    use tempfile::TempDir;
+        | TaskId::ConfigureRecipe(recipe)
-
+        | TaskId::BuildRecipe(recipe)
-    use crate::{config::Config, eval, recipe::RecipeSet};
+        | TaskId::InstallHostRecipe(recipe) => recipe.clone(),
-
+        TaskId::InstallPackageFiles(output) | TaskId::ProduceApk(output) => {
-    use super::{TaskId, TaskPlanner};
+            recipes.output(output)?.recipe().to_owned()
-
+        }
-    fn write_config(root: &TempDir) {
+    })
-        fs::write(
-            root.path().join("config.star"),
-            r#"
-container_runtime = "podman"
-container_image = "local/test:latest"
-container_dockerfile = "Dockerfile"
-arch = "x86_64"
-options = dict(target_arch = arch)
-"#,
-        )
-        .unwrap();
-        fs::write(root.path().join("Dockerfile"), "FROM scratch\n").unwrap();
-    }
-
-    fn write_recipe(root: &TempDir, dir: &str, name: &str, extra: &str) {
-        let recipe_dir = root.path().join(dir);
-        fs::create_dir_all(&recipe_dir).unwrap();
-        fs::write(
-            recipe_dir.join(format!("{name}.star")),
-            format!(
-                r#"
-version = "1.0"
-revision = 1
-source = tarball_source(url = "file:///tmp/{name}.tar", sha256 = "hash-{name}")
-{extra}
-def build(ctx):
-    ctx.run(["true"])
-def install(ctx, pkg):
-    ctx.run(["true"])
-"#
-            ),
-        )
-        .unwrap();
-    }
-
-    fn load(root: &TempDir) -> (Config, RecipeSet) {
-        let config = Config::load(&root.path().join("config.star")).unwrap();
-        let lib = eval::eval_lib(&root.path().join("lib"), Some(&config.options)).unwrap();
-        let recipes = RecipeSet::load(root.path(), &config.options, lib.as_ref()).unwrap();
-        (config, recipes)
-    }
-
-    #[test]
-    fn inactive_seed_does_not_pull_dependencies() {
-        let root = TempDir::new().unwrap();
-        write_config(&root);
-        write_recipe(&root, "recipes", "dep", "");
-        write_recipe(&root, "recipes", "app", r#"deps = ["dep"]"#);
-        let (config, recipes) = load(&root);
-
-        let planner = TaskPlanner::new(root.path(), &config.arch, &recipes);
-        let output = recipes.output("app").unwrap();
-        let recipe = recipes.recipe(output.recipe()).unwrap();
-        fs::create_dir_all(root.path().join("build/pkgs/x86_64")).unwrap();
-        fs::write(root.path().join("build/pkgs/x86_64/app-1.0-r1.apk"), "").unwrap();
-        fs::create_dir_all(root.path().join("build/tasks")).unwrap();
-        fs::write(
-            root.path().join("build/tasks/app.apk"),
-            planner.output_fingerprint(recipe, output).unwrap(),
-        )
-        .unwrap();
-
-        let plan = TaskPlanner::new(root.path(), &config.arch, &recipes)
-            .build_plan(&["app".to_owned()], false)
-            .unwrap();
-        assert!(plan.is_empty());
-    }
-
-    #[test]
-    fn active_target_recipe_keeps_edges_and_topo_order() {
-        let root = TempDir::new().unwrap();
-        write_config(&root);
-        write_recipe(&root, "recipes", "app", "");
-        let (config, recipes) = load(&root);
-
-        let plan = TaskPlanner::new(root.path(), &config.arch, &recipes)
-            .build_plan(&["app".to_owned()], false)
-            .unwrap();
-        assert_eq!(
-            plan.order(),
-            &[
-                TaskId::FetchSources("app".to_owned()),
-                TaskId::PrepareSources("app".to_owned()),
-                TaskId::ConfigureRecipe("app".to_owned()),
-                TaskId::BuildRecipe("app".to_owned()),
-                TaskId::InstallPackageFiles("app".to_owned()),
-                TaskId::ProduceApk("app".to_owned()),
-            ]
-        );
-        assert_eq!(
-            plan.dependencies(&TaskId::ProduceApk("app".to_owned())),
-            Some([TaskId::InstallPackageFiles("app".to_owned())].as_slice())
-        );
-    }
-
-    #[test]
-    fn host_dependencies_are_installed_as_host_recipes() {
-        let root = TempDir::new().unwrap();
-        write_config(&root);
-        write_recipe(&root, "host-recipes", "binutils", "");
-        write_recipe(&root, "recipes", "app", r#"host_deps = ["binutils"]"#);
-        let (config, recipes) = load(&root);
-
-        let plan = TaskPlanner::new(root.path(), &config.arch, &recipes)
-            .build_plan(&["app".to_owned()], false)
-            .unwrap();
-        assert!(
-            plan.order()
-                .contains(&TaskId::InstallHostRecipe("host:binutils".to_owned()))
-        );
-    }
 }

src/layout.rs

@@ -0,0 +1,131 @@
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
+
+use anyhow::Context;
+use sha2::{Digest, Sha256};
+
+use crate::recipe::{OutputPackage, Recipe};
+
+pub struct Layout<'a> {
+    pub root: &'a Path,
+    pub arch: &'a str,
+}
+
+impl<'a> Layout<'a> {
+    pub fn new(root: &'a Path, arch: &'a str) -> Self {
+        Self { root, arch }
+    }
+
+    pub fn source_cache_dir(&self) -> PathBuf {
+        self.root.join("build/cache/sources")
+    }
+
+    pub fn source_cache_path(&self, key: &str) -> PathBuf {
+        self.source_cache_dir().join(key)
+    }
+
+    pub fn source_workdir(&self, recipe: &Recipe) -> PathBuf {
+        self.root.join("build/sources").join(recipe.slug())
+    }
+
+    pub fn build_workdir(&self, recipe: &Recipe) -> PathBuf {
+        self.root.join("build/builds").join(recipe.slug())
+    }
+
+    pub fn host_install_dir(&self, recipe: &Recipe) -> PathBuf {
+        self.root
+            .join("build/host-pkgs")
+            .join(recipe.slug())
+            .join("usr/local")
+    }
+
+    pub fn host_install_root(&self, recipe: &Recipe) -> PathBuf {
+        self.root.join("build/host-pkgs").join(recipe.slug())
+    }
+
+    pub fn apk_path(&self, recipe: &Recipe, output: &OutputPackage) -> PathBuf {
+        self.root.join("build/pkgs").join(self.arch).join(format!(
+            "{}-{}-r{}.apk",
+            output.name(),
+            recipe.version(),
+            recipe.revision()
+        ))
+    }
+
+    pub fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
+        self.root
+            .join("build/sources")
+            .join(format!("{}.{kind}", recipe.slug()))
+    }
+
+    pub fn recipe_task_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
+        self.root
+            .join("build/tasks")
+            .join(format!("{}.{kind}", recipe.slug()))
+    }
+
+    pub fn output_task_stamp(&self, output: &OutputPackage, kind: &str) -> PathBuf {
+        self.root
+            .join("build/tasks")
+            .join(format!("{}.{kind}", output.key().replace(':', "-")))
+    }
+
+    pub fn recipe_fingerprint(&self, recipe: &Recipe) -> anyhow::Result<String> {
+        let mut hasher = Sha256::new();
+        hasher.update(self.arch.as_bytes());
+        hasher.update(recipe.key().as_bytes());
+        hasher.update(recipe.version().as_bytes());
+        hasher.update(recipe.revision().to_le_bytes());
+        hasher.update(
+            fs::read(recipe.path())
+                .with_context(|| format!("reading recipe {}", recipe.path().display()))?,
+        );
+        for (name, source) in recipe.sources().entries() {
+            hasher.update(name.unwrap_or("").as_bytes());
+            hasher.update(source.url().as_bytes());
+            hasher.update(source.cache_key().as_bytes());
+        }
+        for patch in self.recipe_patches(recipe)? {
+            hasher.update(patch.display().to_string().as_bytes());
+            hasher
+                .update(fs::read(&patch).with_context(|| format!("reading {}", patch.display()))?);
+        }
+        Ok(hex::encode(hasher.finalize()))
+    }
+
+    pub fn output_fingerprint(
+        &self,
+        recipe: &Recipe,
+        output: &OutputPackage,
+    ) -> anyhow::Result<String> {
+        let mut hasher = Sha256::new();
+        hasher.update(self.recipe_fingerprint(recipe)?.as_bytes());
+        hasher.update(output.key().as_bytes());
+        Ok(hex::encode(hasher.finalize()))
+    }
+
+    pub fn recipe_has_patches(&self, recipe: &Recipe) -> anyhow::Result<bool> {
+        Ok(!self.recipe_patches(recipe)?.is_empty())
+    }
+
+    pub fn recipe_patches(&self, recipe: &Recipe) -> anyhow::Result<Vec<PathBuf>> {
+        let patches_dir = recipe.dir().join("patches");
+        if !patches_dir.exists() {
+            return Ok(Vec::new());
+        }
+        let mut patches = Vec::new();
+        for entry in fs::read_dir(&patches_dir)
+            .with_context(|| format!("reading patches directory {}", patches_dir.display()))?
+        {
+            let entry = entry?;
+            let path = entry.path();
+            if path.is_file() {
+                patches.push(path);
+            }
+        }
+        patches.sort();
+        Ok(patches)
+    }
+}

src/main.rs

@@ -1,10 +1,13 @@
 mod builder;
 mod cli;
 mod config;
+mod container;
 mod eval;
 mod graph;
+mod layout;
 mod log;
 mod options;
+mod phase;
 mod recipe;
 
 fn main() -> anyhow::Result<()> {

src/phase.rs

@@ -0,0 +1,296 @@
+use std::{cell::RefCell, collections::BTreeMap, rc::Rc};
+
+use allocative::Allocative;
+use anyhow::anyhow;
+use starlark::{
+    collections::SmallMap,
+    environment::{Methods, MethodsBuilder, MethodsStatic, Module},
+    eval::Evaluator,
+    values::{Heap, OwnedFrozenValue, StarlarkValue, Value, list::UnpackList, none::NoneType},
+};
+use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_module, starlark_value};
+
+use crate::container::Container;
+
+thread_local! {
+    static CURRENT: RefCell<Option<PhaseRuntime>> = const { RefCell::new(None) };
+}
+
+#[derive(Clone)]
+pub struct PhaseRuntime {
+    pub container: Rc<RefCell<Container>>,
+    pub base_path: String,
+    pub base_env: Vec<(String, String)>,
+}
+
+pub struct PhaseRuntimeGuard;
+
+impl PhaseRuntimeGuard {
+    pub fn enter(runtime: PhaseRuntime) -> Self {
+        CURRENT.with(|cell| {
+            let prev = cell.borrow_mut().replace(runtime);
+            assert!(prev.is_none(), "phase runtime already set");
+        });
+        Self
+    }
+}
+
+impl Drop for PhaseRuntimeGuard {
+    fn drop(&mut self) {
+        CURRENT.with(|cell| {
+            cell.borrow_mut().take();
+        });
+    }
+}
+
+fn with_current<R>(f: impl FnOnce(&PhaseRuntime) -> R) -> anyhow::Result<R> {
+    CURRENT.with(|cell| {
+        let borrow = cell.borrow();
+        let runtime = borrow
+            .as_ref()
+            .ok_or_else(|| anyhow!("ctx.run called outside of a phase invocation"))?;
+        Ok(f(runtime))
+    })
+}
+
+#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
+pub struct SourceDir {
+    default: String,
+    entries: BTreeMap<String, String>,
+}
+
+impl SourceDir {
+    pub fn single(path: impl Into<String>) -> Self {
+        Self {
+            default: path.into(),
+            entries: BTreeMap::new(),
+        }
+    }
+
+    pub fn named<I, K, V>(entries: I) -> Self
+    where
+        I: IntoIterator<Item = (K, V)>,
+        K: Into<String>,
+        V: Into<String>,
+    {
+        let entries: BTreeMap<String, String> = entries
+            .into_iter()
+            .map(|(k, v)| (k.into(), v.into()))
+            .collect();
+        let default = entries
+            .values()
+            .next()
+            .cloned()
+            .unwrap_or_else(|| "/sources".to_owned());
+        Self { default, entries }
+    }
+}
+
+impl std::fmt::Display for SourceDir {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&self.default)
+    }
+}
+
+starlark::starlark_simple_value!(SourceDir);
+
+#[starlark_value(type = "source_dir")]
+impl<'v> StarlarkValue<'v> for SourceDir {
+    fn at(&self, index: Value<'v>, heap: &'v Heap) -> starlark::Result<Value<'v>> {
+        let key = index.unpack_str().ok_or_else(|| {
+            starlark::Error::new_other(anyhow!("source_dir index must be a string"))
+        })?;
+        let path = self.entries.get(key).ok_or_else(|| {
+            starlark::Error::new_other(anyhow!(
+                "no source named `{key}` (available: {})",
+                if self.entries.is_empty() {
+                    "<none>".to_owned()
+                } else {
+                    self.entries
+                        .keys()
+                        .map(String::as_str)
+                        .collect::<Vec<_>>()
+                        .join(", ")
+                }
+            ))
+        })?;
+        Ok(heap.alloc(path.as_str()))
+    }
+
+    fn add(&self, rhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
+        let suffix = rhs.unpack_str()?;
+        Some(Ok(heap.alloc(format!("{}{}", self.default, suffix))))
+    }
+
+    fn radd(&self, lhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
+        let prefix = lhs.unpack_str()?;
+        Some(Ok(heap.alloc(format!("{}{}", prefix, self.default))))
+    }
+}
+
+#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
+pub struct PhaseContext {
+    source_dir: SourceDir,
+    build_dir: String,
+    prefix: String,
+    sysroot: String,
+    jobs: i32,
+}
+
+impl PhaseContext {
+    pub fn new(source_dir: SourceDir, prefix: &str, jobs: i32) -> Self {
+        Self {
+            source_dir,
+            build_dir: "/build".to_owned(),
+            prefix: prefix.to_owned(),
+            sysroot: "/sysroot".to_owned(),
+            jobs,
+        }
+    }
+}
+
+impl std::fmt::Display for PhaseContext {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "ctx")
+    }
+}
+
+starlark::starlark_simple_value!(PhaseContext);
+
+#[starlark_value(type = "phase_context")]
+impl<'v> StarlarkValue<'v> for PhaseContext {
+    fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
+        Some(match attr {
+            "source_dir" => heap.alloc(self.source_dir.clone()),
+            "build_dir" => heap.alloc(self.build_dir.as_str()),
+            "prefix" => heap.alloc(self.prefix.as_str()),
+            "sysroot" => heap.alloc(self.sysroot.as_str()),
+            "jobs" => heap.alloc(self.jobs),
+            _ => return None,
+        })
+    }
+
+    fn has_attr(&self, attr: &str, _heap: &'v Heap) -> bool {
+        matches!(
+            attr,
+            "source_dir" | "build_dir" | "prefix" | "sysroot" | "jobs" | "run"
+        )
+    }
+
+    fn dir_attr(&self) -> Vec<String> {
+        [
+            "source_dir",
+            "build_dir",
+            "prefix",
+            "sysroot",
+            "jobs",
+            "run",
+        ]
+        .into_iter()
+        .map(String::from)
+        .collect()
+    }
+
+    fn get_methods() -> Option<&'static Methods> {
+        static RES: MethodsStatic = MethodsStatic::new();
+        RES.methods(phase_context_methods)
+    }
+}
+
+#[starlark_module]
+fn phase_context_methods(builder: &mut MethodsBuilder) {
+    fn run<'v>(
+        #[starlark(this)] _this: Value<'v>,
+        #[starlark(require = pos)] argv: UnpackList<String>,
+        #[starlark(require = named)] env: Option<SmallMap<String, String>>,
+    ) -> anyhow::Result<NoneType> {
+        run_in_container(&argv.items, env.unwrap_or_default())?;
+        Ok(NoneType)
+    }
+}
+
+fn run_in_container(
+    argv: &[String],
+    env_overrides: SmallMap<String, String>,
+) -> anyhow::Result<()> {
+    let (container, env) = with_current(|runtime| {
+        let mut env: Vec<(String, String)> = runtime
+            .base_env
+            .iter()
+            .cloned()
+            .map(|(k, v)| {
+                if k == "PATH" {
+                    (k, runtime.base_path.clone())
+                } else {
+                    (k, v)
+                }
+            })
+            .collect();
+        for (k, v) in env_overrides {
+            if let Some(slot) = env.iter_mut().find(|(existing, _)| existing == &k) {
+                slot.1 = v;
+            } else {
+                env.push((k, v));
+            }
+        }
+        (runtime.container.clone(), env)
+    })?;
+    container.borrow().exec(argv, &env, "/build")
+}
+
+#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
+pub struct PackageContext {
+    dest_dir: String,
+}
+
+impl PackageContext {
+    pub fn new(dest_dir: String) -> Self {
+        Self { dest_dir }
+    }
+}
+
+impl std::fmt::Display for PackageContext {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "pkg")
+    }
+}
+
+starlark::starlark_simple_value!(PackageContext);
+
+#[starlark_value(type = "package_context")]
+impl<'v> StarlarkValue<'v> for PackageContext {
+    fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
+        match attr {
+            "dest_dir" => Some(heap.alloc(self.dest_dir.as_str())),
+            _ => None,
+        }
+    }
+
+    fn has_attr(&self, attr: &str, _heap: &'v Heap) -> bool {
+        attr == "dest_dir"
+    }
+
+    fn dir_attr(&self) -> Vec<String> {
+        vec!["dest_dir".to_owned()]
+    }
+}
+
+pub fn invoke_phase(func: &OwnedFrozenValue, args: &[PhaseArg]) -> anyhow::Result<()> {
+    let module = Module::new();
+    let mut eval = Evaluator::new(&module);
+    let allocated: Vec<Value<'_>> = args
+        .iter()
+        .map(|arg| match arg {
+            PhaseArg::Ctx(ctx) => module.heap().alloc(ctx.clone()),
+            PhaseArg::Pkg(pkg) => module.heap().alloc(pkg.clone()),
+        })
+        .collect();
+    eval.eval_function(func.value(), &allocated, &[])
+        .map_err(|err| anyhow!("{err}"))?;
+    Ok(())
+}
+
+pub enum PhaseArg {
+    Ctx(PhaseContext),
+    Pkg(PackageContext),
+}

src/recipe/mod.rs

@@ -5,7 +5,10 @@ mod subpackage;
 use anyhow::{Context, bail};
 use starlark::{
     environment::{FrozenModule, Module},
-    values::{OwnedFrozenValue, UnpackValue, ValueLike, list::ListRef, typing::StarlarkCallable},
+    values::{
+        OwnedFrozenValue, UnpackValue, ValueLike, dict::DictRef, list::ListRef,
+        typing::StarlarkCallable,
+    },
 };
 use std::{
     collections::HashMap,
@@ -141,14 +144,47 @@ impl Recipe {
                 .clone(),
         };
 
-        let source_value = module
+        let source_value = module.get("source");
-            .get("source")
+        let sources_value = module.get("sources");
-            .ok_or_else(|| anyhow::anyhow!("field `source`: missing"))?;
+        let sources = match (source_value, sources_value) {
-        let source = source_value
+            (None, None) => bail!("recipe must define either `source` or `sources`"),
-            .downcast_ref::<Source>()
+            (Some(_), Some(_)) => {
-            .ok_or_else(|| anyhow::anyhow!("field `source`: expected a source value"))?
+                bail!("recipe must define exactly one of `source` or `sources`, not both")
-            .clone();
+            }
-        let sources = Sources::Single(source);
+            (Some(value), None) => {
+                let source = value
+                    .downcast_ref::<Source>()
+                    .ok_or_else(|| anyhow::anyhow!("field `source`: expected a source value"))?
+                    .clone();
+                Sources::Single(source)
+            }
+            (None, Some(value)) => {
+                let dict = DictRef::from_value(value).ok_or_else(|| {
+                    anyhow::anyhow!("field `sources`: expected a dict of name -> source")
+                })?;
+                if dict.iter().len() == 0 {
+                    bail!("field `sources`: must contain at least one entry");
+                }
+                let mut map: HashMap<String, source::Source> = HashMap::new();
+                for (key, value) in dict.iter() {
+                    let key_str = key
+                        .unpack_str()
+                        .ok_or_else(|| anyhow::anyhow!("field `sources`: keys must be strings"))?;
+                    if !is_valid_source_name(key_str) {
+                        bail!(
+                            "field `sources`: invalid source name `{key_str}` (allowed: letters, digits, `-`, `_`; must start with a letter or digit)"
+                        );
+                    }
+                    let source = value.downcast_ref::<Source>().ok_or_else(|| {
+                        anyhow::anyhow!("field `sources`: entry `{key_str}` is not a source value")
+                    })?;
+                    if map.insert(key_str.to_owned(), source.clone()).is_some() {
+                        bail!("field `sources`: duplicate key `{key_str}`");
+                    }
+                }
+                Sources::Multiple(map)
+            }
+        };
 
         let host_deps = optional_string_list(&module, "host_deps")?;
         let build_deps = optional_string_list(&module, "build_deps")?;
@@ -275,6 +311,17 @@ fn optional_string_list(module: &Module, key: &str) -> anyhow::Result<Vec<String
     }
 }
 
+fn is_valid_source_name(name: &str) -> bool {
+    let mut chars = name.chars();
+    let Some(first) = chars.next() else {
+        return false;
+    };
+    if !first.is_ascii_alphanumeric() {
+        return false;
+    }
+    chars.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
+}
+
 pub struct RecipePhases {
     configure: Option<OwnedFrozenValue>,
     build: OwnedFrozenValue,
@@ -368,6 +415,10 @@ impl OutputPackage {
     pub fn name(&self) -> &str {
         &self.name
     }
+
+    pub fn metadata(&self) -> &Metadata {
+        &self.metadata
+    }
 }
 
 #[derive(Debug)]