1144 lines
36 KiB
Rust
1144 lines
36 KiB
Rust
use std::{
|
|
cell::RefCell,
|
|
collections::{BTreeSet, VecDeque},
|
|
fs,
|
|
io::Write,
|
|
path::{Path, PathBuf},
|
|
process::{Command, Stdio},
|
|
rc::Rc,
|
|
thread,
|
|
time::{SystemTime, UNIX_EPOCH},
|
|
};
|
|
|
|
use anyhow::{Context, bail};
|
|
use sha2::{Digest, Sha256};
|
|
use starlark::values::OwnedFrozenValue;
|
|
|
|
use crate::{
|
|
config::Config,
|
|
container::{Container, Mount},
|
|
graph::{TaskId, TaskPlan, TaskPlanner, task_recipe_slug},
|
|
layout::Layout,
|
|
log,
|
|
phase::{
|
|
self, PackageContext, PhaseArg, PhaseContext, PhaseRuntime, PhaseRuntimeGuard, SourceDir,
|
|
},
|
|
recipe::{GitSource, OutputPackage, Recipe, RecipeKind, RecipeSet, Source},
|
|
};
|
|
|
|
const SPLIT_SUBPACKAGE_SCRIPT: &str = r#"
|
|
base=$1
|
|
dest=$2
|
|
package=$3
|
|
shift 3
|
|
|
|
mkdir -p "$dest"
|
|
for pattern do
|
|
matches=$(mktemp)
|
|
find "$base" -depth -path "$base/$pattern" -print > "$matches"
|
|
if ! [ -s "$matches" ]; then
|
|
echo "subpackage $package: glob '$pattern' matched no files under $base" >&2
|
|
rm -f "$matches"
|
|
exit 1
|
|
fi
|
|
|
|
while IFS= read -r path; do
|
|
[ "$path" != "$base" ] || continue
|
|
rel=${path#"$base"/}
|
|
target=$dest/$rel
|
|
mkdir -p "$(dirname "$target")"
|
|
echo "$rel"
|
|
mv "$path" "$target"
|
|
done < "$matches"
|
|
rm -f "$matches"
|
|
done
|
|
|
|
find "$base" -depth -type d -empty -delete
|
|
"#;
|
|
|
|
#[derive(Debug)]
|
|
pub struct Builder {
|
|
root: PathBuf,
|
|
config: Config,
|
|
container_ready: bool,
|
|
}
|
|
|
|
impl Builder {
|
|
pub fn new(root: PathBuf, config: Config) -> Self {
|
|
Self {
|
|
root,
|
|
config,
|
|
container_ready: false,
|
|
}
|
|
}
|
|
|
|
pub fn build(
|
|
&mut self,
|
|
recipes: &RecipeSet,
|
|
requested: &[String],
|
|
rebuild: bool,
|
|
dry_run: bool,
|
|
) -> anyhow::Result<()> {
|
|
let requested = filter_skipped(recipes, requested);
|
|
let plan = TaskPlanner::new(&self.root, &self.config.arch, recipes)
|
|
.build_plan(&requested, rebuild)?;
|
|
self.print_plan(recipes, &plan)?;
|
|
if dry_run {
|
|
return Ok(());
|
|
}
|
|
self.execute_plan(recipes, &plan)
|
|
}
|
|
|
|
pub fn fetch(
|
|
&mut self,
|
|
recipes: &RecipeSet,
|
|
requested: &[String],
|
|
dry_run: bool,
|
|
) -> anyhow::Result<()> {
|
|
let requested = filter_skipped(recipes, requested);
|
|
let plan =
|
|
TaskPlanner::new(&self.root, &self.config.arch, recipes).fetch_plan(&requested)?;
|
|
self.print_plan(recipes, &plan)?;
|
|
if dry_run {
|
|
return Ok(());
|
|
}
|
|
self.execute_plan(recipes, &plan)
|
|
}
|
|
|
|
pub fn ensure_container_ready(&mut self) -> anyhow::Result<()> {
|
|
if !self.container_ready {
|
|
self.ensure_container_image(&self.abs_config_path(&self.config.container_dockerfile))?;
|
|
self.container_ready = true;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn execute_plan(&mut self, recipes: &RecipeSet, plan: &TaskPlan) -> anyhow::Result<()> {
|
|
if plan.is_empty() {
|
|
return Ok(());
|
|
}
|
|
|
|
let mut active: Option<ActiveContainer> = None;
|
|
|
|
for task in plan.order() {
|
|
let recipe_slug = task_recipe_slug(task, recipes)?;
|
|
|
|
if let Some(current) = active.as_ref() {
|
|
if current.recipe_key != recipe_slug {
|
|
active = None;
|
|
}
|
|
}
|
|
|
|
if task_needs_container(task) && active.is_none() {
|
|
self.ensure_container_ready()?;
|
|
active = Some(self.start_recipe_container(recipes, &recipe_slug)?);
|
|
}
|
|
|
|
let layout = Layout::new(&self.root, &self.config.arch);
|
|
self.run_task(&layout, recipes, task, active.as_ref())?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn run_task(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipes: &RecipeSet,
|
|
task: &TaskId,
|
|
active: Option<&ActiveContainer>,
|
|
) -> anyhow::Result<()> {
|
|
match task {
|
|
TaskId::FetchSources(key) => {
|
|
let recipe = recipes.recipe(key)?;
|
|
self.task_fetch_sources(layout, recipe)
|
|
}
|
|
TaskId::PrepareSources(key) => {
|
|
let recipe = recipes.recipe(key)?;
|
|
self.task_prepare_sources(layout, recipe)
|
|
}
|
|
TaskId::PrepareRecipe(key) => {
|
|
let recipe = recipes.recipe(key)?;
|
|
let active = active.expect("recipe prepare task requires an active container");
|
|
self.task_prepare_recipe(layout, recipe, active)
|
|
}
|
|
TaskId::ConfigureRecipe(key) => {
|
|
let recipe = recipes.recipe(key)?;
|
|
let active = active.expect("configure task requires an active container");
|
|
self.task_configure(layout, recipe, active)
|
|
}
|
|
TaskId::BuildRecipe(key) => {
|
|
let recipe = recipes.recipe(key)?;
|
|
let active = active.expect("build task requires an active container");
|
|
self.task_build(layout, recipe, active)
|
|
}
|
|
TaskId::InstallPackageFiles(output_key) => {
|
|
let output = recipes.output(output_key)?;
|
|
let recipe = recipes.recipe(output.recipe())?;
|
|
let active = active.expect("install task requires an active container");
|
|
self.task_install_package(layout, recipe, output, active)
|
|
}
|
|
TaskId::ProduceApk(output_key) => {
|
|
let output = recipes.output(output_key)?;
|
|
let recipe = recipes.recipe(output.recipe())?;
|
|
let active = active.expect("apk task requires an active container");
|
|
self.task_produce_apk(layout, recipe, output, active)
|
|
}
|
|
TaskId::InstallHostRecipe(key) => {
|
|
let recipe = recipes.recipe(key)?;
|
|
let active = active.expect("host install task requires an active container");
|
|
self.task_install_host(layout, recipe, active)
|
|
}
|
|
}
|
|
}
|
|
|
|
fn task_fetch_sources(&self, layout: &Layout<'_>, recipe: &Recipe) -> anyhow::Result<()> {
|
|
fs::create_dir_all(layout.source_cache_dir())?;
|
|
for (name, source) in recipe.sources().entries() {
|
|
let label = name.unwrap_or("source");
|
|
if !source.is_unknown_cache_key()
|
|
&& layout.source_cache_path(source.cache_key()).exists()
|
|
{
|
|
log::skip("fetch", &format!("{} ({})", recipe.key(), label));
|
|
continue;
|
|
}
|
|
log::step(
|
|
"fetch",
|
|
&format!("{} ({}) from {}", recipe.key(), label, source.url()),
|
|
);
|
|
match source {
|
|
Source::Tarball(_) => self.fetch_tarball(layout, recipe, source)?,
|
|
Source::Git(git) => self.fetch_git(layout, recipe, git)?,
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn fetch_tarball(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
source: &Source,
|
|
) -> anyhow::Result<()> {
|
|
let url = source.url();
|
|
let expected = source.cache_key().to_owned();
|
|
let unknown = source.is_unknown_cache_key();
|
|
|
|
let client = reqwest::blocking::Client::builder()
|
|
.user_agent("distro-builder")
|
|
.build()
|
|
.context("building HTTP client")?;
|
|
let mut response = client
|
|
.get(url)
|
|
.send()
|
|
.with_context(|| format!("downloading {url}"))?
|
|
.error_for_status()
|
|
.with_context(|| format!("downloading {url}"))?;
|
|
|
|
let cache_dir = layout.source_cache_dir();
|
|
fs::create_dir_all(&cache_dir)?;
|
|
let mut tmp = tempfile::NamedTempFile::new_in(&cache_dir)
|
|
.with_context(|| format!("creating temp file in {}", cache_dir.display()))?;
|
|
let mut hasher = Sha256::new();
|
|
let mut buf: [u8; 65536] = [0u8; 64 * 1024];
|
|
loop {
|
|
let n = std::io::Read::read(&mut response, &mut buf)
|
|
.with_context(|| format!("reading response body for {url}"))?;
|
|
if n == 0 {
|
|
break;
|
|
}
|
|
hasher.update(&buf[..n]);
|
|
tmp.write_all(&buf[..n])?;
|
|
}
|
|
tmp.flush()?;
|
|
let hash = hex::encode(hasher.finalize());
|
|
|
|
if unknown {
|
|
log::info(
|
|
"fetch",
|
|
&format!("{}: computed sha256 = {hash}", recipe.key()),
|
|
);
|
|
} else if hash != expected {
|
|
bail!("sha256 mismatch for {url}: expected {expected}, got {hash}");
|
|
}
|
|
|
|
let final_path = layout.source_cache_path(&hash);
|
|
if !final_path.exists() {
|
|
tmp.persist(&final_path).with_context(|| {
|
|
format!("renaming downloaded archive to {}", final_path.display())
|
|
})?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn fetch_git(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
source: &GitSource,
|
|
) -> anyhow::Result<()> {
|
|
let url = source.url();
|
|
let commit = source.commit();
|
|
let unknown = matches!(commit, "?" | "???");
|
|
|
|
let cache_dir = layout.source_cache_dir();
|
|
fs::create_dir_all(&cache_dir)?;
|
|
let tmp = tempfile::tempdir_in(&cache_dir)?;
|
|
let work = tmp.path().join("repo");
|
|
|
|
let status = Command::new("git")
|
|
.arg("clone")
|
|
.arg("--bare")
|
|
.arg(url)
|
|
.arg(&work)
|
|
.status()
|
|
.with_context(|| format!("spawning git clone {url}"))?;
|
|
if !status.success() {
|
|
bail!("git clone failed for {url} with {status}");
|
|
}
|
|
|
|
let rev = if unknown {
|
|
let out = Command::new("git")
|
|
.arg("-C")
|
|
.arg(&work)
|
|
.arg("rev-parse")
|
|
.arg("HEAD")
|
|
.output()
|
|
.context("spawning git rev-parse HEAD")?;
|
|
if !out.status.success() {
|
|
bail!("git rev-parse HEAD failed with {}", out.status);
|
|
}
|
|
let rev = String::from_utf8(out.stdout)?.trim().to_owned();
|
|
log::info(
|
|
"fetch",
|
|
&format!("{}: resolved git HEAD = {rev}", recipe.key()),
|
|
);
|
|
rev
|
|
} else {
|
|
let status = Command::new("git")
|
|
.arg("-C")
|
|
.arg(&work)
|
|
.arg("cat-file")
|
|
.arg("-e")
|
|
.arg(commit)
|
|
.status()
|
|
.context("spawning git cat-file")?;
|
|
if !status.success() {
|
|
bail!("commit {commit} not found in {url}");
|
|
}
|
|
commit.to_owned()
|
|
};
|
|
|
|
let final_path = layout.source_cache_path(&rev);
|
|
if !final_path.exists() {
|
|
fs::rename(&work, &final_path).with_context(|| {
|
|
format!(
|
|
"moving git clone to {} from {}",
|
|
final_path.display(),
|
|
work.display()
|
|
)
|
|
})?;
|
|
}
|
|
|
|
if unknown {
|
|
bail!(
|
|
"{}: source commit is unknown; update the recipe with commit = \"{rev}\"",
|
|
recipe.key()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn task_prepare_sources(&self, layout: &Layout<'_>, recipe: &Recipe) -> anyhow::Result<()> {
|
|
log::step("prepare", &recipe.key());
|
|
let src = layout.source_workdir(recipe);
|
|
if src.exists() {
|
|
fs::remove_dir_all(&src).with_context(|| format!("clearing {}", src.display()))?;
|
|
}
|
|
fs::create_dir_all(&src)?;
|
|
let build = layout.build_workdir(recipe);
|
|
if build.exists() {
|
|
fs::remove_dir_all(&build).with_context(|| format!("clearing {}", build.display()))?;
|
|
}
|
|
fs::create_dir_all(&build)?;
|
|
|
|
for (name, source) in recipe.sources().entries() {
|
|
if source.is_unknown_cache_key() {
|
|
bail!(
|
|
"source for {} has an unknown cache key; run `distro fetch` first",
|
|
recipe.key()
|
|
);
|
|
}
|
|
let cache_path = layout.source_cache_path(source.cache_key());
|
|
if !cache_path.exists() {
|
|
bail!(
|
|
"missing cached source for {} at {}",
|
|
recipe.key(),
|
|
cache_path.display()
|
|
);
|
|
}
|
|
let dst = match name {
|
|
None => src.clone(),
|
|
Some(named) => {
|
|
let dst = src.join(named);
|
|
fs::create_dir_all(&dst)?;
|
|
dst
|
|
}
|
|
};
|
|
match source {
|
|
Source::Tarball(tar) => {
|
|
let mut cmd = Command::new("tar");
|
|
cmd.arg("-xf").arg(&cache_path);
|
|
if tar.strip_components() > 0 {
|
|
cmd.arg(format!("--strip-components={}", tar.strip_components()));
|
|
}
|
|
cmd.arg("-C").arg(&dst);
|
|
let status = cmd
|
|
.status()
|
|
.with_context(|| format!("spawning tar -xf {}", cache_path.display()))?;
|
|
if !status.success() {
|
|
bail!(
|
|
"tar extraction of {} failed with {status}",
|
|
cache_path.display()
|
|
);
|
|
}
|
|
}
|
|
Source::Git(git) => {
|
|
let status = Command::new("git")
|
|
.arg("clone")
|
|
.arg(&cache_path)
|
|
.arg(&dst)
|
|
.status()
|
|
.context("spawning git clone from cache")?;
|
|
if !status.success() {
|
|
bail!("git clone from cache failed with {status}");
|
|
}
|
|
let status = Command::new("git")
|
|
.arg("-C")
|
|
.arg(&dst)
|
|
.arg("checkout")
|
|
.arg(git.commit())
|
|
.status()
|
|
.context("spawning git checkout")?;
|
|
if !status.success() {
|
|
bail!("git checkout {} failed with {status}", git.commit());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let version_stamp = layout.source_stamp(recipe, "version");
|
|
fs::create_dir_all(version_stamp.parent().unwrap())?;
|
|
fs::write(
|
|
&version_stamp,
|
|
format!("{}-r{}", recipe.version(), recipe.revision()),
|
|
)?;
|
|
|
|
if layout.recipe_has_patches(recipe)? {
|
|
log::skip(
|
|
"patches",
|
|
&format!("{}: patch application not implemented", recipe.key()),
|
|
);
|
|
fs::write(layout.source_stamp(recipe, "patched"), b"skipped")?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn task_prepare_recipe(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
active: &ActiveContainer,
|
|
) -> anyhow::Result<()> {
|
|
let Some(func) = recipe.phases().prepare() else {
|
|
return Ok(());
|
|
};
|
|
|
|
log::step("prepare", &format!("{} (recipe phase)", recipe.key()));
|
|
let ctx = phase_context_for(recipe);
|
|
self.invoke_with_runtime(active, &[PhaseArg::Ctx(ctx)], func)?;
|
|
self.write_recipe_stamp(layout, recipe, "prepare")
|
|
}
|
|
|
|
fn task_configure(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
active: &ActiveContainer,
|
|
) -> anyhow::Result<()> {
|
|
log::step("configure", &recipe.key());
|
|
if let Some(func) = recipe.phases().configure() {
|
|
let ctx = phase_context_for(recipe);
|
|
self.invoke_with_runtime(active, &[PhaseArg::Ctx(ctx)], func)?;
|
|
}
|
|
self.write_recipe_stamp(layout, recipe, "configure")
|
|
}
|
|
|
|
fn task_build(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
active: &ActiveContainer,
|
|
) -> anyhow::Result<()> {
|
|
log::step("build", &recipe.key());
|
|
let ctx = phase_context_for(recipe);
|
|
self.invoke_with_runtime(active, &[PhaseArg::Ctx(ctx)], recipe.phases().build())?;
|
|
self.write_recipe_stamp(layout, recipe, "build")
|
|
}
|
|
|
|
fn task_install_package(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
output: &OutputPackage,
|
|
active: &ActiveContainer,
|
|
) -> anyhow::Result<()> {
|
|
log::step("install", &output.key());
|
|
if !output.is_base() {
|
|
return self.task_split_subpackage(layout, recipe, output, active);
|
|
}
|
|
|
|
let dest = format!("/dest/{}", output.name());
|
|
active.container.borrow().exec(
|
|
&["mkdir".to_owned(), "-p".to_owned(), dest.clone()],
|
|
&base_env(&active.base_path),
|
|
"/",
|
|
)?;
|
|
let ctx = phase_context_for(recipe);
|
|
let pkg = PackageContext::new(dest);
|
|
self.invoke_with_runtime(
|
|
active,
|
|
&[PhaseArg::Ctx(ctx), PhaseArg::Pkg(pkg)],
|
|
recipe.phases().install(),
|
|
)?;
|
|
self.write_output_stamp(layout, recipe, output, "install")
|
|
}
|
|
|
|
fn task_split_subpackage(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
output: &OutputPackage,
|
|
active: &ActiveContainer,
|
|
) -> anyhow::Result<()> {
|
|
let base = recipe
|
|
.base_output()
|
|
.ok_or_else(|| anyhow::anyhow!("recipe `{}` has no base output", recipe.key()))?;
|
|
let base_dest = format!("/dest/{}", base.name());
|
|
let dest = format!("/dest/{}", output.name());
|
|
let mut argv: Vec<String> = vec![
|
|
"sh".to_owned(),
|
|
"-eu".to_owned(),
|
|
"-c".to_owned(),
|
|
SPLIT_SUBPACKAGE_SCRIPT.to_owned(),
|
|
"split-subpackage".to_owned(),
|
|
base_dest,
|
|
dest,
|
|
output.name().to_owned(),
|
|
];
|
|
argv.extend(output.file_globs().iter().cloned());
|
|
active
|
|
.container
|
|
.borrow()
|
|
.exec(&argv, &base_env(&active.base_path), "/")?;
|
|
self.write_output_stamp(layout, recipe, output, "install")
|
|
}
|
|
|
|
fn task_install_host(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
active: &ActiveContainer,
|
|
) -> anyhow::Result<()> {
|
|
log::step("host-install", &recipe.key());
|
|
let dest = format!("/dest/{}", recipe.slug());
|
|
active.container.borrow().exec(
|
|
&["mkdir".to_owned(), "-p".to_owned(), dest.clone()],
|
|
&base_env(&active.base_path),
|
|
"/",
|
|
)?;
|
|
let ctx = phase_context_for(recipe);
|
|
let pkg = PackageContext::new(dest.clone());
|
|
self.invoke_with_runtime(
|
|
active,
|
|
&[PhaseArg::Ctx(ctx), PhaseArg::Pkg(pkg)],
|
|
recipe.phases().install(),
|
|
)?;
|
|
|
|
let host_root = layout.host_install_root(recipe);
|
|
if host_root.exists() {
|
|
fs::remove_dir_all(&host_root)
|
|
.with_context(|| format!("clearing {}", host_root.display()))?;
|
|
}
|
|
fs::create_dir_all(&host_root)?;
|
|
active
|
|
.container
|
|
.borrow()
|
|
.cp_out(&format!("{dest}/."), &host_root)?;
|
|
|
|
self.write_recipe_stamp(layout, recipe, "host-install")
|
|
}
|
|
|
|
fn task_produce_apk(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
output: &OutputPackage,
|
|
active: &ActiveContainer,
|
|
) -> anyhow::Result<()> {
|
|
log::step("apk", &output.key());
|
|
let files_dir = format!("/dest/{}", output.name());
|
|
let file_name = format!(
|
|
"{}-{}-r{}.apk",
|
|
output.name(),
|
|
recipe.version(),
|
|
recipe.revision()
|
|
);
|
|
let out_in_container = format!("/pkgs/{file_name}");
|
|
let version = format!("{}-r{}", recipe.version(), recipe.revision());
|
|
|
|
let mut argv: Vec<String> = vec![
|
|
"apk".to_owned(),
|
|
"mkpkg".to_owned(),
|
|
"--files".to_owned(),
|
|
files_dir,
|
|
"--output".to_owned(),
|
|
out_in_container,
|
|
"--info".to_owned(),
|
|
format!("name:{}", output.name()),
|
|
"--info".to_owned(),
|
|
format!("version:{version}"),
|
|
"--info".to_owned(),
|
|
format!("arch:{}", self.config.arch),
|
|
"--info".to_owned(),
|
|
format!("origin:{}", recipe.key()),
|
|
];
|
|
if let Some(desc) = output.metadata().description() {
|
|
argv.push("--info".to_owned());
|
|
argv.push(format!("description:{desc}"));
|
|
}
|
|
if let Some(license) = output.metadata().license() {
|
|
argv.push("--info".to_owned());
|
|
argv.push(format!("license:{license}"));
|
|
}
|
|
if let Some(url) = output.metadata().website() {
|
|
argv.push("--info".to_owned());
|
|
argv.push(format!("url:{url}"));
|
|
}
|
|
if let Some(packager) = output.metadata().maintainer() {
|
|
argv.push("--info".to_owned());
|
|
argv.push(format!("packager:{packager}"));
|
|
}
|
|
for dep in recipe.deps().iter().chain(recipe.run_deps().iter()) {
|
|
argv.push("--info".to_owned());
|
|
argv.push(format!("depends:{dep}"));
|
|
}
|
|
|
|
active
|
|
.container
|
|
.borrow()
|
|
.exec(&argv, &base_env(&active.base_path), "/")?;
|
|
|
|
self.write_output_stamp(layout, recipe, output, "apk")
|
|
}
|
|
|
|
fn invoke_with_runtime(
|
|
&self,
|
|
active: &ActiveContainer,
|
|
args: &[PhaseArg],
|
|
func: &OwnedFrozenValue,
|
|
) -> anyhow::Result<()> {
|
|
let runtime = PhaseRuntime {
|
|
container: active.container.clone(),
|
|
base_path: active.base_path.clone(),
|
|
base_env: bare_env(),
|
|
};
|
|
let _guard = PhaseRuntimeGuard::enter(runtime);
|
|
phase::invoke_phase(func, args)
|
|
}
|
|
|
|
fn populate_sysroot(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipes: &RecipeSet,
|
|
recipe: &Recipe,
|
|
active: &ActiveContainer,
|
|
deps: &[String],
|
|
) -> anyhow::Result<()> {
|
|
if deps.is_empty() {
|
|
return Ok(());
|
|
}
|
|
|
|
let env = base_env(&active.base_path);
|
|
log::step(
|
|
"sysroot",
|
|
&format!("{} from {} target apk(s)", recipe.key(), deps.len()),
|
|
);
|
|
for dep in deps {
|
|
let output = recipes.output(dep)?;
|
|
let owning = recipes.recipe(output.recipe())?;
|
|
let apk_host = layout.apk_path(owning, output);
|
|
if !apk_host.exists() {
|
|
bail!(
|
|
"missing apk for target dependency `{dep}` at {}; \
|
|
rebuild it first (e.g. `distro build -r {dep}`)",
|
|
apk_host.display()
|
|
);
|
|
}
|
|
let file_name = apk_host
|
|
.file_name()
|
|
.and_then(|n| n.to_str())
|
|
.ok_or_else(|| {
|
|
anyhow::anyhow!("apk path {} has no UTF-8 file name", apk_host.display())
|
|
})?;
|
|
active.container.borrow().exec(
|
|
&[
|
|
"apk".to_owned(),
|
|
"extract".to_owned(),
|
|
"--allow-untrusted".to_owned(),
|
|
"--force-overwrite".to_owned(),
|
|
"--destination".to_owned(),
|
|
"/sysroot".to_owned(),
|
|
format!("/pkgs/{file_name}"),
|
|
],
|
|
&env,
|
|
"/",
|
|
)?;
|
|
}
|
|
|
|
log::info(
|
|
"sysroot",
|
|
&format!(
|
|
"{}: extracted /sysroot from {} target apk(s)",
|
|
recipe.key(),
|
|
deps.len()
|
|
),
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
fn start_recipe_container(
|
|
&self,
|
|
recipes: &RecipeSet,
|
|
recipe_key: &str,
|
|
) -> anyhow::Result<ActiveContainer> {
|
|
let recipe = recipes.recipe(recipe_key)?;
|
|
let layout = Layout::new(&self.root, &self.config.arch);
|
|
|
|
let source_dir = layout.source_workdir(recipe);
|
|
let build_dir = layout.build_workdir(recipe);
|
|
fs::create_dir_all(&source_dir)?;
|
|
fs::create_dir_all(&build_dir)?;
|
|
|
|
let host_deps = transitive_host_deps(recipes, recipe)?;
|
|
let target_deps = transitive_target_deps(recipes, recipe)?;
|
|
let pkgs_dir = self.root.join("build/pkgs").join(&self.config.arch);
|
|
fs::create_dir_all(&pkgs_dir)?;
|
|
|
|
let mut mounts = vec![
|
|
Mount {
|
|
host: source_dir,
|
|
container: "/sources".to_owned(),
|
|
read_only: false,
|
|
},
|
|
Mount {
|
|
host: build_dir,
|
|
container: "/build".to_owned(),
|
|
read_only: false,
|
|
},
|
|
Mount {
|
|
host: pkgs_dir,
|
|
container: "/pkgs".to_owned(),
|
|
read_only: false,
|
|
},
|
|
];
|
|
|
|
if let Some(files_dir) = recipe.files_dir() {
|
|
mounts.push(Mount {
|
|
host: files_dir,
|
|
container: "/files".to_owned(),
|
|
read_only: true,
|
|
});
|
|
}
|
|
|
|
let mut tools_bins: Vec<String> = Vec::new();
|
|
for dep_key in &host_deps {
|
|
let dep_recipe = recipes.recipe(dep_key)?;
|
|
let install = layout.host_install_dir(dep_recipe);
|
|
if !install.exists() {
|
|
bail!(
|
|
"missing host install for {dep_key} at {}; build it first",
|
|
install.display()
|
|
);
|
|
}
|
|
let bare = dep_key.strip_prefix("host:").unwrap_or(dep_key);
|
|
mounts.push(Mount {
|
|
host: install,
|
|
container: format!("/tools/{bare}"),
|
|
read_only: true,
|
|
});
|
|
tools_bins.push(format!("/tools/{bare}/bin"));
|
|
}
|
|
|
|
let name = format!(
|
|
"distro-builder-{}-{:x}",
|
|
std::process::id(),
|
|
random_suffix()
|
|
);
|
|
let container = Container::start(
|
|
&self.config.container_runtime,
|
|
&self.config.container_image,
|
|
&name,
|
|
&mounts,
|
|
)?;
|
|
|
|
let mut path_segments = tools_bins;
|
|
path_segments.push("/usr/local/sbin".to_owned());
|
|
path_segments.push("/usr/local/bin".to_owned());
|
|
path_segments.push("/usr/sbin".to_owned());
|
|
path_segments.push("/usr/bin".to_owned());
|
|
path_segments.push("/sbin".to_owned());
|
|
path_segments.push("/bin".to_owned());
|
|
let base_path = path_segments.join(":");
|
|
|
|
let active = ActiveContainer {
|
|
recipe_key: recipe_key.to_owned(),
|
|
container: Rc::new(RefCell::new(container)),
|
|
base_path,
|
|
};
|
|
|
|
self.populate_sysroot(&layout, recipes, recipe, &active, &target_deps)?;
|
|
|
|
Ok(active)
|
|
}
|
|
|
|
fn write_recipe_stamp(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
kind: &str,
|
|
) -> anyhow::Result<()> {
|
|
let path = layout.recipe_task_stamp(recipe, kind);
|
|
fs::create_dir_all(path.parent().unwrap())?;
|
|
fs::write(path, layout.recipe_fingerprint(recipe)?)?;
|
|
Ok(())
|
|
}
|
|
|
|
fn write_output_stamp(
|
|
&self,
|
|
layout: &Layout<'_>,
|
|
recipe: &Recipe,
|
|
output: &OutputPackage,
|
|
kind: &str,
|
|
) -> anyhow::Result<()> {
|
|
let path = layout.output_task_stamp(output, kind);
|
|
fs::create_dir_all(path.parent().unwrap())?;
|
|
fs::write(path, layout.output_fingerprint(recipe, output)?)?;
|
|
Ok(())
|
|
}
|
|
|
|
fn ensure_container_image(&self, dockerfile: &Path) -> anyhow::Result<()> {
|
|
if !dockerfile.exists() {
|
|
bail!(
|
|
"configured container Dockerfile does not exist: {}",
|
|
dockerfile.display()
|
|
);
|
|
}
|
|
|
|
let hash = self.container_build_hash(dockerfile)?;
|
|
let stamp = self.root.join("build/container-image.hash");
|
|
if fs::read_to_string(&stamp).ok().as_deref() == Some(hash.as_str())
|
|
&& self.container_image_exists()?
|
|
{
|
|
log::skip(
|
|
"image",
|
|
&format!("using cached {}", self.config.container_image),
|
|
);
|
|
return Ok(());
|
|
}
|
|
|
|
log::step(
|
|
"image",
|
|
&format!(
|
|
"building {} from {}",
|
|
self.config.container_image,
|
|
dockerfile.display()
|
|
),
|
|
);
|
|
let runtime = self.config.container_runtime.as_str();
|
|
let status = Command::new(runtime)
|
|
.arg("build")
|
|
.arg("-f")
|
|
.arg(dockerfile)
|
|
.arg("-t")
|
|
.arg(&self.config.container_image)
|
|
.arg(&self.root)
|
|
.status()
|
|
.with_context(|| {
|
|
format!(
|
|
"failed to build container image `{}` from {}",
|
|
self.config.container_image,
|
|
dockerfile.display()
|
|
)
|
|
})?;
|
|
if !status.success() {
|
|
bail!(
|
|
"container image build failed for `{}` with {status}",
|
|
self.config.container_image
|
|
);
|
|
}
|
|
|
|
fs::create_dir_all(stamp.parent().unwrap())?;
|
|
fs::write(stamp, hash)?;
|
|
Ok(())
|
|
}
|
|
|
|
fn container_image_exists(&self) -> anyhow::Result<bool> {
|
|
let runtime = self.config.container_runtime.as_str();
|
|
let status = Command::new(runtime)
|
|
.arg("image")
|
|
.arg("exists")
|
|
.arg(&self.config.container_image)
|
|
.stdout(Stdio::null())
|
|
.stderr(Stdio::null())
|
|
.status()
|
|
.with_context(|| {
|
|
format!(
|
|
"failed to inspect container image `{}`",
|
|
self.config.container_image
|
|
)
|
|
})?;
|
|
Ok(status.success())
|
|
}
|
|
|
|
fn container_build_hash(&self, dockerfile: &Path) -> anyhow::Result<String> {
|
|
let mut hasher = Sha256::new();
|
|
hasher.update(fs::read(dockerfile)?);
|
|
hasher.update(self.config.container_image.as_bytes());
|
|
Ok(hex::encode(hasher.finalize()))
|
|
}
|
|
|
|
fn print_plan(&self, recipes: &RecipeSet, plan: &TaskPlan) -> anyhow::Result<()> {
|
|
if plan.is_empty() {
|
|
log::skip("plan", "nothing to do");
|
|
return Ok(());
|
|
}
|
|
|
|
let task_count = plan.order().len();
|
|
let edge_count = plan.dependency_count();
|
|
log::step(
|
|
"plan",
|
|
&format!(
|
|
"{} {}, {} {}",
|
|
task_count,
|
|
plural(task_count, "task", "tasks"),
|
|
edge_count,
|
|
plural(edge_count, "dependency edge", "dependency edges")
|
|
),
|
|
);
|
|
|
|
let step_width = task_count.to_string().len() * 2 + 1;
|
|
let action_width = plan
|
|
.order()
|
|
.iter()
|
|
.map(|task| plan_task_parts(task).0.len())
|
|
.max()
|
|
.unwrap_or(0);
|
|
let mut current_recipe: Option<String> = None;
|
|
|
|
for (index, task) in plan.order().iter().enumerate() {
|
|
let recipe_slug = task_recipe_slug(task, recipes)?;
|
|
if current_recipe.as_deref() != Some(recipe_slug.as_str()) {
|
|
if current_recipe.is_some() {
|
|
println!();
|
|
}
|
|
println!(" {recipe_slug}");
|
|
current_recipe = Some(recipe_slug.clone());
|
|
}
|
|
|
|
let step = format!("{}/{}", index + 1, task_count);
|
|
let (action, target) = plan_task_parts(task);
|
|
if target == recipe_slug {
|
|
println!(" {step:>step_width$} {action}");
|
|
} else {
|
|
println!(" {step:>step_width$} {action:<action_width$} {target}");
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn abs_config_path(&self, path: &Path) -> PathBuf {
|
|
if path.is_absolute() {
|
|
path.to_path_buf()
|
|
} else {
|
|
self.root.join(path)
|
|
}
|
|
}
|
|
}
|
|
|
|
struct ActiveContainer {
|
|
recipe_key: String,
|
|
container: Rc<RefCell<Container>>,
|
|
base_path: String,
|
|
}
|
|
|
|
fn filter_skipped(recipes: &RecipeSet, requested: &[String]) -> Vec<String> {
|
|
requested
|
|
.iter()
|
|
.filter(|key| !recipes.is_skipped(key))
|
|
.cloned()
|
|
.collect()
|
|
}
|
|
|
|
fn task_needs_container(task: &TaskId) -> bool {
|
|
matches!(
|
|
task,
|
|
TaskId::ConfigureRecipe(_)
|
|
| TaskId::BuildRecipe(_)
|
|
| TaskId::InstallPackageFiles(_)
|
|
| TaskId::ProduceApk(_)
|
|
| TaskId::InstallHostRecipe(_)
|
|
| TaskId::PrepareRecipe(_)
|
|
)
|
|
}
|
|
|
|
fn plan_task_parts(task: &TaskId) -> (&'static str, &str) {
|
|
match task {
|
|
TaskId::FetchSources(recipe) => ("fetch sources", recipe),
|
|
TaskId::PrepareSources(recipe) => ("prepare sources", recipe),
|
|
TaskId::PrepareRecipe(recipe) => ("recipe prepare", recipe),
|
|
TaskId::ConfigureRecipe(recipe) => ("configure", recipe),
|
|
TaskId::BuildRecipe(recipe) => ("build", recipe),
|
|
TaskId::InstallPackageFiles(output) => ("install files", output),
|
|
TaskId::ProduceApk(output) => ("produce apk", output),
|
|
TaskId::InstallHostRecipe(recipe) => ("install host", recipe),
|
|
}
|
|
}
|
|
|
|
fn plural<'a>(count: usize, singular: &'a str, plural: &'a str) -> &'a str {
|
|
if count == 1 { singular } else { plural }
|
|
}
|
|
|
|
fn phase_context_for(recipe: &Recipe) -> PhaseContext {
|
|
let files = recipe.files_dir().map(|_| "/files".to_owned());
|
|
PhaseContext::new(source_dir_for(recipe), default_jobs(), files)
|
|
}
|
|
|
|
fn source_dir_for(recipe: &Recipe) -> SourceDir {
|
|
let entries = recipe.sources().entries();
|
|
let named: Vec<(&str, &crate::recipe::Source)> = entries
|
|
.iter()
|
|
.filter_map(|(name, src)| name.map(|n| (n, *src)))
|
|
.collect();
|
|
if named.is_empty() {
|
|
SourceDir::single("/sources")
|
|
} else {
|
|
SourceDir::named(
|
|
named
|
|
.into_iter()
|
|
.map(|(n, _)| (n.to_owned(), format!("/sources/{n}"))),
|
|
)
|
|
}
|
|
}
|
|
|
|
fn default_jobs() -> i32 {
|
|
thread::available_parallelism()
|
|
.map(|n| n.get() as i32)
|
|
.unwrap_or(1)
|
|
}
|
|
|
|
const TERMINAL_ENV_VARS: &[&str] = &[
|
|
"TERM",
|
|
"COLORTERM",
|
|
"NO_COLOR",
|
|
"CLICOLOR",
|
|
"CLICOLOR_FORCE",
|
|
"FORCE_COLOR",
|
|
"TERM_PROGRAM",
|
|
"TERM_PROGRAM_VERSION",
|
|
];
|
|
|
|
fn bare_env() -> Vec<(String, String)> {
|
|
let mut env = vec![
|
|
("PATH".to_owned(), String::new()),
|
|
("HOME".to_owned(), "/tmp".to_owned()),
|
|
("LC_ALL".to_owned(), "C".to_owned()),
|
|
];
|
|
env.extend(TERMINAL_ENV_VARS.iter().filter_map(|key| {
|
|
std::env::var(key)
|
|
.ok()
|
|
.filter(|value| !value.is_empty())
|
|
.map(|value| ((*key).to_owned(), value))
|
|
}));
|
|
env
|
|
}
|
|
|
|
fn base_env(path: &str) -> Vec<(String, String)> {
|
|
let mut env = bare_env();
|
|
if let Some(slot) = env.iter_mut().find(|(k, _)| k == "PATH") {
|
|
slot.1 = path.to_owned();
|
|
}
|
|
env
|
|
}
|
|
|
|
fn transitive_host_deps(recipes: &RecipeSet, recipe: &Recipe) -> anyhow::Result<Vec<String>> {
|
|
let mut order: Vec<String> = Vec::new();
|
|
let mut seen: BTreeSet<String> = BTreeSet::new();
|
|
let mut queue: VecDeque<String> = recipe
|
|
.host_deps()
|
|
.iter()
|
|
.map(|d| RecipeKind::HostPackage.key(d))
|
|
.collect();
|
|
while let Some(key) = queue.pop_front() {
|
|
if !seen.insert(key.clone()) {
|
|
continue;
|
|
}
|
|
let dep = recipes.recipe(&key)?;
|
|
for sub in dep.host_deps() {
|
|
queue.push_back(RecipeKind::HostPackage.key(sub));
|
|
}
|
|
order.push(key);
|
|
}
|
|
Ok(order)
|
|
}
|
|
|
|
/// Compute the transitive closure of a recipe's target package dependencies.
|
|
/// The recipe's own `build_deps` and `deps` seed the queue; for each visited
|
|
/// dependency we follow its `deps` (runtime+build-needed) recursively, but
|
|
/// not its `build_deps` (build-time-only relative to *that* package, hence
|
|
/// not propagated outward).
|
|
fn transitive_target_deps(recipes: &RecipeSet, recipe: &Recipe) -> anyhow::Result<Vec<String>> {
|
|
let mut order: Vec<String> = Vec::new();
|
|
let mut seen: BTreeSet<String> = BTreeSet::new();
|
|
let mut queue: VecDeque<String> = recipe
|
|
.build_deps()
|
|
.iter()
|
|
.chain(recipe.deps().iter())
|
|
.cloned()
|
|
.collect();
|
|
while let Some(dep) = queue.pop_front() {
|
|
if !seen.insert(dep.clone()) {
|
|
continue;
|
|
}
|
|
let output = recipes.output(&dep)?;
|
|
let owning = recipes.recipe(output.recipe())?;
|
|
for sub in owning.deps() {
|
|
queue.push_back(sub.clone());
|
|
}
|
|
order.push(dep);
|
|
}
|
|
Ok(order)
|
|
}
|
|
|
|
fn random_suffix() -> u64 {
|
|
let nanos = SystemTime::now()
|
|
.duration_since(UNIX_EPOCH)
|
|
.map(|d| d.subsec_nanos() as u64)
|
|
.unwrap_or(0);
|
|
nanos.wrapping_mul(0x9E3779B97F4A7C15)
|
|
}
|