marv7000/distro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: marv7000/distro:0c9a3fde9452a8f4a16a45779b3428715acc7480
Branches Tags
marv7000/distro:main marv7000/distro:python marv7000/distro:rewrite2 marv7000/distro:rewrite
..
compare: marv7000/distro:rewrite
Branches Tags
marv7000/distro:python marv7000/distro:rewrite2 marv7000/distro:rewrite marv7000/distro:main

5 Commits
0c9a3fde94 .. rewrite

Author SHA1 Message Date
iretq f9b6036c95 get rid of ctx.prefix and add run cwd kwarg 2026-05-20 20:52:42 +02:00
marv7000 16d81f509f things 2026-05-20 00:30:38 +02:00
Marvin Friedrich 312750c61b More stuff, didn't test 2026-05-19 18:12:11 +02:00
marv7000 b71906f402 idk 2026-05-19 03:24:20 +02:00
marv7000 45d47e8d84 wip3 2026-05-19 03:24:10 +02:00
36 changed files with 15485 additions and 466 deletions
Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -458,6 +458,7 @@ dependencies = [
"anyhow",
"clap",
"hex",
"libc",
"reqwest",
"serde",
"serde_json",
Cargo.toml
+1
View File
@@ -8,6 +8,7 @@ license = "MIT"
anyhow = "1.0"
clap = { version = "4.5", features = ["derive"] }
hex = "0.4"
libc = "0.2"
reqwest = { version = "0.12", default-features = false, features = [
"blocking",
"rustls-tls",
Dockerfile
+7 -1
View File
@@ -1,4 +1,4 @@
FROM alpine:3.22.4
FROM alpine:edge
RUN apk upgrade --no-cache && apk add --no-cache \
alpine-sdk \
@@ -16,8 +16,10 @@ RUN apk upgrade --no-cache && apk add --no-cache \
file \
findutils \
flex \
gawk \
gettext-dev \
git \
grep \
gzip \
elfutils-dev \
gmp-dev \
@@ -26,12 +28,16 @@ RUN apk upgrade --no-cache && apk add --no-cache \
libtool \
linux-headers \
meson \
mtools \
nasm \
ncurses \
ninja \
openssl \
openssl-dev \
patch \
pkgconf \
python3 \
rsync \
tar \
texinfo \
xz \
config.star
+17 -3
View File
@@ -3,10 +3,15 @@ container_image = "local/distro-builder:latest"
container_dockerfile = "Dockerfile"
arch = "x86_64"
libc = "musl"
libc = "glibc"
if libc == "glibc":
env = "gnu"
else:
env = libc
host_cflags = "-O2 -pipe"
host_cxxflags = ""
host_cxxflags = host_cflags
host_ldflags = "-Wl,-O1 -Wl,--sort-common -Wl,--as-needed"
target_cflags = host_cflags
@@ -22,7 +27,7 @@ if arch == "x86_64":
options = dict(
target_arch = arch,
target_triple = f"{arch}-linux-{libc}",
target_triple = f"{arch}-linux-{env}",
host_cflags = host_cflags,
host_cxxflags = host_cxxflags,
@@ -33,4 +38,13 @@ options = dict(
ldflags = target_ldflags,
libc = libc,
prefix = "/usr",
bindir = "/usr/bin",
sbindir = "/usr/bin",
libdir = "/usr/lib",
libexecdir = "/usr/libexec",
includedir = "/usr/include",
sysconfdir = "/etc",
localstatedir = "/var",
)
host-recipes/binutils.star
+9 -7
View File
@@ -5,15 +5,15 @@ metadata = meta(
license = "GPL-3.0-or-later",
)
source = tarball_source(
url = "https://ftp.gnu.org/gnu/binutils/binutils-" + version + ".tar.xz",
sha256 = "?",
url = f"https://ftp.gnu.org/gnu/binutils/binutils-{version}.tar.xz",
sha256 = "d75a94f4d73e7a4086f7513e67e439e8fcdcbb726ffe63f4661744e6256b2cf2",
strip_components = 1,
)
def configure(ctx):
ctx.run([
configure_args = [
ctx.source_dir / "configure",
"--prefix=" + ctx.prefix,
"--prefix=/",
"--target=" + options.target_triple,
"--with-sysroot=" + ctx.sysroot,
"--with-pic",
@@ -26,11 +26,13 @@ def configure(ctx):
"--enable-relro",
"--enable-separate-code",
"--enable-threads",
# gprofng's libcollector does not build against musl.
"--disable-gprofng",
"--disable-nls",
"--disable-werror",
], env = {
# gprofng's libcollector relies on glibc-specific internals.
"--disable-gprofng",
]
ctx.run(configure_args, env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
host-recipes/gcc-bootstrap.star
+2 -2
View File
@@ -13,9 +13,9 @@ host_deps = ["binutils"]
def configure(ctx):
ctx.run([
ctx.source_dir + "/configure",
ctx.source_dir / "configure",
"--target=" + options.target_triple,
"--prefix=" + ctx.prefix,
"--prefix=/",
"--with-sysroot=" + ctx.sysroot,
"--without-headers",
"--with-newlib",
host-recipes/gcc.star
+53
View File
@@ -0,0 +1,53 @@
version = "16.1.0"
revision = 1
metadata = meta(
description = "GNU GCC cross-compiler targeting the system triple",
license = "GPL-3.0-or-later",
website = "https://gcc.gnu.org/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz",
sha256 = "50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79",
strip_components = 1,
)
host_deps = ["binutils", "gcc-bootstrap"]
deps = [options.libc, "linux-headers"]
def configure(ctx):
ctx.run([
ctx.source_dir / "configure",
"--target=" + options.target_triple,
"--prefix=/",
"--with-sysroot=" + ctx.sysroot,
"--with-build-sysroot=" + ctx.sysroot,
"--enable-languages=c,c++,lto",
"--disable-bootstrap",
"--enable-default-pie",
"--enable-default-ssp",
"--enable-lto",
"--enable-threads=posix",
"--enable-tls",
"--enable-libstdcxx-time",
"--enable-checking=release",
"--enable-cet=auto",
"--enable-linker-build-id",
"--disable-nls",
"--disable-multilib",
"--disable-fixed-point",
"--disable-werror",
"--disable-libsanitizer",
"--disable-symvers",
], env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
})
def build(ctx):
ctx.run(["make", "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["make", "install-strip"], env = {"DESTDIR": pkg.dest_dir})
# Drop libtool archives.
ctx.run(["find", pkg.dest_dir, "-name", "*.la", "-delete"])
host-recipes/llvm.star
+52
View File
@@ -0,0 +1,52 @@
version = "20.1.0"
revision = 1
metadata = meta(
description = "LLVM compiler infrastructure with clang and lld",
license = "Apache-2.0 WITH LLVM-exception",
website = "https://llvm.org/",
)
source = tarball_source(
url = f"https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/llvm-project-{version}.src.tar.xz",
sha256 = "?",
strip_components = 1,
)
host_deps = ["binutils"]
def configure(ctx):
ctx.run([
"cmake",
"-S", ctx.source_dir / "llvm",
"-B", ctx.build_dir,
"-G", "Ninja",
"-DCMAKE_BUILD_TYPE=Release",
"-DCMAKE_INSTALL_PREFIX=/",
"-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;lld",
"-DLLVM_ENABLE_RUNTIMES=compiler-rt",
"-DLLVM_TARGETS_TO_BUILD=X86;AArch64;RISCV",
"-DLLVM_DEFAULT_TARGET_TRIPLE=" + options.target_triple,
"-DLLVM_HOST_TRIPLE=" + options.target_triple,
"-DLLVM_ENABLE_LIBXML2=OFF",
"-DLLVM_ENABLE_LIBEDIT=OFF",
"-DLLVM_ENABLE_TERMINFO=OFF",
"-DLLVM_ENABLE_ASSERTIONS=OFF",
"-DLLVM_ENABLE_PIC=ON",
"-DLLVM_BUILD_LLVM_DYLIB=ON",
"-DLLVM_LINK_LLVM_DYLIB=ON",
"-DLLVM_INSTALL_UTILS=ON",
"-DLLVM_INCLUDE_TESTS=OFF",
"-DLLVM_INCLUDE_EXAMPLES=OFF",
"-DLLVM_INCLUDE_BENCHMARKS=OFF",
"-DCLANG_DEFAULT_LINKER=lld",
"-DCLANG_DEFAULT_RTLIB=compiler-rt",
"-DCLANG_DEFAULT_CXX_STDLIB=libstdc++",
], env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
})
def build(ctx):
ctx.run(["cmake", "--build", ctx.build_dir, "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["cmake", "--install", ctx.build_dir], env = {"DESTDIR": pkg.dest_dir})
lib/common.star
+67 -9
View File
@@ -1,4 +1,4 @@
# Commonly used helpers.
# Autotools
def autotools_configure(ctx, extra_args = [], extra_env = {}):
env = {
@@ -11,12 +11,12 @@ def autotools_configure(ctx, extra_args = [], extra_env = {}):
ctx.source_dir / "configure",
"--host=" + options.target_triple,
"--with-sysroot=" + ctx.sysroot,
"--prefix=" + ctx.prefix,
"--sysconfdir=/etc",
"--localstatedir=/var",
"--bindir=" + ctx.prefix + "/bin",
"--sbindir=" + ctx.prefix + "/bin",
"--libdir=" + ctx.prefix + "/lib",
"--prefix=" + options.prefix,
"--sysconfdir=" + options.sysconfdir,
"--localstatedir=" + options.localstatedir,
"--bindir=" + options.bindir,
"--sbindir=" + options.sbindir,
"--libdir=" + options.libdir,
"--disable-static",
"--enable-shared",
] + extra_args, env = env)
@@ -25,9 +25,9 @@ def autotools_build(ctx, extra_args = []):
ctx.run(["make", "-j" + str(ctx.jobs)] + extra_args)
def autotools_install(ctx, pkg, extra_args = []):
ctx.run(["make", "install"] + extra_args, env = {"DESTDIR": pkg.destdir})
ctx.run(["make", "install"] + extra_args, env = { "DESTDIR": pkg.dest_dir })
def autotools(configure_args = [], configure_env = [], build_args = [], install_args = []):
def autotools(configure_args = [], configure_env = {}, build_args = [], install_args = []):
def _configure(ctx):
autotools_configure(ctx, extra_args = configure_args, extra_env = configure_env)
def _build(ctx):
@@ -35,3 +35,61 @@ def autotools(configure_args = [], configure_env = [], build_args = [], install_
def _install(ctx, pkg):
autotools_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install
# CMake
def cmake_configure(ctx, extra_args = [], extra_env = {}, host = False):
if host:
env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
}
toolchain_args = []
else:
env = {
"CFLAGS": options.cflags,
"CXXFLAGS": options.cxxflags,
"LDFLAGS": options.ldflags,
}
toolchain_args = [
"-DCMAKE_SYSTEM_NAME=Linux",
"-DCMAKE_SYSTEM_PROCESSOR=" + options.target_arch,
"-DCMAKE_SYSROOT=" + ctx.sysroot,
"-DCMAKE_C_COMPILER=" + options.target_triple + "-gcc",
"-DCMAKE_CXX_COMPILER=" + options.target_triple + "-g++",
"-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
"-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
"-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
"-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY",
]
env.update(extra_env)
ctx.run([
"cmake",
"-S", ctx.source_dir,
"-B", ctx.build_dir,
"-G", "Ninja",
"-DCMAKE_BUILD_TYPE=Release",
"-DCMAKE_INSTALL_PREFIX=" + options.prefix,
"-DCMAKE_INSTALL_SYSCONFDIR=" + options.sysconfdir,
"-DCMAKE_INSTALL_LOCALSTATEDIR=" + options.localstatedir,
] + toolchain_args + extra_args, env = env)
def cmake_build(ctx, extra_args = []):
ctx.run(["cmake", "--build", ctx.build_dir, "-j", str(ctx.jobs)] + extra_args)
def cmake_install(ctx, pkg, extra_args = []):
ctx.run(
["cmake", "--install", ctx.build_dir] + extra_args,
env = {"DESTDIR": pkg.dest_dir},
)
def cmake(configure_args = [], configure_env = {}, build_args = [], install_args = [], host = False):
def _configure(ctx):
cmake_configure(ctx, extra_args = configure_args, extra_env = configure_env, host = host)
def _build(ctx):
cmake_build(ctx, extra_args = build_args)
def _install(ctx, pkg):
cmake_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install
recipes/bash.star
+22
View File
@@ -0,0 +1,22 @@
version = "5.2.32"
revision = 1
metadata = meta(
description = "GNU Bourne-Again SHell",
license = "GPL-3.0-or-later",
website = "https://www.gnu.org/software/bash/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/bash/bash-{version}.tar.gz",
sha256 = "d3ef80d2b67d8cbbe4d3265c63a72c46f9b278ead6e0e06d61801b58f23f50b5",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc, "ncurses", "readline"]
configure, build, install = autotools(configure_args = [
"--without-bash-malloc",
"--with-installed-readline",
"--enable-readline",
"--enable-history",
"--enable-job-control",
], configure_env = {"CFLAGS": "-std=gnu17"})
recipes/coreutils.star
+25
View File
@@ -0,0 +1,25 @@
version = "9.6"
revision = 1
metadata = meta(
description = "GNU core utilities (file, shell, and text manipulation)",
license = "GPL-3.0-or-later",
website = "https://www.gnu.org/software/coreutils/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/coreutils/coreutils-{version}.tar.xz",
sha256 = "7a0124327b398fd9eb1a6abde583389821422c744ffa10734b24f557610d3283",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, install = autotools(
configure_args = [
"--enable-no-install-program=kill,uptime",
"--without-selinux",
"--without-openssl",
],
# coreutils' configure runs link tests that require a working executable;
# cross builds need this hint to skip a known false positive.
configure_env = {"FORCE_UNSAFE_CONFIGURE": "1"},
)
recipes/glibc.star
+36
View File
@@ -0,0 +1,36 @@
version = "2.41"
revision = 1
metadata = meta(
description = "GNU C library",
license = "LGPL-2.1-or-later",
website = "https://www.gnu.org/software/libc/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/glibc/glibc-{version}.tar.xz",
sha256 = "a5a26b22f545d6b7d7b3dd828e11e428f24f4fac43c934fb071b6a7d0828e901",
strip_components = 1,
)
host_deps = ["binutils", "gcc-bootstrap"]
deps = ["linux-headers"]
build_if = options.libc == "glibc"
def configure(ctx):
autotools_configure(ctx, [
"--build=" + options.target_triple,
"--with-headers=" + ctx.sysroot / options.prefix / "include",
"--enable-kernel=5.4",
"--enable-bind-now",
"--enable-stack-protector=strong",
"--enable-cet",
"--disable-werror",
"--disable-profile",
"--disable-nscd",
"--without-selinux",
"--without-gd",
"libc_cv_slibdir=/lib",
"libc_cv_rtlddir=/lib",
"libc_cv_forced_unwind=yes",
])
_, build, install = autotools()
recipes/limine.star
+33 -7
View File
@@ -3,6 +3,7 @@ revision = 1
metadata = meta(
description = "Modern, secure, portable, multiprotocol bootloader and boot manager",
license = "BSD-2-Clause",
website = "https://limine-bootloader.org"
)
source = tarball_source(
url = f"https://github.com/Limine-Bootloader/Limine/releases/download/v{version}/limine-{version}.tar.gz",
@@ -11,15 +12,40 @@ source = tarball_source(
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
build_if = options.target_arch in ["x86_64", "aarch64", "riscv64", "loongarch64"]
arch_configure_args = {
"x86_64": ["--enable-uefi-x86-64", "--enable-uefi-ia32", "--enable-bios", "--enable-bios-cd"],
"aarch64": ["--enable-uefi-aarch64"],
"riscv64": ["--enable-uefi-riscv64"],
"loongarch64": ["--enable-uefi-loongarch64"],
}
configure, build, install = autotools(
configure_args = ["--enable-uefi-cd"] + arch_configure_args.get(options.target_arch),
configure_env = {"TOOLCHAIN_FOR_TARGET": options.target_triple + "-"},
)
subpackages = [
subpackage(
name = "limine-bios",
"limine-uefi",
meta(description = "UEFI files"),
[
"usr/share/limine/BOOT*.EFI",
"usr/share/limine/limine-uefi-*.bin",
],
),
]
configure, build, install = autotools(configure_env = {
"TOOLCHAIN_FOR_TARGET": options.target_triple + "-",
"LD_FOR_TARGET": options.target_triple + "-" + "ld",
"OBJCOPY_FOR_TARGET": options.target_triple + "-" + "objcopy",
"OBJDUMP_FOR_TARGET": options.target_triple + "-" + "objdump",
})
if options.target_arch == "x86_64":
subpackages += [
subpackage(
"limine-bios",
meta(description = "BIOS files"),
[
"usr/share/limine/limine-bios*",
],
)
]
recipes/linux-headers.star
+2 -2
View File
@@ -16,5 +16,5 @@ def build(ctx):
ctx.run(["find", ctx.build_dir / "usr/include", "-type", "f", "!", "-name", "*.h", "-delete"])
def install(ctx, pkg):
ctx.run(["mkdir", "-p", pkg.dest_dir / ctx.prefix])
ctx.run(["cp", "-rp", ctx.build_dir / "usr/include", pkg.dest_dir / ctx.prefix])
ctx.run(["mkdir", "-p", pkg.dest_dir / options.prefix])
ctx.run(["cp", "-rp", ctx.build_dir / "usr/include", pkg.dest_dir / options.prefix])
recipes/linux.star.old
-37
View File
@@ -1,37 +0,0 @@
name = "linux"
version = "7.0.9"
revision = 1
description = "Linux kernel"
license = "GPL-2.0-only"
source = {
"url": f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz",
"sha256": "ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4",
"strip_components": 1,
}
host_deps = ["binutils", "gcc"]
def _make_args(ctx, *args):
result = [
"make",
"-C", ctx.source_dir,
"O=" + ctx.build_dir,
"ARCH=x86_64",
"CROSS_COMPILE=" + OPTIONS.target_triple + "-",
"-j" + str(ctx.jobs),
]
result.extend(args)
return result
def configure(ctx):
ctx.run(_make_args(ctx, "defconfig"))
def build(ctx):
ctx.run(_make_args(ctx, "bzImage"))
def install(ctx, pkg):
ctx.install(
ctx.build_dir + "/arch/x86/boot/bzImage",
pkg.destdir + "/boot/vmlinuz-" + version,
)
recipes/linux/files/config.x86_64
+12665
View File
File diff suppressed because it is too large Load Diff
recipes/linux/recipe.star
+44
View File
@@ -0,0 +1,44 @@
version = "7.0.9"
revision = 1
metadata = meta(
description = "Linux kernel",
license = "GPL-2.0-only",
)
source = tarball_source(
url = f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz",
sha256 = "ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
def make_args(ctx, *args):
# Translate arch name
if options.target_arch == "aarch64":
linux_arch = "arm64"
else:
linux_arch = options.target_arch
result = [
"make",
"ARCH=" + linux_arch,
"CROSS_COMPILE=" + options.target_triple + "-",
"-j" + str(ctx.jobs),
]
result.extend(args)
return result
def configure(ctx):
ctx.run(["cp", "-rp", ctx.source_dir / ".", ctx.build_dir])
ctx.run(["cp", ctx.files / "config." + options.target_arch, ctx.build_dir / ".config"])
ctx.run(make_args(ctx, "olddefconfig"))
def build(ctx):
ctx.run(make_args(ctx))
def install(ctx, pkg):
ctx.install(
ctx.build_dir + "/arch/x86/boot/bzImage",
pkg.destdir + "/boot/vmlinuz-" + version,
)
recipes/make.star
+18
View File
@@ -0,0 +1,18 @@
version = "4.4.1"
revision = 1
metadata = meta(
description = "GNU make build automation tool",
license = "GPL-3.0-or-later",
website = "https://www.gnu.org/software/make/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/make/make-{version}.tar.gz",
sha256 = "dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, install = autotools(configure_args = [
"--without-guile",
])
recipes/musl.star
+3 -2
View File
@@ -6,16 +6,17 @@ metadata = meta(
)
source = tarball_source(
url = f"https://musl.libc.org/releases/musl-{version}.tar.gz",
sha256 = "?",
sha256 = "d585fd3b613c66151fc3249e8ed44f77020cb5e6c1e635a616d3f9f82460512a",
strip_components = 1,
)
host_deps = ["binutils", "gcc-bootstrap"]
build_if = options.libc == "musl"
def configure(ctx):
ctx.run([
ctx.source_dir / "configure",
"--target=" + options.target_triple,
"--prefix=" + ctx.prefix,
"--prefix=" + options.prefix,
"--syslibdir=/lib",
], env = {
"CC": options.target_triple + "-gcc",
recipes/ncurses.star
+38
View File
@@ -0,0 +1,38 @@
version = "6.5"
revision = 1
metadata = meta(
description = "Terminal control library with wide-character support",
license = "MIT",
website = "https://invisible-island.net/ncurses/",
)
source = tarball_source(
url = f"https://invisible-mirror.net/archives/ncurses/ncurses-{version}.tar.gz",
sha256 = "136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, _ = autotools(configure_args = [
"--with-shared",
"--without-debug",
"--without-ada",
"--enable-pc-files",
"--enable-widec",
"--with-termlib",
"--with-cxx-binding",
"--with-cxx-shared",
"--with-pkg-config-libdir=/usr/lib/pkgconfig",
"--mandir=/usr/share/man",
], configure_env = {
# Conflicts with GCC 16 headers
"cf_cv_type_of_bool": "bool",
"cf_cv_cc_bool_type": "1",
"cf_cv_builtin_bool": "1",
"ac_cv_header_stdbool_h": "yes",
})
def install(ctx, pkg):
autotools_install(ctx, pkg, extra_args = [
"DESTDIR=" + pkg.dest_dir
])
recipes/openssl.star
+51
View File
@@ -0,0 +1,51 @@
version = "3.4.1"
revision = 1
metadata = meta(
description = "Cryptography and TLS library (OpenSSL)",
license = "Apache-2.0",
website = "https://www.openssl.org/",
)
source = tarball_source(
url = f"https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz",
sha256 = "?",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = ["zlib"]
def configure(ctx):
# OpenSSL uses its own perl-based Configure script. The first argument is
# the OpenSSL "target" — pick the one matching our triple.
if options.target_arch == "x86_64":
ossl_target = "linux-x86_64"
elif options.target_arch == "aarch64":
ossl_target = "linux-aarch64"
elif options.target_arch == "riscv64":
ossl_target = "linux64-riscv64"
else:
fail("openssl: unsupported target_arch " + options.target_arch)
ctx.run([
ctx.source_dir / "Configure",
ossl_target,
"--prefix=" + options.prefix,
"--openssldir=/etc/ssl",
"--libdir=lib",
"shared",
"zlib",
"no-tests",
"no-static-engine",
"enable-ktls",
], env = {
"CC": options.target_triple + "-gcc",
"AR": options.target_triple + "-ar",
"RANLIB": options.target_triple + "-ranlib",
"CFLAGS": options.cflags,
"LDFLAGS": options.ldflags,
})
def build(ctx):
ctx.run(["make", "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["make", "install_sw", "install_ssldirs"], env = {"DESTDIR": pkg.dest_dir})
recipes/pkgconf.star
+19
View File
@@ -0,0 +1,19 @@
version = "3.3.3"
revision = 1
metadata = meta(
description = "Lightweight pkg-config implementation",
license = "ISC",
website = "http://pkgconf.org/",
)
source = tarball_source(
url = f"https://distfiles.ariadne.space/pkgconf/pkgconf-{version}.tar.xz",
sha256 = "?",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, install = autotools(configure_args = [
"--with-system-libdir=" + options.libdir,
"--with-system-includedir=" + options.includedir,
])
recipes/readline.star
+29
View File
@@ -0,0 +1,29 @@
version = "8.2"
revision = 1
metadata = meta(
description = "Library for command-line editing",
license = "GPL-3.0-or-later",
website = "https://tiswww.case.edu/php/chet/readline/rltop.html",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/readline/readline-{version}.tar.gz",
sha256 = "3feb7171f16a84ee82ca18a36d7b9be109a52c04f492a053331d7d1095007c35",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc, "ncurses"]
configure, build, _ = autotools(
configure_args = ["--with-curses"],
configure_env = {
# Force linking against the system curses; otherwise readline's
# configure may pick a static libtermcap stub it ships internally.
"bash_cv_termcap_lib": "ncursesw",
},
)
# Readline overwrites DESTDIR
def install(ctx, pkg):
autotools_install(ctx, pkg, extra_args = [
"DESTDIR=" + pkg.dest_dir
])
recipes/xz.star
+18
View File
@@ -0,0 +1,18 @@
version = "5.6.3"
revision = 1
metadata = meta(
description = "XZ Utils — LZMA/XZ compression tools and library",
license = "0BSD AND GPL-2.0-or-later AND LGPL-2.1-or-later",
website = "https://tukaani.org/xz/",
)
source = tarball_source(
url = f"https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz",
sha256 = "db0590629b6f0fa36e74aea5f9731dc6f8df068ce7b7bafa45301832a5eebc3a",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, install = autotools(configure_args = [
"--disable-doc",
])
recipes/zlib.star
+36
View File
@@ -0,0 +1,36 @@
version = "1.3.2"
revision = 1
metadata = meta(
description = "Lossless data-compression library",
license = "Zlib",
website = "https://zlib.net/",
)
source = tarball_source(
url = f"https://zlib.net/zlib-{version}.tar.xz",
sha256 = "d7a0654783a4da529d1bb793b7ad9c3318020af77667bcae35f95d0e42a792f3",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
def configure(ctx):
# zlib ships its own ./configure that does not understand the usual
# autoconf flags (no --host, --build, etc.), so it is invoked directly.
ctx.run([
ctx.source_dir / "configure",
"--prefix=" + options.prefix,
"--libdir=" + options.libdir,
"--sharedlibdir=" + options.libdir,
], env = {
"CC": options.target_triple + "-gcc",
"AR": options.target_triple + "-ar",
"RANLIB": options.target_triple + "-ranlib",
"CFLAGS": options.cflags,
"LDFLAGS": options.ldflags,
})
def build(ctx):
ctx.run(["make", "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["make", "install"], env = {"DESTDIR": pkg.dest_dir})
src/builder.rs
+985 -20
View File
File diff suppressed because it is too large Load Diff
src/container.rs
+292
View File
@@ -0,0 +1,292 @@
use std::{
fs::File,
io::{self, Read},
os::{fd::FromRawFd, unix::io::RawFd},
path::{Path, PathBuf},
process::{Command, Stdio},
ptr,
sync::{
Mutex,
atomic::{AtomicBool, AtomicI32, Ordering},
},
thread,
};
use anyhow::{Context, bail};
use crate::config::ContainerRuntime;
#[derive(Debug, Clone)]
pub struct Mount {
pub host: PathBuf,
pub container: String,
pub read_only: bool,
}
#[derive(Debug)]
pub struct Container {
runtime: &'static str,
id: String,
stopped: bool,
}
#[derive(Clone, Debug)]
struct RegisteredContainer {
runtime: &'static str,
id: String,
}
static ACTIVE_CONTAINER: Mutex<Option<RegisteredContainer>> = Mutex::new(None);
static SIGNAL_CLEANUP_INSTALLED: AtomicBool = AtomicBool::new(false);
static SIGNAL_WRITE_FD: AtomicI32 = AtomicI32::new(-1);
pub fn install_signals() -> anyhow::Result<()> {
if SIGNAL_CLEANUP_INSTALLED
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_err()
{
return Ok(());
}
let mut fds = [0; 2];
if unsafe { libc::pipe(fds.as_mut_ptr()) } == -1 {
SIGNAL_CLEANUP_INSTALLED.store(false, Ordering::SeqCst);
return Err(io::Error::last_os_error()).context("creating signal cleanup pipe");
}
set_close_on_exec(fds[0]);
set_close_on_exec(fds[1]);
SIGNAL_WRITE_FD.store(fds[1], Ordering::SeqCst);
thread::spawn(move || signal_cleanup_loop(fds[0]));
install_signal_handler(libc::SIGINT)?;
install_signal_handler(libc::SIGTERM)?;
Ok(())
}
impl Container {
pub fn start(
runtime: &ContainerRuntime,
image: &str,
name: &str,
mounts: &[Mount],
) -> anyhow::Result<Self> {
let runtime_str = runtime.as_str();
let mut cmd = Command::new(runtime_str);
cmd.arg("run")
.arg("-d")
.arg("--rm")
.arg("--name")
.arg(name)
.arg("--read-only")
.arg("--tmpfs")
.arg("/tmp")
.arg("--tmpfs")
.arg("/dest")
.arg("--tmpfs")
.arg("/sysroot")
.arg("--network=none");
if matches!(runtime, ContainerRuntime::Podman) {
cmd.arg("--userns=keep-id");
}
for mount in mounts {
let mut spec = format!("{}:{}", mount.host.display(), mount.container);
if mount.read_only {
spec.push_str(":ro");
}
cmd.arg("-v").arg(spec);
}
cmd.arg(image).arg("sleep").arg("infinity");
cmd.stdout(Stdio::piped()).stderr(Stdio::inherit());
let output = cmd
.output()
.with_context(|| format!("spawning `{runtime_str} run` for image `{image}`"))?;
if !output.status.success() {
bail!(
"`{runtime_str} run` failed with {} for image `{image}`",
output.status
);
}
let id = String::from_utf8(output.stdout)
.context("container id is not valid UTF-8")?
.trim()
.to_owned();
if id.is_empty() {
bail!("`{runtime_str} run` returned an empty container id");
}
register_container(runtime_str, &id);
Ok(Self {
runtime: runtime_str,
id,
stopped: false,
})
}
pub fn exec(&self, argv: &[String], env: &[(String, String)], cwd: &str) -> anyhow::Result<()> {
if argv.is_empty() {
bail!("ctx.run called with an empty argv");
}
let mut cmd = Command::new(self.runtime);
cmd.arg("exec").arg("-w").arg(cwd);
for (k, v) in env {
cmd.arg("-e").arg(format!("{k}={v}"));
}
cmd.arg(&self.id);
cmd.args(argv);
let status = cmd.status().with_context(|| {
format!("spawning `{} exec` in container {}", self.runtime, self.id)
})?;
if !status.success() {
bail!("command {argv:?} failed with {status}");
}
Ok(())
}
pub fn cp_out(&self, src_in_container: &str, host_dst: &Path) -> anyhow::Result<()> {
let spec = format!("{}:{}", self.id, src_in_container);
let status = Command::new(self.runtime)
.arg("cp")
.arg(spec)
.arg(host_dst)
.status()
.with_context(|| format!("spawning `{} cp`", self.runtime))?;
if !status.success() {
bail!(
"`{} cp` failed with {status} for {src_in_container} -> {}",
self.runtime,
host_dst.display()
);
}
Ok(())
}
}
impl Drop for Container {
fn drop(&mut self) {
if !self.stopped {
let _ = Command::new(self.runtime)
.arg("rm")
.arg("-f")
.arg(&self.id)
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
self.stopped = true;
}
unregister_container(&self.id);
}
}
fn register_container(runtime: &'static str, id: &str) {
if let Ok(mut active) = ACTIVE_CONTAINER.lock() {
*active = Some(RegisteredContainer {
runtime,
id: id.to_owned(),
});
}
}
fn unregister_container(id: &str) {
if let Ok(mut active) = ACTIVE_CONTAINER.lock()
&& active.as_ref().is_some_and(|container| container.id == id)
{
*active = None;
}
}
fn active_container() -> Option<RegisteredContainer> {
ACTIVE_CONTAINER
.lock()
.ok()
.and_then(|active| active.clone())
}
fn signal_cleanup_loop(read_fd: RawFd) {
let mut signals = unsafe { File::from_raw_fd(read_fd) };
let mut buffer = [0_u8; 16];
loop {
match signals.read(&mut buffer) {
Ok(0) => return,
Ok(len) => {
let signal = buffer[..len]
.iter()
.copied()
.find(|signal| *signal != 0)
.unwrap_or(libc::SIGINT as u8);
cleanup_after_signal(signal as i32);
}
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
Err(_) => return,
}
}
}
fn cleanup_after_signal(signal: i32) -> ! {
if let Some(container) = active_container() {
eprintln!(
"\nreceived signal {signal}; killing container {}",
container.id
);
kill_container_detached(container.runtime, &container.id);
}
std::process::exit(128 + signal);
}
fn kill_container_detached(runtime: &str, id: &str) {
let _ = Command::new(runtime)
.arg("kill")
.arg(id)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::inherit())
.spawn();
}
fn set_close_on_exec(fd: RawFd) {
unsafe {
let flags = libc::fcntl(fd, libc::F_GETFD);
if flags != -1 {
let _ = libc::fcntl(fd, libc::F_SETFD, flags | libc::FD_CLOEXEC);
}
}
}
fn install_signal_handler(signal: i32) -> anyhow::Result<()> {
let mut action = unsafe { std::mem::zeroed::<libc::sigaction>() };
action.sa_sigaction = handle_signal as *const () as usize;
action.sa_flags = 0;
unsafe {
libc::sigemptyset(&mut action.sa_mask);
if libc::sigaction(signal, &action, ptr::null_mut()) == -1 {
return Err(io::Error::last_os_error())
.with_context(|| format!("installing handler for signal {signal}"));
}
}
Ok(())
}
extern "C" fn handle_signal(signal: libc::c_int) {
let fd = SIGNAL_WRITE_FD.load(Ordering::Relaxed);
if fd < 0 {
return;
}
let signal = signal as u8;
unsafe {
libc::write(fd, ptr::addr_of!(signal).cast(), 1);
}
}
src/eval.rs
+24 -8
View File
@@ -3,13 +3,14 @@ use starlark::{
environment::{FrozenModule, Globals, GlobalsBuilder, Module},
eval::Evaluator,
syntax::{AstModule, Dialect},
values::list::ListRef,
values::list::{ListRef, UnpackList},
};
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
use crate::{
options::Options,
phase::Path as StarPath,
recipe::{GitSource, Metadata, Source, Subpackage, TarballSource},
};
@@ -45,23 +46,38 @@ fn builder_globals(builder: &mut GlobalsBuilder) {
url: String,
sha256: String,
strip_components: Option<u32>,
patches: Option<UnpackList<String>>,
) -> anyhow::Result<Source> {
Ok(Source::Tarball(TarballSource::new(
url,
sha256,
strip_components.unwrap_or(0),
patches.map(|p| p.items).unwrap_or_default(),
)))
}
fn git_source(url: String, commit: String) -> anyhow::Result<Source> {
Ok(Source::Git(GitSource::new(url, commit)))
fn git_source(
url: String,
commit: String,
patches: Option<UnpackList<String>>,
) -> anyhow::Result<Source> {
Ok(Source::Git(GitSource::new(
url,
commit,
patches.map(|p| p.items).unwrap_or_default(),
)))
}
fn subpackage(name: String, metadata: Option<&Metadata>) -> anyhow::Result<Subpackage> {
let metadata = metadata
.cloned()
.unwrap_or_else(|| Metadata::new(None, None, None, None));
Ok(Subpackage::new(name, metadata))
fn subpackage(
name: String,
metadata: &Metadata,
files: UnpackList<String>,
) -> anyhow::Result<Subpackage> {
Ok(Subpackage::new(name, files.items, metadata.clone()))
}
fn path(value: String) -> anyhow::Result<StarPath> {
Ok(StarPath::new(value))
}
}
src/graph.rs
+177 -295
View File
@@ -1,18 +1,21 @@
use std::{
collections::{BTreeMap, BTreeSet},
fmt, fs,
path::{Path, PathBuf},
path::Path,
};
use anyhow::{Context, bail};
use sha2::{Digest, Sha256};
use anyhow::bail;
use crate::recipe::{OutputPackage, Recipe, RecipeKind, RecipeSet};
use crate::{
layout::Layout,
recipe::{OutputPackage, Recipe, RecipeKind, RecipeSet},
};
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub enum TaskId {
FetchSources(String),
PrepareSources(String),
PrepareRecipe(String),
ConfigureRecipe(String),
BuildRecipe(String),
InstallPackageFiles(String),
@@ -25,6 +28,7 @@ impl fmt::Display for TaskId {
match self {
Self::FetchSources(recipe) => write!(f, "fetch sources {recipe}"),
Self::PrepareSources(recipe) => write!(f, "prepare sources {recipe}"),
Self::PrepareRecipe(recipe) => write!(f, "prepare recipe {recipe}"),
Self::ConfigureRecipe(recipe) => write!(f, "configure {recipe}"),
Self::BuildRecipe(recipe) => write!(f, "build {recipe}"),
Self::InstallPackageFiles(output) => write!(f, "install package files {output}"),
@@ -52,18 +56,12 @@ impl TaskPlan {
pub fn dependency_count(&self) -> usize {
self.dependencies.values().map(Vec::len).sum()
}
#[cfg(test)]
pub fn dependencies(&self, task: &TaskId) -> Option<&[TaskId]> {
self.dependencies.get(task).map(Vec::as_slice)
}
}
pub struct TaskPlanner<'a> {
root: &'a Path,
arch: &'a str,
layout: Layout<'a>,
recipes: &'a RecipeSet,
force: bool,
forced_recipes: BTreeSet<String>,
dependencies: BTreeMap<TaskId, Vec<TaskId>>,
inactive: BTreeSet<TaskId>,
visiting: BTreeSet<TaskId>,
@@ -73,10 +71,9 @@ pub struct TaskPlanner<'a> {
impl<'a> TaskPlanner<'a> {
pub fn new(root: &'a Path, arch: &'a str, recipes: &'a RecipeSet) -> Self {
Self {
root,
arch,
layout: Layout::new(root, arch),
recipes,
force: false,
forced_recipes: BTreeSet::new(),
dependencies: BTreeMap::new(),
inactive: BTreeSet::new(),
visiting: BTreeSet::new(),
@@ -85,9 +82,11 @@ impl<'a> TaskPlanner<'a> {
}
pub fn build_plan(mut self, requests: &[String], force: bool) -> anyhow::Result<TaskPlan> {
self.force = force;
for request in requests {
let recipe = self.recipes.recipe(request)?;
if force {
self.forced_recipes.insert(recipe.key());
}
match recipe.kind() {
RecipeKind::Package => {
for output in recipe.outputs() {
@@ -138,18 +137,7 @@ impl<'a> TaskPlanner<'a> {
}
fn into_plan(self) -> anyhow::Result<TaskPlan> {
let mut order = Vec::new();
let mut visiting = BTreeSet::new();
let mut visited = BTreeSet::new();
for task in self.dependencies.keys() {
topo_visit(
task,
&self.dependencies,
&mut visiting,
&mut visited,
&mut order,
)?;
}
let order = recipe_contiguous_order(&self.dependencies, self.recipes)?;
Ok(TaskPlan {
dependencies: self.dependencies,
order,
@@ -160,9 +148,13 @@ impl<'a> TaskPlanner<'a> {
match task {
TaskId::FetchSources(_) => Ok(Vec::new()),
TaskId::PrepareSources(recipe) => Ok(vec![TaskId::FetchSources(recipe.clone())]),
TaskId::PrepareRecipe(recipe) => Ok(vec![TaskId::PrepareSources(recipe.clone())]),
TaskId::ConfigureRecipe(recipe) => {
let recipe = self.recipes.recipe(recipe)?;
let mut deps = vec![TaskId::PrepareSources(recipe.key())];
let mut deps = vec![
TaskId::PrepareSources(recipe.key()),
TaskId::PrepareRecipe(recipe.key()),
];
deps.extend(
recipe
.host_deps()
@@ -181,12 +173,29 @@ impl<'a> TaskPlanner<'a> {
TaskId::BuildRecipe(recipe) => Ok(vec![TaskId::ConfigureRecipe(recipe.clone())]),
TaskId::InstallPackageFiles(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
if output.is_base() {
Ok(vec![TaskId::BuildRecipe(output.recipe().to_owned())])
} else {
let base = recipe.base_output().ok_or_else(|| {
anyhow::anyhow!("recipe `{}` has no base output", recipe.key())
})?;
Ok(vec![TaskId::InstallPackageFiles(base.key())])
}
}
TaskId::ProduceApk(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
let mut deps = vec![TaskId::InstallPackageFiles(output.key())];
if output.is_base() {
deps.extend(
recipe
.outputs()
.iter()
.filter(|candidate| !candidate.is_base())
.map(|subpackage| TaskId::InstallPackageFiles(subpackage.key())),
);
}
deps.extend(
recipe
.deps()
@@ -209,6 +218,9 @@ impl<'a> TaskPlanner<'a> {
TaskId::PrepareSources(recipe) => {
self.prepare_sources_active(self.recipes.recipe(recipe)?)
}
TaskId::PrepareRecipe(recipe) => {
self.prepare_recipe_active(self.recipes.recipe(recipe)?)
}
TaskId::ConfigureRecipe(recipe) => {
self.recipe_task_active(self.recipes.recipe(recipe)?, "configure")
}
@@ -218,7 +230,19 @@ impl<'a> TaskPlanner<'a> {
TaskId::InstallPackageFiles(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
self.output_task_active(recipe, output, "install")
if output.is_base() {
return recipe.outputs().iter().try_fold(false, |active, output| {
Ok(active
|| self.output_task_active(recipe, output, "install")?
|| self.produce_apk_active(recipe, output)?)
});
}
let base = recipe.base_output().ok_or_else(|| {
anyhow::anyhow!("recipe `{}` has no base output", recipe.key())
})?;
Ok(self.output_task_active(recipe, output, "install")?
|| self.produce_apk_active(recipe, output)?
|| self.produce_apk_active(recipe, base)?)
}
TaskId::ProduceApk(output) => {
let output = self.recipes.output(output)?;
@@ -234,36 +258,55 @@ impl<'a> TaskPlanner<'a> {
fn fetch_sources_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
Ok(recipe.sources().entries().iter().any(|(_, source)| {
source.is_unknown_cache_key() || !self.source_cache_path(source.cache_key()).exists()
source.is_unknown_cache_key()
|| !self.layout.source_cache_path(source.cache_key()).exists()
}))
}
fn is_recipe_forced(&self, recipe: &Recipe) -> bool {
self.forced_recipes.contains(&recipe.key())
}
fn prepare_sources_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
if self.force {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
let want_version = format!("{}-r{}", recipe.version(), recipe.revision());
if fs::read_to_string(self.source_stamp(recipe, "version"))
if fs::read_to_string(self.layout.source_stamp(recipe, "version"))
.ok()
.as_deref()
!= Some(want_version.as_str())
{
return Ok(true);
}
if self.recipe_has_patches(recipe)? && !self.source_stamp(recipe, "patched").exists() {
if self.layout.recipe_has_patches(recipe)?
&& !self.layout.source_stamp(recipe, "patched").exists()
{
return Ok(true);
}
Ok(false)
}
fn recipe_task_active(&self, recipe: &Recipe, kind: &str) -> anyhow::Result<bool> {
if self.force {
fn prepare_recipe_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
if recipe.phases().prepare().is_none() {
return Ok(false);
}
if self.prepare_sources_active(recipe)? {
return Ok(true);
}
Ok(fs::read_to_string(self.recipe_task_stamp(recipe, kind))
self.recipe_task_active(recipe, "prepare")
}
fn recipe_task_active(&self, recipe: &Recipe, kind: &str) -> anyhow::Result<bool> {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
Ok(
fs::read_to_string(self.layout.recipe_task_stamp(recipe, kind))
.ok()
.as_deref()
!= Some(self.recipe_fingerprint(recipe)?.as_str()))
!= Some(self.layout.recipe_fingerprint(recipe)?.as_str()),
)
}
fn output_task_active(
@@ -272,281 +315,120 @@ impl<'a> TaskPlanner<'a> {
output: &OutputPackage,
kind: &str,
) -> anyhow::Result<bool> {
if self.force {
return Ok(true);
}
Ok(fs::read_to_string(self.output_task_stamp(output, kind))
.ok()
.as_deref()
!= Some(self.output_fingerprint(recipe, output)?.as_str()))
}
fn produce_apk_active(&self, recipe: &Recipe, output: &OutputPackage) -> anyhow::Result<bool> {
if self.force {
return Ok(true);
}
if !self.apk_path(recipe, output).exists() {
return Ok(true);
}
Ok(fs::read_to_string(self.output_task_stamp(output, "apk"))
.ok()
.as_deref()
!= Some(self.output_fingerprint(recipe, output)?.as_str()))
}
fn install_host_recipe_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
if self.force {
return Ok(true);
}
if !self.host_install_dir(recipe).exists() {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
Ok(
fs::read_to_string(self.recipe_task_stamp(recipe, "host-install"))
fs::read_to_string(self.layout.output_task_stamp(output, kind))
.ok()
.as_deref()
!= Some(self.recipe_fingerprint(recipe)?.as_str()),
!= Some(self.layout.output_fingerprint(recipe, output)?.as_str()),
)
}
fn recipe_fingerprint(&self, recipe: &Recipe) -> anyhow::Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.arch.as_bytes());
hasher.update(recipe.key().as_bytes());
hasher.update(recipe.version().as_bytes());
hasher.update(recipe.revision().to_le_bytes());
hasher.update(
fs::read(recipe.path())
.with_context(|| format!("reading recipe {}", recipe.path().display()))?,
);
for (name, source) in recipe.sources().entries() {
hasher.update(name.unwrap_or("").as_bytes());
hasher.update(source.url().as_bytes());
hasher.update(source.cache_key().as_bytes());
fn produce_apk_active(&self, recipe: &Recipe, output: &OutputPackage) -> anyhow::Result<bool> {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
for patch in self.recipe_patches(recipe)? {
hasher.update(patch.display().to_string().as_bytes());
hasher
.update(fs::read(&patch).with_context(|| format!("reading {}", patch.display()))?);
if !self.layout.apk_path(recipe, output).exists() {
return Ok(true);
}
Ok(hex::encode(hasher.finalize()))
Ok(
fs::read_to_string(self.layout.output_task_stamp(output, "apk"))
.ok()
.as_deref()
!= Some(self.layout.output_fingerprint(recipe, output)?.as_str()),
)
}
fn output_fingerprint(
&self,
recipe: &Recipe,
output: &OutputPackage,
) -> anyhow::Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.recipe_fingerprint(recipe)?.as_bytes());
hasher.update(output.key().as_bytes());
Ok(hex::encode(hasher.finalize()))
fn install_host_recipe_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
fn recipe_has_patches(&self, recipe: &Recipe) -> anyhow::Result<bool> {
Ok(!self.recipe_patches(recipe)?.is_empty())
if !self.layout.host_install_dir(recipe).exists() {
return Ok(true);
}
fn recipe_patches(&self, recipe: &Recipe) -> anyhow::Result<Vec<PathBuf>> {
let patches_dir = recipe.dir().join("patches");
if !patches_dir.exists() {
return Ok(Vec::new());
}
let mut patches = Vec::new();
for entry in fs::read_dir(&patches_dir)
.with_context(|| format!("reading patches directory {}", patches_dir.display()))?
{
let entry = entry?;
let path = entry.path();
if path.is_file() {
patches.push(path);
}
}
patches.sort();
Ok(patches)
}
fn source_cache_path(&self, key: &str) -> PathBuf {
self.root.join("build/cache/sources").join(key)
}
fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.root
.join("build/sources")
.join(format!("{}.{kind}", recipe.slug()))
}
fn recipe_task_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.root
.join("build/tasks")
.join(format!("{}.{kind}", recipe.slug()))
}
fn output_task_stamp(&self, output: &OutputPackage, kind: &str) -> PathBuf {
self.root
.join("build/tasks")
.join(format!("{}.{kind}", output.key().replace(':', "-")))
}
fn apk_path(&self, recipe: &Recipe, output: &OutputPackage) -> PathBuf {
self.root.join("build/pkgs").join(self.arch).join(format!(
"{}-{}-r{}.apk",
output.name(),
recipe.version(),
recipe.revision()
))
}
fn host_install_dir(&self, recipe: &Recipe) -> PathBuf {
self.root
.join("build/host-pkgs")
.join(recipe.slug())
.join("usr/local")
Ok(
fs::read_to_string(self.layout.recipe_task_stamp(recipe, "host-install"))
.ok()
.as_deref()
!= Some(self.layout.recipe_fingerprint(recipe)?.as_str()),
)
}
}
fn topo_visit(
task: &TaskId,
fn recipe_contiguous_order(
dependencies: &BTreeMap<TaskId, Vec<TaskId>>,
visiting: &mut BTreeSet<TaskId>,
visited: &mut BTreeSet<TaskId>,
order: &mut Vec<TaskId>,
) -> anyhow::Result<()> {
if visited.contains(task) {
return Ok(());
}
if !visiting.insert(task.clone()) {
bail!("task dependency cycle involving `{task}`");
}
for dependency in dependencies.get(task).into_iter().flatten() {
topo_visit(dependency, dependencies, visiting, visited, order)?;
}
visiting.remove(task);
visited.insert(task.clone());
order.push(task.clone());
Ok(())
}
recipes: &RecipeSet,
) -> anyhow::Result<Vec<TaskId>> {
let total = dependencies.len();
let mut remaining: BTreeMap<TaskId, BTreeSet<TaskId>> = dependencies
.iter()
.map(|(task, deps)| (task.clone(), deps.iter().cloned().collect()))
.collect();
#[cfg(test)]
mod tests {
use std::fs;
use tempfile::TempDir;
use crate::{config::Config, eval, recipe::RecipeSet};
use super::{TaskId, TaskPlanner};
fn write_config(root: &TempDir) {
fs::write(
root.path().join("config.star"),
r#"
container_runtime = "podman"
container_image = "local/test:latest"
container_dockerfile = "Dockerfile"
arch = "x86_64"
options = dict(target_arch = arch)
"#,
)
.unwrap();
fs::write(root.path().join("Dockerfile"), "FROM scratch\n").unwrap();
}
fn write_recipe(root: &TempDir, dir: &str, name: &str, extra: &str) {
let recipe_dir = root.path().join(dir);
fs::create_dir_all(&recipe_dir).unwrap();
fs::write(
recipe_dir.join(format!("{name}.star")),
format!(
r#"
version = "1.0"
revision = 1
source = tarball_source(url = "file:///tmp/{name}.tar", sha256 = "hash-{name}")
{extra}
def build(ctx):
ctx.run(["true"])
def install(ctx, pkg):
ctx.run(["true"])
"#
),
)
.unwrap();
}
fn load(root: &TempDir) -> (Config, RecipeSet) {
let config = Config::load(&root.path().join("config.star")).unwrap();
let lib = eval::eval_lib(&root.path().join("lib"), Some(&config.options)).unwrap();
let recipes = RecipeSet::load(root.path(), &config.options, lib.as_ref()).unwrap();
(config, recipes)
}
#[test]
fn inactive_seed_does_not_pull_dependencies() {
let root = TempDir::new().unwrap();
write_config(&root);
write_recipe(&root, "recipes", "dep", "");
write_recipe(&root, "recipes", "app", r#"deps = ["dep"]"#);
let (config, recipes) = load(&root);
let planner = TaskPlanner::new(root.path(), &config.arch, &recipes);
let output = recipes.output("app").unwrap();
let recipe = recipes.recipe(output.recipe()).unwrap();
fs::create_dir_all(root.path().join("build/pkgs/x86_64")).unwrap();
fs::write(root.path().join("build/pkgs/x86_64/app-1.0-r1.apk"), "").unwrap();
fs::create_dir_all(root.path().join("build/tasks")).unwrap();
fs::write(
root.path().join("build/tasks/app.apk"),
planner.output_fingerprint(recipe, output).unwrap(),
)
.unwrap();
let plan = TaskPlanner::new(root.path(), &config.arch, &recipes)
.build_plan(&["app".to_owned()], false)
.unwrap();
assert!(plan.is_empty());
}
#[test]
fn active_target_recipe_keeps_edges_and_topo_order() {
let root = TempDir::new().unwrap();
write_config(&root);
write_recipe(&root, "recipes", "app", "");
let (config, recipes) = load(&root);
let plan = TaskPlanner::new(root.path(), &config.arch, &recipes)
.build_plan(&["app".to_owned()], false)
.unwrap();
assert_eq!(
plan.order(),
&[
TaskId::FetchSources("app".to_owned()),
TaskId::PrepareSources("app".to_owned()),
TaskId::ConfigureRecipe("app".to_owned()),
TaskId::BuildRecipe("app".to_owned()),
TaskId::InstallPackageFiles("app".to_owned()),
TaskId::ProduceApk("app".to_owned()),
]
);
assert_eq!(
plan.dependencies(&TaskId::ProduceApk("app".to_owned())),
Some([TaskId::InstallPackageFiles("app".to_owned())].as_slice())
);
}
#[test]
fn host_dependencies_are_installed_as_host_recipes() {
let root = TempDir::new().unwrap();
write_config(&root);
write_recipe(&root, "host-recipes", "binutils", "");
write_recipe(&root, "recipes", "app", r#"host_deps = ["binutils"]"#);
let (config, recipes) = load(&root);
let plan = TaskPlanner::new(root.path(), &config.arch, &recipes)
.build_plan(&["app".to_owned()], false)
.unwrap();
assert!(
plan.order()
.contains(&TaskId::InstallHostRecipe("host:binutils".to_owned()))
);
let mut dependents: BTreeMap<TaskId, Vec<TaskId>> = BTreeMap::new();
for (task, deps) in dependencies {
for dep in deps {
dependents
.entry(dep.clone())
.or_default()
.push(task.clone());
}
}
let mut order = Vec::with_capacity(total);
let mut current_recipe: Option<String> = None;
while order.len() < total {
let next = pick_next(&remaining, current_recipe.as_deref(), recipes)?;
let slug = task_recipe_slug(&next, recipes)?;
current_recipe = Some(slug);
remaining.remove(&next);
if let Some(children) = dependents.get(&next) {
for child in children {
if let Some(deps) = remaining.get_mut(child) {
deps.remove(&next);
}
}
}
order.push(next);
}
Ok(order)
}
fn pick_next(
remaining: &BTreeMap<TaskId, BTreeSet<TaskId>>,
current_recipe: Option<&str>,
recipes: &RecipeSet,
) -> anyhow::Result<TaskId> {
let mut fallback: Option<TaskId> = None;
for (task, deps) in remaining {
if !deps.is_empty() {
continue;
}
if let Some(active) = current_recipe {
let slug = task_recipe_slug(task, recipes)?;
if slug == active {
return Ok(task.clone());
}
}
if fallback.is_none() {
fallback = Some(task.clone());
}
}
fallback.ok_or_else(|| anyhow::anyhow!("task dependency cycle detected"))
}
pub fn task_recipe_slug(task: &TaskId, recipes: &RecipeSet) -> anyhow::Result<String> {
Ok(match task {
TaskId::FetchSources(recipe)
| TaskId::PrepareSources(recipe)
| TaskId::PrepareRecipe(recipe)
| TaskId::ConfigureRecipe(recipe)
| TaskId::BuildRecipe(recipe)
| TaskId::InstallHostRecipe(recipe) => recipe.clone(),
TaskId::InstallPackageFiles(output) | TaskId::ProduceApk(output) => {
recipes.output(output)?.recipe().to_owned()
}
})
}
src/layout.rs
+123
View File
@@ -0,0 +1,123 @@
use std::{
fs,
path::{Path, PathBuf},
};
use anyhow::Context;
use sha2::{Digest, Sha256};
use crate::recipe::{OutputPackage, Recipe};
pub struct Layout<'a> {
pub root: &'a Path,
pub arch: &'a str,
}
impl<'a> Layout<'a> {
pub fn new(root: &'a Path, arch: &'a str) -> Self {
Self { root, arch }
}
pub fn source_cache_dir(&self) -> PathBuf {
self.root.join("build/cache/sources")
}
pub fn source_cache_path(&self, key: &str) -> PathBuf {
self.source_cache_dir().join(key)
}
pub fn source_workdir(&self, recipe: &Recipe) -> PathBuf {
self.root.join("build/sources").join(recipe.slug())
}
pub fn build_workdir(&self, recipe: &Recipe) -> PathBuf {
self.root.join("build/builds").join(recipe.slug())
}
pub fn host_install_dir(&self, recipe: &Recipe) -> PathBuf {
self.root.join("build/host-pkgs").join(recipe.slug())
}
pub fn host_install_root(&self, recipe: &Recipe) -> PathBuf {
self.root.join("build/host-pkgs").join(recipe.slug())
}
pub fn apk_path(&self, recipe: &Recipe, output: &OutputPackage) -> PathBuf {
self.root.join("build/pkgs").join(self.arch).join(format!(
"{}-{}-r{}.apk",
output.name(),
recipe.version(),
recipe.revision()
))
}
pub fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.root
.join("build/sources")
.join(format!("{}.{kind}", recipe.slug()))
}
pub fn recipe_task_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.root
.join("build/tasks")
.join(format!("{}.{kind}", recipe.slug()))
}
pub fn output_task_stamp(&self, output: &OutputPackage, kind: &str) -> PathBuf {
self.root
.join("build/tasks")
.join(format!("{}.{kind}", output.key().replace(':', "-")))
}
pub fn recipe_fingerprint(&self, recipe: &Recipe) -> anyhow::Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.arch.as_bytes());
hasher.update(recipe.key().as_bytes());
hasher.update(recipe.version().as_bytes());
hasher.update(recipe.revision().to_le_bytes());
hasher.update(
fs::read(recipe.path())
.with_context(|| format!("reading recipe {}", recipe.path().display()))?,
);
for (name, source) in recipe.sources().entries() {
hasher.update(name.unwrap_or("").as_bytes());
hasher.update(source.url().as_bytes());
hasher.update(source.cache_key().as_bytes());
}
for patch in self.recipe_patches(recipe)? {
hasher.update(patch.display().to_string().as_bytes());
hasher
.update(fs::read(&patch).with_context(|| format!("reading {}", patch.display()))?);
}
Ok(hex::encode(hasher.finalize()))
}
pub fn output_fingerprint(
&self,
recipe: &Recipe,
output: &OutputPackage,
) -> anyhow::Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.recipe_fingerprint(recipe)?.as_bytes());
hasher.update(output.key().as_bytes());
Ok(hex::encode(hasher.finalize()))
}
pub fn recipe_has_patches(&self, recipe: &Recipe) -> anyhow::Result<bool> {
Ok(!self.recipe_patches(recipe)?.is_empty())
}
pub fn recipe_patches(&self, recipe: &Recipe) -> anyhow::Result<Vec<PathBuf>> {
let Some(data_dir) = recipe.data_dir() else {
return Ok(Vec::new());
};
let patches_dir = data_dir.join("patches");
let mut out = Vec::new();
for (_, source) in recipe.sources().entries() {
for name in source.patches() {
out.push(patches_dir.join(name));
}
}
Ok(out)
}
}
src/main.rs
+4
View File
@@ -1,12 +1,16 @@
mod builder;
mod cli;
mod config;
mod container;
mod eval;
mod graph;
mod layout;
mod log;
mod options;
mod phase;
mod recipe;
fn main() -> anyhow::Result<()> {
container::install_signals()?;
cli::run()
}
src/phase.rs
+389
View File
@@ -0,0 +1,389 @@
use std::{cell::RefCell, collections::BTreeMap, rc::Rc};
use allocative::Allocative;
use anyhow::anyhow;
use starlark::{
collections::SmallMap,
environment::{Methods, MethodsBuilder, MethodsStatic, Module},
eval::Evaluator,
values::{
Heap, OwnedFrozenValue, StarlarkValue, Value, ValueLike, list::UnpackList, none::NoneType,
},
};
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_module, starlark_value};
use crate::container::Container;
thread_local! {
static CURRENT: RefCell<Option<PhaseRuntime>> = const { RefCell::new(None) };
}
#[derive(Clone)]
pub struct PhaseRuntime {
pub container: Rc<RefCell<Container>>,
pub base_path: String,
pub base_env: Vec<(String, String)>,
}
pub struct PhaseRuntimeGuard;
impl PhaseRuntimeGuard {
pub fn enter(runtime: PhaseRuntime) -> Self {
CURRENT.with(|cell| {
let prev = cell.borrow_mut().replace(runtime);
assert!(prev.is_none(), "phase runtime already set");
});
Self
}
}
impl Drop for PhaseRuntimeGuard {
fn drop(&mut self) {
CURRENT.with(|cell| {
cell.borrow_mut().take();
});
}
}
fn with_current<R>(f: impl FnOnce(&PhaseRuntime) -> R) -> anyhow::Result<R> {
CURRENT.with(|cell| {
let borrow = cell.borrow();
let runtime = borrow
.as_ref()
.ok_or_else(|| anyhow!("ctx.run called outside of a phase invocation"))?;
Ok(f(runtime))
})
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct Path {
inner: String,
}
impl Path {
pub fn new(inner: impl Into<String>) -> Self {
Self {
inner: inner.into(),
}
}
pub fn as_str(&self) -> &str {
&self.inner
}
}
fn join_paths(base: &str, rhs: &str) -> String {
let trimmed = base.trim_end_matches('/');
if trimmed.is_empty() {
format!("/{rhs}")
} else {
format!("{trimmed}/{rhs}")
}
}
fn coerce_path_string(value: Value<'_>) -> anyhow::Result<String> {
if let Some(s) = value.unpack_str() {
return Ok(s.to_owned());
}
if let Some(p) = value.downcast_ref::<Path>() {
return Ok(p.inner.clone());
}
if let Some(s) = value.downcast_ref::<SourceDir>() {
return Ok(s.default.clone());
}
anyhow::bail!("expected a string or path, got `{}`", value.get_type())
}
impl std::fmt::Display for Path {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.inner)
}
}
starlark::starlark_simple_value!(Path);
#[starlark_value(type = "path")]
impl<'v> StarlarkValue<'v> for Path {
fn div(&self, other: Value<'v>, heap: &'v Heap) -> starlark::Result<Value<'v>> {
let rhs = coerce_path_string(other).map_err(starlark::Error::new_other)?;
Ok(heap.alloc(Path::new(join_paths(&self.inner, &rhs))))
}
fn add(&self, rhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
let suffix = rhs.unpack_str()?;
Some(Ok(heap.alloc(format!("{}{}", self.inner, suffix))))
}
fn radd(&self, lhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
let prefix = lhs.unpack_str()?;
Some(Ok(heap.alloc(format!("{}{}", prefix, self.inner))))
}
fn equals(&self, other: Value<'v>) -> starlark::Result<bool> {
Ok(other
.downcast_ref::<Self>()
.map(|o| o.inner == self.inner)
.unwrap_or(false))
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct SourceDir {
default: String,
entries: BTreeMap<String, String>,
}
impl SourceDir {
pub fn single(path: impl Into<String>) -> Self {
Self {
default: path.into(),
entries: BTreeMap::new(),
}
}
pub fn named<I, K, V>(entries: I) -> Self
where
I: IntoIterator<Item = (K, V)>,
K: Into<String>,
V: Into<String>,
{
let entries: BTreeMap<String, String> = entries
.into_iter()
.map(|(k, v)| (k.into(), v.into()))
.collect();
let default = entries
.values()
.next()
.cloned()
.unwrap_or_else(|| "/sources".to_owned());
Self { default, entries }
}
}
impl std::fmt::Display for SourceDir {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.default)
}
}
starlark::starlark_simple_value!(SourceDir);
#[starlark_value(type = "source_dir")]
impl<'v> StarlarkValue<'v> for SourceDir {
fn at(&self, index: Value<'v>, heap: &'v Heap) -> starlark::Result<Value<'v>> {
let key = index.unpack_str().ok_or_else(|| {
starlark::Error::new_other(anyhow!("source_dir index must be a string"))
})?;
let path = self.entries.get(key).ok_or_else(|| {
starlark::Error::new_other(anyhow!(
"no source named `{key}` (available: {})",
if self.entries.is_empty() {
"<none>".to_owned()
} else {
self.entries
.keys()
.map(String::as_str)
.collect::<Vec<_>>()
.join(", ")
}
))
})?;
Ok(heap.alloc(path.as_str()))
}
fn add(&self, rhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
let suffix = rhs.unpack_str()?;
Some(Ok(heap.alloc(format!("{}{}", self.default, suffix))))
}
fn radd(&self, lhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
let prefix = lhs.unpack_str()?;
Some(Ok(heap.alloc(format!("{}{}", prefix, self.default))))
}
fn div(&self, other: Value<'v>, heap: &'v Heap) -> starlark::Result<Value<'v>> {
let rhs = coerce_path_string(other).map_err(starlark::Error::new_other)?;
Ok(heap.alloc(Path::new(join_paths(&self.default, &rhs))))
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct PhaseContext {
source_dir: SourceDir,
build_dir: String,
sysroot: String,
files: Option<String>,
jobs: i32,
}
impl PhaseContext {
pub fn new(source_dir: SourceDir, jobs: i32, files: Option<String>) -> Self {
Self {
source_dir,
build_dir: "/build".to_owned(),
sysroot: "/sysroot".to_owned(),
files,
jobs,
}
}
}
impl std::fmt::Display for PhaseContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "ctx")
}
}
starlark::starlark_simple_value!(PhaseContext);
#[starlark_value(type = "phase_context")]
impl<'v> StarlarkValue<'v> for PhaseContext {
fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
Some(match attr {
"source_dir" => heap.alloc(self.source_dir.clone()),
"build_dir" => heap.alloc(Path::new(self.build_dir.clone())),
"sysroot" => heap.alloc(Path::new(self.sysroot.clone())),
"files" => heap.alloc(Path::new(self.files.as_ref()?.clone())),
"jobs" => heap.alloc(self.jobs),
_ => return None,
})
}
fn has_attr(&self, attr: &str, _heap: &'v Heap) -> bool {
match attr {
"source_dir" | "build_dir" | "sysroot" | "jobs" | "run" => true,
"files" => self.files.is_some(),
_ => false,
}
}
fn dir_attr(&self) -> Vec<String> {
let mut attrs = vec![
"source_dir".to_owned(),
"build_dir".to_owned(),
"sysroot".to_owned(),
"jobs".to_owned(),
"run".to_owned(),
];
if self.files.is_some() {
attrs.push("files".to_owned());
}
attrs
}
fn get_methods() -> Option<&'static Methods> {
static RES: MethodsStatic = MethodsStatic::new();
RES.methods(phase_context_methods)
}
}
#[starlark_module]
fn phase_context_methods(builder: &mut MethodsBuilder) {
fn run<'v>(
#[starlark(this)] _this: Value<'v>,
#[starlark(require = pos)] argv: UnpackList<Value<'v>>,
#[starlark(require = named)] env: Option<SmallMap<String, Value<'v>>>,
#[starlark(require = named)] cwd: Option<String>,
) -> anyhow::Result<NoneType> {
let argv: Vec<String> = argv
.items
.iter()
.map(|v| coerce_path_string(*v))
.collect::<anyhow::Result<Vec<_>>>()?;
let mut env_strings: SmallMap<String, String> = SmallMap::new();
if let Some(env) = env {
for (k, v) in env {
env_strings.insert(k, coerce_path_string(v)?);
}
}
run_in_container(&argv, env_strings, cwd.as_deref().unwrap_or("/build"))?;
Ok(NoneType)
}
}
fn run_in_container(
argv: &[String],
env_overrides: SmallMap<String, String>,
cwd: &str,
) -> anyhow::Result<()> {
let (container, env) = with_current(|runtime| {
let mut env: Vec<(String, String)> = runtime
.base_env
.iter()
.cloned()
.map(|(k, v)| {
if k == "PATH" {
(k, runtime.base_path.clone())
} else {
(k, v)
}
})
.collect();
for (k, v) in env_overrides {
if let Some(slot) = env.iter_mut().find(|(existing, _)| existing == &k) {
slot.1 = v;
} else {
env.push((k, v));
}
}
(runtime.container.clone(), env)
})?;
container.borrow().exec(argv, &env, cwd)
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct PackageContext {
dest_dir: String,
}
impl PackageContext {
pub fn new(dest_dir: String) -> Self {
Self { dest_dir }
}
}
impl std::fmt::Display for PackageContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "pkg")
}
}
starlark::starlark_simple_value!(PackageContext);
#[starlark_value(type = "package_context")]
impl<'v> StarlarkValue<'v> for PackageContext {
fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
match attr {
"dest_dir" => Some(heap.alloc(Path::new(self.dest_dir.clone()))),
_ => None,
}
}
fn has_attr(&self, attr: &str, _heap: &'v Heap) -> bool {
attr == "dest_dir"
}
fn dir_attr(&self) -> Vec<String> {
vec!["dest_dir".to_owned()]
}
}
pub fn invoke_phase(func: &OwnedFrozenValue, args: &[PhaseArg]) -> anyhow::Result<()> {
let module = Module::new();
let mut eval = Evaluator::new(&module);
let allocated: Vec<Value<'_>> = args
.iter()
.map(|arg| match arg {
PhaseArg::Ctx(ctx) => module.heap().alloc(ctx.clone()),
PhaseArg::Pkg(pkg) => module.heap().alloc(pkg.clone()),
})
.collect();
eval.eval_function(func.value(), &allocated, &[])
.map_err(|err| anyhow!("{err}"))?;
Ok(())
}
pub enum PhaseArg {
Ctx(PhaseContext),
Pkg(PackageContext),
}
src/recipe/metadata.rs
+17
View File
@@ -51,4 +51,21 @@ impl Metadata {
pub fn website(&self) -> Option<&str> {
self.website.as_deref()
}
pub fn inherit(&self, from: &Self) -> Self {
Self {
maintainer: self
.maintainer
.as_ref()
.or(from.maintainer.as_ref())
.cloned(),
description: self
.description
.as_ref()
.or(from.description.as_ref())
.cloned(),
license: self.license.as_ref().or(from.license.as_ref()).cloned(),
website: self.website.as_ref().or(from.website.as_ref()).cloned(),
}
}
}
src/recipe/mod.rs
+179 -57
View File
@@ -5,11 +5,14 @@ mod subpackage;
use anyhow::{Context, bail};
use starlark::{
environment::{FrozenModule, Module},
values::{OwnedFrozenValue, UnpackValue, ValueLike, list::ListRef, typing::StarlarkCallable},
eval::Evaluator,
values::{
OwnedFrozenValue, UnpackValue, ValueLike, dict::DictRef, list::ListRef,
typing::StarlarkCallable,
},
};
use std::{
collections::HashMap,
fmt,
collections::{HashMap, HashSet},
path::{Path, PathBuf},
};
use walkdir::WalkDir;
@@ -95,24 +98,6 @@ pub struct Recipe {
phases: RecipePhases,
}
impl fmt::Debug for Recipe {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Recipe")
.field("path", &self.path)
.field("kind", &self.kind)
.field("version", &self.version)
.field("revision", &self.revision)
.field("sources", &self.sources)
.field("outputs", &self.outputs)
.field("host_deps", &self.host_deps)
.field("build_deps", &self.build_deps)
.field("deps", &self.deps)
.field("run_deps", &self.run_deps)
.field("phases", &self.phases)
.finish()
}
}
impl Recipe {
pub fn load(
path: &Path,
@@ -120,10 +105,14 @@ impl Recipe {
kind: RecipeKind,
options: &Options,
lib: Option<&FrozenModule>,
) -> anyhow::Result<Self> {
) -> anyhow::Result<Option<Self>> {
let module = eval::eval_file(path, Some(options), lib)
.with_context(|| format!("evaluating recipe {}", path.display()))?;
if !Self::eval_build_if(&module)? {
return Ok(None);
}
let version = eval::extract_string(&module, "version")
.map_err(|e| anyhow::anyhow!("field `version`: {e}"))?;
@@ -141,19 +130,52 @@ impl Recipe {
.clone(),
};
let source_value = module
.get("source")
.ok_or_else(|| anyhow::anyhow!("field `source`: missing"))?;
let source = source_value
let source_value = module.get("source");
let sources_value = module.get("sources");
let sources = match (source_value, sources_value) {
(None, None) => bail!("recipe must define either `source` or `sources`"),
(Some(_), Some(_)) => {
bail!("recipe must define exactly one of `source` or `sources`, not both")
}
(Some(value), None) => {
let source = value
.downcast_ref::<Source>()
.ok_or_else(|| anyhow::anyhow!("field `source`: expected a source value"))?
.clone();
let sources = Sources::Single(source);
Sources::Single(source)
}
(None, Some(value)) => {
let dict = DictRef::from_value(value).ok_or_else(|| {
anyhow::anyhow!("field `sources`: expected a dict of name -> source")
})?;
if dict.iter().len() == 0 {
bail!("field `sources`: must contain at least one entry");
}
let mut map: HashMap<String, source::Source> = HashMap::new();
for (key, value) in dict.iter() {
let key_str = key
.unpack_str()
.ok_or_else(|| anyhow::anyhow!("field `sources`: keys must be strings"))?;
if !Self::is_valid_source_name(key_str) {
bail!(
"field `sources`: invalid source name `{key_str}` (allowed: letters, digits, `-`, `_`; must start with a letter or digit)"
);
}
let source = value.downcast_ref::<Source>().ok_or_else(|| {
anyhow::anyhow!("field `sources`: entry `{key_str}` is not a source value")
})?;
if map.insert(key_str.to_owned(), source.clone()).is_some() {
bail!("field `sources`: duplicate key `{key_str}`");
}
}
Sources::Multiple(map)
}
};
let host_deps = optional_string_list(&module, "host_deps")?;
let build_deps = optional_string_list(&module, "build_deps")?;
let deps = optional_string_list(&module, "deps")?;
let run_deps = optional_string_list(&module, "run_deps")?;
let host_deps = Self::optional_string_list(&module, "host_deps")?;
let build_deps = Self::optional_string_list(&module, "build_deps")?;
let deps = Self::optional_string_list(&module, "deps")?;
let run_deps = Self::optional_string_list(&module, "run_deps")?;
let recipe_key = kind.key(name);
let outputs = match kind {
@@ -162,6 +184,8 @@ impl Recipe {
recipe: recipe_key.clone(),
name: name.to_owned(),
metadata: metadata.clone(),
file_globs: Vec::new(),
base: true,
}];
if let Some(value) = module.get("subpackages") {
let list = ListRef::from_value(value)
@@ -172,10 +196,13 @@ impl Recipe {
"field `subpackages`: each entry must be a subpackage value"
)
})?;
Self::check_subpackage_file_globs(sub.name(), sub.file_globs())?;
outputs.push(OutputPackage {
recipe: recipe_key.clone(),
name: sub.name().to_owned(),
metadata: sub.metadata().clone(),
metadata: sub.metadata().inherit(&metadata),
file_globs: sub.file_globs().to_vec(),
base: false,
});
}
}
@@ -194,7 +221,7 @@ impl Recipe {
.map_err(|err| anyhow::anyhow!("freezing recipe module {}: {err:?}", path.display()))?;
let phases = RecipePhases::load(&module)?;
Ok(Recipe {
let recipe = Recipe {
name: name.to_owned(),
path: path.to_path_buf(),
kind,
@@ -207,7 +234,8 @@ impl Recipe {
deps,
run_deps,
phases,
})
};
Ok(Some(recipe))
}
pub fn phases(&self) -> &RecipePhases {
@@ -222,8 +250,21 @@ impl Recipe {
self.kind.slug(&self.name)
}
pub fn dir(&self) -> &Path {
self.path.parent().unwrap_or_else(|| Path::new("."))
pub fn data_dir(&self) -> Option<&Path> {
if self.path.file_name().is_some_and(|n| n == "recipe.star") {
self.path.parent()
} else {
None
}
}
pub fn files_dir(&self) -> Option<PathBuf> {
let candidate = self.data_dir()?.join("files");
if candidate.is_dir() {
Some(candidate)
} else {
None
}
}
pub fn path(&self) -> &Path {
@@ -250,6 +291,10 @@ impl Recipe {
&self.outputs
}
pub fn base_output(&self) -> Option<&OutputPackage> {
self.outputs.iter().find(|output| output.is_base())
}
pub fn host_deps(&self) -> &[String] {
&self.host_deps
}
@@ -265,7 +310,6 @@ impl Recipe {
pub fn run_deps(&self) -> &[String] {
&self.run_deps
}
}
fn optional_string_list(module: &Module, key: &str) -> anyhow::Result<Vec<String>> {
match eval::extract_string_list(module, key) {
@@ -275,31 +319,85 @@ fn optional_string_list(module: &Module, key: &str) -> anyhow::Result<Vec<String
}
}
/// Evaluates "build_if" functions.
fn eval_build_if(module: &Module) -> anyhow::Result<bool> {
let Some(value) = module.get("build_if") else {
return Ok(true);
};
if let Some(b) = value.unpack_bool() {
return Ok(b);
}
let callable: Option<StarlarkCallable<'_>> = StarlarkCallable::unpack_value_opt(value);
if callable.is_none() {
bail!("field `build_if`: expected a bool or a callable returning a bool");
}
let mut eval = Evaluator::new(module);
let result = eval
.eval_function(value, &[], &[])
.map_err(|err| anyhow::anyhow!("calling `build_if`: {err}"))?;
result
.unpack_bool()
.ok_or_else(|| anyhow::anyhow!("field `build_if`: must return a bool"))
}
fn is_valid_source_name(name: &str) -> bool {
let mut chars = name.chars();
let Some(first) = chars.next() else {
return false;
};
if !first.is_ascii_alphanumeric() {
return false;
}
chars.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
}
fn check_subpackage_file_globs(name: &str, file_globs: &[String]) -> anyhow::Result<()> {
if file_globs.is_empty() {
bail!("subpackage '{name}' must declare at least one file glob");
}
for glob in file_globs {
if glob.is_empty() {
bail!("subpackage '{name}' has an empty file glob");
}
if glob.starts_with('/') {
bail!("subpackage '{name}' glob '{glob}' must be relative to the package dest dir");
}
if Path::new(glob)
.components()
.any(|component| matches!(component, std::path::Component::ParentDir))
{
bail!("subpackage '{name}' glob '{glob}' must not contain '..'");
}
}
Ok(())
}
}
pub struct RecipePhases {
prepare: Option<OwnedFrozenValue>,
configure: Option<OwnedFrozenValue>,
build: OwnedFrozenValue,
install: OwnedFrozenValue,
}
impl fmt::Debug for RecipePhases {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RecipePhases")
.field("configure", &self.configure.is_some())
.field("build", &true)
.field("install", &true)
.finish()
}
}
impl RecipePhases {
fn load(module: &FrozenModule) -> anyhow::Result<Self> {
Ok(Self {
prepare: optional_phase_function(module, "prepare")?,
configure: optional_phase_function(module, "configure")?,
build: required_phase_function(module, "build")?,
install: required_phase_function(module, "install")?,
})
}
pub fn prepare(&self) -> Option<&OwnedFrozenValue> {
self.prepare.as_ref()
}
pub fn configure(&self) -> Option<&OwnedFrozenValue> {
self.configure.as_ref()
}
@@ -346,7 +444,7 @@ fn validate_phase_function(name: &str, value: &OwnedFrozenValue) -> anyhow::Resu
Ok(())
}
#[derive(Clone, Debug)]
#[derive(Clone)]
pub struct OutputPackage {
/// Canonical key of the owning recipe.
recipe: String,
@@ -354,6 +452,10 @@ pub struct OutputPackage {
name: String,
/// Metadata attached to the output package.
metadata: Metadata,
/// File globs claimed from the base package dest dir.
file_globs: Vec<String>,
/// Whether this is the recipe's base output.
base: bool,
}
impl OutputPackage {
@@ -368,12 +470,24 @@ impl OutputPackage {
pub fn name(&self) -> &str {
&self.name
}
pub fn metadata(&self) -> &Metadata {
&self.metadata
}
pub fn file_globs(&self) -> &[String] {
&self.file_globs
}
pub fn is_base(&self) -> bool {
self.base
}
}
#[derive(Debug)]
pub struct RecipeSet {
recipes: HashMap<String, Recipe>,
outputs: HashMap<String, OutputPackage>,
skipped: HashSet<String>,
}
impl RecipeSet {
@@ -384,6 +498,7 @@ impl RecipeSet {
) -> anyhow::Result<Self> {
let mut recipes = HashMap::new();
let mut outputs = HashMap::new();
let mut skipped: HashSet<String> = HashSet::new();
for (path, kind) in [
("recipes", RecipeKind::Package),
@@ -396,9 +511,13 @@ impl RecipeSet {
}
for (name, path) in discover_recipes(&recipes_dir)? {
let recipe = Recipe::load(&path, &name, kind, options, lib)
.with_context(|| format!("loading recipe `{name}`"))?;
let key = kind.key(&name);
let loaded = Recipe::load(&path, &name, kind, options, lib)
.with_context(|| format!("loading recipe `{name}`"))?;
let Some(recipe) = loaded else {
skipped.insert(key);
continue;
};
if recipes.insert(key.clone(), recipe).is_some() {
bail!("duplicate recipe `{key}`");
@@ -416,7 +535,11 @@ impl RecipeSet {
}
}
Ok(Self { recipes, outputs })
Ok(Self {
recipes,
outputs,
skipped,
})
}
pub fn recipe(&self, key: &str) -> anyhow::Result<&Recipe> {
@@ -430,13 +553,12 @@ impl RecipeSet {
.get(key)
.ok_or_else(|| anyhow::anyhow!("unknown output package `{key}`"))
}
pub fn is_skipped(&self, key: &str) -> bool {
self.skipped.contains(key)
}
}
/// Find all recipe `.star` files under `dir`, returning a map of recipe name
/// to its `.star` file. Intermediate directories act as categories and are
/// not themselves recipes. Within any subtree, a recipe takes either form:
/// - `.../<name>.star` — name is the file stem
/// - `.../<name>/recipe.star` — name is the parent directory
fn discover_recipes(dir: &Path) -> anyhow::Result<HashMap<String, PathBuf>> {
let mut recipes: HashMap<String, PathBuf> = HashMap::new();
src/recipe/source.rs
+25 -3
View File
@@ -7,6 +7,7 @@ pub struct TarballSource {
url: String,
sha256: String,
strip_components: u32,
patches: Vec<String>,
}
impl std::fmt::Display for TarballSource {
@@ -21,11 +22,12 @@ starlark::starlark_simple_value!(TarballSource);
impl<'v> StarlarkValue<'v> for TarballSource {}
impl TarballSource {
pub fn new(url: String, sha256: String, strip_components: u32) -> Self {
pub fn new(url: String, sha256: String, strip_components: u32, patches: Vec<String>) -> Self {
Self {
url,
sha256,
strip_components,
patches,
}
}
@@ -40,12 +42,17 @@ impl TarballSource {
pub fn strip_components(&self) -> u32 {
self.strip_components
}
pub fn patches(&self) -> &[String] {
&self.patches
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct GitSource {
url: String,
commit: String,
patches: Vec<String>,
}
impl std::fmt::Display for GitSource {
@@ -60,8 +67,12 @@ starlark::starlark_simple_value!(GitSource);
impl<'v> StarlarkValue<'v> for GitSource {}
impl GitSource {
pub fn new(url: String, commit: String) -> Self {
Self { url, commit }
pub fn new(url: String, commit: String, patches: Vec<String>) -> Self {
Self {
url,
commit,
patches,
}
}
pub fn url(&self) -> &str {
@@ -71,6 +82,10 @@ impl GitSource {
pub fn commit(&self) -> &str {
&self.commit
}
pub fn patches(&self) -> &[String] {
&self.patches
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
@@ -108,4 +123,11 @@ impl Source {
pub fn is_unknown_cache_key(&self) -> bool {
matches!(self.cache_key(), "?" | "???")
}
pub fn patches(&self) -> &[String] {
match self {
Self::Tarball(source) => source.patches(),
Self::Git(source) => source.patches(),
}
}
}
src/recipe/subpackage.rs
+11 -2
View File
@@ -7,18 +7,27 @@ use crate::recipe::Metadata;
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct Subpackage {
name: String,
file_globs: Vec<String>,
metadata: Metadata,
}
impl Subpackage {
pub fn new(name: String, metadata: Metadata) -> Self {
Self { name, metadata }
pub fn new(name: String, file_globs: Vec<String>, metadata: Metadata) -> Self {
Self {
name,
file_globs,
metadata,
}
}
pub fn name(&self) -> &str {
&self.name
}
pub fn file_globs(&self) -> &[String] {
&self.file_globs
}
pub fn metadata(&self) -> &Metadata {
&self.metadata
}