4 Commits

Author SHA1 Message Date
marv7000 b6e18c474e *: Switch to python 2026-05-26 03:06:26 +02:00
marv7000 2e6704516a recipe: Implement run 2026-05-23 00:10:08 +02:00
marv7000 1a7c817fb9 recipe: basic recipe parsing 2026-05-22 20:54:44 +02:00
iretq a525868969 meta: initial rewrite (final) 2026-05-22 18:48:11 +02:00
84 changed files with 2982 additions and 7450 deletions
-4
View File
@@ -1,4 +0,0 @@
target
build
.git
*.bak
+5 -4
View File
@@ -1,4 +1,5 @@
/target
/build
*.bak
*.lock
__pycache__/
/cache
/build*
/sources
/sysroot
Generated
-2901
View File
File diff suppressed because it is too large Load Diff
-24
View File
@@ -1,24 +0,0 @@
[package]
name = "distro"
version = "0.1.0"
edition = "2024"
license = "MIT"
[dependencies]
anyhow = "1.0"
clap = { version = "4.5", features = ["derive"] }
hex = "0.4"
libc = "0.2"
reqwest = { version = "0.12", default-features = false, features = [
"blocking",
"rustls-tls",
] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
sha2 = "0.10"
shell-escape = "0.1"
starlark = "0.13"
starlark_derive = "0.13"
allocative = "0.3"
tempfile = "3.10"
walkdir = "2.5"
+46 -30
View File
@@ -1,46 +1,62 @@
FROM alpine:edge
RUN apk upgrade --no-cache && apk add --no-cache \
alpine-sdk \
RUN mkdir -p /sources /build /pkgs /sysroot /dest /tools /files
RUN apk add --no-cache \
apk-tools \
autoconf \
automake \
build-base \
bash \
bc \
bison \
bzip2 \
ca-certificates \
cmake \
coreutils \
curl \
patch \
tar \
xz \
zstd \
file \
findutils \
flex \
gawk \
gettext-dev \
git \
coreutils \
diffutils \
grep \
gzip \
elfutils-dev \
sed \
gawk \
musl-dev \
linux-headers \
gmp-dev \
mpfr-dev \
mpc1-dev \
isl-dev \
zlib-dev \
git \
pkgconf \
patchelf \
gperf \
python3 \
python3-dev \
py3-mako \
py3-yaml \
py3-packaging \
py3-docutils \
py3-passlib \
perl \
m4 \
libtool \
linux-headers \
meson \
gettext-dev \
bison \
flex \
which \
ca-certificates \
rsync \
mtools \
nasm \
ncurses \
cmake \
ninja \
openssl \
meson \
glslang \
elfutils-dev \
libffi-dev \
expat-dev \
libxml2-dev \
pcre2-dev \
openssl-dev \
patch \
pkgconf \
python3 \
rsync \
tar \
texinfo \
xz \
zstd
openssl \
ncurses
WORKDIR /work
WORKDIR /build
-18
View File
@@ -1,18 +0,0 @@
MIT License
Copyright (c) 2026 marv7000
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial
portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.
-50
View File
@@ -1,50 +0,0 @@
container_runtime = "podman"
container_image = "local/distro-builder:latest"
container_dockerfile = "Dockerfile"
arch = "x86_64"
libc = "glibc"
if libc == "glibc":
env = "gnu"
else:
env = libc
host_cflags = "-O2 -pipe"
host_cxxflags = host_cflags
host_ldflags = "-Wl,-O1 -Wl,--sort-common -Wl,--as-needed"
target_cflags = host_cflags
target_cxxflags = host_cxxflags
target_ldflags = host_ldflags + " -Wl,-z,now"
if arch == "x86_64":
flags = " -march=x86-64-v3 -mtune=generic -fstack-clash-protection -fstack-protector-strong -fcf-protection"
target_cflags += flags
target_cxxflags += flags
target_ldflags += " -Wl,-z,pack-relative-relocs"
options = dict(
target_arch = arch,
target_triple = f"{arch}-linux-{env}",
host_cflags = host_cflags,
host_cxxflags = host_cxxflags,
host_ldflags = host_ldflags,
cflags = target_cflags,
cxxflags = target_cxxflags,
ldflags = target_ldflags,
libc = libc,
prefix = "/usr",
bindir = "/usr/bin",
sbindir = "/usr/bin",
libdir = "/usr/lib",
libexecdir = "/usr/libexec",
includedir = "/usr/include",
sysconfdir = "/etc",
localstatedir = "/var",
)
+21
View File
@@ -0,0 +1,21 @@
version = "2.72"
revision = 1
description = "GNU autoconf"
license = "GPL-3.0-or-later"
url = "https://www.gnu.org/software/autoconf/"
source = tarball(
url=f"https://ftp.gnu.org/gnu/autoconf/autoconf-{version}.tar.xz",
sha256="ba885c1319578d6c94d46e9b0dceb4014caafe2490e437a0dbca3f270a223f5a",
)
def configure(self):
self.run(self.source_dir / "configure", f"--prefix={self.prefix}")
def build(self):
autotools_build(self)
def install(self):
autotools_install(self)
+22
View File
@@ -0,0 +1,22 @@
version = "1.17"
revision = 1
description = "GNU automake"
license = "GPL-2.0-or-later"
url = "https://www.gnu.org/software/automake/"
source = tarball(
url=f"https://ftp.gnu.org/gnu/automake/automake-{version}.tar.xz",
sha256="8920c1fc411e13b90bf704ef9db6f29d540e76d232cb3b2c9f4dc4cc599bd990",
)
host_deps = ["autoconf"]
def configure(self):
self.run(self.source_dir / "configure", f"--prefix={self.prefix}")
def build(self):
autotools_build(self)
def install(self):
autotools_install(self)
+39
View File
@@ -0,0 +1,39 @@
version = "2.46.0"
revision = 1
description = "GNU binutils cross-compiled for the target triple"
license = "GPL-3.0-or-later"
source = tarball(
url=f"https://ftp.gnu.org/gnu/binutils/binutils-{version}.tar.xz",
sha256="d75a94f4d73e7a4086f7513e67e439e8fcdcbb726ffe63f4661744e6256b2cf2",
)
def configure(self):
self.run(
self.source_dir / "configure",
f"--prefix={self.prefix}",
f"--target={self.triple}",
f"--with-sysroot={self.sysroot}",
"--with-pic",
"--enable-cet",
"--enable-default-execstack=no",
"--enable-deterministic-archives",
"--enable-ld=default",
"--enable-new-dtags",
"--enable-plugins",
"--enable-relro",
"--enable-separate-code",
"--enable-threads",
"--disable-nls",
"--disable-werror",
# gprofng's libcollector relies on glibc-specific internals.
"--disable-gprofng",
env={
"CFLAGS": self.profile["host_cflags"],
"CXXFLAGS": self.profile["host_cxxflags"],
"LDFLAGS": self.profile["host_ldflags"],
},
)
_, build, install = autotools()
-41
View File
@@ -1,41 +0,0 @@
version = "2.46.0"
revision = 1
metadata = meta(
description = "GNU binutils cross-compiled for the target triple",
license = "GPL-3.0-or-later",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/binutils/binutils-{version}.tar.xz",
sha256 = "d75a94f4d73e7a4086f7513e67e439e8fcdcbb726ffe63f4661744e6256b2cf2",
strip_components = 1,
)
def configure(ctx):
configure_args = [
ctx.source_dir / "configure",
"--prefix=/",
"--target=" + options.target_triple,
"--with-sysroot=" + ctx.sysroot,
"--with-pic",
"--enable-cet",
"--enable-default-execstack=no",
"--enable-deterministic-archives",
"--enable-ld=default",
"--enable-new-dtags",
"--enable-plugins",
"--enable-relro",
"--enable-separate-code",
"--enable-threads",
"--disable-nls",
"--disable-werror",
# gprofng's libcollector relies on glibc-specific internals.
"--disable-gprofng",
]
ctx.run(configure_args, env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
})
_, build, install = autotools()
+47
View File
@@ -0,0 +1,47 @@
version = "16.1.0"
revision = 1
description = "GNU GCC cross-compiler (bootstrap stage, C/C++ only)"
license = "GPL-3.0-or-later"
source = tarball(
url=f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz",
sha256="50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79",
)
host_deps = ["binutils"]
def configure(self):
self.run(
self.source_dir / "configure",
f"--target={self.triple}",
f"--prefix={self.prefix}",
f"--with-sysroot={self.sysroot}",
"--without-headers",
"--with-newlib",
"--enable-languages=c,c++",
"--enable-default-pie",
"--enable-default-ssp",
"--disable-nls",
"--disable-shared",
"--disable-threads",
"--disable-libssp",
"--disable-libgomp",
"--disable-libquadmath",
"--disable-libatomic",
"--disable-libvtv",
"--disable-multilib",
env={
"CFLAGS": self.profile["host_cflags"],
"CXXFLAGS": self.profile["host_cxxflags"],
"LDFLAGS": self.profile["host_ldflags"],
},
)
def build(self):
self.run("make", f"-j{self.jobs}", "all-gcc")
self.run("make", f"-j{self.jobs}", "all-target-libgcc")
def install(self):
self.run("make", "install-gcc", env={"DESTDIR": str(self.dest_dir)})
self.run("make", "install-target-libgcc", env={"DESTDIR": str(self.dest_dir)})
-47
View File
@@ -1,47 +0,0 @@
version = "16.1.0"
revision = 1
metadata = meta(
description = "GNU GCC cross-compiler (bootstrap stage, C/C++ only)",
license = "GPL-3.0-or-later",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz",
sha256 = "50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79",
strip_components = 1,
)
host_deps = ["binutils"]
def configure(ctx):
ctx.run([
ctx.source_dir / "configure",
"--target=" + options.target_triple,
"--prefix=/",
"--with-sysroot=" + ctx.sysroot,
"--without-headers",
"--with-newlib",
"--enable-languages=c,c++",
"--enable-default-pie",
"--enable-default-ssp",
"--disable-nls",
"--disable-shared",
"--disable-threads",
"--disable-libssp",
"--disable-libgomp",
"--disable-libquadmath",
"--disable-libatomic",
"--disable-libvtv",
"--disable-multilib",
], env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
})
def build(ctx):
jobs = "-j" + str(ctx.jobs)
ctx.run(["make", jobs, "all-gcc"])
ctx.run(["make", jobs, "all-target-libgcc"])
def install(ctx, pkg):
ctx.run(["make", "install-gcc"], env = {"DESTDIR": pkg.dest_dir})
ctx.run(["make", "install-target-libgcc"], env = {"DESTDIR": pkg.dest_dir})
+54
View File
@@ -0,0 +1,54 @@
version = "16.1.0"
revision = 1
description = "GNU GCC cross-compiler targeting the system triple"
license = "GPL-3.0-or-later"
url = "https://gcc.gnu.org/"
source = tarball(
url=f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz",
sha256="50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79",
)
host_deps = ["binutils", "gcc-bootstrap"]
deps = [profile["libc"], "linux-headers"]
def configure(self):
self.run(
self.source_dir / "configure",
f"--target={self.triple}",
f"--prefix={self.prefix}",
f"--with-sysroot={self.sysroot}",
f"--with-build-sysroot={self.sysroot}",
f"--with-gxx-include-dir={self.sysroot}{self.profile['includedir']}/c++/{version}",
"--enable-languages=c,c++,lto",
"--disable-bootstrap",
"--enable-default-pie",
"--enable-default-ssp",
"--enable-lto",
"--enable-threads=posix",
"--enable-tls",
"--enable-libstdcxx-time",
"--enable-checking=release",
"--enable-cet=auto",
"--enable-linker-build-id",
"--disable-nls",
"--disable-multilib",
"--disable-fixed-point",
"--disable-werror",
"--disable-libsanitizer",
"--disable-symvers",
env={
"CFLAGS": self.profile["host_cflags"],
"CXXFLAGS": self.profile["host_cxxflags"],
"LDFLAGS": self.profile["host_ldflags"],
},
)
def build(self):
self.run("make", f"-j{self.jobs}")
def install(self):
self.run("make", "install-strip", env={"DESTDIR": str(self.dest_dir)})
# Drop libtool archives.
self.run("find", self.dest_dir, "-name", "*.la", "-delete")
-53
View File
@@ -1,53 +0,0 @@
version = "16.1.0"
revision = 1
metadata = meta(
description = "GNU GCC cross-compiler targeting the system triple",
license = "GPL-3.0-or-later",
website = "https://gcc.gnu.org/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz",
sha256 = "50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79",
strip_components = 1,
)
host_deps = ["binutils", "gcc-bootstrap"]
deps = [options.libc, "linux-headers"]
def configure(ctx):
ctx.run([
ctx.source_dir / "configure",
"--target=" + options.target_triple,
"--prefix=/",
"--with-sysroot=" + ctx.sysroot,
"--with-build-sysroot=" + ctx.sysroot,
"--enable-languages=c,c++,lto",
"--disable-bootstrap",
"--enable-default-pie",
"--enable-default-ssp",
"--enable-lto",
"--enable-threads=posix",
"--enable-tls",
"--enable-libstdcxx-time",
"--enable-checking=release",
"--enable-cet=auto",
"--enable-linker-build-id",
"--disable-nls",
"--disable-multilib",
"--disable-fixed-point",
"--disable-werror",
"--disable-libsanitizer",
"--disable-symvers",
], env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
})
def build(ctx):
ctx.run(["make", "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["make", "install-strip"], env = {"DESTDIR": pkg.dest_dir})
# Drop libtool archives.
ctx.run(["find", pkg.dest_dir, "-name", "*.la", "-delete"])
+56
View File
@@ -0,0 +1,56 @@
version = "20.1.0"
revision = 1
description = "LLVM compiler infrastructure with clang and lld"
license = "Apache-2.0 WITH LLVM-exception"
url = "https://llvm.org/"
source = tarball(
url=f"https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/llvm-project-{version}.src.tar.xz",
sha256="4579051e3c255fb4bb795d54324f5a7f3ef79bd9181e44293d7ee9a7f62aad9a",
)
host_deps = ["binutils"]
def configure(self):
self.run(
"cmake",
"-S",
self.source_dir / "llvm",
"-B",
self.build_dir,
"-G",
"Ninja",
"-DCMAKE_BUILD_TYPE=Release",
f"-DCMAKE_INSTALL_PREFIX={self.prefix}",
"-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;lld",
"-DLLVM_ENABLE_RUNTIMES=compiler-rt",
"-DLLVM_TARGETS_TO_BUILD=X86;AArch64;RISCV",
f"-DLLVM_DEFAULT_TARGET_TRIPLE={self.triple}",
f"-DLLVM_HOST_TRIPLE={self.triple}",
"-DLLVM_ENABLE_LIBXML2=OFF",
"-DLLVM_ENABLE_LIBEDIT=OFF",
"-DLLVM_ENABLE_TERMINFO=OFF",
"-DLLVM_ENABLE_ASSERTIONS=OFF",
"-DLLVM_ENABLE_PIC=ON",
"-DLLVM_BUILD_LLVM_DYLIB=ON",
"-DLLVM_LINK_LLVM_DYLIB=ON",
"-DLLVM_INSTALL_UTILS=ON",
"-DLLVM_INCLUDE_TESTS=OFF",
"-DLLVM_INCLUDE_EXAMPLES=OFF",
"-DLLVM_INCLUDE_BENCHMARKS=OFF",
"-DCLANG_DEFAULT_LINKER=lld",
"-DCLANG_DEFAULT_RTLIB=compiler-rt",
"-DCLANG_DEFAULT_CXX_STDLIB=libstdc++",
env={
"CFLAGS": self.profile["host_cflags"],
"CXXFLAGS": self.profile["host_cxxflags"],
"LDFLAGS": self.profile["host_ldflags"],
},
)
def build(self):
self.run("cmake", "--build", self.build_dir, f"-j{self.jobs}")
def install(self):
self.run("cmake", "--install", self.build_dir, env={"DESTDIR": str(self.dest_dir)})
-52
View File
@@ -1,52 +0,0 @@
version = "20.1.0"
revision = 1
metadata = meta(
description = "LLVM compiler infrastructure with clang and lld",
license = "Apache-2.0 WITH LLVM-exception",
website = "https://llvm.org/",
)
source = tarball_source(
url = f"https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/llvm-project-{version}.src.tar.xz",
sha256 = "?",
strip_components = 1,
)
host_deps = ["binutils"]
def configure(ctx):
ctx.run([
"cmake",
"-S", ctx.source_dir / "llvm",
"-B", ctx.build_dir,
"-G", "Ninja",
"-DCMAKE_BUILD_TYPE=Release",
"-DCMAKE_INSTALL_PREFIX=/",
"-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;lld",
"-DLLVM_ENABLE_RUNTIMES=compiler-rt",
"-DLLVM_TARGETS_TO_BUILD=X86;AArch64;RISCV",
"-DLLVM_DEFAULT_TARGET_TRIPLE=" + options.target_triple,
"-DLLVM_HOST_TRIPLE=" + options.target_triple,
"-DLLVM_ENABLE_LIBXML2=OFF",
"-DLLVM_ENABLE_LIBEDIT=OFF",
"-DLLVM_ENABLE_TERMINFO=OFF",
"-DLLVM_ENABLE_ASSERTIONS=OFF",
"-DLLVM_ENABLE_PIC=ON",
"-DLLVM_BUILD_LLVM_DYLIB=ON",
"-DLLVM_LINK_LLVM_DYLIB=ON",
"-DLLVM_INSTALL_UTILS=ON",
"-DLLVM_INCLUDE_TESTS=OFF",
"-DLLVM_INCLUDE_EXAMPLES=OFF",
"-DLLVM_INCLUDE_BENCHMARKS=OFF",
"-DCLANG_DEFAULT_LINKER=lld",
"-DCLANG_DEFAULT_RTLIB=compiler-rt",
"-DCLANG_DEFAULT_CXX_STDLIB=libstdc++",
], env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
})
def build(ctx):
ctx.run(["cmake", "--build", ctx.build_dir, "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["cmake", "--install", ctx.build_dir], env = {"DESTDIR": pkg.dest_dir})
-95
View File
@@ -1,95 +0,0 @@
# Autotools
def autotools_configure(ctx, extra_args = [], extra_env = {}):
env = {
"CFLAGS": options.cflags,
"CXXFLAGS": options.cxxflags,
"LDFLAGS": options.ldflags,
}
env.update(extra_env)
ctx.run([
ctx.source_dir / "configure",
"--host=" + options.target_triple,
"--with-sysroot=" + ctx.sysroot,
"--prefix=" + options.prefix,
"--sysconfdir=" + options.sysconfdir,
"--localstatedir=" + options.localstatedir,
"--bindir=" + options.bindir,
"--sbindir=" + options.sbindir,
"--libdir=" + options.libdir,
"--disable-static",
"--enable-shared",
] + extra_args, env = env)
def autotools_build(ctx, extra_args = []):
ctx.run(["make", "-j" + str(ctx.jobs)] + extra_args)
def autotools_install(ctx, pkg, extra_args = []):
ctx.run(["make", "install"] + extra_args, env = { "DESTDIR": pkg.dest_dir })
def autotools(configure_args = [], configure_env = {}, build_args = [], install_args = []):
def _configure(ctx):
autotools_configure(ctx, extra_args = configure_args, extra_env = configure_env)
def _build(ctx):
autotools_build(ctx, extra_args = build_args)
def _install(ctx, pkg):
autotools_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install
# CMake
def cmake_configure(ctx, extra_args = [], extra_env = {}, host = False):
if host:
env = {
"CFLAGS": options.host_cflags,
"CXXFLAGS": options.host_cxxflags,
"LDFLAGS": options.host_ldflags,
}
toolchain_args = []
else:
env = {
"CFLAGS": options.cflags,
"CXXFLAGS": options.cxxflags,
"LDFLAGS": options.ldflags,
}
toolchain_args = [
"-DCMAKE_SYSTEM_NAME=Linux",
"-DCMAKE_SYSTEM_PROCESSOR=" + options.target_arch,
"-DCMAKE_SYSROOT=" + ctx.sysroot,
"-DCMAKE_C_COMPILER=" + options.target_triple + "-gcc",
"-DCMAKE_CXX_COMPILER=" + options.target_triple + "-g++",
"-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
"-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
"-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
"-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY",
]
env.update(extra_env)
ctx.run([
"cmake",
"-S", ctx.source_dir,
"-B", ctx.build_dir,
"-G", "Ninja",
"-DCMAKE_BUILD_TYPE=Release",
"-DCMAKE_INSTALL_PREFIX=" + options.prefix,
"-DCMAKE_INSTALL_SYSCONFDIR=" + options.sysconfdir,
"-DCMAKE_INSTALL_LOCALSTATEDIR=" + options.localstatedir,
] + toolchain_args + extra_args, env = env)
def cmake_build(ctx, extra_args = []):
ctx.run(["cmake", "--build", ctx.build_dir, "-j", str(ctx.jobs)] + extra_args)
def cmake_install(ctx, pkg, extra_args = []):
ctx.run(
["cmake", "--install", ctx.build_dir] + extra_args,
env = {"DESTDIR": pkg.dest_dir},
)
def cmake(configure_args = [], configure_env = {}, build_args = [], install_args = [], host = False):
def _configure(ctx):
cmake_configure(ctx, extra_args = configure_args, extra_env = configure_env, host = host)
def _build(ctx):
cmake_build(ctx, extra_args = build_args)
def _install(ctx, pkg):
cmake_install(ctx, pkg, extra_args = install_args)
return _configure, _build, _install
Symlink
+1
View File
@@ -0,0 +1 @@
src/__main__.py
+34
View File
@@ -0,0 +1,34 @@
def profile():
arch = "x86_64"
libc = "glibc"
triple = f"{arch}-orchid-linux-gnu"
host_cflags = "-O2 -pipe"
host_cxxflags = host_cflags
host_ldflags = "-Wl,-O1 -Wl,--sort-common -Wl,--as-needed"
target_flags = " -march=x86-64-v3 -mtune=generic -fstack-clash-protection -fstack-protector-strong -fcf-protection"
cflags = host_cflags + target_flags
cxxflags = cflags
ldflags = host_ldflags + " -Wl,-z,now -Wl,-z,pack-relative-relocs"
return {
"arch": arch,
"libc": libc,
"triple": triple,
"container_image": "localhost/orchid-builder:latest",
"host_cflags": host_cflags,
"host_cxxflags": host_cxxflags,
"host_ldflags": host_ldflags,
"cflags": cflags,
"cxxflags": cxxflags,
"ldflags": ldflags,
"prefix": "/usr",
"bindir": "/usr/bin",
"sbindir": "/usr/bin",
"libdir": "/usr/lib",
"libexecdir": "/usr/libexec",
"includedir": "/usr/include",
"sysconfdir": "/etc",
"localstatedir": "/var",
}
+34
View File
@@ -0,0 +1,34 @@
def profile():
arch = "x86_64"
libc = "musl"
triple = f"{arch}-orchid-linux-{libc}"
host_cflags = "-O2 -pipe"
host_cxxflags = host_cflags
host_ldflags = "-Wl,-O1 -Wl,--sort-common -Wl,--as-needed"
target_flags = " -march=x86-64-v3 -mtune=generic -fstack-clash-protection -fstack-protector-strong -fcf-protection"
cflags = host_cflags + target_flags
cxxflags = cflags
ldflags = host_ldflags + " -Wl,-z,now -Wl,-z,pack-relative-relocs"
return {
"arch": arch,
"libc": libc,
"triple": triple,
"container_image": "localhost/orchid-builder:latest",
"host_cflags": host_cflags,
"host_cxxflags": host_cxxflags,
"host_ldflags": host_ldflags,
"cflags": cflags,
"cxxflags": cxxflags,
"ldflags": ldflags,
"prefix": "/usr",
"bindir": "/usr/bin",
"sbindir": "/usr/bin",
"libdir": "/usr/lib",
"libexecdir": "/usr/libexec",
"includedir": "/usr/include",
"sysconfdir": "/etc",
"localstatedir": "/var",
}
+6
View File
@@ -0,0 +1,6 @@
{
"ignore": [
"recipes",
"host-recipes"
]
}
+28
View File
@@ -0,0 +1,28 @@
version = "5.2.32"
revision = 1
description = "GNU Bourne-Again SHell"
license = "GPL-3.0-or-later"
url = "https://www.gnu.org/software/bash/"
source = tarball(
url=f"https://ftp.gnu.org/gnu/bash/bash-{version}.tar.gz",
sha256="d3ef80d2b67d8cbbe4d3265c63a72c46f9b278ead6e0e06d61801b58f23f50b5",
)
host_deps = ["autoconf", "automake", "binutils", "gcc"]
deps = [profile["libc"], "ncurses", "readline"]
build_if = False # TODO: Doesn't build yet
configure, build, install = autotools(
configure_args=[
"--without-bash-malloc",
"--disable-nls",
"--with-installed-readline",
"--with-curses",
"--enable-readline",
"--with-installed-readline=/sysroot/usr", # TODO: get /sysroot from context
],
configure_env={
"CFLAGS": profile["cflags"] + " -std=gnu17",
"CFLAGS_FOR_BUILD": profile["cflags"] + " -std=gnu17",
},
)
-22
View File
@@ -1,22 +0,0 @@
version = "5.2.32"
revision = 1
metadata = meta(
description = "GNU Bourne-Again SHell",
license = "GPL-3.0-or-later",
website = "https://www.gnu.org/software/bash/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/bash/bash-{version}.tar.gz",
sha256 = "d3ef80d2b67d8cbbe4d3265c63a72c46f9b278ead6e0e06d61801b58f23f50b5",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc, "ncurses", "readline"]
configure, build, install = autotools(configure_args = [
"--without-bash-malloc",
"--with-installed-readline",
"--enable-readline",
"--enable-history",
"--enable-job-control",
], configure_env = {"CFLAGS": "-std=gnu17"})
+20
View File
@@ -0,0 +1,20 @@
version = "9.6"
revision = 1
description = "GNU core utilities (file, shell, and text manipulation)"
license = "GPL-3.0-or-later"
url = "https://www.gnu.org/software/coreutils/"
source = tarball(
url=f"https://ftp.gnu.org/gnu/coreutils/coreutils-{version}.tar.xz",
sha256="7a0124327b398fd9eb1a6abde583389821422c744ffa10734b24f557610d3283",
)
host_deps = ["autoconf", "automake", "binutils", "gcc"]
deps = [profile["libc"]]
configure, build, install = autotools(
configure_args=[
"--enable-no-install-program=kill,uptime",
"--without-selinux",
"--without-openssl",
],
configure_env={"FORCE_UNSAFE_CONFIGURE": "1"},
)
-25
View File
@@ -1,25 +0,0 @@
version = "9.6"
revision = 1
metadata = meta(
description = "GNU core utilities (file, shell, and text manipulation)",
license = "GPL-3.0-or-later",
website = "https://www.gnu.org/software/coreutils/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/coreutils/coreutils-{version}.tar.xz",
sha256 = "7a0124327b398fd9eb1a6abde583389821422c744ffa10734b24f557610d3283",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, install = autotools(
configure_args = [
"--enable-no-install-program=kill,uptime",
"--without-selinux",
"--without-openssl",
],
# coreutils' configure runs link tests that require a working executable;
# cross builds need this hint to skip a known false positive.
configure_env = {"FORCE_UNSAFE_CONFIGURE": "1"},
)
+38
View File
@@ -0,0 +1,38 @@
version = "2.41"
revision = 1
description = "GNU C library"
license = "LGPL-2.1-or-later"
url = "https://www.gnu.org/software/libc/"
source = tarball(
url=f"https://ftp.gnu.org/gnu/glibc/glibc-{version}.tar.xz",
sha256="a5a26b22f545d6b7d7b3dd828e11e428f24f4fac43c934fb071b6a7d0828e901",
)
host_deps = ["autoconf", "automake", "binutils", "gcc-bootstrap"]
deps = ["linux-headers"]
build_if = profile["libc"] == "glibc"
def configure(self):
autotools_configure(
self,
[
f"--build={self.triple}",
f"--with-headers={self.sysroot}{self.profile['prefix']}/include",
"--enable-kernel=5.4",
"--enable-bind-now",
"--enable-stack-protector=strong",
"--enable-cet",
"--disable-werror",
"--disable-profile",
"--disable-nscd",
"--without-selinux",
"--without-gd",
"libc_cv_slibdir=/lib",
"libc_cv_rtlddir=/lib",
"libc_cv_forced_unwind=yes",
],
)
_, build, install = autotools()
-36
View File
@@ -1,36 +0,0 @@
version = "2.41"
revision = 1
metadata = meta(
description = "GNU C library",
license = "LGPL-2.1-or-later",
website = "https://www.gnu.org/software/libc/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/glibc/glibc-{version}.tar.xz",
sha256 = "a5a26b22f545d6b7d7b3dd828e11e428f24f4fac43c934fb071b6a7d0828e901",
strip_components = 1,
)
host_deps = ["binutils", "gcc-bootstrap"]
deps = ["linux-headers"]
build_if = options.libc == "glibc"
def configure(ctx):
autotools_configure(ctx, [
"--build=" + options.target_triple,
"--with-headers=" + ctx.sysroot / options.prefix / "include",
"--enable-kernel=5.4",
"--enable-bind-now",
"--enable-stack-protector=strong",
"--enable-cet",
"--disable-werror",
"--disable-profile",
"--disable-nscd",
"--without-selinux",
"--without-gd",
"libc_cv_slibdir=/lib",
"libc_cv_rtlddir=/lib",
"libc_cv_forced_unwind=yes",
])
_, build, install = autotools()
+59
View File
@@ -0,0 +1,59 @@
version = "16.1.0"
revision = 1
description = "GNU C++ standard library"
license = "GPL-3.0-or-later"
url = "https://gcc.gnu.org/"
source = tarball(
url=f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz",
sha256="50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79",
)
host_deps = ["binutils", "gcc"]
deps = [profile["libc"]]
def configure(self):
self.run(
self.source_dir / "configure",
f"--target={self.triple}",
f"--prefix={self.profile['prefix']}",
f"--libdir={self.profile['libdir']}",
f"--with-sysroot={self.sysroot}",
f"--with-build-sysroot={self.sysroot}",
f"--with-gxx-include-dir={self.profile['includedir']}/c++/{version}",
f"--with-toolexeclibdir={self.profile['libdir']}",
"--enable-languages=c,c++",
"--disable-bootstrap",
"--enable-default-pie",
"--enable-default-ssp",
"--enable-lto",
"--enable-threads=posix",
"--enable-tls",
"--enable-libstdcxx-time",
"--enable-checking=release",
"--enable-cet=auto",
"--enable-linker-build-id",
"--disable-nls",
"--disable-multilib",
"--disable-fixed-point",
"--disable-werror",
"--disable-libsanitizer",
"--disable-symvers",
env={
"CFLAGS": self.profile["host_cflags"],
"CXXFLAGS": self.profile["host_cxxflags"],
"LDFLAGS": self.profile["host_ldflags"],
},
)
def build(self):
self.run("make", f"-j{self.jobs}", "all-target-libstdc++-v3")
def install(self):
self.run(
"make",
"install-target-libstdc++-v3",
env={"DESTDIR": str(self.dest_dir)},
)
self.run("find", self.dest_dir, "-name", "*.la", "-delete")
+50
View File
@@ -0,0 +1,50 @@
version = "12.2.0"
revision = 1
description = "Modern, secure, portable, multiprotocol bootloader and boot manager"
license = "BSD-2-Clause"
url = "https://limine-bootloader.org"
source = tarball(
url=f"https://github.com/Limine-Bootloader/Limine/releases/download/v{version}/limine-{version}.tar.gz",
sha256="db8a119878cfeead63c0a78236c577c40539c5759496950ea0ed32a6cf567865",
)
host_deps = ["autoconf", "automake", "binutils", "gcc"]
deps = [profile["libc"]]
build_if = profile["arch"] in ("x86_64", "aarch64", "riscv64", "loongarch64")
_arch_args = {
"x86_64": [
"--enable-uefi-x86-64",
"--enable-uefi-ia32",
"--enable-bios",
"--enable-bios-cd",
],
"aarch64": ["--enable-uefi-aarch64"],
"riscv64": ["--enable-uefi-riscv64"],
"loongarch64": ["--enable-uefi-loongarch64"],
}
configure, build, install = autotools(
configure_args=["--enable-uefi-cd", *_arch_args.get(profile["arch"], [])],
configure_env={"TOOLCHAIN_FOR_TARGET": profile["triple"] + "-"},
)
subpackages = [
subpackage(
"limine-uefi",
description="UEFI files",
files=[
"usr/share/limine/BOOT*.EFI",
"usr/share/limine/limine-uefi-*.bin",
],
),
]
if profile["arch"] == "x86_64":
subpackages.append(
subpackage(
"limine-bios",
description="BIOS files",
files=["usr/share/limine/limine-bios*"],
)
)
-51
View File
@@ -1,51 +0,0 @@
version = "12.2.0"
revision = 1
metadata = meta(
description = "Modern, secure, portable, multiprotocol bootloader and boot manager",
license = "BSD-2-Clause",
website = "https://limine-bootloader.org"
)
source = tarball_source(
url = f"https://github.com/Limine-Bootloader/Limine/releases/download/v{version}/limine-{version}.tar.gz",
sha256 = "db8a119878cfeead63c0a78236c577c40539c5759496950ea0ed32a6cf567865",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
build_if = options.target_arch in ["x86_64", "aarch64", "riscv64", "loongarch64"]
arch_configure_args = {
"x86_64": ["--enable-uefi-x86-64", "--enable-uefi-ia32", "--enable-bios", "--enable-bios-cd"],
"aarch64": ["--enable-uefi-aarch64"],
"riscv64": ["--enable-uefi-riscv64"],
"loongarch64": ["--enable-uefi-loongarch64"],
}
configure, build, install = autotools(
configure_args = ["--enable-uefi-cd"] + arch_configure_args.get(options.target_arch),
configure_env = {"TOOLCHAIN_FOR_TARGET": options.target_triple + "-"},
)
subpackages = [
subpackage(
"limine-uefi",
meta(description = "UEFI files"),
[
"usr/share/limine/BOOT*.EFI",
"usr/share/limine/limine-uefi-*.bin",
],
),
]
if options.target_arch == "x86_64":
subpackages += [
subpackage(
"limine-bios",
meta(description = "BIOS files"),
[
"usr/share/limine/limine-bios*",
],
)
]
+38
View File
@@ -0,0 +1,38 @@
version = "7.0.9"
revision = 2
description = "Linux kernel headers for userspace development"
license = "GPL-2.0-only"
source = tarball(
url=f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz",
sha256="ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4",
)
linux_archs = {"aarch64": "arm64"}
def build(self):
# Stage the source into the (writable) build dir before invoking the kernel
# build system, which is not happy with a read-only source tree.
self.run("cp", "-rp", f"{self.source_dir}/.", self.build_dir)
arch = linux_archs.get(self.profile["arch"], self.profile["arch"])
self.run("make", "headers_install", f"ARCH={arch}")
self.run(
"find",
self.build_dir / "usr/include",
"-type",
"f",
"!",
"-name",
"*.h",
"-delete",
)
def install(self):
self.run("mkdir", "-p", self.dest_dir / self.profile["prefix"].lstrip("/"))
self.run(
"cp",
"-rp",
self.build_dir / "usr/include",
str(self.dest_dir) + self.profile["prefix"],
)
-20
View File
@@ -1,20 +0,0 @@
version = "7.0.9"
revision = 1
metadata = meta(
description = "Linux kernel headers for userspace development",
license = "GPL-2.0-only",
)
source = tarball_source(
url = f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz",
sha256 = "ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4",
strip_components = 1,
)
def build(ctx):
ctx.run(["cp", "-rp", ctx.source_dir / ".", ctx.build_dir])
ctx.run(["make", "headers_install", "ARCH=" + options.target_arch])
ctx.run(["find", ctx.build_dir / "usr/include", "-type", "f", "!", "-name", "*.h", "-delete"])
def install(ctx, pkg):
ctx.run(["mkdir", "-p", pkg.dest_dir / options.prefix])
ctx.run(["cp", "-rp", ctx.build_dir / "usr/include", pkg.dest_dir / options.prefix])
+48
View File
@@ -0,0 +1,48 @@
version = "7.0.9"
revision = 2
description = "Linux kernel"
license = "GPL-2.0-only"
source = tarball(
url=f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz",
sha256="ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4",
)
host_deps = ["binutils", "gcc"]
deps = [profile["libc"]]
linux_archs = {"aarch64": "arm64"}
linux_subarchs = {"x86_64": "x86"}
def _make(self, *extra):
arch = linux_archs.get(self.profile["arch"], self.profile["arch"])
subarch = linux_subarchs.get(self.profile["arch"])
subarch_arg = (f"SUBARCH={subarch}",) if subarch else ()
return (
"make",
f"ARCH={arch}",
*subarch_arg,
f"CROSS_COMPILE={self.triple}-",
f"-j{self.jobs}",
*extra,
)
def configure(self):
self.run("cp", "-rp", f"{self.source_dir}/.", self.build_dir)
self.run(
"cp", self.files / f"config.{self.profile['arch']}", self.build_dir / ".config"
)
self.run(*_make(self, "olddefconfig"))
def build(self):
self.run(*_make(self))
def install(self):
self.run(
"install",
"-Dm644",
self.build_dir / "arch/x86/boot/bzImage",
self.dest_dir / f"boot/vmlinuz-{self.version}",
)
-44
View File
@@ -1,44 +0,0 @@
version = "7.0.9"
revision = 1
metadata = meta(
description = "Linux kernel",
license = "GPL-2.0-only",
)
source = tarball_source(
url = f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz",
sha256 = "ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
def make_args(ctx, *args):
# Translate arch name
if options.target_arch == "aarch64":
linux_arch = "arm64"
else:
linux_arch = options.target_arch
result = [
"make",
"ARCH=" + linux_arch,
"CROSS_COMPILE=" + options.target_triple + "-",
"-j" + str(ctx.jobs),
]
result.extend(args)
return result
def configure(ctx):
ctx.run(["cp", "-rp", ctx.source_dir / ".", ctx.build_dir])
ctx.run(["cp", ctx.files / "config." + options.target_arch, ctx.build_dir / ".config"])
ctx.run(make_args(ctx, "olddefconfig"))
def build(ctx):
ctx.run(make_args(ctx))
def install(ctx, pkg):
ctx.install(
ctx.build_dir + "/arch/x86/boot/bzImage",
pkg.destdir + "/boot/vmlinuz-" + version,
)
+13
View File
@@ -0,0 +1,13 @@
version = "4.4.1"
revision = 1
description = "GNU make build automation tool"
license = "GPL-3.0-or-later"
url = "https://www.gnu.org/software/make/"
source = tarball(
url=f"https://ftp.gnu.org/gnu/make/make-{version}.tar.gz",
sha256="dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3",
)
host_deps = ["autoconf", "automake", "binutils", "gcc"]
deps = [profile["libc"]]
configure, build, install = autotools(configure_args=["--without-guile"])
-18
View File
@@ -1,18 +0,0 @@
version = "4.4.1"
revision = 1
metadata = meta(
description = "GNU make build automation tool",
license = "GPL-3.0-or-later",
website = "https://www.gnu.org/software/make/",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/make/make-{version}.tar.gz",
sha256 = "dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, install = autotools(configure_args = [
"--without-guile",
])
+28
View File
@@ -0,0 +1,28 @@
version = "1.2.6"
revision = 1
description = "Small, standards-conformant implementation of libc"
license = "MIT"
source = tarball(
url=f"https://musl.libc.org/releases/musl-{version}.tar.gz",
sha256="d585fd3b613c66151fc3249e8ed44f77020cb5e6c1e635a616d3f9f82460512a",
)
host_deps = ["binutils", "gcc-bootstrap"]
build_if = profile["libc"] == "musl"
def configure(self):
self.run(
self.source_dir / "configure",
f"--target={self.triple}",
f"--prefix={self.profile['prefix']}",
"--syslibdir=/lib",
env={
"CC": f"{self.triple}-gcc",
"CFLAGS": self.profile["cflags"],
"LDFLAGS": self.profile["ldflags"],
},
)
_, build, install = autotools()
-27
View File
@@ -1,27 +0,0 @@
version = "1.2.6"
revision = 1
metadata = meta(
description = "Small, standards-conformant implementation of libc",
license = "MIT",
)
source = tarball_source(
url = f"https://musl.libc.org/releases/musl-{version}.tar.gz",
sha256 = "d585fd3b613c66151fc3249e8ed44f77020cb5e6c1e635a616d3f9f82460512a",
strip_components = 1,
)
host_deps = ["binutils", "gcc-bootstrap"]
build_if = options.libc == "musl"
def configure(ctx):
ctx.run([
ctx.source_dir / "configure",
"--target=" + options.target_triple,
"--prefix=" + options.prefix,
"--syslibdir=/lib",
], env = {
"CC": options.target_triple + "-gcc",
"CFLAGS": options.cflags,
"LDFLAGS": options.ldflags,
})
_, build, install = autotools()
+39
View File
@@ -0,0 +1,39 @@
version = "6.5"
revision = 1
description = "Terminal control library with wide-character support"
license = "MIT"
url = "https://invisible-island.net/ncurses/"
source = tarball(
url=f"https://invisible-mirror.net/archives/ncurses/ncurses-{version}.tar.gz",
sha256="136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6",
)
host_deps = ["autoconf", "automake", "binutils", "gcc"]
deps = [profile["libc"], "libstdc++"]
configure, build, _ = autotools(
configure_args=[
"--with-shared",
"--without-debug",
"--without-ada",
"--enable-pc-files",
"--enable-widec",
"--with-termlib",
"--with-cxx-binding",
"--with-cxx-shared",
"--with-pkg-config-libdir=/usr/lib/pkgconfig",
"--with-tic-path=/usr/bin/tic",
"--with-infocmp-path=/usr/bin/infocmp",
"--mandir=/usr/share/man",
],
configure_env={
# Conflicts with GCC 16 headers.
"cf_cv_type_of_bool": "bool",
"cf_cv_cc_bool_type": "1",
"cf_cv_builtin_bool": "1",
"ac_cv_header_stdbool_h": "yes",
},
)
def install(self):
autotools_install(self, [f"DESTDIR={self.dest_dir}"])
-38
View File
@@ -1,38 +0,0 @@
version = "6.5"
revision = 1
metadata = meta(
description = "Terminal control library with wide-character support",
license = "MIT",
website = "https://invisible-island.net/ncurses/",
)
source = tarball_source(
url = f"https://invisible-mirror.net/archives/ncurses/ncurses-{version}.tar.gz",
sha256 = "136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, _ = autotools(configure_args = [
"--with-shared",
"--without-debug",
"--without-ada",
"--enable-pc-files",
"--enable-widec",
"--with-termlib",
"--with-cxx-binding",
"--with-cxx-shared",
"--with-pkg-config-libdir=/usr/lib/pkgconfig",
"--mandir=/usr/share/man",
], configure_env = {
# Conflicts with GCC 16 headers
"cf_cv_type_of_bool": "bool",
"cf_cv_cc_bool_type": "1",
"cf_cv_builtin_bool": "1",
"ac_cv_header_stdbool_h": "yes",
})
def install(ctx, pkg):
autotools_install(ctx, pkg, extra_args = [
"DESTDIR=" + pkg.dest_dir
])
+53
View File
@@ -0,0 +1,53 @@
version = "3.4.1"
revision = 1
description = "Cryptography and TLS library (OpenSSL)"
license = "Apache-2.0"
url = "https://www.openssl.org/"
source = tarball(
url=f"https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz",
sha256="?",
)
host_deps = ["binutils", "gcc"]
deps = ["zlib"]
ossl_targets = {
"x86_64": "linux-x86_64",
"aarch64": "linux-aarch64",
"riscv64": "linux64-riscv64",
}
def configure(self):
target = ossl_targets.get(self.profile["arch"])
if target is None:
raise ValueError(f"openssl: unsupported arch {self.profile['arch']}")
self.run(
self.source_dir / "Configure",
target,
f"--prefix={self.profile['prefix']}",
"--openssldir=/etc/ssl",
"--libdir=lib",
"shared",
"zlib",
"no-tests",
"no-static-engine",
"enable-ktls",
env={
"CC": f"{self.triple}-gcc",
"AR": f"{self.triple}-ar",
"RANLIB": f"{self.triple}-ranlib",
"CFLAGS": self.profile["cflags"],
"LDFLAGS": self.profile["ldflags"],
},
)
def build(self):
self.run("make", f"-j{self.jobs}")
def install(self):
self.run(
"make", "install_sw", "install_ssldirs", env={"DESTDIR": str(self.dest_dir)}
)
-51
View File
@@ -1,51 +0,0 @@
version = "3.4.1"
revision = 1
metadata = meta(
description = "Cryptography and TLS library (OpenSSL)",
license = "Apache-2.0",
website = "https://www.openssl.org/",
)
source = tarball_source(
url = f"https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz",
sha256 = "?",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = ["zlib"]
def configure(ctx):
# OpenSSL uses its own perl-based Configure script. The first argument is
# the OpenSSL "target" — pick the one matching our triple.
if options.target_arch == "x86_64":
ossl_target = "linux-x86_64"
elif options.target_arch == "aarch64":
ossl_target = "linux-aarch64"
elif options.target_arch == "riscv64":
ossl_target = "linux64-riscv64"
else:
fail("openssl: unsupported target_arch " + options.target_arch)
ctx.run([
ctx.source_dir / "Configure",
ossl_target,
"--prefix=" + options.prefix,
"--openssldir=/etc/ssl",
"--libdir=lib",
"shared",
"zlib",
"no-tests",
"no-static-engine",
"enable-ktls",
], env = {
"CC": options.target_triple + "-gcc",
"AR": options.target_triple + "-ar",
"RANLIB": options.target_triple + "-ranlib",
"CFLAGS": options.cflags,
"LDFLAGS": options.ldflags,
})
def build(ctx):
ctx.run(["make", "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["make", "install_sw", "install_ssldirs"], env = {"DESTDIR": pkg.dest_dir})
+18
View File
@@ -0,0 +1,18 @@
version = "3.3.3"
revision = 1
description = "Lightweight pkg-config implementation"
license = "ISC"
url = "http://pkgconf.org/"
source = tarball(
url=f"https://distfiles.ariadne.space/pkgconf/pkgconf-{version}.tar.xz",
sha256="?",
)
host_deps = ["autoconf", "automake", "binutils", "gcc"]
deps = [profile["libc"]]
configure, build, install = autotools(
configure_args=[
f"--with-system-libdir={profile['libdir']}",
f"--with-system-includedir={profile['includedir']}",
]
)
-19
View File
@@ -1,19 +0,0 @@
version = "3.3.3"
revision = 1
metadata = meta(
description = "Lightweight pkg-config implementation",
license = "ISC",
website = "http://pkgconf.org/",
)
source = tarball_source(
url = f"https://distfiles.ariadne.space/pkgconf/pkgconf-{version}.tar.xz",
sha256 = "?",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, install = autotools(configure_args = [
"--with-system-libdir=" + options.libdir,
"--with-system-includedir=" + options.includedir,
])
+22
View File
@@ -0,0 +1,22 @@
version = "8.2"
revision = 1
description = "Library for command-line editing"
license = "GPL-3.0-or-later"
url = "https://tiswww.case.edu/php/chet/readline/rltop.html"
source = tarball(
url=f"https://ftp.gnu.org/gnu/readline/readline-{version}.tar.gz",
sha256="3feb7171f16a84ee82ca18a36d7b9be109a52c04f492a053331d7d1095007c35",
)
host_deps = ["autoconf", "automake", "binutils", "gcc"]
deps = [profile["libc"], "ncurses"]
configure, build, _ = autotools(
configure_args=["--with-curses"],
# Force linking against system ncurses, not readline's internal termcap stub.
configure_env={"bash_cv_termcap_lib": "ncursesw"},
)
def install(self):
# readline overwrites DESTDIR on its own; pass explicitly.
autotools_install(self, [f"DESTDIR={self.dest_dir}"])
-29
View File
@@ -1,29 +0,0 @@
version = "8.2"
revision = 1
metadata = meta(
description = "Library for command-line editing",
license = "GPL-3.0-or-later",
website = "https://tiswww.case.edu/php/chet/readline/rltop.html",
)
source = tarball_source(
url = f"https://ftp.gnu.org/gnu/readline/readline-{version}.tar.gz",
sha256 = "3feb7171f16a84ee82ca18a36d7b9be109a52c04f492a053331d7d1095007c35",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc, "ncurses"]
configure, build, _ = autotools(
configure_args = ["--with-curses"],
configure_env = {
# Force linking against the system curses; otherwise readline's
# configure may pick a static libtermcap stub it ships internally.
"bash_cv_termcap_lib": "ncursesw",
},
)
# Readline overwrites DESTDIR
def install(ctx, pkg):
autotools_install(ctx, pkg, extra_args = [
"DESTDIR=" + pkg.dest_dir
])
+13
View File
@@ -0,0 +1,13 @@
version = "5.6.3"
revision = 1
description = "LZMA/XZ compression tools and library"
license = "0BSD AND GPL-2.0-or-later AND LGPL-2.1-or-later"
url = "https://tukaani.org/xz/"
source = tarball(
url=f"https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz",
sha256="db0590629b6f0fa36e74aea5f9731dc6f8df068ce7b7bafa45301832a5eebc3a",
)
host_deps = ["autoconf", "automake", "binutils", "gcc"]
deps = [profile["libc"]]
configure, build, install = autotools(configure_args=["--disable-doc"])
-18
View File
@@ -1,18 +0,0 @@
version = "5.6.3"
revision = 1
metadata = meta(
description = "XZ Utils — LZMA/XZ compression tools and library",
license = "0BSD AND GPL-2.0-or-later AND LGPL-2.1-or-later",
website = "https://tukaani.org/xz/",
)
source = tarball_source(
url = f"https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz",
sha256 = "db0590629b6f0fa36e74aea5f9731dc6f8df068ce7b7bafa45301832a5eebc3a",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
configure, build, install = autotools(configure_args = [
"--disable-doc",
])
+36
View File
@@ -0,0 +1,36 @@
version = "1.3.2"
revision = 1
description = "Lossless data-compression library"
license = "Zlib"
url = "https://zlib.net/"
source = tarball(
url=f"https://github.com/madler/zlib/releases/download/v{version}/zlib-{version}.tar.gz",
sha256="?",
)
host_deps = ["binutils", "gcc"]
deps = [profile["libc"]]
def configure(self):
# zlib ships its own ./configure; doesn't grok autoconf flags.
self.run(
self.source_dir / "configure",
f"--prefix={self.profile['prefix']}",
f"--libdir={self.profile['libdir']}",
f"--sharedlibdir={self.profile['libdir']}",
env={
"CC": f"{self.triple}-gcc",
"AR": f"{self.triple}-ar",
"RANLIB": f"{self.triple}-ranlib",
"CFLAGS": self.profile["cflags"],
"LDFLAGS": self.profile["ldflags"],
},
)
def build(self):
self.run("make", f"-j{self.jobs}")
def install(self):
self.run("make", "install", env={"DESTDIR": str(self.dest_dir)})
-36
View File
@@ -1,36 +0,0 @@
version = "1.3.2"
revision = 1
metadata = meta(
description = "Lossless data-compression library",
license = "Zlib",
website = "https://zlib.net/",
)
source = tarball_source(
url = f"https://zlib.net/zlib-{version}.tar.xz",
sha256 = "d7a0654783a4da529d1bb793b7ad9c3318020af77667bcae35f95d0e42a792f3",
strip_components = 1,
)
host_deps = ["binutils", "gcc"]
deps = [options.libc]
def configure(ctx):
# zlib ships its own ./configure that does not understand the usual
# autoconf flags (no --host, --build, etc.), so it is invoked directly.
ctx.run([
ctx.source_dir / "configure",
"--prefix=" + options.prefix,
"--libdir=" + options.libdir,
"--sharedlibdir=" + options.libdir,
], env = {
"CC": options.target_triple + "-gcc",
"AR": options.target_triple + "-ar",
"RANLIB": options.target_triple + "-ranlib",
"CFLAGS": options.cflags,
"LDFLAGS": options.ldflags,
})
def build(ctx):
ctx.run(["make", "-j" + str(ctx.jobs)])
def install(ctx, pkg):
ctx.run(["make", "install"], env = {"DESTDIR": pkg.dest_dir})
+3
View File
@@ -0,0 +1,3 @@
"""Orchid build system"""
__version__ = "0.1.0"
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/python3
import os
import sys
_HERE = os.path.dirname(os.path.realpath(__file__))
_ROOT = os.path.dirname(_HERE)
if _ROOT not in sys.path:
sys.path.insert(0, _ROOT)
from src.cli import main
raise SystemExit(main())
+135
View File
@@ -0,0 +1,135 @@
from src.container import Container
from src.recipe import Recipe
from src.source import Subpackage
def _info_args(
name: str,
version: str,
revision: int,
arch: str,
origin: str,
description: str,
license: str,
url: str,
maintainer: str,
depends: list[str],
) -> list[str]:
args = [
"--info",
f"name:{name}",
"--info",
f"version:{version}-r{revision}",
"--info",
f"arch:{arch}",
"--info",
f"origin:{origin}",
]
if description:
args += ["--info", f"description:{description}"]
if license:
args += ["--info", f"license:{license}"]
if url:
args += ["--info", f"url:{url}"]
if maintainer:
args += ["--info", f"packager:{maintainer}"]
for d in depends:
args += ["--info", f"depends:{d}"]
return args
def mkpkg_base(container: Container, recipe: Recipe, arch: str) -> None:
name = recipe.name
deps = list(recipe.deps) + list(recipe.run_deps)
args = [
"apk",
"mkpkg",
"--files",
f"/dest/{name}",
"--output",
f"/pkgs/{name}-{recipe.version}-r{recipe.revision}.apk",
*_info_args(
name,
recipe.version,
recipe.revision,
arch,
recipe.name,
recipe.description,
recipe.license,
recipe.url,
recipe.maintainer,
deps,
),
]
container.exec(args)
def mkpkg_subpackage(
container: Container, recipe: Recipe, sub: Subpackage, arch: str
) -> None:
args = [
"apk",
"mkpkg",
"--files",
f"/dest/{sub.name}",
"--output",
f"/pkgs/{sub.name}-{recipe.version}-r{recipe.revision}.apk",
*_info_args(
sub.name,
recipe.version,
recipe.revision,
arch,
recipe.name,
sub.description,
sub.license,
sub.url,
sub.maintainer,
[recipe.name],
),
]
container.exec(args)
def index(container: Container) -> None:
container.exec_shell(
"apk mkndx --allow-untrusted -o /pkgs/Packages.adb /pkgs/*.apk"
)
def sysroot_install(container: Container, pkgs: list[str], *, initdb: bool) -> None:
if not pkgs:
return
args = [
"apk",
"add",
"--root",
"/sysroot",
"--allow-untrusted",
"--no-network",
"--repository",
"/pkgs/Packages.adb",
]
if initdb:
args.append("--initdb")
args += pkgs
container.exec(args)
def sysroot_initialized(container: Container) -> bool:
rc = container.exec(["test", "-f", "/sysroot/lib/apk/db/installed"], check=False)
return rc == 0
def split_subpackage_script(base: str, dest: str, pattern: str) -> str:
# Move matching files from /dest/<base> to /dest/<dest>, preserving dirs.
# TODO: Make this cleaner.
return (
f"set -e; "
f"cd /dest/{base}; "
f"find . -depth -path './{pattern}' -print 2>/dev/null | while IFS= read -r f; do "
f' rel="${{f#./}}"; '
f' target=/dest/{dest}/"$rel"; '
f' mkdir -p "$(dirname "$target")"; '
f' mv "$f" "$target"; '
f"done"
)
+283
View File
@@ -0,0 +1,283 @@
import os
import re
import shutil
from pathlib import Path
from src import apk, fetch, log
from src.container import Container, Mount
from src.context import RecipeContext
from src.layout import Layout
from src.plan import (
PHASE_STAMPS,
Plan,
stamp_dir,
stamp_token,
stamp_valid,
transitive_host_deps,
)
from src.recipe import Recipe, RecipeSet
def _container_name(*parts: str) -> str:
name = "-".join(parts)
name = re.sub(
r"[^a-zA-Z0-9_.-]",
lambda m: f"_{ord(m.group(0)):02x}",
name,
)
if not name or not name[0].isalnum():
name = f"orchid-{name}"
return name
def _source_tree(layout: Layout, r: Recipe, key: str | None) -> Path:
base = layout.source_tree(r.name, r.version)
if len(r.sources) == 1:
return base
if key is None:
raise ValueError(f"{r.name}: multi-source entries must be named")
return base / key
def _prepare_all_sources(layout: Layout, r: Recipe) -> None:
for key, src in r.sources.items():
tree = _source_tree(layout, r, key)
tree.parent.mkdir(parents=True, exist_ok=True)
fetch.prepare_source(layout, r.dir, src, tree)
def _build_path_env(host_deps_order: list[str], layout: Layout) -> str:
parts: list[str] = []
for h in reversed(host_deps_order):
parts += [
f"/tools/{h}/sbin",
f"/tools/{h}/bin",
]
parts += [
"/usr/local/sbin",
"/usr/local/bin",
"/usr/sbin",
"/usr/bin",
"/sbin",
"/bin",
]
return ":".join(parts)
def _container_for(
rs: RecipeSet, layout: Layout, profile: dict, r: Recipe
) -> tuple[Container, list[str]]:
host_order = transitive_host_deps(rs, r)
name = _container_name("orchid", layout.build.name, r.kind, r.name)
mounts: list[Mount] = []
tmpfs: list[str] = ["/sysroot"]
# sources
if len(r.sources) == 1:
key = next(iter(r.sources.keys()))
mounts.append(Mount(_source_tree(layout, r, key), "/sources", readonly=True))
else:
for k in r.sources.keys():
mounts.append(
Mount(_source_tree(layout, r, k), f"/sources/{k}", readonly=True)
)
# build dir
wd = layout.build_workdir(r.name if r.kind == "target" else f"host-{r.name}")
wd.mkdir(parents=True, exist_ok=True)
mounts.append(Mount(wd, "/build", readonly=False))
# files dir
if r.files_dir is not None:
mounts.append(Mount(r.files_dir, "/files", readonly=True))
# tools
for h in host_order:
mounts.append(Mount(layout.host_pkg_dir(h), f"/tools/{h}", readonly=True))
# pkgs (produced packages persist on disk); sysroot lives in tmpfs
mounts.append(Mount(layout.pkgs_dir, "/pkgs", readonly=False))
env = {
"PATH": _build_path_env(host_order, layout),
"ORCHID_ARCH": profile["arch"],
"ORCHID_TRIPLE": profile["triple"],
"ORCHID_JOBS": str(os.cpu_count() or 1),
}
c = Container(
name=name,
image=profile["container_image"],
mounts=mounts,
tmpfs=tmpfs,
network=False,
extra_env=env,
)
return c, host_order
def _recipe_ref(r: Recipe) -> str:
return f"{r.key} ({r.version}-r{r.revision})"
def _mark_stamp(layout: Layout, r: Recipe, phase: str) -> None:
name = PHASE_STAMPS.get(phase)
if name is None:
return
d = stamp_dir(layout, r)
d.mkdir(parents=True, exist_ok=True)
(d / name).write_text(stamp_token(r))
def _clear_stamps(layout: Layout, r: Recipe) -> None:
d = stamp_dir(layout, r)
if d.is_dir():
shutil.rmtree(d)
def _wipe_workdir(layout: Layout, r: Recipe) -> None:
wd = layout.build_workdir(r.name if r.kind == "target" else f"host-{r.name}")
if wd.is_dir():
shutil.rmtree(wd)
def _run_phases(ctx: RecipeContext, r: Recipe, layout: Layout) -> None:
for phase in ("prepare", "configure", "build"):
fn = r.phases.get(phase)
if fn is None:
continue
if stamp_valid(layout, r, phase):
log.info_field(phase, f"{_recipe_ref(r)} (cached)")
continue
log.info_field(phase, _recipe_ref(r))
fn(ctx)
_mark_stamp(layout, r, phase)
def _do_install(ctx: RecipeContext, r: Recipe) -> None:
fn = r.phases.get("install")
if fn is None:
return
log.info_field("install", _recipe_ref(r))
ctx._dest_output = r.name
try:
fn(ctx)
finally:
ctx._dest_output = None
def _split_subpackages(c: Container, r: Recipe) -> None:
for sub in r.subpackages:
c.exec(["mkdir", "-p", f"/dest/{sub.name}"])
for pat in sub.files:
c.exec_shell(apk.split_subpackage_script(r.name, sub.name, pat))
def _package_target(c: Container, r: Recipe, arch: str) -> None:
apk.mkpkg_base(c, r, arch)
for sub in r.subpackages:
apk.mkpkg_subpackage(c, r, sub, arch)
apk.index(c)
def _finalize_host(c: Container, layout: Layout, r: Recipe) -> None:
import subprocess
out = layout.host_pkg_dir(r.name)
if out.exists():
shutil.rmtree(out)
out.mkdir(parents=True)
p1 = subprocess.Popen(
[
"podman",
"exec",
c.name,
"sh",
"-c",
f"cd /dest/{r.name}/tools/{r.name} && tar -cf - .",
],
stdout=subprocess.PIPE,
)
p2 = subprocess.Popen(["tar", "-xf", "-", "-C", str(out)], stdin=p1.stdout)
if p1.stdout is not None:
p1.stdout.close()
rc = p2.wait()
p1.wait()
if rc != 0 or p1.returncode != 0:
raise RuntimeError(f"host {r.name}: copy-out failed")
layout.host_pkg_marker(r.name, r.version, r.revision).write_text("ok\n")
def _sysroot_sync(c: Container, r: Recipe) -> None:
direct_deps = list(r.deps)
if not direct_deps:
return
initdb = not apk.sysroot_initialized(c)
apk.sysroot_install(c, direct_deps, initdb=initdb)
def build_one(
rs: RecipeSet, layout: Layout, profile: dict, r: Recipe, *, forced: bool = False
) -> None:
log.info_field("recipe", _recipe_ref(r))
if forced:
_clear_stamps(layout, r)
_wipe_workdir(layout, r)
_prepare_all_sources(layout, r)
c, _host_order = _container_for(rs, layout, profile, r)
c.start()
try:
# Ensure base output dest dir exists.
c.exec(["mkdir", "-p", f"/dest/{r.name}"])
_sysroot_sync(c, r)
ctx = RecipeContext(
recipe=r, profile=profile, container=c, jobs=os.cpu_count() or 1
)
_run_phases(ctx, r, layout)
_do_install(ctx, r)
if r.kind == "target":
log.info_field("package", _recipe_ref(r))
_split_subpackages(c, r)
_package_target(c, r, profile["arch"])
else:
log.info_field("finalize", _recipe_ref(r))
_finalize_host(c, layout, r)
finally:
c.stop()
log.ok_field("done", _recipe_ref(r))
def execute(plan: Plan, rs: RecipeSet, layout: Layout, profile: dict) -> None:
if not plan.order:
log.info_field("plan", "nothing to do")
return
for k in plan.order:
r = plan.recipes[k]
build_one(rs, layout, profile, r, forced=k in plan.forced)
def install_to(
layout: Layout,
profile: dict,
dest: Path,
pkgs: list[str],
*,
initdb: bool = True,
) -> None:
if not pkgs:
log.info_field("install", "nothing to install")
return
c = Container(
name=_container_name("orchid", layout.build.name, "install"),
image=profile["container_image"],
mounts=[
Mount(layout.pkgs_dir, "/pkgs", readonly=True),
Mount(dest, "/sysroot", readonly=False),
],
network=False,
)
c.start()
try:
log.info_field("install", f"{dest}: {', '.join(pkgs)}")
apk.sysroot_install(c, pkgs, initdb=initdb)
finally:
c.stop()
-1143
View File
File diff suppressed because it is too large Load Diff
+240
View File
@@ -0,0 +1,240 @@
import argparse
import os
import sys
from pathlib import Path
from src import (
builder,
container,
log,
plan,
profile as profile_mod,
recipe as recipe_mod,
)
from src.layout import Layout, find_repo_root
def _resolve_build_dir(p: str | None) -> Path:
if p:
return Path(p).resolve()
env = os.environ.get("ORCHID_BUILD")
if env:
return Path(env).resolve()
cwd = Path.cwd()
if (cwd / "profile").is_symlink() or (cwd / "profile").exists():
return cwd
raise SystemExit("error: -C <build-dir> required (or run inside a build dir)")
def _layout(build_dir: Path) -> Layout:
repo = find_repo_root(build_dir)
return Layout(repo=repo, build=build_dir)
def _load(layout: Layout):
prof = profile_mod.load_profile(layout)
rs = recipe_mod.load_recipes(layout, prof)
return prof, rs
def cmd_init(args) -> int:
target = Path(args.build_dir).resolve()
repo = find_repo_root(Path.cwd())
profile_mod.init_build_dir(target, repo, args.profile)
log.ok(f"initialized {target} (profile: {args.profile})")
return 0
def cmd_image(args) -> int:
layout = _layout(_resolve_build_dir(args.build_dir))
layout.ensure()
prof = profile_mod.load_profile(layout)
container.ensure_image(
layout.dockerfile, prof["container_image"], layout.image_hash_file
)
return 0
def _recipe_version(r) -> str:
return f"{r.version}-r{r.revision}"
def _print_plan(p: plan.Plan) -> None:
out = sys.stdout
if not p.order:
print(
f"{log.tag('info', stream=out)} "
f"{log.field('plan', 'nothing to do', stream=out)}"
)
return
count = len(p.order)
suffix = "" if count == 1 else "s"
print(
f"{log.tag('info', stream=out)} "
f"{log.field('plan', f'{count} recipe{suffix}', stream=out)}"
)
rows: list[tuple[str, str, str, str]] = []
for i, k in enumerate(p.order, start=1):
r = p.recipes[k]
rows.append((str(i), k, _recipe_version(r), ", ".join(p.stages.get(k, ()))))
widths = [
max(len(row[0]) for row in rows),
max(len("recipe"), *(len(row[1]) for row in rows)),
max(len("version"), *(len(row[2]) for row in rows)),
]
header = (
f" {'#':>{widths[0]}} "
f"{'recipe':<{widths[1]}} "
f"{'version':<{widths[2]}} "
"stages"
)
print(log.bold(header, stream=out))
for num, name, version, stages in rows:
print(
f" {num:>{widths[0]}} "
f"{name:<{widths[1]}} "
f"{version:<{widths[2]}} "
f"{stages}"
)
def cmd_plan(args) -> int:
layout = _layout(_resolve_build_dir(args.build_dir))
layout.ensure()
prof, rs = _load(layout)
p = plan.build_plan(rs, layout, args.recipes or None, rebuild=args.rebuild)
_print_plan(p)
return 0
def cmd_build(args) -> int:
layout = _layout(_resolve_build_dir(args.build_dir))
layout.ensure()
prof, rs = _load(layout)
container.ensure_image(
layout.dockerfile, prof["container_image"], layout.image_hash_file
)
p = plan.build_plan(rs, layout, args.recipes or None, rebuild=args.rebuild)
if args.dry_run:
return cmd_plan(args)
builder.execute(p, rs, layout, prof)
return 0
def cmd_install(args) -> int:
layout = _layout(_resolve_build_dir(args.build_dir))
layout.ensure()
prof, rs = _load(layout)
container.ensure_image(
layout.dockerfile, prof["container_image"], layout.image_hash_file
)
dest = Path(args.dest).resolve()
dest.mkdir(parents=True, exist_ok=True)
if args.recipes:
pkgs: list[str] = []
for k in args.recipes:
r = rs.get(k)
if r.kind != "target":
raise SystemExit(f"error: cannot install host recipe {k!r}")
pkgs.extend(r.outputs)
else:
pkgs = [r.name for r in rs.target.values() if r.enabled]
builder.install_to(layout, prof, dest, pkgs, initdb=args.initdb)
log.ok(f"installed {len(pkgs)} package(s) to {dest}")
return 0
def cmd_fetch(args) -> int:
layout = _layout(_resolve_build_dir(args.build_dir))
layout.ensure()
prof, rs = _load(layout)
from . import fetch as fetch_mod
targets = args.recipes or [r.key for r in rs.all() if r.enabled]
for k in targets:
r = rs.get(k)
for key, src in r.sources.items():
fetch_mod.fetch(layout, src)
return 0
def _common(p: argparse.ArgumentParser, with_recipes: bool = True) -> None:
p.add_argument(
"-C", "--build-dir", help="build directory (defaults to $ORCHID_BUILD or cwd)"
)
if with_recipes:
p.add_argument(
"recipes", nargs="*", help="recipe keys (target name or host:<name>)"
)
def make_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="orchid", description="Orchid distribution builder"
)
sub = parser.add_subparsers(dest="cmd", required=True)
p_init = sub.add_parser("init", help="create a build directory")
p_init.add_argument("build_dir")
p_init.add_argument("--profile", required=True)
p_init.set_defaults(func=cmd_init)
p_img = sub.add_parser("image", help="build/refresh the container image")
_common(p_img, with_recipes=False)
p_img.set_defaults(func=cmd_image)
p_plan = sub.add_parser("plan", help="print build plan")
_common(p_plan)
p_plan.add_argument("--rebuild", action="store_true")
p_plan.set_defaults(func=cmd_plan)
p_build = sub.add_parser("build", help="build recipes")
_common(p_build)
p_build.add_argument("--rebuild", action="store_true")
p_build.add_argument("-n", "--dry-run", action="store_true")
p_build.set_defaults(func=cmd_build)
p_inst = sub.add_parser(
"install", help="install built packages into a sysroot directory"
)
p_inst.add_argument(
"-C", "--build-dir", help="build directory (defaults to $ORCHID_BUILD or cwd)"
)
p_inst.add_argument("dest", help="destination sysroot directory")
p_inst.add_argument(
"recipes",
nargs="*",
help="target recipes to install (defaults to all enabled targets)",
)
p_inst.add_argument(
"--no-initdb",
dest="initdb",
action="store_false",
help="do not initialize the apk database (use for incremental installs)",
)
p_inst.set_defaults(func=cmd_install, initdb=True)
p_fetch = sub.add_parser("fetch", help="fetch sources only")
_common(p_fetch)
p_fetch.set_defaults(func=cmd_fetch)
return parser
def main(argv: list[str] | None = None) -> int:
parser = make_parser()
args = parser.parse_args(argv)
try:
return args.func(args)
except (SystemExit, KeyboardInterrupt):
raise
except Exception as e:
log.error(f"{type(e).__name__}: {e}")
if os.environ.get("ORCHID_DEBUG"):
raise
return 1
-77
View File
@@ -1,77 +0,0 @@
use clap::{Parser, Subcommand};
use std::path::PathBuf;
use crate::{builder::Builder, config::Config, eval, recipe::RecipeSet};
#[derive(Debug, Parser)]
struct Cli {
#[arg(
long,
short,
default_value = ".",
help = "Directory containing the configuration and recipe files"
)]
root: PathBuf,
#[command(subcommand)]
command: Command,
}
#[derive(Debug, Parser)]
#[command(about = "Fetch sources for the given recipes")]
struct FetchCommand {
#[arg(
required = true,
help = "List of recipes to fetch, host recipes should be prefixed with `host:`"
)]
recipes: Vec<String>,
#[arg(long, short = 'n', help = "Print what will be done and exit")]
dry_run: bool,
}
#[derive(Debug, Parser)]
#[command(about = "Build the given recipes")]
struct BuildCommand {
#[arg(
required = true,
help = "List of recipes to build, host recipes should be prefixed with `host:`"
)]
recipes: Vec<String>,
#[arg(long, short, help = "Perform a full rebuild of the given recipes")]
rebuild: bool,
#[arg(long, short = 'n', help = "Print what will be done and exit")]
dry_run: bool,
}
#[derive(Debug, Subcommand)]
enum Command {
Fetch(FetchCommand),
Build(BuildCommand),
#[command(about = "Create or refresh the configured build container image")]
Image,
}
pub fn run() -> anyhow::Result<()> {
let cli = Cli::parse();
let root_path = cli.root.canonicalize().unwrap_or(cli.root);
let config = Config::load(&root_path.join("config.star"))?;
match cli.command {
Command::Fetch(command) => {
let lib = eval::eval_lib(&root_path.join("lib"), Some(&config.options))?;
let recipes = RecipeSet::load(&root_path, &config.options, lib.as_ref())?;
let mut builder = Builder::new(root_path, config);
builder.fetch(&recipes, &command.recipes, command.dry_run)
}
Command::Build(command) => {
let lib = eval::eval_lib(&root_path.join("lib"), Some(&config.options))?;
let recipes = RecipeSet::load(&root_path, &config.options, lib.as_ref())?;
let mut builder = Builder::new(root_path, config);
builder.build(&recipes, &command.recipes, command.rebuild, command.dry_run)
}
Command::Image => {
let mut builder = Builder::new(root_path, config);
builder.ensure_container_ready()
}
}
}
-117
View File
@@ -1,117 +0,0 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
};
use starlark::values::dict::FrozenDictRef;
use crate::{
eval::{ExtractError, eval_file, extract_string},
options::Options,
};
#[derive(Debug)]
pub enum ContainerRuntime {
Docker,
Podman,
}
impl ContainerRuntime {
pub fn as_str(&self) -> &'static str {
match self {
Self::Docker => "docker",
Self::Podman => "podman",
}
}
}
impl std::fmt::Display for ContainerRuntime {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
impl TryFrom<&str> for ContainerRuntime {
type Error = anyhow::Error;
fn try_from(value: &str) -> anyhow::Result<Self> {
match value {
"docker" => Ok(Self::Docker),
"podman" => Ok(Self::Podman),
_ => anyhow::bail!("invalid runtime: {value}"),
}
}
}
#[derive(Debug)]
pub struct Config {
pub container_runtime: ContainerRuntime,
pub container_image: String,
pub container_dockerfile: PathBuf,
pub arch: String,
pub options: Options,
}
impl Config {
pub fn load(path: &Path) -> anyhow::Result<Self> {
let module = eval_file(path, None, None)?;
let container_runtime = match extract_string(&module, "container_runtime") {
Ok(v) => ContainerRuntime::try_from(v.as_str())?,
Err(ExtractError::NotFound) => ContainerRuntime::Podman,
Err(ExtractError::TypeMismatch) => anyhow::bail!("`container_runtime` is not a string"),
};
let container_image = match extract_string(&module, "container_image") {
Ok(container_image) => container_image,
Err(ExtractError::NotFound) => {
anyhow::bail!("`container_image` config variable not set")
}
Err(ExtractError::TypeMismatch) => anyhow::bail!("`container_image` is not a string"),
};
let container_dockerfile = match extract_string(&module, "container_dockerfile") {
Ok(container_dockerfile) => PathBuf::from(container_dockerfile),
Err(ExtractError::NotFound) => PathBuf::from("Dockerfile"),
Err(ExtractError::TypeMismatch) => {
anyhow::bail!("`container_dockerfile` is not a string")
}
};
let arch = match extract_string(&module, "arch") {
Ok(arch) => arch,
Err(ExtractError::NotFound) => anyhow::bail!("`arch` config variable not set"),
Err(ExtractError::TypeMismatch) => anyhow::bail!("`arch` is not a string"),
};
let frozen_module = module.freeze()?;
let options_value = frozen_module
.get_option("options")?
.ok_or_else(|| anyhow::anyhow!("`options` config variable not set"))?;
let entries = {
// SAFETY: the FrozenValue is only used to construct a FrozenDictRef whose
// lifetime is bounded by `options_value`, which keeps the frozen heap alive.
let dict =
FrozenDictRef::from_frozen_value(unsafe { options_value.unchecked_frozen_value() })
.ok_or_else(|| anyhow::anyhow!("`options` is not a dict"))?;
dict.iter()
.map(|(k, v)| {
let key = k
.to_value()
.unpack_str()
.ok_or_else(|| anyhow::anyhow!("non-string key in `options`"))?
.to_owned();
Ok((key, options_value.map(|_| v)))
})
.collect::<anyhow::Result<HashMap<_, _>>>()?
};
let options = Options::new(entries);
Ok(Self {
container_runtime,
container_image,
container_dockerfile,
arch,
options,
})
}
}
+200
View File
@@ -0,0 +1,200 @@
import atexit
import hashlib
import os
import signal
import subprocess
import threading
from dataclasses import dataclass, field
from pathlib import Path
from src import log
@dataclass
class Mount:
src: Path
dst: str
readonly: bool = False
def as_arg(self) -> str:
flag = ":ro" if self.readonly else ""
return f"{self.src}:{self.dst}{flag}"
_active: set["Container"] = set()
_lock = threading.Lock()
_handlers_installed = False
def _install_handlers() -> None:
global _handlers_installed
if _handlers_installed:
return
_handlers_installed = True
def cleanup(*_):
_shutdown_all()
atexit.register(_shutdown_all)
for s in (signal.SIGINT, signal.SIGTERM):
try:
signal.signal(s, lambda *_: (_shutdown_all(), os._exit(130)))
except (ValueError, OSError):
pass
def _shutdown_all() -> None:
with _lock:
cs = list(_active)
_active.clear()
for c in cs:
try:
subprocess.run(
["podman", "rm", "-f", c.name],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
except Exception:
pass
@dataclass
class Container:
name: str
image: str
mounts: list[Mount] = field(default_factory=list)
tmpfs: list[str] = field(default_factory=list)
network: bool = False
extra_env: dict[str, str] = field(default_factory=dict)
started: bool = False
def __hash__(self) -> int:
return id(self)
def __eq__(self, other) -> bool:
return self is other
def start(self) -> None:
if self.started:
return
_install_handlers()
# Ensure no stale container.
subprocess.run(
["podman", "rm", "-f", self.name],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
argv = [
"podman",
"run",
"-d",
"--rm",
"--name",
self.name,
]
for t in ("/tmp", "/dest", "/var/cache", *self.tmpfs):
argv += ["--tmpfs", t]
argv += ["--network=none"] if not self.network else []
for m in self.mounts:
m.src.mkdir(parents=True, exist_ok=True) if not m.src.exists() else None
argv += ["-v", m.as_arg()]
argv += [self.image, "sleep", "infinity"]
log.debug(f"container start: {' '.join(argv)}")
r = subprocess.run(argv, capture_output=True, text=True)
if r.returncode != 0:
raise RuntimeError(f"podman run failed: {r.stderr.strip()}")
with _lock:
_active.add(self)
self.started = True
def stop(self) -> None:
if not self.started:
return
subprocess.run(
["podman", "rm", "-f", self.name],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
with _lock:
_active.discard(self)
self.started = False
def exec(
self,
argv: list[str],
*,
env: dict[str, str] | None = None,
cwd: str | None = None,
check: bool = True,
user: str | None = None,
) -> int:
if not self.started:
raise RuntimeError("container not started")
cmd = ["podman", "exec"]
if cwd:
cmd += ["-w", cwd]
if user:
cmd += ["-u", user]
merged: dict[str, str] = {}
merged.update(self.extra_env)
if env:
merged.update(env)
for k, v in merged.items():
cmd += ["-e", f"{k}={v}"]
cmd += [self.name, *argv]
log.debug(f"exec: {' '.join(argv)}")
rc = subprocess.call(cmd)
if check and rc != 0:
raise RuntimeError(f"command failed (exit {rc}): {' '.join(argv)}")
return rc
def exec_shell(
self,
script: str,
*,
env: dict[str, str] | None = None,
cwd: str | None = None,
check: bool = True,
) -> int:
return self.exec(["sh", "-ec", script], env=env, cwd=cwd, check=check)
def cp_out(self, src_in: str, dst_host: Path) -> None:
dst_host.parent.mkdir(parents=True, exist_ok=True)
r = subprocess.run(
["podman", "cp", f"{self.name}:{src_in}", str(dst_host)],
capture_output=True,
text=True,
)
if r.returncode != 0:
raise RuntimeError(f"podman cp failed: {r.stderr.strip()}")
def hash_dockerfile(p: Path) -> str:
return hashlib.sha256(p.read_bytes()).hexdigest()
def image_exists(tag: str) -> bool:
r = subprocess.run(["podman", "image", "exists", tag])
return r.returncode == 0
def build_image(dockerfile: Path, tag: str) -> None:
log.info(f"building container image {tag}")
r = subprocess.run(
["podman", "build", "-t", tag, "-f", str(dockerfile), str(dockerfile.parent)]
)
if r.returncode != 0:
raise RuntimeError("podman build failed")
def ensure_image(dockerfile: Path, tag: str, hash_file: Path) -> None:
cur = hash_dockerfile(dockerfile) + "\n" + tag
if hash_file.exists() and hash_file.read_text() == cur and image_exists(tag):
log.debug(f"image {tag} up-to-date")
return
build_image(dockerfile, tag)
hash_file.parent.mkdir(parents=True, exist_ok=True)
hash_file.write_text(cur)
-292
View File
@@ -1,292 +0,0 @@
use std::{
fs::File,
io::{self, Read},
os::{fd::FromRawFd, unix::io::RawFd},
path::{Path, PathBuf},
process::{Command, Stdio},
ptr,
sync::{
Mutex,
atomic::{AtomicBool, AtomicI32, Ordering},
},
thread,
};
use anyhow::{Context, bail};
use crate::config::ContainerRuntime;
#[derive(Debug, Clone)]
pub struct Mount {
pub host: PathBuf,
pub container: String,
pub read_only: bool,
}
#[derive(Debug)]
pub struct Container {
runtime: &'static str,
id: String,
stopped: bool,
}
#[derive(Clone, Debug)]
struct RegisteredContainer {
runtime: &'static str,
id: String,
}
static ACTIVE_CONTAINER: Mutex<Option<RegisteredContainer>> = Mutex::new(None);
static SIGNAL_CLEANUP_INSTALLED: AtomicBool = AtomicBool::new(false);
static SIGNAL_WRITE_FD: AtomicI32 = AtomicI32::new(-1);
pub fn install_signals() -> anyhow::Result<()> {
if SIGNAL_CLEANUP_INSTALLED
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_err()
{
return Ok(());
}
let mut fds = [0; 2];
if unsafe { libc::pipe(fds.as_mut_ptr()) } == -1 {
SIGNAL_CLEANUP_INSTALLED.store(false, Ordering::SeqCst);
return Err(io::Error::last_os_error()).context("creating signal cleanup pipe");
}
set_close_on_exec(fds[0]);
set_close_on_exec(fds[1]);
SIGNAL_WRITE_FD.store(fds[1], Ordering::SeqCst);
thread::spawn(move || signal_cleanup_loop(fds[0]));
install_signal_handler(libc::SIGINT)?;
install_signal_handler(libc::SIGTERM)?;
Ok(())
}
impl Container {
pub fn start(
runtime: &ContainerRuntime,
image: &str,
name: &str,
mounts: &[Mount],
) -> anyhow::Result<Self> {
let runtime_str = runtime.as_str();
let mut cmd = Command::new(runtime_str);
cmd.arg("run")
.arg("-d")
.arg("--rm")
.arg("--name")
.arg(name)
.arg("--read-only")
.arg("--tmpfs")
.arg("/tmp")
.arg("--tmpfs")
.arg("/dest")
.arg("--tmpfs")
.arg("/sysroot")
.arg("--network=none");
if matches!(runtime, ContainerRuntime::Podman) {
cmd.arg("--userns=keep-id");
}
for mount in mounts {
let mut spec = format!("{}:{}", mount.host.display(), mount.container);
if mount.read_only {
spec.push_str(":ro");
}
cmd.arg("-v").arg(spec);
}
cmd.arg(image).arg("sleep").arg("infinity");
cmd.stdout(Stdio::piped()).stderr(Stdio::inherit());
let output = cmd
.output()
.with_context(|| format!("spawning `{runtime_str} run` for image `{image}`"))?;
if !output.status.success() {
bail!(
"`{runtime_str} run` failed with {} for image `{image}`",
output.status
);
}
let id = String::from_utf8(output.stdout)
.context("container id is not valid UTF-8")?
.trim()
.to_owned();
if id.is_empty() {
bail!("`{runtime_str} run` returned an empty container id");
}
register_container(runtime_str, &id);
Ok(Self {
runtime: runtime_str,
id,
stopped: false,
})
}
pub fn exec(&self, argv: &[String], env: &[(String, String)], cwd: &str) -> anyhow::Result<()> {
if argv.is_empty() {
bail!("ctx.run called with an empty argv");
}
let mut cmd = Command::new(self.runtime);
cmd.arg("exec").arg("-w").arg(cwd);
for (k, v) in env {
cmd.arg("-e").arg(format!("{k}={v}"));
}
cmd.arg(&self.id);
cmd.args(argv);
let status = cmd.status().with_context(|| {
format!("spawning `{} exec` in container {}", self.runtime, self.id)
})?;
if !status.success() {
bail!("command {argv:?} failed with {status}");
}
Ok(())
}
pub fn cp_out(&self, src_in_container: &str, host_dst: &Path) -> anyhow::Result<()> {
let spec = format!("{}:{}", self.id, src_in_container);
let status = Command::new(self.runtime)
.arg("cp")
.arg(spec)
.arg(host_dst)
.status()
.with_context(|| format!("spawning `{} cp`", self.runtime))?;
if !status.success() {
bail!(
"`{} cp` failed with {status} for {src_in_container} -> {}",
self.runtime,
host_dst.display()
);
}
Ok(())
}
}
impl Drop for Container {
fn drop(&mut self) {
if !self.stopped {
let _ = Command::new(self.runtime)
.arg("rm")
.arg("-f")
.arg(&self.id)
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
self.stopped = true;
}
unregister_container(&self.id);
}
}
fn register_container(runtime: &'static str, id: &str) {
if let Ok(mut active) = ACTIVE_CONTAINER.lock() {
*active = Some(RegisteredContainer {
runtime,
id: id.to_owned(),
});
}
}
fn unregister_container(id: &str) {
if let Ok(mut active) = ACTIVE_CONTAINER.lock()
&& active.as_ref().is_some_and(|container| container.id == id)
{
*active = None;
}
}
fn active_container() -> Option<RegisteredContainer> {
ACTIVE_CONTAINER
.lock()
.ok()
.and_then(|active| active.clone())
}
fn signal_cleanup_loop(read_fd: RawFd) {
let mut signals = unsafe { File::from_raw_fd(read_fd) };
let mut buffer = [0_u8; 16];
loop {
match signals.read(&mut buffer) {
Ok(0) => return,
Ok(len) => {
let signal = buffer[..len]
.iter()
.copied()
.find(|signal| *signal != 0)
.unwrap_or(libc::SIGINT as u8);
cleanup_after_signal(signal as i32);
}
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
Err(_) => return,
}
}
}
fn cleanup_after_signal(signal: i32) -> ! {
if let Some(container) = active_container() {
eprintln!(
"\nreceived signal {signal}; killing container {}",
container.id
);
kill_container_detached(container.runtime, &container.id);
}
std::process::exit(128 + signal);
}
fn kill_container_detached(runtime: &str, id: &str) {
let _ = Command::new(runtime)
.arg("kill")
.arg(id)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::inherit())
.spawn();
}
fn set_close_on_exec(fd: RawFd) {
unsafe {
let flags = libc::fcntl(fd, libc::F_GETFD);
if flags != -1 {
let _ = libc::fcntl(fd, libc::F_SETFD, flags | libc::FD_CLOEXEC);
}
}
}
fn install_signal_handler(signal: i32) -> anyhow::Result<()> {
let mut action = unsafe { std::mem::zeroed::<libc::sigaction>() };
action.sa_sigaction = handle_signal as *const () as usize;
action.sa_flags = 0;
unsafe {
libc::sigemptyset(&mut action.sa_mask);
if libc::sigaction(signal, &action, ptr::null_mut()) == -1 {
return Err(io::Error::last_os_error())
.with_context(|| format!("installing handler for signal {signal}"));
}
}
Ok(())
}
extern "C" fn handle_signal(signal: libc::c_int) {
let fd = SIGNAL_WRITE_FD.load(Ordering::Relaxed);
if fd < 0 {
return;
}
let signal = signal as u8;
unsafe {
libc::write(fd, ptr::addr_of!(signal).cast(), 1);
}
}
+104
View File
@@ -0,0 +1,104 @@
import os
from dataclasses import dataclass, field
from pathlib import PurePosixPath
from typing import Any
from src.container import Container
from src.recipe import Recipe
@dataclass
class RecipeContext:
"""The `self` value passed to recipe phase functions."""
recipe: Recipe
profile: dict
container: Container
jobs: int
_dest_output: str | None = None
env: dict[str, str] = field(default_factory=dict)
@property
def name(self) -> str:
return self.recipe.name
@property
def version(self) -> str:
return self.recipe.version
@property
def revision(self) -> int:
return self.recipe.revision
@property
def source_dir(self) -> PurePosixPath:
srcs = self.recipe.sources
if len(srcs) == 1:
return PurePosixPath("/sources")
raise RuntimeError(f"{self.name}: multiple sources; use self.sources")
@property
def sources(self) -> dict[str, PurePosixPath]:
srcs = self.recipe.sources
if len(srcs) == 1:
raise RuntimeError(f"{self.name}: only one source; use self.source_dir")
out: dict[str, PurePosixPath] = {}
for k in srcs.keys():
if k is None:
raise RuntimeError(f"{self.name}: multiple sources must be named")
out[k] = PurePosixPath("/sources") / k
return out
@property
def build_dir(self) -> PurePosixPath:
return PurePosixPath("/build")
@property
def sysroot(self) -> PurePosixPath:
return PurePosixPath("/sysroot")
@property
def dest_dir(self) -> PurePosixPath:
if self._dest_output is None:
raise RuntimeError("dest_dir only available during install/package phases")
return PurePosixPath("/dest") / self._dest_output
@property
def files(self) -> PurePosixPath:
if self.recipe.files_dir is None:
raise RuntimeError(f"{self.name}: no files/ dir")
return PurePosixPath("/files")
@property
def prefix(self) -> str:
return f"/tools/{self.name}" if self.recipe.kind == "host" else "/usr"
@property
def triple(self) -> str:
return self.profile["triple"]
@property
def arch(self) -> str:
return self.profile["arch"]
def run(
self,
*argv,
env: dict[str, str] | None = None,
cwd: str | os.PathLike | None = None,
) -> None:
flat: list[str] = []
for a in argv:
if isinstance(a, (PurePosixPath, os.PathLike)):
flat.append(str(a))
elif isinstance(a, bool):
raise TypeError("bool not allowed in run(); use str")
elif isinstance(a, (str, int)):
flat.append(str(a))
else:
raise TypeError(f"unsupported arg type in run(): {type(a).__name__}")
merged = dict(self.env)
if env:
merged.update(env)
cwd_s = str(cwd) if cwd is not None else "/build"
self.container.exec(flat, env=merged, cwd=cwd_s)
-193
View File
@@ -1,193 +0,0 @@
use anyhow::Context;
use starlark::{
environment::{FrozenModule, Globals, GlobalsBuilder, Module},
eval::Evaluator,
syntax::{AstModule, Dialect},
values::list::{ListRef, UnpackList},
};
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
use crate::{
options::Options,
phase::Path as StarPath,
recipe::{GitSource, Metadata, Source, Subpackage, TarballSource},
};
#[derive(Debug)]
pub enum ExtractError {
NotFound,
TypeMismatch,
}
impl std::fmt::Display for ExtractError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ExtractError::NotFound => write!(f, "missing"),
ExtractError::TypeMismatch => write!(f, "wrong type"),
}
}
}
impl std::error::Error for ExtractError {}
#[starlark::starlark_module]
fn builder_globals(builder: &mut GlobalsBuilder) {
fn meta(
maintainer: Option<String>,
description: Option<String>,
license: Option<String>,
website: Option<String>,
) -> anyhow::Result<Metadata> {
Ok(Metadata::new(maintainer, description, license, website))
}
fn tarball_source(
url: String,
sha256: String,
strip_components: Option<u32>,
patches: Option<UnpackList<String>>,
) -> anyhow::Result<Source> {
Ok(Source::Tarball(TarballSource::new(
url,
sha256,
strip_components.unwrap_or(0),
patches.map(|p| p.items).unwrap_or_default(),
)))
}
fn git_source(
url: String,
commit: String,
patches: Option<UnpackList<String>>,
) -> anyhow::Result<Source> {
Ok(Source::Git(GitSource::new(
url,
commit,
patches.map(|p| p.items).unwrap_or_default(),
)))
}
fn subpackage(
name: String,
metadata: &Metadata,
files: UnpackList<String>,
) -> anyhow::Result<Subpackage> {
Ok(Subpackage::new(name, files.items, metadata.clone()))
}
fn path(value: String) -> anyhow::Result<StarPath> {
Ok(StarPath::new(value))
}
}
pub fn eval_file(
path: &Path,
options: Option<&Options>,
lib: Option<&FrozenModule>,
) -> anyhow::Result<Module> {
let module = Module::new();
if let Some(lib) = lib {
module.import_public_symbols(lib);
}
if let Some(options) = options {
inject_options(&module, options);
}
let ast = AstModule::parse_file(path, &dialect()).map_err(|err| anyhow::anyhow!("{err}"))?;
let globals = globals();
let mut eval = Evaluator::new(&module);
eval.eval_module(ast, &globals)
.map_err(|err| anyhow::anyhow!("{err}"))?;
drop(eval);
Ok(module)
}
/// Parse and evaluate every `.star` file under `dir` into a single frozen
/// module whose public bindings can be imported into recipe modules. Returns
/// `Ok(None)` if `dir` doesn't exist or contains no `.star` files.
pub fn eval_lib(dir: &Path, options: Option<&Options>) -> anyhow::Result<Option<FrozenModule>> {
if !dir.exists() {
return Ok(None);
}
let mut files: Vec<PathBuf> = Vec::new();
for entry in WalkDir::new(dir) {
let entry = entry.with_context(|| format!("walking lib directory {}", dir.display()))?;
let path = entry.path();
if entry.file_type().is_file() && path.extension().is_some_and(|ext| ext == "star") {
files.push(path.to_path_buf());
}
}
if files.is_empty() {
return Ok(None);
}
// Sorted for deterministic ordering when later definitions shadow earlier ones.
files.sort();
let module = Module::new();
if let Some(options) = options {
inject_options(&module, options);
}
let dialect = dialect();
let globals = globals();
for file in &files {
let ast = AstModule::parse_file(file, &dialect)
.map_err(|err| anyhow::anyhow!("parsing {}: {err}", file.display()))?;
let mut eval = Evaluator::new(&module);
eval.eval_module(ast, &globals)
.map_err(|err| anyhow::anyhow!("evaluating {}: {err}", file.display()))?;
}
Ok(Some(module.freeze()?))
}
fn dialect() -> Dialect {
Dialect {
enable_top_level_stmt: true,
enable_f_strings: true,
..Dialect::Standard
}
}
fn globals() -> Globals {
GlobalsBuilder::standard().with(builder_globals).build()
}
fn inject_options(module: &Module, options: &Options) {
let value = module.heap().alloc(options.clone());
module.set("options", value);
}
pub fn extract_string(module: &Module, key: &str) -> Result<String, ExtractError> {
module
.get(key)
.ok_or_else(|| ExtractError::NotFound)
.and_then(|v| {
v.unpack_str()
.map(|v| v.to_string())
.ok_or_else(|| ExtractError::TypeMismatch)
})
}
pub fn extract_i32(module: &Module, key: &str) -> Result<i32, ExtractError> {
module
.get(key)
.ok_or(ExtractError::NotFound)
.and_then(|v| v.unpack_i32().ok_or(ExtractError::TypeMismatch))
}
pub fn extract_string_list(module: &Module, key: &str) -> Result<Vec<String>, ExtractError> {
let value = module.get(key).ok_or(ExtractError::NotFound)?;
let list = ListRef::from_value(value).ok_or(ExtractError::TypeMismatch)?;
list.iter()
.map(|v| {
v.unpack_str()
.map(|s| s.to_string())
.ok_or(ExtractError::TypeMismatch)
})
.collect()
}
+179
View File
@@ -0,0 +1,179 @@
import contextlib
import fcntl
import hashlib
import os
import shutil
import subprocess
import tarfile
import tempfile
import urllib.request
from pathlib import Path
from src import log
from src.layout import Layout
from src.source import Git, Tarball
@contextlib.contextmanager
def cache_lock(layout: Layout):
layout.cache_dir.mkdir(parents=True, exist_ok=True)
f = open(layout.cache_lock, "w")
try:
fcntl.flock(f, fcntl.LOCK_EX)
yield
finally:
fcntl.flock(f, fcntl.LOCK_UN)
f.close()
def fetch_tarball(layout: Layout, src: Tarball) -> Path:
dest = layout.tarball_cache / src.sha256
if dest.is_file():
return dest
if src.sha256 == "?":
log.warn(f"fetching {src.url} (sha256 unknown)")
else:
log.info(f"fetching {src.url}")
layout.tarball_cache.mkdir(parents=True, exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(dir=layout.tarball_cache, prefix=".tmp-")
tmp = Path(tmp_path)
h = hashlib.sha256()
try:
with os.fdopen(tmp_fd, "wb") as out, urllib.request.urlopen(src.url) as resp:
while True:
chunk = resp.read(1 << 20)
if not chunk:
break
out.write(chunk)
h.update(chunk)
got = h.hexdigest()
if src.sha256 == "?":
log.warn(f"computed sha256 = {got} (paste into recipe)")
final = layout.tarball_cache / got
os.replace(tmp, final)
return final
if got != src.sha256:
raise RuntimeError(
f"sha256 mismatch for {src.url}: expected {src.sha256}, got {got}"
)
os.replace(tmp, dest)
return dest
except BaseException:
with contextlib.suppress(FileNotFoundError):
tmp.unlink()
raise
def fetch_git(layout: Layout, src: Git) -> Path:
dest = layout.git_cache / src.commit
if dest.is_dir():
return dest
log.info(f"git clone {src.url} (commit {src.commit[:12]})")
layout.git_cache.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=layout.git_cache, prefix=".tmp-") as td:
bare = Path(td) / "bare"
subprocess.run(
["git", "clone", "--bare", "--quiet", src.url, str(bare)], check=True
)
r = subprocess.run(["git", "-C", str(bare), "cat-file", "-e", src.commit])
if r.returncode != 0:
raise RuntimeError(f"commit {src.commit} not found in {src.url}")
os.replace(bare, dest)
return dest
def fetch(layout: Layout, src) -> Path:
with cache_lock(layout):
if isinstance(src, Tarball):
return fetch_tarball(layout, src)
if isinstance(src, Git):
return fetch_git(layout, src)
raise TypeError(f"unknown source type {type(src).__name__}")
def _extracted_marker(tree: Path) -> Path:
return tree.with_name(tree.name + ".extracted")
def _patched_marker(tree: Path) -> Path:
return tree.with_name(tree.name + ".patched")
def _wipe(tree: Path) -> None:
if tree.exists():
shutil.rmtree(tree)
for m in (_extracted_marker(tree), _patched_marker(tree)):
with contextlib.suppress(FileNotFoundError):
m.unlink()
def extract_tarball(cache_path: Path, tree: Path, strip: int) -> None:
tree.mkdir(parents=True, exist_ok=True)
# Use tar binary to support all formats including zstd
cmd = ["tar", "-xf", str(cache_path), "-C", str(tree)]
if strip:
cmd += [f"--strip-components={strip}"]
subprocess.run(cmd, check=True)
def extract_git(cache_path: Path, tree: Path, commit: str) -> None:
subprocess.run(["git", "clone", "--quiet", str(cache_path), str(tree)], check=True)
subprocess.run(
[
"git",
"-C",
str(tree),
"-c",
"advice.detachedHead=false",
"checkout",
"--quiet",
commit,
],
check=True,
)
def extract(cache_path: Path, src, tree: Path) -> None:
_wipe(tree)
if isinstance(src, Tarball):
extract_tarball(cache_path, tree, src.strip_components)
elif isinstance(src, Git):
extract_git(cache_path, tree, src.commit)
else:
raise TypeError(f"unknown source type {type(src).__name__}")
_extracted_marker(tree).write_text("ok\n")
def apply_patches(tree: Path, recipe_dir: Path, patches: tuple[str, ...]) -> None:
if not patches:
_patched_marker(tree).write_text("\n")
return
pdir = recipe_dir / "patches"
for name in patches:
p = pdir / name
if not p.is_file():
raise FileNotFoundError(f"patch not found: {p}")
log.info(f" patch {name}")
with open(p, "rb") as fh:
r = subprocess.run(
["patch", "-p1", "--no-backup-if-mismatch", "--quiet"],
cwd=tree,
stdin=fh,
)
if r.returncode != 0:
raise RuntimeError(f"patch {name} failed in {tree}")
_patched_marker(tree).write_text("\n".join(patches) + "\n")
def prepare_source(layout: Layout, recipe_dir: Path, src, tree: Path) -> None:
"""Fetch + extract + patch into `tree`. Idempotent via marker files."""
cache_path = fetch(layout, src)
expected_patches = "\n".join(src.patches) + "\n" if src.patches else "\n"
if (
_patched_marker(tree).is_file()
and _patched_marker(tree).read_text() == expected_patches
):
return
log.info(f"extract {tree.name}")
extract(cache_path, src, tree)
apply_patches(tree, recipe_dir, src.patches)
-434
View File
@@ -1,434 +0,0 @@
use std::{
collections::{BTreeMap, BTreeSet},
fmt, fs,
path::Path,
};
use anyhow::bail;
use crate::{
layout::Layout,
recipe::{OutputPackage, Recipe, RecipeKind, RecipeSet},
};
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub enum TaskId {
FetchSources(String),
PrepareSources(String),
PrepareRecipe(String),
ConfigureRecipe(String),
BuildRecipe(String),
InstallPackageFiles(String),
ProduceApk(String),
InstallHostRecipe(String),
}
impl fmt::Display for TaskId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::FetchSources(recipe) => write!(f, "fetch sources {recipe}"),
Self::PrepareSources(recipe) => write!(f, "prepare sources {recipe}"),
Self::PrepareRecipe(recipe) => write!(f, "prepare recipe {recipe}"),
Self::ConfigureRecipe(recipe) => write!(f, "configure {recipe}"),
Self::BuildRecipe(recipe) => write!(f, "build {recipe}"),
Self::InstallPackageFiles(output) => write!(f, "install package files {output}"),
Self::ProduceApk(output) => write!(f, "produce apk {output}"),
Self::InstallHostRecipe(recipe) => write!(f, "install host recipe {recipe}"),
}
}
}
#[derive(Debug)]
pub struct TaskPlan {
dependencies: BTreeMap<TaskId, Vec<TaskId>>,
order: Vec<TaskId>,
}
impl TaskPlan {
pub fn order(&self) -> &[TaskId] {
&self.order
}
pub fn is_empty(&self) -> bool {
self.order.is_empty()
}
pub fn dependency_count(&self) -> usize {
self.dependencies.values().map(Vec::len).sum()
}
}
pub struct TaskPlanner<'a> {
layout: Layout<'a>,
recipes: &'a RecipeSet,
forced_recipes: BTreeSet<String>,
dependencies: BTreeMap<TaskId, Vec<TaskId>>,
inactive: BTreeSet<TaskId>,
visiting: BTreeSet<TaskId>,
visited: BTreeSet<TaskId>,
}
impl<'a> TaskPlanner<'a> {
pub fn new(root: &'a Path, arch: &'a str, recipes: &'a RecipeSet) -> Self {
Self {
layout: Layout::new(root, arch),
recipes,
forced_recipes: BTreeSet::new(),
dependencies: BTreeMap::new(),
inactive: BTreeSet::new(),
visiting: BTreeSet::new(),
visited: BTreeSet::new(),
}
}
pub fn build_plan(mut self, requests: &[String], force: bool) -> anyhow::Result<TaskPlan> {
for request in requests {
let recipe = self.recipes.recipe(request)?;
if force {
self.forced_recipes.insert(recipe.key());
}
match recipe.kind() {
RecipeKind::Package => {
for output in recipe.outputs() {
self.visit(TaskId::ProduceApk(output.key()))?;
}
}
RecipeKind::HostPackage => {
self.visit(TaskId::InstallHostRecipe(recipe.key()))?;
}
}
}
self.into_plan()
}
pub fn fetch_plan(mut self, requests: &[String]) -> anyhow::Result<TaskPlan> {
for request in requests {
let recipe = self.recipes.recipe(request)?;
self.visit(TaskId::FetchSources(recipe.key()))?;
}
self.into_plan()
}
fn visit(&mut self, task: TaskId) -> anyhow::Result<()> {
if self.visited.contains(&task) || self.inactive.contains(&task) {
return Ok(());
}
if !self.is_active(&task)? {
self.inactive.insert(task);
return Ok(());
}
if !self.visiting.insert(task.clone()) {
bail!("task dependency cycle involving `{task}`");
}
let dependencies = self.dependencies(&task)?;
let mut active_dependencies = Vec::new();
for dependency in dependencies {
self.visit(dependency.clone())?;
if self.dependencies.contains_key(&dependency) {
active_dependencies.push(dependency);
}
}
self.visiting.remove(&task);
self.visited.insert(task.clone());
self.dependencies.insert(task, active_dependencies);
Ok(())
}
fn into_plan(self) -> anyhow::Result<TaskPlan> {
let order = recipe_contiguous_order(&self.dependencies, self.recipes)?;
Ok(TaskPlan {
dependencies: self.dependencies,
order,
})
}
fn dependencies(&self, task: &TaskId) -> anyhow::Result<Vec<TaskId>> {
match task {
TaskId::FetchSources(_) => Ok(Vec::new()),
TaskId::PrepareSources(recipe) => Ok(vec![TaskId::FetchSources(recipe.clone())]),
TaskId::PrepareRecipe(recipe) => Ok(vec![TaskId::PrepareSources(recipe.clone())]),
TaskId::ConfigureRecipe(recipe) => {
let recipe = self.recipes.recipe(recipe)?;
let mut deps = vec![
TaskId::PrepareSources(recipe.key()),
TaskId::PrepareRecipe(recipe.key()),
];
deps.extend(
recipe
.host_deps()
.iter()
.map(|dep| TaskId::InstallHostRecipe(RecipeKind::HostPackage.key(dep))),
);
deps.extend(
recipe
.build_deps()
.iter()
.chain(recipe.deps().iter())
.map(|dep| TaskId::ProduceApk(dep.clone())),
);
Ok(deps)
}
TaskId::BuildRecipe(recipe) => Ok(vec![TaskId::ConfigureRecipe(recipe.clone())]),
TaskId::InstallPackageFiles(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
if output.is_base() {
Ok(vec![TaskId::BuildRecipe(output.recipe().to_owned())])
} else {
let base = recipe.base_output().ok_or_else(|| {
anyhow::anyhow!("recipe `{}` has no base output", recipe.key())
})?;
Ok(vec![TaskId::InstallPackageFiles(base.key())])
}
}
TaskId::ProduceApk(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
let mut deps = vec![TaskId::InstallPackageFiles(output.key())];
if output.is_base() {
deps.extend(
recipe
.outputs()
.iter()
.filter(|candidate| !candidate.is_base())
.map(|subpackage| TaskId::InstallPackageFiles(subpackage.key())),
);
}
deps.extend(
recipe
.deps()
.iter()
.chain(recipe.run_deps().iter())
.map(|dep| TaskId::ProduceApk(dep.clone())),
);
Ok(deps)
}
TaskId::InstallHostRecipe(recipe) => {
self.recipes.recipe(recipe)?;
Ok(vec![TaskId::BuildRecipe(recipe.clone())])
}
}
}
fn is_active(&self, task: &TaskId) -> anyhow::Result<bool> {
match task {
TaskId::FetchSources(recipe) => self.fetch_sources_active(self.recipes.recipe(recipe)?),
TaskId::PrepareSources(recipe) => {
self.prepare_sources_active(self.recipes.recipe(recipe)?)
}
TaskId::PrepareRecipe(recipe) => {
self.prepare_recipe_active(self.recipes.recipe(recipe)?)
}
TaskId::ConfigureRecipe(recipe) => {
self.recipe_task_active(self.recipes.recipe(recipe)?, "configure")
}
TaskId::BuildRecipe(recipe) => {
self.recipe_task_active(self.recipes.recipe(recipe)?, "build")
}
TaskId::InstallPackageFiles(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
if output.is_base() {
return recipe.outputs().iter().try_fold(false, |active, output| {
Ok(active
|| self.output_task_active(recipe, output, "install")?
|| self.produce_apk_active(recipe, output)?)
});
}
let base = recipe.base_output().ok_or_else(|| {
anyhow::anyhow!("recipe `{}` has no base output", recipe.key())
})?;
Ok(self.output_task_active(recipe, output, "install")?
|| self.produce_apk_active(recipe, output)?
|| self.produce_apk_active(recipe, base)?)
}
TaskId::ProduceApk(output) => {
let output = self.recipes.output(output)?;
let recipe = self.recipes.recipe(output.recipe())?;
self.produce_apk_active(recipe, output)
}
TaskId::InstallHostRecipe(recipe) => {
let recipe = self.recipes.recipe(recipe)?;
self.install_host_recipe_active(recipe)
}
}
}
fn fetch_sources_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
Ok(recipe.sources().entries().iter().any(|(_, source)| {
source.is_unknown_cache_key()
|| !self.layout.source_cache_path(source.cache_key()).exists()
}))
}
fn is_recipe_forced(&self, recipe: &Recipe) -> bool {
self.forced_recipes.contains(&recipe.key())
}
fn prepare_sources_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
let want_version = format!("{}-r{}", recipe.version(), recipe.revision());
if fs::read_to_string(self.layout.source_stamp(recipe, "version"))
.ok()
.as_deref()
!= Some(want_version.as_str())
{
return Ok(true);
}
if self.layout.recipe_has_patches(recipe)?
&& !self.layout.source_stamp(recipe, "patched").exists()
{
return Ok(true);
}
Ok(false)
}
fn prepare_recipe_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
if recipe.phases().prepare().is_none() {
return Ok(false);
}
if self.prepare_sources_active(recipe)? {
return Ok(true);
}
self.recipe_task_active(recipe, "prepare")
}
fn recipe_task_active(&self, recipe: &Recipe, kind: &str) -> anyhow::Result<bool> {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
Ok(
fs::read_to_string(self.layout.recipe_task_stamp(recipe, kind))
.ok()
.as_deref()
!= Some(self.layout.recipe_fingerprint(recipe)?.as_str()),
)
}
fn output_task_active(
&self,
recipe: &Recipe,
output: &OutputPackage,
kind: &str,
) -> anyhow::Result<bool> {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
Ok(
fs::read_to_string(self.layout.output_task_stamp(output, kind))
.ok()
.as_deref()
!= Some(self.layout.output_fingerprint(recipe, output)?.as_str()),
)
}
fn produce_apk_active(&self, recipe: &Recipe, output: &OutputPackage) -> anyhow::Result<bool> {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
if !self.layout.apk_path(recipe, output).exists() {
return Ok(true);
}
Ok(
fs::read_to_string(self.layout.output_task_stamp(output, "apk"))
.ok()
.as_deref()
!= Some(self.layout.output_fingerprint(recipe, output)?.as_str()),
)
}
fn install_host_recipe_active(&self, recipe: &Recipe) -> anyhow::Result<bool> {
if self.is_recipe_forced(recipe) {
return Ok(true);
}
if !self.layout.host_install_dir(recipe).exists() {
return Ok(true);
}
Ok(
fs::read_to_string(self.layout.recipe_task_stamp(recipe, "host-install"))
.ok()
.as_deref()
!= Some(self.layout.recipe_fingerprint(recipe)?.as_str()),
)
}
}
fn recipe_contiguous_order(
dependencies: &BTreeMap<TaskId, Vec<TaskId>>,
recipes: &RecipeSet,
) -> anyhow::Result<Vec<TaskId>> {
let total = dependencies.len();
let mut remaining: BTreeMap<TaskId, BTreeSet<TaskId>> = dependencies
.iter()
.map(|(task, deps)| (task.clone(), deps.iter().cloned().collect()))
.collect();
let mut dependents: BTreeMap<TaskId, Vec<TaskId>> = BTreeMap::new();
for (task, deps) in dependencies {
for dep in deps {
dependents
.entry(dep.clone())
.or_default()
.push(task.clone());
}
}
let mut order = Vec::with_capacity(total);
let mut current_recipe: Option<String> = None;
while order.len() < total {
let next = pick_next(&remaining, current_recipe.as_deref(), recipes)?;
let slug = task_recipe_slug(&next, recipes)?;
current_recipe = Some(slug);
remaining.remove(&next);
if let Some(children) = dependents.get(&next) {
for child in children {
if let Some(deps) = remaining.get_mut(child) {
deps.remove(&next);
}
}
}
order.push(next);
}
Ok(order)
}
fn pick_next(
remaining: &BTreeMap<TaskId, BTreeSet<TaskId>>,
current_recipe: Option<&str>,
recipes: &RecipeSet,
) -> anyhow::Result<TaskId> {
let mut fallback: Option<TaskId> = None;
for (task, deps) in remaining {
if !deps.is_empty() {
continue;
}
if let Some(active) = current_recipe {
let slug = task_recipe_slug(task, recipes)?;
if slug == active {
return Ok(task.clone());
}
}
if fallback.is_none() {
fallback = Some(task.clone());
}
}
fallback.ok_or_else(|| anyhow::anyhow!("task dependency cycle detected"))
}
pub fn task_recipe_slug(task: &TaskId, recipes: &RecipeSet) -> anyhow::Result<String> {
Ok(match task {
TaskId::FetchSources(recipe)
| TaskId::PrepareSources(recipe)
| TaskId::PrepareRecipe(recipe)
| TaskId::ConfigureRecipe(recipe)
| TaskId::BuildRecipe(recipe)
| TaskId::InstallHostRecipe(recipe) => recipe.clone(),
TaskId::InstallPackageFiles(output) | TaskId::ProduceApk(output) => {
recipes.output(output)?.recipe().to_owned()
}
})
}
+107
View File
@@ -0,0 +1,107 @@
from dataclasses import dataclass
from pathlib import Path
@dataclass(frozen=True)
class Layout:
"""All filesystem paths used by the builder."""
repo: Path
build: Path
@property
def recipes_dir(self) -> Path:
return self.repo / "recipes"
@property
def host_recipes_dir(self) -> Path:
return self.repo / "host-recipes"
@property
def profiles_dir(self) -> Path:
return self.repo / "profiles"
@property
def dockerfile(self) -> Path:
return self.repo / "Dockerfile"
@property
def cache_dir(self) -> Path:
return self.repo / "cache"
@property
def tarball_cache(self) -> Path:
return self.cache_dir / "tarballs"
@property
def git_cache(self) -> Path:
return self.cache_dir / "git"
@property
def cache_lock(self) -> Path:
return self.cache_dir / ".lock"
@property
def sources_dir(self) -> Path:
return self.repo / "sources"
def source_tree(self, name: str, version: str) -> Path:
return self.sources_dir / f"{name}-{version}"
@property
def profile_link(self) -> Path:
return self.build / "profile"
@property
def build_workdirs(self) -> Path:
return self.build / "builds"
def build_workdir(self, name: str) -> Path:
return self.build_workdirs / name
@property
def host_pkgs_dir(self) -> Path:
return self.build / "host-pkgs"
def host_pkg_dir(self, name: str) -> Path:
return self.host_pkgs_dir / name
def host_pkg_marker(self, name: str, version: str, revision: int) -> Path:
return self.host_pkg_dir(name) / f".built-{version}-r{revision}"
@property
def pkgs_dir(self) -> Path:
return self.build / "pkgs"
def apk_path(self, output: str, version: str, revision: int) -> Path:
return self.pkgs_dir / f"{output}-{version}-r{revision}.apk"
@property
def apkindex(self) -> Path:
return self.pkgs_dir / "Packages.adb"
@property
def image_hash_file(self) -> Path:
return self.build / ".image-hash"
def ensure(self) -> None:
for p in (
self.cache_dir,
self.tarball_cache,
self.git_cache,
self.sources_dir,
self.build_workdirs,
self.host_pkgs_dir,
self.pkgs_dir,
):
p.mkdir(parents=True, exist_ok=True)
def find_repo_root(start: Path) -> Path:
cur = start.resolve()
for p in (cur, *cur.parents):
if (p / "Dockerfile").is_file() and (p / "profiles").is_dir():
return p
raise FileNotFoundError(
f"Could not find Orchid repo root (looking for Dockerfile + profiles/) from {start}"
)
-123
View File
@@ -1,123 +0,0 @@
use std::{
fs,
path::{Path, PathBuf},
};
use anyhow::Context;
use sha2::{Digest, Sha256};
use crate::recipe::{OutputPackage, Recipe};
pub struct Layout<'a> {
pub root: &'a Path,
pub arch: &'a str,
}
impl<'a> Layout<'a> {
pub fn new(root: &'a Path, arch: &'a str) -> Self {
Self { root, arch }
}
pub fn source_cache_dir(&self) -> PathBuf {
self.root.join("build/cache/sources")
}
pub fn source_cache_path(&self, key: &str) -> PathBuf {
self.source_cache_dir().join(key)
}
pub fn source_workdir(&self, recipe: &Recipe) -> PathBuf {
self.root.join("build/sources").join(recipe.slug())
}
pub fn build_workdir(&self, recipe: &Recipe) -> PathBuf {
self.root.join("build/builds").join(recipe.slug())
}
pub fn host_install_dir(&self, recipe: &Recipe) -> PathBuf {
self.root.join("build/host-pkgs").join(recipe.slug())
}
pub fn host_install_root(&self, recipe: &Recipe) -> PathBuf {
self.root.join("build/host-pkgs").join(recipe.slug())
}
pub fn apk_path(&self, recipe: &Recipe, output: &OutputPackage) -> PathBuf {
self.root.join("build/pkgs").join(self.arch).join(format!(
"{}-{}-r{}.apk",
output.name(),
recipe.version(),
recipe.revision()
))
}
pub fn source_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.root
.join("build/sources")
.join(format!("{}.{kind}", recipe.slug()))
}
pub fn recipe_task_stamp(&self, recipe: &Recipe, kind: &str) -> PathBuf {
self.root
.join("build/tasks")
.join(format!("{}.{kind}", recipe.slug()))
}
pub fn output_task_stamp(&self, output: &OutputPackage, kind: &str) -> PathBuf {
self.root
.join("build/tasks")
.join(format!("{}.{kind}", output.key().replace(':', "-")))
}
pub fn recipe_fingerprint(&self, recipe: &Recipe) -> anyhow::Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.arch.as_bytes());
hasher.update(recipe.key().as_bytes());
hasher.update(recipe.version().as_bytes());
hasher.update(recipe.revision().to_le_bytes());
hasher.update(
fs::read(recipe.path())
.with_context(|| format!("reading recipe {}", recipe.path().display()))?,
);
for (name, source) in recipe.sources().entries() {
hasher.update(name.unwrap_or("").as_bytes());
hasher.update(source.url().as_bytes());
hasher.update(source.cache_key().as_bytes());
}
for patch in self.recipe_patches(recipe)? {
hasher.update(patch.display().to_string().as_bytes());
hasher
.update(fs::read(&patch).with_context(|| format!("reading {}", patch.display()))?);
}
Ok(hex::encode(hasher.finalize()))
}
pub fn output_fingerprint(
&self,
recipe: &Recipe,
output: &OutputPackage,
) -> anyhow::Result<String> {
let mut hasher = Sha256::new();
hasher.update(self.recipe_fingerprint(recipe)?.as_bytes());
hasher.update(output.key().as_bytes());
Ok(hex::encode(hasher.finalize()))
}
pub fn recipe_has_patches(&self, recipe: &Recipe) -> anyhow::Result<bool> {
Ok(!self.recipe_patches(recipe)?.is_empty())
}
pub fn recipe_patches(&self, recipe: &Recipe) -> anyhow::Result<Vec<PathBuf>> {
let Some(data_dir) = recipe.data_dir() else {
return Ok(Vec::new());
};
let patches_dir = data_dir.join("patches");
let mut out = Vec::new();
for (_, source) in recipe.sources().entries() {
for name in source.patches() {
out.push(patches_dir.join(name));
}
}
Ok(out)
}
}
+45
View File
@@ -0,0 +1,45 @@
def autotools_configure(self, extra_args=(), extra_env=None):
p = self.profile
env = {
"CFLAGS": p.get("cflags", ""),
"CXXFLAGS": p.get("cxxflags", ""),
"LDFLAGS": p.get("ldflags", ""),
}
if extra_env:
env.update(extra_env)
args = [
self.source_dir / "configure",
f"--host={p['triple']}",
f"--with-sysroot={self.sysroot}",
f"--prefix={p.get('prefix', '/usr')}",
f"--sysconfdir={p.get('sysconfdir', '/etc')}",
f"--localstatedir={p.get('localstatedir', '/var')}",
f"--bindir={p.get('bindir', '/usr/bin')}",
f"--sbindir={p.get('sbindir', '/usr/bin')}",
f"--libdir={p.get('libdir', '/usr/lib')}",
"--disable-static",
"--enable-shared",
*extra_args,
]
self.run(*args, env=env)
def autotools_build(self, extra_args=()):
self.run("make", f"-j{self.jobs}", *extra_args)
def autotools_install(self, extra_args=()):
self.run("make", "install", *extra_args, env={"DESTDIR": str(self.dest_dir)})
def autotools(*, configure_args=(), configure_env=None, build_args=(), install_args=()):
def _configure(self):
autotools_configure(self, configure_args, configure_env)
def _build(self):
autotools_build(self, build_args)
def _install(self):
autotools_install(self, (*install_args, f"DESTDIR={self.dest_dir}"))
return _configure, _build, _install
+73
View File
@@ -0,0 +1,73 @@
def cmake_configure(self, extra_args=(), extra_env=None, *, host=False):
p = self.profile
if host:
env = {
"CFLAGS": p.get("host_cflags", ""),
"CXXFLAGS": p.get("host_cxxflags", ""),
"LDFLAGS": p.get("host_ldflags", ""),
}
toolchain = []
else:
env = {
"CFLAGS": p.get("cflags", ""),
"CXXFLAGS": p.get("cxxflags", ""),
"LDFLAGS": p.get("ldflags", ""),
}
toolchain = [
"-DCMAKE_SYSTEM_NAME=Linux",
f"-DCMAKE_SYSTEM_PROCESSOR={p['arch']}",
f"-DCMAKE_SYSROOT={self.sysroot}",
f"-DCMAKE_C_COMPILER={p['triple']}-gcc",
f"-DCMAKE_CXX_COMPILER={p['triple']}-g++",
"-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
"-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
"-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
"-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY",
]
if extra_env:
env.update(extra_env)
self.run(
"cmake",
"-S",
self.source_dir,
"-B",
self.build_dir,
"-G",
"Ninja",
"-DCMAKE_BUILD_TYPE=Release",
f"-DCMAKE_INSTALL_PREFIX={p.get('prefix', '/usr')}",
f"-DCMAKE_INSTALL_SYSCONFDIR={p.get('sysconfdir', '/etc')}",
f"-DCMAKE_INSTALL_LOCALSTATEDIR={p.get('localstatedir', '/var')}",
*toolchain,
*extra_args,
env=env,
)
def cmake_build(self, extra_args=()):
self.run("cmake", "--build", self.build_dir, "-j", self.jobs, *extra_args)
def cmake_install(self, extra_args=()):
self.run(
"cmake",
"--install",
self.build_dir,
*extra_args,
env={"DESTDIR": str(self.dest_dir)},
)
def cmake(
*, configure_args=(), configure_env=None, build_args=(), install_args=(), host=False
):
def _configure(self):
cmake_configure(self, configure_args, configure_env, host=host)
def _build(self):
cmake_build(self, build_args)
def _install(self):
cmake_install(self, install_args)
return _configure, _build, _install
+9
View File
@@ -0,0 +1,9 @@
def make_build(self, extra_args=(), *, env=None):
self.run("make", f"-j{self.jobs}", *extra_args, env=env)
def make_install(self, extra_args=(), *, env=None):
merged = {"DESTDIR": str(self.dest_dir)}
if env:
merged.update(env)
self.run("make", "install", *extra_args, env=merged)
+85
View File
@@ -0,0 +1,85 @@
def meson_cross_file(self):
cross = self.build_dir / "meson-cross.ini"
self.write_text(
cross,
f"""\
[binaries]
c = '{self.triple}-gcc'
cpp = '{self.triple}-g++'
ar = '{self.triple}-gcc-ar'
nm = '{self.triple}-nm'
objcopy = '{self.triple}-objcopy'
ranlib = '{self.triple}-ranlib'
strip = '{self.triple}-strip'
pkg-config = '{self.triple}-pkg-config'
[host_machine]
system = 'linux'
cpu_family = '{self.arch}'
cpu = '{self.arch}'
endian = 'little'
""",
)
return cross
def meson_configure(
self, extra_args=(), extra_env=None, *, source_dir=None, flags=True, host=False
):
p = self.options
env = {}
if flags:
env.update(
{
"CFLAGS": p.get("cflags", ""),
"CXXFLAGS": p.get("cxxflags", ""),
"LDFLAGS": p.get("ldflags", ""),
}
)
if extra_env:
env.update(extra_env)
cross_args = [] if host else ["--cross-file", meson_cross_file(self)]
self.run(
"meson",
"setup",
source_dir or self.source_dir,
*cross_args,
f"--prefix={p.get('prefix', '/usr')}",
f"--sysconfdir={p.get('sysconfdir', '/etc')}",
f"--localstatedir={p.get('localstatedir', '/var')}",
"--libdir=lib",
"--sbindir=bin",
"--bindir=bin",
"--datadir=share",
"--buildtype=release",
"-Ddefault_library=shared",
*extra_args,
env=env,
)
def meson_build(self, extra_args=()):
self.run("meson", "compile", f"-j{self.jobs}", *extra_args)
def meson_install(self, extra_args=()):
self.run(
"meson",
"install",
"--no-rebuild",
*extra_args,
env={"DESTDIR": str(self.dest_dir)},
)
def meson(*, configure_args=(), configure_env=None, build_args=(), install_args=()):
def _configure(self):
meson_configure(self, configure_args, configure_env)
def _build(self):
meson_build(self, build_args)
def _install(self):
meson_install(self, install_args)
return _configure, _build, _install
+70
View File
@@ -0,0 +1,70 @@
import os
import sys
from typing import TextIO
_COLORS = {
"debug": "\033[2m",
"info": "\033[34m",
"warn": "\033[33m",
"error": "\033[31m",
"ok": "\033[32m",
}
_RESET = "\033[0m"
_BOLD = "\033[1m"
_TAGS = {"debug": "..", "info": "->", "warn": "!!", "error": "xx", "ok": "**"}
def _enabled(stream: TextIO | None = None) -> bool:
stream = stream or sys.stderr
return stream.isatty() and os.environ.get("NO_COLOR") is None
def tag(level: str, *, stream: TextIO | None = None) -> str:
tag_text = _TAGS[level]
if _enabled(stream):
return f"{_COLORS[level]}{tag_text}{_RESET}"
return tag_text
def bold(text: str, *, stream: TextIO | None = None) -> str:
if _enabled(stream):
return f"{_BOLD}{text}{_RESET}"
return text
def _emit(level: str, msg: str) -> None:
sys.stderr.write(f"{tag(level)} {msg}\n")
sys.stderr.flush()
def field(label: str, value: str, *, stream: TextIO | None = None) -> str:
return f"{bold(label, stream=stream)}: {value}"
def debug(msg: str) -> None:
if os.environ.get("ORCHID_DEBUG"):
_emit("debug", msg)
def info(msg: str) -> None:
_emit("info", msg)
def info_field(label: str, value: str) -> None:
_emit("info", field(label, value))
def warn(msg: str) -> None:
_emit("warn", msg)
def error(msg: str) -> None:
_emit("error", msg)
def ok(msg: str) -> None:
_emit("ok", msg)
def ok_field(label: str, value: str) -> None:
_emit("ok", field(label, value))
-26
View File
@@ -1,26 +0,0 @@
use std::{io::IsTerminal, sync::LazyLock};
static IS_STDERR_TERMINAL: LazyLock<bool> = LazyLock::new(|| std::io::stderr().is_terminal());
const ARROW: &str = "==>";
fn emit(color: &str, action: &str, details: &str) {
if *IS_STDERR_TERMINAL {
eprintln!("\x1b[{color}m{ARROW} \x1b[1m{action} \x1b[0m{details}");
} else {
eprintln!("{ARROW} {action} {details}");
}
}
pub fn step(action: &str, details: &str) {
emit("1;34", action, details);
}
pub fn skip(action: &str, details: &str) {
emit("1;33", action, details);
}
#[allow(dead_code)]
pub fn info(action: &str, details: &str) {
emit("1;32", action, details);
}
-16
View File
@@ -1,16 +0,0 @@
mod builder;
mod cli;
mod config;
mod container;
mod eval;
mod graph;
mod layout;
mod log;
mod options;
mod phase;
mod recipe;
fn main() -> anyhow::Result<()> {
container::install_signals()?;
cli::run()
}
-35
View File
@@ -1,35 +0,0 @@
use allocative::Allocative;
use starlark::values::{Heap, OwnedFrozenValue, StarlarkValue, Value};
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value};
use std::collections::HashMap;
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct Options {
entries: HashMap<String, OwnedFrozenValue>,
}
impl Options {
pub fn new(entries: HashMap<String, OwnedFrozenValue>) -> Self {
Self { entries }
}
}
impl std::fmt::Display for Options {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "options")
}
}
starlark::starlark_simple_value!(Options);
#[starlark_value(type = "options")]
impl<'v> StarlarkValue<'v> for Options {
fn get_attr(&self, attribute: &str, _heap: &'v Heap) -> Option<Value<'v>> {
let owned = self.entries.get(attribute)?;
// SAFETY: `self` is kept alive by the module heap into which it was
// allocated, and `owned` holds an Arc to its source frozen heap. The
// returned Value therefore remains valid for as long as the receiving
// module is alive.
Some(unsafe { owned.unchecked_frozen_value() }.to_value())
}
}
-389
View File
@@ -1,389 +0,0 @@
use std::{cell::RefCell, collections::BTreeMap, rc::Rc};
use allocative::Allocative;
use anyhow::anyhow;
use starlark::{
collections::SmallMap,
environment::{Methods, MethodsBuilder, MethodsStatic, Module},
eval::Evaluator,
values::{
Heap, OwnedFrozenValue, StarlarkValue, Value, ValueLike, list::UnpackList, none::NoneType,
},
};
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_module, starlark_value};
use crate::container::Container;
thread_local! {
static CURRENT: RefCell<Option<PhaseRuntime>> = const { RefCell::new(None) };
}
#[derive(Clone)]
pub struct PhaseRuntime {
pub container: Rc<RefCell<Container>>,
pub base_path: String,
pub base_env: Vec<(String, String)>,
}
pub struct PhaseRuntimeGuard;
impl PhaseRuntimeGuard {
pub fn enter(runtime: PhaseRuntime) -> Self {
CURRENT.with(|cell| {
let prev = cell.borrow_mut().replace(runtime);
assert!(prev.is_none(), "phase runtime already set");
});
Self
}
}
impl Drop for PhaseRuntimeGuard {
fn drop(&mut self) {
CURRENT.with(|cell| {
cell.borrow_mut().take();
});
}
}
fn with_current<R>(f: impl FnOnce(&PhaseRuntime) -> R) -> anyhow::Result<R> {
CURRENT.with(|cell| {
let borrow = cell.borrow();
let runtime = borrow
.as_ref()
.ok_or_else(|| anyhow!("ctx.run called outside of a phase invocation"))?;
Ok(f(runtime))
})
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct Path {
inner: String,
}
impl Path {
pub fn new(inner: impl Into<String>) -> Self {
Self {
inner: inner.into(),
}
}
pub fn as_str(&self) -> &str {
&self.inner
}
}
fn join_paths(base: &str, rhs: &str) -> String {
let trimmed = base.trim_end_matches('/');
if trimmed.is_empty() {
format!("/{rhs}")
} else {
format!("{trimmed}/{rhs}")
}
}
fn coerce_path_string(value: Value<'_>) -> anyhow::Result<String> {
if let Some(s) = value.unpack_str() {
return Ok(s.to_owned());
}
if let Some(p) = value.downcast_ref::<Path>() {
return Ok(p.inner.clone());
}
if let Some(s) = value.downcast_ref::<SourceDir>() {
return Ok(s.default.clone());
}
anyhow::bail!("expected a string or path, got `{}`", value.get_type())
}
impl std::fmt::Display for Path {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.inner)
}
}
starlark::starlark_simple_value!(Path);
#[starlark_value(type = "path")]
impl<'v> StarlarkValue<'v> for Path {
fn div(&self, other: Value<'v>, heap: &'v Heap) -> starlark::Result<Value<'v>> {
let rhs = coerce_path_string(other).map_err(starlark::Error::new_other)?;
Ok(heap.alloc(Path::new(join_paths(&self.inner, &rhs))))
}
fn add(&self, rhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
let suffix = rhs.unpack_str()?;
Some(Ok(heap.alloc(format!("{}{}", self.inner, suffix))))
}
fn radd(&self, lhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
let prefix = lhs.unpack_str()?;
Some(Ok(heap.alloc(format!("{}{}", prefix, self.inner))))
}
fn equals(&self, other: Value<'v>) -> starlark::Result<bool> {
Ok(other
.downcast_ref::<Self>()
.map(|o| o.inner == self.inner)
.unwrap_or(false))
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct SourceDir {
default: String,
entries: BTreeMap<String, String>,
}
impl SourceDir {
pub fn single(path: impl Into<String>) -> Self {
Self {
default: path.into(),
entries: BTreeMap::new(),
}
}
pub fn named<I, K, V>(entries: I) -> Self
where
I: IntoIterator<Item = (K, V)>,
K: Into<String>,
V: Into<String>,
{
let entries: BTreeMap<String, String> = entries
.into_iter()
.map(|(k, v)| (k.into(), v.into()))
.collect();
let default = entries
.values()
.next()
.cloned()
.unwrap_or_else(|| "/sources".to_owned());
Self { default, entries }
}
}
impl std::fmt::Display for SourceDir {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.default)
}
}
starlark::starlark_simple_value!(SourceDir);
#[starlark_value(type = "source_dir")]
impl<'v> StarlarkValue<'v> for SourceDir {
fn at(&self, index: Value<'v>, heap: &'v Heap) -> starlark::Result<Value<'v>> {
let key = index.unpack_str().ok_or_else(|| {
starlark::Error::new_other(anyhow!("source_dir index must be a string"))
})?;
let path = self.entries.get(key).ok_or_else(|| {
starlark::Error::new_other(anyhow!(
"no source named `{key}` (available: {})",
if self.entries.is_empty() {
"<none>".to_owned()
} else {
self.entries
.keys()
.map(String::as_str)
.collect::<Vec<_>>()
.join(", ")
}
))
})?;
Ok(heap.alloc(path.as_str()))
}
fn add(&self, rhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
let suffix = rhs.unpack_str()?;
Some(Ok(heap.alloc(format!("{}{}", self.default, suffix))))
}
fn radd(&self, lhs: Value<'v>, heap: &'v Heap) -> Option<starlark::Result<Value<'v>>> {
let prefix = lhs.unpack_str()?;
Some(Ok(heap.alloc(format!("{}{}", prefix, self.default))))
}
fn div(&self, other: Value<'v>, heap: &'v Heap) -> starlark::Result<Value<'v>> {
let rhs = coerce_path_string(other).map_err(starlark::Error::new_other)?;
Ok(heap.alloc(Path::new(join_paths(&self.default, &rhs))))
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct PhaseContext {
source_dir: SourceDir,
build_dir: String,
sysroot: String,
files: Option<String>,
jobs: i32,
}
impl PhaseContext {
pub fn new(source_dir: SourceDir, jobs: i32, files: Option<String>) -> Self {
Self {
source_dir,
build_dir: "/build".to_owned(),
sysroot: "/sysroot".to_owned(),
files,
jobs,
}
}
}
impl std::fmt::Display for PhaseContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "ctx")
}
}
starlark::starlark_simple_value!(PhaseContext);
#[starlark_value(type = "phase_context")]
impl<'v> StarlarkValue<'v> for PhaseContext {
fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
Some(match attr {
"source_dir" => heap.alloc(self.source_dir.clone()),
"build_dir" => heap.alloc(Path::new(self.build_dir.clone())),
"sysroot" => heap.alloc(Path::new(self.sysroot.clone())),
"files" => heap.alloc(Path::new(self.files.as_ref()?.clone())),
"jobs" => heap.alloc(self.jobs),
_ => return None,
})
}
fn has_attr(&self, attr: &str, _heap: &'v Heap) -> bool {
match attr {
"source_dir" | "build_dir" | "sysroot" | "jobs" | "run" => true,
"files" => self.files.is_some(),
_ => false,
}
}
fn dir_attr(&self) -> Vec<String> {
let mut attrs = vec![
"source_dir".to_owned(),
"build_dir".to_owned(),
"sysroot".to_owned(),
"jobs".to_owned(),
"run".to_owned(),
];
if self.files.is_some() {
attrs.push("files".to_owned());
}
attrs
}
fn get_methods() -> Option<&'static Methods> {
static RES: MethodsStatic = MethodsStatic::new();
RES.methods(phase_context_methods)
}
}
#[starlark_module]
fn phase_context_methods(builder: &mut MethodsBuilder) {
fn run<'v>(
#[starlark(this)] _this: Value<'v>,
#[starlark(require = pos)] argv: UnpackList<Value<'v>>,
#[starlark(require = named)] env: Option<SmallMap<String, Value<'v>>>,
#[starlark(require = named)] cwd: Option<String>,
) -> anyhow::Result<NoneType> {
let argv: Vec<String> = argv
.items
.iter()
.map(|v| coerce_path_string(*v))
.collect::<anyhow::Result<Vec<_>>>()?;
let mut env_strings: SmallMap<String, String> = SmallMap::new();
if let Some(env) = env {
for (k, v) in env {
env_strings.insert(k, coerce_path_string(v)?);
}
}
run_in_container(&argv, env_strings, cwd.as_deref().unwrap_or("/build"))?;
Ok(NoneType)
}
}
fn run_in_container(
argv: &[String],
env_overrides: SmallMap<String, String>,
cwd: &str,
) -> anyhow::Result<()> {
let (container, env) = with_current(|runtime| {
let mut env: Vec<(String, String)> = runtime
.base_env
.iter()
.cloned()
.map(|(k, v)| {
if k == "PATH" {
(k, runtime.base_path.clone())
} else {
(k, v)
}
})
.collect();
for (k, v) in env_overrides {
if let Some(slot) = env.iter_mut().find(|(existing, _)| existing == &k) {
slot.1 = v;
} else {
env.push((k, v));
}
}
(runtime.container.clone(), env)
})?;
container.borrow().exec(argv, &env, cwd)
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct PackageContext {
dest_dir: String,
}
impl PackageContext {
pub fn new(dest_dir: String) -> Self {
Self { dest_dir }
}
}
impl std::fmt::Display for PackageContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "pkg")
}
}
starlark::starlark_simple_value!(PackageContext);
#[starlark_value(type = "package_context")]
impl<'v> StarlarkValue<'v> for PackageContext {
fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option<Value<'v>> {
match attr {
"dest_dir" => Some(heap.alloc(Path::new(self.dest_dir.clone()))),
_ => None,
}
}
fn has_attr(&self, attr: &str, _heap: &'v Heap) -> bool {
attr == "dest_dir"
}
fn dir_attr(&self) -> Vec<String> {
vec!["dest_dir".to_owned()]
}
}
pub fn invoke_phase(func: &OwnedFrozenValue, args: &[PhaseArg]) -> anyhow::Result<()> {
let module = Module::new();
let mut eval = Evaluator::new(&module);
let allocated: Vec<Value<'_>> = args
.iter()
.map(|arg| match arg {
PhaseArg::Ctx(ctx) => module.heap().alloc(ctx.clone()),
PhaseArg::Pkg(pkg) => module.heap().alloc(pkg.clone()),
})
.collect();
eval.eval_function(func.value(), &allocated, &[])
.map_err(|err| anyhow!("{err}"))?;
Ok(())
}
pub enum PhaseArg {
Ctx(PhaseContext),
Pkg(PackageContext),
}
+171
View File
@@ -0,0 +1,171 @@
import graphlib
from dataclasses import dataclass
from pathlib import Path
from src.layout import Layout
from src.recipe import Recipe, RecipeSet
BUILD_PHASES = ("prepare", "configure", "build")
PHASE_STAMPS = {
"prepare": "prepared",
"configure": "configured",
"build": "built",
}
def _key(r: Recipe) -> str:
return r.key # "host:<name>" or "<name>"
def transitive_host_deps(rs: RecipeSet, r: Recipe) -> list[str]:
"""Returns transitive host_deps for `r`, in topological order (deepest first)."""
seen: dict[str, Recipe] = {}
order: list[str] = []
def visit(name: str) -> None:
if name in seen:
return
h = rs.host.get(name)
if h is None:
raise KeyError(f"unknown host dep: {name}")
seen[name] = h
for hh in h.host_deps:
visit(hh)
order.append(name)
for d in r.host_deps:
visit(d)
return order
def is_built(layout: Layout, r: Recipe) -> bool:
if r.kind == "host":
return layout.host_pkg_marker(r.name, r.version, r.revision).is_file()
for out in r.outputs:
if not layout.apk_path(out, r.version, r.revision).is_file():
return False
return True
def stamp_dir(layout: Layout, r: Recipe) -> Path:
wd = layout.build_workdir(r.name if r.kind == "target" else f"host-{r.name}")
return wd / ".orchid-stamps"
def stamp_token(r: Recipe) -> str:
return f"{r.version}-r{r.revision}\n"
def stamp_valid(layout: Layout, r: Recipe, phase: str) -> bool:
name = PHASE_STAMPS.get(phase)
if name is None:
return False
p = stamp_dir(layout, r) / name
try:
return p.read_text() == stamp_token(r)
except FileNotFoundError:
return False
def planned_stages(
layout: Layout, r: Recipe, *, forced: bool = False
) -> tuple[str, ...]:
stages: list[str] = []
for phase in BUILD_PHASES:
if phase in r.phases and (forced or not stamp_valid(layout, r, phase)):
stages.append(phase)
if "install" in r.phases:
stages.append("install")
stages.append("package" if r.kind == "target" else "finalize")
return tuple(stages)
@dataclass
class Plan:
order: list[str] # ordered list of recipe keys
recipes: dict[str, Recipe] # all referenced recipes
forced: set[str] # keys to rebuild even if built
stages: dict[str, tuple[str, ...]] # stages each planned recipe will run
def __iter__(self):
return iter(self.order)
def _collect_targets(rs: RecipeSet, requested: list[str] | None) -> list[Recipe]:
if not requested:
return [r for r in rs.target.values() if r.enabled] + [
r for r in rs.host.values() if r.enabled
]
out: list[Recipe] = []
for spec in requested:
r = rs.get(spec)
if not r.enabled:
raise ValueError(f"{spec}: disabled by build_if = False")
out.append(r)
return out
def build_plan(
rs: RecipeSet, layout: Layout, requested: list[str] | None, *, rebuild: bool = False
) -> Plan:
seen: dict[str, Recipe] = {}
ts: graphlib.TopologicalSorter[str] = graphlib.TopologicalSorter()
forced: set[str] = set()
def add(r: Recipe) -> None:
k = _key(r)
if k in seen:
return
seen[k] = r
deps: list[str] = []
for h in r.host_deps:
hr = rs.host.get(h)
if hr is None:
raise KeyError(f"{r.name}: unknown host dep {h!r}")
if not hr.enabled:
raise ValueError(f"{r.name}: host dep {h!r} disabled by build_if")
add(hr)
deps.append(_key(hr))
if r.kind == "target":
for d in (*r.deps, *r.run_deps):
tr = rs.target.get(d)
if tr is None:
raise KeyError(f"{r.name}: unknown dep {d!r}")
if not tr.enabled:
raise ValueError(f"{r.name}: dep {d!r} disabled by build_if")
add(tr)
deps.append(_key(tr))
else:
# host recipes may declare target `deps` that need to land in /sysroot
for d in r.deps:
tr = rs.target.get(d)
if tr is None:
raise KeyError(f"host:{r.name}: unknown target dep {d!r}")
if not tr.enabled:
raise ValueError(f"host:{r.name}: dep {d!r} disabled by build_if")
add(tr)
deps.append(_key(tr))
ts.add(k, *deps)
requested_recipes = _collect_targets(rs, requested)
for r in requested_recipes:
add(r)
if rebuild:
forced.add(_key(r))
order = list(ts.static_order())
stages: dict[str, tuple[str, ...]] = {}
final_order: list[str] = []
for k in order:
r = seen[k]
if k in forced:
stages[k] = planned_stages(layout, r, forced=True)
final_order.append(k)
continue
if is_built(layout, r):
continue
stages[k] = planned_stages(layout, r)
final_order.append(k)
return Plan(order=final_order, recipes=seen, forced=forced, stages=stages)
+49
View File
@@ -0,0 +1,49 @@
import importlib.util
from pathlib import Path
from typing import Any
from src.layout import Layout
REQUIRED_KEYS = ("arch", "triple", "container_image")
def _load_module(path: Path, name: str):
spec = importlib.util.spec_from_file_location(name, path)
if spec is None or spec.loader is None:
raise RuntimeError(f"cannot load profile module {path}")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def load_profile(layout: Layout) -> dict[str, Any]:
link = layout.profile_link
if not link.exists():
raise FileNotFoundError(f"{link} missing a profile")
target = link.resolve()
if not target.is_file():
raise FileNotFoundError(f"profile {target.name}: missing config.py")
mod = _load_module(target, f"orchid_profile_{target.name}")
if not hasattr(mod, "profile"):
raise AttributeError(f"profile {target.name}: config.py must define profile()")
data = mod.profile()
if not isinstance(data, dict):
raise TypeError(f"profile {target.name}: profile() must return a dict")
missing = [k for k in REQUIRED_KEYS if k not in data]
if missing:
raise ValueError(f"profile {target.name}: missing keys {missing}")
data["__name__"] = target.name
return data
def init_build_dir(build: Path, repo: Path, profile_name: str) -> None:
profile_src = repo / "profiles" / (profile_name + ".py")
if not (profile_src).is_file():
raise FileNotFoundError(f"profile {profile_name!r} not found at {profile_src}")
build.mkdir(parents=True, exist_ok=False)
link = build / "profile"
# Use relative symlink so the build dir can be moved with the repo.
import os
rel = Path(os.path.relpath(profile_src, build))
link.symlink_to(rel)
+258
View File
@@ -0,0 +1,258 @@
import importlib
import importlib.util
import pkgutil
from dataclasses import dataclass
from functools import cache
from pathlib import Path, PurePosixPath
from typing import Any, Callable
from src import source as src_mod
from src.layout import Layout
from src.source import Subpackage, Tarball, subpackage, tarball, git
PHASE_NAMES = ("prepare", "configure", "build", "install")
LIB_DIR = Path(__file__).with_name("lib")
@dataclass
class Recipe:
name: str
kind: str # "target" | "host"
dir: Path # recipe directory or parent (for pure form)
pure: bool
version: str
revision: int
description: str
license: str
url: str
maintainer: str
sources: dict[str | None, Tarball | src_mod.Git] # key=None for single source
host_deps: tuple[str, ...]
deps: tuple[str, ...]
run_deps: tuple[str, ...]
subpackages: tuple[Subpackage, ...]
phases: dict[str, Callable[[Any], None]]
enabled: bool
@property
def key(self) -> str:
return f"host:{self.name}" if self.kind == "host" else self.name
@property
def patches_dir(self) -> Path | None:
if self.pure:
return None
p = self.dir / "patches"
return p if p.is_dir() else None
@property
def files_dir(self) -> Path | None:
if self.pure:
return None
p = self.dir / "files"
return p if p.is_dir() else None
@property
def outputs(self) -> list[str]:
if self.kind == "host":
return [self.name]
return [self.name, *(s.name for s in self.subpackages)]
# Import everything in lib/ for recipes
@cache
def _lib_symbols() -> dict:
ns = {}
if not LIB_DIR.is_dir():
return ns
for info in sorted(pkgutil.iter_modules([str(LIB_DIR)]), key=lambda i: i.name):
if info.ispkg:
continue
mod = importlib.import_module(f"src.lib.{info.name}")
names = getattr(mod, "__all__", None)
if names is None:
names = [n for n in mod.__dict__ if not n.startswith("_")]
for n in names:
ns[n] = getattr(mod, n)
return ns
# Symbols which are injected into the recipe
def _builtins(profile: dict) -> dict:
return {
**_lib_symbols(),
"tarball": tarball,
"git": git,
"subpackage": subpackage,
"path": PurePosixPath,
"profile": profile,
}
def _load_module(path: Path, ns: dict):
spec = importlib.util.spec_from_file_location(
f"orchid_recipe_{path.stem}_{id(path)}", path
)
if spec is None or spec.loader is None:
raise RuntimeError(f"cannot load {path}")
mod = importlib.util.module_from_spec(spec)
mod.__dict__.update(ns)
spec.loader.exec_module(mod)
return mod
def _plain_field(mod, recipe_name: str, field_name: str) -> str:
value = getattr(mod, field_name, "")
if value is None:
return ""
if not isinstance(value, str):
raise TypeError(f"{recipe_name}: '{field_name}' must be a string")
return value
def _load_one(
name: str, kind: str, recipe_file: Path, recipe_dir: Path, pure: bool, profile: dict
) -> Recipe | None:
mod = _load_module(recipe_file, _builtins(profile))
version = getattr(mod, "version", None)
if not isinstance(version, str) or not version:
raise ValueError(f"{name}: 'version' (str) required")
revision = int(getattr(mod, "revision", 1))
if hasattr(mod, "metadata"):
raise TypeError(
f"{name}: use plain description/license/url/maintainer fields, not metadata = meta(...)"
)
description = _plain_field(mod, name, "description")
license = _plain_field(mod, name, "license")
url = _plain_field(mod, name, "url")
maintainer = _plain_field(mod, name, "maintainer")
single = getattr(mod, "source", None)
multi = getattr(mod, "sources", None)
if single is not None and multi is not None:
raise ValueError(f"{name}: define either 'source' or 'sources', not both")
sources = {}
if single is not None:
if not isinstance(single, (Tarball, src_mod.Git)):
raise TypeError(f"{name}: 'source' must be tarball()/git()")
sources[None] = single
elif multi is not None:
if not isinstance(multi, dict) or not multi:
raise TypeError(f"{name}: 'sources' must be a non-empty dict")
multi_items = list(multi.items())
for k, v in multi_items:
if not isinstance(k, str) or not k:
raise TypeError(f"{name}: 'sources' keys must be non-empty strings")
if not isinstance(v, (Tarball, src_mod.Git)):
raise TypeError(f"{name}: source {k!r} must be tarball()/git()")
if len(multi_items) == 1:
sources[None] = multi_items[0][1]
else:
for k, v in multi_items:
sources[k] = v
else:
raise ValueError(f"{name}: 'source' or 'sources' required")
if pure:
for s in sources.values():
if s.patches:
raise ValueError(
f"{name}: pure-form recipe cannot declare patches; convert to {name}/recipe.py + patches/"
)
host_deps = tuple(getattr(mod, "host_deps", ()) or ())
deps = tuple(getattr(mod, "deps", ()) or ())
run_deps = tuple(getattr(mod, "run_deps", ()) or ())
subs = tuple(getattr(mod, "subpackages", ()) or ())
for s in subs:
if not isinstance(s, Subpackage):
raise TypeError(
f"{name}: subpackages entries must be subpackage(...) values"
)
if kind == "host" and subs:
raise ValueError(f"{name}: host recipes do not support subpackages")
phases: dict[str, Callable] = {}
for pn in PHASE_NAMES:
fn = getattr(mod, pn, None)
if fn is not None:
if not callable(fn):
raise TypeError(f"{name}: '{pn}' must be a function")
phases[pn] = fn
if kind == "target" and "build" not in phases and "install" not in phases:
# Purely declarative pkgs are unusual but allowed
pass
enabled = bool(getattr(mod, "build_if", True))
return Recipe(
name=name,
kind=kind,
dir=recipe_dir,
pure=pure,
version=version,
revision=revision,
description=description,
license=license,
url=url,
maintainer=maintainer,
sources=sources,
host_deps=host_deps,
deps=deps,
run_deps=run_deps,
subpackages=subs,
phases=phases,
enabled=enabled,
)
def _discover(root: Path, kind: str, profile: dict) -> dict[str, Recipe]:
out: dict[str, Recipe] = {}
if not root.is_dir():
return out
for entry in sorted(root.iterdir()):
if entry.name.startswith(".") or entry.name.startswith("_"):
continue
if entry.is_file() and entry.suffix == ".py":
name = entry.stem
r = _load_one(name, kind, entry, entry.parent, pure=True, profile=profile)
elif entry.is_dir():
rf = entry / "recipe.py"
if not rf.is_file():
continue
r = _load_one(entry.name, kind, rf, entry, pure=False, profile=profile)
else:
continue
if r is None:
continue
if r.name in out:
raise ValueError(f"duplicate {kind} recipe: {r.name}")
out[r.name] = r
return out
@dataclass
class RecipeSet:
target: dict[str, Recipe]
host: dict[str, Recipe]
def get(self, key: str) -> Recipe:
if key.startswith("host:"):
n = key[5:]
if n not in self.host:
raise KeyError(f"host recipe not found: {n}")
return self.host[n]
if key not in self.target:
raise KeyError(f"recipe not found: {key}")
return self.target[key]
def all(self) -> list[Recipe]:
return [*self.target.values(), *self.host.values()]
def load_recipes(layout: Layout, profile: dict) -> RecipeSet:
target = _discover(layout.recipes_dir, "target", profile)
host = _discover(layout.host_recipes_dir, "host", profile)
return RecipeSet(target=target, host=host)
-71
View File
@@ -1,71 +0,0 @@
use allocative::Allocative;
use starlark::values::StarlarkValue;
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value};
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct Metadata {
maintainer: Option<String>,
description: Option<String>,
license: Option<String>,
website: Option<String>,
}
impl std::fmt::Display for Metadata {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "metadata")
}
}
starlark::starlark_simple_value!(Metadata);
#[starlark_value(type = "metadata")]
impl<'v> StarlarkValue<'v> for Metadata {}
impl Metadata {
pub fn new(
maintainer: Option<String>,
description: Option<String>,
license: Option<String>,
website: Option<String>,
) -> Self {
Self {
maintainer,
description,
license,
website,
}
}
pub fn maintainer(&self) -> Option<&str> {
self.maintainer.as_deref()
}
pub fn description(&self) -> Option<&str> {
self.description.as_deref()
}
pub fn license(&self) -> Option<&str> {
self.license.as_deref()
}
pub fn website(&self) -> Option<&str> {
self.website.as_deref()
}
pub fn inherit(&self, from: &Self) -> Self {
Self {
maintainer: self
.maintainer
.as_ref()
.or(from.maintainer.as_ref())
.cloned(),
description: self
.description
.as_ref()
.or(from.description.as_ref())
.cloned(),
license: self.license.as_ref().or(from.license.as_ref()).cloned(),
website: self.website.as_ref().or(from.website.as_ref()).cloned(),
}
}
}
-603
View File
@@ -1,603 +0,0 @@
mod metadata;
mod source;
mod subpackage;
use anyhow::{Context, bail};
use starlark::{
environment::{FrozenModule, Module},
eval::Evaluator,
values::{
OwnedFrozenValue, UnpackValue, ValueLike, dict::DictRef, list::ListRef,
typing::StarlarkCallable,
},
};
use std::{
collections::{HashMap, HashSet},
path::{Path, PathBuf},
};
use walkdir::WalkDir;
use crate::{
eval::{self, ExtractError},
options::Options,
};
pub use metadata::Metadata;
pub use source::{GitSource, Source, TarballSource};
pub use subpackage::Subpackage;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RecipeKind {
Package,
HostPackage,
}
impl RecipeKind {
pub fn key(self, name: &str) -> String {
match self {
Self::Package => name.to_string(),
Self::HostPackage => format!("host:{name}"),
}
}
pub fn slug(self, name: &str) -> String {
self.key(name).replace(':', "-")
}
}
#[derive(Debug)]
pub enum Sources {
Single(source::Source),
Multiple(HashMap<String, source::Source>),
}
impl Sources {
pub fn entries(&self) -> Vec<(Option<&str>, &source::Source)> {
match self {
Self::Single(source) => vec![(None, source)],
Self::Multiple(sources) => {
let mut entries = sources
.iter()
.map(|(name, source)| (Some(name.as_str()), source))
.collect::<Vec<_>>();
entries.sort_by_key(|(name, _)| *name);
entries
}
}
}
}
pub struct Recipe {
/// Recipe name without namespace prefix.
name: String,
/// Path to the recipe's .star file.
path: PathBuf,
/// What kind of a recipe is that?
kind: RecipeKind,
/// Version shared by every package output of this recipe.
version: String,
/// Revision shared by every package output of this recipe.
revision: i32,
/// List of sources required to build this recipe.
sources: Sources,
/// All packages produced by this recipe.
/// This is empty for host recipes.
outputs: Vec<OutputPackage>,
/// Host packages requires for this recipe.
host_deps: Vec<String>,
/// Packages installed to the system root during build of the recipe, but
/// not listed as part of the `apk` dependencies.
build_deps: Vec<String>,
/// Packages installed to the system root during build of the recipe AND
/// listed as part of the `apk` dependencies.
deps: Vec<String>,
/// Packages NOT installed to the system root during build, only listed
/// as part of the `apk` dependencies.
run_deps: Vec<String>,
/// Starlark phase functions defined by the recipe.
phases: RecipePhases,
}
impl Recipe {
pub fn load(
path: &Path,
name: &str,
kind: RecipeKind,
options: &Options,
lib: Option<&FrozenModule>,
) -> anyhow::Result<Option<Self>> {
let module = eval::eval_file(path, Some(options), lib)
.with_context(|| format!("evaluating recipe {}", path.display()))?;
if !Self::eval_build_if(&module)? {
return Ok(None);
}
let version = eval::extract_string(&module, "version")
.map_err(|e| anyhow::anyhow!("field `version`: {e}"))?;
let revision = match eval::extract_i32(&module, "revision") {
Ok(v) => v,
Err(ExtractError::NotFound) => 1,
Err(e) => bail!("field `revision`: {e}"),
};
let metadata = match module.get("metadata") {
None => Metadata::new(None, None, None, None),
Some(value) => value
.downcast_ref::<Metadata>()
.ok_or_else(|| anyhow::anyhow!("field `metadata`: expected a metadata value"))?
.clone(),
};
let source_value = module.get("source");
let sources_value = module.get("sources");
let sources = match (source_value, sources_value) {
(None, None) => bail!("recipe must define either `source` or `sources`"),
(Some(_), Some(_)) => {
bail!("recipe must define exactly one of `source` or `sources`, not both")
}
(Some(value), None) => {
let source = value
.downcast_ref::<Source>()
.ok_or_else(|| anyhow::anyhow!("field `source`: expected a source value"))?
.clone();
Sources::Single(source)
}
(None, Some(value)) => {
let dict = DictRef::from_value(value).ok_or_else(|| {
anyhow::anyhow!("field `sources`: expected a dict of name -> source")
})?;
if dict.iter().len() == 0 {
bail!("field `sources`: must contain at least one entry");
}
let mut map: HashMap<String, source::Source> = HashMap::new();
for (key, value) in dict.iter() {
let key_str = key
.unpack_str()
.ok_or_else(|| anyhow::anyhow!("field `sources`: keys must be strings"))?;
if !Self::is_valid_source_name(key_str) {
bail!(
"field `sources`: invalid source name `{key_str}` (allowed: letters, digits, `-`, `_`; must start with a letter or digit)"
);
}
let source = value.downcast_ref::<Source>().ok_or_else(|| {
anyhow::anyhow!("field `sources`: entry `{key_str}` is not a source value")
})?;
if map.insert(key_str.to_owned(), source.clone()).is_some() {
bail!("field `sources`: duplicate key `{key_str}`");
}
}
Sources::Multiple(map)
}
};
let host_deps = Self::optional_string_list(&module, "host_deps")?;
let build_deps = Self::optional_string_list(&module, "build_deps")?;
let deps = Self::optional_string_list(&module, "deps")?;
let run_deps = Self::optional_string_list(&module, "run_deps")?;
let recipe_key = kind.key(name);
let outputs = match kind {
RecipeKind::Package => {
let mut outputs = vec![OutputPackage {
recipe: recipe_key.clone(),
name: name.to_owned(),
metadata: metadata.clone(),
file_globs: Vec::new(),
base: true,
}];
if let Some(value) = module.get("subpackages") {
let list = ListRef::from_value(value)
.ok_or_else(|| anyhow::anyhow!("field `subpackages`: expected a list"))?;
for item in list.iter() {
let sub = item.downcast_ref::<Subpackage>().ok_or_else(|| {
anyhow::anyhow!(
"field `subpackages`: each entry must be a subpackage value"
)
})?;
Self::check_subpackage_file_globs(sub.name(), sub.file_globs())?;
outputs.push(OutputPackage {
recipe: recipe_key.clone(),
name: sub.name().to_owned(),
metadata: sub.metadata().inherit(&metadata),
file_globs: sub.file_globs().to_vec(),
base: false,
});
}
}
outputs
}
RecipeKind::HostPackage => {
if module.get("subpackages").is_some() {
bail!("host recipes cannot declare `subpackages`");
}
Vec::new()
}
};
let module = module
.freeze()
.map_err(|err| anyhow::anyhow!("freezing recipe module {}: {err:?}", path.display()))?;
let phases = RecipePhases::load(&module)?;
let recipe = Recipe {
name: name.to_owned(),
path: path.to_path_buf(),
kind,
version,
revision,
sources,
outputs,
host_deps,
build_deps,
deps,
run_deps,
phases,
};
Ok(Some(recipe))
}
pub fn phases(&self) -> &RecipePhases {
&self.phases
}
pub fn key(&self) -> String {
self.kind.key(&self.name)
}
pub fn slug(&self) -> String {
self.kind.slug(&self.name)
}
pub fn data_dir(&self) -> Option<&Path> {
if self.path.file_name().is_some_and(|n| n == "recipe.star") {
self.path.parent()
} else {
None
}
}
pub fn files_dir(&self) -> Option<PathBuf> {
let candidate = self.data_dir()?.join("files");
if candidate.is_dir() {
Some(candidate)
} else {
None
}
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn kind(&self) -> RecipeKind {
self.kind
}
pub fn version(&self) -> &str {
&self.version
}
pub fn revision(&self) -> i32 {
self.revision
}
pub fn sources(&self) -> &Sources {
&self.sources
}
pub fn outputs(&self) -> &[OutputPackage] {
&self.outputs
}
pub fn base_output(&self) -> Option<&OutputPackage> {
self.outputs.iter().find(|output| output.is_base())
}
pub fn host_deps(&self) -> &[String] {
&self.host_deps
}
pub fn build_deps(&self) -> &[String] {
&self.build_deps
}
pub fn deps(&self) -> &[String] {
&self.deps
}
pub fn run_deps(&self) -> &[String] {
&self.run_deps
}
fn optional_string_list(module: &Module, key: &str) -> anyhow::Result<Vec<String>> {
match eval::extract_string_list(module, key) {
Ok(v) => Ok(v),
Err(ExtractError::NotFound) => Ok(Vec::new()),
Err(e) => Err(anyhow::anyhow!("field `{key}`: {e}")),
}
}
/// Evaluates "build_if" functions.
fn eval_build_if(module: &Module) -> anyhow::Result<bool> {
let Some(value) = module.get("build_if") else {
return Ok(true);
};
if let Some(b) = value.unpack_bool() {
return Ok(b);
}
let callable: Option<StarlarkCallable<'_>> = StarlarkCallable::unpack_value_opt(value);
if callable.is_none() {
bail!("field `build_if`: expected a bool or a callable returning a bool");
}
let mut eval = Evaluator::new(module);
let result = eval
.eval_function(value, &[], &[])
.map_err(|err| anyhow::anyhow!("calling `build_if`: {err}"))?;
result
.unpack_bool()
.ok_or_else(|| anyhow::anyhow!("field `build_if`: must return a bool"))
}
fn is_valid_source_name(name: &str) -> bool {
let mut chars = name.chars();
let Some(first) = chars.next() else {
return false;
};
if !first.is_ascii_alphanumeric() {
return false;
}
chars.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
}
fn check_subpackage_file_globs(name: &str, file_globs: &[String]) -> anyhow::Result<()> {
if file_globs.is_empty() {
bail!("subpackage '{name}' must declare at least one file glob");
}
for glob in file_globs {
if glob.is_empty() {
bail!("subpackage '{name}' has an empty file glob");
}
if glob.starts_with('/') {
bail!("subpackage '{name}' glob '{glob}' must be relative to the package dest dir");
}
if Path::new(glob)
.components()
.any(|component| matches!(component, std::path::Component::ParentDir))
{
bail!("subpackage '{name}' glob '{glob}' must not contain '..'");
}
}
Ok(())
}
}
pub struct RecipePhases {
prepare: Option<OwnedFrozenValue>,
configure: Option<OwnedFrozenValue>,
build: OwnedFrozenValue,
install: OwnedFrozenValue,
}
impl RecipePhases {
fn load(module: &FrozenModule) -> anyhow::Result<Self> {
Ok(Self {
prepare: optional_phase_function(module, "prepare")?,
configure: optional_phase_function(module, "configure")?,
build: required_phase_function(module, "build")?,
install: required_phase_function(module, "install")?,
})
}
pub fn prepare(&self) -> Option<&OwnedFrozenValue> {
self.prepare.as_ref()
}
pub fn configure(&self) -> Option<&OwnedFrozenValue> {
self.configure.as_ref()
}
pub fn build(&self) -> &OwnedFrozenValue {
&self.build
}
pub fn install(&self) -> &OwnedFrozenValue {
&self.install
}
}
fn optional_phase_function(
module: &FrozenModule,
name: &str,
) -> anyhow::Result<Option<OwnedFrozenValue>> {
let Some(value) = module
.get_option(name)
.with_context(|| format!("field `{name}`"))?
else {
return Ok(None);
};
validate_phase_function(name, &value)?;
Ok(Some(value))
}
fn required_phase_function(module: &FrozenModule, name: &str) -> anyhow::Result<OwnedFrozenValue> {
let value = module
.get_option(name)
.with_context(|| format!("field `{name}`"))?
.ok_or_else(|| anyhow::anyhow!("field `{name}`: missing"))?;
validate_phase_function(name, &value)?;
Ok(value)
}
fn validate_phase_function(name: &str, value: &OwnedFrozenValue) -> anyhow::Result<()> {
let callable: Option<StarlarkCallable<'_>> = StarlarkCallable::unpack_value_opt(value.value());
if callable.is_none() {
bail!("field `{name}`: expected a callable value");
}
Ok(())
}
#[derive(Clone)]
pub struct OutputPackage {
/// Canonical key of the owning recipe.
recipe: String,
/// Name of the output package.
name: String,
/// Metadata attached to the output package.
metadata: Metadata,
/// File globs claimed from the base package dest dir.
file_globs: Vec<String>,
/// Whether this is the recipe's base output.
base: bool,
}
impl OutputPackage {
pub fn key(&self) -> String {
self.name.clone()
}
pub fn recipe(&self) -> &str {
&self.recipe
}
pub fn name(&self) -> &str {
&self.name
}
pub fn metadata(&self) -> &Metadata {
&self.metadata
}
pub fn file_globs(&self) -> &[String] {
&self.file_globs
}
pub fn is_base(&self) -> bool {
self.base
}
}
pub struct RecipeSet {
recipes: HashMap<String, Recipe>,
outputs: HashMap<String, OutputPackage>,
skipped: HashSet<String>,
}
impl RecipeSet {
pub fn load(
root_path: &Path,
options: &Options,
lib: Option<&FrozenModule>,
) -> anyhow::Result<Self> {
let mut recipes = HashMap::new();
let mut outputs = HashMap::new();
let mut skipped: HashSet<String> = HashSet::new();
for (path, kind) in [
("recipes", RecipeKind::Package),
("host-recipes", RecipeKind::HostPackage),
] {
let recipes_dir = root_path.join(path);
if !recipes_dir.exists() {
continue;
}
for (name, path) in discover_recipes(&recipes_dir)? {
let key = kind.key(&name);
let loaded = Recipe::load(&path, &name, kind, options, lib)
.with_context(|| format!("loading recipe `{name}`"))?;
let Some(recipe) = loaded else {
skipped.insert(key);
continue;
};
if recipes.insert(key.clone(), recipe).is_some() {
bail!("duplicate recipe `{key}`");
}
}
}
for recipe in recipes.values() {
for output in &recipe.outputs {
let key = recipe.kind.key(&output.name);
if outputs.insert(key.clone(), output.clone()).is_some() {
bail!("duplicate package output `{key}`");
}
}
}
Ok(Self {
recipes,
outputs,
skipped,
})
}
pub fn recipe(&self, key: &str) -> anyhow::Result<&Recipe> {
self.recipes
.get(key)
.ok_or_else(|| anyhow::anyhow!("unknown recipe `{key}`"))
}
pub fn output(&self, key: &str) -> anyhow::Result<&OutputPackage> {
self.outputs
.get(key)
.ok_or_else(|| anyhow::anyhow!("unknown output package `{key}`"))
}
pub fn is_skipped(&self, key: &str) -> bool {
self.skipped.contains(key)
}
}
fn discover_recipes(dir: &Path) -> anyhow::Result<HashMap<String, PathBuf>> {
let mut recipes: HashMap<String, PathBuf> = HashMap::new();
let walker = WalkDir::new(dir).follow_links(false);
for entry in walker {
let entry =
entry.with_context(|| format!("walking recipes directory {}", dir.display()))?;
if !entry.file_type().is_file() {
continue;
}
let path = entry.path();
let Some(file_name) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
let name = if file_name == "recipe.star" {
let Some(parent_name) = path
.parent()
.and_then(|p| p.file_name())
.and_then(|n| n.to_str())
else {
continue;
};
parent_name.to_owned()
} else if let Some(stem) = file_name.strip_suffix(".star") {
stem.to_owned()
} else {
continue;
};
if let Some(existing) = recipes.insert(name.clone(), path.to_path_buf()) {
bail!(
"recipe `{name}` is defined twice: {} and {}",
existing.display(),
recipes[&name].display(),
);
}
}
Ok(recipes)
}
-133
View File
@@ -1,133 +0,0 @@
use allocative::Allocative;
use starlark::values::StarlarkValue;
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value};
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct TarballSource {
url: String,
sha256: String,
strip_components: u32,
patches: Vec<String>,
}
impl std::fmt::Display for TarballSource {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "tarball_source")
}
}
starlark::starlark_simple_value!(TarballSource);
#[starlark_value(type = "tarball_source")]
impl<'v> StarlarkValue<'v> for TarballSource {}
impl TarballSource {
pub fn new(url: String, sha256: String, strip_components: u32, patches: Vec<String>) -> Self {
Self {
url,
sha256,
strip_components,
patches,
}
}
pub fn url(&self) -> &str {
&self.url
}
pub fn sha256(&self) -> &str {
&self.sha256
}
pub fn strip_components(&self) -> u32 {
self.strip_components
}
pub fn patches(&self) -> &[String] {
&self.patches
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct GitSource {
url: String,
commit: String,
patches: Vec<String>,
}
impl std::fmt::Display for GitSource {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "git_source")
}
}
starlark::starlark_simple_value!(GitSource);
#[starlark_value(type = "git_source")]
impl<'v> StarlarkValue<'v> for GitSource {}
impl GitSource {
pub fn new(url: String, commit: String, patches: Vec<String>) -> Self {
Self {
url,
commit,
patches,
}
}
pub fn url(&self) -> &str {
&self.url
}
pub fn commit(&self) -> &str {
&self.commit
}
pub fn patches(&self) -> &[String] {
&self.patches
}
}
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub enum Source {
Tarball(TarballSource),
Git(GitSource),
}
impl std::fmt::Display for Source {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "source")
}
}
starlark::starlark_simple_value!(Source);
#[starlark_value(type = "source")]
impl<'v> StarlarkValue<'v> for Source {}
impl Source {
pub fn url(&self) -> &str {
match self {
Self::Tarball(source) => source.url(),
Self::Git(source) => source.url(),
}
}
pub fn cache_key(&self) -> &str {
match self {
Self::Tarball(source) => source.sha256(),
Self::Git(source) => source.commit(),
}
}
pub fn is_unknown_cache_key(&self) -> bool {
matches!(self.cache_key(), "?" | "???")
}
pub fn patches(&self) -> &[String] {
match self {
Self::Tarball(source) => source.patches(),
Self::Git(source) => source.patches(),
}
}
}
-45
View File
@@ -1,45 +0,0 @@
use allocative::Allocative;
use starlark::values::StarlarkValue;
use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_value};
use crate::recipe::Metadata;
#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)]
pub struct Subpackage {
name: String,
file_globs: Vec<String>,
metadata: Metadata,
}
impl Subpackage {
pub fn new(name: String, file_globs: Vec<String>, metadata: Metadata) -> Self {
Self {
name,
file_globs,
metadata,
}
}
pub fn name(&self) -> &str {
&self.name
}
pub fn file_globs(&self) -> &[String] {
&self.file_globs
}
pub fn metadata(&self) -> &Metadata {
&self.metadata
}
}
impl std::fmt::Display for Subpackage {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "subpackage")
}
}
starlark::starlark_simple_value!(Subpackage);
#[starlark_value(type = "subpackage")]
impl<'v> StarlarkValue<'v> for Subpackage {}
+91
View File
@@ -0,0 +1,91 @@
from dataclasses import dataclass
@dataclass(frozen=True)
class Tarball:
url: str
sha256: str
strip_components: int = 1
patches: tuple[str, ...] = ()
@property
def cache_key(self) -> str:
return self.sha256
@dataclass(frozen=True)
class Git:
url: str
commit: str
patches: tuple[str, ...] = ()
@property
def cache_key(self) -> str:
return self.commit
def tarball(
*,
url: str,
sha256: str,
strip_components: int = 1,
patches: list[str] | tuple[str, ...] = (),
) -> Tarball:
return Tarball(
url=url,
sha256=sha256,
strip_components=strip_components,
patches=tuple(patches),
)
def git(*, url: str, commit: str, patches: list[str] | tuple[str, ...] = ()) -> Git:
if not commit or commit == "?":
raise ValueError("git source requires an explicit commit SHA")
return Git(url=url, commit=commit, patches=tuple(patches))
@dataclass(frozen=True)
class Subpackage:
name: str
files: tuple[str, ...]
description: str = ""
license: str = ""
url: str = ""
maintainer: str = ""
def subpackage(
name: str,
*,
description: str = "",
license: str = "",
url: str = "",
maintainer: str = "",
files: list[str] | tuple[str, ...] = (),
) -> Subpackage:
for field_name, value in (
("description", description),
("license", license),
("url", url),
("maintainer", maintainer),
):
if not isinstance(value, str):
raise TypeError(f"subpackage {name}: '{field_name}' must be a string")
for pat in files:
if pat.startswith("/") or ".." in pat.split("/"):
raise ValueError(
f"subpackage {name}: file pattern {pat!r} must be relative and free of '..'"
)
return Subpackage(
name=name,
description=description,
license=license,
url=url,
maintainer=maintainer,
files=tuple(files),
)
def patches_of(src) -> tuple[str, ...]:
return src.patches