diff --git a/Dockerfile b/Dockerfile index a6d50a1..96030b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,7 @@ RUN apk upgrade --no-cache && apk add --no-cache \ patch \ pkgconf \ python3 \ + rsync \ tar \ texinfo \ xz \ diff --git a/config.star b/config.star index de8492a..17b30c7 100644 --- a/config.star +++ b/config.star @@ -3,10 +3,15 @@ container_image = "local/distro-builder:latest" container_dockerfile = "Dockerfile" arch = "x86_64" -libc = "musl" +libc = "glibc" + +if libc == "glibc": + env = "gnu" +else: + env = libc host_cflags = "-O2 -pipe" -host_cxxflags = "" +host_cxxflags = host_cflags host_ldflags = "-Wl,-O1 -Wl,--sort-common -Wl,--as-needed" target_cflags = host_cflags @@ -22,7 +27,7 @@ if arch == "x86_64": options = dict( target_arch = arch, - target_triple = f"{arch}-linux-{libc}", + target_triple = f"{arch}-linux-{env}", host_cflags = host_cflags, host_cxxflags = host_cxxflags, diff --git a/host-recipes/binutils.star b/host-recipes/binutils.star index 3f574b4..adadba9 100644 --- a/host-recipes/binutils.star +++ b/host-recipes/binutils.star @@ -5,14 +5,14 @@ metadata = meta( license = "GPL-3.0-or-later", ) source = tarball_source( - url = "https://ftp.gnu.org/gnu/binutils/binutils-" + version + ".tar.xz", + url = f"https://ftp.gnu.org/gnu/binutils/binutils-{version}.tar.xz", sha256 = "d75a94f4d73e7a4086f7513e67e439e8fcdcbb726ffe63f4661744e6256b2cf2", strip_components = 1, ) def configure(ctx): - ctx.run([ - ctx.source_dir + "/configure", + configure_args = [ + ctx.source_dir / "configure", "--prefix=" + ctx.prefix, "--target=" + options.target_triple, "--with-sysroot=" + ctx.sysroot, @@ -26,11 +26,14 @@ def configure(ctx): "--enable-relro", "--enable-separate-code", "--enable-threads", - # gprofng's libcollector does not build against musl. - "--disable-gprofng", "--disable-nls", "--disable-werror", - ], env = { + ] + # gprofng's libcollector relies on glibc-specific internals. + if options.libc == "musl": + configure_args.append("--disable-gprofng") + + ctx.run(configure_args, env = { "CFLAGS": options.host_cflags, "CXXFLAGS": options.host_cxxflags, "LDFLAGS": options.host_ldflags, diff --git a/host-recipes/gcc-bootstrap.star b/host-recipes/gcc-bootstrap.star index dd23f84..981f8aa 100644 --- a/host-recipes/gcc-bootstrap.star +++ b/host-recipes/gcc-bootstrap.star @@ -13,7 +13,7 @@ host_deps = ["binutils"] def configure(ctx): ctx.run([ - ctx.source_dir + "/configure", + ctx.source_dir / "configure", "--target=" + options.target_triple, "--prefix=" + ctx.prefix, "--with-sysroot=" + ctx.sysroot, diff --git a/host-recipes/gcc.star b/host-recipes/gcc.star new file mode 100644 index 0000000..f5c7a2e --- /dev/null +++ b/host-recipes/gcc.star @@ -0,0 +1,54 @@ +version = "16.1.0" +revision = 1 +metadata = meta( + description = "GNU GCC cross-compiler targeting the system triple", + license = "GPL-3.0-or-later", + website = "https://gcc.gnu.org/", +) +source = tarball_source( + url = f"https://ftp.gnu.org/gnu/gcc/gcc-{version}/gcc-{version}.tar.xz", + sha256 = "50efb4d94c3397aff3b0d61a5abd748b4dd31d9d3f2ab7be05b171d36a510f79", + strip_components = 1, +) +host_deps = ["binutils", "gcc-bootstrap"] +deps = [options.libc, "linux-headers"] + +def configure(ctx): + ctx.run([ + ctx.source_dir / "configure", + "--target=" + options.target_triple, + "--prefix=" + ctx.prefix, + "--with-sysroot=" + ctx.sysroot, + "--with-build-sysroot=" + ctx.sysroot, + "--enable-languages=c,c++,lto", + "--disable-bootstrap", + "--enable-default-pie", + "--enable-default-ssp", + "--enable-lto", + "--enable-threads=posix", + "--enable-tls", + "--enable-libstdcxx-time", + "--enable-checking=release", + "--enable-cet=auto", + "--enable-linker-build-id", + "--disable-nls", + "--disable-multilib", + "--disable-fixed-point", + "--disable-werror", + "--disable-libsanitizer", + "--disable-symvers", + ], env = { + "CFLAGS": options.host_cflags, + "CXXFLAGS": options.host_cxxflags, + "LDFLAGS": options.host_ldflags, + }) + +def build(ctx): + ctx.run(["make", "-j" + str(ctx.jobs)]) + +def install(ctx, pkg): + ctx.run(["make", "install-strip"], env = {"DESTDIR": pkg.dest_dir}) + # Drop libtool archives — they bake build-time paths into target programs. + ctx.run([ + "find", pkg.dest_dir + ctx.prefix, "-name", "*.la", "-delete", + ]) diff --git a/host-recipes/llvm.star b/host-recipes/llvm.star new file mode 100644 index 0000000..a22d492 --- /dev/null +++ b/host-recipes/llvm.star @@ -0,0 +1,52 @@ +version = "20.1.0" +revision = 1 +metadata = meta( + description = "LLVM compiler infrastructure with clang and lld", + license = "Apache-2.0 WITH LLVM-exception", + website = "https://llvm.org/", +) +source = tarball_source( + url = f"https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/llvm-project-{version}.src.tar.xz", + sha256 = "?", + strip_components = 1, +) +host_deps = ["binutils"] + +def configure(ctx): + ctx.run([ + "cmake", + "-S", ctx.source_dir / "llvm", + "-B", ctx.build_dir, + "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_INSTALL_PREFIX=" + ctx.prefix, + "-DLLVM_ENABLE_PROJECTS=clang;clang-tools-extra;lld", + "-DLLVM_ENABLE_RUNTIMES=compiler-rt", + "-DLLVM_TARGETS_TO_BUILD=X86;AArch64;RISCV", + "-DLLVM_DEFAULT_TARGET_TRIPLE=" + options.target_triple, + "-DLLVM_HOST_TRIPLE=" + options.target_triple, + "-DLLVM_ENABLE_LIBXML2=OFF", + "-DLLVM_ENABLE_LIBEDIT=OFF", + "-DLLVM_ENABLE_TERMINFO=OFF", + "-DLLVM_ENABLE_ASSERTIONS=OFF", + "-DLLVM_ENABLE_PIC=ON", + "-DLLVM_BUILD_LLVM_DYLIB=ON", + "-DLLVM_LINK_LLVM_DYLIB=ON", + "-DLLVM_INSTALL_UTILS=ON", + "-DLLVM_INCLUDE_TESTS=OFF", + "-DLLVM_INCLUDE_EXAMPLES=OFF", + "-DLLVM_INCLUDE_BENCHMARKS=OFF", + "-DCLANG_DEFAULT_LINKER=lld", + "-DCLANG_DEFAULT_RTLIB=compiler-rt", + "-DCLANG_DEFAULT_CXX_STDLIB=libstdc++", + ], env = { + "CFLAGS": options.host_cflags, + "CXXFLAGS": options.host_cxxflags, + "LDFLAGS": options.host_ldflags, + }) + +def build(ctx): + ctx.run(["cmake", "--build", ctx.build_dir, "-j" + str(ctx.jobs)]) + +def install(ctx, pkg): + ctx.run(["cmake", "--install", ctx.build_dir], env = {"DESTDIR": pkg.dest_dir}) diff --git a/lib/common.star b/lib/common.star index 779181c..d38b818 100644 --- a/lib/common.star +++ b/lib/common.star @@ -8,15 +8,15 @@ def autotools_configure(ctx, extra_args = [], extra_env = {}): } env.update(extra_env) ctx.run([ - ctx.source_dir + "/configure", + ctx.source_dir / "configure", "--host=" + options.target_triple, "--with-sysroot=" + ctx.sysroot, "--prefix=" + ctx.prefix, "--sysconfdir=/etc", "--localstatedir=/var", - "--bindir=" + ctx.prefix + "/bin", - "--sbindir=" + ctx.prefix + "/bin", - "--libdir=" + ctx.prefix + "/lib", + "--bindir=" + ctx.prefix / "bin", + "--sbindir=" + ctx.prefix / "bin", + "--libdir=" + ctx.prefix / "lib", "--disable-static", "--enable-shared", ] + extra_args, env = env) @@ -25,7 +25,7 @@ def autotools_build(ctx, extra_args = []): ctx.run(["make", "-j" + str(ctx.jobs)] + extra_args) def autotools_install(ctx, pkg, extra_args = []): - ctx.run(["make", "install"] + extra_args, env = {"DESTDIR": pkg.dest_dir}) + ctx.run(["make", "install", "DESTDIR=" + pkg.dest_dir] + extra_args) def autotools(configure_args = [], configure_env = {}, build_args = [], install_args = []): def _configure(ctx): @@ -35,3 +35,82 @@ def autotools(configure_args = [], configure_env = {}, build_args = [], install_ def _install(ctx, pkg): autotools_install(ctx, pkg, extra_args = install_args) return _configure, _build, _install + +def host_autotools_configure(ctx, extra_args = [], extra_env = {}): + env = { + "CFLAGS": options.host_cflags, + "CXXFLAGS": options.host_cxxflags, + "LDFLAGS": options.host_ldflags, + } + env.update(extra_env) + ctx.run([ + ctx.source_dir / "configure", + "--prefix=" + ctx.prefix, + "--sysconfdir=/etc", + "--localstatedir=/var", + "--disable-nls", + ] + extra_args, env = env) + +def host_autotools(configure_args = [], configure_env = {}, build_args = [], install_args = []): + def _configure(ctx): + host_autotools_configure(ctx, extra_args = configure_args, extra_env = configure_env) + def _build(ctx): + autotools_build(ctx, extra_args = build_args) + def _install(ctx, pkg): + autotools_install(ctx, pkg, extra_args = install_args) + return _configure, _build, _install + +def cmake_configure(ctx, extra_args = [], extra_env = {}, host = False): + if host: + env = { + "CFLAGS": options.host_cflags, + "CXXFLAGS": options.host_cxxflags, + "LDFLAGS": options.host_ldflags, + } + toolchain_args = [] + else: + env = { + "CFLAGS": options.cflags, + "CXXFLAGS": options.cxxflags, + "LDFLAGS": options.ldflags, + } + toolchain_args = [ + "-DCMAKE_SYSTEM_NAME=Linux", + "-DCMAKE_SYSTEM_PROCESSOR=" + options.target_arch, + "-DCMAKE_SYSROOT=" + ctx.sysroot, + "-DCMAKE_C_COMPILER=" + options.target_triple + "-gcc", + "-DCMAKE_CXX_COMPILER=" + options.target_triple + "-g++", + "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER", + "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY", + "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY", + "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ONLY", + ] + env.update(extra_env) + ctx.run([ + "cmake", + "-S", ctx.source_dir, + "-B", ctx.build_dir, + "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_INSTALL_PREFIX=" + ctx.prefix, + "-DCMAKE_INSTALL_SYSCONFDIR=/etc", + "-DCMAKE_INSTALL_LOCALSTATEDIR=/var", + ] + toolchain_args + extra_args, env = env) + +def cmake_build(ctx, extra_args = []): + ctx.run(["cmake", "--build", ctx.build_dir, "-j", str(ctx.jobs)] + extra_args) + +def cmake_install(ctx, pkg, extra_args = []): + ctx.run( + ["cmake", "--install", ctx.build_dir] + extra_args, + env = {"DESTDIR": pkg.dest_dir}, + ) + +def cmake(configure_args = [], configure_env = {}, build_args = [], install_args = [], host = False): + def _configure(ctx): + cmake_configure(ctx, extra_args = configure_args, extra_env = configure_env, host = host) + def _build(ctx): + cmake_build(ctx, extra_args = build_args) + def _install(ctx, pkg): + cmake_install(ctx, pkg, extra_args = install_args) + return _configure, _build, _install diff --git a/recipes/bash.star b/recipes/bash.star new file mode 100644 index 0000000..c359607 --- /dev/null +++ b/recipes/bash.star @@ -0,0 +1,22 @@ +version = "5.2.32" +revision = 1 +metadata = meta( + description = "GNU Bourne-Again SHell", + license = "GPL-3.0-or-later", + website = "https://www.gnu.org/software/bash/", +) +source = tarball_source( + url = f"https://ftp.gnu.org/gnu/bash/bash-{version}.tar.gz", + sha256 = "d3ef80d2b67d8cbbe4d3265c63a72c46f9b278ead6e0e06d61801b58f23f50b5", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] +deps = ["ncurses", "readline"] + +configure, build, install = autotools(configure_args = [ + "--without-bash-malloc", + "--with-installed-readline", + "--enable-readline", + "--enable-history", + "--enable-job-control", +]) diff --git a/recipes/bzip2.star b/recipes/bzip2.star new file mode 100644 index 0000000..66621f5 --- /dev/null +++ b/recipes/bzip2.star @@ -0,0 +1,39 @@ +version = "1.0.8" +revision = 1 +metadata = meta( + description = "Block-sorting file compressor", + license = "bzip2-1.0.6", + website = "https://sourceware.org/bzip2/", +) +source = tarball_source( + url = f"https://sourceware.org/pub/bzip2/bzip2-{version}.tar.gz", + sha256 = "?", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] + +# bzip2 ships only a plain Makefile, no configure script. + +def build(ctx): + # Copy sources into the build dir so the in-tree Makefile can write here. + ctx.run(["cp", "-rp", ctx.source_dir / ".", ctx.build_dir]) + jobs = "-j" + str(ctx.jobs) + common = [ + "CC=" + options.target_triple + "-gcc", + "AR=" + options.target_triple + "-ar", + "RANLIB=" + options.target_triple + "-ranlib", + "CFLAGS=" + options.cflags + " -D_FILE_OFFSET_BITS=64", + ] + ctx.run(["make", jobs, "-f", "Makefile-libbz2_so"] + common) + ctx.run(["make", jobs, "libbz2.a", "bzip2", "bzip2recover"] + common) + +def install(ctx, pkg): + ctx.run(["make", "install", "PREFIX=" + pkg.dest_dir + ctx.prefix]) + # Install the shared library that the auxiliary Makefile produced. + libdir = pkg.dest_dir + ctx.prefix / "lib" + bindir = pkg.dest_dir + ctx.prefix / "bin" + ctx.run(["mkdir", "-p", libdir, bindir]) + ctx.run(["cp", "-a", "libbz2.so." + version, libdir]) + ctx.run(["ln", "-sf", "libbz2.so." + version, libdir + "/libbz2.so.1.0"]) + ctx.run(["ln", "-sf", "libbz2.so." + version, libdir + "/libbz2.so.1"]) + ctx.run(["ln", "-sf", "libbz2.so." + version, libdir + "/libbz2.so"]) diff --git a/recipes/coreutils.star b/recipes/coreutils.star new file mode 100644 index 0000000..ff8ee26 --- /dev/null +++ b/recipes/coreutils.star @@ -0,0 +1,25 @@ +version = "9.6" +revision = 1 +metadata = meta( + description = "GNU core utilities (file, shell, and text manipulation)", + license = "GPL-3.0-or-later", + website = "https://www.gnu.org/software/coreutils/", +) +source = tarball_source( + url = f"https://ftp.gnu.org/gnu/coreutils/coreutils-{version}.tar.xz", + sha256 = "?", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] +deps = [options.libc] + +configure, build, install = autotools( + configure_args = [ + "--enable-no-install-program=kill,uptime", + "--without-selinux", + "--without-openssl", + ], + # coreutils' configure runs link tests that require a working executable; + # cross builds need this hint to skip a known false positive. + configure_env = {"FORCE_UNSAFE_CONFIGURE": "1"}, +) diff --git a/recipes/glibc.star b/recipes/glibc.star new file mode 100644 index 0000000..ee7e070 --- /dev/null +++ b/recipes/glibc.star @@ -0,0 +1,47 @@ +version = "2.41" +revision = 1 +metadata = meta( + description = "GNU C library", + license = "LGPL-2.1-or-later", + website = "https://www.gnu.org/software/libc/", +) +source = tarball_source( + url = f"https://ftp.gnu.org/gnu/glibc/glibc-{version}.tar.xz", + sha256 = "a5a26b22f545d6b7d7b3dd828e11e428f24f4fac43c934fb071b6a7d0828e901", + strip_components = 1, +) +host_deps = ["binutils", "gcc-bootstrap"] +deps = ["linux-headers"] + +build_if = options.libc == "glibc" + +def configure(ctx): + ctx.run([ + ctx.source_dir / "configure", + "--host=" + options.target_triple, + "--build=" + options.target_triple, + "--prefix=/usr", + "--with-headers=" + ctx.sysroot / "usr/include", + "--enable-kernel=5.4", + "--enable-bind-now", + "--enable-stack-protector=strong", + "--enable-cet", + "--disable-werror", + "--disable-profile", + "--disable-nscd", + "--without-selinux", + "--without-gd", + "libc_cv_forced_unwind=yes", + "libc_cv_ssp=no", + "libc_cv_ssp_strong=yes", + ], env = { + "CC": options.target_triple + "-gcc", + "CXX": options.target_triple + "-g++", + "CFLAGS": options.cflags, + }) + +def build(ctx): + ctx.run(["make", "-j" + str(ctx.jobs)]) + +def install(ctx, pkg): + ctx.run(["make", "install", "DESTDIR=" + pkg.dest_dir]) diff --git a/recipes/limine.star b/recipes/limine.star index d769e0a..de3743a 100644 --- a/recipes/limine.star +++ b/recipes/limine.star @@ -3,6 +3,7 @@ revision = 1 metadata = meta( description = "Modern, secure, portable, multiprotocol bootloader and boot manager", license = "BSD-2-Clause", + website = "https://limine-bootloader.org" ) source = tarball_source( url = f"https://github.com/Limine-Bootloader/Limine/releases/download/v{version}/limine-{version}.tar.gz", @@ -11,11 +12,8 @@ source = tarball_source( ) host_deps = ["binutils", "gcc"] deps = [options.libc] -subpackages = [ - subpackage( - name = "limine-bios", - ), -] + +build_if = options.target_arch in ["x86_64", "aarch64", "riscv64", "loongarch64"] configure, build, install = autotools(configure_env = { "TOOLCHAIN_FOR_TARGET": options.target_triple + "-", diff --git a/recipes/linux-headers.star b/recipes/linux-headers.star index 04d83db..d764536 100644 --- a/recipes/linux-headers.star +++ b/recipes/linux-headers.star @@ -11,10 +11,10 @@ source = tarball_source( ) def build(ctx): - ctx.run(["cp", "-rp", ctx.source_dir + "/.", ctx.build_dir]) + ctx.run(["cp", "-rp", ctx.source_dir / ".", ctx.build_dir]) ctx.run(["make", "headers_install", "ARCH=" + options.target_arch]) - ctx.run(["find", ctx.build_dir + "/usr/include", "-type", "f", "!", "-name", "*.h", "-delete"]) + ctx.run(["find", ctx.build_dir / "usr/include", "-type", "f", "!", "-name", "*.h", "-delete"]) def install(ctx, pkg): ctx.run(["mkdir", "-p", pkg.dest_dir + ctx.prefix]) - ctx.run(["cp", "-rp", ctx.build_dir + "/usr/include", pkg.dest_dir + ctx.prefix]) + ctx.run(["cp", "-rp", ctx.build_dir / "usr/include", pkg.dest_dir / ctx.prefix]) diff --git a/recipes/linux.star.old b/recipes/linux.star.old deleted file mode 100644 index f1c3758..0000000 --- a/recipes/linux.star.old +++ /dev/null @@ -1,37 +0,0 @@ -name = "linux" -version = "7.0.9" -revision = 1 -description = "Linux kernel" -license = "GPL-2.0-only" - -source = { - "url": f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz", - "sha256": "ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4", - "strip_components": 1, -} - -host_deps = ["binutils", "gcc"] - -def _make_args(ctx, *args): - result = [ - "make", - "-C", ctx.source_dir, - "O=" + ctx.build_dir, - "ARCH=x86_64", - "CROSS_COMPILE=" + OPTIONS.target_triple + "-", - "-j" + str(ctx.jobs), - ] - result.extend(args) - return result - -def configure(ctx): - ctx.run(_make_args(ctx, "defconfig")) - -def build(ctx): - ctx.run(_make_args(ctx, "bzImage")) - -def install(ctx, pkg): - ctx.install( - ctx.build_dir + "/arch/x86/boot/bzImage", - pkg.destdir + "/boot/vmlinuz-" + version, - ) diff --git a/recipes/linux/files/.config b/recipes/linux/files/.config new file mode 100644 index 0000000..4640904 --- /dev/null +++ b/recipes/linux/files/.config @@ -0,0 +1 @@ +# TODO diff --git a/recipes/linux/recipe.star b/recipes/linux/recipe.star new file mode 100644 index 0000000..a6dbc81 --- /dev/null +++ b/recipes/linux/recipe.star @@ -0,0 +1,44 @@ +version = "7.0.9" +revision = 1 +metadata = meta( + description = "Linux kernel", + license = "GPL-2.0-only", +) + +source = tarball_source( + url = f"https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-{version}.tar.xz", + sha256 = "ac07acdf76cf4621cc5187a2670270a1a699533c8a6b225e4878c416ad83f1c4", + strip_components = 1, +) + +host_deps = ["binutils", "gcc"] + +def _make_args(ctx, *args): + # Translate arch name + if options.target_arch == "aarch64": + linux_arch = "arm64" + else: + linux_arch = options.target_arch + + result = [ + "make", + "-C", ctx.source_dir, + "O=" + ctx.build_dir, + "ARCH=" + linux_arch, + "CROSS_COMPILE=" + options.target_triple + "-", + "-j" + str(ctx.jobs), + ] + result.extend(args) + return result + +def configure(ctx): + ctx.run(_make_args(ctx, "defconfig")) + +def build(ctx): + ctx.run(_make_args(ctx, "bzImage")) + +def install(ctx, pkg): + ctx.install( + ctx.build_dir + "/arch/x86/boot/bzImage", + pkg.destdir + "/boot/vmlinuz-" + version, + ) diff --git a/recipes/make.star b/recipes/make.star new file mode 100644 index 0000000..52a8ae9 --- /dev/null +++ b/recipes/make.star @@ -0,0 +1,17 @@ +version = "4.4.1" +revision = 1 +metadata = meta( + description = "GNU make build automation tool", + license = "GPL-3.0-or-later", + website = "https://www.gnu.org/software/make/", +) +source = tarball_source( + url = f"https://ftp.gnu.org/gnu/make/make-{version}.tar.gz", + sha256 = "?", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] + +configure, build, install = autotools(configure_args = [ + "--without-guile", +]) diff --git a/recipes/musl.star b/recipes/musl.star index c796e82..6527b78 100644 --- a/recipes/musl.star +++ b/recipes/musl.star @@ -10,10 +10,11 @@ source = tarball_source( strip_components = 1, ) host_deps = ["binutils", "gcc-bootstrap"] +build_if = options.libc == "musl" def configure(ctx): ctx.run([ - ctx.source_dir + "/configure", + ctx.source_dir / "configure", "--target=" + options.target_triple, "--prefix=" + ctx.prefix, "--syslibdir=/lib", diff --git a/recipes/ncurses.star b/recipes/ncurses.star new file mode 100644 index 0000000..d035409 --- /dev/null +++ b/recipes/ncurses.star @@ -0,0 +1,33 @@ +version = "6.5" +revision = 1 +metadata = meta( + description = "Terminal control library with wide-character support", + license = "MIT", + website = "https://invisible-island.net/ncurses/", +) +source = tarball_source( + url = f"https://invisible-mirror.net/archives/ncurses/ncurses-{version}.tar.gz", + sha256 = "136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] +deps = [options.libc] + +configure, build, install = autotools(configure_args = [ + "--with-shared", + "--without-debug", + "--without-ada", + "--enable-pc-files", + "--enable-widec", + "--with-termlib", + "--with-cxx-binding", + "--with-cxx-shared", + "--with-pkg-config-libdir=/usr/lib/pkgconfig", + "--mandir=/usr/share/man", +], configure_env = { + # Conflicts with GCC 16 headers + "cf_cv_type_of_bool": "bool", + "cf_cv_cc_bool_type": "1", + "cf_cv_builtin_bool": "1", + "ac_cv_header_stdbool_h": "yes", +}) diff --git a/recipes/openssl.star b/recipes/openssl.star new file mode 100644 index 0000000..4aac9f3 --- /dev/null +++ b/recipes/openssl.star @@ -0,0 +1,51 @@ +version = "3.4.1" +revision = 1 +metadata = meta( + description = "Cryptography and TLS library (OpenSSL)", + license = "Apache-2.0", + website = "https://www.openssl.org/", +) +source = tarball_source( + url = f"https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz", + sha256 = "?", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] +deps = ["zlib"] + +def configure(ctx): + # OpenSSL uses its own perl-based Configure script. The first argument is + # the OpenSSL "target" — pick the one matching our triple. + if options.target_arch == "x86_64": + ossl_target = "linux-x86_64" + elif options.target_arch == "aarch64": + ossl_target = "linux-aarch64" + elif options.target_arch == "riscv64": + ossl_target = "linux64-riscv64" + else: + fail("openssl: unsupported target_arch " + options.target_arch) + + ctx.run([ + ctx.source_dir / "Configure", + ossl_target, + "--prefix=" + ctx.prefix, + "--openssldir=/etc/ssl", + "--libdir=lib", + "shared", + "zlib", + "no-tests", + "no-static-engine", + "enable-ktls", + ], env = { + "CC": options.target_triple + "-gcc", + "AR": options.target_triple + "-ar", + "RANLIB": options.target_triple + "-ranlib", + "CFLAGS": options.cflags, + "LDFLAGS": options.ldflags, + }) + +def build(ctx): + ctx.run(["make", "-j" + str(ctx.jobs)]) + +def install(ctx, pkg): + ctx.run(["make", "install_sw", "install_ssldirs"], env = {"DESTDIR": pkg.dest_dir}) diff --git a/recipes/pkgconf.star b/recipes/pkgconf.star new file mode 100644 index 0000000..c5176f7 --- /dev/null +++ b/recipes/pkgconf.star @@ -0,0 +1,18 @@ +version = "3.3.3" +revision = 1 +metadata = meta( + description = "Lightweight pkg-config implementation", + license = "ISC", + website = "http://pkgconf.org/", +) +source = tarball_source( + url = f"https://distfiles.ariadne.space/pkgconf/pkgconf-{version}.tar.xz", + sha256 = "?", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] + +configure, build, install = autotools(configure_args = [ + "--with-system-libdir=/usr/lib", + "--with-system-includedir=/usr/include", +]) diff --git a/recipes/readline.star b/recipes/readline.star new file mode 100644 index 0000000..e0b9ea0 --- /dev/null +++ b/recipes/readline.star @@ -0,0 +1,23 @@ +version = "8.2" +revision = 1 +metadata = meta( + description = "Library for command-line editing", + license = "GPL-3.0-or-later", + website = "https://tiswww.case.edu/php/chet/readline/rltop.html", +) +source = tarball_source( + url = f"https://ftp.gnu.org/gnu/readline/readline-{version}.tar.gz", + sha256 = "3feb7171f16a84ee82ca18a36d7b9be109a52c04f492a053331d7d1095007c35", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] +deps = ["ncurses"] + +configure, build, install = autotools( + configure_args = ["--with-curses"], + configure_env = { + # Force linking against the system curses; otherwise readline's + # configure may pick a static libtermcap stub it ships internally. + "bash_cv_termcap_lib": "ncursesw", + }, +) diff --git a/recipes/xz.star b/recipes/xz.star new file mode 100644 index 0000000..8ea3fec --- /dev/null +++ b/recipes/xz.star @@ -0,0 +1,17 @@ +version = "5.6.3" +revision = 1 +metadata = meta( + description = "XZ Utils — LZMA/XZ compression tools and library", + license = "0BSD AND GPL-2.0-or-later AND LGPL-2.1-or-later", + website = "https://tukaani.org/xz/", +) +source = tarball_source( + url = f"https://github.com/tukaani-project/xz/releases/download/v{version}/xz-{version}.tar.xz", + sha256 = "?", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] + +configure, build, install = autotools(configure_args = [ + "--disable-doc", +]) diff --git a/recipes/zlib.star b/recipes/zlib.star new file mode 100644 index 0000000..80e03a2 --- /dev/null +++ b/recipes/zlib.star @@ -0,0 +1,35 @@ +version = "1.3.1" +revision = 1 +metadata = meta( + description = "Lossless data-compression library", + license = "Zlib", + website = "https://zlib.net/", +) +source = tarball_source( + url = f"https://zlib.net/zlib-{version}.tar.xz", + sha256 = "?", + strip_components = 1, +) +host_deps = ["binutils", "gcc"] + +def configure(ctx): + # zlib ships its own ./configure that does not understand the usual + # autoconf flags (no --host, --build, etc.), so it is invoked directly. + ctx.run([ + ctx.source_dir / "configure", + "--prefix=" + ctx.prefix, + "--libdir=" + ctx.prefix / "lib", + "--sharedlibdir=" + ctx.prefix / "lib", + ], env = { + "CC": options.target_triple + "-gcc", + "AR": options.target_triple + "-ar", + "RANLIB": options.target_triple + "-ranlib", + "CFLAGS": options.cflags, + "LDFLAGS": options.ldflags, + }) + +def build(ctx): + ctx.run(["make", "-j" + str(ctx.jobs)]) + +def install(ctx, pkg): + ctx.run(["make", "install"], env = {"DESTDIR": pkg.dest_dir}) diff --git a/src/builder.rs b/src/builder.rs index 2354420..fff3258 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -49,8 +49,9 @@ impl Builder { rebuild: bool, dry_run: bool, ) -> anyhow::Result<()> { + let requested = filter_skipped(recipes, requested); let plan = TaskPlanner::new(&self.root, &self.config.arch, recipes) - .build_plan(requested, rebuild)?; + .build_plan(&requested, rebuild)?; self.print_plan(&plan); if dry_run { return Ok(()); @@ -64,8 +65,9 @@ impl Builder { requested: &[String], dry_run: bool, ) -> anyhow::Result<()> { + let requested = filter_skipped(recipes, requested); let plan = - TaskPlanner::new(&self.root, &self.config.arch, recipes).fetch_plan(requested)?; + TaskPlanner::new(&self.root, &self.config.arch, recipes).fetch_plan(&requested)?; self.print_plan(&plan); if dry_run { return Ok(()); @@ -424,11 +426,7 @@ impl Builder { ) -> anyhow::Result<()> { log::step("configure", &recipe.key()); if let Some(func) = recipe.phases().configure() { - let ctx = PhaseContext::new( - source_dir_for(recipe), - prefix_for(recipe.kind()), - default_jobs(), - ); + let ctx = phase_context_for(recipe); self.invoke_with_runtime(active, &[PhaseArg::Ctx(ctx)], func)?; } self.write_recipe_stamp(layout, recipe, "configure") @@ -441,11 +439,7 @@ impl Builder { active: &ActiveContainer, ) -> anyhow::Result<()> { log::step("build", &recipe.key()); - let ctx = PhaseContext::new( - source_dir_for(recipe), - prefix_for(recipe.kind()), - default_jobs(), - ); + let ctx = phase_context_for(recipe); self.invoke_with_runtime(active, &[PhaseArg::Ctx(ctx)], recipe.phases().build())?; self.write_recipe_stamp(layout, recipe, "build") } @@ -464,11 +458,7 @@ impl Builder { &base_env(&active.base_path), "/", )?; - let ctx = PhaseContext::new( - source_dir_for(recipe), - prefix_for(recipe.kind()), - default_jobs(), - ); + let ctx = phase_context_for(recipe); let pkg = PackageContext::new(dest); self.invoke_with_runtime( active, @@ -491,11 +481,7 @@ impl Builder { &base_env(&active.base_path), "/", )?; - let ctx = PhaseContext::new( - source_dir_for(recipe), - prefix_for(recipe.kind()), - default_jobs(), - ); + let ctx = phase_context_for(recipe); let pkg = PackageContext::new(dest.clone()); self.invoke_with_runtime( active, @@ -595,6 +581,61 @@ impl Builder { phase::invoke_phase(func, args) } + fn populate_sysroot( + &self, + layout: &Layout<'_>, + recipes: &RecipeSet, + recipe: &Recipe, + active: &ActiveContainer, + deps: &[String], + ) -> anyhow::Result<()> { + if deps.is_empty() { + return Ok(()); + } + + let env = base_env(&active.base_path); + for dep in deps { + let output = recipes.output(dep)?; + let owning = recipes.recipe(output.recipe())?; + let apk_host = layout.apk_path(owning, output); + if !apk_host.exists() { + bail!( + "missing apk for target dependency `{dep}` at {}; \ + rebuild it first (e.g. `distro build -r {dep}`)", + apk_host.display() + ); + } + let file_name = apk_host + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| { + anyhow::anyhow!("apk path {} has no UTF-8 file name", apk_host.display()) + })?; + active.container.borrow().exec( + &[ + "apk".to_owned(), + "extract".to_owned(), + "--allow-untrusted".to_owned(), + "--destination".to_owned(), + "/sysroot".to_owned(), + format!("/pkgs/{file_name}"), + ], + &env, + "/", + )?; + } + + log::info( + "sysroot", + &format!( + "{}: extracted /sysroot from {} target apk(s)", + recipe.key(), + deps.len() + ), + ); + Ok(()) + } + fn start_recipe_container( &self, recipes: &RecipeSet, @@ -609,8 +650,10 @@ impl Builder { fs::create_dir_all(&build_dir)?; let host_deps = transitive_host_deps(recipes, recipe)?; + let target_deps = transitive_target_deps(recipes, recipe)?; let pkgs_dir = self.root.join("build/pkgs").join(&self.config.arch); fs::create_dir_all(&pkgs_dir)?; + let mut mounts = vec![ Mount { host: source_dir, @@ -628,6 +671,15 @@ impl Builder { read_only: false, }, ]; + + if let Some(files_dir) = recipe.files_dir() { + mounts.push(Mount { + host: files_dir, + container: "/files".to_owned(), + read_only: true, + }); + } + let mut tools_bins: Vec = Vec::new(); for dep_key in &host_deps { let dep_recipe = recipes.recipe(dep_key)?; @@ -674,6 +726,8 @@ impl Builder { base_path, }; + self.populate_sysroot(&layout, recipes, recipe, &active, &target_deps)?; + Ok(active) } @@ -817,6 +871,21 @@ struct ActiveContainer { base_path: String, } +fn filter_skipped(recipes: &RecipeSet, requested: &[String]) -> Vec { + requested + .iter() + .filter(|key| { + if recipes.is_skipped(key) { + log::skip("skip", &format!("{key} (build_if returned false)")); + false + } else { + true + } + }) + .cloned() + .collect() +} + fn task_needs_container(task: &TaskId) -> bool { matches!( task, @@ -835,6 +904,16 @@ fn prefix_for(kind: RecipeKind) -> &'static str { } } +fn phase_context_for(recipe: &Recipe) -> PhaseContext { + let files = recipe.files_dir().map(|_| "/files".to_owned()); + PhaseContext::new( + source_dir_for(recipe), + prefix_for(recipe.kind()), + default_jobs(), + files, + ) +} + fn source_dir_for(recipe: &Recipe) -> SourceDir { let entries = recipe.sources().entries(); let named: Vec<(&str, &crate::recipe::Source)> = entries @@ -896,6 +975,34 @@ fn transitive_host_deps(recipes: &RecipeSet, recipe: &Recipe) -> anyhow::Result< Ok(order) } +/// Compute the transitive closure of a recipe's target package dependencies. +/// The recipe's own `build_deps` and `deps` seed the queue; for each visited +/// dependency we follow its `deps` (runtime+build-needed) recursively, but +/// not its `build_deps` (build-time-only relative to *that* package, hence +/// not propagated outward). +fn transitive_target_deps(recipes: &RecipeSet, recipe: &Recipe) -> anyhow::Result> { + let mut order: Vec = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + let mut queue: VecDeque = recipe + .build_deps() + .iter() + .chain(recipe.deps().iter()) + .cloned() + .collect(); + while let Some(dep) = queue.pop_front() { + if !seen.insert(dep.clone()) { + continue; + } + let output = recipes.output(&dep)?; + let owning = recipes.recipe(output.recipe())?; + for sub in owning.deps() { + queue.push_back(sub.clone()); + } + order.push(dep); + } + Ok(order) +} + fn random_suffix() -> u64 { let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/src/container.rs b/src/container.rs index 6af66b4..170d749 100644 --- a/src/container.rs +++ b/src/container.rs @@ -123,29 +123,6 @@ impl Container { } Ok(()) } - - pub fn stop(mut self) -> anyhow::Result<()> { - self.stop_inner() - } - - fn stop_inner(&mut self) -> anyhow::Result<()> { - if self.stopped { - return Ok(()); - } - self.stopped = true; - let status = Command::new(self.runtime) - .arg("rm") - .arg("-f") - .arg(&self.id) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .with_context(|| format!("spawning `{} rm`", self.runtime))?; - if !status.success() { - bail!("`{} rm -f {}` failed with {status}", self.runtime, self.id); - } - Ok(()) - } } impl Drop for Container { diff --git a/src/eval.rs b/src/eval.rs index 0e10148..2fb1c1f 100644 --- a/src/eval.rs +++ b/src/eval.rs @@ -10,6 +10,7 @@ use walkdir::WalkDir; use crate::{ options::Options, + phase::Path as StarPath, recipe::{GitSource, Metadata, Source, Subpackage, TarballSource}, }; @@ -45,16 +46,26 @@ fn builder_globals(builder: &mut GlobalsBuilder) { url: String, sha256: String, strip_components: Option, + patches: Option>, ) -> anyhow::Result { Ok(Source::Tarball(TarballSource::new( url, sha256, strip_components.unwrap_or(0), + patches.map(|p| p.items).unwrap_or_default(), ))) } - fn git_source(url: String, commit: String) -> anyhow::Result { - Ok(Source::Git(GitSource::new(url, commit))) + fn git_source( + url: String, + commit: String, + patches: Option>, + ) -> anyhow::Result { + Ok(Source::Git(GitSource::new( + url, + commit, + patches.map(|p| p.items).unwrap_or_default(), + ))) } fn subpackage(name: String, metadata: Option<&Metadata>) -> anyhow::Result { @@ -63,6 +74,13 @@ fn builder_globals(builder: &mut GlobalsBuilder) { .unwrap_or_else(|| Metadata::new(None, None, None, None)); Ok(Subpackage::new(name, metadata)) } + + /// Wrap a string as a `path`. Paths support the `/` operator for + /// meson-style joining (`path("/usr") / "lib" / "pkgconfig"`), with + /// the same semantics as `pathlib.PurePosixPath`. + fn path(value: String) -> anyhow::Result { + Ok(StarPath::new(value)) + } } pub fn eval_file( diff --git a/src/graph.rs b/src/graph.rs index 5f03611..2f4cb86 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -54,17 +54,12 @@ impl TaskPlan { pub fn dependency_count(&self) -> usize { self.dependencies.values().map(Vec::len).sum() } - - #[cfg(test)] - pub fn dependencies(&self, task: &TaskId) -> Option<&[TaskId]> { - self.dependencies.get(task).map(Vec::as_slice) - } } pub struct TaskPlanner<'a> { layout: Layout<'a>, recipes: &'a RecipeSet, - force: bool, + forced_recipes: BTreeSet, dependencies: BTreeMap>, inactive: BTreeSet, visiting: BTreeSet, @@ -76,7 +71,7 @@ impl<'a> TaskPlanner<'a> { Self { layout: Layout::new(root, arch), recipes, - force: false, + forced_recipes: BTreeSet::new(), dependencies: BTreeMap::new(), inactive: BTreeSet::new(), visiting: BTreeSet::new(), @@ -85,9 +80,11 @@ impl<'a> TaskPlanner<'a> { } pub fn build_plan(mut self, requests: &[String], force: bool) -> anyhow::Result { - self.force = force; for request in requests { let recipe = self.recipes.recipe(request)?; + if force { + self.forced_recipes.insert(recipe.key()); + } match recipe.kind() { RecipeKind::Package => { for output in recipe.outputs() { @@ -229,8 +226,12 @@ impl<'a> TaskPlanner<'a> { })) } + fn is_recipe_forced(&self, recipe: &Recipe) -> bool { + self.forced_recipes.contains(&recipe.key()) + } + fn prepare_sources_active(&self, recipe: &Recipe) -> anyhow::Result { - if self.force { + if self.is_recipe_forced(recipe) { return Ok(true); } let want_version = format!("{}-r{}", recipe.version(), recipe.revision()); @@ -250,7 +251,7 @@ impl<'a> TaskPlanner<'a> { } fn recipe_task_active(&self, recipe: &Recipe, kind: &str) -> anyhow::Result { - if self.force { + if self.is_recipe_forced(recipe) { return Ok(true); } Ok( @@ -267,7 +268,7 @@ impl<'a> TaskPlanner<'a> { output: &OutputPackage, kind: &str, ) -> anyhow::Result { - if self.force { + if self.is_recipe_forced(recipe) { return Ok(true); } Ok( @@ -279,7 +280,7 @@ impl<'a> TaskPlanner<'a> { } fn produce_apk_active(&self, recipe: &Recipe, output: &OutputPackage) -> anyhow::Result { - if self.force { + if self.is_recipe_forced(recipe) { return Ok(true); } if !self.layout.apk_path(recipe, output).exists() { @@ -294,7 +295,7 @@ impl<'a> TaskPlanner<'a> { } fn install_host_recipe_active(&self, recipe: &Recipe) -> anyhow::Result { - if self.force { + if self.is_recipe_forced(recipe) { return Ok(true); } if !self.layout.host_install_dir(recipe).exists() { diff --git a/src/layout.rs b/src/layout.rs index 35524b8..e95f09e 100644 --- a/src/layout.rs +++ b/src/layout.rs @@ -111,21 +111,16 @@ impl<'a> Layout<'a> { } pub fn recipe_patches(&self, recipe: &Recipe) -> anyhow::Result> { - let patches_dir = recipe.dir().join("patches"); - if !patches_dir.exists() { + let Some(data_dir) = recipe.data_dir() else { return Ok(Vec::new()); - } - let mut patches = Vec::new(); - for entry in fs::read_dir(&patches_dir) - .with_context(|| format!("reading patches directory {}", patches_dir.display()))? - { - let entry = entry?; - let path = entry.path(); - if path.is_file() { - patches.push(path); + }; + let patches_dir = data_dir.join("patches"); + let mut out = Vec::new(); + for (_, source) in recipe.sources().entries() { + for name in source.patches() { + out.push(patches_dir.join(name)); } } - patches.sort(); - Ok(patches) + Ok(out) } } diff --git a/src/phase.rs b/src/phase.rs index 1544c1a..1d29870 100644 --- a/src/phase.rs +++ b/src/phase.rs @@ -6,7 +6,9 @@ use starlark::{ collections::SmallMap, environment::{Methods, MethodsBuilder, MethodsStatic, Module}, eval::Evaluator, - values::{Heap, OwnedFrozenValue, StarlarkValue, Value, list::UnpackList, none::NoneType}, + values::{ + Heap, OwnedFrozenValue, StarlarkValue, Value, ValueLike, list::UnpackList, none::NoneType, + }, }; use starlark_derive::{NoSerialize, ProvidesStaticType, starlark_module, starlark_value}; @@ -53,6 +55,81 @@ fn with_current(f: impl FnOnce(&PhaseRuntime) -> R) -> anyhow::Result { }) } +#[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] +pub struct Path { + inner: String, +} + +impl Path { + pub fn new(inner: impl Into) -> Self { + Self { + inner: inner.into(), + } + } + + pub fn as_str(&self) -> &str { + &self.inner + } +} + +fn join_paths(base: &str, rhs: &str) -> String { + if rhs.starts_with('/') { + return rhs.to_owned(); + } + let trimmed = base.trim_end_matches('/'); + if trimmed.is_empty() { + format!("/{rhs}") + } else { + format!("{trimmed}/{rhs}") + } +} + +fn coerce_path_string(value: Value<'_>) -> anyhow::Result { + if let Some(s) = value.unpack_str() { + return Ok(s.to_owned()); + } + if let Some(p) = value.downcast_ref::() { + return Ok(p.inner.clone()); + } + if let Some(s) = value.downcast_ref::() { + return Ok(s.default.clone()); + } + anyhow::bail!("expected a string or path, got `{}`", value.get_type()) +} + +impl std::fmt::Display for Path { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.inner) + } +} + +starlark::starlark_simple_value!(Path); + +#[starlark_value(type = "path")] +impl<'v> StarlarkValue<'v> for Path { + fn div(&self, other: Value<'v>, heap: &'v Heap) -> starlark::Result> { + let rhs = coerce_path_string(other).map_err(starlark::Error::new_other)?; + Ok(heap.alloc(Path::new(join_paths(&self.inner, &rhs)))) + } + + fn add(&self, rhs: Value<'v>, heap: &'v Heap) -> Option>> { + let suffix = rhs.unpack_str()?; + Some(Ok(heap.alloc(format!("{}{}", self.inner, suffix)))) + } + + fn radd(&self, lhs: Value<'v>, heap: &'v Heap) -> Option>> { + let prefix = lhs.unpack_str()?; + Some(Ok(heap.alloc(format!("{}{}", prefix, self.inner)))) + } + + fn equals(&self, other: Value<'v>) -> starlark::Result { + Ok(other + .downcast_ref::() + .map(|o| o.inner == self.inner) + .unwrap_or(false)) + } +} + #[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] pub struct SourceDir { default: String, @@ -126,6 +203,11 @@ impl<'v> StarlarkValue<'v> for SourceDir { let prefix = lhs.unpack_str()?; Some(Ok(heap.alloc(format!("{}{}", prefix, self.default)))) } + + fn div(&self, other: Value<'v>, heap: &'v Heap) -> starlark::Result> { + let rhs = coerce_path_string(other).map_err(starlark::Error::new_other)?; + Ok(heap.alloc(Path::new(join_paths(&self.default, &rhs)))) + } } #[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] @@ -134,16 +216,18 @@ pub struct PhaseContext { build_dir: String, prefix: String, sysroot: String, + files: Option, jobs: i32, } impl PhaseContext { - pub fn new(source_dir: SourceDir, prefix: &str, jobs: i32) -> Self { + pub fn new(source_dir: SourceDir, prefix: &str, jobs: i32, files: Option) -> Self { Self { source_dir, build_dir: "/build".to_owned(), prefix: prefix.to_owned(), sysroot: "/sysroot".to_owned(), + files, jobs, } } @@ -162,33 +246,36 @@ impl<'v> StarlarkValue<'v> for PhaseContext { fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option> { Some(match attr { "source_dir" => heap.alloc(self.source_dir.clone()), - "build_dir" => heap.alloc(self.build_dir.as_str()), - "prefix" => heap.alloc(self.prefix.as_str()), - "sysroot" => heap.alloc(self.sysroot.as_str()), + "build_dir" => heap.alloc(Path::new(self.build_dir.clone())), + "prefix" => heap.alloc(Path::new(self.prefix.clone())), + "sysroot" => heap.alloc(Path::new(self.sysroot.clone())), + "files" => heap.alloc(Path::new(self.files.as_ref()?.clone())), "jobs" => heap.alloc(self.jobs), _ => return None, }) } fn has_attr(&self, attr: &str, _heap: &'v Heap) -> bool { - matches!( - attr, - "source_dir" | "build_dir" | "prefix" | "sysroot" | "jobs" | "run" - ) + match attr { + "source_dir" | "build_dir" | "prefix" | "sysroot" | "jobs" | "run" => true, + "files" => self.files.is_some(), + _ => false, + } } fn dir_attr(&self) -> Vec { - [ - "source_dir", - "build_dir", - "prefix", - "sysroot", - "jobs", - "run", - ] - .into_iter() - .map(String::from) - .collect() + let mut attrs = vec![ + "source_dir".to_owned(), + "build_dir".to_owned(), + "prefix".to_owned(), + "sysroot".to_owned(), + "jobs".to_owned(), + "run".to_owned(), + ]; + if self.files.is_some() { + attrs.push("files".to_owned()); + } + attrs } fn get_methods() -> Option<&'static Methods> { @@ -201,10 +288,21 @@ impl<'v> StarlarkValue<'v> for PhaseContext { fn phase_context_methods(builder: &mut MethodsBuilder) { fn run<'v>( #[starlark(this)] _this: Value<'v>, - #[starlark(require = pos)] argv: UnpackList, - #[starlark(require = named)] env: Option>, + #[starlark(require = pos)] argv: UnpackList>, + #[starlark(require = named)] env: Option>>, ) -> anyhow::Result { - run_in_container(&argv.items, env.unwrap_or_default())?; + let argv: Vec = argv + .items + .iter() + .map(|v| coerce_path_string(*v)) + .collect::>>()?; + let mut env_strings: SmallMap = SmallMap::new(); + if let Some(env) = env { + for (k, v) in env { + env_strings.insert(k, coerce_path_string(v)?); + } + } + run_in_container(&argv, env_strings)?; Ok(NoneType) } } @@ -261,7 +359,7 @@ starlark::starlark_simple_value!(PackageContext); impl<'v> StarlarkValue<'v> for PackageContext { fn get_attr(&self, attr: &str, heap: &'v Heap) -> Option> { match attr { - "dest_dir" => Some(heap.alloc(self.dest_dir.as_str())), + "dest_dir" => Some(heap.alloc(Path::new(self.dest_dir.clone()))), _ => None, } } diff --git a/src/recipe/mod.rs b/src/recipe/mod.rs index df20b98..da4b8b9 100644 --- a/src/recipe/mod.rs +++ b/src/recipe/mod.rs @@ -5,13 +5,14 @@ mod subpackage; use anyhow::{Context, bail}; use starlark::{ environment::{FrozenModule, Module}, + eval::Evaluator, values::{ OwnedFrozenValue, UnpackValue, ValueLike, dict::DictRef, list::ListRef, typing::StarlarkCallable, }, }; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, fmt, path::{Path, PathBuf}, }; @@ -19,6 +20,7 @@ use walkdir::WalkDir; use crate::{ eval::{self, ExtractError}, + log, options::Options, }; @@ -123,10 +125,14 @@ impl Recipe { kind: RecipeKind, options: &Options, lib: Option<&FrozenModule>, - ) -> anyhow::Result { + ) -> anyhow::Result> { let module = eval::eval_file(path, Some(options), lib) .with_context(|| format!("evaluating recipe {}", path.display()))?; + if !evaluate_build_if(&module)? { + return Ok(None); + } + let version = eval::extract_string(&module, "version") .map_err(|e| anyhow::anyhow!("field `version`: {e}"))?; @@ -230,7 +236,7 @@ impl Recipe { .map_err(|err| anyhow::anyhow!("freezing recipe module {}: {err:?}", path.display()))?; let phases = RecipePhases::load(&module)?; - Ok(Recipe { + let recipe = Recipe { name: name.to_owned(), path: path.to_path_buf(), kind, @@ -243,7 +249,8 @@ impl Recipe { deps, run_deps, phases, - }) + }; + Ok(Some(recipe)) } pub fn phases(&self) -> &RecipePhases { @@ -258,8 +265,21 @@ impl Recipe { self.kind.slug(&self.name) } - pub fn dir(&self) -> &Path { - self.path.parent().unwrap_or_else(|| Path::new(".")) + pub fn data_dir(&self) -> Option<&Path> { + if self.path.file_name().is_some_and(|n| n == "recipe.star") { + self.path.parent() + } else { + None + } + } + + pub fn files_dir(&self) -> Option { + let candidate = self.data_dir()?.join("files"); + if candidate.is_dir() { + Some(candidate) + } else { + None + } } pub fn path(&self) -> &Path { @@ -311,6 +331,33 @@ fn optional_string_list(module: &Module, key: &str) -> anyhow::Result anyhow::Result { + let Some(value) = module.get("build_if") else { + return Ok(true); + }; + + if let Some(b) = value.unpack_bool() { + return Ok(b); + } + + let callable: Option> = StarlarkCallable::unpack_value_opt(value); + if callable.is_none() { + bail!("field `build_if`: expected a bool or a callable returning a bool"); + } + + let mut eval = Evaluator::new(module); + let result = eval + .eval_function(value, &[], &[]) + .map_err(|err| anyhow::anyhow!("calling `build_if`: {err}"))?; + result + .unpack_bool() + .ok_or_else(|| anyhow::anyhow!("field `build_if`: must return a bool")) +} + fn is_valid_source_name(name: &str) -> bool { let mut chars = name.chars(); let Some(first) = chars.next() else { @@ -425,6 +472,7 @@ impl OutputPackage { pub struct RecipeSet { recipes: HashMap, outputs: HashMap, + skipped: HashSet, } impl RecipeSet { @@ -435,6 +483,7 @@ impl RecipeSet { ) -> anyhow::Result { let mut recipes = HashMap::new(); let mut outputs = HashMap::new(); + let mut skipped: HashSet = HashSet::new(); for (path, kind) in [ ("recipes", RecipeKind::Package), @@ -447,9 +496,14 @@ impl RecipeSet { } for (name, path) in discover_recipes(&recipes_dir)? { - let recipe = Recipe::load(&path, &name, kind, options, lib) - .with_context(|| format!("loading recipe `{name}`"))?; let key = kind.key(&name); + let loaded = Recipe::load(&path, &name, kind, options, lib) + .with_context(|| format!("loading recipe `{name}`"))?; + let Some(recipe) = loaded else { + log::skip("load", &format!("{key} (build_if returned false)")); + skipped.insert(key); + continue; + }; if recipes.insert(key.clone(), recipe).is_some() { bail!("duplicate recipe `{key}`"); @@ -467,7 +521,11 @@ impl RecipeSet { } } - Ok(Self { recipes, outputs }) + Ok(Self { + recipes, + outputs, + skipped, + }) } pub fn recipe(&self, key: &str) -> anyhow::Result<&Recipe> { @@ -481,6 +539,10 @@ impl RecipeSet { .get(key) .ok_or_else(|| anyhow::anyhow!("unknown output package `{key}`")) } + + pub fn is_skipped(&self, key: &str) -> bool { + self.skipped.contains(key) + } } /// Find all recipe `.star` files under `dir`, returning a map of recipe name diff --git a/src/recipe/source.rs b/src/recipe/source.rs index 61d870c..42736c4 100644 --- a/src/recipe/source.rs +++ b/src/recipe/source.rs @@ -7,6 +7,7 @@ pub struct TarballSource { url: String, sha256: String, strip_components: u32, + patches: Vec, } impl std::fmt::Display for TarballSource { @@ -21,11 +22,12 @@ starlark::starlark_simple_value!(TarballSource); impl<'v> StarlarkValue<'v> for TarballSource {} impl TarballSource { - pub fn new(url: String, sha256: String, strip_components: u32) -> Self { + pub fn new(url: String, sha256: String, strip_components: u32, patches: Vec) -> Self { Self { url, sha256, strip_components, + patches, } } @@ -40,12 +42,17 @@ impl TarballSource { pub fn strip_components(&self) -> u32 { self.strip_components } + + pub fn patches(&self) -> &[String] { + &self.patches + } } #[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] pub struct GitSource { url: String, commit: String, + patches: Vec, } impl std::fmt::Display for GitSource { @@ -60,8 +67,12 @@ starlark::starlark_simple_value!(GitSource); impl<'v> StarlarkValue<'v> for GitSource {} impl GitSource { - pub fn new(url: String, commit: String) -> Self { - Self { url, commit } + pub fn new(url: String, commit: String, patches: Vec) -> Self { + Self { + url, + commit, + patches, + } } pub fn url(&self) -> &str { @@ -71,6 +82,10 @@ impl GitSource { pub fn commit(&self) -> &str { &self.commit } + + pub fn patches(&self) -> &[String] { + &self.patches + } } #[derive(Debug, Clone, Allocative, ProvidesStaticType, NoSerialize)] @@ -108,4 +123,11 @@ impl Source { pub fn is_unknown_cache_key(&self) -> bool { matches!(self.cache_key(), "?" | "???") } + + pub fn patches(&self) -> &[String] { + match self { + Self::Tarball(source) => source.patches(), + Self::Git(source) => source.patches(), + } + } }