flua: support our flua modules in the bootstrap flua

This version builds every module into the flua binary itself, since all
of the bootstrap tools are built -DNO_SHARED.  As a result, we also
cannot dlsym(), so we can't really discover the names of our newly
builtin modules.  Instead, just build out a linker set with all of our
luaopen_*() functions to register everything up-front.

Building in all of the modules isn't strictly necessary, but it means
that we have an example of how to add a bootstrap module everywhere you
go and one doesn't need to consider whether bootstrap flua can use a
module when writing scripts.  On my build machine, the consequence on
our binary size is an increase from around 1.6M -> 1.9M, which isn't
really that bad.

.lua modules can install into their usual path below $WORLDTMP/legacy
and we'll pick them up automagically by way of the ctor that sets up
LUA_PATH early on.

This re-lands bootstrap module support with a more sensible subset, and
after having verified that it cross-builds fine on macOS and Linux -- we
cannot do libfreebsd on !FreeBSD because it's more system header
dependant.  We also need to bootstrap libmd to bring in libhash, and
libucl + libyaml.

Reviewed by:	bapt, emaste (both previous version)
Differential Revision:	https://reviews.freebsd.org/D51890

Makefile.inc1

@@ -2548,6 +2548,10 @@ ${_bt}-usr.sbin/kldxref: ${_bt_libelf_depend} ${_bt_libkldelf_depend}
 .if ${BOOTSTRAPPING} < 1300059
 ${_bt}-libexec/flua: ${_bt}-lib/liblua
 _flua= lib/liblua libexec/flua
+.if ${BOOTSTRAPPING} == 0
+_flua+= lib/libmd lib/libucl lib/libyaml
+${_bt}-libexec/flua: ${_bt}-lib/libmd ${_bt}-lib/libucl ${_bt}-lib/libyaml
+.endif # BOOTSTRAPPING == 0
 .endif
 
 # r245440 mtree -N support added

contrib/libucl/lua/lua_ucl.c

@@ -30,6 +30,8 @@
 #include "lua_ucl.h"
 #include <strings.h>
 
+#include "bootstrap.h"
+
 /***
  * @module ucl
  * This lua module allows to parse objects from strings and to store data into
@@ -1571,3 +1573,5 @@ ucl_object_toclosure (const ucl_object_t *obj)
 
 	return (struct ucl_lua_funcdata*)obj->value.ud;
 }
+
+FLUA_MODULE(ucl);

contrib/lyaml/ext/yaml/yaml.c

@@ -35,6 +35,8 @@
 
 #include "lyaml.h"
 
+#include "bootstrap.h"
+
 #define MYNAME		"yaml"
 #define MYVERSION	MYNAME " library for " LUA_VERSION " / " VERSION
 
@@ -64,3 +66,5 @@ luaopen_yaml (lua_State *L)
 
    return 1;
 }
+
+FLUA_MODULE(yaml);

libexec/flua/Makefile

@@ -1,10 +1,40 @@
 .include <src.lua.mk>
 
-SUBDIR+=	libfreebsd
-SUBDIR+=	libhash
-SUBDIR+=	libjail
-SUBDIR+=	libucl
-SUBDIR+=	liblyaml
+# New flua modules should be added here rather than to SUBDIR so that we can do
+# the right thing for both bootstrap flua and target flua.  The former does not
+# do any shared libs, so we just build them all straight into flua itself rather
+# than mucking about with the infrastructure to make them linkable -- thus, why
+# these are all structured to have a Makefile that describes what we want
+# *installed*, and a Makefile.inc that describes what we need to *build*.
+FLUA_MODULES+=	libhash
+.ifndef BOOTSTRAPPING
+# Bootstrap flua can't usefully do anything with libjail anyways, because it
+# can't assume it's being run on a system that even supports jails.
+FLUA_MODULES+=	libjail
+.endif
+FLUA_MODULES+=	libucl
+FLUA_MODULES+=	liblyaml
+
+.ifdef BOOTSTRAPPING
+# libfreebsd is generally omitted from the bootstrap flua because its
+# functionality largely assumes a FreeBSD kernel/system headers, so it doesn't
+# really offer functionality that we can use in bootstrap.
+CFLAGS+=	-I${.CURDIR} -DBOOTSTRAPPING
+
+SHAREDIR=	${WORLDTMP}/legacy/usr/share/flua
+FLUA_PATH=	${SHAREDIR}/?.lua;${SHAREDIR}/?/init.lua
+CFLAGS+=	-DBOOTSTRAP_FLUA_PATH=\"${FLUA_PATH:Q}\"
+
+.for mod in ${FLUA_MODULES}
+.include "${mod}/Makefile.inc"
+.endfor
+
+.else
+
+FLUA_MODULES+=	libfreebsd
+SUBDIR+=	${FLUA_MODULES}
+
+.endif
 
 LUASRC?=	${SRCTOP}/contrib/lua/src
 .PATH: ${LUASRC}
@@ -14,7 +44,7 @@ WARNS?=	3
 
 CWARNFLAGS.gcc+=	-Wno-format-nonliteral
 
-LIBADD=	lua
+LIBADD+=	lua
 
 # Entry point
 SRCS+=	lua.c

libexec/flua/Makefile.inc

@@ -2,4 +2,9 @@ SHLIBDIR?=	${LIBDIR}/flua
 
 CFLAGS+=	\
 		-I${SRCTOP}/contrib/lua/src \
-		-I${SRCTOP}/lib/liblua
+		-I${SRCTOP}/lib/liblua \
+		-I${SRCTOP}/libexec/flua
+
+.ifdef BOOTSTRAPPING
+CFLAGS+=	-DBOOTSTRAPPING
+.endif

libexec/flua/bootstrap.h

@@ -0,0 +1,32 @@
+/*-
+ * Copyright (c) 2025 Kyle Evans <kevans@FreeBSD.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef FLUA_BOOTSTRAP_H
+#define	FLUA_BOOTSTRAP_H
+
+#ifdef BOOTSTRAPPING
+#include <sys/linker_set.h>
+
+#include <lauxlib.h>
+
+#define	FLUA_MODULE_SETNAME	flua_modules
+
+SET_DECLARE(FLUA_MODULE_SETNAME, const luaL_Reg);
+#define	FLUA_MODULE_DEF(ident, modname, openfn)			\
+	static const luaL_Reg ident = {	modname, openfn };	\
+	DATA_SET(FLUA_MODULE_SETNAME, ident)
+
+#define	FLUA_MODULE_NAMED(mod, name)	\
+	FLUA_MODULE_DEF(module_ ## mod, name, luaopen_ ## mod)
+#define	FLUA_MODULE(mod)		\
+	FLUA_MODULE_DEF(module_ ## mod, #mod, luaopen_ ## mod)
+#else	/* !BOOTSTRAPPING */
+#define	FLUA_MODULE_DEF(ident, modname, openfn)
+#define	FLUA_MODULE_NAMED(mod, name)
+#define	FLUA_MODULE(modname)
+#endif	/* BOOTSTRAPPING */
+
+#endif	/* FLUA_BOOTSTRAP_H */

libexec/flua/libfreebsd/kenv/Makefile

@@ -1,5 +1,5 @@
 SHLIB_NAME=	kenv.so
-SRCS+=		kenv.c
 MAN=		freebsd.kenv.3lua
 
+.include "Makefile.inc"
 .include <bsd.lib.mk>

libexec/flua/libfreebsd/kenv/Makefile.inc

@@ -0,0 +1,2 @@
+.PATH:		${.PARSEDIR}
+SRCS+=		kenv.c

libexec/flua/libfreebsd/kenv/kenv.c

@@ -14,6 +14,8 @@
 #include <lualib.h>
 #include <lauxlib.h>
 
+#include "bootstrap.h"
+
 int luaopen_freebsd_kenv(lua_State *L);
 
 static int
@@ -94,3 +96,5 @@ luaopen_freebsd_kenv(lua_State *L)
 
 	return (1);
 }
+
+FLUA_MODULE_NAMED(freebsd_kenv, "freebsd.kenv");

libexec/flua/libfreebsd/sys/linker/Makefile

@@ -1,7 +1,6 @@
 SHLIB_NAME=	linker.so
 
-SRCS+=		linker.c
-
 MAN=	freebsd.sys.linker.3lua
 
+.include "Makefile.inc"
 .include <bsd.lib.mk>

libexec/flua/libfreebsd/sys/linker/Makefile.inc

@@ -0,0 +1,2 @@
+.PATH:		${.PARSEDIR}
+SRCS+=		linker.c

libexec/flua/libfreebsd/sys/linker/linker.c

@@ -15,6 +15,8 @@
 #include <lualib.h>
 #include <lauxlib.h>
 
+#include "bootstrap.h"
+
 int luaopen_freebsd_sys_linker(lua_State *L);
 
 static int
@@ -80,3 +82,5 @@ luaopen_freebsd_sys_linker(lua_State *L)
 
 	return (1);
 }
+
+FLUA_MODULE_NAMED(freebsd_sys_linker, "freebsd.sys.linker");

libexec/flua/libhash/Makefile

@@ -1,9 +1,6 @@
 SHLIB_NAME=	hash.so
 
-SRCS+=		lhash.c
-
-LIBADD+=	md
-
 MAN=	hash.3lua
 
+.include "Makefile.inc"
 .include <bsd.lib.mk>

libexec/flua/libhash/Makefile.inc

@@ -0,0 +1,3 @@
+.PATH:		${.PARSEDIR}
+SRCS+=		lhash.c
+LIBADD+=	md

libexec/flua/libhash/lhash.c

@@ -11,6 +11,8 @@
 #include <sha256.h>
 #include <string.h>
 
+#include "bootstrap.h"
+
 #define SHA256_META "SHA256 meta table"
 #define SHA256_DIGEST_LEN 32
 
@@ -175,3 +177,5 @@ luaopen_hash(lua_State *L)
 
 	return 1;
 }
+
+FLUA_MODULE(hash);

libexec/flua/libjail/Makefile

@@ -1,9 +1,6 @@
 SHLIB_NAME=	jail.so
 
-SRCS+=		lua_jail.c
-
-LIBADD+=	jail
-
 MAN=	jail.3lua
 
+.include "Makefile.inc"
 .include <bsd.lib.mk>

libexec/flua/libjail/Makefile.inc

@@ -0,0 +1,3 @@
+.PATH:		${.PARSEDIR}
+SRCS+=		lua_jail.c
+LIBADD+=	jail

libexec/flua/libjail/lua_jail.c

@@ -38,6 +38,8 @@
 #include <lauxlib.h>
 #include <lualib.h>
 
+#include "bootstrap.h"
+
 #define	JAIL_METATABLE "jail iterator metatable"
 
 /*
@@ -716,3 +718,5 @@ luaopen_jail(lua_State *L)
 
 	return (1);
 }
+
+FLUA_MODULE(jail);

libexec/flua/liblyaml/Makefile

@@ -1,22 +1,4 @@
 SHLIB_NAME=	yaml.so
 
-WARNS=	1
-LYAMLSRC?=	${SRCTOP}/contrib/lyaml
-.PATH:		${LYAMLSRC}/ext/yaml ${LYAMLSRC}/lib/lyaml
-SRCS=		emitter.c \
-		parser.c \
-		scanner.c \
-		yaml.c
-CFLAGS+=	\
-		-I${LYAMLSRC}/ext/yaml \
-		-I${SRCTOP}/contrib/libyaml/include \
-		-DVERSION=\"6.2.8\"
-LIBADD+=	yaml
-
-FILES=		explicit.lua \
-		functional.lua \
-		implicit.lua \
-		init.lua
-FILESDIR=	${SHAREDIR}/flua/lyaml
-
+.include "Makefile.inc"
 .include <bsd.lib.mk>

libexec/flua/liblyaml/Makefile.inc

@@ -0,0 +1,20 @@
+WARNS=	1
+
+LYAMLSRC?=	${SRCTOP}/contrib/lyaml
+.PATH:		${LYAMLSRC}/ext/yaml ${LYAMLSRC}/lib/lyaml
+SRCS+=		emitter.c \
+		parser.c \
+		scanner.c \
+		yaml.c
+CFLAGS+=	\
+		-I${LYAMLSRC}/ext/yaml \
+		-I${SRCTOP}/contrib/libyaml/include \
+		-DVERSION=\"6.2.8\"
+LIBADD+=	yaml
+
+FILESGROUPS+=	YAML
+YAML=		explicit.lua \
+		functional.lua \
+		implicit.lua \
+		init.lua
+YAMLDIR=	${SHAREDIR}/flua/lyaml

libexec/flua/libucl/Makefile

@@ -1,14 +1,4 @@
 SHLIB_NAME=	ucl.so
 
-WARNS=		2
-
-UCLSRC?= 	${SRCTOP}/contrib/libucl
-.PATH: ${UCLSRC}/lua
-SRCS+=		lua_ucl.c
-CFLAGS+=	\
-		-I${UCLSRC}/include \
-		-I${UCLSRC}/src \
-		-I${UCLSRC}/uthash
-LIBADD+=	ucl
-
+.include "Makefile.inc"
 .include <bsd.lib.mk>

libexec/flua/libucl/Makefile.inc

@@ -0,0 +1,12 @@
+.if ${WARNS:U6} > 2
+WARNS=		2
+.endif
+
+UCLSRC?=	${SRCTOP}/contrib/libucl
+.PATH: ${UCLSRC}/lua
+SRCS+=		lua_ucl.c
+CFLAGS+=	\
+		-I${UCLSRC}/include \
+		-I${UCLSRC}/src \
+		-I${UCLSRC}/uthash
+LIBADD+=	ucl

libexec/flua/linit_flua.c

@@ -26,8 +26,8 @@
 
 #include "lprefix.h"
 
-
 #include <stddef.h>
+#include <stdlib.h>
 
 #include "lua.h"
 
@@ -37,6 +37,8 @@
 #include "lposix.h"
 #include "lfbsd.h"
 
+#include "bootstrap.h"
+
 /*
 ** these libs are loaded by lua.c and are readily available to any Lua
 ** program
@@ -62,6 +64,28 @@ static const luaL_Reg loadedlibs[] = {
   {NULL, NULL}
 };
 
+#ifdef BOOTSTRAPPING
+static void __attribute__((constructor)) flua_init_env(void) {
+  /*
+   * This happens in the middle of luaopen_package().  We could move it into
+   * flua_setup_mods(), but it seems better to avoid its timing being so
+   * important that it would break some of our bootstrap modules if someone
+   * were to reorder things.
+   */
+  if (getenv("LUA_PATH") == NULL)
+    setenv("LUA_PATH", BOOTSTRAP_FLUA_PATH, 1);
+}
+
+static void flua_setup_mods (lua_State *L) {
+  const luaL_Reg **flib;
+
+  SET_FOREACH(flib, FLUA_MODULE_SETNAME) {
+    luaL_requiref(L, (*flib)->name, (*flib)->func, 1);
+    lua_pop(L, 1);  /* remove lib */
+  }
+};
+#endif
+
 LUALIB_API void luaL_openlibs (lua_State *L) {
   const luaL_Reg *lib;
   /* "require" functions from 'loadedlibs' and set results to global table */
@@ -69,4 +93,7 @@ LUALIB_API void luaL_openlibs (lua_State *L) {
     luaL_requiref(L, lib->name, lib->func, 1);
     lua_pop(L, 1);  /* remove lib */
   }
+#ifdef BOOTSTRAPPING
+  flua_setup_mods(L);
+#endif
 }

tools/build/Makefile

@@ -497,6 +497,7 @@ INSTALLDIR_LIST= \
 	usr/include/casper \
 	usr/include/openssl \
 	usr/include/private/ucl \
+	usr/include/private/yaml \
 	usr/include/private/zstd \
 	usr/lib \
 	usr/libdata/pkgconfig \