release: Add -DPKGBASE option to include pkgbase packages

If this option is set, an offline repo of pkgbase packages corresponding
to base.txz and kernel.txz will be included in the disc1 release media
rather than the base.txz and kernel.txz tarballs.

Reviewed by:	bapt
Sponsored by:	The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D50346

release/Makefile

@@ -21,6 +21,8 @@
 # Variables affecting the build process:
 #  WORLDDIR: location of src tree -- must have built world and default kernel
 #            (by default, the directory above this one)
+#  PKGBASE:  if set, include pkgbase packages rather than dist tarballs in
+#            disc1 and dvd1 installation media
 #  PORTSDIR: location of ports tree to distribute (default: /usr/ports)
 #  XTRADIR:  xtra-bits-dir argument for <arch>/mkisoimages.sh
 #  NOPKG:    if set, do not distribute third-party packages
@@ -124,7 +126,7 @@ CLEANFILES+=	${I}.xz
 .if defined(WITH_DVD) && !empty(WITH_DVD)
 CLEANFILES+=	pkg-stage
 .endif
-CLEANDIRS=	dist ftp disc1 disc1-disc1 disc1-memstick bootonly bootonly-bootonly bootonly-memstick dvd
+CLEANDIRS=	dist pkgbase-repo pkgbase-repo-conf ftp disc1 disc1-disc1 disc1-memstick bootonly bootonly-bootonly bootonly-memstick dvd
 beforeclean:
 	chflags -R noschg .
 .include <bsd.obj.mk>
@@ -170,7 +172,16 @@ ports.txz:
 	    --exclude 'usr/ports/INDEX*' --exclude work usr/ports | \
 	    ${XZ_CMD} > ${.OBJDIR}/ports.txz )
 
-disc1: packagesystem
+.if defined(PKGBASE)
+PKGBASE_REPO=	pkgbase-repo
+PKG_ABI=	$$(${PKG_CMD} -o ABI_FILE=${.TARGET}/usr/bin/uname config ABI)
+.endif
+
+pkgbase-repo:
+	mkdir -p pkgbase-repo
+	( ${IMAKE} -C ${WORLDDIR} packages REPODIR=${.OBJDIR}/pkgbase-repo )
+
+disc1: packagesystem ${PKGBASE_REPO}
 # Install system
 	mkdir -p ${.TARGET}
 	( cd ${WORLDDIR} && ${IMAKE} installworld installkernel distribution \
@@ -181,6 +192,24 @@ disc1: packagesystem
 	    MK_RESCUE=no MK_DICT=no \
 	    MK_KERNEL_SYMBOLS=no MK_TESTS=no MK_DEBUG_FILES=no \
 	    -DDB_FROM_SRC -DNO_ROOT)
+.if defined(PKGBASE)
+# Create offline pkgbase repo on release media
+	mkdir -p ${.TARGET}/usr/freebsd-packages/repos/
+	${.CURDIR}/scripts/pkgbase-stage.lua disc \
+	    ${.OBJDIR}/pkgbase-repo/${PKG_ABI}/latest \
+	    ${.TARGET}/usr/freebsd-packages/offline \
+	    "${_ALL_libcompats}"
+	cp ${.CURDIR}/scripts/FreeBSD-base-offline.conf \
+		${.TARGET}/usr/freebsd-packages/repos/
+	mtree -c -p ${.TARGET}/usr/freebsd-packages | \
+	    mtree -C -k type,mode,link,size | \
+	    sed 's|^\.|./usr/freebsd-packages|g' >> ${.TARGET}/METALOG
+# Copy manifest only (no distfiles) to get checksums
+	mkdir -p ${.TARGET}/usr/freebsd-dist
+	cp MANIFEST ${.TARGET}/usr/freebsd-dist
+	echo "./usr/freebsd-dist type=dir uname=root gname=wheel mode=0755" >> ${.TARGET}/METALOG
+	echo "./usr/freebsd-dist/MANIFEST type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG
+.else
 # Copy distfiles
 	mkdir -p ${.TARGET}/usr/freebsd-dist
 	for dist in MANIFEST $$(ls *.txz | grep -v container | grep -vE -- '(${base ${_ALL_libcompats}:L:ts|})-dbg'); \
@@ -190,6 +219,7 @@ disc1: packagesystem
 	for dist in MANIFEST $$(ls *.txz | grep -v container | grep -vE -- '(${base ${_ALL_libcompats}:L:ts|})-dbg'); \
 	    do echo "./usr/freebsd-dist/$${dist} type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG; \
 	done
+.endif
 .if ${.MAKE.OS} == "FreeBSD" && (!defined(NOPKG) || empty(NOPKG))
 # Install packages onto release media.
 	${PKG_INSTALL} pkg || true
@@ -256,6 +286,24 @@ dvd: packagesystem
 		DESTDIR=${.OBJDIR}/${.TARGET} MK_RESCUE=no MK_KERNEL_SYMBOLS=no \
 		MK_TESTS=no MK_DEBUG_FILES=no \
 		-DDB_FROM_SRC -DNO_ROOT)
+.if defined(PKGBASE)
+# Create offline pkgbase repo on release media
+	mkdir -p ${.TARGET}/usr/freebsd-packages/repos/
+	${.CURDIR}/scripts/pkgbase-stage.lua dvd \
+	    ${.OBJDIR}/pkgbase-repo/${PKG_ABI}/latest \
+	    ${.TARGET}/usr/freebsd-packages/offline \
+	    "${_ALL_libcompats}"
+	cp ${.CURDIR}/scripts/FreeBSD-base-offline.conf \
+		${.TARGET}/usr/freebsd-packages/repos/
+	mtree -c -p ${.TARGET}/usr/freebsd-packages | \
+	    mtree -C -k type,mode,link,size | \
+	    sed 's|^\.|./usr/freebsd-packages|g' >> ${.TARGET}/METALOG
+# Copy manifest only (no distfiles) to get checksums
+	mkdir -p ${.TARGET}/usr/freebsd-dist
+	cp MANIFEST ${.TARGET}/usr/freebsd-dist
+	echo "./usr/freebsd-dist type=dir uname=root gname=wheel mode=0755" >> ${.TARGET}/METALOG
+	echo "./usr/freebsd-dist/MANIFEST type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG
+.else
 # Copy distfiles
 	mkdir -p ${.TARGET}/usr/freebsd-dist
 	for dist in MANIFEST $$(ls *.txz | grep -v container); \
@@ -265,6 +313,7 @@ dvd: packagesystem
 	for dist in MANIFEST $$(ls *.txz | grep -v container); \
 	    do echo "./usr/freebsd-dist/$${dist} type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG; \
 	done
+.endif
 .if ${.MAKE.OS} == "FreeBSD" && (!defined(NOPKG) || empty(NOPKG))
 # Install packages onto release media.
 	${PKG_INSTALL} pkg || true

release/scripts/FreeBSD-base-offline.conf

@@ -0,0 +1,4 @@
+FreeBSD-base: {
+  url: "file:///usr/freebsd-packages/offline",
+  enabled: yes
+}

release/scripts/pkgbase-stage.lua

@@ -0,0 +1,138 @@
+#!/usr/libexec/flua
+
+-- SPDX-License-Identifier: BSD-2-Clause
+--
+-- Copyright(c) 2025 The FreeBSD Foundation.
+--
+-- This software was developed by Isaac Freund <ifreund@freebsdfoundation.org>
+-- under sponsorship from the FreeBSD Foundation.
+
+-- Run a command using the OS shell and capture the stdout
+-- Strips exactly one trailing newline if present, does not strip any other whitespace.
+-- Asserts that the command exits cleanly
+local function capture(command)
+	local p = io.popen(command)
+	local output = p:read("*a")
+	assert(p:close())
+	-- Strip exactly one trailing newline from the output, if there is one
+	return output:match("(.-)\n$") or output
+end
+
+local function append_list(list, other)
+	for _, item in ipairs(other) do
+		table.insert(list, item)
+	end
+end
+
+-- Returns a list of packages to be included in the given media
+local function select_packages(pkg, media, all_libcompats)
+	local components = {
+		kernel = {},
+		kernel_dbg = {},
+		base = {},
+		base_dbg = {},
+		src = {},
+		tests = {},
+	}
+
+	for compat in all_libcompats:gmatch("%S+") do
+		components["lib" .. compat] = {}
+		components["lib" .. compat .. "_dbg"] = {}
+	end
+
+	local rquery = capture(pkg .. "rquery -U -r FreeBSD-base %n")
+	for package in rquery:gmatch("[^\n]+") do
+		if package == "FreeBSD-src" or package:match("^FreeBSD%-src%-.*") then
+			table.insert(components["src"], package)
+		elseif package == "FreeBSD-tests" or package:match("^FreeBSD%-tests%-.*") then
+			table.insert(components["tests"], package)
+		elseif package:match("^FreeBSD%-kernel%-.*") then
+			-- Kernels other than FreeBSD-kernel-generic are ignored
+			if package == "FreeBSD-kernel-generic" then
+				table.insert(components["kernel"], package)
+			elseif package == "FreeBSD-kernel-generic-dbg" then
+				table.insert(components["kernel_dbg"], package)
+			end
+		elseif package:match(".*%-dbg$") then
+			table.insert(components["base_dbg"], package)
+		else
+			local found = false
+			for compat in all_libcompats:gmatch("%S+") do
+				if package:match(".*%-dbg%-lib" .. compat .. "$") then
+					table.insert(components["lib" .. compat .. "_dbg"], package)
+					found = true
+					break
+				elseif package:match(".*%-lib" .. compat .. "$") then
+					table.insert(components["lib" .. compat], package)
+					found = true
+					break
+				end
+			end
+			if not found then
+				table.insert(components["base"], package)
+			end
+		end
+	end
+	assert(#components["kernel"] == 1)
+	assert(#components["base"] > 0)
+
+	local selected = {}
+	if media == "disc" then
+		append_list(selected, components["base"])
+		append_list(selected, components["kernel"])
+		append_list(selected, components["kernel_dbg"])
+		append_list(selected, components["src"])
+		append_list(selected, components["tests"])
+		for compat in all_libcompats:gmatch("%S+") do
+			append_list(selected, components["lib" .. compat])
+		end
+	else
+		assert(media == "dvd")
+		append_list(selected, components["base"])
+		append_list(selected, components["base_dbg"])
+		append_list(selected, components["kernel"])
+		append_list(selected, components["kernel_dbg"])
+		append_list(selected, components["src"])
+		append_list(selected, components["tests"])
+		for compat in all_libcompats:gmatch("%S+") do
+			append_list(selected, components["lib" .. compat])
+			append_list(selected, components["lib" .. compat .. "_dbg"])
+		end
+	end
+
+	return selected
+end
+
+local function main()
+	-- Determines package subset selected
+	local media = assert(arg[1])
+	assert(media == "disc" or media == "dvd")
+	-- Local repository to fetch from
+	local source = assert(arg[2])
+	-- Directory to create new repository
+	local target = assert(arg[3])
+	-- =hitespace separated list of all libcompat names (e.g. "32")
+	local all_libcompats = assert(arg[4])
+
+	assert(os.execute("mkdir -p pkgbase-repo-conf"))
+	local f <close> = assert(io.open("pkgbase-repo-conf/FreeBSD-base.conf", "w"))
+	assert(f:write(string.format([[
+	FreeBSD-base: {
+	  url: "file://%s",
+	  enabled: yes
+	}
+	]], source)))
+	assert(f:close())
+
+	local pkg = "pkg -o ASSUME_ALWAYS_YES=yes -o IGNORE_OSVERSION=yes " ..
+	    "-o INSTALL_AS_USER=1 -o PKG_DBDIR=./pkgdb -R ./pkgbase-repo-conf "
+
+	assert(os.execute(pkg .. "update"))
+
+	local packages = select_packages(pkg, media, all_libcompats)
+
+	assert(os.execute(pkg .. "fetch -o " .. target .. " " .. table.concat(packages, " ")))
+	assert(os.execute(pkg .. "repo " .. target))
+end
+
+main()