libexec/kgdb: Add new modules and install them together with debug info

This change simplifies integration of gdb python scripts with our kernel
debugging infrastructure.  Rather than putting debugging scripts in
/usr/libexec/kgdb, move them to <path-to-kernel-debug-symbols>/gdb, and
add a kernel-gdb.py which automatically loads modules from that
directory.  kernel-gdb.py will be automatically executed by kgdb when
loading kernel debug symbols (assuming a default configuration), so one
no longer needs to do anything to use these modules.

The change also adds a couple of new modules, vnet.py and pcpu.py, for
conveniently accessing VNET symbols and PCPU/DPCPU fields, respectively.
Note that these require a change to the kernel linker when accessing
symbols from a loadable kernel module.

sys/tools/gdb/README.txt describes the scheme in more detail and
provides some rudiementary documentation for the commands and functions
added by these modules.  It should be updated when adding new features.

sys/tools/gdb/selftest.py can be used to do some primitive testing of
the modules.  All it does is execute a number of gdb commands making use
of commands and functions added by these modules.  The developer is
expected to verify that the commands complete without errors and that
the output looks sane.

Discussed with:	kp, avg, jhb, glebius
MFC after:	2 weeks
Differential Revision:	https://reviews.freebsd.org/D50825

ObsoleteFiles.inc

@@ -51,6 +51,10 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20251003: kgdb python scripts moved
+OLD_FILES+=usr/libexec/kgdb/acttrace.py
+OLD_DIRS+=usr/libexec/kgdb
+
 # 20251001: test helper sendto-IP_MULTICAST_IF renamed
 OLD_FILES+=usr/tests/sys/netinet/sendto-IP_MULTICAST_IF
 

etc/mtree/BSD.usr.dist

@@ -181,8 +181,6 @@
         ..
         hyperv
         ..
-        kgdb
-        ..
         lpr
             ru
             ..

libexec/Makefile

@@ -10,7 +10,6 @@ SUBDIR=	${_atf} \
 	flua \
 	getty \
 	${_hyperv} \
-	kgdb \
 	${_mail.local} \
 	${_makewhatis.local} \
 	${_mknetid} \

libexec/kgdb/Makefile

@@ -1,5 +0,0 @@
-FILESDIR?= /usr/libexec/kgdb
-
-FILES= acttrace.py
-
-.include <bsd.prog.mk>

sys/conf/kern.post.mk

@@ -398,6 +398,14 @@ CFLAGS+= -fdebug-prefix-map=./${_link}=${PREFIX_SYSDIR}/${_link}/include
 .endif
 .endfor
 
+# Install GDB plugins that are useful for kernel debugging.  See the
+# README in sys/tools/gdb for more information.
+GDB_FILES= acttrace.py \
+	   freebsd.py \
+	   pcpu.py \
+	   selftest.py \
+	   vnet.py
+
 ${_ILINKS}:
 	@case ${.TARGET} in \
 	machine) \
@@ -447,6 +455,13 @@ kernel-install: .PHONY
 .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no"
 	mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR}
 	${INSTALL} -p -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/
+	${INSTALL} -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} \
+	    $S/tools/kernel-gdb.py ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/${KERNEL_KO}-gdb.py
+	mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/gdb
+.for file in ${GDB_FILES}
+	${INSTALL} -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} \
+	    $S/tools/gdb/${file} ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/gdb/${file}
+.endfor
 .endif
 .if defined(KERNEL_EXTRA_INSTALL)
 	${INSTALL} -p -m ${KMODMODE} -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_EXTRA_INSTALL} ${DESTDIR}${KODIR}/

sys/tools/gdb/README.txt

@@ -0,0 +1,21 @@
+This directory contains Python scripts that can be loaded by GDB to help debug
+FreeBSD kernel crashes.
+
+Add new commands and functions in their own files.  Functions with general
+utility should be added to freebsd.py.  sys/tools/kernel-gdb.py is installed
+into the kernel debug directory (typically /usr/lib/debug/boot/kernel).  It will
+be automatically loaded by kgdb when opening a vmcore, so if you add new GDB
+commands or functions, that script should be updated to import them, and you
+should document them here.
+
+To provide some rudimentary testing, selftest.py tries to exercise all of the
+commands and functions defined here.  To use it, run selftest.sh to panic the
+system.  Then, create a kernel dump or attach to the panicked kernel, and invoke
+the script with "python import selftest" in (k)gdb.
+
+Commands:
+acttrace	Display a backtrace for all on-CPU threads
+
+Functions:
+$PCPU(<field>[, <cpuid>])	Display the value of a PCPU/DPCPU field
+$V(<variable>[, <vnet>])	Display the value of a VNET variable

sys/tools/gdb/acttrace.py

@@ -1,38 +1,23 @@
-#-
+#
 # Copyright (c) 2022 The FreeBSD Foundation
 #
 # This software was developed by Mark Johnston under sponsorship from the
 # FreeBSD Foundation.
 #
+# SPDX-License-Identifier: BSD-2-Clause
+#
 
 import gdb
-
-
-def symval(name):
-    return gdb.lookup_global_symbol(name).value()
-
-
-def tid_to_gdb_thread(tid):
-    for thread in gdb.inferiors()[0].threads():
-        if thread.ptid[2] == tid:
-            return thread
-    else:
-        return None
-
-
-def all_pcpus():
-    mp_maxid = symval("mp_maxid")
-    cpuid_to_pcpu = symval("cpuid_to_pcpu")
-
-    cpu = 0
-    while cpu <= mp_maxid:
-        pcpu = cpuid_to_pcpu[cpu]
-        if pcpu:
-            yield pcpu
-        cpu = cpu + 1
-
+from freebsd import *
+from pcpu import *
 
 class acttrace(gdb.Command):
+    """
+    Register an acttrace command with gdb.
+
+    When run, acttrace prints the stack trace of all threads that were on-CPU
+    at the time of the panic.
+    """
     def __init__(self):
         super(acttrace, self).__init__("acttrace", gdb.COMMAND_USER)
 
@@ -40,13 +25,13 @@ class acttrace(gdb.Command):
         # Save the current thread so that we can switch back after.
         curthread = gdb.selected_thread()
 
-        for pcpu in all_pcpus():
+        for pcpu in pcpu_foreach():
             td = pcpu['pc_curthread']
             tid = td['td_tid']
 
             gdb_thread = tid_to_gdb_thread(tid)
             if gdb_thread is None:
-                print("failed to find GDB thread with TID {}".format(tid))
+                raise gdb.error(f"failed to find GDB thread with TID {tid}")
             else:
                 gdb_thread.switch()
 

sys/tools/gdb/freebsd.py

@@ -0,0 +1,75 @@
+#
+# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org>
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+import gdb
+
+def symval(name):
+    sym = gdb.lookup_global_symbol(name)
+    if sym is None:
+        sym = gdb.lookup_static_symbol(name)
+        if sym is None:
+            raise gdb.GdbError(f"Symbol '{name}' not found")
+    return sym.value()
+
+
+def _queue_foreach(head, field, headf, nextf):
+    elm = head[headf]
+    while elm != 0:
+        yield elm
+        elm = elm[field][nextf]
+
+
+def list_foreach(head, field):
+    """sys/queue.h-style iterator."""
+    return _queue_foreach(head, field, "lh_first", "le_next")
+
+
+def tailq_foreach(head, field):
+    """sys/queue.h-style iterator."""
+    return _queue_foreach(head, field, "tqh_first", "tqe_next")
+
+
+def linker_file_foreach():
+    """Iterate over loaded linker files."""
+    return tailq_foreach(symval("linker_files"), "link")
+
+
+def pcpu_foreach():
+    mp_maxid = symval("mp_maxid")
+    cpuid_to_pcpu = symval("cpuid_to_pcpu")
+
+    cpu = 0
+    while cpu <= mp_maxid:
+        pcpu = cpuid_to_pcpu[cpu]
+        if pcpu:
+            yield pcpu
+        cpu = cpu + 1
+
+
+def tid_to_gdb_thread(tid):
+    """Convert a FreeBSD kernel thread ID to a gdb inferior thread."""
+    for thread in gdb.inferiors()[0].threads():
+        if thread.ptid[2] == tid:
+            return thread
+    else:
+        return None
+
+
+def tdfind(tid, pid=-1):
+    """Convert a FreeBSD kernel thread ID to a struct thread pointer."""
+    td = tdfind.cached_threads.get(int(tid))
+    if td:
+        return td
+
+    for p in list_foreach(symval("allproc"), "p_list"):
+        if pid != -1 and pid != p['p_pid']:
+            continue
+        for td in tailq_foreach(p['p_threads'], "td_plist"):
+            ntid = td['td_tid']
+            tdfind.cached_threads[int(ntid)] = td
+            if ntid == tid:
+                return td
+tdfind.cached_threads = dict()

sys/tools/gdb/pcpu.py

@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org>
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+import gdb
+from freebsd import *
+
+class pcpu(gdb.Function):
+    """
+    Register a function to lookup PCPU and DPCPU variables by name.
+
+    To look up the value of the PCPU field foo on CPU n, use
+    $PCPU("foo", n).  This works for DPCPU fields too.  If the CPU ID is
+    omitted, and the currently selected thread is on-CPU, that CPU is
+    used, otherwise an error is raised.
+    """
+    def __init__(self):
+        super(pcpu, self).__init__("PCPU")
+
+    def invoke(self, field, cpuid=-1):
+        if cpuid == -1:
+            cpuid = tdfind(gdb.selected_thread().ptid[2])['td_oncpu']
+            if cpuid == -1:
+                raise gdb.error("Currently selected thread is off-CPU")
+            if cpuid < 0 or cpuid > symval("mp_maxid"):
+                raise gdb.error(f"Currently selected on invalid CPU {cpuid}")
+        pcpu = symval("cpuid_to_pcpu")[cpuid]
+
+        # Are we dealing with a PCPU or DPCPU field?
+        field = field.string()
+        for f in gdb.lookup_type("struct pcpu").fields():
+            if f.name == "pc_" + field:
+                return pcpu["pc_" + field]
+
+        def uintptr_t(val):
+            return val.cast(gdb.lookup_type("uintptr_t"))
+
+        # We're dealing with a DPCPU field.  This is handled similarly
+        # to VNET symbols, see vnet.py for comments.
+        pcpu_base = pcpu['pc_dynamic']
+        pcpu_entry = symval("pcpu_entry_" + field)
+        pcpu_entry_addr = uintptr_t(pcpu_entry.address)
+
+        for lf in linker_file_foreach():
+            block = gdb.block_for_pc(lf['ops']['cls']['methods'][0]['func'])
+            elf_file_t = gdb.lookup_type("elf_file_t", block).target()
+            ef = lf.cast(elf_file_t)
+
+            file_type = lf['ops']['cls']['name'].string()
+            if file_type == "elf64":
+                start = uintptr_t(ef['pcpu_start'])
+                if start == 0:
+                    continue
+                end = uintptr_t(ef['pcpu_stop'])
+                base = uintptr_t(ef['pcpu_base'])
+            elif file_type == "elf64_obj":
+                for i in range(ef['nprogtab']):
+                    pe = ef['progtab'][i]
+                    if pe['name'].string() == "set_pcpu":
+                        start = uintptr_t(pe['origaddr'])
+                        end = start + uintptr_t(pe['size'])
+                        base = uintptr_t(pe['addr'])
+                        break
+                else:
+                    continue
+            else:
+                path = lf['pathname'].string()
+                raise gdb.error(f"{path} has unexpected linker file type {file_type}")
+
+            if pcpu_entry_addr >= start and pcpu_entry_addr < end:
+                obj = gdb.Value(pcpu_base + pcpu_entry_addr - start + base)
+                return obj.cast(pcpu_entry.type.pointer()).dereference()
+
+# Register with gdb.
+pcpu()

sys/tools/gdb/selftest.py

@@ -0,0 +1,31 @@
+#
+# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org>
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+import gdb
+
+cmds = ["acttrace",
+        "p $V(\"tcbinfo\")",
+        "p $V(\"tcbinfo\", vnet0)",
+        "p $V(\"pf_status\")",
+        "p $V(\"pf_status\", \"gdbselftest\")",
+        "p $PCPU(\"curthread\")",
+        "p $PCPU(\"curthread\", 0)",
+        "p/x $PCPU(\"hardclocktime\", 1)",
+        "p $PCPU(\"pqbatch\")[0][0]",
+        "p $PCPU(\"ss\", 1)",
+        ]
+
+for cmd in cmds:
+    try:
+        print(f"Running command: '{cmd}'")
+        gdb.execute(cmd)
+    except gdb.error as e:
+        print(f"Command '{cmd}' failed: {e}")
+        break
+
+# We didn't hit any unexpected errors.  This isn't as good as actually
+# verifying the output, but it's better than nothing.
+print("Everything seems OK")

sys/tools/gdb/selftest.sh

@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org>
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+set -e
+
+n=$(sysctl -n hw.ncpu)
+if [ $n -lt 2 ]; then
+    echo "This test requires at least 2 CPUs"
+    exit 1
+fi
+
+# Set up some things expected by selftest.py.
+kldload -n pf siftr
+pfctl -e || true
+jail -c name=gdbselftest vnet persist
+
+echo "I'm about to panic your system, ctrl-C now if that's not what you want."
+sleep 10
+sysctl debug.debugger_on_panic=0
+sysctl debug.kdb.panic=1

sys/tools/gdb/vnet.py

@@ -0,0 +1,100 @@
+#
+# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org>
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+import gdb
+import traceback
+from freebsd import *
+
+class vnet(gdb.Function):
+    """
+    Register a function to look up VNET variables by name.
+
+    To look at the value of a VNET variable V_foo, print $V("foo").  The
+    currently selected thread's VNET is used by default, but can be optionally
+    specified as a second parameter, e.g., $V("foo", <vnet>), where <vnet> is a
+    pointer to a struct vnet (e.g., vnet0 or allprison.tqh_first->pr_vnet) or a
+    string naming a jail.
+    """
+    def __init__(self):
+        super(vnet, self).__init__("V")
+
+    def invoke(self, sym, vnet=None):
+        sym = sym.string()
+        if sym.startswith("V_"):
+            sym = sym[len("V_"):]
+        if gdb.lookup_symbol("sysctl___kern_features_vimage")[0] is None:
+            return symval(sym)
+
+        # Look up the VNET's base address.
+        if vnet is None:
+            vnet = tdfind(gdb.selected_thread().ptid[2])['td_vnet']
+            if not vnet:
+                # If curthread->td_vnet == NULL, vnet0 is the current vnet.
+                vnet = symval("vnet0")
+        elif vnet.type.is_string_like:
+            vnet = vnet.string()
+            for prison in tailq_foreach(symval("allprison"), "pr_list"):
+                if prison['pr_name'].string() == vnet:
+                    vnet = prison['pr_vnet']
+                    break
+            else:
+                raise gdb.error(f"No prison named {vnet}")
+
+        def uintptr_t(val):
+            return val.cast(gdb.lookup_type("uintptr_t"))
+
+        # Now the tricky part: compute the address of the symbol relative
+        # to the selected VNET.  In the compiled kernel this is done at
+        # load time by applying a magic transformation to relocations
+        # against symbols in the vnet linker set.  Here we have to apply
+        # the transformation manually.
+        vnet_data_base = vnet['vnet_data_base']
+        vnet_entry = symval("vnet_entry_" + sym)
+        vnet_entry_addr = uintptr_t(vnet_entry.address)
+
+        # First, which kernel module does the symbol belong to?
+        for lf in linker_file_foreach():
+            # Find the bounds of this linker file's VNET linker set.  The
+            # struct containing the bounds depends on the type of the linker
+            # file, and unfortunately both are called elf_file_t.  So we use a
+            # PC value from the compilation unit (either link_elf.c or
+            # link_elf_obj.c) to disambiguate.
+            block = gdb.block_for_pc(lf['ops']['cls']['methods'][0]['func'])
+            elf_file_t = gdb.lookup_type("elf_file_t", block).target()
+            ef = lf.cast(elf_file_t)
+
+            file_type = lf['ops']['cls']['name'].string()
+            if file_type == "elf64":
+                start = uintptr_t(ef['vnet_start'])
+                if start == 0:
+                    # This linker file doesn't have a VNET linker set.
+                    continue
+                end = uintptr_t(ef['vnet_stop'])
+                base = uintptr_t(ef['vnet_base'])
+            elif file_type == "elf64_obj":
+                for i in range(ef['nprogtab']):
+                    pe = ef['progtab'][i]
+                    if pe['name'].string() == "set_vnet":
+                        start = uintptr_t(pe['origaddr'])
+                        end = start + uintptr_t(pe['size'])
+                        base = uintptr_t(pe['addr'])
+                        break
+                else:
+                    # This linker file doesn't have a VNET linker set.
+                    continue
+            else:
+                path = lf['pathname'].string()
+                raise gdb.error(f"{path} has unexpected linker file type {file_type}")
+
+            if vnet_entry_addr >= start and vnet_entry_addr < end:
+                # The symbol belongs to this linker file, so compute the final
+                # address.
+                obj = gdb.Value(vnet_data_base + vnet_entry_addr - start + base)
+                return obj.cast(vnet_entry.type.pointer()).dereference()
+
+
+# Register with gdb.
+vnet()

sys/tools/kernel-gdb.py

@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2025 Mark Johnston <markj@FreeBSD.org>
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), "gdb"))
+
+# Import FreeBSD kernel debugging commands and modules below.
+import acttrace
+import pcpu
+import vnet

usr.sbin/crashinfo/crashinfo.sh

@@ -217,10 +217,7 @@ echo
 
 file=`mktemp /tmp/crashinfo.XXXXXX`
 if [ $? -eq 0 ]; then
-	scriptdir=/usr/libexec/kgdb
-
 	echo "bt -full" >> $file
-	echo "source ${scriptdir}/acttrace.py" >> $file
 	echo "acttrace" >> $file
 	echo "quit" >> $file
 	${GDB%gdb}kgdb -q $KERNEL $VMCORE < $file