From: John Ogness <john.ogness@linutronix.de> Date: Wed, 13 Sep 2023 08:35:23 +0000 Subject: [PATCH 14/48] printk: nbcon: Implement processing in port->lock wrapper Currently the port->lock wrappers uart_port_lock(), uart_port_unlock() (and their variants) only lock/unlock the spin_lock. If the port is an nbcon console, the wrappers must also acquire/release the console and mark the region as unsafe. This allows general port->lock synchronization to be synchronized with the nbcon console ownership. Introduce a new struct nbcon_drvdata within struct console that provides the necessary components for the port lock wrappers to acquire the nbcon console and track its ownership. Also introduce uart_port_set_cons() as a wrapper to set @cons of a uart_port. The wrapper sets @cons under the port lock in order to prevent @cons from disappearing while another context owns the port lock via the port lock wrappers. Also cleanup the description of the console_srcu_read_flags() function. It is used by the port lock wrappers to ensure a console cannot be fully unregistered while another context owns the port lock via the port lock wrappers. Signed-off-by: John Ogness <john.ogness@linutronix.de> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> --- drivers/tty/serial/8250/8250_core.c | 6 +- drivers/tty/serial/amba-pl011.c | 2 drivers/tty/serial/serial_core.c | 16 ++--- include/linux/console.h | 57 ++++++++++++++++---- include/linux/printk.h | 13 ++++ include/linux/serial_core.h | 98 +++++++++++++++++++++++++++++++++++- kernel/printk/nbcon.c | 52 +++++++++++++++++++ 7 files changed, 219 insertions(+), 25 deletions(-) Index: linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c =================================================================== @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:630 @ static int univ8250_console_setup(struct port = &serial8250_ports[co->index].port; /* link port to console */ - port->cons = co; + uart_port_set_cons(port, co); retval = serial8250_console_setup(port, options, false); if (retval != 0) - port->cons = NULL; + uart_port_set_cons(port, NULL); return retval; } @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:692 @ static int univ8250_console_match(struct continue; co->index = i; - port->cons = co; + uart_port_set_cons(port, co); return serial8250_console_setup(port, options, true); } Index: linux-6.8.2-rt10/drivers/tty/serial/amba-pl011.c =================================================================== --- linux-6.8.2-rt10.orig/drivers/tty/serial/amba-pl011.c +++ linux-6.8.2-rt10/drivers/tty/serial/amba-pl011.c @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:2491 @ static int pl011_console_match(struct co continue; co->index = i; - port->cons = co; + uart_port_set_cons(port, co); return pl011_console_setup(co, options); } Index: linux-6.8.2-rt10/drivers/tty/serial/serial_core.c =================================================================== --- linux-6.8.2-rt10.orig/drivers/tty/serial/serial_core.c +++ linux-6.8.2-rt10/drivers/tty/serial/serial_core.c @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:3148 @ static int serial_core_add_one_port(stru state->uart_port = uport; uport->state = state; + /* + * If this port is in use as a console then the spinlock is already + * initialised. + */ + if (!uart_console_registered(uport)) + uart_port_spin_lock_init(uport); + state->pm_state = UART_PM_STATE_UNDEFINED; - uport->cons = drv->cons; + uart_port_set_cons(uport, drv->cons); uport->minor = drv->tty_driver->minor_start + uport->line; uport->name = kasprintf(GFP_KERNEL, "%s%d", drv->dev_name, drv->tty_driver->name_base + uport->line); @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:3165 @ static int serial_core_add_one_port(stru goto out; } - /* - * If this port is in use as a console then the spinlock is already - * initialised. - */ - if (!uart_console_registered(uport)) - uart_port_spin_lock_init(uport); - if (uport->cons && uport->dev) of_console_check(uport->dev->of_node, uport->cons->name, uport->line); Index: linux-6.8.2-rt10/include/linux/console.h =================================================================== --- linux-6.8.2-rt10.orig/include/linux/console.h +++ linux-6.8.2-rt10/include/linux/console.h @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:286 @ struct nbcon_write_context { }; /** + * struct nbcon_drvdata - Data to allow nbcon acquire in non-print context + * @ctxt: The core console context + * @srcu_cookie: Storage for a console_srcu_lock cookie, if needed + * @owner_index: Storage for the owning console index, if needed + * @locked: Storage for the locked state, if needed + * + * All fields (except for @ctxt) are available exclusively to the driver to + * use as needed. They are not used by the printk subsystem. + */ +struct nbcon_drvdata { + struct nbcon_context __private ctxt; + + /* reserved for driver use */ + int srcu_cookie; + short owner_index; + bool locked; +}; + +/** * struct console - The console descriptor structure * @name: The name of the console driver * @write: Legacy write callback to output messages (Optional) @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:418 @ struct console { atomic_t __private nbcon_state; atomic_long_t __private nbcon_seq; + + /** + * @nbcon_drvdata: + * + * Data for nbcon ownership tracking to allow acquiring nbcon consoles + * in non-printing contexts. + * + * Drivers may need to acquire nbcon consoles in non-printing + * contexts. This is achieved by providing a struct nbcon_drvdata. + * Then the driver can call nbcon_driver_acquire() and + * nbcon_driver_release(). The struct does not require any special + * initialization. + */ + struct nbcon_drvdata *nbcon_drvdata; + struct printk_buffers *pbufs; }; @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:462 @ extern void console_list_unlock(void) __ extern struct hlist_head console_list; /** - * console_srcu_read_flags - Locklessly read the console flags + * console_srcu_read_flags - Locklessly read flags of a possibly registered + * console * @con: struct console pointer of console to read flags from * - * This function provides the necessary READ_ONCE() and data_race() - * notation for locklessly reading the console flags. The READ_ONCE() - * in this function matches the WRITE_ONCE() when @flags are modified - * for registered consoles with console_srcu_write_flags(). + * Locklessly reading @con->flags provides a consistent read value because + * there is at most one CPU modifying @con->flags and that CPU is using only + * read-modify-write operations to do so. * - * Only use this function to read console flags when locklessly - * iterating the console list via srcu. + * Requires console_srcu_read_lock to be held, which implies that @con might + * be a registered console. If the caller is holding the console_list_lock or + * it is certain that the console is not registered, the caller may read + * @con->flags directly instead. * * Context: Any context. + * Return: The current value of the @con->flags field. */ static inline short console_srcu_read_flags(const struct console *con) { WARN_ON_ONCE(!console_srcu_read_lock_is_held()); /* - * Locklessly reading console->flags provides a consistent - * read value because there is at most one CPU modifying - * console->flags and that CPU is using only read-modify-write - * operations to do so. + * The READ_ONCE() matches the WRITE_ONCE() when @flags are modified + * for registered consoles with console_srcu_write_flags(). */ return data_race(READ_ONCE(con->flags)); } Index: linux-6.8.2-rt10/include/linux/printk.h =================================================================== --- linux-6.8.2-rt10.orig/include/linux/printk.h +++ linux-6.8.2-rt10/include/linux/printk.h @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:12 @ #include <linux/ratelimit_types.h> #include <linux/once_lite.h> +struct console; + extern const char linux_banner[]; extern const char linux_proc_banner[]; @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:198 @ void show_regs_print_info(const char *lo extern asmlinkage void dump_stack_lvl(const char *log_lvl) __cold; extern asmlinkage void dump_stack(void) __cold; void printk_trigger_flush(void); +extern void nbcon_driver_acquire(struct console *con); +extern void nbcon_driver_release(struct console *con); #else static inline __printf(1, 0) int vprintk(const char *s, va_list args) @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:279 @ static inline void dump_stack(void) static inline void printk_trigger_flush(void) { } + +static inline void nbcon_driver_acquire(struct console *con) +{ +} + +static inline void nbcon_driver_release(struct console *con) +{ +} + #endif bool this_cpu_in_panic(void); Index: linux-6.8.2-rt10/include/linux/serial_core.h =================================================================== --- linux-6.8.2-rt10.orig/include/linux/serial_core.h +++ linux-6.8.2-rt10/include/linux/serial_core.h @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:11 @ #define LINUX_SERIAL_CORE_H #include <linux/bitops.h> +#include <linux/bug.h> #include <linux/compiler.h> #include <linux/console.h> #include <linux/interrupt.h> #include <linux/circ_buf.h> +#include <linux/lockdep.h> +#include <linux/printk.h> #include <linux/spinlock.h> #include <linux/sched.h> #include <linux/tty.h> @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:613 @ static inline void __uart_port_unlock_ir } /** + * uart_port_set_cons - Safely set the @cons field for a uart + * @up: The uart port to set + * @con: The new console to set to + * + * This function must be used to set @up->cons. It uses the port lock to + * synchronize with the port lock wrappers in order to ensure that the console + * cannot change or disappear while another context is holding the port lock. + */ +static inline void uart_port_set_cons(struct uart_port *up, struct console *con) +{ + unsigned long flags; + + __uart_port_lock_irqsave(up, &flags); + up->cons = con; + __uart_port_unlock_irqrestore(up, flags); +} + +/* Only for internal port lock wrapper usage. */ +static inline void __uart_port_nbcon_acquire(struct uart_port *up) +{ + lockdep_assert_held_once(&up->lock); + + if (likely(!uart_console(up))) + return; + + if (up->cons->nbcon_drvdata) { + /* + * If @up->cons is registered, prevent it from fully + * unregistering until this context releases the nbcon. + */ + int cookie = console_srcu_read_lock(); + + /* Ensure console is registered and is an nbcon console. */ + if (!hlist_unhashed_lockless(&up->cons->node) && + (console_srcu_read_flags(up->cons) & CON_NBCON)) { + WARN_ON_ONCE(up->cons->nbcon_drvdata->locked); + + nbcon_driver_acquire(up->cons); + + /* + * Record @up->line to be used during release because + * @up->cons->index can change while the port and + * nbcon are locked. + */ + up->cons->nbcon_drvdata->owner_index = up->line; + up->cons->nbcon_drvdata->srcu_cookie = cookie; + up->cons->nbcon_drvdata->locked = true; + } else { + console_srcu_read_unlock(cookie); + } + } +} + +/* Only for internal port lock wrapper usage. */ +static inline void __uart_port_nbcon_release(struct uart_port *up) +{ + lockdep_assert_held_once(&up->lock); + + /* + * uart_console() cannot be used here because @up->cons->index might + * have changed. Check against @up->cons->nbcon_drvdata->owner_index + * instead. + */ + + if (unlikely(up->cons && + up->cons->nbcon_drvdata && + up->cons->nbcon_drvdata->locked && + up->cons->nbcon_drvdata->owner_index == up->line)) { + WARN_ON_ONCE(!up->cons->nbcon_drvdata->locked); + + up->cons->nbcon_drvdata->locked = false; + nbcon_driver_release(up->cons); + console_srcu_read_unlock(up->cons->nbcon_drvdata->srcu_cookie); + } +} + +/** * uart_port_lock - Lock the UART port * @up: Pointer to UART port structure */ static inline void uart_port_lock(struct uart_port *up) { spin_lock(&up->lock); + __uart_port_nbcon_acquire(up); } /** @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:706 @ static inline void uart_port_lock(struct static inline void uart_port_lock_irq(struct uart_port *up) { spin_lock_irq(&up->lock); + __uart_port_nbcon_acquire(up); } /** @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:717 @ static inline void uart_port_lock_irq(st static inline void uart_port_lock_irqsave(struct uart_port *up, unsigned long *flags) { spin_lock_irqsave(&up->lock, *flags); + __uart_port_nbcon_acquire(up); } /** @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:728 @ static inline void uart_port_lock_irqsav */ static inline bool uart_port_trylock(struct uart_port *up) { - return spin_trylock(&up->lock); + if (!spin_trylock(&up->lock)) + return false; + + __uart_port_nbcon_acquire(up); + return true; } /** @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:744 @ static inline bool uart_port_trylock(str */ static inline bool uart_port_trylock_irqsave(struct uart_port *up, unsigned long *flags) { - return spin_trylock_irqsave(&up->lock, *flags); + if (!spin_trylock_irqsave(&up->lock, *flags)) + return false; + + __uart_port_nbcon_acquire(up); + return true; } /** @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:757 @ static inline bool uart_port_trylock_irq */ static inline void uart_port_unlock(struct uart_port *up) { + __uart_port_nbcon_release(up); spin_unlock(&up->lock); } @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:767 @ static inline void uart_port_unlock(stru */ static inline void uart_port_unlock_irq(struct uart_port *up) { + __uart_port_nbcon_release(up); spin_unlock_irq(&up->lock); } @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:778 @ static inline void uart_port_unlock_irq( */ static inline void uart_port_unlock_irqrestore(struct uart_port *up, unsigned long flags) { + __uart_port_nbcon_release(up); spin_unlock_irqrestore(&up->lock, flags); } Index: linux-6.8.2-rt10/kernel/printk/nbcon.c =================================================================== --- linux-6.8.2-rt10.orig/kernel/printk/nbcon.c +++ linux-6.8.2-rt10/kernel/printk/nbcon.c @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:6 @ // Copyright (C) 2022 Intel, Thomas Gleixner #include <linux/kernel.h> +#include <linux/bug.h> #include <linux/console.h> #include <linux/delay.h> +#include <linux/export.h> #include <linux/slab.h> +#include <linux/string.h> #include "internal.h" /* * Printk console printing implementation for consoles which does not depend @ linux-6.8.2-rt10/drivers/tty/serial/8250/8250_core.c:994 @ void nbcon_free(struct console *con) con->pbufs = NULL; } + +/** + * nbcon_driver_acquire - Acquire nbcon console and enter unsafe section + * @con: The nbcon console to acquire + * + * Context: Any context which could not be migrated to another CPU. + * + * Console drivers will usually use their own internal synchronization + * mechasism to synchronize between console printing and non-printing + * activities (such as setting baud rates). However, nbcon console drivers + * supporting atomic consoles may also want to mark unsafe sections when + * performing non-printing activities. + * + * This function acquires the nbcon console using priority NBCON_PRIO_NORMAL + * and marks it unsafe for handover/takeover. + * + * Console drivers using this function must have provided @nbcon_drvdata in + * their struct console, which is used to track ownership and state + * information. + */ +void nbcon_driver_acquire(struct console *con) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(con->nbcon_drvdata, ctxt); + + cant_migrate(); + + do { + do { + memset(ctxt, 0, sizeof(*ctxt)); + ctxt->console = con; + ctxt->prio = NBCON_PRIO_NORMAL; + } while (!nbcon_context_try_acquire(ctxt)); + + } while (!nbcon_context_enter_unsafe(ctxt)); +} +EXPORT_SYMBOL_GPL(nbcon_driver_acquire); + +/** + * nbcon_driver_release - Exit unsafe section and release the nbcon console + * @con: The nbcon console acquired in nbcon_driver_acquire() + */ +void nbcon_driver_release(struct console *con) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(con->nbcon_drvdata, ctxt); + + if (nbcon_context_exit_unsafe(ctxt)) + nbcon_context_release(ctxt); +} +EXPORT_SYMBOL_GPL(nbcon_driver_release);