From: Thomas Gleixner <tglx@linutronix.de> Date: Sun, 11 Sep 2022 00:28:04 +0200 Subject: [PATCH 07/24] printk: nobkl: Add buffer management In case of hostile takeovers it must be ensured that the previous owner cannot scribble over the output buffer of the emergency/panic context. This is achieved by: - Adding a global output buffer instance for early boot (pre per CPU data being available). - Allocating an output buffer per console for threaded printers once printer threads become available. - Allocating per CPU output buffers per console for printing from all contexts not covered by the other buffers. - Choosing the appropriate buffer is handled in the acquire/release functions. The output buffer is wrapped into a separate data structure so other context related fields can be added in later steps. Co-developed-by: John Ogness <john.ogness@linutronix.de> Signed-off-by: John Ogness <john.ogness@linutronix.de> Signed-off-by: Thomas Gleixner (Intel) <tglx@linutronix.de> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> --- include/linux/console.h | 13 ++++++ kernel/printk/internal.h | 22 +++++++++- kernel/printk/printk.c | 26 +++++++----- kernel/printk/printk_nobkl.c | 90 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 137 insertions(+), 14 deletions(-) Index: linux-6.3.0-rt11/include/linux/console.h =================================================================== @ linux-6.3.0-rt11/include/linux/console.h:182 @ enum cons_flags { * * @locked: Console is locked by a writer * @unsafe: Console is busy in a non takeover region + * @thread: Current owner is the printk thread * @cur_prio: The priority of the current output * @req_prio: The priority of a handover request * @cpu: The CPU on which the writer runs @ linux-6.3.0-rt11/include/linux/console.h:208 @ struct cons_state { struct { u32 locked : 1; u32 unsafe : 1; + u32 thread : 1; u32 cur_prio : 2; u32 req_prio : 2; u32 cpu : 18; @ linux-6.3.0-rt11/include/linux/console.h:239 @ enum cons_prio { }; struct console; +struct printk_buffers; /** * struct cons_context - Context for console acquire/release @ linux-6.3.0-rt11/include/linux/console.h:251 @ struct console; * @req_state: The request state for spin and cleanup * @spinwait_max_us: Limit for spinwait acquire * @prio: Priority of the context + * @pbufs: Pointer to the text buffer for this context + * @thread: The acquire is printk thread context * @hostile: Hostile takeover requested. Cleared on normal * acquire or friendly handover * @spinwait: Spinwait on acquire if possible @ linux-6.3.0-rt11/include/linux/console.h:265 @ struct cons_context { struct cons_state req_state; unsigned int spinwait_max_us; enum cons_prio prio; + struct printk_buffers *pbufs; + unsigned int thread : 1; unsigned int hostile : 1; unsigned int spinwait : 1; }; @ linux-6.3.0-rt11/include/linux/console.h:285 @ struct cons_write_context { bool unsafe; }; +struct cons_context_data; + /** * struct console - The console descriptor structure * @name: The name of the console driver @ linux-6.3.0-rt11/include/linux/console.h:308 @ struct cons_write_context { * @node: hlist node for the console list * * @atomic_state: State array for NOBKL consoles; real and handover + * @thread_pbufs: Pointer to thread private buffer + * @pcpu_data: Pointer to percpu context data */ struct console { char name[16]; @ linux-6.3.0-rt11/include/linux/console.h:332 @ struct console { /* NOBKL console specific members */ atomic_long_t __private atomic_state[2]; + struct printk_buffers *thread_pbufs; + struct cons_context_data __percpu *pcpu_data; }; #ifdef CONFIG_LOCKDEP Index: linux-6.3.0-rt11/kernel/printk/internal.h =================================================================== --- linux-6.3.0-rt11.orig/kernel/printk/internal.h +++ linux-6.3.0-rt11/kernel/printk/internal.h @ linux-6.3.0-rt11/include/linux/console.h:16 @ int devkmsg_sysctl_set_loglvl(struct ctl #define printk_sysctl_init() do { } while (0) #endif -#ifdef CONFIG_PRINTK +#define con_printk(lvl, con, fmt, ...) \ + printk(lvl pr_fmt("%s%sconsole [%s%d] " fmt), \ + (con->flags & CON_NO_BKL) ? "" : "legacy ", \ + (con->flags & CON_BOOT) ? "boot" : "", \ + con->name, con->index, ##__VA_ARGS__) +#ifdef CONFIG_PRINTK #ifdef CONFIG_PRINTK_CALLER #define PRINTK_PREFIX_MAX 48 #else @ linux-6.3.0-rt11/include/linux/console.h:72 @ u16 printk_parse_prefix(const char *text enum printk_info_flags *flags); void cons_nobkl_cleanup(struct console *con); -void cons_nobkl_init(struct console *con); +bool cons_nobkl_init(struct console *con); +bool cons_alloc_percpu_data(struct console *con); #else @ linux-6.3.0-rt11/include/linux/console.h:90 @ void cons_nobkl_init(struct console *con #define printk_safe_exit_irqrestore(flags) local_irq_restore(flags) static inline bool printk_percpu_data_ready(void) { return false; } -static inline void cons_nobkl_init(struct console *con) { } +static inline bool cons_nobkl_init(struct console *con) { return true; } static inline void cons_nobkl_cleanup(struct console *con) { } #endif /* CONFIG_PRINTK */ @ linux-6.3.0-rt11/include/linux/console.h:122 @ struct printk_message { u64 seq; unsigned long dropped; }; + +/** + * struct cons_context_data - console context data + * @pbufs: Buffer for storing the text + * + * Used for early boot and for per CPU data. + */ +struct cons_context_data { + struct printk_buffers pbufs; +}; Index: linux-6.3.0-rt11/kernel/printk/printk.c =================================================================== --- linux-6.3.0-rt11.orig/kernel/printk/printk.c +++ linux-6.3.0-rt11/kernel/printk/printk.c @ linux-6.3.0-rt11/include/linux/console.h:458 @ static bool have_bkl_console; */ bool have_boot_console; +static int unregister_console_locked(struct console *console); + #ifdef CONFIG_PRINTK DECLARE_WAIT_QUEUE_HEAD(log_wait); /* All 3 protected by @syslog_lock. */ @ linux-6.3.0-rt11/include/linux/console.h:1118 @ static inline void log_buf_add_cpu(void) static void __init set_percpu_data_ready(void) { + struct hlist_node *tmp; + struct console *con; + + console_list_lock(); + + hlist_for_each_entry_safe(con, tmp, &console_list, node) { + if (!cons_alloc_percpu_data(con)) + unregister_console_locked(con); + } + __printk_percpu_data_ready = true; + + console_list_unlock(); } static unsigned int __init add_to_rb(struct printk_ringbuffer *rb, @ linux-6.3.0-rt11/include/linux/console.h:3351 @ static void try_enable_default_console(s newcon->flags |= CON_CONSDEV; } -#define con_printk(lvl, con, fmt, ...) \ - printk(lvl pr_fmt("%s%sconsole [%s%d] " fmt), \ - (con->flags & CON_NO_BKL) ? "" : "legacy ", \ - (con->flags & CON_BOOT) ? "boot" : "", \ - con->name, con->index, ##__VA_ARGS__) - static void console_init_seq(struct console *newcon, bool bootcon_registered) { struct console *con; @ linux-6.3.0-rt11/include/linux/console.h:3415 @ static void console_init_seq(struct cons #define console_first() \ hlist_entry(console_list.first, struct console, node) -static int unregister_console_locked(struct console *console); - /* * The console driver calls this routine during kernel initialization * to register the console printing procedure with printk() and to @ linux-6.3.0-rt11/include/linux/console.h:3508 @ void register_console(struct console *ne if (!(newcon->flags & CON_NO_BKL)) have_bkl_console = true; - else - cons_nobkl_init(newcon); + else if (!cons_nobkl_init(newcon)) + goto unlock; if (newcon->flags & CON_BOOT) have_boot_console = true; Index: linux-6.3.0-rt11/kernel/printk/printk_nobkl.c =================================================================== --- linux-6.3.0-rt11.orig/kernel/printk/printk_nobkl.c +++ linux-6.3.0-rt11/kernel/printk/printk_nobkl.c @ linux-6.3.0-rt11/include/linux/console.h:169 @ static inline bool cons_check_panic(void return pcpu != PANIC_CPU_INVALID && pcpu != smp_processor_id(); } +static struct cons_context_data early_cons_ctxt_data __initdata; + +/** + * cons_context_set_pbufs - Set the output text buffer for the current context + * @ctxt: Pointer to the acquire context + * + * Buffer selection: + * 1) Early boot uses the global (initdata) buffer + * 2) Printer threads use the dynamically allocated per-console buffers + * 3) All other contexts use the per CPU buffers + * + * This guarantees that there is no concurrency on the output records ever. + * Early boot and per CPU nesting is not a problem. The takeover logic + * tells the interrupted context that the buffer has been overwritten. + * + * There are two critical regions that matter: + * + * 1) Context is filling the buffer with a record. After interruption + * it continues to sprintf() the record and before it goes to + * write it out, it checks the state, notices the takeover, discards + * the content and backs out. + * + * 2) Context is in a unsafe critical region in the driver. After + * interruption it might read overwritten data from the output + * buffer. When it leaves the critical region it notices and backs + * out. Hostile takeovers in driver critical regions are best effort + * and there is not much that can be done about that. + */ +static __ref void cons_context_set_pbufs(struct cons_context *ctxt) +{ + struct console *con = ctxt->console; + + /* Thread context or early boot? */ + if (ctxt->thread) + ctxt->pbufs = con->thread_pbufs; + else if (!con->pcpu_data) + ctxt->pbufs = &early_cons_ctxt_data.pbufs; + else + ctxt->pbufs = &(this_cpu_ptr(con->pcpu_data)->pbufs); +} + /** * cons_cleanup_handover - Cleanup a handover request * @ctxt: Pointer to acquire context @ linux-6.3.0-rt11/include/linux/console.h:545 @ again: } success: /* Common updates on success */ + cons_context_set_pbufs(ctxt); return true; check_hostile: @ linux-6.3.0-rt11/include/linux/console.h:668 @ static bool cons_release(struct cons_con { bool ret = __cons_release(ctxt); + /* Invalidate the buffer pointer. It is no longer valid. */ + ctxt->pbufs = NULL; + ctxt->state.atom = 0; return ret; } @ linux-6.3.0-rt11/include/linux/console.h:692 @ bool console_release(struct cons_write_c EXPORT_SYMBOL_GPL(console_release); /** + * cons_alloc_percpu_data - Allocate percpu data for a console + * @con: Console to allocate for + * + * Returns: True on success. False otherwise and the console cannot be used. + * + * If it is not yet possible to allocate per CPU data, success is returned. + * When per CPU data becomes possible, set_percpu_data_ready() will call + * this function again for all registered consoles. + */ +bool cons_alloc_percpu_data(struct console *con) +{ + if (!printk_percpu_data_ready()) + return true; + + con->pcpu_data = alloc_percpu(typeof(*con->pcpu_data)); + if (con->pcpu_data) + return true; + + con_printk(KERN_WARNING, con, "failed to allocate percpu buffers\n"); + return false; +} + +/** + * cons_free_percpu_data - Free percpu data of a console on unregister + * @con: Console to clean up + */ +static void cons_free_percpu_data(struct console *con) +{ + if (!con->pcpu_data) + return; + + free_percpu(con->pcpu_data); + con->pcpu_data = NULL; +} + +/** * cons_nobkl_init - Initialize the NOBKL console specific data * @con: Console to initialize + * + * Returns: True on success. False otherwise and the console cannot be used. */ -void cons_nobkl_init(struct console *con) +bool cons_nobkl_init(struct console *con) { struct cons_state state = { }; + if (!cons_alloc_percpu_data(con)) + return false; + cons_state_set(con, CON_STATE_CUR, &state); cons_state_set(con, CON_STATE_REQ, &state); + return true; } /** @ linux-6.3.0-rt11/include/linux/console.h:755 @ void cons_nobkl_cleanup(struct console * cons_state_set(con, CON_STATE_CUR, &state); cons_state_set(con, CON_STATE_REQ, &state); + cons_free_percpu_data(con); }