Subject: [PATCH 07/24] printk: nobkl: Add buffer management
From: Thomas Gleixner <[email-masked]>
Date: Sun, 11 Sep 2022 00:28:04 +0200

In case of hostile takeovers it must be ensured that the previous
owner cannot scribble over the output buffer of the emergency/panic
context. This is achieved by:

 - Adding a global output buffer instance for early boot (pre per CPU
   data being available).

 - Allocating an output buffer per console for threaded printers once
   printer threads become available.

 - Allocating per CPU output buffers per console for printing from
   all contexts not covered by the other buffers.

 - Choosing the appropriate buffer is handled in the acquire/release
   functions.

The output buffer is wrapped into a separate data structure so other
context related fields can be added in later steps.

Co-developed-by: John Ogness <[email-masked]>
Signed-off-by: John Ogness <[email-masked]>
Signed-off-by: Thomas Gleixner (Intel) <[email-masked]>
Signed-off-by: Sebastian Andrzej Siewior <[email-masked]>
---
 include/linux/console.h      |   13 ++++++
 kernel/printk/internal.h     |   22 +++++++++-
 kernel/printk/printk.c       |   26 +++++++-----
 kernel/printk/printk_nobkl.c |   90 ++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 137 insertions(+), 14 deletions(-)

Index: linux-6.3.0-rt11/include/linux/console.h
===================================================================
--- linux-6.3.0-rt11.orig/include/linux/console.h
+++ linux-6.3.0-rt11/include/linux/console.h
@@ -179,6 +179,7 @@ enum cons_flags {
  *
  * @locked:	Console is locked by a writer
  * @unsafe:	Console is busy in a non takeover region
+ * @thread:	Current owner is the printk thread
  * @cur_prio:	The priority of the current output
  * @req_prio:	The priority of a handover request
  * @cpu:	The CPU on which the writer runs
@@ -204,6 +205,7 @@ struct cons_state {
 				struct {
 					u32 locked	:  1;
 					u32 unsafe	:  1;
+					u32 thread	:  1;
 					u32 cur_prio	:  2;
 					u32 req_prio	:  2;
 					u32 cpu		: 18;
@@ -234,6 +236,7 @@ enum cons_prio {
 };
 
 struct console;
+struct printk_buffers;
 
 /**
  * struct cons_context - Context for console acquire/release
@@ -245,6 +248,8 @@ struct console;
  * @req_state:		The request state for spin and cleanup
  * @spinwait_max_us:	Limit for spinwait acquire
  * @prio:		Priority of the context
+ * @pbufs:		Pointer to the text buffer for this context
+ * @thread:		The acquire is printk thread context
  * @hostile:		Hostile takeover requested. Cleared on normal
  *			acquire or friendly handover
  * @spinwait:		Spinwait on acquire if possible
@@ -257,6 +262,8 @@ struct cons_context {
 	struct cons_state	req_state;
 	unsigned int		spinwait_max_us;
 	enum cons_prio		prio;
+	struct printk_buffers	*pbufs;
+	unsigned int		thread		: 1;
 	unsigned int		hostile		: 1;
 	unsigned int		spinwait	: 1;
 };
@@ -275,6 +282,8 @@ struct cons_write_context {
 	bool			unsafe;
 };
 
+struct cons_context_data;
+
 /**
  * struct console - The console descriptor structure
  * @name:		The name of the console driver
@@ -296,6 +305,8 @@ struct cons_write_context {
  * @node:		hlist node for the console list
  *
  * @atomic_state:	State array for NOBKL consoles; real and handover
+ * @thread_pbufs:	Pointer to thread private buffer
+ * @pcpu_data:		Pointer to percpu context data
  */
 struct console {
 	char			name[16];
@@ -318,6 +329,8 @@ struct console {
 
 	/* NOBKL console specific members */
 	atomic_long_t		__private atomic_state[2];
+	struct printk_buffers	*thread_pbufs;
+	struct cons_context_data	__percpu *pcpu_data;
 };
 
 #ifdef CONFIG_LOCKDEP
Index: linux-6.3.0-rt11/kernel/printk/internal.h
===================================================================
--- linux-6.3.0-rt11.orig/kernel/printk/internal.h
+++ linux-6.3.0-rt11/kernel/printk/internal.h
@@ -13,8 +13,13 @@ int devkmsg_sysctl_set_loglvl(struct ctl
 #define printk_sysctl_init() do { } while (0)
 #endif
 
-#ifdef CONFIG_PRINTK
+#define con_printk(lvl, con, fmt, ...)				\
+	printk(lvl pr_fmt("%s%sconsole [%s%d] " fmt),		\
+	       (con->flags & CON_NO_BKL) ? "" : "legacy ",	\
+	       (con->flags & CON_BOOT) ? "boot" : "",		\
+	       con->name, con->index, ##__VA_ARGS__)
 
+#ifdef CONFIG_PRINTK
 #ifdef CONFIG_PRINTK_CALLER
 #define PRINTK_PREFIX_MAX	48
 #else
@@ -64,7 +69,8 @@ u16 printk_parse_prefix(const char *text
 			enum printk_info_flags *flags);
 
 void cons_nobkl_cleanup(struct console *con);
-void cons_nobkl_init(struct console *con);
+bool cons_nobkl_init(struct console *con);
+bool cons_alloc_percpu_data(struct console *con);
 
 #else
 
@@ -81,7 +87,7 @@ void cons_nobkl_init(struct console *con
 #define printk_safe_exit_irqrestore(flags) local_irq_restore(flags)
 
 static inline bool printk_percpu_data_ready(void) { return false; }
-static inline void cons_nobkl_init(struct console *con) { }
+static inline bool cons_nobkl_init(struct console *con) { return true; }
 static inline void cons_nobkl_cleanup(struct console *con) { }
 
 #endif /* CONFIG_PRINTK */
@@ -113,3 +119,13 @@ struct printk_message {
 	u64			seq;
 	unsigned long		dropped;
 };
+
+/**
+ * struct cons_context_data - console context data
+ * @pbufs:		Buffer for storing the text
+ *
+ * Used for early boot and for per CPU data.
+ */
+struct cons_context_data {
+	struct printk_buffers		pbufs;
+};
Index: linux-6.3.0-rt11/kernel/printk/printk.c
===================================================================
--- linux-6.3.0-rt11.orig/kernel/printk/printk.c
+++ linux-6.3.0-rt11/kernel/printk/printk.c
@@ -455,6 +455,8 @@ static bool have_bkl_console;
  */
 bool have_boot_console;
 
+static int unregister_console_locked(struct console *console);
+
 #ifdef CONFIG_PRINTK
 DECLARE_WAIT_QUEUE_HEAD(log_wait);
 /* All 3 protected by @syslog_lock. */
@@ -1113,7 +1115,19 @@ static inline void log_buf_add_cpu(void)
 
 static void __init set_percpu_data_ready(void)
 {
+	struct hlist_node *tmp;
+	struct console *con;
+
+	console_list_lock();
+
+	hlist_for_each_entry_safe(con, tmp, &console_list, node) {
+		if (!cons_alloc_percpu_data(con))
+			unregister_console_locked(con);
+	}
+
 	__printk_percpu_data_ready = true;
+
+	console_list_unlock();
 }
 
 static unsigned int __init add_to_rb(struct printk_ringbuffer *rb,
@@ -3334,12 +3348,6 @@ static void try_enable_default_console(s
 		newcon->flags |= CON_CONSDEV;
 }
 
-#define con_printk(lvl, con, fmt, ...)				\
-	printk(lvl pr_fmt("%s%sconsole [%s%d] " fmt),		\
-	       (con->flags & CON_NO_BKL) ? "" : "legacy ",	\
-	       (con->flags & CON_BOOT) ? "boot" : "",		\
-	       con->name, con->index, ##__VA_ARGS__)
-
 static void console_init_seq(struct console *newcon, bool bootcon_registered)
 {
 	struct console *con;
@@ -3404,8 +3412,6 @@ static void console_init_seq(struct cons
 #define console_first()				\
 	hlist_entry(console_list.first, struct console, node)
 
-static int unregister_console_locked(struct console *console);
-
 /*
  * The console driver calls this routine during kernel initialization
  * to register the console printing procedure with printk() and to
@@ -3499,8 +3505,8 @@ void register_console(struct console *ne
 
 	if (!(newcon->flags & CON_NO_BKL))
 		have_bkl_console = true;
-	else
-		cons_nobkl_init(newcon);
+	else if (!cons_nobkl_init(newcon))
+		goto unlock;
 
 	if (newcon->flags & CON_BOOT)
 		have_boot_console = true;
Index: linux-6.3.0-rt11/kernel/printk/printk_nobkl.c
===================================================================
--- linux-6.3.0-rt11.orig/kernel/printk/printk_nobkl.c
+++ linux-6.3.0-rt11/kernel/printk/printk_nobkl.c
@@ -166,6 +166,47 @@ static inline bool cons_check_panic(void
 	return pcpu != PANIC_CPU_INVALID && pcpu != smp_processor_id();
 }
 
+static struct cons_context_data early_cons_ctxt_data __initdata;
+
+/**
+ * cons_context_set_pbufs - Set the output text buffer for the current context
+ * @ctxt:	Pointer to the acquire context
+ *
+ * Buffer selection:
+ *   1) Early boot uses the global (initdata) buffer
+ *   2) Printer threads use the dynamically allocated per-console buffers
+ *   3) All other contexts use the per CPU buffers
+ *
+ * This guarantees that there is no concurrency on the output records ever.
+ * Early boot and per CPU nesting is not a problem. The takeover logic
+ * tells the interrupted context that the buffer has been overwritten.
+ *
+ * There are two critical regions that matter:
+ *
+ * 1) Context is filling the buffer with a record. After interruption
+ *    it continues to sprintf() the record and before it goes to
+ *    write it out, it checks the state, notices the takeover, discards
+ *    the content and backs out.
+ *
+ * 2) Context is in a unsafe critical region in the driver. After
+ *    interruption it might read overwritten data from the output
+ *    buffer. When it leaves the critical region it notices and backs
+ *    out. Hostile takeovers in driver critical regions are best effort
+ *    and there is not much that can be done about that.
+ */
+static __ref void cons_context_set_pbufs(struct cons_context *ctxt)
+{
+	struct console *con = ctxt->console;
+
+	/* Thread context or early boot? */
+	if (ctxt->thread)
+		ctxt->pbufs = con->thread_pbufs;
+	else if (!con->pcpu_data)
+		ctxt->pbufs = &early_cons_ctxt_data.pbufs;
+	else
+		ctxt->pbufs = &(this_cpu_ptr(con->pcpu_data)->pbufs);
+}
+
 /**
  * cons_cleanup_handover - Cleanup a handover request
  * @ctxt:	Pointer to acquire context
@@ -501,6 +542,7 @@ again:
 	}
 success:
 	/* Common updates on success */
+	cons_context_set_pbufs(ctxt);
 	return true;
 
 check_hostile:
@@ -623,6 +665,9 @@ static bool cons_release(struct cons_con
 {
 	bool ret = __cons_release(ctxt);
 
+	/* Invalidate the buffer pointer. It is no longer valid. */
+	ctxt->pbufs = NULL;
+
 	ctxt->state.atom = 0;
 	return ret;
 }
@@ -644,15 +689,57 @@ bool console_release(struct cons_write_c
 EXPORT_SYMBOL_GPL(console_release);
 
 /**
+ * cons_alloc_percpu_data - Allocate percpu data for a console
+ * @con:	Console to allocate for
+ *
+ * Returns: True on success. False otherwise and the console cannot be used.
+ *
+ * If it is not yet possible to allocate per CPU data, success is returned.
+ * When per CPU data becomes possible, set_percpu_data_ready() will call
+ * this function again for all registered consoles.
+ */
+bool cons_alloc_percpu_data(struct console *con)
+{
+	if (!printk_percpu_data_ready())
+		return true;
+
+	con->pcpu_data = alloc_percpu(typeof(*con->pcpu_data));
+	if (con->pcpu_data)
+		return true;
+
+	con_printk(KERN_WARNING, con, "failed to allocate percpu buffers\n");
+	return false;
+}
+
+/**
+ * cons_free_percpu_data - Free percpu data of a console on unregister
+ * @con:	Console to clean up
+ */
+static void cons_free_percpu_data(struct console *con)
+{
+	if (!con->pcpu_data)
+		return;
+
+	free_percpu(con->pcpu_data);
+	con->pcpu_data = NULL;
+}
+
+/**
  * cons_nobkl_init - Initialize the NOBKL console specific data
  * @con:	Console to initialize
+ *
+ * Returns: True on success. False otherwise and the console cannot be used.
  */
-void cons_nobkl_init(struct console *con)
+bool cons_nobkl_init(struct console *con)
 {
 	struct cons_state state = { };
 
+	if (!cons_alloc_percpu_data(con))
+		return false;
+
 	cons_state_set(con, CON_STATE_CUR, &state);
 	cons_state_set(con, CON_STATE_REQ, &state);
+	return true;
 }
 
 /**
@@ -665,4 +752,5 @@ void cons_nobkl_cleanup(struct console *
 
 	cons_state_set(con, CON_STATE_CUR, &state);
 	cons_state_set(con, CON_STATE_REQ, &state);
+	cons_free_percpu_data(con);
 }