Subject: x86: Support for lazy preemption From: Thomas Gleixner Date: Thu Nov 1 11:03:47 2012 +0100 From: Thomas Gleixner Implement the x86 pieces for lazy preempt. Signed-off-by: Thomas Gleixner --- arch/x86/Kconfig | 1 + arch/x86/include/asm/preempt.h | 33 ++++++++++++++++++++++++++++++++- arch/x86/include/asm/thread_info.h | 7 +++++++ include/linux/entry-common.h | 2 +- kernel/entry/common.c | 2 +- 5 files changed, 42 insertions(+), 3 deletions(-) --- Index: linux-6.3.0-rt11/arch/x86/Kconfig =================================================================== --- linux-6.3.0-rt11.orig/arch/x86/Kconfig +++ linux-6.3.0-rt11/arch/x86/Kconfig @@ -253,6 +253,7 @@ config X86 select HAVE_PCI select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_PREEMPT_LAZY select MMU_GATHER_RCU_TABLE_FREE if PARAVIRT select MMU_GATHER_MERGE_VMAS select HAVE_POSIX_CPU_TIMERS_TASK_WORK Index: linux-6.3.0-rt11/arch/x86/include/asm/preempt.h =================================================================== --- linux-6.3.0-rt11.orig/arch/x86/include/asm/preempt.h +++ linux-6.3.0-rt11/arch/x86/include/asm/preempt.h @@ -90,18 +90,49 @@ static __always_inline void __preempt_co * a decrement which hits zero means we have no preempt_count and should * reschedule. */ -static __always_inline bool __preempt_count_dec_and_test(void) +static __always_inline bool ____preempt_count_dec_and_test(void) { return GEN_UNARY_RMWcc("decl", pcpu_hot.preempt_count, e, __percpu_arg([var])); } +static __always_inline bool __preempt_count_dec_and_test(void) +{ + if (____preempt_count_dec_and_test()) + return true; +#ifdef CONFIG_PREEMPT_LAZY + if (preempt_count()) + return false; + if (current_thread_info()->preempt_lazy_count) + return false; + return test_thread_flag(TIF_NEED_RESCHED_LAZY); +#else + return false; +#endif +} + /* * Returns true when we need to resched and can (barring IRQ state). */ static __always_inline bool should_resched(int preempt_offset) { +#ifdef CONFIG_PREEMPT_LAZY + u32 tmp; + tmp = raw_cpu_read_4(pcpu_hot.preempt_count); + if (tmp == preempt_offset) + return true; + + /* preempt count == 0 ? */ + tmp &= ~PREEMPT_NEED_RESCHED; + if (tmp != preempt_offset) + return false; + /* XXX PREEMPT_LOCK_OFFSET */ + if (current_thread_info()->preempt_lazy_count) + return false; + return test_thread_flag(TIF_NEED_RESCHED_LAZY); +#else return unlikely(raw_cpu_read_4(pcpu_hot.preempt_count) == preempt_offset); +#endif } #ifdef CONFIG_PREEMPTION Index: linux-6.3.0-rt11/arch/x86/include/asm/thread_info.h =================================================================== --- linux-6.3.0-rt11.orig/arch/x86/include/asm/thread_info.h +++ linux-6.3.0-rt11/arch/x86/include/asm/thread_info.h @@ -57,6 +57,8 @@ struct thread_info { unsigned long flags; /* low level flags */ unsigned long syscall_work; /* SYSCALL_WORK_ flags */ u32 status; /* thread synchronous flags */ + int preempt_lazy_count; /* 0 => lazy preemptable + <0 => BUG */ #ifdef CONFIG_SMP u32 cpu; /* current CPU */ #endif @@ -65,6 +67,7 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .flags = 0, \ + .preempt_lazy_count = 0, \ } #else /* !__ASSEMBLY__ */ @@ -92,6 +95,7 @@ struct thread_info { #define TIF_NOCPUID 15 /* CPUID is not accessible in userland */ #define TIF_NOTSC 16 /* TSC is not accessible in userland */ #define TIF_NOTIFY_SIGNAL 17 /* signal notifications exist */ +#define TIF_NEED_RESCHED_LAZY 19 /* lazy rescheduling necessary */ #define TIF_MEMDIE 20 /* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 21 /* idle is polling for TIF_NEED_RESCHED */ #define TIF_IO_BITMAP 22 /* uses I/O bitmap */ @@ -115,6 +119,7 @@ struct thread_info { #define _TIF_NOCPUID (1 << TIF_NOCPUID) #define _TIF_NOTSC (1 << TIF_NOTSC) #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY) #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) #define _TIF_SPEC_FORCE_UPDATE (1 << TIF_SPEC_FORCE_UPDATE) @@ -146,6 +151,8 @@ struct thread_info { #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW) +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY) + #define STACK_WARN (THREAD_SIZE/8) /* Index: linux-6.3.0-rt11/include/linux/entry-common.h =================================================================== --- linux-6.3.0-rt11.orig/include/linux/entry-common.h +++ linux-6.3.0-rt11/include/linux/entry-common.h @@ -59,7 +59,7 @@ #define EXIT_TO_USER_MODE_WORK \ (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ - _TIF_NEED_RESCHED | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | \ + _TIF_NEED_RESCHED_MASK | _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | \ ARCH_EXIT_TO_USER_MODE_WORK) /** Index: linux-6.3.0-rt11/kernel/entry/common.c =================================================================== --- linux-6.3.0-rt11.orig/kernel/entry/common.c +++ linux-6.3.0-rt11/kernel/entry/common.c @@ -155,7 +155,7 @@ static unsigned long exit_to_user_mode_l local_irq_enable_exit_to_user(ti_work); - if (ti_work & _TIF_NEED_RESCHED) + if (ti_work & _TIF_NEED_RESCHED_MASK) schedule(); if (ti_work & _TIF_UPROBE)