printk: Revert console kthreads for 5.19
Subsystems (5)
Petr Mladek
maintainercover printk: Revert console kthreads for 5.19
We have had several discussions with the other printk developers and together decided that we would like more time to investigate the implicit expectations [1] of each and every console driver because now we are concerned that there may be more hidden implicit expectations.
We much appreciate the chance to have this in 5.19 release candidates because the wider user base helped us to get feedback from user cases and architectures that we were not able to cover. And we see that the patchset needs some more love.
We are sorry for the inconvenience and thanks a lot for the feedback and help with debugging and testing.
This patchset reverts only the final patches adding the kthreads and per-console. It keeps the refactoring of the console_unlock() code that makes sense on its own.
[1] https://lore.kernel.org/all/YrBdjVwBOVgLfHyb@alley/
PS: I am going to prepare a proper pull request. I just wanted
to send this in advance just in case anyone wants to comment on it.
Petr Mladek (6):
Revert "printk: Wait for the global console lock when the system is
going down"
Revert "printk: Block console kthreads when direct printing will be
required"
Revert "printk: remove @console_locked"
Revert "printk: extend console_lock for per-console locking" Revert "printk: add kthread console printers"
Revert "printk: add functions to prefer direct printing"
drivers/tty/sysrq.c | 2 - include/linux/console.h | 17 -- include/linux/printk.h | 16 - kernel/hung_task.c | 11 +- kernel/panic.c | 6 - kernel/printk/internal.h | 2 - kernel/printk/printk.c | 593 ++++-------------------------------- kernel/printk/printk_safe.c | 32 -- kernel/rcu/tree_stall.h | 2 - kernel/reboot.c | 16 +- kernel/watchdog.c | 4 - kernel/watchdog_hld.c | 4 - 12 files changed, 65 insertions(+), 640 deletions(-)
Signature · 1 line
2.35.3
Petr Mladek
maintainerPATCH 1/6 Revert "printk: Wait for the global console lock when the system is going down"
This reverts commit b87f02307d3cfbda768520f0687c51ca77e14fc3.
The testing of 5.19 release candidates revealed missing synchronization between early and regular console functionality.
It would be possible to start the console kthreads later as a workaround. But it is clear that console lock serialized console drivers between each other. It opens a big area of possible problems that were not considered by people involved in the development and review.
printk() is crucial for debugging kernel issues and console output is very important part of it. The number of consoles is huge and a proper review would take some time. As a result it need to be reverted for 5.19.
--- include/linux/printk.h | 5 ----- kernel/panic.c | 2 -- kernel/printk/internal.h | 2 -- kernel/printk/printk.c | 4 ---- kernel/printk/printk_safe.c | 32 -------------------------------- kernel/reboot.c | 2 -- 6 files changed, 47 deletions(-)
include/linux/printk.h +0 -5
@@ -173,7 +173,6 @@ extern void printk_prefer_direct_enter(void);extern void printk_prefer_direct_exit (void );extern bool pr_flush (int timeout_ms,bool reset_on_progress); -extern void try_block_console_kthreads (int timeout_ms);/* * Please don't use printk_ratelimit(), because it shares ratelimiting state
@@ -238,10 +237,6 @@ static inline bool pr_flush(int timeout_ms, bool reset_on_progress)return true; } -static inline void try_block_console_kthreads (int timeout_ms) -{ -} -static inline int printk_ratelimit (void ) {return 0 ;
kernel/panic.c +0 -2
@@ -297,7 +297,6 @@ void panic(const char *fmt, ...)* unfortunately means it may not be hardened to work in a * panic situation. */ -try_block_console_kthreads (10000 );smp_send_stop (); }else {/*
@@ -305,7 +304,6 @@ void panic(const char *fmt, ...)* kmsg_dump, we will need architecture dependent extra * works in addition to stopping other CPUs. */ -try_block_console_kthreads (10000 );crash_smp_send_stop (); }
kernel/printk/internal.h +0 -2
@@ -20,8 +20,6 @@ enum printk_info_flags {LOG_CONT =8 ,/* text is a fragment of a continuation line */ }; -extern bool block_console_kthreads; - __printf(4, 0) int vprintk_store(int facility, int level, const struct dev_printk_info *dev_info,
kernel/printk/printk.c +0 -4
@@ -250,9 +250,6 @@ static atomic_t console_kthreads_active = ATOMIC_INIT(0);#define console_kthread_printing_exit () \ atomic_dec(&console_kthreads_active) -/* Block console kthreads to avoid processing new messages. */ -bool block_console_kthreads; -/* * Helper macros to handle lockdep when locking/unlocking console_sem. We use * macros instead of functions so that _RET_IP_ contains useful information.
@@ -3733,7 +3730,6 @@ static bool printer_should_wake(struct console *con, u64 seq)if (con->blocked ||console_kthreads_atomically_blocked () || - block_console_kthreads || system_state >SYSTEM_RUNNING || oops_in_progress) {return false;
kernel/printk/printk_safe.c +0 -32
@@ -8,9 +8,7 @@#include <linux/smp.h> #include <linux/cpumask.h> #include <linux/printk.h> -#include <linux/console.h> #include <linux/kprobes.h> -#include <linux/delay.h> #include "internal.h"
@@ -52,33 +50,3 @@ asmlinkage int vprintk(const char *fmt, va_list args)return vprintk_default (fmt, args); }EXPORT_SYMBOL (vprintk); - -/** -* try_block_console_kthreads() - Try to block console kthreads and -* make the global console_lock() avaialble -* -* @timeout_ms: The maximum time (in ms) to wait. -* -* Prevent console kthreads from starting processing new messages. Wait -* until the global console_lock() become available. -* -* Context: Can be called in any context. -*/ -void try_block_console_kthreads (int timeout_ms) -{ - block_console_kthreads = true; - -/* Do not wait when the console lock could not be safely taken. */ -if (this_cpu_read (printk_context) ||in_nmi ()) -return ; - -while (timeout_ms >0 ) { -if (console_trylock ()) { -console_unlock (); -return ; - } - -udelay (1000 ); - timeout_ms -=1 ; - } -}
kernel/reboot.c +0 -2
@@ -82,7 +82,6 @@ void kernel_restart_prepare(char *cmd) {blocking_notifier_call_chain (&reboot_notifier_list,SYS_RESTART , cmd); system_state =SYSTEM_RESTART ; -try_block_console_kthreads (10000 );usermodehelper_disable ();device_shutdown (); }
@@ -271,7 +270,6 @@ static void kernel_shutdown_prepare(enum system_states state)blocking_notifier_call_chain (&reboot_notifier_list, (state ==SYSTEM_HALT ) ?SYS_HALT :SYS_POWER_OFF ,NULL ); system_state = state; -try_block_console_kthreads (10000 );usermodehelper_disable ();device_shutdown (); }
Signature · 1 line
2.35.3
Petr Mladek
maintainerPATCH 2/6 Revert "printk: Block console kthreads when direct printing will be required"
In reply to Petr MladekThis reverts commit c3230283e2819a69dad2cf7a63143fde8bab8b5c.
The testing of 5.19 release candidates revealed missing synchronization between early and regular console functionality.
It would be possible to start the console kthreads later as a workaround. But it is clear that console lock serialized console drivers between each other. It opens a big area of possible problems that were not considered by people involved in the development and review.
printk() is crucial for debugging kernel issues and console output is very important part of it. The number of consoles is huge and a proper review would take some time. As a result it need to be reverted for 5.19.
--- kernel/printk/printk.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
kernel/printk/printk.c +1 -3
@@ -3729,9 +3729,7 @@ static bool printer_should_wake(struct console *con, u64 seq)return true;if (con->blocked || -console_kthreads_atomically_blocked () || - system_state >SYSTEM_RUNNING || - oops_in_progress) { +console_kthreads_atomically_blocked ()) {return false; }
Signature · 1 line
2.35.3
Petr Mladek
maintainerPATCH 3/6 Revert "printk: remove @console_locked"
In reply to Petr MladekThis reverts commit ab406816fca009349b89cbde885daf68a8c77e33.
The testing of 5.19 release candidates revealed missing synchronization between early and regular console functionality.
It would be possible to start the console kthreads later as a workaround. But it is clear that console lock serialized console drivers between each other. It opens a big area of possible problems that were not considered by people involved in the development and review.
printk() is crucial for debugging kernel issues and console output is very important part of it. The number of consoles is huge and a proper review would take some time. As a result it need to be reverted for 5.19.
--- kernel/printk/printk.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-)
kernel/printk/printk.c +15 -14
@@ -340,7 +340,15 @@ static void console_kthreads_unblock(void) console_kthreads_blocked = false; } -static int console_suspended; +/* +* This is used for debugging the mess that is the VT code by +* keeping track if we have the console semaphore held. It's +* definitely not the perfect debug tool (we don't know if _WE_ +* hold it and are racing, but it helps tracking those weird code +* paths in the console code where we end up in places I want +* locked without the console semaphore held). +*/ +static int console_locked, console_suspended;/* * Array of consoles built from command line options (console=)
@@ -2711,6 +2719,7 @@ void console_lock(void)if (console_suspended)return ;console_kthreads_block (); + console_locked =1 ; console_may_schedule =1 ; }EXPORT_SYMBOL (console_lock);
@@ -2735,26 +2744,15 @@ int console_trylock(void)up_console_sem ();return 0 ; } + console_locked =1 ; console_may_schedule =0 ;return 1 ; }EXPORT_SYMBOL (console_trylock); -/* -* This is used to help to make sure that certain paths within the VT code are -* running with the console lock held. It is definitely not the perfect debug -* tool (it is not known if the VT code is the task holding the console lock), -* but it helps tracking those weird code paths in the console code such as -* when the console is suspended: where the console is not locked but no -* console printing may occur. -* -* Note: This returns true when the console is suspended but is not locked. -* This is intentional because the VT code must consider that situation -* the same as if the console was locked. -*/ int is_console_locked (void ) { -return (console_kthreads_blocked ||atomic_read (&console_kthreads_active)); +return (console_locked ||atomic_read (&console_kthreads_active)); }EXPORT_SYMBOL (is_console_locked);
@@ -2810,6 +2808,8 @@ static inline bool console_is_usable(struct console *con)static void __console_unlock (void ) { + console_locked =0 ; +/* * Depending on whether console_lock() or console_trylock() was used, * appropriately allow the kthread printers to continue.
@@ -3127,6 +3127,7 @@ void console_unblank(void) }else console_lock (); + console_locked =1 ; console_may_schedule =0 ;for_each_console (c)if ((c->flags &CON_ENABLED ) && c->unblank)
Signature · 1 line
2.35.3
Petr Mladek
maintainerPATCH 4/6 Revert "printk: extend console_lock for per-console locking"
In reply to Petr MladekThis reverts commit 8e274732115f63c1d09136284431b3555bd5cc56.
The testing of 5.19 release candidates revealed missing synchronization between early and regular console functionality.
It would be possible to start the console kthreads later as a workaround. But it is clear that console lock serialized console drivers between each other. It opens a big area of possible problems that were not considered by people involved in the development and review.
printk() is crucial for debugging kernel issues and console output is very important part of it. The number of consoles is huge and a proper review would take some time. As a result it need to be reverted for 5.19.
--- include/linux/console.h | 15 --- kernel/printk/printk.c | 261 +++++++++------------------------------- 2 files changed, 56 insertions(+), 220 deletions(-)
include/linux/console.h +0 -15
@@ -16,7 +16,6 @@#include <linux/atomic.h> #include <linux/types.h> -#include <linux/mutex.h> struct vc_data ;struct console_font_op ;
@@ -155,20 +154,6 @@ struct console {u64 seq;unsigned long dropped;struct task_struct *thread; -bool blocked; - -/* -* The per-console lock is used by printing kthreads to synchronize -* this console with callers of console_lock(). This is necessary in -* order to allow printing kthreads to run in parallel to each other, -* while each safely accessing the @blocked field and synchronizing -* against direct printing via console_lock/console_unlock. -* -* Note: For synchronizing against direct printing via -* console_trylock/console_unlock, see the static global -* variable @console_kthreads_active. -*/ -struct mutex lock;void *data;struct console *next;
kernel/printk/printk.c +56 -205
417 lines not on this page · raw message
Signature · 1 line
2.35.3
Petr Mladek
maintainerPATCH 5/6 Revert "printk: add kthread console printers"
In reply to Petr MladekThis reverts commit 09c5ba0aa2fcfdadb17d045c3ee6f86d69270df7.
This reverts commit b87f02307d3cfbda768520f0687c51ca77e14fc3.
The testing of 5.19 release candidates revealed missing synchronization between early and regular console functionality.
It would be possible to start the console kthreads later as a workaround. But it is clear that console lock serialized console drivers between each other. It opens a big area of possible problems that were not considered by people involved in the development and review.
printk() is crucial for debugging kernel issues and console output is very important part of it. The number of consoles is huge and a proper review would take some time. As a result it need to be reverted for 5.19.
--- include/linux/console.h | 2 - kernel/printk/printk.c | 329 +++------------------------------------- 2 files changed, 22 insertions(+), 309 deletions(-)
include/linux/console.h +0 -2
@@ -153,8 +153,6 @@ struct console {uint ospeed;u64 seq;unsigned long dropped; -struct task_struct *thread; -void *data;struct console *next; };
kernel/printk/printk.c +22 -307
472 lines not on this page · raw message
Signature · 1 line
2.35.3
Petr Mladek
maintainerPATCH 6/6 Revert "printk: add functions to prefer direct printing"
In reply to Petr MladekThis reverts commit 2bb2b7b57f81255c13f4395ea911d6bdc70c9fe2.
The testing of 5.19 release candidates revealed missing synchronization between early and regular console functionality.
It would be possible to start the console kthreads later as a workaround. But it is clear that console lock serialized console drivers between each other. It opens a big area of possible problems that were not considered by people involved in the development and review.
printk() is crucial for debugging kernel issues and console output is very important part of it. The number of consoles is huge and a proper review would take some time. As a result it need to be reverted for 5.19.
--- drivers/tty/sysrq.c | 2 -- include/linux/printk.h | 11 ----------- kernel/hung_task.c | 11 +---------- kernel/panic.c | 4 ---- kernel/printk/printk.c | 28 ---------------------------- kernel/rcu/tree_stall.h | 2 -- kernel/reboot.c | 14 +------------- kernel/watchdog.c | 4 ---- kernel/watchdog_hld.c | 4 ---- 9 files changed, 2 insertions(+), 78 deletions(-)
drivers/tty/sysrq.c +0 -2
@@ -581,7 +581,6 @@ void __handle_sysrq(int key, bool check_mask)rcu_sysrq_start ();rcu_read_lock (); -printk_prefer_direct_enter ();/* * Raise the apparent loglevel to maximum so that the sysrq header * is shown to provide the user with positive feedback. We do not
@@ -623,7 +622,6 @@ void __handle_sysrq(int key, bool check_mask)pr_cont ("\n" ); console_loglevel = orig_log_level; } -printk_prefer_direct_exit ();rcu_read_unlock ();rcu_sysrq_end ();
include/linux/printk.h +0 -11
@@ -169,9 +169,6 @@ extern void __printk_safe_exit(void);#define printk_deferred_enter __printk_safe_enter#define printk_deferred_exit __printk_safe_exit -extern void printk_prefer_direct_enter (void ); -extern void printk_prefer_direct_exit (void ); -extern bool pr_flush (int timeout_ms,bool reset_on_progress);/*
@@ -224,14 +221,6 @@ static inline void printk_deferred_exit(void) { } -static inline void printk_prefer_direct_enter (void ) -{ -} - -static inline void printk_prefer_direct_exit (void ) -{ -} -static inline bool pr_flush (int timeout_ms,bool reset_on_progress) {return true;
kernel/hung_task.c +1 -10
@@ -127,8 +127,6 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout)* complain: */ if (sysctl_hung_task_warnings) { -printk_prefer_direct_enter (); -if (sysctl_hung_task_warnings >0 ) sysctl_hung_task_warnings--; pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
@@ -144,8 +142,6 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout)if (sysctl_hung_task_all_cpu_backtrace) hung_task_show_all_bt = true; - -printk_prefer_direct_exit (); }touch_nmi_watchdog ();
@@ -208,17 +204,12 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout) } unlock:rcu_read_unlock (); -if (hung_task_show_lock) { -printk_prefer_direct_enter (); +if (hung_task_show_lock)debug_show_all_locks (); -printk_prefer_direct_exit (); - }if (hung_task_show_all_bt) { hung_task_show_all_bt = false; -printk_prefer_direct_enter ();trigger_all_cpu_backtrace (); -printk_prefer_direct_exit (); }if (hung_task_call_panic)
kernel/panic.c +0 -4
@@ -603,8 +603,6 @@ void __warn(const char *file, int line, void *caller, unsigned taint, {disable_trace_on_warning (); -printk_prefer_direct_enter (); -if (file) pr_warn("WARNING: CPU: %d PID: %d at %s:%d %pS\n", raw_smp_processor_id(), current->pid, file, line,
@@ -634,8 +632,6 @@ void __warn(const char *file, int line, void *caller, unsigned taint,/* Just a warning, don't kill lockdep. */ add_taint (taint,LOCKDEP_STILL_OK ); - -printk_prefer_direct_exit (); }#ifndef __WARN_FLAGS
kernel/printk/printk.c +0 -28
@@ -362,34 +362,6 @@ static int console_msg_format = MSG_FORMAT_DEFAULT;static DEFINE_MUTEX (syslog_lock);#ifdef CONFIG_PRINTK -static atomic_t printk_prefer_direct =ATOMIC_INIT (0 ); - -/** -* printk_prefer_direct_enter - cause printk() calls to attempt direct -* printing to all enabled consoles -* -* Since it is not possible to call into the console printing code from any -* context, there is no guarantee that direct printing will occur. -* -* This globally effects all printk() callers. -* -* Context: Any context. -*/ -void printk_prefer_direct_enter (void ) -{ -atomic_inc (&printk_prefer_direct); -} - -/** -* printk_prefer_direct_exit - restore printk() behavior -* -* Context: Any context. -*/ -void printk_prefer_direct_exit (void ) -{ -WARN_ON (atomic_dec_if_positive (&printk_prefer_direct) <0 ); -} -DECLARE_WAIT_QUEUE_HEAD (log_wait);/* All 3 protected by @syslog_lock. */ /* the next printk record to read by syslog(READ) or /proc/kmsg */
kernel/rcu/tree_stall.h +0 -2
@@ -647,7 +647,6 @@ static void print_cpu_stall(unsigned long gps)* See Documentation/RCU/stallwarn.rst for info on how to debug * RCU CPU stall warnings. */ -printk_prefer_direct_enter ();trace_rcu_stall_warning (rcu_state.name,TPS ("SelfDetected" ));pr_err ("INFO: %s self-detected stall on CPU\n" , rcu_state.name);raw_spin_lock_irqsave_rcu_node (rdp->mynode, flags);
@@ -685,7 +684,6 @@ static void print_cpu_stall(unsigned long gps)*/ set_tsk_need_resched (current);set_preempt_need_resched (); -printk_prefer_direct_exit (); }static void check_cpu_stall (struct rcu_data *rdp)
kernel/reboot.c +1 -13
@@ -819,11 +819,9 @@ static int __orderly_reboot(void) ret =run_cmd (reboot_cmd);if (ret) { -printk_prefer_direct_enter ();pr_warn ("Failed to start orderly reboot: forcing the issue\n" );emergency_sync ();kernel_restart (NULL ); -printk_prefer_direct_exit (); }return ret;
@@ -836,7 +834,6 @@ static int __orderly_poweroff(bool force) ret =run_cmd (poweroff_cmd);if (ret && force) { -printk_prefer_direct_enter ();pr_warn ("Failed to start orderly shutdown: forcing the issue\n" );/*
@@ -846,7 +843,6 @@ static int __orderly_poweroff(bool force)*/ emergency_sync ();kernel_power_off (); -printk_prefer_direct_exit (); }return ret;
@@ -904,8 +900,6 @@ EXPORT_SYMBOL_GPL(orderly_reboot);*/ static void hw_failure_emergency_poweroff_func (struct work_struct *work) { -printk_prefer_direct_enter (); -/* * We have reached here after the emergency shutdown waiting period has * expired. This means orderly_poweroff has not been able to shut off
@@ -922,8 +916,6 @@ static void hw_failure_emergency_poweroff_func(struct work_struct *work)*/ pr_emerg ("Hardware protection shutdown failed. Trying emergency restart\n" );emergency_restart (); - -printk_prefer_direct_exit (); } static DECLARE_DELAYED_WORK(hw_failure_emergency_poweroff_work,
@@ -962,13 +954,11 @@ void hw_protection_shutdown(const char *reason, int ms_until_forced) {static atomic_t allow_proceed =ATOMIC_INIT (1 ); -printk_prefer_direct_enter (); -pr_emerg ("HARDWARE PROTECTION shutdown (%s)\n" , reason);/* Shutdown should be initiated only once. */ if (!atomic_dec_and_test (&allow_proceed)) - goto out; +return ;/* * Queue a backup emergency shutdown in the event of
@@ -976,8 +966,6 @@ void hw_protection_shutdown(const char *reason, int ms_until_forced)*/ hw_failure_emergency_poweroff (ms_until_forced);orderly_poweroff (true); -out: -printk_prefer_direct_exit (); }EXPORT_SYMBOL_GPL (hw_protection_shutdown);
kernel/watchdog.c +0 -4
@@ -424,8 +424,6 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)/* Start period for the next softlockup warning. */ update_report_ts (); -printk_prefer_direct_enter (); -pr_emerg ("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n" ,smp_processor_id (), duration, current->comm,task_pid_nr (current));
@@ -444,8 +442,6 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)add_taint (TAINT_SOFTLOCKUP ,LOCKDEP_STILL_OK );if (softlockup_panic)panic ("softlockup: hung tasks" ); - -printk_prefer_direct_exit (); }return HRTIMER_RESTART ;
kernel/watchdog_hld.c +0 -4
@@ -135,8 +135,6 @@ static void watchdog_overflow_callback(struct perf_event *event,if (__this_cpu_read (hard_watchdog_warn) == true)return ; -printk_prefer_direct_enter (); -pr_emerg ("Watchdog detected hard LOCKUP on cpu %d\n" , this_cpu);print_modules ();
@@ -157,8 +155,6 @@ static void watchdog_overflow_callback(struct perf_event *event,if (hardlockup_panic)nmi_panic (regs,"Hard LOCKUP" ); -printk_prefer_direct_exit (); -__this_cpu_write (hard_watchdog_warn, true);return ; }
Signature · 1 line
2.35.3