include/linux/sched/idle.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_SCHED_IDLE_H
   3 #define _LINUX_SCHED_IDLE_H
   4
   5 #include <linux/sched.h>
   6
   7 enum cpu_idle_type {
   8         __CPU_NOT_IDLE = 0,
   9         CPU_IDLE,
  10         CPU_NEWLY_IDLE,
  11         CPU_MAX_IDLE_TYPES
  12 };
  13
  14 #ifdef CONFIG_SMP
  15 extern void wake_up_if_idle(int cpu);
  16 #else
  17 static inline void wake_up_if_idle(int cpu) { }
  18 #endif
  19
  20 /*
  21  * Idle thread specific functions to determine the need_resched
  22  * polling state.
  23  */
  24 #ifdef TIF_POLLING_NRFLAG
  25
  26 #ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H
  27
  28 static __always_inline void __current_set_polling(void)
  29 {
  30         arch_set_bit(TIF_POLLING_NRFLAG,
  31                      (unsigned long *)(&current_thread_info()->flags));
  32 }
  33
  34 static __always_inline void __current_clr_polling(void)
  35 {
  36         arch_clear_bit(TIF_POLLING_NRFLAG,
  37                        (unsigned long *)(&current_thread_info()->flags));
  38 }
  39
  40 #else
  41
  42 static __always_inline void __current_set_polling(void)
  43 {
  44         set_bit(TIF_POLLING_NRFLAG,
  45                 (unsigned long *)(&current_thread_info()->flags));
  46 }
  47
  48 static __always_inline void __current_clr_polling(void)
  49 {
  50         clear_bit(TIF_POLLING_NRFLAG,
  51                   (unsigned long *)(&current_thread_info()->flags));
  52 }
  53
  54 #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H */
  55
  56 static __always_inline bool __must_check current_set_polling_and_test(void)
  57 {
  58         __current_set_polling();
  59
  60         /*
  61          * Polling state must be visible before we test NEED_RESCHED,
  62          * paired by resched_curr()
  63          */
  64         smp_mb__after_atomic();
  65
  66         return unlikely(tif_need_resched());
  67 }
  68
  69 static __always_inline bool __must_check current_clr_polling_and_test(void)
  70 {
  71         __current_clr_polling();
  72
  73         /*
  74          * Polling state must be visible before we test NEED_RESCHED,
  75          * paired by resched_curr()
  76          */
  77         smp_mb__after_atomic();
  78
  79         return unlikely(tif_need_resched());
  80 }
  81
  82 #else
  83 static inline void __current_set_polling(void) { }
  84 static inline void __current_clr_polling(void) { }
  85
  86 static inline bool __must_check current_set_polling_and_test(void)
  87 {
  88         return unlikely(tif_need_resched());
  89 }
  90 static inline bool __must_check current_clr_polling_and_test(void)
  91 {
  92         return unlikely(tif_need_resched());
  93 }
  94 #endif
  95
  96 static __always_inline void current_clr_polling(void)
  97 {
  98         __current_clr_polling();
  99
 100         /*
 101          * Ensure we check TIF_NEED_RESCHED after we clear the polling bit.
 102          * Once the bit is cleared, we'll get IPIs with every new
 103          * TIF_NEED_RESCHED and the IPI handler, scheduler_ipi(), will also
 104          * fold.
 105          */
 106         smp_mb(); /* paired with resched_curr() */
 107
 108         preempt_fold_need_resched();
 109 }
 110
 111 #endif /* _LINUX_SCHED_IDLE_H */