include/linux/context_tracking_state.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_CONTEXT_TRACKING_STATE_H
   3 #define _LINUX_CONTEXT_TRACKING_STATE_H
   4
   5 #include <linux/percpu.h>
   6 #include <linux/static_key.h>
   7 #include <linux/context_tracking_irq.h>
   8
   9 /* Offset to allow distinguishing irq vs. task-based idle entry/exit. */
  10 #define DYNTICK_IRQ_NONIDLE     ((LONG_MAX / 2) + 1)
  11
  12 enum ctx_state {
  13         CONTEXT_DISABLED        = -1,   /* returned by ct_state() if unknown */
  14         CONTEXT_KERNEL          = 0,
  15         CONTEXT_IDLE            = 1,
  16         CONTEXT_USER            = 2,
  17         CONTEXT_GUEST           = 3,
  18         CONTEXT_MAX             = 4,
  19 };
  20
  21 /* Even value for idle, else odd. */
  22 #define RCU_DYNTICKS_IDX CONTEXT_MAX
  23
  24 #define CT_STATE_MASK (CONTEXT_MAX - 1)
  25 #define CT_DYNTICKS_MASK (~CT_STATE_MASK)
  26
  27 struct context_tracking {
  28 #ifdef CONFIG_CONTEXT_TRACKING_USER
  29         /*
  30          * When active is false, probes are unset in order
  31          * to minimize overhead: TIF flags are cleared
  32          * and calls to user_enter/exit are ignored. This
  33          * may be further optimized using static keys.
  34          */
  35         bool active;
  36         int recursion;
  37 #endif
  38 #ifdef CONFIG_CONTEXT_TRACKING
  39         atomic_t state;
  40 #endif
  41 #ifdef CONFIG_CONTEXT_TRACKING_IDLE
  42         long dynticks_nesting;          /* Track process nesting level. */
  43         long dynticks_nmi_nesting;      /* Track irq/NMI nesting level. */
  44 #endif
  45 };
  46
  47 #ifdef CONFIG_CONTEXT_TRACKING
  48 DECLARE_PER_CPU(struct context_tracking, context_tracking);
  49
  50 static __always_inline int __ct_state(void)
  51 {
  52         return arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK;
  53 }
  54 #endif
  55
  56 #ifdef CONFIG_CONTEXT_TRACKING_IDLE
  57 static __always_inline int ct_dynticks(void)
  58 {
  59         return atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_DYNTICKS_MASK;
  60 }
  61
  62 static __always_inline int ct_dynticks_cpu(int cpu)
  63 {
  64         struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
  65
  66         return atomic_read(&ct->state) & CT_DYNTICKS_MASK;
  67 }
  68
  69 static __always_inline int ct_dynticks_cpu_acquire(int cpu)
  70 {
  71         struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
  72
  73         return atomic_read_acquire(&ct->state) & CT_DYNTICKS_MASK;
  74 }
  75
  76 static __always_inline long ct_dynticks_nesting(void)
  77 {
  78         return __this_cpu_read(context_tracking.dynticks_nesting);
  79 }
  80
  81 static __always_inline long ct_dynticks_nesting_cpu(int cpu)
  82 {
  83         struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
  84
  85         return ct->dynticks_nesting;
  86 }
  87
  88 static __always_inline long ct_dynticks_nmi_nesting(void)
  89 {
  90         return __this_cpu_read(context_tracking.dynticks_nmi_nesting);
  91 }
  92
  93 static __always_inline long ct_dynticks_nmi_nesting_cpu(int cpu)
  94 {
  95         struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
  96
  97         return ct->dynticks_nmi_nesting;
  98 }
  99 #endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */
 100
 101 #ifdef CONFIG_CONTEXT_TRACKING_USER
 102 extern struct static_key_false context_tracking_key;
 103
 104 static __always_inline bool context_tracking_enabled(void)
 105 {
 106         return static_branch_unlikely(&context_tracking_key);
 107 }
 108
 109 static __always_inline bool context_tracking_enabled_cpu(int cpu)
 110 {
 111         return context_tracking_enabled() && per_cpu(context_tracking.active, cpu);
 112 }
 113
 114 static inline bool context_tracking_enabled_this_cpu(void)
 115 {
 116         return context_tracking_enabled() && __this_cpu_read(context_tracking.active);
 117 }
 118
 119 /**
 120  * ct_state() - return the current context tracking state if known
 121  *
 122  * Returns the current cpu's context tracking state if context tracking
 123  * is enabled.  If context tracking is disabled, returns
 124  * CONTEXT_DISABLED.  This should be used primarily for debugging.
 125  */
 126 static __always_inline int ct_state(void)
 127 {
 128         int ret;
 129
 130         if (!context_tracking_enabled())
 131                 return CONTEXT_DISABLED;
 132
 133         preempt_disable();
 134         ret = __ct_state();
 135         preempt_enable();
 136
 137         return ret;
 138 }
 139
 140 #else
 141 static __always_inline bool context_tracking_enabled(void) { return false; }
 142 static __always_inline bool context_tracking_enabled_cpu(int cpu) { return false; }
 143 static __always_inline bool context_tracking_enabled_this_cpu(void) { return false; }
 144 #endif /* CONFIG_CONTEXT_TRACKING_USER */
 145
 146 #endif