Commit | Line | Data |
---|---|---|
00de9d74 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
9b1d82fa PM |
2 | /* |
3 | * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. | |
4 | * | |
9b1d82fa PM |
5 | * Copyright IBM Corporation, 2008 |
6 | * | |
00de9d74 | 7 | * Author: Paul E. McKenney <paulmck@linux.ibm.com> |
9b1d82fa PM |
8 | * |
9 | * For detailed explanation of Read-Copy Update mechanism see - | |
4ce5b903 | 10 | * Documentation/RCU |
9b1d82fa | 11 | */ |
4ce5b903 IM |
12 | #include <linux/completion.h> |
13 | #include <linux/interrupt.h> | |
9b1d82fa | 14 | #include <linux/notifier.h> |
f9411ebe | 15 | #include <linux/rcupdate_wait.h> |
4ce5b903 | 16 | #include <linux/kernel.h> |
9984de1a | 17 | #include <linux/export.h> |
9b1d82fa | 18 | #include <linux/mutex.h> |
4ce5b903 IM |
19 | #include <linux/sched.h> |
20 | #include <linux/types.h> | |
21 | #include <linux/init.h> | |
9b1d82fa | 22 | #include <linux/time.h> |
4ce5b903 | 23 | #include <linux/cpu.h> |
268bb0ce | 24 | #include <linux/prefetch.h> |
9b1d82fa | 25 | |
29c00b4a PM |
26 | #include "rcu.h" |
27 | ||
6d48152e PM |
28 | /* Global control variables for rcupdate callback mechanism. */ |
29 | struct rcu_ctrlblk { | |
30 | struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */ | |
31 | struct rcu_head **donetail; /* ->next pointer of last "done" CB. */ | |
32 | struct rcu_head **curtail; /* ->next pointer of last CB. */ | |
33 | }; | |
34 | ||
35 | /* Definition for rcupdate control block. */ | |
709fdce7 PM |
36 | static struct rcu_ctrlblk rcu_ctrlblk = { |
37 | .donetail = &rcu_ctrlblk.rcucblist, | |
38 | .curtail = &rcu_ctrlblk.rcucblist, | |
6d48152e PM |
39 | }; |
40 | ||
709fdce7 | 41 | void rcu_barrier(void) |
9b1d82fa | 42 | { |
709fdce7 | 43 | wait_rcu_gp(call_rcu); |
9b1d82fa | 44 | } |
709fdce7 | 45 | EXPORT_SYMBOL(rcu_barrier); |
9b1d82fa | 46 | |
65cfe358 | 47 | /* Record an rcu quiescent state. */ |
709fdce7 | 48 | void rcu_qs(void) |
9b1d82fa | 49 | { |
b554d7de ED |
50 | unsigned long flags; |
51 | ||
52 | local_irq_save(flags); | |
709fdce7 PM |
53 | if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { |
54 | rcu_ctrlblk.donetail = rcu_ctrlblk.curtail; | |
18d7e406 | 55 | raise_softirq_irqoff(RCU_SOFTIRQ); |
65cfe358 | 56 | } |
b554d7de | 57 | local_irq_restore(flags); |
9b1d82fa PM |
58 | } |
59 | ||
60 | /* | |
61 | * Check to see if the scheduling-clock interrupt came from an extended | |
9b2e4f18 PM |
62 | * quiescent state, and, if so, tell RCU about it. This function must |
63 | * be called from hardirq context. It is normally called from the | |
64 | * scheduling-clock interrupt. | |
9b1d82fa | 65 | */ |
c98cac60 | 66 | void rcu_sched_clock_irq(int user) |
9b1d82fa | 67 | { |
c5bacd94 | 68 | if (user) { |
709fdce7 | 69 | rcu_qs(); |
c5bacd94 PM |
70 | } else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { |
71 | set_tsk_need_resched(current); | |
72 | set_preempt_need_resched(); | |
73 | } | |
9b1d82fa PM |
74 | } |
75 | ||
65cfe358 PM |
76 | /* Invoke the RCU callbacks whose grace period has elapsed. */ |
77 | static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) | |
9b1d82fa | 78 | { |
9b1d82fa | 79 | struct rcu_head *next, *list; |
4ce5b903 | 80 | unsigned long flags; |
9b1d82fa | 81 | |
9b1d82fa PM |
82 | /* Move the ready-to-invoke callbacks to a local list. */ |
83 | local_irq_save(flags); | |
709fdce7 | 84 | if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) { |
6e91f8cb PM |
85 | /* No callbacks ready, so just leave. */ |
86 | local_irq_restore(flags); | |
87 | return; | |
88 | } | |
709fdce7 PM |
89 | list = rcu_ctrlblk.rcucblist; |
90 | rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail; | |
91 | *rcu_ctrlblk.donetail = NULL; | |
92 | if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail) | |
93 | rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist; | |
94 | rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist; | |
9b1d82fa PM |
95 | local_irq_restore(flags); |
96 | ||
97 | /* Invoke the callbacks on the local list. */ | |
98 | while (list) { | |
99 | next = list->next; | |
100 | prefetch(next); | |
551d55a9 | 101 | debug_rcu_head_unqueue(list); |
b2c0710c | 102 | local_bh_disable(); |
6d48152e | 103 | __rcu_reclaim("", list); |
b2c0710c | 104 | local_bh_enable(); |
9b1d82fa PM |
105 | list = next; |
106 | } | |
107 | } | |
108 | ||
9b1d82fa PM |
109 | /* |
110 | * Wait for a grace period to elapse. But it is illegal to invoke | |
679d3f30 PM |
111 | * synchronize_rcu() from within an RCU read-side critical section. |
112 | * Therefore, any legal call to synchronize_rcu() is a quiescent | |
113 | * state, and so on a UP system, synchronize_rcu() need do nothing. | |
65cfe358 PM |
114 | * (But Lai Jiangshan points out the benefits of doing might_sleep() |
115 | * to reduce latency.) | |
9b1d82fa PM |
116 | * |
117 | * Cool, huh? (Due to Josh Triplett.) | |
9b1d82fa | 118 | */ |
709fdce7 | 119 | void synchronize_rcu(void) |
9b1d82fa | 120 | { |
f78f5b90 PM |
121 | RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || |
122 | lock_is_held(&rcu_lock_map) || | |
123 | lock_is_held(&rcu_sched_lock_map), | |
679d3f30 | 124 | "Illegal synchronize_rcu() in RCU read-side critical section"); |
9b1d82fa | 125 | } |
709fdce7 | 126 | EXPORT_SYMBOL_GPL(synchronize_rcu); |
9b1d82fa | 127 | |
9b1d82fa | 128 | /* |
679d3f30 | 129 | * Post an RCU callback to be invoked after the end of an RCU grace |
65cfe358 PM |
130 | * period. But since we have but one CPU, that would be after any |
131 | * quiescent state. | |
9b1d82fa | 132 | */ |
709fdce7 | 133 | void call_rcu(struct rcu_head *head, rcu_callback_t func) |
9b1d82fa PM |
134 | { |
135 | unsigned long flags; | |
136 | ||
551d55a9 | 137 | debug_rcu_head_queue(head); |
9b1d82fa PM |
138 | head->func = func; |
139 | head->next = NULL; | |
4ce5b903 | 140 | |
9b1d82fa | 141 | local_irq_save(flags); |
709fdce7 PM |
142 | *rcu_ctrlblk.curtail = head; |
143 | rcu_ctrlblk.curtail = &head->next; | |
9b1d82fa | 144 | local_irq_restore(flags); |
5f6130fa LJ |
145 | |
146 | if (unlikely(is_idle_task(current))) { | |
709fdce7 | 147 | /* force scheduling for rcu_qs() */ |
5f6130fa LJ |
148 | resched_cpu(0); |
149 | } | |
9b1d82fa | 150 | } |
709fdce7 | 151 | EXPORT_SYMBOL_GPL(call_rcu); |
9dc5ad32 | 152 | |
aa23c6fb | 153 | void __init rcu_init(void) |
9dc5ad32 PM |
154 | { |
155 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); | |
aa23c6fb | 156 | rcu_early_boot_tests(); |
e0fcba9a | 157 | srcu_init(); |
9dc5ad32 | 158 | } |