Commit | Line | Data |
---|---|---|
8c366db0 | 1 | /* SPDX-License-Identifier: GPL-2.0+ */ |
d8be8173 PM |
2 | /* |
3 | * Sleepable Read-Copy Update mechanism for mutual exclusion, | |
4 | * tiny variant. | |
5 | * | |
d8be8173 PM |
6 | * Copyright (C) IBM Corporation, 2017 |
7 | * | |
8c366db0 | 8 | * Author: Paul McKenney <paulmck@linux.ibm.com> |
d8be8173 PM |
9 | */ |
10 | ||
11 | #ifndef _LINUX_SRCU_TINY_H | |
12 | #define _LINUX_SRCU_TINY_H | |
13 | ||
14 | #include <linux/swait.h> | |
15 | ||
16 | struct srcu_struct { | |
3ddf20c9 PM |
17 | short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ |
18 | short srcu_idx; /* Current reader array element. */ | |
19 | u8 srcu_gp_running; /* GP workqueue running? */ | |
20 | u8 srcu_gp_waiting; /* GP waiting for readers? */ | |
d8be8173 PM |
21 | struct swait_queue_head srcu_wq; |
22 | /* Last srcu_read_unlock() wakes GP. */ | |
2464dd94 PM |
23 | struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ |
24 | struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ | |
d8be8173 PM |
25 | struct work_struct srcu_work; /* For driving grace periods. */ |
26 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
27 | struct lockdep_map dep_map; | |
28 | #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
29 | }; | |
30 | ||
31 | void srcu_drive_gp(struct work_struct *wp); | |
32 | ||
9c80172b | 33 | #define __SRCU_STRUCT_INIT(name, __ignored) \ |
d8be8173 PM |
34 | { \ |
35 | .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \ | |
2464dd94 | 36 | .srcu_cb_tail = &name.srcu_cb_head, \ |
d8be8173 PM |
37 | .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \ |
38 | __SRCU_DEP_MAP_INIT(name) \ | |
39 | } | |
40 | ||
41 | /* | |
42 | * This odd _STATIC_ arrangement is needed for API compatibility with | |
43 | * Tree SRCU, which needs some per-CPU data. | |
44 | */ | |
45 | #define DEFINE_SRCU(name) \ | |
9c80172b | 46 | struct srcu_struct name = __SRCU_STRUCT_INIT(name, name) |
d8be8173 | 47 | #define DEFINE_STATIC_SRCU(name) \ |
9c80172b | 48 | static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name) |
d8be8173 | 49 | |
aacb5d91 | 50 | void synchronize_srcu(struct srcu_struct *ssp); |
d8be8173 | 51 | |
d4efe6c5 PM |
52 | /* |
53 | * Counts the new reader in the appropriate per-CPU element of the | |
54 | * srcu_struct. Can be invoked from irq/bh handlers, but the matching | |
55 | * __srcu_read_unlock() must be in the same handler instance. Returns an | |
56 | * index that must be passed to the matching srcu_read_unlock(). | |
57 | */ | |
aacb5d91 | 58 | static inline int __srcu_read_lock(struct srcu_struct *ssp) |
d4efe6c5 PM |
59 | { |
60 | int idx; | |
61 | ||
aacb5d91 PM |
62 | idx = READ_ONCE(ssp->srcu_idx); |
63 | WRITE_ONCE(ssp->srcu_lock_nesting[idx], ssp->srcu_lock_nesting[idx] + 1); | |
d4efe6c5 PM |
64 | return idx; |
65 | } | |
66 | ||
aacb5d91 | 67 | static inline void synchronize_srcu_expedited(struct srcu_struct *ssp) |
d8be8173 | 68 | { |
aacb5d91 | 69 | synchronize_srcu(ssp); |
d8be8173 PM |
70 | } |
71 | ||
aacb5d91 | 72 | static inline void srcu_barrier(struct srcu_struct *ssp) |
d8be8173 | 73 | { |
aacb5d91 | 74 | synchronize_srcu(ssp); |
d8be8173 PM |
75 | } |
76 | ||
115a1a52 | 77 | /* Defined here to avoid size increase for non-torture kernels. */ |
aacb5d91 | 78 | static inline void srcu_torture_stats_print(struct srcu_struct *ssp, |
115a1a52 PM |
79 | char *tt, char *tf) |
80 | { | |
81 | int idx; | |
82 | ||
aacb5d91 | 83 | idx = READ_ONCE(ssp->srcu_idx) & 0x1; |
115a1a52 PM |
84 | pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n", |
85 | tt, tf, idx, | |
aacb5d91 PM |
86 | READ_ONCE(ssp->srcu_lock_nesting[!idx]), |
87 | READ_ONCE(ssp->srcu_lock_nesting[idx])); | |
115a1a52 PM |
88 | } |
89 | ||
d8be8173 | 90 | #endif |