Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
f6180121 MJ |
2 | /* Event cache for netfilter. */ |
3 | ||
f229f6ce PM |
4 | /* |
5 | * (C) 2005 Harald Welte <laforge@gnumonks.org> | |
6 | * (C) 2005 Patrick McHardy <kaber@trash.net> | |
7 | * (C) 2005-2006 Netfilter Core Team <coreteam@netfilter.org> | |
8 | * (C) 2005 USAGI/WIDE Project <http://www.linux-ipv6.org> | |
f6180121 MJ |
9 | */ |
10 | ||
5191d70f AS |
11 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
12 | ||
f6180121 MJ |
13 | #include <linux/types.h> |
14 | #include <linux/netfilter.h> | |
15 | #include <linux/skbuff.h> | |
16 | #include <linux/vmalloc.h> | |
17 | #include <linux/stddef.h> | |
18 | #include <linux/err.h> | |
f6180121 MJ |
19 | #include <linux/kernel.h> |
20 | #include <linux/netdevice.h> | |
5a0e3ad6 | 21 | #include <linux/slab.h> |
bc3b2d7f | 22 | #include <linux/export.h> |
f6180121 MJ |
23 | |
24 | #include <net/netfilter/nf_conntrack.h> | |
f6180121 | 25 | #include <net/netfilter/nf_conntrack_core.h> |
40d102cd | 26 | #include <net/netfilter/nf_conntrack_ecache.h> |
a0891aa6 | 27 | #include <net/netfilter/nf_conntrack_extend.h> |
f6180121 | 28 | |
e34d5c1a | 29 | static DEFINE_MUTEX(nf_ct_ecache_mutex); |
13b18339 | 30 | |
2ed3bf18 FW |
31 | #define DYING_NULLS_VAL ((1 << 30) + 1) |
32 | #define ECACHE_MAX_JIFFIES msecs_to_jiffies(10) | |
33 | #define ECACHE_RETRY_JIFFIES msecs_to_jiffies(10) | |
9500507c FW |
34 | |
35 | enum retry_state { | |
36 | STATE_CONGESTED, | |
37 | STATE_RESTART, | |
38 | STATE_DONE, | |
39 | }; | |
40 | ||
0d3cc504 FW |
41 | struct nf_conntrack_net_ecache *nf_conn_pernet_ecache(const struct net *net) |
42 | { | |
43 | struct nf_conntrack_net *cnet = nf_ct_pernet(net); | |
44 | ||
45 | return &cnet->ecache; | |
46 | } | |
47 | #if IS_MODULE(CONFIG_NF_CT_NETLINK) | |
48 | EXPORT_SYMBOL_GPL(nf_conn_pernet_ecache); | |
49 | #endif | |
50 | ||
2ed3bf18 | 51 | static enum retry_state ecache_work_evict_list(struct nf_conntrack_net *cnet) |
9500507c | 52 | { |
2ed3bf18 FW |
53 | unsigned long stop = jiffies + ECACHE_MAX_JIFFIES; |
54 | struct hlist_nulls_head evicted_list; | |
63f55acf | 55 | enum retry_state ret = STATE_DONE; |
9500507c FW |
56 | struct nf_conntrack_tuple_hash *h; |
57 | struct hlist_nulls_node *n; | |
2ed3bf18 | 58 | unsigned int sent; |
9500507c | 59 | |
2ed3bf18 | 60 | INIT_HLIST_NULLS_HEAD(&evicted_list, DYING_NULLS_VAL); |
9500507c | 61 | |
2ed3bf18 FW |
62 | next: |
63 | sent = 0; | |
64 | spin_lock_bh(&cnet->ecache.dying_lock); | |
65 | ||
66 | hlist_nulls_for_each_entry_safe(h, n, &cnet->ecache.dying_list, hnnode) { | |
9500507c | 67 | struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); |
9500507c | 68 | |
2ed3bf18 FW |
69 | /* The worker owns all entries, ct remains valid until nf_ct_put |
70 | * in the loop below. | |
63f55acf | 71 | */ |
9500507c FW |
72 | if (nf_conntrack_event(IPCT_DESTROY, ct)) { |
73 | ret = STATE_CONGESTED; | |
74 | break; | |
75 | } | |
76 | ||
2ed3bf18 FW |
77 | hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); |
78 | hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode, &evicted_list); | |
9500507c | 79 | |
2ed3bf18 | 80 | if (time_after(stop, jiffies)) { |
9500507c FW |
81 | ret = STATE_RESTART; |
82 | break; | |
83 | } | |
2ed3bf18 FW |
84 | |
85 | if (sent++ > 16) { | |
86 | spin_unlock_bh(&cnet->ecache.dying_lock); | |
87 | cond_resched(); | |
88 | goto next; | |
89 | } | |
9500507c FW |
90 | } |
91 | ||
2ed3bf18 | 92 | spin_unlock_bh(&cnet->ecache.dying_lock); |
9500507c | 93 | |
2ed3bf18 FW |
94 | hlist_nulls_for_each_entry_safe(h, n, &evicted_list, hnnode) { |
95 | struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h); | |
96 | ||
2ed3bf18 FW |
97 | hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode); |
98 | nf_ct_put(ct); | |
99 | ||
100 | cond_resched(); | |
101 | } | |
9500507c FW |
102 | |
103 | return ret; | |
104 | } | |
105 | ||
106 | static void ecache_work(struct work_struct *work) | |
107 | { | |
9027ce0b | 108 | struct nf_conntrack_net *cnet = container_of(work, struct nf_conntrack_net, ecache.dwork.work); |
2ed3bf18 FW |
109 | int ret, delay = -1; |
110 | ||
111 | ret = ecache_work_evict_list(cnet); | |
112 | switch (ret) { | |
113 | case STATE_CONGESTED: | |
114 | delay = ECACHE_RETRY_JIFFIES; | |
115 | break; | |
116 | case STATE_RESTART: | |
117 | delay = 0; | |
118 | break; | |
119 | case STATE_DONE: | |
120 | break; | |
9500507c FW |
121 | } |
122 | ||
9500507c | 123 | if (delay >= 0) |
9027ce0b | 124 | schedule_delayed_work(&cnet->ecache.dwork, delay); |
9500507c FW |
125 | } |
126 | ||
b3afdc17 | 127 | static int __nf_conntrack_eventmask_report(struct nf_conntrack_ecache *e, |
8dd8678e FW |
128 | const u32 events, |
129 | const u32 missed, | |
b3afdc17 | 130 | const struct nf_ct_event *item) |
3c435e2e | 131 | { |
b3afdc17 | 132 | struct net *net = nf_ct_net(item->ct); |
3c435e2e | 133 | struct nf_ct_event_notifier *notify; |
8dd8678e | 134 | u32 old, want; |
b3afdc17 FW |
135 | int ret; |
136 | ||
137 | if (!((events | missed) & e->ctmask)) | |
138 | return 0; | |
139 | ||
140 | rcu_read_lock(); | |
141 | ||
142 | notify = rcu_dereference(net->ct.nf_conntrack_event_cb); | |
143 | if (!notify) { | |
144 | rcu_read_unlock(); | |
145 | return 0; | |
146 | } | |
147 | ||
b86c0e64 | 148 | ret = notify->ct_event(events | missed, item); |
b3afdc17 FW |
149 | rcu_read_unlock(); |
150 | ||
151 | if (likely(ret >= 0 && missed == 0)) | |
152 | return 0; | |
153 | ||
8dd8678e FW |
154 | do { |
155 | old = READ_ONCE(e->missed); | |
156 | if (ret < 0) | |
157 | want = old | events; | |
158 | else | |
159 | want = old & ~missed; | |
160 | } while (cmpxchg(&e->missed, old, want) != old); | |
b3afdc17 FW |
161 | |
162 | return ret; | |
163 | } | |
164 | ||
165 | int nf_conntrack_eventmask_report(unsigned int events, struct nf_conn *ct, | |
166 | u32 portid, int report) | |
167 | { | |
3c435e2e | 168 | struct nf_conntrack_ecache *e; |
478374a3 | 169 | struct nf_ct_event item; |
8dd8678e | 170 | unsigned int missed; |
b3afdc17 | 171 | int ret; |
478374a3 FW |
172 | |
173 | if (!nf_ct_is_confirmed(ct)) | |
b3afdc17 | 174 | return 0; |
3c435e2e FW |
175 | |
176 | e = nf_ct_ecache_find(ct); | |
177 | if (!e) | |
b3afdc17 | 178 | return 0; |
3c435e2e | 179 | |
478374a3 FW |
180 | memset(&item, 0, sizeof(item)); |
181 | ||
182 | item.ct = ct; | |
183 | item.portid = e->portid ? e->portid : portid; | |
184 | item.report = report; | |
185 | ||
186 | /* This is a resent of a destroy event? If so, skip missed */ | |
187 | missed = e->portid ? 0 : e->missed; | |
188 | ||
b3afdc17 FW |
189 | ret = __nf_conntrack_eventmask_report(e, events, missed, &item); |
190 | if (unlikely(ret < 0 && (events & (1 << IPCT_DESTROY)))) { | |
191 | /* This is a destroy event that has been triggered by a process, | |
192 | * we store the PORTID to include it in the retransmission. | |
9291f090 | 193 | */ |
b3afdc17 FW |
194 | if (e->portid == 0 && portid != 0) |
195 | e->portid = portid; | |
3c435e2e | 196 | } |
9291f090 | 197 | |
3c435e2e FW |
198 | return ret; |
199 | } | |
200 | EXPORT_SYMBOL_GPL(nf_conntrack_eventmask_report); | |
201 | ||
f6180121 MJ |
202 | /* deliver cached events and clear cache entry - must be called with locally |
203 | * disabled softirqs */ | |
a0891aa6 | 204 | void nf_ct_deliver_cached_events(struct nf_conn *ct) |
f6180121 | 205 | { |
a0891aa6 | 206 | struct nf_conntrack_ecache *e; |
58020f77 | 207 | struct nf_ct_event item; |
8dd8678e | 208 | unsigned int events; |
e34d5c1a | 209 | |
ad88b7a6 | 210 | if (!nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct)) |
b3afdc17 | 211 | return; |
ad88b7a6 | 212 | |
a0891aa6 PNA |
213 | e = nf_ct_ecache_find(ct); |
214 | if (e == NULL) | |
b3afdc17 | 215 | return; |
a0891aa6 PNA |
216 | |
217 | events = xchg(&e->cache, 0); | |
218 | ||
58020f77 | 219 | item.ct = ct; |
15e47304 | 220 | item.portid = 0; |
58020f77 TZ |
221 | item.report = 0; |
222 | ||
b3afdc17 FW |
223 | /* We make a copy of the missed event cache without taking |
224 | * the lock, thus we may send missed events twice. However, | |
225 | * this does not harm and it happens very rarely. | |
226 | */ | |
227 | __nf_conntrack_eventmask_report(e, events, e->missed, &item); | |
f6180121 | 228 | } |
13b18339 | 229 | EXPORT_SYMBOL_GPL(nf_ct_deliver_cached_events); |
f6180121 | 230 | |
ecdfb48c FW |
231 | void nf_ct_expect_event_report(enum ip_conntrack_expect_events event, |
232 | struct nf_conntrack_expect *exp, | |
233 | u32 portid, int report) | |
234 | ||
235 | { | |
236 | struct net *net = nf_ct_exp_net(exp); | |
bd1431db | 237 | struct nf_ct_event_notifier *notify; |
ecdfb48c FW |
238 | struct nf_conntrack_ecache *e; |
239 | ||
240 | rcu_read_lock(); | |
bd1431db | 241 | notify = rcu_dereference(net->ct.nf_conntrack_event_cb); |
ecdfb48c FW |
242 | if (!notify) |
243 | goto out_unlock; | |
244 | ||
245 | e = nf_ct_ecache_find(exp->master); | |
246 | if (!e) | |
247 | goto out_unlock; | |
248 | ||
249 | if (e->expmask & (1 << event)) { | |
250 | struct nf_exp_event item = { | |
251 | .exp = exp, | |
252 | .portid = portid, | |
253 | .report = report | |
254 | }; | |
b86c0e64 | 255 | notify->exp_event(1 << event, &item); |
ecdfb48c FW |
256 | } |
257 | out_unlock: | |
258 | rcu_read_unlock(); | |
259 | } | |
260 | ||
b86c0e64 FW |
261 | void nf_conntrack_register_notifier(struct net *net, |
262 | const struct nf_ct_event_notifier *new) | |
010c7d6f | 263 | { |
b56f2d55 | 264 | struct nf_ct_event_notifier *notify; |
e34d5c1a PNA |
265 | |
266 | mutex_lock(&nf_ct_ecache_mutex); | |
70e9942f | 267 | notify = rcu_dereference_protected(net->ct.nf_conntrack_event_cb, |
b56f2d55 | 268 | lockdep_is_held(&nf_ct_ecache_mutex)); |
b86c0e64 | 269 | WARN_ON_ONCE(notify); |
cf778b00 | 270 | rcu_assign_pointer(net->ct.nf_conntrack_event_cb, new); |
e34d5c1a | 271 | mutex_unlock(&nf_ct_ecache_mutex); |
010c7d6f PM |
272 | } |
273 | EXPORT_SYMBOL_GPL(nf_conntrack_register_notifier); | |
274 | ||
b86c0e64 | 275 | void nf_conntrack_unregister_notifier(struct net *net) |
010c7d6f | 276 | { |
e34d5c1a | 277 | mutex_lock(&nf_ct_ecache_mutex); |
70e9942f | 278 | RCU_INIT_POINTER(net->ct.nf_conntrack_event_cb, NULL); |
e34d5c1a | 279 | mutex_unlock(&nf_ct_ecache_mutex); |
bd1431db | 280 | /* synchronize_rcu() is called after netns pre_exit */ |
010c7d6f PM |
281 | } |
282 | EXPORT_SYMBOL_GPL(nf_conntrack_unregister_notifier); | |
283 | ||
1379940b FW |
284 | void nf_conntrack_ecache_work(struct net *net, enum nf_ct_ecache_state state) |
285 | { | |
0418b989 | 286 | struct nf_conntrack_net *cnet = nf_ct_pernet(net); |
1379940b FW |
287 | |
288 | if (state == NFCT_ECACHE_DESTROY_FAIL && | |
9027ce0b FW |
289 | !delayed_work_pending(&cnet->ecache.dwork)) { |
290 | schedule_delayed_work(&cnet->ecache.dwork, HZ); | |
1379940b FW |
291 | net->ct.ecache_dwork_pending = true; |
292 | } else if (state == NFCT_ECACHE_DESTROY_SENT) { | |
2ed3bf18 FW |
293 | if (!hlist_nulls_empty(&cnet->ecache.dying_list)) |
294 | mod_delayed_work(system_wq, &cnet->ecache.dwork, 0); | |
295 | else | |
296 | net->ct.ecache_dwork_pending = false; | |
1379940b FW |
297 | } |
298 | } | |
299 | ||
b0a7ab4a FW |
300 | bool nf_ct_ecache_ext_add(struct nf_conn *ct, u16 ctmask, u16 expmask, gfp_t gfp) |
301 | { | |
302 | struct net *net = nf_ct_net(ct); | |
303 | struct nf_conntrack_ecache *e; | |
304 | ||
90d1daa4 FW |
305 | switch (net->ct.sysctl_events) { |
306 | case 0: | |
307 | /* assignment via template / ruleset? ignore sysctl. */ | |
308 | if (ctmask || expmask) | |
309 | break; | |
310 | return true; | |
311 | case 2: /* autodetect: no event listener, don't allocate extension. */ | |
312 | if (!READ_ONCE(net->ct.ctnetlink_has_listener)) | |
313 | return true; | |
314 | fallthrough; | |
315 | case 1: | |
316 | /* always allocate an extension. */ | |
317 | if (!ctmask && !expmask) { | |
318 | ctmask = ~0; | |
319 | expmask = ~0; | |
320 | } | |
321 | break; | |
322 | default: | |
323 | WARN_ON_ONCE(1); | |
324 | return true; | |
b0a7ab4a | 325 | } |
b0a7ab4a FW |
326 | |
327 | e = nf_ct_ext_add(ct, NF_CT_EXT_ECACHE, gfp); | |
328 | if (e) { | |
329 | e->ctmask = ctmask; | |
330 | e->expmask = expmask; | |
331 | } | |
332 | ||
333 | return e != NULL; | |
334 | } | |
335 | EXPORT_SYMBOL_GPL(nf_ct_ecache_ext_add); | |
336 | ||
90d1daa4 | 337 | #define NF_CT_EVENTS_DEFAULT 2 |
a0891aa6 PNA |
338 | static int nf_ct_events __read_mostly = NF_CT_EVENTS_DEFAULT; |
339 | ||
fc3893fd | 340 | void nf_conntrack_ecache_pernet_init(struct net *net) |
a0891aa6 | 341 | { |
0418b989 | 342 | struct nf_conntrack_net *cnet = nf_ct_pernet(net); |
1379940b | 343 | |
a0891aa6 | 344 | net->ct.sysctl_events = nf_ct_events; |
9027ce0b | 345 | |
9027ce0b | 346 | INIT_DELAYED_WORK(&cnet->ecache.dwork, ecache_work); |
2ed3bf18 FW |
347 | INIT_HLIST_NULLS_HEAD(&cnet->ecache.dying_list, DYING_NULLS_VAL); |
348 | spin_lock_init(&cnet->ecache.dying_lock); | |
1015c3de | 349 | |
8dd8678e | 350 | BUILD_BUG_ON(__IPCT_MAX >= 16); /* e->ctmask is u16 */ |
3fe0f943 | 351 | } |
a0891aa6 | 352 | |
3fe0f943 G |
353 | void nf_conntrack_ecache_pernet_fini(struct net *net) |
354 | { | |
0418b989 | 355 | struct nf_conntrack_net *cnet = nf_ct_pernet(net); |
1379940b | 356 | |
9027ce0b | 357 | cancel_delayed_work_sync(&cnet->ecache.dwork); |
3fe0f943 | 358 | } |