Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
1da177e4 LT |
2 | /* |
3 | * net/core/dst.c Protocol independent destination cache. | |
4 | * | |
5 | * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> | |
6 | * | |
7 | */ | |
8 | ||
9 | #include <linux/bitops.h> | |
10 | #include <linux/errno.h> | |
11 | #include <linux/init.h> | |
12 | #include <linux/kernel.h> | |
86bba269 | 13 | #include <linux/workqueue.h> |
1da177e4 LT |
14 | #include <linux/mm.h> |
15 | #include <linux/module.h> | |
5a0e3ad6 | 16 | #include <linux/slab.h> |
1da177e4 | 17 | #include <linux/netdevice.h> |
1da177e4 LT |
18 | #include <linux/skbuff.h> |
19 | #include <linux/string.h> | |
20 | #include <linux/types.h> | |
e9dc8653 | 21 | #include <net/net_namespace.h> |
2fc1b5dd | 22 | #include <linux/sched.h> |
268bb0ce | 23 | #include <linux/prefetch.h> |
61adedf3 | 24 | #include <net/lwtunnel.h> |
b6ca8bd5 | 25 | #include <net/xfrm.h> |
1da177e4 LT |
26 | |
27 | #include <net/dst.h> | |
f38a9eb1 | 28 | #include <net/dst_metadata.h> |
1da177e4 | 29 | |
ede2059d | 30 | int dst_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb) |
1da177e4 LT |
31 | { |
32 | kfree_skb(skb); | |
33 | return 0; | |
34 | } | |
ede2059d | 35 | EXPORT_SYMBOL(dst_discard_out); |
1da177e4 | 36 | |
3fb07daf | 37 | const struct dst_metrics dst_default_metrics = { |
a37e6e34 ED |
38 | /* This initializer is needed to force linker to place this variable |
39 | * into const section. Otherwise it might end into bss section. | |
40 | * We really want to avoid false sharing on this variable, and catch | |
41 | * any writes on it. | |
42 | */ | |
9620fef2 | 43 | .refcnt = REFCOUNT_INIT(1), |
a37e6e34 | 44 | }; |
d4ead6b3 | 45 | EXPORT_SYMBOL(dst_default_metrics); |
a37e6e34 | 46 | |
f38a9eb1 | 47 | void dst_init(struct dst_entry *dst, struct dst_ops *ops, |
762c8dc7 | 48 | struct net_device *dev, int initial_obsolete, |
f38a9eb1 | 49 | unsigned short flags) |
1da177e4 | 50 | { |
5c1e6aa3 | 51 | dst->dev = dev; |
d62607c3 | 52 | netdev_hold(dev, &dst->dev_tracker, GFP_ATOMIC); |
1da177e4 | 53 | dst->ops = ops; |
3fb07daf | 54 | dst_init_metrics(dst, dst_default_metrics.metrics, true); |
cf911662 | 55 | dst->expires = 0UL; |
cf911662 DM |
56 | #ifdef CONFIG_XFRM |
57 | dst->xfrm = NULL; | |
58 | #endif | |
5c1e6aa3 | 59 | dst->input = dst_discard; |
ede2059d | 60 | dst->output = dst_discard_out; |
cf911662 | 61 | dst->error = 0; |
5c1e6aa3 | 62 | dst->obsolete = initial_obsolete; |
cf911662 DM |
63 | dst->header_len = 0; |
64 | dst->trailer_len = 0; | |
65 | #ifdef CONFIG_IP_ROUTE_CLASSID | |
66 | dst->tclassid = 0; | |
1da177e4 | 67 | #endif |
61adedf3 | 68 | dst->lwtstate = NULL; |
762c8dc7 | 69 | rcuref_init(&dst->__rcuref, 1); |
418a7307 | 70 | INIT_LIST_HEAD(&dst->rt_uncached); |
cf911662 | 71 | dst->__use = 0; |
5c1e6aa3 DM |
72 | dst->lastuse = jiffies; |
73 | dst->flags = flags; | |
957c665f DM |
74 | if (!(flags & DST_NOCOUNT)) |
75 | dst_entries_add(ops, 1); | |
f38a9eb1 TG |
76 | } |
77 | EXPORT_SYMBOL(dst_init); | |
78 | ||
79 | void *dst_alloc(struct dst_ops *ops, struct net_device *dev, | |
762c8dc7 | 80 | int initial_obsolete, unsigned short flags) |
f38a9eb1 TG |
81 | { |
82 | struct dst_entry *dst; | |
83 | ||
cf86a086 ED |
84 | if (ops->gc && |
85 | !(flags & DST_NOCOUNT) && | |
af6d1034 JM |
86 | dst_entries_get_fast(ops) > ops->gc_thresh) |
87 | ops->gc(ops); | |
f38a9eb1 TG |
88 | |
89 | dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC); | |
90 | if (!dst) | |
91 | return NULL; | |
92 | ||
762c8dc7 | 93 | dst_init(dst, ops, dev, initial_obsolete, flags); |
f38a9eb1 | 94 | |
1da177e4 LT |
95 | return dst; |
96 | } | |
598ed936 | 97 | EXPORT_SYMBOL(dst_alloc); |
1da177e4 | 98 | |
1da177e4 LT |
99 | struct dst_entry *dst_destroy(struct dst_entry * dst) |
100 | { | |
b92cf4aa | 101 | struct dst_entry *child = NULL; |
1da177e4 LT |
102 | |
103 | smp_rmb(); | |
104 | ||
b92cf4aa | 105 | #ifdef CONFIG_XFRM |
b6ca8bd5 DM |
106 | if (dst->xfrm) { |
107 | struct xfrm_dst *xdst = (struct xfrm_dst *) dst; | |
108 | ||
109 | child = xdst->child; | |
110 | } | |
b92cf4aa | 111 | #endif |
957c665f DM |
112 | if (!(dst->flags & DST_NOCOUNT)) |
113 | dst_entries_add(dst->ops, -1); | |
1da177e4 LT |
114 | |
115 | if (dst->ops->destroy) | |
116 | dst->ops->destroy(dst); | |
d62607c3 | 117 | netdev_put(dst->dev, &dst->dev_tracker); |
f38a9eb1 | 118 | |
e252b3d1 WC |
119 | lwtstate_put(dst->lwtstate); |
120 | ||
f38a9eb1 | 121 | if (dst->flags & DST_METADATA) |
d71785ff | 122 | metadata_dst_free((struct metadata_dst *)dst); |
f38a9eb1 TG |
123 | else |
124 | kmem_cache_free(dst->ops->kmem_cachep, dst); | |
1da177e4 LT |
125 | |
126 | dst = child; | |
52df157f WW |
127 | if (dst) |
128 | dst_release_immediate(dst); | |
1da177e4 LT |
129 | return NULL; |
130 | } | |
598ed936 | 131 | EXPORT_SYMBOL(dst_destroy); |
1da177e4 | 132 | |
f8864972 ED |
133 | static void dst_destroy_rcu(struct rcu_head *head) |
134 | { | |
135 | struct dst_entry *dst = container_of(head, struct dst_entry, rcu_head); | |
136 | ||
137 | dst = dst_destroy(dst); | |
f8864972 ED |
138 | } |
139 | ||
4a6ce2b6 WW |
140 | /* Operations to mark dst as DEAD and clean up the net device referenced |
141 | * by dst: | |
1be107de | 142 | * 1. put the dst under blackhole interface and discard all tx/rx packets |
4a6ce2b6 WW |
143 | * on this route. |
144 | * 2. release the net_device | |
145 | * This function should be called when removing routes from the fib tree | |
146 | * in preparation for a NETDEV_DOWN/NETDEV_UNREGISTER event and also to | |
147 | * make the next dst_ops->check() fail. | |
148 | */ | |
149 | void dst_dev_put(struct dst_entry *dst) | |
150 | { | |
151 | struct net_device *dev = dst->dev; | |
152 | ||
153 | dst->obsolete = DST_OBSOLETE_DEAD; | |
154 | if (dst->ops->ifdown) | |
43c28172 | 155 | dst->ops->ifdown(dst, dev); |
4a6ce2b6 WW |
156 | dst->input = dst_discard; |
157 | dst->output = dst_discard_out; | |
8d7017fd | 158 | dst->dev = blackhole_netdev; |
d62607c3 JK |
159 | netdev_ref_replace(dev, blackhole_netdev, &dst->dev_tracker, |
160 | GFP_ATOMIC); | |
4a6ce2b6 WW |
161 | } |
162 | EXPORT_SYMBOL(dst_dev_put); | |
163 | ||
8d330868 IJ |
164 | void dst_release(struct dst_entry *dst) |
165 | { | |
bc9d3a9f TG |
166 | if (dst && rcuref_put(&dst->__rcuref)) |
167 | call_rcu_hurry(&dst->rcu_head, dst_destroy_rcu); | |
8d330868 IJ |
168 | } |
169 | EXPORT_SYMBOL(dst_release); | |
170 | ||
5f56f409 WW |
171 | void dst_release_immediate(struct dst_entry *dst) |
172 | { | |
bc9d3a9f TG |
173 | if (dst && rcuref_put(&dst->__rcuref)) |
174 | dst_destroy(dst); | |
5f56f409 WW |
175 | } |
176 | EXPORT_SYMBOL(dst_release_immediate); | |
177 | ||
62fa8a84 DM |
178 | u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old) |
179 | { | |
3fb07daf | 180 | struct dst_metrics *p = kmalloc(sizeof(*p), GFP_ATOMIC); |
62fa8a84 DM |
181 | |
182 | if (p) { | |
3fb07daf | 183 | struct dst_metrics *old_p = (struct dst_metrics *)__DST_METRICS_PTR(old); |
62fa8a84 DM |
184 | unsigned long prev, new; |
185 | ||
9620fef2 | 186 | refcount_set(&p->refcnt, 1); |
3fb07daf | 187 | memcpy(p->metrics, old_p->metrics, sizeof(p->metrics)); |
62fa8a84 DM |
188 | |
189 | new = (unsigned long) p; | |
190 | prev = cmpxchg(&dst->_metrics, old, new); | |
191 | ||
192 | if (prev != old) { | |
193 | kfree(p); | |
3fb07daf | 194 | p = (struct dst_metrics *)__DST_METRICS_PTR(prev); |
62fa8a84 DM |
195 | if (prev & DST_METRICS_READ_ONLY) |
196 | p = NULL; | |
3fb07daf | 197 | } else if (prev & DST_METRICS_REFCOUNTED) { |
9620fef2 | 198 | if (refcount_dec_and_test(&old_p->refcnt)) |
3fb07daf | 199 | kfree(old_p); |
62fa8a84 DM |
200 | } |
201 | } | |
3fb07daf ED |
202 | BUILD_BUG_ON(offsetof(struct dst_metrics, metrics) != 0); |
203 | return (u32 *)p; | |
62fa8a84 DM |
204 | } |
205 | EXPORT_SYMBOL(dst_cow_metrics_generic); | |
206 | ||
207 | /* Caller asserts that dst_metrics_read_only(dst) is false. */ | |
208 | void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old) | |
209 | { | |
210 | unsigned long prev, new; | |
211 | ||
3fb07daf | 212 | new = ((unsigned long) &dst_default_metrics) | DST_METRICS_READ_ONLY; |
62fa8a84 DM |
213 | prev = cmpxchg(&dst->_metrics, old, new); |
214 | if (prev == old) | |
215 | kfree(__DST_METRICS_PTR(old)); | |
216 | } | |
217 | EXPORT_SYMBOL(__dst_destroy_metrics_generic); | |
218 | ||
c4c877b2 DB |
219 | struct dst_entry *dst_blackhole_check(struct dst_entry *dst, u32 cookie) |
220 | { | |
221 | return NULL; | |
222 | } | |
223 | ||
224 | u32 *dst_blackhole_cow_metrics(struct dst_entry *dst, unsigned long old) | |
225 | { | |
226 | return NULL; | |
227 | } | |
228 | ||
229 | struct neighbour *dst_blackhole_neigh_lookup(const struct dst_entry *dst, | |
230 | struct sk_buff *skb, | |
231 | const void *daddr) | |
232 | { | |
233 | return NULL; | |
234 | } | |
235 | ||
236 | void dst_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk, | |
237 | struct sk_buff *skb, u32 mtu, | |
238 | bool confirm_neigh) | |
239 | { | |
240 | } | |
241 | EXPORT_SYMBOL_GPL(dst_blackhole_update_pmtu); | |
242 | ||
243 | void dst_blackhole_redirect(struct dst_entry *dst, struct sock *sk, | |
244 | struct sk_buff *skb) | |
245 | { | |
246 | } | |
247 | EXPORT_SYMBOL_GPL(dst_blackhole_redirect); | |
248 | ||
249 | unsigned int dst_blackhole_mtu(const struct dst_entry *dst) | |
250 | { | |
251 | unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); | |
252 | ||
253 | return mtu ? : dst->dev->mtu; | |
254 | } | |
255 | EXPORT_SYMBOL_GPL(dst_blackhole_mtu); | |
256 | ||
a188bb56 DB |
257 | static struct dst_ops dst_blackhole_ops = { |
258 | .family = AF_UNSPEC, | |
259 | .neigh_lookup = dst_blackhole_neigh_lookup, | |
260 | .check = dst_blackhole_check, | |
261 | .cow_metrics = dst_blackhole_cow_metrics, | |
262 | .update_pmtu = dst_blackhole_update_pmtu, | |
263 | .redirect = dst_blackhole_redirect, | |
264 | .mtu = dst_blackhole_mtu, | |
f38a9eb1 TG |
265 | }; |
266 | ||
3fcece12 JK |
267 | static void __metadata_dst_init(struct metadata_dst *md_dst, |
268 | enum metadata_type type, u8 optslen) | |
f38a9eb1 | 269 | { |
f38a9eb1 TG |
270 | struct dst_entry *dst; |
271 | ||
f38a9eb1 | 272 | dst = &md_dst->dst; |
762c8dc7 | 273 | dst_init(dst, &dst_blackhole_ops, NULL, DST_OBSOLETE_NONE, |
a4c2fd7f | 274 | DST_METADATA | DST_NOCOUNT); |
f38a9eb1 | 275 | memset(dst + 1, 0, sizeof(*md_dst) + optslen - sizeof(*dst)); |
3fcece12 | 276 | md_dst->type = type; |
d3aa45ce AS |
277 | } |
278 | ||
3fcece12 JK |
279 | struct metadata_dst *metadata_dst_alloc(u8 optslen, enum metadata_type type, |
280 | gfp_t flags) | |
d3aa45ce AS |
281 | { |
282 | struct metadata_dst *md_dst; | |
283 | ||
284 | md_dst = kmalloc(sizeof(*md_dst) + optslen, flags); | |
285 | if (!md_dst) | |
286 | return NULL; | |
287 | ||
3fcece12 | 288 | __metadata_dst_init(md_dst, type, optslen); |
f38a9eb1 TG |
289 | |
290 | return md_dst; | |
291 | } | |
292 | EXPORT_SYMBOL_GPL(metadata_dst_alloc); | |
293 | ||
d71785ff PA |
294 | void metadata_dst_free(struct metadata_dst *md_dst) |
295 | { | |
296 | #ifdef CONFIG_DST_CACHE | |
e65a4955 DL |
297 | if (md_dst->type == METADATA_IP_TUNNEL) |
298 | dst_cache_destroy(&md_dst->u.tun_info.dst_cache); | |
d71785ff | 299 | #endif |
94151f5a EB |
300 | if (md_dst->type == METADATA_XFRM) |
301 | dst_release(md_dst->u.xfrm_info.dst_orig); | |
d71785ff PA |
302 | kfree(md_dst); |
303 | } | |
af308b94 | 304 | EXPORT_SYMBOL_GPL(metadata_dst_free); |
d71785ff | 305 | |
3fcece12 JK |
306 | struct metadata_dst __percpu * |
307 | metadata_dst_alloc_percpu(u8 optslen, enum metadata_type type, gfp_t flags) | |
d3aa45ce AS |
308 | { |
309 | int cpu; | |
310 | struct metadata_dst __percpu *md_dst; | |
311 | ||
312 | md_dst = __alloc_percpu_gfp(sizeof(struct metadata_dst) + optslen, | |
313 | __alignof__(struct metadata_dst), flags); | |
314 | if (!md_dst) | |
315 | return NULL; | |
316 | ||
317 | for_each_possible_cpu(cpu) | |
3fcece12 | 318 | __metadata_dst_init(per_cpu_ptr(md_dst, cpu), type, optslen); |
d3aa45ce AS |
319 | |
320 | return md_dst; | |
321 | } | |
322 | EXPORT_SYMBOL_GPL(metadata_dst_alloc_percpu); | |
d66f2b91 JK |
323 | |
324 | void metadata_dst_free_percpu(struct metadata_dst __percpu *md_dst) | |
325 | { | |
326 | int cpu; | |
327 | ||
d66f2b91 JK |
328 | for_each_possible_cpu(cpu) { |
329 | struct metadata_dst *one_md_dst = per_cpu_ptr(md_dst, cpu); | |
330 | ||
94151f5a | 331 | #ifdef CONFIG_DST_CACHE |
d66f2b91 JK |
332 | if (one_md_dst->type == METADATA_IP_TUNNEL) |
333 | dst_cache_destroy(&one_md_dst->u.tun_info.dst_cache); | |
d66f2b91 | 334 | #endif |
94151f5a EB |
335 | if (one_md_dst->type == METADATA_XFRM) |
336 | dst_release(one_md_dst->u.xfrm_info.dst_orig); | |
337 | } | |
d66f2b91 JK |
338 | free_percpu(md_dst); |
339 | } | |
340 | EXPORT_SYMBOL_GPL(metadata_dst_free_percpu); |