Commit | Line | Data |
---|---|---|
06021292 | 1 | /* |
8b80cda5 | 2 | * net busy poll support |
06021292 ET |
3 | * Copyright(c) 2013 Intel Corporation. |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., | |
16 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
17 | * | |
18 | * Author: Eliezer Tamir | |
19 | * | |
20 | * Contact Information: | |
21 | * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> | |
22 | */ | |
23 | ||
8b80cda5 ET |
24 | #ifndef _LINUX_NET_BUSY_POLL_H |
25 | #define _LINUX_NET_BUSY_POLL_H | |
06021292 ET |
26 | |
27 | #include <linux/netdevice.h> | |
28 | #include <net/ip.h> | |
29 | ||
e0d1095a | 30 | #ifdef CONFIG_NET_RX_BUSY_POLL |
06021292 ET |
31 | |
32 | struct napi_struct; | |
64b0dc51 ET |
33 | extern unsigned int sysctl_net_busy_read __read_mostly; |
34 | extern unsigned int sysctl_net_busy_poll __read_mostly; | |
06021292 ET |
35 | |
36 | /* return values from ndo_ll_poll */ | |
37 | #define LL_FLUSH_FAILED -1 | |
38 | #define LL_FLUSH_BUSY -2 | |
39 | ||
cbf55001 | 40 | static inline bool net_busy_loop_on(void) |
91e2fd33 | 41 | { |
64b0dc51 | 42 | return sysctl_net_busy_poll; |
91e2fd33 ET |
43 | } |
44 | ||
ad6276e0 ET |
45 | /* a wrapper to make debug_smp_processor_id() happy |
46 | * we can use sched_clock() because we don't care much about precision | |
9a3c71aa | 47 | * we only care that the average is bounded |
ad6276e0 ET |
48 | */ |
49 | #ifdef CONFIG_DEBUG_PREEMPT | |
76b1e9b9 | 50 | static inline u64 busy_loop_us_clock(void) |
ad6276e0 ET |
51 | { |
52 | u64 rc; | |
53 | ||
54 | preempt_disable_notrace(); | |
55 | rc = sched_clock(); | |
56 | preempt_enable_no_resched_notrace(); | |
57 | ||
76b1e9b9 | 58 | return rc >> 10; |
ad6276e0 ET |
59 | } |
60 | #else /* CONFIG_DEBUG_PREEMPT */ | |
76b1e9b9 | 61 | static inline u64 busy_loop_us_clock(void) |
ad6276e0 | 62 | { |
76b1e9b9 | 63 | return sched_clock() >> 10; |
ad6276e0 ET |
64 | } |
65 | #endif /* CONFIG_DEBUG_PREEMPT */ | |
66 | ||
76b1e9b9 | 67 | static inline unsigned long sk_busy_loop_end_time(struct sock *sk) |
06021292 | 68 | { |
76b1e9b9 | 69 | return busy_loop_us_clock() + ACCESS_ONCE(sk->sk_ll_usec); |
2d48d67f | 70 | } |
9a3c71aa | 71 | |
76b1e9b9 ET |
72 | /* in poll/select we use the global sysctl_net_ll_poll value */ |
73 | static inline unsigned long busy_loop_end_time(void) | |
1bc2774d | 74 | { |
64b0dc51 | 75 | return busy_loop_us_clock() + ACCESS_ONCE(sysctl_net_busy_poll); |
06021292 ET |
76 | } |
77 | ||
cbf55001 | 78 | static inline bool sk_can_busy_loop(struct sock *sk) |
06021292 | 79 | { |
dafcc438 | 80 | return sk->sk_ll_usec && sk->sk_napi_id && |
06021292 ET |
81 | !need_resched() && !signal_pending(current); |
82 | } | |
83 | ||
76b1e9b9 ET |
84 | |
85 | static inline bool busy_loop_timeout(unsigned long end_time) | |
06021292 | 86 | { |
76b1e9b9 | 87 | unsigned long now = busy_loop_us_clock(); |
1bc2774d | 88 | |
76b1e9b9 | 89 | return time_after(now, end_time); |
06021292 ET |
90 | } |
91 | ||
2d48d67f ET |
92 | /* when used in sock_poll() nonblock is known at compile time to be true |
93 | * so the loop and end_time will be optimized out | |
94 | */ | |
cbf55001 | 95 | static inline bool sk_busy_loop(struct sock *sk, int nonblock) |
06021292 | 96 | { |
76b1e9b9 | 97 | unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0; |
06021292 ET |
98 | const struct net_device_ops *ops; |
99 | struct napi_struct *napi; | |
100 | int rc = false; | |
101 | ||
102 | /* | |
103 | * rcu read lock for napi hash | |
104 | * bh so we don't race with net_rx_action | |
105 | */ | |
106 | rcu_read_lock_bh(); | |
107 | ||
108 | napi = napi_by_id(sk->sk_napi_id); | |
109 | if (!napi) | |
110 | goto out; | |
111 | ||
112 | ops = napi->dev->netdev_ops; | |
8b80cda5 | 113 | if (!ops->ndo_busy_poll) |
06021292 ET |
114 | goto out; |
115 | ||
116 | do { | |
8b80cda5 | 117 | rc = ops->ndo_busy_poll(napi); |
06021292 ET |
118 | |
119 | if (rc == LL_FLUSH_FAILED) | |
120 | break; /* permanent failure */ | |
121 | ||
122 | if (rc > 0) | |
123 | /* local bh are disabled so it is ok to use _BH */ | |
124 | NET_ADD_STATS_BH(sock_net(sk), | |
288a9376 | 125 | LINUX_MIB_BUSYPOLLRXPACKETS, rc); |
3046e2f5 | 126 | cpu_relax(); |
06021292 | 127 | |
2d48d67f | 128 | } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) && |
76b1e9b9 | 129 | !need_resched() && !busy_loop_timeout(end_time)); |
06021292 ET |
130 | |
131 | rc = !skb_queue_empty(&sk->sk_receive_queue); | |
132 | out: | |
133 | rcu_read_unlock_bh(); | |
134 | return rc; | |
135 | } | |
136 | ||
137 | /* used in the NIC receive handler to mark the skb */ | |
8b80cda5 ET |
138 | static inline void skb_mark_napi_id(struct sk_buff *skb, |
139 | struct napi_struct *napi) | |
06021292 ET |
140 | { |
141 | skb->napi_id = napi->napi_id; | |
142 | } | |
143 | ||
144 | /* used in the protocol hanlder to propagate the napi_id to the socket */ | |
8b80cda5 | 145 | static inline void sk_mark_napi_id(struct sock *sk, struct sk_buff *skb) |
06021292 ET |
146 | { |
147 | sk->sk_napi_id = skb->napi_id; | |
148 | } | |
149 | ||
e0d1095a | 150 | #else /* CONFIG_NET_RX_BUSY_POLL */ |
cbf55001 | 151 | static inline unsigned long net_busy_loop_on(void) |
91e2fd33 ET |
152 | { |
153 | return 0; | |
154 | } | |
06021292 | 155 | |
76b1e9b9 | 156 | static inline unsigned long busy_loop_end_time(void) |
06021292 ET |
157 | { |
158 | return 0; | |
159 | } | |
160 | ||
cbf55001 | 161 | static inline bool sk_can_busy_loop(struct sock *sk) |
06021292 ET |
162 | { |
163 | return false; | |
164 | } | |
165 | ||
8b80cda5 ET |
166 | static inline void skb_mark_napi_id(struct sk_buff *skb, |
167 | struct napi_struct *napi) | |
06021292 ET |
168 | { |
169 | } | |
170 | ||
8b80cda5 | 171 | static inline void sk_mark_napi_id(struct sock *sk, struct sk_buff *skb) |
06021292 ET |
172 | { |
173 | } | |
174 | ||
76b1e9b9 | 175 | static inline bool busy_loop_timeout(unsigned long end_time) |
06021292 | 176 | { |
76b1e9b9 | 177 | return true; |
06021292 ET |
178 | } |
179 | ||
dfcefb0b CW |
180 | static inline bool sk_busy_loop(struct sock *sk, int nonblock) |
181 | { | |
182 | return false; | |
183 | } | |
184 | ||
e0d1095a | 185 | #endif /* CONFIG_NET_RX_BUSY_POLL */ |
8b80cda5 | 186 | #endif /* _LINUX_NET_BUSY_POLL_H */ |