Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2418f4f2 RZ |
2 | #ifndef _LINUX_MATH64_H |
3 | #define _LINUX_MATH64_H | |
4 | ||
5 | #include <linux/types.h> | |
605a140a | 6 | #include <linux/math.h> |
b874b835 | 7 | #include <vdso/math64.h> |
2418f4f2 RZ |
8 | #include <asm/div64.h> |
9 | ||
10 | #if BITS_PER_LONG == 64 | |
11 | ||
c2853c8d AS |
12 | #define div64_long(x, y) div64_s64((x), (y)) |
13 | #define div64_ul(x, y) div64_u64((x), (y)) | |
f910381a | 14 | |
2418f4f2 RZ |
15 | /** |
16 | * div_u64_rem - unsigned 64bit divide with 32bit divisor with remainder | |
078843f7 RD |
17 | * @dividend: unsigned 64bit dividend |
18 | * @divisor: unsigned 32bit divisor | |
19 | * @remainder: pointer to unsigned 32bit remainder | |
20 | * | |
21 | * Return: sets ``*remainder``, then returns dividend / divisor | |
2418f4f2 RZ |
22 | * |
23 | * This is commonly provided by 32bit archs to provide an optimized 64bit | |
24 | * divide. | |
25 | */ | |
26 | static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) | |
27 | { | |
28 | *remainder = dividend % divisor; | |
29 | return dividend / divisor; | |
30 | } | |
31 | ||
c2a9a645 | 32 | /* |
2418f4f2 | 33 | * div_s64_rem - signed 64bit divide with 32bit divisor with remainder |
078843f7 RD |
34 | * @dividend: signed 64bit dividend |
35 | * @divisor: signed 32bit divisor | |
36 | * @remainder: pointer to signed 32bit remainder | |
37 | * | |
38 | * Return: sets ``*remainder``, then returns dividend / divisor | |
2418f4f2 RZ |
39 | */ |
40 | static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder) | |
41 | { | |
42 | *remainder = dividend % divisor; | |
43 | return dividend / divisor; | |
44 | } | |
45 | ||
c2a9a645 | 46 | /* |
eb18cba7 | 47 | * div64_u64_rem - unsigned 64bit divide with 64bit divisor and remainder |
078843f7 RD |
48 | * @dividend: unsigned 64bit dividend |
49 | * @divisor: unsigned 64bit divisor | |
50 | * @remainder: pointer to unsigned 64bit remainder | |
51 | * | |
52 | * Return: sets ``*remainder``, then returns dividend / divisor | |
eb18cba7 MS |
53 | */ |
54 | static inline u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder) | |
55 | { | |
56 | *remainder = dividend % divisor; | |
57 | return dividend / divisor; | |
58 | } | |
59 | ||
c2a9a645 | 60 | /* |
6f6d6a1a | 61 | * div64_u64 - unsigned 64bit divide with 64bit divisor |
078843f7 RD |
62 | * @dividend: unsigned 64bit dividend |
63 | * @divisor: unsigned 64bit divisor | |
64 | * | |
65 | * Return: dividend / divisor | |
6f6d6a1a RZ |
66 | */ |
67 | static inline u64 div64_u64(u64 dividend, u64 divisor) | |
68 | { | |
69 | return dividend / divisor; | |
70 | } | |
71 | ||
c2a9a645 | 72 | /* |
658716d1 | 73 | * div64_s64 - signed 64bit divide with 64bit divisor |
078843f7 RD |
74 | * @dividend: signed 64bit dividend |
75 | * @divisor: signed 64bit divisor | |
76 | * | |
77 | * Return: dividend / divisor | |
658716d1 BB |
78 | */ |
79 | static inline s64 div64_s64(s64 dividend, s64 divisor) | |
80 | { | |
81 | return dividend / divisor; | |
82 | } | |
83 | ||
2418f4f2 RZ |
84 | #elif BITS_PER_LONG == 32 |
85 | ||
c2853c8d AS |
86 | #define div64_long(x, y) div_s64((x), (y)) |
87 | #define div64_ul(x, y) div_u64((x), (y)) | |
f910381a | 88 | |
2418f4f2 RZ |
89 | #ifndef div_u64_rem |
90 | static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) | |
91 | { | |
92 | *remainder = do_div(dividend, divisor); | |
93 | return dividend; | |
94 | } | |
95 | #endif | |
96 | ||
97 | #ifndef div_s64_rem | |
98 | extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder); | |
99 | #endif | |
100 | ||
eb18cba7 MS |
101 | #ifndef div64_u64_rem |
102 | extern u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder); | |
103 | #endif | |
104 | ||
6f6d6a1a | 105 | #ifndef div64_u64 |
f3002134 | 106 | extern u64 div64_u64(u64 dividend, u64 divisor); |
6f6d6a1a RZ |
107 | #endif |
108 | ||
658716d1 BB |
109 | #ifndef div64_s64 |
110 | extern s64 div64_s64(s64 dividend, s64 divisor); | |
111 | #endif | |
112 | ||
2418f4f2 RZ |
113 | #endif /* BITS_PER_LONG */ |
114 | ||
115 | /** | |
116 | * div_u64 - unsigned 64bit divide with 32bit divisor | |
078843f7 RD |
117 | * @dividend: unsigned 64bit dividend |
118 | * @divisor: unsigned 32bit divisor | |
2418f4f2 RZ |
119 | * |
120 | * This is the most common 64bit divide and should be used if possible, | |
121 | * as many 32bit archs can optimize this variant better than a full 64bit | |
122 | * divide. | |
123 | */ | |
124 | #ifndef div_u64 | |
125 | static inline u64 div_u64(u64 dividend, u32 divisor) | |
126 | { | |
127 | u32 remainder; | |
128 | return div_u64_rem(dividend, divisor, &remainder); | |
129 | } | |
130 | #endif | |
131 | ||
132 | /** | |
133 | * div_s64 - signed 64bit divide with 32bit divisor | |
078843f7 RD |
134 | * @dividend: signed 64bit dividend |
135 | * @divisor: signed 32bit divisor | |
2418f4f2 RZ |
136 | */ |
137 | #ifndef div_s64 | |
138 | static inline s64 div_s64(s64 dividend, s32 divisor) | |
139 | { | |
140 | s32 remainder; | |
141 | return div_s64_rem(dividend, divisor, &remainder); | |
142 | } | |
143 | #endif | |
144 | ||
f595ec96 JF |
145 | u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder); |
146 | ||
9e3d6223 PZ |
147 | #ifndef mul_u32_u32 |
148 | /* | |
149 | * Many a GCC version messes this up and generates a 64x64 mult :-( | |
150 | */ | |
151 | static inline u64 mul_u32_u32(u32 a, u32 b) | |
152 | { | |
153 | return (u64)a * b; | |
154 | } | |
155 | #endif | |
156 | ||
be5e610c PZ |
157 | #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) |
158 | ||
159 | #ifndef mul_u64_u32_shr | |
160 | static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) | |
161 | { | |
162 | return (u64)(((unsigned __int128)a * mul) >> shift); | |
163 | } | |
164 | #endif /* mul_u64_u32_shr */ | |
165 | ||
35181e86 HZ |
166 | #ifndef mul_u64_u64_shr |
167 | static inline u64 mul_u64_u64_shr(u64 a, u64 mul, unsigned int shift) | |
168 | { | |
169 | return (u64)(((unsigned __int128)a * mul) >> shift); | |
170 | } | |
171 | #endif /* mul_u64_u64_shr */ | |
172 | ||
be5e610c PZ |
173 | #else |
174 | ||
175 | #ifndef mul_u64_u32_shr | |
176 | static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) | |
177 | { | |
178 | u32 ah, al; | |
179 | u64 ret; | |
180 | ||
181 | al = a; | |
182 | ah = a >> 32; | |
183 | ||
9e3d6223 | 184 | ret = mul_u32_u32(al, mul) >> shift; |
be5e610c | 185 | if (ah) |
9e3d6223 | 186 | ret += mul_u32_u32(ah, mul) << (32 - shift); |
be5e610c PZ |
187 | |
188 | return ret; | |
189 | } | |
190 | #endif /* mul_u64_u32_shr */ | |
191 | ||
35181e86 HZ |
192 | #ifndef mul_u64_u64_shr |
193 | static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift) | |
194 | { | |
195 | union { | |
196 | u64 ll; | |
197 | struct { | |
198 | #ifdef __BIG_ENDIAN | |
199 | u32 high, low; | |
200 | #else | |
201 | u32 low, high; | |
202 | #endif | |
203 | } l; | |
204 | } rl, rm, rn, rh, a0, b0; | |
205 | u64 c; | |
206 | ||
207 | a0.ll = a; | |
208 | b0.ll = b; | |
209 | ||
9e3d6223 PZ |
210 | rl.ll = mul_u32_u32(a0.l.low, b0.l.low); |
211 | rm.ll = mul_u32_u32(a0.l.low, b0.l.high); | |
212 | rn.ll = mul_u32_u32(a0.l.high, b0.l.low); | |
213 | rh.ll = mul_u32_u32(a0.l.high, b0.l.high); | |
35181e86 HZ |
214 | |
215 | /* | |
216 | * Each of these lines computes a 64-bit intermediate result into "c", | |
217 | * starting at bits 32-95. The low 32-bits go into the result of the | |
218 | * multiplication, the high 32-bits are carried into the next step. | |
219 | */ | |
220 | rl.l.high = c = (u64)rl.l.high + rm.l.low + rn.l.low; | |
221 | rh.l.low = c = (c >> 32) + rm.l.high + rn.l.high + rh.l.low; | |
222 | rh.l.high = (c >> 32) + rh.l.high; | |
223 | ||
224 | /* | |
225 | * The 128-bit result of the multiplication is in rl.ll and rh.ll, | |
226 | * shift it right and throw away the high part of the result. | |
227 | */ | |
228 | if (shift == 0) | |
229 | return rl.ll; | |
230 | if (shift < 64) | |
231 | return (rl.ll >> shift) | (rh.ll << (64 - shift)); | |
232 | return rh.ll >> (shift & 63); | |
233 | } | |
234 | #endif /* mul_u64_u64_shr */ | |
235 | ||
be5e610c PZ |
236 | #endif |
237 | ||
605a140a IS |
238 | #ifndef mul_s64_u64_shr |
239 | static inline u64 mul_s64_u64_shr(s64 a, u64 b, unsigned int shift) | |
240 | { | |
241 | u64 ret; | |
242 | ||
243 | /* | |
244 | * Extract the sign before the multiplication and put it back | |
245 | * afterwards if needed. | |
246 | */ | |
247 | ret = mul_u64_u64_shr(abs(a), b, shift); | |
248 | ||
249 | if (a < 0) | |
250 | ret = -((s64) ret); | |
251 | ||
252 | return ret; | |
253 | } | |
254 | #endif /* mul_s64_u64_shr */ | |
255 | ||
381d585c HZ |
256 | #ifndef mul_u64_u32_div |
257 | static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 divisor) | |
258 | { | |
259 | union { | |
260 | u64 ll; | |
261 | struct { | |
262 | #ifdef __BIG_ENDIAN | |
263 | u32 high, low; | |
264 | #else | |
265 | u32 low, high; | |
266 | #endif | |
267 | } l; | |
268 | } u, rl, rh; | |
269 | ||
270 | u.ll = a; | |
9e3d6223 PZ |
271 | rl.ll = mul_u32_u32(u.l.low, mul); |
272 | rh.ll = mul_u32_u32(u.l.high, mul) + rl.l.high; | |
381d585c HZ |
273 | |
274 | /* Bits 32-63 of the result will be in rh.l.low. */ | |
275 | rl.l.high = do_div(rh.ll, divisor); | |
276 | ||
277 | /* Bits 0-31 of the result will be in rl.l.low. */ | |
278 | do_div(rl.ll, divisor); | |
279 | ||
280 | rl.l.high = rh.l.low; | |
281 | return rl.ll; | |
282 | } | |
283 | #endif /* mul_u64_u32_div */ | |
284 | ||
3dc167ba ON |
285 | u64 mul_u64_u64_div_u64(u64 a, u64 mul, u64 div); |
286 | ||
68600f62 RG |
287 | #define DIV64_U64_ROUND_UP(ll, d) \ |
288 | ({ u64 _tmp = (d); div64_u64((ll) + _tmp - 1, _tmp); }) | |
289 | ||
cb8be119 SH |
290 | /** |
291 | * DIV64_U64_ROUND_CLOSEST - unsigned 64bit divide with 64bit divisor rounded to nearest integer | |
292 | * @dividend: unsigned 64bit dividend | |
293 | * @divisor: unsigned 64bit divisor | |
294 | * | |
295 | * Divide unsigned 64bit dividend by unsigned 64bit divisor | |
296 | * and round to closest integer. | |
297 | * | |
298 | * Return: dividend / divisor rounded to nearest integer | |
299 | */ | |
300 | #define DIV64_U64_ROUND_CLOSEST(dividend, divisor) \ | |
301 | ({ u64 _tmp = (divisor); div64_u64((dividend) + _tmp / 2, _tmp); }) | |
302 | ||
2c861b73 PR |
303 | /* |
304 | * DIV_U64_ROUND_CLOSEST - unsigned 64bit divide with 32bit divisor rounded to nearest integer | |
305 | * @dividend: unsigned 64bit dividend | |
306 | * @divisor: unsigned 32bit divisor | |
307 | * | |
308 | * Divide unsigned 64bit dividend by unsigned 32bit divisor | |
309 | * and round to closest integer. | |
310 | * | |
311 | * Return: dividend / divisor rounded to nearest integer | |
312 | */ | |
313 | #define DIV_U64_ROUND_CLOSEST(dividend, divisor) \ | |
314 | ({ u32 _tmp = (divisor); div_u64((u64)(dividend) + _tmp / 2, _tmp); }) | |
315 | ||
af60459a CZ |
316 | /* |
317 | * DIV_S64_ROUND_CLOSEST - signed 64bit divide with 32bit divisor rounded to nearest integer | |
318 | * @dividend: signed 64bit dividend | |
319 | * @divisor: signed 32bit divisor | |
320 | * | |
321 | * Divide signed 64bit dividend by signed 32bit divisor | |
322 | * and round to closest integer. | |
323 | * | |
324 | * Return: dividend / divisor rounded to nearest integer | |
325 | */ | |
326 | #define DIV_S64_ROUND_CLOSEST(dividend, divisor)( \ | |
327 | { \ | |
328 | s64 __x = (dividend); \ | |
329 | s32 __d = (divisor); \ | |
330 | ((__x > 0) == (__d > 0)) ? \ | |
331 | div_s64((__x + (__d / 2)), __d) : \ | |
332 | div_s64((__x - (__d / 2)), __d); \ | |
333 | } \ | |
334 | ) | |
2418f4f2 | 335 | #endif /* _LINUX_MATH64_H */ |