[linux-block.git] / include / linux / math64.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MATH64_H
#define _LINUX_MATH64_H

#include <linux/types.h>
#include <linux/math.h>
#include <vdso/math64.h>
#include <asm/div64.h>

#if BITS_PER_LONG == 64

#define div64_long(x, y) div64_s64((x), (y))
#define div64_ul(x, y)   div64_u64((x), (y))

/**
 * div_u64_rem - unsigned 64bit divide with 32bit divisor with remainder
 * @dividend: unsigned 64bit dividend
 * @divisor: unsigned 32bit divisor
 * @remainder: pointer to unsigned 32bit remainder
 *
 * Return: sets ``*remainder``, then returns dividend / divisor
 *
 * This is commonly provided by 32bit archs to provide an optimized 64bit
 * divide.
 */
static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
{
	*remainder = dividend % divisor;
	return dividend / divisor;
}

/*
 * div_s64_rem - signed 64bit divide with 32bit divisor with remainder
 * @dividend: signed 64bit dividend
 * @divisor: signed 32bit divisor
 * @remainder: pointer to signed 32bit remainder
 *
 * Return: sets ``*remainder``, then returns dividend / divisor
 */
static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder)
{
	*remainder = dividend % divisor;
	return dividend / divisor;
}

/*
 * div64_u64_rem - unsigned 64bit divide with 64bit divisor and remainder
 * @dividend: unsigned 64bit dividend
 * @divisor: unsigned 64bit divisor
 * @remainder: pointer to unsigned 64bit remainder
 *
 * Return: sets ``*remainder``, then returns dividend / divisor
 */
static inline u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder)
{
	*remainder = dividend % divisor;
	return dividend / divisor;
}

/*
 * div64_u64 - unsigned 64bit divide with 64bit divisor
 * @dividend: unsigned 64bit dividend
 * @divisor: unsigned 64bit divisor
 *
 * Return: dividend / divisor
 */
static inline u64 div64_u64(u64 dividend, u64 divisor)
{
	return dividend / divisor;
}

/*
 * div64_s64 - signed 64bit divide with 64bit divisor
 * @dividend: signed 64bit dividend
 * @divisor: signed 64bit divisor
 *
 * Return: dividend / divisor
 */
static inline s64 div64_s64(s64 dividend, s64 divisor)
{
	return dividend / divisor;
}

#elif BITS_PER_LONG == 32

#define div64_long(x, y) div_s64((x), (y))
#define div64_ul(x, y)   div_u64((x), (y))

#ifndef div_u64_rem
static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
{
	*remainder = do_div(dividend, divisor);
	return dividend;
}
#endif

#ifndef div_s64_rem
extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder);
#endif

#ifndef div64_u64_rem
extern u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder);
#endif

#ifndef div64_u64
extern u64 div64_u64(u64 dividend, u64 divisor);
#endif

#ifndef div64_s64
extern s64 div64_s64(s64 dividend, s64 divisor);
#endif

#endif /* BITS_PER_LONG */

/**
 * div_u64 - unsigned 64bit divide with 32bit divisor
 * @dividend: unsigned 64bit dividend
 * @divisor: unsigned 32bit divisor
 *
 * This is the most common 64bit divide and should be used if possible,
 * as many 32bit archs can optimize this variant better than a full 64bit
 * divide.
 */
#ifndef div_u64
static inline u64 div_u64(u64 dividend, u32 divisor)
{
	u32 remainder;
	return div_u64_rem(dividend, divisor, &remainder);
}
#endif

/**
 * div_s64 - signed 64bit divide with 32bit divisor
 * @dividend: signed 64bit dividend
 * @divisor: signed 32bit divisor
 */
#ifndef div_s64
static inline s64 div_s64(s64 dividend, s32 divisor)
{
	s32 remainder;
	return div_s64_rem(dividend, divisor, &remainder);
}
#endif

u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder);

#ifndef mul_u32_u32
/*
 * Many a GCC version messes this up and generates a 64x64 mult :-(
 */
static inline u64 mul_u32_u32(u32 a, u32 b)
{
	return (u64)a * b;
}
#endif

#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)

#ifndef mul_u64_u32_shr
static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
{
	return (u64)(((unsigned __int128)a * mul) >> shift);
}
#endif /* mul_u64_u32_shr */

#ifndef mul_u64_u64_shr
static inline u64 mul_u64_u64_shr(u64 a, u64 mul, unsigned int shift)
{
	return (u64)(((unsigned __int128)a * mul) >> shift);
}
#endif /* mul_u64_u64_shr */

#else

#ifndef mul_u64_u32_shr
static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
{
	u32 ah, al;
	u64 ret;

	al = a;
	ah = a >> 32;

	ret = mul_u32_u32(al, mul) >> shift;
	if (ah)
		ret += mul_u32_u32(ah, mul) << (32 - shift);

	return ret;
}
#endif /* mul_u64_u32_shr */

#ifndef mul_u64_u64_shr
static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift)
{
	union {
		u64 ll;
		struct {
#ifdef __BIG_ENDIAN
			u32 high, low;
#else
			u32 low, high;
#endif
		} l;
	} rl, rm, rn, rh, a0, b0;
	u64 c;

	a0.ll = a;
	b0.ll = b;

	rl.ll = mul_u32_u32(a0.l.low, b0.l.low);
	rm.ll = mul_u32_u32(a0.l.low, b0.l.high);
	rn.ll = mul_u32_u32(a0.l.high, b0.l.low);
	rh.ll = mul_u32_u32(a0.l.high, b0.l.high);

	/*
	 * Each of these lines computes a 64-bit intermediate result into "c",
	 * starting at bits 32-95.  The low 32-bits go into the result of the
	 * multiplication, the high 32-bits are carried into the next step.
	 */
	rl.l.high = c = (u64)rl.l.high + rm.l.low + rn.l.low;
	rh.l.low = c = (c >> 32) + rm.l.high + rn.l.high + rh.l.low;
	rh.l.high = (c >> 32) + rh.l.high;

	/*
	 * The 128-bit result of the multiplication is in rl.ll and rh.ll,
	 * shift it right and throw away the high part of the result.
	 */
	if (shift == 0)
		return rl.ll;
	if (shift < 64)
		return (rl.ll >> shift) | (rh.ll << (64 - shift));
	return rh.ll >> (shift & 63);
}
#endif /* mul_u64_u64_shr */

#endif

#ifndef mul_s64_u64_shr
static inline u64 mul_s64_u64_shr(s64 a, u64 b, unsigned int shift)
{
	u64 ret;

	/*
	 * Extract the sign before the multiplication and put it back
	 * afterwards if needed.
	 */
	ret = mul_u64_u64_shr(abs(a), b, shift);

	if (a < 0)
		ret = -((s64) ret);

	return ret;
}
#endif /* mul_s64_u64_shr */

#ifndef mul_u64_u32_div
static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 divisor)
{
	union {
		u64 ll;
		struct {
#ifdef __BIG_ENDIAN
			u32 high, low;
#else
			u32 low, high;
#endif
		} l;
	} u, rl, rh;

	u.ll = a;
	rl.ll = mul_u32_u32(u.l.low, mul);
	rh.ll = mul_u32_u32(u.l.high, mul) + rl.l.high;

	/* Bits 32-63 of the result will be in rh.l.low. */
	rl.l.high = do_div(rh.ll, divisor);

	/* Bits 0-31 of the result will be in rl.l.low.	*/
	do_div(rl.ll, divisor);

	rl.l.high = rh.l.low;
	return rl.ll;
}
#endif /* mul_u64_u32_div */

u64 mul_u64_u64_div_u64(u64 a, u64 mul, u64 div);

#define DIV64_U64_ROUND_UP(ll, d)	\
	({ u64 _tmp = (d); div64_u64((ll) + _tmp - 1, _tmp); })

/**
 * DIV64_U64_ROUND_CLOSEST - unsigned 64bit divide with 64bit divisor rounded to nearest integer
 * @dividend: unsigned 64bit dividend
 * @divisor: unsigned 64bit divisor
 *
 * Divide unsigned 64bit dividend by unsigned 64bit divisor
 * and round to closest integer.
 *
 * Return: dividend / divisor rounded to nearest integer
 */
#define DIV64_U64_ROUND_CLOSEST(dividend, divisor)	\
	({ u64 _tmp = (divisor); div64_u64((dividend) + _tmp / 2, _tmp); })

/*
 * DIV_U64_ROUND_CLOSEST - unsigned 64bit divide with 32bit divisor rounded to nearest integer
 * @dividend: unsigned 64bit dividend
 * @divisor: unsigned 32bit divisor
 *
 * Divide unsigned 64bit dividend by unsigned 32bit divisor
 * and round to closest integer.
 *
 * Return: dividend / divisor rounded to nearest integer
 */
#define DIV_U64_ROUND_CLOSEST(dividend, divisor)	\
	({ u32 _tmp = (divisor); div_u64((u64)(dividend) + _tmp / 2, _tmp); })

/*
 * DIV_S64_ROUND_CLOSEST - signed 64bit divide with 32bit divisor rounded to nearest integer
 * @dividend: signed 64bit dividend
 * @divisor: signed 32bit divisor
 *
 * Divide signed 64bit dividend by signed 32bit divisor
 * and round to closest integer.
 *
 * Return: dividend / divisor rounded to nearest integer
 */
#define DIV_S64_ROUND_CLOSEST(dividend, divisor)(	\
{							\
	s64 __x = (dividend);				\
	s32 __d = (divisor);				\
	((__x > 0) == (__d > 0)) ?			\
		div_s64((__x + (__d / 2)), __d) :	\
		div_s64((__x - (__d / 2)), __d);	\
}							\
)
#endif /* _LINUX_MATH64_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
2418f4f2 RZ	2	#ifndef _LINUX_MATH64_H
	3	#define _LINUX_MATH64_H
	4
	5	#include <linux/types.h>
605a140a	6	#include <linux/math.h>
b874b835	7	#include <vdso/math64.h>
2418f4f2 RZ	8	#include <asm/div64.h>
	9
	10	#if BITS_PER_LONG == 64
	11
c2853c8d AS	12	#define div64_long(x, y) div64_s64((x), (y))
c2853c8d AS	13	#define div64_ul(x, y) div64_u64((x), (y))
f910381a	14
2418f4f2 RZ	15	/**
2418f4f2 RZ	16	* div_u64_rem - unsigned 64bit divide with 32bit divisor with remainder
078843f7 RD	17	* @dividend: unsigned 64bit dividend
	18	* @divisor: unsigned 32bit divisor
	19	* @remainder: pointer to unsigned 32bit remainder
	20	*
	21	* Return: sets ``*remainder``, then returns dividend / divisor
2418f4f2 RZ	22	*
	23	* This is commonly provided by 32bit archs to provide an optimized 64bit
	24	* divide.
	25	*/
	26	static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
	27	{
	28	*remainder = dividend % divisor;
	29	return dividend / divisor;
	30	}
	31
c2a9a645	32	/*
2418f4f2	33	* div_s64_rem - signed 64bit divide with 32bit divisor with remainder
078843f7 RD	34	* @dividend: signed 64bit dividend
	35	* @divisor: signed 32bit divisor
	36	* @remainder: pointer to signed 32bit remainder
	37	*
	38	* Return: sets ``*remainder``, then returns dividend / divisor
2418f4f2 RZ	39	*/
	40	static inline s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder)
	41	{
	42	*remainder = dividend % divisor;
	43	return dividend / divisor;
	44	}
	45
c2a9a645	46	/*
eb18cba7	47	* div64_u64_rem - unsigned 64bit divide with 64bit divisor and remainder
078843f7 RD	48	* @dividend: unsigned 64bit dividend
	49	* @divisor: unsigned 64bit divisor
	50	* @remainder: pointer to unsigned 64bit remainder
	51	*
	52	* Return: sets ``*remainder``, then returns dividend / divisor
eb18cba7 MS	53	*/
	54	static inline u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder)
	55	{
	56	*remainder = dividend % divisor;
	57	return dividend / divisor;
	58	}
	59
c2a9a645	60	/*
6f6d6a1a	61	* div64_u64 - unsigned 64bit divide with 64bit divisor
078843f7 RD	62	* @dividend: unsigned 64bit dividend
	63	* @divisor: unsigned 64bit divisor
	64	*
	65	* Return: dividend / divisor
6f6d6a1a RZ	66	*/
	67	static inline u64 div64_u64(u64 dividend, u64 divisor)
	68	{
	69	return dividend / divisor;
	70	}
	71
c2a9a645	72	/*
658716d1	73	* div64_s64 - signed 64bit divide with 64bit divisor
078843f7 RD	74	* @dividend: signed 64bit dividend
	75	* @divisor: signed 64bit divisor
	76	*
	77	* Return: dividend / divisor
658716d1 BB	78	*/
	79	static inline s64 div64_s64(s64 dividend, s64 divisor)
	80	{
	81	return dividend / divisor;
	82	}
	83
2418f4f2 RZ	84	#elif BITS_PER_LONG == 32
2418f4f2 RZ	85
c2853c8d AS	86	#define div64_long(x, y) div_s64((x), (y))
c2853c8d AS	87	#define div64_ul(x, y) div_u64((x), (y))
f910381a	88
2418f4f2 RZ	89	#ifndef div_u64_rem
	90	static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder)
	91	{
	92	*remainder = do_div(dividend, divisor);
	93	return dividend;
	94	}
	95	#endif
	96
	97	#ifndef div_s64_rem
	98	extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder);
	99	#endif
	100
eb18cba7 MS	101	#ifndef div64_u64_rem
	102	extern u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder);
	103	#endif
	104
6f6d6a1a	105	#ifndef div64_u64
f3002134	106	extern u64 div64_u64(u64 dividend, u64 divisor);
6f6d6a1a RZ	107	#endif
6f6d6a1a RZ	108
658716d1 BB	109	#ifndef div64_s64
	110	extern s64 div64_s64(s64 dividend, s64 divisor);
	111	#endif
	112
2418f4f2 RZ	113	#endif /* BITS_PER_LONG */
	114
	115	/**
	116	* div_u64 - unsigned 64bit divide with 32bit divisor
078843f7 RD	117	* @dividend: unsigned 64bit dividend
078843f7 RD	118	* @divisor: unsigned 32bit divisor
2418f4f2 RZ	119	*
	120	* This is the most common 64bit divide and should be used if possible,
	121	* as many 32bit archs can optimize this variant better than a full 64bit
	122	* divide.
	123	*/
	124	#ifndef div_u64
	125	static inline u64 div_u64(u64 dividend, u32 divisor)
	126	{
	127	u32 remainder;
	128	return div_u64_rem(dividend, divisor, &remainder);
	129	}
	130	#endif
	131
	132	/**
	133	* div_s64 - signed 64bit divide with 32bit divisor
078843f7 RD	134	* @dividend: signed 64bit dividend
078843f7 RD	135	* @divisor: signed 32bit divisor
2418f4f2 RZ	136	*/
	137	#ifndef div_s64
	138	static inline s64 div_s64(s64 dividend, s32 divisor)
	139	{
	140	s32 remainder;
	141	return div_s64_rem(dividend, divisor, &remainder);
	142	}
	143	#endif
	144
f595ec96 JF	145	u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder);
f595ec96 JF	146
9e3d6223 PZ	147	#ifndef mul_u32_u32
	148	/*
	149	* Many a GCC version messes this up and generates a 64x64 mult :-(
	150	*/
	151	static inline u64 mul_u32_u32(u32 a, u32 b)
	152	{
	153	return (u64)a * b;
	154	}
	155	#endif
	156
be5e610c PZ	157	#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
	158
	159	#ifndef mul_u64_u32_shr
	160	static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
	161	{
	162	return (u64)(((unsigned __int128)a * mul) >> shift);
	163	}
	164	#endif /* mul_u64_u32_shr */
	165
35181e86 HZ	166	#ifndef mul_u64_u64_shr
	167	static inline u64 mul_u64_u64_shr(u64 a, u64 mul, unsigned int shift)
	168	{
	169	return (u64)(((unsigned __int128)a * mul) >> shift);
	170	}
	171	#endif /* mul_u64_u64_shr */
	172
be5e610c PZ	173	#else
	174
	175	#ifndef mul_u64_u32_shr
	176	static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
	177	{
	178	u32 ah, al;
	179	u64 ret;
	180
	181	al = a;
	182	ah = a >> 32;
	183
9e3d6223	184	ret = mul_u32_u32(al, mul) >> shift;
be5e610c	185	if (ah)
9e3d6223	186	ret += mul_u32_u32(ah, mul) << (32 - shift);
be5e610c PZ	187
	188	return ret;
	189	}
	190	#endif /* mul_u64_u32_shr */
	191
35181e86 HZ	192	#ifndef mul_u64_u64_shr
	193	static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift)
	194	{
	195	union {
	196	u64 ll;
	197	struct {
	198	#ifdef __BIG_ENDIAN
	199	u32 high, low;
	200	#else
	201	u32 low, high;
	202	#endif
	203	} l;
	204	} rl, rm, rn, rh, a0, b0;
	205	u64 c;
	206
	207	a0.ll = a;
	208	b0.ll = b;
	209
9e3d6223 PZ	210	rl.ll = mul_u32_u32(a0.l.low, b0.l.low);
	211	rm.ll = mul_u32_u32(a0.l.low, b0.l.high);
	212	rn.ll = mul_u32_u32(a0.l.high, b0.l.low);
	213	rh.ll = mul_u32_u32(a0.l.high, b0.l.high);
35181e86 HZ	214
	215	/*
	216	* Each of these lines computes a 64-bit intermediate result into "c",
	217	* starting at bits 32-95. The low 32-bits go into the result of the
	218	* multiplication, the high 32-bits are carried into the next step.
	219	*/
	220	rl.l.high = c = (u64)rl.l.high + rm.l.low + rn.l.low;
	221	rh.l.low = c = (c >> 32) + rm.l.high + rn.l.high + rh.l.low;
	222	rh.l.high = (c >> 32) + rh.l.high;
	223
	224	/*
	225	* The 128-bit result of the multiplication is in rl.ll and rh.ll,
	226	* shift it right and throw away the high part of the result.
	227	*/
	228	if (shift == 0)
	229	return rl.ll;
	230	if (shift < 64)
	231	return (rl.ll >> shift) \| (rh.ll << (64 - shift));
	232	return rh.ll >> (shift & 63);
	233	}
	234	#endif /* mul_u64_u64_shr */
	235
be5e610c PZ	236	#endif
be5e610c PZ	237
605a140a IS	238	#ifndef mul_s64_u64_shr
	239	static inline u64 mul_s64_u64_shr(s64 a, u64 b, unsigned int shift)
	240	{
	241	u64 ret;
	242
	243	/*
	244	* Extract the sign before the multiplication and put it back
	245	* afterwards if needed.
	246	*/
	247	ret = mul_u64_u64_shr(abs(a), b, shift);
	248
	249	if (a < 0)
	250	ret = -((s64) ret);
	251
	252	return ret;
	253	}
	254	#endif /* mul_s64_u64_shr */
	255
381d585c HZ	256	#ifndef mul_u64_u32_div
	257	static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 divisor)
	258	{
	259	union {
	260	u64 ll;
	261	struct {
	262	#ifdef __BIG_ENDIAN
	263	u32 high, low;
	264	#else
	265	u32 low, high;
	266	#endif
	267	} l;
	268	} u, rl, rh;
	269
	270	u.ll = a;
9e3d6223 PZ	271	rl.ll = mul_u32_u32(u.l.low, mul);
9e3d6223 PZ	272	rh.ll = mul_u32_u32(u.l.high, mul) + rl.l.high;
381d585c HZ	273
	274	/* Bits 32-63 of the result will be in rh.l.low. */
	275	rl.l.high = do_div(rh.ll, divisor);
	276
	277	/* Bits 0-31 of the result will be in rl.l.low. */
	278	do_div(rl.ll, divisor);
	279
	280	rl.l.high = rh.l.low;
	281	return rl.ll;
	282	}
	283	#endif /* mul_u64_u32_div */
	284
3dc167ba ON	285	u64 mul_u64_u64_div_u64(u64 a, u64 mul, u64 div);
3dc167ba ON	286
68600f62 RG	287	#define DIV64_U64_ROUND_UP(ll, d) \
	288	({ u64 _tmp = (d); div64_u64((ll) + _tmp - 1, _tmp); })
	289
cb8be119 SH	290	/**
	291	* DIV64_U64_ROUND_CLOSEST - unsigned 64bit divide with 64bit divisor rounded to nearest integer
	292	* @dividend: unsigned 64bit dividend
	293	* @divisor: unsigned 64bit divisor
	294	*
	295	* Divide unsigned 64bit dividend by unsigned 64bit divisor
	296	* and round to closest integer.
	297	*
	298	* Return: dividend / divisor rounded to nearest integer
	299	*/
	300	#define DIV64_U64_ROUND_CLOSEST(dividend, divisor) \
	301	({ u64 _tmp = (divisor); div64_u64((dividend) + _tmp / 2, _tmp); })
	302
2c861b73 PR	303	/*
	304	* DIV_U64_ROUND_CLOSEST - unsigned 64bit divide with 32bit divisor rounded to nearest integer
	305	* @dividend: unsigned 64bit dividend
	306	* @divisor: unsigned 32bit divisor
	307	*
	308	* Divide unsigned 64bit dividend by unsigned 32bit divisor
	309	* and round to closest integer.
	310	*
	311	* Return: dividend / divisor rounded to nearest integer
	312	*/
	313	#define DIV_U64_ROUND_CLOSEST(dividend, divisor) \
	314	({ u32 _tmp = (divisor); div_u64((u64)(dividend) + _tmp / 2, _tmp); })
	315
af60459a CZ	316	/*
	317	* DIV_S64_ROUND_CLOSEST - signed 64bit divide with 32bit divisor rounded to nearest integer
	318	* @dividend: signed 64bit dividend
	319	* @divisor: signed 32bit divisor
	320	*
	321	* Divide signed 64bit dividend by signed 32bit divisor
	322	* and round to closest integer.
	323	*
	324	* Return: dividend / divisor rounded to nearest integer
	325	*/
	326	#define DIV_S64_ROUND_CLOSEST(dividend, divisor)( \
	327	{ \
	328	s64 __x = (dividend); \
	329	s32 __d = (divisor); \
	330	((__x > 0) == (__d > 0)) ? \
	331	div_s64((__x + (__d / 2)), __d) : \
	332	div_s64((__x - (__d / 2)), __d); \
	333	} \
	334	)
2418f4f2	335	#endif /* _LINUX_MATH64_H */