[linux-2.6-block.git] / arch / arm64 / include / asm / word-at-a-time.h

/*
 * Copyright (C) 2013 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef __ASM_WORD_AT_A_TIME_H
#define __ASM_WORD_AT_A_TIME_H

#include <linux/uaccess.h>

#ifndef __AARCH64EB__

#include <linux/kernel.h>

struct word_at_a_time {
	const unsigned long one_bits, high_bits;
};

#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }

static inline unsigned long has_zero(unsigned long a, unsigned long *bits,
				     const struct word_at_a_time *c)
{
	unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
	*bits = mask;
	return mask;
}

#define prep_zero_mask(a, bits, c) (bits)

static inline unsigned long create_zero_mask(unsigned long bits)
{
	bits = (bits - 1) & ~bits;
	return bits >> 7;
}

static inline unsigned long find_zero(unsigned long mask)
{
	return fls64(mask) >> 3;
}

#define zero_bytemask(mask) (mask)

#else	/* __AARCH64EB__ */
#include <asm-generic/word-at-a-time.h>
#endif

/*
 * Load an unaligned word from kernel space.
 *
 * In the (very unlikely) case of the word being a page-crosser
 * and the next page not being mapped, take the exception and
 * return zeroes in the non-existing part.
 */
static inline unsigned long load_unaligned_zeropad(const void *addr)
{
	unsigned long ret, offset;

	/* Load word from unaligned pointer addr */
	asm(
	"1:	ldr	%0, %3\n"
	"2:\n"
	"	.pushsection .fixup,\"ax\"\n"
	"	.align 2\n"
	"3:	and	%1, %2, #0x7\n"
	"	bic	%2, %2, #0x7\n"
	"	ldr	%0, [%2]\n"
	"	lsl	%1, %1, #0x3\n"
#ifndef __AARCH64EB__
	"	lsr	%0, %0, %1\n"
#else
	"	lsl	%0, %0, %1\n"
#endif
	"	b	2b\n"
	"	.popsection\n"
	_ASM_EXTABLE(1b, 3b)
	: "=&r" (ret), "=&r" (offset)
	: "r" (addr), "Q" (*(unsigned long *)addr));

	return ret;
}

#endif /* __ASM_WORD_AT_A_TIME_H */
Commit	Line	Data
12a0ef7b WD	1	/*
	2	* Copyright (C) 2013 ARM Ltd.
	3	*
	4	* This program is free software; you can redistribute it and/or modify
	5	* it under the terms of the GNU General Public License version 2 as
	6	* published by the Free Software Foundation.
	7	*
	8	* This program is distributed in the hope that it will be useful,
	9	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	11	* GNU General Public License for more details.
	12	*
	13	* You should have received a copy of the GNU General Public License
	14	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	15	*/
	16	#ifndef __ASM_WORD_AT_A_TIME_H
	17	#define __ASM_WORD_AT_A_TIME_H
	18
7c0f6ba6	19	#include <linux/uaccess.h>
6c94f27a	20
12a0ef7b WD	21	#ifndef __AARCH64EB__
	22
	23	#include <linux/kernel.h>
	24
	25	struct word_at_a_time {
	26	const unsigned long one_bits, high_bits;
	27	};
	28
	29	#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
	30
	31	static inline unsigned long has_zero(unsigned long a, unsigned long *bits,
	32	const struct word_at_a_time *c)
	33	{
	34	unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
	35	*bits = mask;
	36	return mask;
	37	}
	38
	39	#define prep_zero_mask(a, bits, c) (bits)
	40
	41	static inline unsigned long create_zero_mask(unsigned long bits)
	42	{
	43	bits = (bits - 1) & ~bits;
	44	return bits >> 7;
	45	}
	46
	47	static inline unsigned long find_zero(unsigned long mask)
	48	{
	49	return fls64(mask) >> 3;
	50	}
	51
7bc13fd3 WD	52	#define zero_bytemask(mask) (mask)
7bc13fd3 WD	53
12a0ef7b WD	54	#else /* __AARCH64EB__ */
	55	#include <asm-generic/word-at-a-time.h>
	56	#endif
	57
7bc13fd3 WD	58	/*
	59	* Load an unaligned word from kernel space.
	60	*
	61	* In the (very unlikely) case of the word being a page-crosser
	62	* and the next page not being mapped, take the exception and
	63	* return zeroes in the non-existing part.
	64	*/
	65	static inline unsigned long load_unaligned_zeropad(const void *addr)
	66	{
	67	unsigned long ret, offset;
	68
	69	/* Load word from unaligned pointer addr */
	70	asm(
	71	"1: ldr %0, %3\n"
	72	"2:\n"
	73	" .pushsection .fixup,\"ax\"\n"
	74	" .align 2\n"
	75	"3: and %1, %2, #0x7\n"
	76	" bic %2, %2, #0x7\n"
	77	" ldr %0, [%2]\n"
	78	" lsl %1, %1, #0x3\n"
	79	#ifndef __AARCH64EB__
	80	" lsr %0, %0, %1\n"
	81	#else
	82	" lsl %0, %0, %1\n"
	83	#endif
	84	" b 2b\n"
	85	" .popsection\n"
6c94f27a	86	_ASM_EXTABLE(1b, 3b)
7bc13fd3 WD	87	: "=&r" (ret), "=&r" (offset)
	88	: "r" (addr), "Q" ((unsigned long )addr));
	89
	90	return ret;
	91	}
	92
12a0ef7b	93	#endif /* __ASM_WORD_AT_A_TIME_H */