[linux-2.6-block.git] / arch / mips / lib / csum_partial.S

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Quick'n'dirty IP checksum ...
 *
 * Copyright (C) 1998, 1999 Ralf Baechle
 * Copyright (C) 1999 Silicon Graphics, Inc.
 * Copyright (C) 2007  Maciej W. Rozycki
 * Copyright (C) 2014 Imagination Technologies Ltd.
 */
#include <linux/errno.h>
#include <linux/export.h>
#include <asm/asm.h>
#include <asm/asm-offsets.h>
#include <asm/regdef.h>

#ifdef CONFIG_64BIT
/*
 * As we are sharing code base with the mips32 tree (which use the o32 ABI
 * register definitions). We need to redefine the register definitions from
 * the n64 ABI register naming to the o32 ABI register naming.
 */
#undef t0
#undef t1
#undef t2
#undef t3
#define t0	$8
#define t1	$9
#define t2	$10
#define t3	$11
#define t4	$12
#define t5	$13
#define t6	$14
#define t7	$15

#define USE_DOUBLE
#endif

#ifdef USE_DOUBLE

#define LOAD   ld
#define LOAD32 lwu
#define ADD    daddu
#define NBYTES 8

#else

#define LOAD   lw
#define LOAD32 lw
#define ADD    addu
#define NBYTES 4

#endif /* USE_DOUBLE */

#define UNIT(unit)  ((unit)*NBYTES)

#define ADDC(sum,reg)						\
	.set	push;						\
	.set	noat;						\
	ADD	sum, reg;					\
	sltu	v1, sum, reg;					\
	ADD	sum, v1;					\
	.set	pop

#define ADDC32(sum,reg)						\
	.set	push;						\
	.set	noat;						\
	addu	sum, reg;					\
	sltu	v1, sum, reg;					\
	addu	sum, v1;					\
	.set	pop

#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)	\
	LOAD	_t0, (offset + UNIT(0))(src);			\
	LOAD	_t1, (offset + UNIT(1))(src);			\
	LOAD	_t2, (offset + UNIT(2))(src);			\
	LOAD	_t3, (offset + UNIT(3))(src);			\
	ADDC(_t0, _t1);						\
	ADDC(_t2, _t3);						\
	ADDC(sum, _t0);						\
	ADDC(sum, _t2)

#ifdef USE_DOUBLE
#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3)	\
	CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
#else
#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3)	\
	CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3);	\
	CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
#endif

/*
 * a0: source address
 * a1: length of the area to checksum
 * a2: partial checksum
 */

#define src a0
#define sum v0

	.text
	.set	noreorder
	.align	5
LEAF(csum_partial)
EXPORT_SYMBOL(csum_partial)
	move	sum, zero
	move	t7, zero

	sltiu	t8, a1, 0x8
	bnez	t8, .Lsmall_csumcpy		/* < 8 bytes to copy */
	 move	t2, a1

	andi	t7, src, 0x1			/* odd buffer? */

.Lhword_align:
	beqz	t7, .Lword_align
	 andi	t8, src, 0x2

	lbu	t0, (src)
	LONG_SUBU	a1, a1, 0x1
#ifdef __MIPSEL__
	sll	t0, t0, 8
#endif
	ADDC(sum, t0)
	PTR_ADDU	src, src, 0x1
	andi	t8, src, 0x2

.Lword_align:
	beqz	t8, .Ldword_align
	 sltiu	t8, a1, 56

	lhu	t0, (src)
	LONG_SUBU	a1, a1, 0x2
	ADDC(sum, t0)
	sltiu	t8, a1, 56
	PTR_ADDU	src, src, 0x2

.Ldword_align:
	bnez	t8, .Ldo_end_words
	 move	t8, a1

	andi	t8, src, 0x4
	beqz	t8, .Lqword_align
	 andi	t8, src, 0x8

	LOAD32	t0, 0x00(src)
	LONG_SUBU	a1, a1, 0x4
	ADDC(sum, t0)
	PTR_ADDU	src, src, 0x4
	andi	t8, src, 0x8

.Lqword_align:
	beqz	t8, .Loword_align
	 andi	t8, src, 0x10

#ifdef USE_DOUBLE
	ld	t0, 0x00(src)
	LONG_SUBU	a1, a1, 0x8
	ADDC(sum, t0)
#else
	lw	t0, 0x00(src)
	lw	t1, 0x04(src)
	LONG_SUBU	a1, a1, 0x8
	ADDC(sum, t0)
	ADDC(sum, t1)
#endif
	PTR_ADDU	src, src, 0x8
	andi	t8, src, 0x10

.Loword_align:
	beqz	t8, .Lbegin_movement
	 LONG_SRL	t8, a1, 0x7

#ifdef USE_DOUBLE
	ld	t0, 0x00(src)
	ld	t1, 0x08(src)
	ADDC(sum, t0)
	ADDC(sum, t1)
#else
	CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
#endif
	LONG_SUBU	a1, a1, 0x10
	PTR_ADDU	src, src, 0x10
	LONG_SRL	t8, a1, 0x7

.Lbegin_movement:
	beqz	t8, 1f
	 andi	t2, a1, 0x40

.Lmove_128bytes:
	CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
	CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
	CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
	CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
	LONG_SUBU	t8, t8, 0x01
	.set	reorder				/* DADDI_WAR */
	PTR_ADDU	src, src, 0x80
	bnez	t8, .Lmove_128bytes
	.set	noreorder

1:
	beqz	t2, 1f
	 andi	t2, a1, 0x20

.Lmove_64bytes:
	CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
	CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
	PTR_ADDU	src, src, 0x40

1:
	beqz	t2, .Ldo_end_words
	 andi	t8, a1, 0x1c

.Lmove_32bytes:
	CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
	andi	t8, a1, 0x1c
	PTR_ADDU	src, src, 0x20

.Ldo_end_words:
	beqz	t8, .Lsmall_csumcpy
	 andi	t2, a1, 0x3
	LONG_SRL	t8, t8, 0x2

.Lend_words:
	LOAD32	t0, (src)
	LONG_SUBU	t8, t8, 0x1
	ADDC(sum, t0)
	.set	reorder				/* DADDI_WAR */
	PTR_ADDU	src, src, 0x4
	bnez	t8, .Lend_words
	.set	noreorder

/* unknown src alignment and < 8 bytes to go  */
.Lsmall_csumcpy:
	move	a1, t2

	andi	t0, a1, 4
	beqz	t0, 1f
	 andi	t0, a1, 2

	/* Still a full word to go  */
	ulw	t1, (src)
	PTR_ADDIU	src, 4
#ifdef USE_DOUBLE
	dsll	t1, t1, 32			/* clear lower 32bit */
#endif
	ADDC(sum, t1)

1:	move	t1, zero
	beqz	t0, 1f
	 andi	t0, a1, 1

	/* Still a halfword to go  */
	ulhu	t1, (src)
	PTR_ADDIU	src, 2

1:	beqz	t0, 1f
	 sll	t1, t1, 16

	lbu	t2, (src)
	 nop

#ifdef __MIPSEB__
	sll	t2, t2, 8
#endif
	or	t1, t2

1:	ADDC(sum, t1)

	/* fold checksum */
#ifdef USE_DOUBLE
	dsll32	v1, sum, 0
	daddu	sum, v1
	sltu	v1, sum, v1
	dsra32	sum, sum, 0
	addu	sum, v1
#endif

	/* odd buffer alignment? */
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \
    defined(CONFIG_CPU_LOONGSON64)
	.set	push
	.set	arch=mips32r2
	wsbh	v1, sum
	movn	sum, v1, t7
	.set	pop
#else
	beqz	t7, 1f			/* odd buffer alignment? */
	 lui	v1, 0x00ff
	addu	v1, 0x00ff
	and	t0, sum, v1
	sll	t0, t0, 8
	srl	sum, sum, 8
	and	sum, sum, v1
	or	sum, sum, t0
1:
#endif
	.set	reorder
	/* Add the passed partial csum.	 */
	ADDC32(sum, a2)
	jr	ra
	.set	noreorder
	END(csum_partial)


/*
 * checksum and copy routines based on memcpy.S
 *
 *	csum_partial_copy_nocheck(src, dst, len)
 *	__csum_partial_copy_kernel(src, dst, len)
 *
 * See "Spec" in memcpy.S for details.	Unlike __copy_user, all
 * function in this file use the standard calling convention.
 */

#define src a0
#define dst a1
#define len a2
#define sum v0
#define odd t8

/*
 * All exception handlers simply return 0.
 */

/* Instruction type */
#define LD_INSN 1
#define ST_INSN 2
#define LEGACY_MODE 1
#define EVA_MODE    2
#define USEROP   1
#define KERNELOP 2

/*
 * Wrapper to add an entry in the exception table
 * in case the insn causes a memory exception.
 * Arguments:
 * insn    : Load/store instruction
 * type    : Instruction type
 * reg     : Register
 * addr    : Address
 * handler : Exception handler
 */
#define EXC(insn, type, reg, addr)		\
	.if \mode == LEGACY_MODE;		\
9:		insn reg, addr;			\
		.section __ex_table,"a";	\
		PTR_WD	9b, .L_exc;		\
		.previous;			\
	/* This is enabled in EVA mode */	\
	.else;					\
		/* If loading from user or storing to user */	\
		.if ((\from == USEROP) && (type == LD_INSN)) || \
		    ((\to == USEROP) && (type == ST_INSN));	\
9:			__BUILD_EVA_INSN(insn##e, reg, addr);	\
			.section __ex_table,"a";		\
			PTR_WD	9b, .L_exc;			\
			.previous;				\
		.else;						\
			/* EVA without exception */		\
			insn reg, addr;				\
		.endif;						\
	.endif

#undef LOAD

#ifdef USE_DOUBLE

#define LOADK	ld /* No exception */
#define LOAD(reg, addr)		EXC(ld, LD_INSN, reg, addr)
#define LOADBU(reg, addr)	EXC(lbu, LD_INSN, reg, addr)
#define LOADL(reg, addr)	EXC(ldl, LD_INSN, reg, addr)
#define LOADR(reg, addr)	EXC(ldr, LD_INSN, reg, addr)
#define STOREB(reg, addr)	EXC(sb, ST_INSN, reg, addr)
#define STOREL(reg, addr)	EXC(sdl, ST_INSN, reg, addr)
#define STORER(reg, addr)	EXC(sdr, ST_INSN, reg, addr)
#define STORE(reg, addr)	EXC(sd, ST_INSN, reg, addr)
#define ADD    daddu
#define SUB    dsubu
#define SRL    dsrl
#define SLL    dsll
#define SLLV   dsllv
#define SRLV   dsrlv
#define NBYTES 8
#define LOG_NBYTES 3

#else

#define LOADK	lw /* No exception */
#define LOAD(reg, addr)		EXC(lw, LD_INSN, reg, addr)
#define LOADBU(reg, addr)	EXC(lbu, LD_INSN, reg, addr)
#define LOADL(reg, addr)	EXC(lwl, LD_INSN, reg, addr)
#define LOADR(reg, addr)	EXC(lwr, LD_INSN, reg, addr)
#define STOREB(reg, addr)	EXC(sb, ST_INSN, reg, addr)
#define STOREL(reg, addr)	EXC(swl, ST_INSN, reg, addr)
#define STORER(reg, addr)	EXC(swr, ST_INSN, reg, addr)
#define STORE(reg, addr)	EXC(sw, ST_INSN, reg, addr)
#define ADD    addu
#define SUB    subu
#define SRL    srl
#define SLL    sll
#define SLLV   sllv
#define SRLV   srlv
#define NBYTES 4
#define LOG_NBYTES 2

#endif /* USE_DOUBLE */

#ifdef CONFIG_CPU_LITTLE_ENDIAN
#define LDFIRST LOADR
#define LDREST	LOADL
#define STFIRST STORER
#define STREST	STOREL
#define SHIFT_DISCARD SLLV
#define SHIFT_DISCARD_REVERT SRLV
#else
#define LDFIRST LOADL
#define LDREST	LOADR
#define STFIRST STOREL
#define STREST	STORER
#define SHIFT_DISCARD SRLV
#define SHIFT_DISCARD_REVERT SLLV
#endif

#define FIRST(unit) ((unit)*NBYTES)
#define REST(unit)  (FIRST(unit)+NBYTES-1)

#define ADDRMASK (NBYTES-1)

#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
	.set	noat
#else
	.set	at=v1
#endif

	.macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to

	li	sum, -1
	move	odd, zero
	/*
	 * Note: dst & src may be unaligned, len may be 0
	 * Temps
	 */
	/*
	 * The "issue break"s below are very approximate.
	 * Issue delays for dcache fills will perturb the schedule, as will
	 * load queue full replay traps, etc.
	 *
	 * If len < NBYTES use byte operations.
	 */
	sltu	t2, len, NBYTES
	and	t1, dst, ADDRMASK
	bnez	t2, .Lcopy_bytes_checklen\@
	 and	t0, src, ADDRMASK
	andi	odd, dst, 0x1			/* odd buffer? */
	bnez	t1, .Ldst_unaligned\@
	 nop
	bnez	t0, .Lsrc_unaligned_dst_aligned\@
	/*
	 * use delay slot for fall-through
	 * src and dst are aligned; need to compute rem
	 */
.Lboth_aligned\@:
	 SRL	t0, len, LOG_NBYTES+3	 # +3 for 8 units/iter
	beqz	t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
	 nop
	SUB	len, 8*NBYTES		# subtract here for bgez loop
	.align	4
1:
	LOAD(t0, UNIT(0)(src))
	LOAD(t1, UNIT(1)(src))
	LOAD(t2, UNIT(2)(src))
	LOAD(t3, UNIT(3)(src))
	LOAD(t4, UNIT(4)(src))
	LOAD(t5, UNIT(5)(src))
	LOAD(t6, UNIT(6)(src))
	LOAD(t7, UNIT(7)(src))
	SUB	len, len, 8*NBYTES
	ADD	src, src, 8*NBYTES
	STORE(t0, UNIT(0)(dst))
	ADDC(t0, t1)
	STORE(t1, UNIT(1)(dst))
	ADDC(sum, t0)
	STORE(t2, UNIT(2)(dst))
	ADDC(t2, t3)
	STORE(t3, UNIT(3)(dst))
	ADDC(sum, t2)
	STORE(t4, UNIT(4)(dst))
	ADDC(t4, t5)
	STORE(t5, UNIT(5)(dst))
	ADDC(sum, t4)
	STORE(t6, UNIT(6)(dst))
	ADDC(t6, t7)
	STORE(t7, UNIT(7)(dst))
	ADDC(sum, t6)
	.set	reorder				/* DADDI_WAR */
	ADD	dst, dst, 8*NBYTES
	bgez	len, 1b
	.set	noreorder
	ADD	len, 8*NBYTES		# revert len (see above)

	/*
	 * len == the number of bytes left to copy < 8*NBYTES
	 */
.Lcleanup_both_aligned\@:
#define rem t7
	beqz	len, .Ldone\@
	 sltu	t0, len, 4*NBYTES
	bnez	t0, .Lless_than_4units\@
	 and	rem, len, (NBYTES-1)	# rem = len % NBYTES
	/*
	 * len >= 4*NBYTES
	 */
	LOAD(t0, UNIT(0)(src))
	LOAD(t1, UNIT(1)(src))
	LOAD(t2, UNIT(2)(src))
	LOAD(t3, UNIT(3)(src))
	SUB	len, len, 4*NBYTES
	ADD	src, src, 4*NBYTES
	STORE(t0, UNIT(0)(dst))
	ADDC(t0, t1)
	STORE(t1, UNIT(1)(dst))
	ADDC(sum, t0)
	STORE(t2, UNIT(2)(dst))
	ADDC(t2, t3)
	STORE(t3, UNIT(3)(dst))
	ADDC(sum, t2)
	.set	reorder				/* DADDI_WAR */
	ADD	dst, dst, 4*NBYTES
	beqz	len, .Ldone\@
	.set	noreorder
.Lless_than_4units\@:
	/*
	 * rem = len % NBYTES
	 */
	beq	rem, len, .Lcopy_bytes\@
	 nop
1:
	LOAD(t0, 0(src))
	ADD	src, src, NBYTES
	SUB	len, len, NBYTES
	STORE(t0, 0(dst))
	ADDC(sum, t0)
	.set	reorder				/* DADDI_WAR */
	ADD	dst, dst, NBYTES
	bne	rem, len, 1b
	.set	noreorder

	/*
	 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
	 * A loop would do only a byte at a time with possible branch
	 * mispredicts.	 Can't do an explicit LOAD dst,mask,or,STORE
	 * because can't assume read-access to dst.  Instead, use
	 * STREST dst, which doesn't require read access to dst.
	 *
	 * This code should perform better than a simple loop on modern,
	 * wide-issue mips processors because the code has fewer branches and
	 * more instruction-level parallelism.
	 */
#define bits t2
	beqz	len, .Ldone\@
	 ADD	t1, dst, len	# t1 is just past last byte of dst
	li	bits, 8*NBYTES
	SLL	rem, len, 3	# rem = number of bits to keep
	LOAD(t0, 0(src))
	SUB	bits, bits, rem # bits = number of bits to discard
	SHIFT_DISCARD t0, t0, bits
	STREST(t0, -1(t1))
	SHIFT_DISCARD_REVERT t0, t0, bits
	.set reorder
	ADDC(sum, t0)
	b	.Ldone\@
	.set noreorder
.Ldst_unaligned\@:
	/*
	 * dst is unaligned
	 * t0 = src & ADDRMASK
	 * t1 = dst & ADDRMASK; T1 > 0
	 * len >= NBYTES
	 *
	 * Copy enough bytes to align dst
	 * Set match = (src and dst have same alignment)
	 */
#define match rem
	LDFIRST(t3, FIRST(0)(src))
	ADD	t2, zero, NBYTES
	LDREST(t3, REST(0)(src))
	SUB	t2, t2, t1	# t2 = number of bytes copied
	xor	match, t0, t1
	STFIRST(t3, FIRST(0)(dst))
	SLL	t4, t1, 3		# t4 = number of bits to discard
	SHIFT_DISCARD t3, t3, t4
	/* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
	ADDC(sum, t3)
	beq	len, t2, .Ldone\@
	 SUB	len, len, t2
	ADD	dst, dst, t2
	beqz	match, .Lboth_aligned\@
	 ADD	src, src, t2

.Lsrc_unaligned_dst_aligned\@:
	SRL	t0, len, LOG_NBYTES+2	 # +2 for 4 units/iter
	beqz	t0, .Lcleanup_src_unaligned\@
	 and	rem, len, (4*NBYTES-1)	 # rem = len % 4*NBYTES
1:
/*
 * Avoid consecutive LD*'s to the same register since some mips
 * implementations can't issue them in the same cycle.
 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
 * are to the same unit (unless src is aligned, but it's not).
 */
	LDFIRST(t0, FIRST(0)(src))
	LDFIRST(t1, FIRST(1)(src))
	SUB	len, len, 4*NBYTES
	LDREST(t0, REST(0)(src))
	LDREST(t1, REST(1)(src))
	LDFIRST(t2, FIRST(2)(src))
	LDFIRST(t3, FIRST(3)(src))
	LDREST(t2, REST(2)(src))
	LDREST(t3, REST(3)(src))
	ADD	src, src, 4*NBYTES
#ifdef CONFIG_CPU_SB1
	nop				# improves slotting
#endif
	STORE(t0, UNIT(0)(dst))
	ADDC(t0, t1)
	STORE(t1, UNIT(1)(dst))
	ADDC(sum, t0)
	STORE(t2, UNIT(2)(dst))
	ADDC(t2, t3)
	STORE(t3, UNIT(3)(dst))
	ADDC(sum, t2)
	.set	reorder				/* DADDI_WAR */
	ADD	dst, dst, 4*NBYTES
	bne	len, rem, 1b
	.set	noreorder

.Lcleanup_src_unaligned\@:
	beqz	len, .Ldone\@
	 and	rem, len, NBYTES-1  # rem = len % NBYTES
	beq	rem, len, .Lcopy_bytes\@
	 nop
1:
	LDFIRST(t0, FIRST(0)(src))
	LDREST(t0, REST(0)(src))
	ADD	src, src, NBYTES
	SUB	len, len, NBYTES
	STORE(t0, 0(dst))
	ADDC(sum, t0)
	.set	reorder				/* DADDI_WAR */
	ADD	dst, dst, NBYTES
	bne	len, rem, 1b
	.set	noreorder

.Lcopy_bytes_checklen\@:
	beqz	len, .Ldone\@
	 nop
.Lcopy_bytes\@:
	/* 0 < len < NBYTES  */
#ifdef CONFIG_CPU_LITTLE_ENDIAN
#define SHIFT_START 0
#define SHIFT_INC 8
#else
#define SHIFT_START 8*(NBYTES-1)
#define SHIFT_INC -8
#endif
	move	t2, zero	# partial word
	li	t3, SHIFT_START # shift
#define COPY_BYTE(N)			\
	LOADBU(t0, N(src));		\
	SUB	len, len, 1;		\
	STOREB(t0, N(dst));		\
	SLLV	t0, t0, t3;		\
	addu	t3, SHIFT_INC;		\
	beqz	len, .Lcopy_bytes_done\@; \
	 or	t2, t0

	COPY_BYTE(0)
	COPY_BYTE(1)
#ifdef USE_DOUBLE
	COPY_BYTE(2)
	COPY_BYTE(3)
	COPY_BYTE(4)
	COPY_BYTE(5)
#endif
	LOADBU(t0, NBYTES-2(src))
	SUB	len, len, 1
	STOREB(t0, NBYTES-2(dst))
	SLLV	t0, t0, t3
	or	t2, t0
.Lcopy_bytes_done\@:
	ADDC(sum, t2)
.Ldone\@:
	/* fold checksum */
	.set	push
	.set	noat
#ifdef USE_DOUBLE
	dsll32	v1, sum, 0
	daddu	sum, v1
	sltu	v1, sum, v1
	dsra32	sum, sum, 0
	addu	sum, v1
#endif

#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \
    defined(CONFIG_CPU_LOONGSON64)
	.set	push
	.set	arch=mips32r2
	wsbh	v1, sum
	movn	sum, v1, odd
	.set	pop
#else
	beqz	odd, 1f			/* odd buffer alignment? */
	 lui	v1, 0x00ff
	addu	v1, 0x00ff
	and	t0, sum, v1
	sll	t0, t0, 8
	srl	sum, sum, 8
	and	sum, sum, v1
	or	sum, sum, t0
1:
#endif
	.set	pop
	.set reorder
	jr	ra
	.set noreorder
	.endm

	.set noreorder
.L_exc:
	jr	ra
	 li	v0, 0

FEXPORT(__csum_partial_copy_nocheck)
EXPORT_SYMBOL(__csum_partial_copy_nocheck)
#ifndef CONFIG_EVA
FEXPORT(__csum_partial_copy_to_user)
EXPORT_SYMBOL(__csum_partial_copy_to_user)
FEXPORT(__csum_partial_copy_from_user)
EXPORT_SYMBOL(__csum_partial_copy_from_user)
#endif
__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP

#ifdef CONFIG_EVA
LEAF(__csum_partial_copy_to_user)
__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP
END(__csum_partial_copy_to_user)

LEAF(__csum_partial_copy_from_user)
__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP
END(__csum_partial_copy_from_user)
#endif
Commit	Line	Data
	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* Quick'n'dirty IP checksum ...
	7	*
	8	* Copyright (C) 1998, 1999 Ralf Baechle
	9	* Copyright (C) 1999 Silicon Graphics, Inc.
	10	* Copyright (C) 2007 Maciej W. Rozycki
	11	* Copyright (C) 2014 Imagination Technologies Ltd.
	12	*/
	13	#include <linux/errno.h>
	14	#include <linux/export.h>
	15	#include <asm/asm.h>
	16	#include <asm/asm-offsets.h>
	17	#include <asm/regdef.h>
	18
	19	#ifdef CONFIG_64BIT
	20	/*
	21	* As we are sharing code base with the mips32 tree (which use the o32 ABI
	22	* register definitions). We need to redefine the register definitions from
	23	* the n64 ABI register naming to the o32 ABI register naming.
	24	*/
	25	#undef t0
	26	#undef t1
	27	#undef t2
	28	#undef t3
	29	#define t0 $8
	30	#define t1 $9
	31	#define t2 $10
	32	#define t3 $11
	33	#define t4 $12
	34	#define t5 $13
	35	#define t6 $14
	36	#define t7 $15
	37
	38	#define USE_DOUBLE
	39	#endif
	40
	41	#ifdef USE_DOUBLE
	42
	43	#define LOAD ld
	44	#define LOAD32 lwu
	45	#define ADD daddu
	46	#define NBYTES 8
	47
	48	#else
	49
	50	#define LOAD lw
	51	#define LOAD32 lw
	52	#define ADD addu
	53	#define NBYTES 4
	54
	55	#endif /* USE_DOUBLE */
	56
	57	#define UNIT(unit) ((unit)*NBYTES)
	58
	59	#define ADDC(sum,reg) \
	60	.set push; \
	61	.set noat; \
	62	ADD sum, reg; \
	63	sltu v1, sum, reg; \
	64	ADD sum, v1; \
	65	.set pop
	66
	67	#define ADDC32(sum,reg) \
	68	.set push; \
	69	.set noat; \
	70	addu sum, reg; \
	71	sltu v1, sum, reg; \
	72	addu sum, v1; \
	73	.set pop
	74
	75	#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \
	76	LOAD _t0, (offset + UNIT(0))(src); \
	77	LOAD _t1, (offset + UNIT(1))(src); \
	78	LOAD _t2, (offset + UNIT(2))(src); \
	79	LOAD _t3, (offset + UNIT(3))(src); \
	80	ADDC(_t0, _t1); \
	81	ADDC(_t2, _t3); \
	82	ADDC(sum, _t0); \
	83	ADDC(sum, _t2)
	84
	85	#ifdef USE_DOUBLE
	86	#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
	87	CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
	88	#else
	89	#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
	90	CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \
	91	CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
	92	#endif
	93
	94	/*
	95	* a0: source address
	96	* a1: length of the area to checksum
	97	* a2: partial checksum
	98	*/
	99
	100	#define src a0
	101	#define sum v0
	102
	103	.text
	104	.set noreorder
	105	.align 5
	106	LEAF(csum_partial)
	107	EXPORT_SYMBOL(csum_partial)
	108	move sum, zero
	109	move t7, zero
	110
	111	sltiu t8, a1, 0x8
	112	bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */
	113	move t2, a1
	114
	115	andi t7, src, 0x1 /* odd buffer? */
	116
	117	.Lhword_align:
	118	beqz t7, .Lword_align
	119	andi t8, src, 0x2
	120
	121	lbu t0, (src)
	122	LONG_SUBU a1, a1, 0x1
	123	#ifdef __MIPSEL__
	124	sll t0, t0, 8
	125	#endif
	126	ADDC(sum, t0)
	127	PTR_ADDU src, src, 0x1
	128	andi t8, src, 0x2
	129
	130	.Lword_align:
	131	beqz t8, .Ldword_align
	132	sltiu t8, a1, 56
	133
	134	lhu t0, (src)
	135	LONG_SUBU a1, a1, 0x2
	136	ADDC(sum, t0)
	137	sltiu t8, a1, 56
	138	PTR_ADDU src, src, 0x2
	139
	140	.Ldword_align:
	141	bnez t8, .Ldo_end_words
	142	move t8, a1
	143
	144	andi t8, src, 0x4
	145	beqz t8, .Lqword_align
	146	andi t8, src, 0x8
	147
	148	LOAD32 t0, 0x00(src)
	149	LONG_SUBU a1, a1, 0x4
	150	ADDC(sum, t0)
	151	PTR_ADDU src, src, 0x4
	152	andi t8, src, 0x8
	153
	154	.Lqword_align:
	155	beqz t8, .Loword_align
	156	andi t8, src, 0x10
	157
	158	#ifdef USE_DOUBLE
	159	ld t0, 0x00(src)
	160	LONG_SUBU a1, a1, 0x8
	161	ADDC(sum, t0)
	162	#else
	163	lw t0, 0x00(src)
	164	lw t1, 0x04(src)
	165	LONG_SUBU a1, a1, 0x8
	166	ADDC(sum, t0)
	167	ADDC(sum, t1)
	168	#endif
	169	PTR_ADDU src, src, 0x8
	170	andi t8, src, 0x10
	171
	172	.Loword_align:
	173	beqz t8, .Lbegin_movement
	174	LONG_SRL t8, a1, 0x7
	175
	176	#ifdef USE_DOUBLE
	177	ld t0, 0x00(src)
	178	ld t1, 0x08(src)
	179	ADDC(sum, t0)
	180	ADDC(sum, t1)
	181	#else
	182	CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
	183	#endif
	184	LONG_SUBU a1, a1, 0x10
	185	PTR_ADDU src, src, 0x10
	186	LONG_SRL t8, a1, 0x7
	187
	188	.Lbegin_movement:
	189	beqz t8, 1f
	190	andi t2, a1, 0x40
	191
	192	.Lmove_128bytes:
	193	CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
	194	CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
	195	CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
	196	CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
	197	LONG_SUBU t8, t8, 0x01
	198	.set reorder /* DADDI_WAR */
	199	PTR_ADDU src, src, 0x80
	200	bnez t8, .Lmove_128bytes
	201	.set noreorder
	202
	203	1:
	204	beqz t2, 1f
	205	andi t2, a1, 0x20
	206
	207	.Lmove_64bytes:
	208	CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
	209	CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
	210	PTR_ADDU src, src, 0x40
	211
	212	1:
	213	beqz t2, .Ldo_end_words
	214	andi t8, a1, 0x1c
	215
	216	.Lmove_32bytes:
	217	CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
	218	andi t8, a1, 0x1c
	219	PTR_ADDU src, src, 0x20
	220
	221	.Ldo_end_words:
	222	beqz t8, .Lsmall_csumcpy
	223	andi t2, a1, 0x3
	224	LONG_SRL t8, t8, 0x2
	225
	226	.Lend_words:
	227	LOAD32 t0, (src)
	228	LONG_SUBU t8, t8, 0x1
	229	ADDC(sum, t0)
	230	.set reorder /* DADDI_WAR */
	231	PTR_ADDU src, src, 0x4
	232	bnez t8, .Lend_words
	233	.set noreorder
	234
	235	/* unknown src alignment and < 8 bytes to go */
	236	.Lsmall_csumcpy:
	237	move a1, t2
	238
	239	andi t0, a1, 4
	240	beqz t0, 1f
	241	andi t0, a1, 2
	242
	243	/* Still a full word to go */
	244	ulw t1, (src)
	245	PTR_ADDIU src, 4
	246	#ifdef USE_DOUBLE
	247	dsll t1, t1, 32 /* clear lower 32bit */
	248	#endif
	249	ADDC(sum, t1)
	250
	251	1: move t1, zero
	252	beqz t0, 1f
	253	andi t0, a1, 1
	254
	255	/* Still a halfword to go */
	256	ulhu t1, (src)
	257	PTR_ADDIU src, 2
	258
	259	1: beqz t0, 1f
	260	sll t1, t1, 16
	261
	262	lbu t2, (src)
	263	nop
	264
	265	#ifdef __MIPSEB__
	266	sll t2, t2, 8
	267	#endif
	268	or t1, t2
	269
	270	1: ADDC(sum, t1)
	271
	272	/* fold checksum */
	273	#ifdef USE_DOUBLE
	274	dsll32 v1, sum, 0
	275	daddu sum, v1
	276	sltu v1, sum, v1
	277	dsra32 sum, sum, 0
	278	addu sum, v1
	279	#endif
	280
	281	/* odd buffer alignment? */
	282	#if defined(CONFIG_CPU_MIPSR2) \|\| defined(CONFIG_CPU_MIPSR5) \|\| \
	283	defined(CONFIG_CPU_LOONGSON64)
	284	.set push
	285	.set arch=mips32r2
	286	wsbh v1, sum
	287	movn sum, v1, t7
	288	.set pop
	289	#else
	290	beqz t7, 1f /* odd buffer alignment? */
	291	lui v1, 0x00ff
	292	addu v1, 0x00ff
	293	and t0, sum, v1
	294	sll t0, t0, 8
	295	srl sum, sum, 8
	296	and sum, sum, v1
	297	or sum, sum, t0
	298	1:
	299	#endif
	300	.set reorder
	301	/* Add the passed partial csum. */
	302	ADDC32(sum, a2)
	303	jr ra
	304	.set noreorder
	305	END(csum_partial)
	306
	307
	308	/*
	309	* checksum and copy routines based on memcpy.S
	310	*
	311	* csum_partial_copy_nocheck(src, dst, len)
	312	* __csum_partial_copy_kernel(src, dst, len)
	313	*
	314	* See "Spec" in memcpy.S for details. Unlike __copy_user, all
	315	* function in this file use the standard calling convention.
	316	*/
	317
	318	#define src a0
	319	#define dst a1
	320	#define len a2
	321	#define sum v0
	322	#define odd t8
	323
	324	/*
	325	* All exception handlers simply return 0.
	326	*/
	327
	328	/* Instruction type */
	329	#define LD_INSN 1
	330	#define ST_INSN 2
	331	#define LEGACY_MODE 1
	332	#define EVA_MODE 2
	333	#define USEROP 1
	334	#define KERNELOP 2
	335
	336	/*
	337	* Wrapper to add an entry in the exception table
	338	* in case the insn causes a memory exception.
	339	* Arguments:
	340	* insn : Load/store instruction
	341	* type : Instruction type
	342	* reg : Register
	343	* addr : Address
	344	* handler : Exception handler
	345	*/
	346	#define EXC(insn, type, reg, addr) \
	347	.if \mode == LEGACY_MODE; \
	348	9: insn reg, addr; \
	349	.section __ex_table,"a"; \
	350	PTR_WD 9b, .L_exc; \
	351	.previous; \
	352	/* This is enabled in EVA mode */ \
	353	.else; \
	354	/* If loading from user or storing to user */ \
	355	.if ((\from == USEROP) && (type == LD_INSN)) \|\| \
	356	((\to == USEROP) && (type == ST_INSN)); \
	357	9: __BUILD_EVA_INSN(insn##e, reg, addr); \
	358	.section __ex_table,"a"; \
	359	PTR_WD 9b, .L_exc; \
	360	.previous; \
	361	.else; \
	362	/* EVA without exception */ \
	363	insn reg, addr; \
	364	.endif; \
	365	.endif
	366
	367	#undef LOAD
	368
	369	#ifdef USE_DOUBLE
	370
	371	#define LOADK ld /* No exception */
	372	#define LOAD(reg, addr) EXC(ld, LD_INSN, reg, addr)
	373	#define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr)
	374	#define LOADL(reg, addr) EXC(ldl, LD_INSN, reg, addr)
	375	#define LOADR(reg, addr) EXC(ldr, LD_INSN, reg, addr)
	376	#define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr)
	377	#define STOREL(reg, addr) EXC(sdl, ST_INSN, reg, addr)
	378	#define STORER(reg, addr) EXC(sdr, ST_INSN, reg, addr)
	379	#define STORE(reg, addr) EXC(sd, ST_INSN, reg, addr)
	380	#define ADD daddu
	381	#define SUB dsubu
	382	#define SRL dsrl
	383	#define SLL dsll
	384	#define SLLV dsllv
	385	#define SRLV dsrlv
	386	#define NBYTES 8
	387	#define LOG_NBYTES 3
	388
	389	#else
	390
	391	#define LOADK lw /* No exception */
	392	#define LOAD(reg, addr) EXC(lw, LD_INSN, reg, addr)
	393	#define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr)
	394	#define LOADL(reg, addr) EXC(lwl, LD_INSN, reg, addr)
	395	#define LOADR(reg, addr) EXC(lwr, LD_INSN, reg, addr)
	396	#define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr)
	397	#define STOREL(reg, addr) EXC(swl, ST_INSN, reg, addr)
	398	#define STORER(reg, addr) EXC(swr, ST_INSN, reg, addr)
	399	#define STORE(reg, addr) EXC(sw, ST_INSN, reg, addr)
	400	#define ADD addu
	401	#define SUB subu
	402	#define SRL srl
	403	#define SLL sll
	404	#define SLLV sllv
	405	#define SRLV srlv
	406	#define NBYTES 4
	407	#define LOG_NBYTES 2
	408
	409	#endif /* USE_DOUBLE */
	410
	411	#ifdef CONFIG_CPU_LITTLE_ENDIAN
	412	#define LDFIRST LOADR
	413	#define LDREST LOADL
	414	#define STFIRST STORER
	415	#define STREST STOREL
	416	#define SHIFT_DISCARD SLLV
	417	#define SHIFT_DISCARD_REVERT SRLV
	418	#else
	419	#define LDFIRST LOADL
	420	#define LDREST LOADR
	421	#define STFIRST STOREL
	422	#define STREST STORER
	423	#define SHIFT_DISCARD SRLV
	424	#define SHIFT_DISCARD_REVERT SLLV
	425	#endif
	426
	427	#define FIRST(unit) ((unit)*NBYTES)
	428	#define REST(unit) (FIRST(unit)+NBYTES-1)
	429
	430	#define ADDRMASK (NBYTES-1)
	431
	432	#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
	433	.set noat
	434	#else
	435	.set at=v1
	436	#endif
	437
	438	.macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to
	439
	440	li sum, -1
	441	move odd, zero
	442	/*
	443	* Note: dst & src may be unaligned, len may be 0
	444	* Temps
	445	*/
	446	/*
	447	* The "issue break"s below are very approximate.
	448	* Issue delays for dcache fills will perturb the schedule, as will
	449	* load queue full replay traps, etc.
	450	*
	451	* If len < NBYTES use byte operations.
	452	*/
	453	sltu t2, len, NBYTES
	454	and t1, dst, ADDRMASK
	455	bnez t2, .Lcopy_bytes_checklen\@
	456	and t0, src, ADDRMASK
	457	andi odd, dst, 0x1 /* odd buffer? */
	458	bnez t1, .Ldst_unaligned\@
	459	nop
	460	bnez t0, .Lsrc_unaligned_dst_aligned\@
	461	/*
	462	* use delay slot for fall-through
	463	* src and dst are aligned; need to compute rem
	464	*/
	465	.Lboth_aligned\@:
	466	SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
	467	beqz t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
	468	nop
	469	SUB len, 8*NBYTES # subtract here for bgez loop
	470	.align 4
	471	1:
	472	LOAD(t0, UNIT(0)(src))
	473	LOAD(t1, UNIT(1)(src))
	474	LOAD(t2, UNIT(2)(src))
	475	LOAD(t3, UNIT(3)(src))
	476	LOAD(t4, UNIT(4)(src))
	477	LOAD(t5, UNIT(5)(src))
	478	LOAD(t6, UNIT(6)(src))
	479	LOAD(t7, UNIT(7)(src))
	480	SUB len, len, 8*NBYTES
	481	ADD src, src, 8*NBYTES
	482	STORE(t0, UNIT(0)(dst))
	483	ADDC(t0, t1)
	484	STORE(t1, UNIT(1)(dst))
	485	ADDC(sum, t0)
	486	STORE(t2, UNIT(2)(dst))
	487	ADDC(t2, t3)
	488	STORE(t3, UNIT(3)(dst))
	489	ADDC(sum, t2)
	490	STORE(t4, UNIT(4)(dst))
	491	ADDC(t4, t5)
	492	STORE(t5, UNIT(5)(dst))
	493	ADDC(sum, t4)
	494	STORE(t6, UNIT(6)(dst))
	495	ADDC(t6, t7)
	496	STORE(t7, UNIT(7)(dst))
	497	ADDC(sum, t6)
	498	.set reorder /* DADDI_WAR */
	499	ADD dst, dst, 8*NBYTES
	500	bgez len, 1b
	501	.set noreorder
	502	ADD len, 8*NBYTES # revert len (see above)
	503
	504	/*
	505	* len == the number of bytes left to copy < 8*NBYTES
	506	*/
	507	.Lcleanup_both_aligned\@:
	508	#define rem t7
	509	beqz len, .Ldone\@
	510	sltu t0, len, 4*NBYTES
	511	bnez t0, .Lless_than_4units\@
	512	and rem, len, (NBYTES-1) # rem = len % NBYTES
	513	/*
	514	* len >= 4*NBYTES
	515	*/
	516	LOAD(t0, UNIT(0)(src))
	517	LOAD(t1, UNIT(1)(src))
	518	LOAD(t2, UNIT(2)(src))
	519	LOAD(t3, UNIT(3)(src))
	520	SUB len, len, 4*NBYTES
	521	ADD src, src, 4*NBYTES
	522	STORE(t0, UNIT(0)(dst))
	523	ADDC(t0, t1)
	524	STORE(t1, UNIT(1)(dst))
	525	ADDC(sum, t0)
	526	STORE(t2, UNIT(2)(dst))
	527	ADDC(t2, t3)
	528	STORE(t3, UNIT(3)(dst))
	529	ADDC(sum, t2)
	530	.set reorder /* DADDI_WAR */
	531	ADD dst, dst, 4*NBYTES
	532	beqz len, .Ldone\@
	533	.set noreorder
	534	.Lless_than_4units\@:
	535	/*
	536	* rem = len % NBYTES
	537	*/
	538	beq rem, len, .Lcopy_bytes\@
	539	nop
	540	1:
	541	LOAD(t0, 0(src))
	542	ADD src, src, NBYTES
	543	SUB len, len, NBYTES
	544	STORE(t0, 0(dst))
	545	ADDC(sum, t0)
	546	.set reorder /* DADDI_WAR */
	547	ADD dst, dst, NBYTES
	548	bne rem, len, 1b
	549	.set noreorder
	550
	551	/*
	552	* src and dst are aligned, need to copy rem bytes (rem < NBYTES)
	553	* A loop would do only a byte at a time with possible branch
	554	* mispredicts. Can't do an explicit LOAD dst,mask,or,STORE
	555	* because can't assume read-access to dst. Instead, use
	556	* STREST dst, which doesn't require read access to dst.
	557	*
	558	* This code should perform better than a simple loop on modern,
	559	* wide-issue mips processors because the code has fewer branches and
	560	* more instruction-level parallelism.
	561	*/
	562	#define bits t2
	563	beqz len, .Ldone\@
	564	ADD t1, dst, len # t1 is just past last byte of dst
	565	li bits, 8*NBYTES
	566	SLL rem, len, 3 # rem = number of bits to keep
	567	LOAD(t0, 0(src))
	568	SUB bits, bits, rem # bits = number of bits to discard
	569	SHIFT_DISCARD t0, t0, bits
	570	STREST(t0, -1(t1))
	571	SHIFT_DISCARD_REVERT t0, t0, bits
	572	.set reorder
	573	ADDC(sum, t0)
	574	b .Ldone\@
	575	.set noreorder
	576	.Ldst_unaligned\@:
	577	/*
	578	* dst is unaligned
	579	* t0 = src & ADDRMASK
	580	* t1 = dst & ADDRMASK; T1 > 0
	581	* len >= NBYTES
	582	*
	583	* Copy enough bytes to align dst
	584	* Set match = (src and dst have same alignment)
	585	*/
	586	#define match rem
	587	LDFIRST(t3, FIRST(0)(src))
	588	ADD t2, zero, NBYTES
	589	LDREST(t3, REST(0)(src))
	590	SUB t2, t2, t1 # t2 = number of bytes copied
	591	xor match, t0, t1
	592	STFIRST(t3, FIRST(0)(dst))
	593	SLL t4, t1, 3 # t4 = number of bits to discard
	594	SHIFT_DISCARD t3, t3, t4
	595	/* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
	596	ADDC(sum, t3)
	597	beq len, t2, .Ldone\@
	598	SUB len, len, t2
	599	ADD dst, dst, t2
	600	beqz match, .Lboth_aligned\@
	601	ADD src, src, t2
	602
	603	.Lsrc_unaligned_dst_aligned\@:
	604	SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
	605	beqz t0, .Lcleanup_src_unaligned\@
	606	and rem, len, (4NBYTES-1) # rem = len % 4NBYTES
	607	1:
	608	/*
	609	* Avoid consecutive LD*'s to the same register since some mips
	610	* implementations can't issue them in the same cycle.
	611	* It's OK to load FIRST(N+1) before REST(N) because the two addresses
	612	* are to the same unit (unless src is aligned, but it's not).
	613	*/
	614	LDFIRST(t0, FIRST(0)(src))
	615	LDFIRST(t1, FIRST(1)(src))
	616	SUB len, len, 4*NBYTES
	617	LDREST(t0, REST(0)(src))
	618	LDREST(t1, REST(1)(src))
	619	LDFIRST(t2, FIRST(2)(src))
	620	LDFIRST(t3, FIRST(3)(src))
	621	LDREST(t2, REST(2)(src))
	622	LDREST(t3, REST(3)(src))
	623	ADD src, src, 4*NBYTES
	624	#ifdef CONFIG_CPU_SB1
	625	nop # improves slotting
	626	#endif
	627	STORE(t0, UNIT(0)(dst))
	628	ADDC(t0, t1)
	629	STORE(t1, UNIT(1)(dst))
	630	ADDC(sum, t0)
	631	STORE(t2, UNIT(2)(dst))
	632	ADDC(t2, t3)
	633	STORE(t3, UNIT(3)(dst))
	634	ADDC(sum, t2)
	635	.set reorder /* DADDI_WAR */
	636	ADD dst, dst, 4*NBYTES
	637	bne len, rem, 1b
	638	.set noreorder
	639
	640	.Lcleanup_src_unaligned\@:
	641	beqz len, .Ldone\@
	642	and rem, len, NBYTES-1 # rem = len % NBYTES
	643	beq rem, len, .Lcopy_bytes\@
	644	nop
	645	1:
	646	LDFIRST(t0, FIRST(0)(src))
	647	LDREST(t0, REST(0)(src))
	648	ADD src, src, NBYTES
	649	SUB len, len, NBYTES
	650	STORE(t0, 0(dst))
	651	ADDC(sum, t0)
	652	.set reorder /* DADDI_WAR */
	653	ADD dst, dst, NBYTES
	654	bne len, rem, 1b
	655	.set noreorder
	656
	657	.Lcopy_bytes_checklen\@:
	658	beqz len, .Ldone\@
	659	nop
	660	.Lcopy_bytes\@:
	661	/* 0 < len < NBYTES */
	662	#ifdef CONFIG_CPU_LITTLE_ENDIAN
	663	#define SHIFT_START 0
	664	#define SHIFT_INC 8
	665	#else
	666	#define SHIFT_START 8*(NBYTES-1)
	667	#define SHIFT_INC -8
	668	#endif
	669	move t2, zero # partial word
	670	li t3, SHIFT_START # shift
	671	#define COPY_BYTE(N) \
	672	LOADBU(t0, N(src)); \
	673	SUB len, len, 1; \
	674	STOREB(t0, N(dst)); \
	675	SLLV t0, t0, t3; \
	676	addu t3, SHIFT_INC; \
	677	beqz len, .Lcopy_bytes_done\@; \
	678	or t2, t0
	679
	680	COPY_BYTE(0)
	681	COPY_BYTE(1)
	682	#ifdef USE_DOUBLE
	683	COPY_BYTE(2)
	684	COPY_BYTE(3)
	685	COPY_BYTE(4)
	686	COPY_BYTE(5)
	687	#endif
	688	LOADBU(t0, NBYTES-2(src))
	689	SUB len, len, 1
	690	STOREB(t0, NBYTES-2(dst))
	691	SLLV t0, t0, t3
	692	or t2, t0
	693	.Lcopy_bytes_done\@:
	694	ADDC(sum, t2)
	695	.Ldone\@:
	696	/* fold checksum */
	697	.set push
	698	.set noat
	699	#ifdef USE_DOUBLE
	700	dsll32 v1, sum, 0
	701	daddu sum, v1
	702	sltu v1, sum, v1
	703	dsra32 sum, sum, 0
	704	addu sum, v1
	705	#endif
	706
	707	#if defined(CONFIG_CPU_MIPSR2) \|\| defined(CONFIG_CPU_MIPSR5) \|\| \
	708	defined(CONFIG_CPU_LOONGSON64)
	709	.set push
	710	.set arch=mips32r2
	711	wsbh v1, sum
	712	movn sum, v1, odd
	713	.set pop
	714	#else
	715	beqz odd, 1f /* odd buffer alignment? */
	716	lui v1, 0x00ff
	717	addu v1, 0x00ff
	718	and t0, sum, v1
	719	sll t0, t0, 8
	720	srl sum, sum, 8
	721	and sum, sum, v1
	722	or sum, sum, t0
	723	1:
	724	#endif
	725	.set pop
	726	.set reorder
	727	jr ra
	728	.set noreorder
	729	.endm
	730
	731	.set noreorder
	732	.L_exc:
	733	jr ra
	734	li v0, 0
	735
	736	FEXPORT(__csum_partial_copy_nocheck)
	737	EXPORT_SYMBOL(__csum_partial_copy_nocheck)
	738	#ifndef CONFIG_EVA
	739	FEXPORT(__csum_partial_copy_to_user)
	740	EXPORT_SYMBOL(__csum_partial_copy_to_user)
	741	FEXPORT(__csum_partial_copy_from_user)
	742	EXPORT_SYMBOL(__csum_partial_copy_from_user)
	743	#endif
	744	__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP
	745
	746	#ifdef CONFIG_EVA
	747	LEAF(__csum_partial_copy_to_user)
	748	__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP
	749	END(__csum_partial_copy_to_user)
	750
	751	LEAF(__csum_partial_copy_from_user)
	752	__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP
	753	END(__csum_partial_copy_from_user)
	754	#endif