| 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | /* |
| 3 | * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> |
| 4 | * Copyright 2002 Andi Kleen, SuSE Labs. |
| 5 | * |
| 6 | * Functions to copy from and to user space. |
| 7 | */ |
| 8 | |
| 9 | #include <linux/linkage.h> |
| 10 | #include <asm/cpufeatures.h> |
| 11 | #include <asm/alternative.h> |
| 12 | #include <asm/asm.h> |
| 13 | #include <asm/export.h> |
| 14 | |
| 15 | /* |
| 16 | * rep_movs_alternative - memory copy with exception handling. |
| 17 | * This version is for CPUs that don't have FSRM (Fast Short Rep Movs) |
| 18 | * |
| 19 | * Input: |
| 20 | * rdi destination |
| 21 | * rsi source |
| 22 | * rcx count |
| 23 | * |
| 24 | * Output: |
| 25 | * rcx uncopied bytes or 0 if successful. |
| 26 | * |
| 27 | * NOTE! The calling convention is very intentionally the same as |
| 28 | * for 'rep movs', so that we can rewrite the function call with |
| 29 | * just a plain 'rep movs' on machines that have FSRM. But to make |
| 30 | * it simpler for us, we can clobber rsi/rdi and rax freely. |
| 31 | */ |
| 32 | SYM_FUNC_START(rep_movs_alternative) |
| 33 | cmpq $64,%rcx |
| 34 | jae .Llarge |
| 35 | |
| 36 | cmp $8,%ecx |
| 37 | jae .Lword |
| 38 | |
| 39 | testl %ecx,%ecx |
| 40 | je .Lexit |
| 41 | |
| 42 | .Lcopy_user_tail: |
| 43 | 0: movb (%rsi),%al |
| 44 | 1: movb %al,(%rdi) |
| 45 | inc %rdi |
| 46 | inc %rsi |
| 47 | dec %rcx |
| 48 | jne .Lcopy_user_tail |
| 49 | .Lexit: |
| 50 | RET |
| 51 | |
| 52 | _ASM_EXTABLE_UA( 0b, .Lexit) |
| 53 | _ASM_EXTABLE_UA( 1b, .Lexit) |
| 54 | |
| 55 | .p2align 4 |
| 56 | .Lword: |
| 57 | 2: movq (%rsi),%rax |
| 58 | 3: movq %rax,(%rdi) |
| 59 | addq $8,%rsi |
| 60 | addq $8,%rdi |
| 61 | sub $8,%ecx |
| 62 | je .Lexit |
| 63 | cmp $8,%ecx |
| 64 | jae .Lword |
| 65 | jmp .Lcopy_user_tail |
| 66 | |
| 67 | _ASM_EXTABLE_UA( 2b, .Lcopy_user_tail) |
| 68 | _ASM_EXTABLE_UA( 3b, .Lcopy_user_tail) |
| 69 | |
| 70 | .Llarge: |
| 71 | 0: ALTERNATIVE "jmp .Llarge_movsq", "rep movsb", X86_FEATURE_ERMS |
| 72 | 1: RET |
| 73 | |
| 74 | _ASM_EXTABLE_UA( 0b, 1b) |
| 75 | |
| 76 | .Llarge_movsq: |
| 77 | movq %rcx,%rax |
| 78 | shrq $3,%rcx |
| 79 | andl $7,%eax |
| 80 | 0: rep movsq |
| 81 | movl %eax,%ecx |
| 82 | testl %ecx,%ecx |
| 83 | jne .Lcopy_user_tail |
| 84 | RET |
| 85 | |
| 86 | 1: leaq (%rax,%rcx,8),%rcx |
| 87 | jmp .Lcopy_user_tail |
| 88 | |
| 89 | _ASM_EXTABLE_UA( 0b, 1b) |
| 90 | SYM_FUNC_END(rep_movs_alternative) |
| 91 | EXPORT_SYMBOL(rep_movs_alternative) |