Commit | Line | Data |
---|---|---|
559671e0 HC |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* | |
3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
4 | */ | |
5 | ||
a275a82d | 6 | #include <asm/alternative-asm.h> |
559671e0 HC |
7 | #include <asm/asm.h> |
8 | #include <asm/asmmacro.h> | |
508f28c6 | 9 | #include <asm/asm-extable.h> |
a275a82d | 10 | #include <asm/cpu.h> |
559671e0 HC |
11 | #include <asm/export.h> |
12 | #include <asm/regdef.h> | |
13 | ||
a275a82d | 14 | .irp to, 0, 1, 2, 3, 4, 5, 6, 7 |
912bcfaf YT |
15 | .L_fixup_handle_\to\(): |
16 | addi.d a0, a2, (\to) * (-8) | |
559671e0 | 17 | jr ra |
912bcfaf | 18 | .endr |
559671e0 | 19 | |
a275a82d HC |
20 | SYM_FUNC_START(__copy_user) |
21 | /* | |
22 | * Some CPUs support hardware unaligned access | |
23 | */ | |
24 | ALTERNATIVE "b __copy_user_generic", \ | |
25 | "b __copy_user_fast", CPU_FEATURE_UAL | |
26 | SYM_FUNC_END(__copy_user) | |
27 | ||
28 | EXPORT_SYMBOL(__copy_user) | |
29 | ||
559671e0 | 30 | /* |
a275a82d | 31 | * unsigned long __copy_user_generic(void *to, const void *from, size_t n) |
559671e0 HC |
32 | * |
33 | * a0: to | |
34 | * a1: from | |
35 | * a2: n | |
36 | */ | |
a275a82d | 37 | SYM_FUNC_START(__copy_user_generic) |
559671e0 HC |
38 | beqz a2, 3f |
39 | ||
40 | 1: ld.b t0, a1, 0 | |
41 | 2: st.b t0, a0, 0 | |
42 | addi.d a0, a0, 1 | |
43 | addi.d a1, a1, 1 | |
44 | addi.d a2, a2, -1 | |
1fdb9a92 | 45 | bgtz a2, 1b |
559671e0 HC |
46 | |
47 | 3: move a0, a2 | |
48 | jr ra | |
49 | ||
912bcfaf YT |
50 | _asm_extable 1b, .L_fixup_handle_0 |
51 | _asm_extable 2b, .L_fixup_handle_0 | |
a275a82d | 52 | SYM_FUNC_END(__copy_user_generic) |
559671e0 | 53 | |
a275a82d HC |
54 | /* |
55 | * unsigned long __copy_user_fast(void *to, const void *from, unsigned long n) | |
56 | * | |
57 | * a0: to | |
58 | * a1: from | |
59 | * a2: n | |
60 | */ | |
61 | SYM_FUNC_START(__copy_user_fast) | |
62 | beqz a2, 19f | |
63 | ||
64 | ori a3, zero, 64 | |
65 | blt a2, a3, 17f | |
66 | ||
67 | /* copy 64 bytes at a time */ | |
68 | 1: ld.d t0, a1, 0 | |
69 | 2: ld.d t1, a1, 8 | |
70 | 3: ld.d t2, a1, 16 | |
71 | 4: ld.d t3, a1, 24 | |
72 | 5: ld.d t4, a1, 32 | |
73 | 6: ld.d t5, a1, 40 | |
74 | 7: ld.d t6, a1, 48 | |
75 | 8: ld.d t7, a1, 56 | |
76 | 9: st.d t0, a0, 0 | |
77 | 10: st.d t1, a0, 8 | |
78 | 11: st.d t2, a0, 16 | |
79 | 12: st.d t3, a0, 24 | |
80 | 13: st.d t4, a0, 32 | |
81 | 14: st.d t5, a0, 40 | |
82 | 15: st.d t6, a0, 48 | |
83 | 16: st.d t7, a0, 56 | |
84 | ||
85 | addi.d a0, a0, 64 | |
86 | addi.d a1, a1, 64 | |
87 | addi.d a2, a2, -64 | |
88 | bge a2, a3, 1b | |
89 | ||
90 | beqz a2, 19f | |
91 | ||
92 | /* copy the remaining bytes */ | |
93 | 17: ld.b t0, a1, 0 | |
94 | 18: st.b t0, a0, 0 | |
95 | addi.d a0, a0, 1 | |
96 | addi.d a1, a1, 1 | |
97 | addi.d a2, a2, -1 | |
98 | bgt a2, zero, 17b | |
99 | ||
100 | /* return */ | |
101 | 19: move a0, a2 | |
102 | jr ra | |
103 | ||
104 | /* fixup and ex_table */ | |
105 | _asm_extable 1b, .L_fixup_handle_0 | |
106 | _asm_extable 2b, .L_fixup_handle_1 | |
107 | _asm_extable 3b, .L_fixup_handle_2 | |
108 | _asm_extable 4b, .L_fixup_handle_3 | |
109 | _asm_extable 5b, .L_fixup_handle_4 | |
110 | _asm_extable 6b, .L_fixup_handle_5 | |
111 | _asm_extable 7b, .L_fixup_handle_6 | |
112 | _asm_extable 8b, .L_fixup_handle_7 | |
113 | _asm_extable 9b, .L_fixup_handle_0 | |
114 | _asm_extable 10b, .L_fixup_handle_1 | |
115 | _asm_extable 11b, .L_fixup_handle_2 | |
116 | _asm_extable 12b, .L_fixup_handle_3 | |
117 | _asm_extable 13b, .L_fixup_handle_4 | |
118 | _asm_extable 14b, .L_fixup_handle_5 | |
119 | _asm_extable 15b, .L_fixup_handle_6 | |
120 | _asm_extable 16b, .L_fixup_handle_7 | |
121 | _asm_extable 17b, .L_fixup_handle_0 | |
122 | _asm_extable 18b, .L_fixup_handle_0 | |
123 | SYM_FUNC_END(__copy_user_fast) |