Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
535c611d HC |
2 | /* |
3 | * String handling functions. | |
4 | * | |
5 | * Copyright IBM Corp. 2012 | |
6 | */ | |
7 | ||
b8c723f1 | 8 | #include <linux/export.h> |
535c611d | 9 | #include <linux/linkage.h> |
97489e06 MS |
10 | #include <asm/nospec-insn.h> |
11 | ||
12 | GEN_BR_THUNK %r14 | |
535c611d | 13 | |
b4623d4e HC |
14 | /* |
15 | * void *memmove(void *dest, const void *src, size_t n) | |
16 | */ | |
e48b6853 | 17 | SYM_FUNC_START(__memmove) |
b4623d4e HC |
18 | ltgr %r4,%r4 |
19 | lgr %r1,%r2 | |
5eda25b1 | 20 | jz .Lmemmove_exit |
551f4134 | 21 | aghi %r4,-1 |
b4623d4e HC |
22 | clgr %r2,%r3 |
23 | jnh .Lmemmove_forward | |
551f4134 | 24 | la %r5,1(%r4,%r3) |
b4623d4e HC |
25 | clgr %r2,%r5 |
26 | jl .Lmemmove_reverse | |
27 | .Lmemmove_forward: | |
b4623d4e HC |
28 | srlg %r0,%r4,8 |
29 | ltgr %r0,%r0 | |
551f4134 HC |
30 | jz .Lmemmove_forward_remainder |
31 | .Lmemmove_forward_loop: | |
b4623d4e HC |
32 | mvc 0(256,%r1),0(%r3) |
33 | la %r1,256(%r1) | |
34 | la %r3,256(%r3) | |
551f4134 HC |
35 | brctg %r0,.Lmemmove_forward_loop |
36 | .Lmemmove_forward_remainder: | |
b4623d4e HC |
37 | larl %r5,.Lmemmove_mvc |
38 | ex %r4,0(%r5) | |
5eda25b1 | 39 | .Lmemmove_exit: |
97489e06 | 40 | BR_EX %r14 |
b4623d4e | 41 | .Lmemmove_reverse: |
b4623d4e HC |
42 | ic %r0,0(%r4,%r3) |
43 | stc %r0,0(%r4,%r1) | |
551f4134 | 44 | brctg %r4,.Lmemmove_reverse |
b4623d4e HC |
45 | ic %r0,0(%r4,%r3) |
46 | stc %r0,0(%r4,%r1) | |
97489e06 | 47 | BR_EX %r14 |
b4623d4e HC |
48 | .Lmemmove_mvc: |
49 | mvc 0(1,%r1),0(%r3) | |
e48b6853 HC |
50 | SYM_FUNC_END(__memmove) |
51 | EXPORT_SYMBOL(__memmove) | |
52 | ||
53 | SYM_FUNC_ALIAS(memmove, __memmove) | |
b4623d4e HC |
54 | EXPORT_SYMBOL(memmove) |
55 | ||
535c611d HC |
56 | /* |
57 | * memset implementation | |
58 | * | |
59 | * This code corresponds to the C construct below. We do distinguish | |
60 | * between clearing (c == 0) and setting a memory array (c != 0) simply | |
61 | * because nearly all memset invocations in the kernel clear memory and | |
62 | * the xc instruction is preferred in such cases. | |
63 | * | |
64 | * void *memset(void *s, int c, size_t n) | |
65 | * { | |
66 | * if (likely(c == 0)) | |
67 | * return __builtin_memset(s, 0, n); | |
68 | * return __builtin_memset(s, c, n); | |
69 | * } | |
70 | */ | |
e48b6853 | 71 | SYM_FUNC_START(__memset) |
535c611d | 72 | ltgr %r4,%r4 |
5eda25b1 | 73 | jz .Lmemset_exit |
535c611d HC |
74 | ltgr %r3,%r3 |
75 | jnz .Lmemset_fill | |
76 | aghi %r4,-1 | |
77 | srlg %r3,%r4,8 | |
78 | ltgr %r3,%r3 | |
79 | lgr %r1,%r2 | |
551f4134 | 80 | jz .Lmemset_clear_remainder |
535c611d HC |
81 | .Lmemset_clear_loop: |
82 | xc 0(256,%r1),0(%r1) | |
83 | la %r1,256(%r1) | |
84 | brctg %r3,.Lmemset_clear_loop | |
551f4134 | 85 | .Lmemset_clear_remainder: |
535c611d HC |
86 | larl %r3,.Lmemset_xc |
87 | ex %r4,0(%r3) | |
5eda25b1 | 88 | .Lmemset_exit: |
97489e06 | 89 | BR_EX %r14 |
535c611d | 90 | .Lmemset_fill: |
535c611d HC |
91 | cghi %r4,1 |
92 | lgr %r1,%r2 | |
993fef95 | 93 | je .Lmemset_fill_exit |
535c611d | 94 | aghi %r4,-2 |
993fef95 HC |
95 | srlg %r5,%r4,8 |
96 | ltgr %r5,%r5 | |
551f4134 | 97 | jz .Lmemset_fill_remainder |
535c611d | 98 | .Lmemset_fill_loop: |
993fef95 HC |
99 | stc %r3,0(%r1) |
100 | mvc 1(255,%r1),0(%r1) | |
535c611d | 101 | la %r1,256(%r1) |
993fef95 | 102 | brctg %r5,.Lmemset_fill_loop |
551f4134 | 103 | .Lmemset_fill_remainder: |
993fef95 HC |
104 | stc %r3,0(%r1) |
105 | larl %r5,.Lmemset_mvc | |
106 | ex %r4,0(%r5) | |
97489e06 | 107 | BR_EX %r14 |
993fef95 HC |
108 | .Lmemset_fill_exit: |
109 | stc %r3,0(%r1) | |
97489e06 | 110 | BR_EX %r14 |
535c611d HC |
111 | .Lmemset_xc: |
112 | xc 0(1,%r1),0(%r1) | |
113 | .Lmemset_mvc: | |
114 | mvc 1(1,%r1),0(%r1) | |
e48b6853 HC |
115 | SYM_FUNC_END(__memset) |
116 | EXPORT_SYMBOL(__memset) | |
117 | ||
118 | SYM_FUNC_ALIAS(memset, __memset) | |
711f5df7 | 119 | EXPORT_SYMBOL(memset) |
535c611d HC |
120 | |
121 | /* | |
122 | * memcpy implementation | |
123 | * | |
124 | * void *memcpy(void *dest, const void *src, size_t n) | |
125 | */ | |
e48b6853 | 126 | SYM_FUNC_START(__memcpy) |
535c611d | 127 | ltgr %r4,%r4 |
5eda25b1 | 128 | jz .Lmemcpy_exit |
535c611d HC |
129 | aghi %r4,-1 |
130 | srlg %r5,%r4,8 | |
131 | ltgr %r5,%r5 | |
132 | lgr %r1,%r2 | |
133 | jnz .Lmemcpy_loop | |
551f4134 | 134 | .Lmemcpy_remainder: |
535c611d HC |
135 | larl %r5,.Lmemcpy_mvc |
136 | ex %r4,0(%r5) | |
5eda25b1 | 137 | .Lmemcpy_exit: |
97489e06 | 138 | BR_EX %r14 |
535c611d HC |
139 | .Lmemcpy_loop: |
140 | mvc 0(256,%r1),0(%r3) | |
141 | la %r1,256(%r1) | |
142 | la %r3,256(%r3) | |
143 | brctg %r5,.Lmemcpy_loop | |
551f4134 | 144 | j .Lmemcpy_remainder |
535c611d HC |
145 | .Lmemcpy_mvc: |
146 | mvc 0(1,%r1),0(%r3) | |
e48b6853 HC |
147 | SYM_FUNC_END(__memcpy) |
148 | EXPORT_SYMBOL(__memcpy) | |
149 | ||
150 | SYM_FUNC_ALIAS(memcpy, __memcpy) | |
711f5df7 | 151 | EXPORT_SYMBOL(memcpy) |
0b77d670 HC |
152 | |
153 | /* | |
154 | * __memset16/32/64 | |
155 | * | |
156 | * void *__memset16(uint16_t *s, uint16_t v, size_t count) | |
157 | * void *__memset32(uint32_t *s, uint32_t v, size_t count) | |
158 | * void *__memset64(uint64_t *s, uint64_t v, size_t count) | |
159 | */ | |
160 | .macro __MEMSET bits,bytes,insn | |
45769052 | 161 | SYM_FUNC_START(__memset\bits) |
0b77d670 | 162 | ltgr %r4,%r4 |
5eda25b1 | 163 | jz .L__memset_exit\bits |
0b77d670 | 164 | cghi %r4,\bytes |
5eda25b1 | 165 | je .L__memset_store\bits |
0b77d670 HC |
166 | aghi %r4,-(\bytes+1) |
167 | srlg %r5,%r4,8 | |
168 | ltgr %r5,%r5 | |
169 | lgr %r1,%r2 | |
170 | jz .L__memset_remainder\bits | |
171 | .L__memset_loop\bits: | |
172 | \insn %r3,0(%r1) | |
173 | mvc \bytes(256-\bytes,%r1),0(%r1) | |
174 | la %r1,256(%r1) | |
175 | brctg %r5,.L__memset_loop\bits | |
176 | .L__memset_remainder\bits: | |
177 | \insn %r3,0(%r1) | |
178 | larl %r5,.L__memset_mvc\bits | |
179 | ex %r4,0(%r5) | |
97489e06 | 180 | BR_EX %r14 |
5eda25b1 | 181 | .L__memset_store\bits: |
0b77d670 | 182 | \insn %r3,0(%r2) |
5eda25b1 | 183 | .L__memset_exit\bits: |
97489e06 | 184 | BR_EX %r14 |
0b77d670 HC |
185 | .L__memset_mvc\bits: |
186 | mvc \bytes(1,%r1),0(%r1) | |
45769052 | 187 | SYM_FUNC_END(__memset\bits) |
0b77d670 HC |
188 | .endm |
189 | ||
190 | __MEMSET 16,2,sth | |
191 | EXPORT_SYMBOL(__memset16) | |
192 | ||
193 | __MEMSET 32,4,st | |
194 | EXPORT_SYMBOL(__memset32) | |
195 | ||
196 | __MEMSET 64,8,stg | |
197 | EXPORT_SYMBOL(__memset64) |