Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * ppc64 MMU hashtable management routines | |
3 | * | |
3c726f8d | 4 | * (c) Copyright IBM Corp. 2003, 2005 |
1da177e4 LT |
5 | * |
6 | * Maintained by: Benjamin Herrenschmidt | |
7 | * <benh@kernel.crashing.org> | |
8 | * | |
9 | * This file is covered by the GNU Public Licence v2 as | |
10 | * described in the kernel's COPYING file. | |
11 | */ | |
12 | ||
ab1f9dac | 13 | #include <asm/reg.h> |
1da177e4 LT |
14 | #include <asm/pgtable.h> |
15 | #include <asm/mmu.h> | |
16 | #include <asm/page.h> | |
17 | #include <asm/types.h> | |
18 | #include <asm/ppc_asm.h> | |
0013a854 | 19 | #include <asm/asm-offsets.h> |
1da177e4 LT |
20 | #include <asm/cputable.h> |
21 | ||
22 | .text | |
23 | ||
24 | /* | |
25 | * Stackframe: | |
26 | * | |
27 | * +-> Back chain (SP + 256) | |
28 | * | General register save area (SP + 112) | |
29 | * | Parameter save area (SP + 48) | |
30 | * | TOC save area (SP + 40) | |
31 | * | link editor doubleword (SP + 32) | |
32 | * | compiler doubleword (SP + 24) | |
33 | * | LR save area (SP + 16) | |
34 | * | CR save area (SP + 8) | |
35 | * SP ---> +-- Back chain (SP + 0) | |
36 | */ | |
3c726f8d BH |
37 | |
38 | #ifndef CONFIG_PPC_64K_PAGES | |
39 | ||
40 | /***************************************************************************** | |
41 | * * | |
42 | * 4K SW & 4K HW pages implementation * | |
43 | * * | |
44 | *****************************************************************************/ | |
45 | ||
46 | ||
1da177e4 | 47 | /* |
3c726f8d | 48 | * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, |
aefa5688 AK |
49 | * pte_t *ptep, unsigned long trap, unsigned long flags, |
50 | * int ssize) | |
1da177e4 | 51 | * |
3c726f8d | 52 | * Adds a 4K page to the hash table in a segment of 4K pages only |
1da177e4 LT |
53 | */ |
54 | ||
3c726f8d | 55 | _GLOBAL(__hash_page_4K) |
1da177e4 LT |
56 | mflr r0 |
57 | std r0,16(r1) | |
58 | stdu r1,-STACKFRAMESIZE(r1) | |
59 | /* Save all params that we need after a function call */ | |
44ce6a5e MN |
60 | std r6,STK_PARAM(R6)(r1) |
61 | std r8,STK_PARAM(R8)(r1) | |
62 | std r9,STK_PARAM(R9)(r1) | |
1da177e4 | 63 | |
1da177e4 LT |
64 | /* Save non-volatile registers. |
65 | * r31 will hold "old PTE" | |
66 | * r30 is "new PTE" | |
5524a27d | 67 | * r29 is vpn |
1da177e4 LT |
68 | * r28 is a hash value |
69 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
70 | */ | |
c75df6f9 MN |
71 | std r27,STK_REG(R27)(r1) |
72 | std r28,STK_REG(R28)(r1) | |
73 | std r29,STK_REG(R29)(r1) | |
74 | std r30,STK_REG(R30)(r1) | |
75 | std r31,STK_REG(R31)(r1) | |
1da177e4 LT |
76 | |
77 | /* Step 1: | |
78 | * | |
79 | * Check permissions, atomically mark the linux PTE busy | |
80 | * and hashed. | |
81 | */ | |
82 | 1: | |
83 | ldarx r31,0,r6 | |
84 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
85 | andc. r0,r4,r31 | |
86 | bne- htab_wrong_access | |
87 | /* Check if PTE is busy */ | |
88 | andi. r0,r31,_PAGE_BUSY | |
d03853d5 OJ |
89 | /* If so, just bail out and refault if needed. Someone else |
90 | * is changing this PTE anyway and might hash it. | |
91 | */ | |
3c726f8d BH |
92 | bne- htab_bail_ok |
93 | ||
1da177e4 LT |
94 | /* Prepare new PTE value (turn access RW into DIRTY, then |
95 | * add BUSY,HASHPTE and ACCESSED) | |
96 | */ | |
97 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
98 | or r30,r30,r31 | |
99 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | |
100 | /* Write the linux PTE atomically (setting busy) */ | |
101 | stdcx. r30,0,r6 | |
102 | bne- 1b | |
103 | isync | |
104 | ||
105 | /* Step 2: | |
106 | * | |
107 | * Insert/Update the HPTE in the hash table. At this point, | |
108 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
109 | */ | |
110 | ||
1189be65 PM |
111 | BEGIN_FTR_SECTION |
112 | cmpdi r9,0 /* check segment size */ | |
113 | bne 3f | |
44ae3ab3 | 114 | END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) |
5524a27d AK |
115 | /* Calc vpn and put it in r29 */ |
116 | sldi r29,r5,SID_SHIFT - VPN_SHIFT | |
117 | rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT) | |
118 | or r29,r28,r29 | |
eda8eebd AK |
119 | /* |
120 | * Calculate hash value for primary slot and store it in r28 | |
121 | * r3 = va, r5 = vsid | |
122 | * r0 = (va >> 12) & ((1ul << (28 - 12)) -1) | |
123 | */ | |
124 | rldicl r0,r3,64-12,48 | |
125 | xor r28,r5,r0 /* hash */ | |
1189be65 PM |
126 | b 4f |
127 | ||
5524a27d AK |
128 | 3: /* Calc vpn and put it in r29 */ |
129 | sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT | |
130 | rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT) | |
131 | or r29,r28,r29 | |
132 | ||
133 | /* | |
134 | * calculate hash value for primary slot and | |
135 | * store it in r28 for 1T segment | |
eda8eebd | 136 | * r3 = va, r5 = vsid |
5524a27d | 137 | */ |
eda8eebd AK |
138 | sldi r28,r5,25 /* vsid << 25 */ |
139 | /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */ | |
140 | rldicl r0,r3,64-12,36 | |
141 | xor r28,r28,r5 /* vsid ^ ( vsid << 25) */ | |
1189be65 | 142 | xor r28,r28,r0 /* hash */ |
1da177e4 LT |
143 | |
144 | /* Convert linux PTE bits into HW equivalents */ | |
1189be65 | 145 | 4: andi. r3,r30,0x1fe /* Get basic set of flags */ |
3c726f8d | 146 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ |
1da177e4 LT |
147 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ |
148 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
3c726f8d | 149 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ |
1da177e4 LT |
150 | andc r0,r30,r0 /* r0 = pte & ~r0 */ |
151 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
c8c06f5a AK |
152 | /* |
153 | * Always add "C" bit for perf. Memory coherence is always enabled | |
154 | */ | |
155 | ori r3,r3,HPTE_R_C | HPTE_R_M | |
1da177e4 LT |
156 | |
157 | /* We eventually do the icache sync here (maybe inline that | |
158 | * code rather than call a C function...) | |
159 | */ | |
1da177e4 LT |
160 | BEGIN_FTR_SECTION |
161 | mr r4,r30 | |
162 | mr r5,r7 | |
b1576fec | 163 | bl hash_page_do_lazy_icache |
8913ca1c | 164 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) |
1da177e4 LT |
165 | |
166 | /* At this point, r3 contains new PP bits, save them in | |
167 | * place of "access" in the param area (sic) | |
168 | */ | |
44ce6a5e | 169 | std r3,STK_PARAM(R4)(r1) |
1da177e4 LT |
170 | |
171 | /* Get htab_hash_mask */ | |
172 | ld r4,htab_hash_mask@got(2) | |
173 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
174 | ||
175 | /* Check if we may already be in the hashtable, in this case, we | |
176 | * go to out-of-line code to try to modify the HPTE | |
177 | */ | |
178 | andi. r0,r31,_PAGE_HASHPTE | |
179 | bne htab_modify_pte | |
180 | ||
181 | htab_insert_pte: | |
182 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | |
183 | * add _PAGE_HASHPTE | |
184 | */ | |
185 | lis r0,_PAGE_HPTEFLAGS@h | |
186 | ori r0,r0,_PAGE_HPTEFLAGS@l | |
187 | andc r30,r30,r0 | |
188 | ori r30,r30,_PAGE_HASHPTE | |
189 | ||
3c726f8d BH |
190 | /* physical address r5 */ |
191 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
192 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
193 | |
194 | /* Calculate primary group hash */ | |
195 | and r0,r28,r27 | |
3c726f8d | 196 | rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ |
1da177e4 LT |
197 | |
198 | /* Call ppc_md.hpte_insert */ | |
44ce6a5e | 199 | ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ |
5524a27d | 200 | mr r4,r29 /* Retrieve vpn */ |
3c726f8d BH |
201 | li r7,0 /* !bolted, !secondary */ |
202 | li r8,MMU_PAGE_4K /* page size */ | |
b1022fbd AK |
203 | li r9,MMU_PAGE_4K /* actual page size */ |
204 | ld r10,STK_PARAM(R9)(r1) /* segment size */ | |
b86206e4 AB |
205 | .globl htab_call_hpte_insert1 |
206 | htab_call_hpte_insert1: | |
3c726f8d | 207 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
208 | cmpdi 0,r3,0 |
209 | bge htab_pte_insert_ok /* Insertion successful */ | |
210 | cmpdi 0,r3,-2 /* Critical failure */ | |
211 | beq- htab_pte_insert_failure | |
212 | ||
213 | /* Now try secondary slot */ | |
214 | ||
3c726f8d BH |
215 | /* physical address r5 */ |
216 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
217 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
218 | |
219 | /* Calculate secondary group hash */ | |
220 | andc r0,r27,r28 | |
221 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
222 | ||
223 | /* Call ppc_md.hpte_insert */ | |
44ce6a5e | 224 | ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ |
5524a27d | 225 | mr r4,r29 /* Retrieve vpn */ |
3c726f8d BH |
226 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ |
227 | li r8,MMU_PAGE_4K /* page size */ | |
b1022fbd AK |
228 | li r9,MMU_PAGE_4K /* actual page size */ |
229 | ld r10,STK_PARAM(R9)(r1) /* segment size */ | |
b86206e4 AB |
230 | .globl htab_call_hpte_insert2 |
231 | htab_call_hpte_insert2: | |
3c726f8d | 232 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
233 | cmpdi 0,r3,0 |
234 | bge+ htab_pte_insert_ok /* Insertion successful */ | |
235 | cmpdi 0,r3,-2 /* Critical failure */ | |
236 | beq- htab_pte_insert_failure | |
237 | ||
238 | /* Both are full, we need to evict something */ | |
239 | mftb r0 | |
240 | /* Pick a random group based on TB */ | |
241 | andi. r0,r0,1 | |
242 | mr r5,r28 | |
243 | bne 2f | |
244 | not r5,r5 | |
245 | 2: and r0,r5,r27 | |
246 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
247 | /* Call ppc_md.hpte_remove */ | |
b86206e4 AB |
248 | .globl htab_call_hpte_remove |
249 | htab_call_hpte_remove: | |
3c726f8d | 250 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
251 | |
252 | /* Try all again */ | |
253 | b htab_insert_pte | |
254 | ||
3c726f8d | 255 | htab_bail_ok: |
d03853d5 | 256 | li r3,0 |
3c726f8d | 257 | b htab_bail |
d03853d5 | 258 | |
1da177e4 LT |
259 | htab_pte_insert_ok: |
260 | /* Insert slot number & secondary bit in PTE */ | |
261 | rldimi r30,r3,12,63-15 | |
262 | ||
263 | /* Write out the PTE with a normal write | |
264 | * (maybe add eieio may be good still ?) | |
265 | */ | |
266 | htab_write_out_pte: | |
44ce6a5e | 267 | ld r6,STK_PARAM(R6)(r1) |
1da177e4 LT |
268 | std r30,0(r6) |
269 | li r3, 0 | |
3c726f8d | 270 | htab_bail: |
c75df6f9 MN |
271 | ld r27,STK_REG(R27)(r1) |
272 | ld r28,STK_REG(R28)(r1) | |
273 | ld r29,STK_REG(R29)(r1) | |
274 | ld r30,STK_REG(R30)(r1) | |
275 | ld r31,STK_REG(R31)(r1) | |
1da177e4 LT |
276 | addi r1,r1,STACKFRAMESIZE |
277 | ld r0,16(r1) | |
278 | mtlr r0 | |
279 | blr | |
280 | ||
281 | htab_modify_pte: | |
282 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
283 | mr r4,r3 | |
284 | rlwinm r3,r31,32-12,29,31 | |
285 | ||
286 | /* Secondary group ? if yes, get a inverted hash value */ | |
287 | mr r5,r28 | |
bf680d51 | 288 | andi. r0,r31,_PAGE_F_SECOND |
1da177e4 LT |
289 | beq 1f |
290 | not r5,r5 | |
291 | 1: | |
292 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
293 | and r0,r5,r27 | |
294 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
295 | add r3,r0,r3 /* add slot idx */ | |
296 | ||
297 | /* Call ppc_md.hpte_updatepp */ | |
5524a27d | 298 | mr r5,r29 /* vpn */ |
db3d8534 AK |
299 | li r6,MMU_PAGE_4K /* base page size */ |
300 | li r7,MMU_PAGE_4K /* actual page size */ | |
301 | ld r8,STK_PARAM(R9)(r1) /* segment size */ | |
aefa5688 | 302 | ld r9,STK_PARAM(R8)(r1) /* get "flags" param */ |
b86206e4 AB |
303 | .globl htab_call_hpte_updatepp |
304 | htab_call_hpte_updatepp: | |
3c726f8d | 305 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
306 | |
307 | /* if we failed because typically the HPTE wasn't really here | |
308 | * we try an insertion. | |
309 | */ | |
310 | cmpdi 0,r3,-1 | |
311 | beq- htab_insert_pte | |
312 | ||
313 | /* Clear the BUSY bit and Write out the PTE */ | |
314 | li r0,_PAGE_BUSY | |
315 | andc r30,r30,r0 | |
316 | b htab_write_out_pte | |
317 | ||
318 | htab_wrong_access: | |
319 | /* Bail out clearing reservation */ | |
320 | stdcx. r31,0,r6 | |
321 | li r3,1 | |
3c726f8d BH |
322 | b htab_bail |
323 | ||
324 | htab_pte_insert_failure: | |
325 | /* Bail out restoring old PTE */ | |
44ce6a5e | 326 | ld r6,STK_PARAM(R6)(r1) |
3c726f8d BH |
327 | std r31,0(r6) |
328 | li r3,-1 | |
329 | b htab_bail | |
330 | ||
89ff7250 | 331 | #endif |