56585086413a4f9876c70eb6b6ea9f3cf1c1ef69
[~shefty/rdma-dev.git] / arch / powerpc / mm / hash_low_64.S
1 /*
2  * ppc64 MMU hashtable management routines
3  *
4  * (c) Copyright IBM Corp. 2003, 2005
5  *
6  * Maintained by: Benjamin Herrenschmidt
7  *                <benh@kernel.crashing.org>
8  *
9  * This file is covered by the GNU Public Licence v2 as
10  * described in the kernel's COPYING file.
11  */
12
13 #include <asm/reg.h>
14 #include <asm/pgtable.h>
15 #include <asm/mmu.h>
16 #include <asm/page.h>
17 #include <asm/types.h>
18 #include <asm/ppc_asm.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/cputable.h>
21
22         .text
23
24 /*
25  * Stackframe:
26  *              
27  *         +-> Back chain                       (SP + 256)
28  *         |   General register save area       (SP + 112)
29  *         |   Parameter save area              (SP + 48)
30  *         |   TOC save area                    (SP + 40)
31  *         |   link editor doubleword           (SP + 32)
32  *         |   compiler doubleword              (SP + 24)
33  *         |   LR save area                     (SP + 16)
34  *         |   CR save area                     (SP + 8)
35  * SP ---> +-- Back chain                       (SP + 0)
36  */
37
38 #ifndef CONFIG_PPC_64K_PAGES
39
40 /*****************************************************************************
41  *                                                                           *
42  *           4K SW & 4K HW pages implementation                              *
43  *                                                                           *
44  *****************************************************************************/
45
46
47 /*
48  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
49  *               pte_t *ptep, unsigned long trap, int local, int ssize)
50  *
51  * Adds a 4K page to the hash table in a segment of 4K pages only
52  */
53
54 _GLOBAL(__hash_page_4K)
55         mflr    r0
56         std     r0,16(r1)
57         stdu    r1,-STACKFRAMESIZE(r1)
58         /* Save all params that we need after a function call */
59         std     r6,STK_PARAM(R6)(r1)
60         std     r8,STK_PARAM(R8)(r1)
61         std     r9,STK_PARAM(R9)(r1)
62         
63         /* Save non-volatile registers.
64          * r31 will hold "old PTE"
65          * r30 is "new PTE"
66          * r29 is vpn
67          * r28 is a hash value
68          * r27 is hashtab mask (maybe dynamic patched instead ?)
69          */
70         std     r27,STK_REG(R27)(r1)
71         std     r28,STK_REG(R28)(r1)
72         std     r29,STK_REG(R29)(r1)
73         std     r30,STK_REG(R30)(r1)
74         std     r31,STK_REG(R31)(r1)
75         
76         /* Step 1:
77          *
78          * Check permissions, atomically mark the linux PTE busy
79          * and hashed.
80          */ 
81 1:
82         ldarx   r31,0,r6
83         /* Check access rights (access & ~(pte_val(*ptep))) */
84         andc.   r0,r4,r31
85         bne-    htab_wrong_access
86         /* Check if PTE is busy */
87         andi.   r0,r31,_PAGE_BUSY
88         /* If so, just bail out and refault if needed. Someone else
89          * is changing this PTE anyway and might hash it.
90          */
91         bne-    htab_bail_ok
92
93         /* Prepare new PTE value (turn access RW into DIRTY, then
94          * add BUSY,HASHPTE and ACCESSED)
95          */
96         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
97         or      r30,r30,r31
98         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
99         /* Write the linux PTE atomically (setting busy) */
100         stdcx.  r30,0,r6
101         bne-    1b
102         isync
103
104         /* Step 2:
105          *
106          * Insert/Update the HPTE in the hash table. At this point,
107          * r4 (access) is re-useable, we use it for the new HPTE flags
108          */
109
110 BEGIN_FTR_SECTION
111         cmpdi   r9,0                    /* check segment size */
112         bne     3f
113 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
114         /* Calc vpn and put it in r29 */
115         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
116         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
117         or      r29,r28,r29
118
119         /* Calculate hash value for primary slot and store it in r28 */
120         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
121         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
122         xor     r28,r5,r0
123         b       4f
124
125 3:      /* Calc vpn and put it in r29 */
126         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
127         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
128         or      r29,r28,r29
129
130         /*
131          * calculate hash value for primary slot and
132          * store it in r28 for 1T segment
133          */
134         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
135         clrldi  r5,r5,40                /* vsid & 0xffffff */
136         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
137         xor     r28,r28,r5
138         xor     r28,r28,r0              /* hash */
139
140         /* Convert linux PTE bits into HW equivalents */
141 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
142         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
143         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
144         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
145         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
146         andc    r0,r30,r0               /* r0 = pte & ~r0 */
147         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
148         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
149
150         /* We eventually do the icache sync here (maybe inline that
151          * code rather than call a C function...) 
152          */
153 BEGIN_FTR_SECTION
154         mr      r4,r30
155         mr      r5,r7
156         bl      .hash_page_do_lazy_icache
157 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
158
159         /* At this point, r3 contains new PP bits, save them in
160          * place of "access" in the param area (sic)
161          */
162         std     r3,STK_PARAM(R4)(r1)
163
164         /* Get htab_hash_mask */
165         ld      r4,htab_hash_mask@got(2)
166         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
167
168         /* Check if we may already be in the hashtable, in this case, we
169          * go to out-of-line code to try to modify the HPTE
170          */
171         andi.   r0,r31,_PAGE_HASHPTE
172         bne     htab_modify_pte
173
174 htab_insert_pte:
175         /* Clear hpte bits in new pte (we also clear BUSY btw) and
176          * add _PAGE_HASHPTE
177          */
178         lis     r0,_PAGE_HPTEFLAGS@h
179         ori     r0,r0,_PAGE_HPTEFLAGS@l
180         andc    r30,r30,r0
181         ori     r30,r30,_PAGE_HASHPTE
182
183         /* physical address r5 */
184         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
185         sldi    r5,r5,PAGE_SHIFT
186
187         /* Calculate primary group hash */
188         and     r0,r28,r27
189         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
190
191         /* Call ppc_md.hpte_insert */
192         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
193         mr      r4,r29                  /* Retrieve vpn */
194         li      r7,0                    /* !bolted, !secondary */
195         li      r8,MMU_PAGE_4K          /* page size */
196         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
197 _GLOBAL(htab_call_hpte_insert1)
198         bl      .                       /* Patched by htab_finish_init() */
199         cmpdi   0,r3,0
200         bge     htab_pte_insert_ok      /* Insertion successful */
201         cmpdi   0,r3,-2                 /* Critical failure */
202         beq-    htab_pte_insert_failure
203
204         /* Now try secondary slot */
205         
206         /* physical address r5 */
207         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
208         sldi    r5,r5,PAGE_SHIFT
209
210         /* Calculate secondary group hash */
211         andc    r0,r27,r28
212         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
213         
214         /* Call ppc_md.hpte_insert */
215         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
216         mr      r4,r29                  /* Retrieve vpn */
217         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
218         li      r8,MMU_PAGE_4K          /* page size */
219         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
220 _GLOBAL(htab_call_hpte_insert2)
221         bl      .                       /* Patched by htab_finish_init() */
222         cmpdi   0,r3,0
223         bge+    htab_pte_insert_ok      /* Insertion successful */
224         cmpdi   0,r3,-2                 /* Critical failure */
225         beq-    htab_pte_insert_failure
226
227         /* Both are full, we need to evict something */
228         mftb    r0
229         /* Pick a random group based on TB */
230         andi.   r0,r0,1
231         mr      r5,r28
232         bne     2f
233         not     r5,r5
234 2:      and     r0,r5,r27
235         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */   
236         /* Call ppc_md.hpte_remove */
237 _GLOBAL(htab_call_hpte_remove)
238         bl      .                       /* Patched by htab_finish_init() */
239
240         /* Try all again */
241         b       htab_insert_pte 
242
243 htab_bail_ok:
244         li      r3,0
245         b       htab_bail
246
247 htab_pte_insert_ok:
248         /* Insert slot number & secondary bit in PTE */
249         rldimi  r30,r3,12,63-15
250                 
251         /* Write out the PTE with a normal write
252          * (maybe add eieio may be good still ?)
253          */
254 htab_write_out_pte:
255         ld      r6,STK_PARAM(R6)(r1)
256         std     r30,0(r6)
257         li      r3, 0
258 htab_bail:
259         ld      r27,STK_REG(R27)(r1)
260         ld      r28,STK_REG(R28)(r1)
261         ld      r29,STK_REG(R29)(r1)
262         ld      r30,STK_REG(R30)(r1)
263         ld      r31,STK_REG(R31)(r1)
264         addi    r1,r1,STACKFRAMESIZE
265         ld      r0,16(r1)
266         mtlr    r0
267         blr
268
269 htab_modify_pte:
270         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
271         mr      r4,r3
272         rlwinm  r3,r31,32-12,29,31
273
274         /* Secondary group ? if yes, get a inverted hash value */
275         mr      r5,r28
276         andi.   r0,r31,_PAGE_SECONDARY
277         beq     1f
278         not     r5,r5
279 1:
280         /* Calculate proper slot value for ppc_md.hpte_updatepp */
281         and     r0,r5,r27
282         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
283         add     r3,r0,r3        /* add slot idx */
284
285         /* Call ppc_md.hpte_updatepp */
286         mr      r5,r29                  /* vpn */
287         li      r6,MMU_PAGE_4K          /* page size */
288         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
289         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
290 _GLOBAL(htab_call_hpte_updatepp)
291         bl      .                       /* Patched by htab_finish_init() */
292
293         /* if we failed because typically the HPTE wasn't really here
294          * we try an insertion. 
295          */
296         cmpdi   0,r3,-1
297         beq-    htab_insert_pte
298
299         /* Clear the BUSY bit and Write out the PTE */
300         li      r0,_PAGE_BUSY
301         andc    r30,r30,r0
302         b       htab_write_out_pte
303
304 htab_wrong_access:
305         /* Bail out clearing reservation */
306         stdcx.  r31,0,r6
307         li      r3,1
308         b       htab_bail
309
310 htab_pte_insert_failure:
311         /* Bail out restoring old PTE */
312         ld      r6,STK_PARAM(R6)(r1)
313         std     r31,0(r6)
314         li      r3,-1
315         b       htab_bail
316
317
318 #else /* CONFIG_PPC_64K_PAGES */
319
320
321 /*****************************************************************************
322  *                                                                           *
323  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
324  *                                                                           *
325  *****************************************************************************/
326
327 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
328  *               pte_t *ptep, unsigned long trap, int local, int ssize,
329  *               int subpg_prot)
330  */
331
332 /*
333  * For now, we do NOT implement Admixed pages
334  */
335 _GLOBAL(__hash_page_4K)
336         mflr    r0
337         std     r0,16(r1)
338         stdu    r1,-STACKFRAMESIZE(r1)
339         /* Save all params that we need after a function call */
340         std     r6,STK_PARAM(R6)(r1)
341         std     r8,STK_PARAM(R8)(r1)
342         std     r9,STK_PARAM(R9)(r1)
343
344         /* Save non-volatile registers.
345          * r31 will hold "old PTE"
346          * r30 is "new PTE"
347          * r29 is vpn
348          * r28 is a hash value
349          * r27 is hashtab mask (maybe dynamic patched instead ?)
350          * r26 is the hidx mask
351          * r25 is the index in combo page
352          */
353         std     r25,STK_REG(R25)(r1)
354         std     r26,STK_REG(R26)(r1)
355         std     r27,STK_REG(R27)(r1)
356         std     r28,STK_REG(R28)(r1)
357         std     r29,STK_REG(R29)(r1)
358         std     r30,STK_REG(R30)(r1)
359         std     r31,STK_REG(R31)(r1)
360
361         /* Step 1:
362          *
363          * Check permissions, atomically mark the linux PTE busy
364          * and hashed.
365          */
366 1:
367         ldarx   r31,0,r6
368         /* Check access rights (access & ~(pte_val(*ptep))) */
369         andc.   r0,r4,r31
370         bne-    htab_wrong_access
371         /* Check if PTE is busy */
372         andi.   r0,r31,_PAGE_BUSY
373         /* If so, just bail out and refault if needed. Someone else
374          * is changing this PTE anyway and might hash it.
375          */
376         bne-    htab_bail_ok
377         /* Prepare new PTE value (turn access RW into DIRTY, then
378          * add BUSY and ACCESSED)
379          */
380         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
381         or      r30,r30,r31
382         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
383         oris    r30,r30,_PAGE_COMBO@h
384         /* Write the linux PTE atomically (setting busy) */
385         stdcx.  r30,0,r6
386         bne-    1b
387         isync
388
389         /* Step 2:
390          *
391          * Insert/Update the HPTE in the hash table. At this point,
392          * r4 (access) is re-useable, we use it for the new HPTE flags
393          */
394
395         /* Load the hidx index */
396         rldicl  r25,r3,64-12,60
397
398 BEGIN_FTR_SECTION
399         cmpdi   r9,0                    /* check segment size */
400         bne     3f
401 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
402         /* Calc vpn and put it in r29 */
403         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
404         /*
405          * clrldi r3,r3,64 - SID_SHIFT -->  ea & 0xfffffff
406          * srdi  r28,r3,VPN_SHIFT
407          */
408         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
409         or      r29,r28,r29
410
411         /* Calculate hash value for primary slot and store it in r28 */
412         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
413         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
414         xor     r28,r5,r0
415         b       4f
416
417 3:      /* Calc vpn and put it in r29 */
418         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
419         /*
420          * clrldi r3,r3,64 - SID_SHIFT_1T -->  ea & 0xffffffffff
421          * srdi r28,r3,VPN_SHIFT
422          */
423         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
424         or      r29,r28,r29
425
426         /*
427          * Calculate hash value for primary slot and
428          * store it in r28  for 1T segment
429          */
430         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
431         clrldi  r5,r5,40                /* vsid & 0xffffff */
432         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
433         xor     r28,r28,r5
434         xor     r28,r28,r0              /* hash */
435
436         /* Convert linux PTE bits into HW equivalents */
437 4:
438 #ifdef CONFIG_PPC_SUBPAGE_PROT
439         andc    r10,r30,r10
440         andi.   r3,r10,0x1fe            /* Get basic set of flags */
441         rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
442 #else
443         andi.   r3,r30,0x1fe            /* Get basic set of flags */
444         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
445 #endif
446         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
447         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
448         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
449         andc    r0,r3,r0                /* r0 = pte & ~r0 */
450         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
451         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
452
453         /* We eventually do the icache sync here (maybe inline that
454          * code rather than call a C function...)
455          */
456 BEGIN_FTR_SECTION
457         mr      r4,r30
458         mr      r5,r7
459         bl      .hash_page_do_lazy_icache
460 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
461
462         /* At this point, r3 contains new PP bits, save them in
463          * place of "access" in the param area (sic)
464          */
465         std     r3,STK_PARAM(R4)(r1)
466
467         /* Get htab_hash_mask */
468         ld      r4,htab_hash_mask@got(2)
469         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
470
471         /* Check if we may already be in the hashtable, in this case, we
472          * go to out-of-line code to try to modify the HPTE. We look for
473          * the bit at (1 >> (index + 32))
474          */
475         rldicl. r0,r31,64-12,48
476         li      r26,0                   /* Default hidx */
477         beq     htab_insert_pte
478
479         /*
480          * Check if the pte was already inserted into the hash table
481          * as a 64k HW page, and invalidate the 64k HPTE if so.
482          */
483         andis.  r0,r31,_PAGE_COMBO@h
484         beq     htab_inval_old_hpte
485
486         ld      r6,STK_PARAM(R6)(r1)
487         ori     r26,r6,0x8000           /* Load the hidx mask */
488         ld      r26,0(r26)
489         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
490         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
491         bne     htab_modify_pte
492
493 htab_insert_pte:
494         /* real page number in r5, PTE RPN value + index */
495         andis.  r0,r31,_PAGE_4K_PFN@h
496         srdi    r5,r31,PTE_RPN_SHIFT
497         bne-    htab_special_pfn
498         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
499         add     r5,r5,r25
500 htab_special_pfn:
501         sldi    r5,r5,HW_PAGE_SHIFT
502
503         /* Calculate primary group hash */
504         and     r0,r28,r27
505         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
506
507         /* Call ppc_md.hpte_insert */
508         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
509         mr      r4,r29                  /* Retrieve vpn */
510         li      r7,0                    /* !bolted, !secondary */
511         li      r8,MMU_PAGE_4K          /* page size */
512         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
513 _GLOBAL(htab_call_hpte_insert1)
514         bl      .                       /* patched by htab_finish_init() */
515         cmpdi   0,r3,0
516         bge     htab_pte_insert_ok      /* Insertion successful */
517         cmpdi   0,r3,-2                 /* Critical failure */
518         beq-    htab_pte_insert_failure
519
520         /* Now try secondary slot */
521
522         /* real page number in r5, PTE RPN value + index */
523         andis.  r0,r31,_PAGE_4K_PFN@h
524         srdi    r5,r31,PTE_RPN_SHIFT
525         bne-    3f
526         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
527         add     r5,r5,r25
528 3:      sldi    r5,r5,HW_PAGE_SHIFT
529
530         /* Calculate secondary group hash */
531         andc    r0,r27,r28
532         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
533
534         /* Call ppc_md.hpte_insert */
535         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
536         mr      r4,r29                  /* Retrieve vpn */
537         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
538         li      r8,MMU_PAGE_4K          /* page size */
539         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
540 _GLOBAL(htab_call_hpte_insert2)
541         bl      .                       /* patched by htab_finish_init() */
542         cmpdi   0,r3,0
543         bge+    htab_pte_insert_ok      /* Insertion successful */
544         cmpdi   0,r3,-2                 /* Critical failure */
545         beq-    htab_pte_insert_failure
546
547         /* Both are full, we need to evict something */
548         mftb    r0
549         /* Pick a random group based on TB */
550         andi.   r0,r0,1
551         mr      r5,r28
552         bne     2f
553         not     r5,r5
554 2:      and     r0,r5,r27
555         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
556         /* Call ppc_md.hpte_remove */
557 _GLOBAL(htab_call_hpte_remove)
558         bl      .                       /* patched by htab_finish_init() */
559
560         /* Try all again */
561         b       htab_insert_pte
562
563         /*
564          * Call out to C code to invalidate an 64k HW HPTE that is
565          * useless now that the segment has been switched to 4k pages.
566          */
567 htab_inval_old_hpte:
568         mr      r3,r29                  /* vpn */
569         mr      r4,r31                  /* PTE.pte */
570         li      r5,0                    /* PTE.hidx */
571         li      r6,MMU_PAGE_64K         /* psize */
572         ld      r7,STK_PARAM(R9)(r1)    /* ssize */
573         ld      r8,STK_PARAM(R8)(r1)    /* local */
574         bl      .flush_hash_page
575         /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
576         lis     r0,_PAGE_HPTE_SUB@h
577         ori     r0,r0,_PAGE_HPTE_SUB@l
578         andc    r30,r30,r0
579         b       htab_insert_pte
580         
581 htab_bail_ok:
582         li      r3,0
583         b       htab_bail
584
585 htab_pte_insert_ok:
586         /* Insert slot number & secondary bit in PTE second half,
587          * clear _PAGE_BUSY and set approriate HPTE slot bit
588          */
589         ld      r6,STK_PARAM(R6)(r1)
590         li      r0,_PAGE_BUSY
591         andc    r30,r30,r0
592         /* HPTE SUB bit */
593         li      r0,1
594         subfic  r5,r25,27               /* Must match bit position in */
595         sld     r0,r0,r5                /* pgtable.h */
596         or      r30,r30,r0
597         /* hindx */
598         sldi    r5,r25,2
599         sld     r3,r3,r5
600         li      r4,0xf
601         sld     r4,r4,r5
602         andc    r26,r26,r4
603         or      r26,r26,r3
604         ori     r5,r6,0x8000
605         std     r26,0(r5)
606         lwsync
607         std     r30,0(r6)
608         li      r3, 0
609 htab_bail:
610         ld      r25,STK_REG(R25)(r1)
611         ld      r26,STK_REG(R26)(r1)
612         ld      r27,STK_REG(R27)(r1)
613         ld      r28,STK_REG(R28)(r1)
614         ld      r29,STK_REG(R29)(r1)
615         ld      r30,STK_REG(R30)(r1)
616         ld      r31,STK_REG(R31)(r1)
617         addi    r1,r1,STACKFRAMESIZE
618         ld      r0,16(r1)
619         mtlr    r0
620         blr
621
622 htab_modify_pte:
623         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
624         mr      r4,r3
625         sldi    r5,r25,2
626         srd     r3,r26,r5
627
628         /* Secondary group ? if yes, get a inverted hash value */
629         mr      r5,r28
630         andi.   r0,r3,0x8 /* page secondary ? */
631         beq     1f
632         not     r5,r5
633 1:      andi.   r3,r3,0x7 /* extract idx alone */
634
635         /* Calculate proper slot value for ppc_md.hpte_updatepp */
636         and     r0,r5,r27
637         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
638         add     r3,r0,r3        /* add slot idx */
639
640         /* Call ppc_md.hpte_updatepp */
641         mr      r5,r29                  /* vpn */
642         li      r6,MMU_PAGE_4K          /* page size */
643         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
644         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
645 _GLOBAL(htab_call_hpte_updatepp)
646         bl      .                       /* patched by htab_finish_init() */
647
648         /* if we failed because typically the HPTE wasn't really here
649          * we try an insertion.
650          */
651         cmpdi   0,r3,-1
652         beq-    htab_insert_pte
653
654         /* Clear the BUSY bit and Write out the PTE */
655         li      r0,_PAGE_BUSY
656         andc    r30,r30,r0
657         ld      r6,STK_PARAM(R6)(r1)
658         std     r30,0(r6)
659         li      r3,0
660         b       htab_bail
661
662 htab_wrong_access:
663         /* Bail out clearing reservation */
664         stdcx.  r31,0,r6
665         li      r3,1
666         b       htab_bail
667
668 htab_pte_insert_failure:
669         /* Bail out restoring old PTE */
670         ld      r6,STK_PARAM(R6)(r1)
671         std     r31,0(r6)
672         li      r3,-1
673         b       htab_bail
674
675 #endif /* CONFIG_PPC_64K_PAGES */
676
677 #ifdef CONFIG_PPC_HAS_HASH_64K
678
679 /*****************************************************************************
680  *                                                                           *
681  *           64K SW & 64K HW in a 64K segment pages implementation           *
682  *                                                                           *
683  *****************************************************************************/
684
685 _GLOBAL(__hash_page_64K)
686         mflr    r0
687         std     r0,16(r1)
688         stdu    r1,-STACKFRAMESIZE(r1)
689         /* Save all params that we need after a function call */
690         std     r6,STK_PARAM(R6)(r1)
691         std     r8,STK_PARAM(R8)(r1)
692         std     r9,STK_PARAM(R9)(r1)
693
694         /* Save non-volatile registers.
695          * r31 will hold "old PTE"
696          * r30 is "new PTE"
697          * r29 is vpn
698          * r28 is a hash value
699          * r27 is hashtab mask (maybe dynamic patched instead ?)
700          */
701         std     r27,STK_REG(R27)(r1)
702         std     r28,STK_REG(R28)(r1)
703         std     r29,STK_REG(R29)(r1)
704         std     r30,STK_REG(R30)(r1)
705         std     r31,STK_REG(R31)(r1)
706
707         /* Step 1:
708          *
709          * Check permissions, atomically mark the linux PTE busy
710          * and hashed.
711          */
712 1:
713         ldarx   r31,0,r6
714         /* Check access rights (access & ~(pte_val(*ptep))) */
715         andc.   r0,r4,r31
716         bne-    ht64_wrong_access
717         /* Check if PTE is busy */
718         andi.   r0,r31,_PAGE_BUSY
719         /* If so, just bail out and refault if needed. Someone else
720          * is changing this PTE anyway and might hash it.
721          */
722         bne-    ht64_bail_ok
723 BEGIN_FTR_SECTION
724         /* Check if PTE has the cache-inhibit bit set */
725         andi.   r0,r31,_PAGE_NO_CACHE
726         /* If so, bail out and refault as a 4k page */
727         bne-    ht64_bail_ok
728 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
729         /* Prepare new PTE value (turn access RW into DIRTY, then
730          * add BUSY and ACCESSED)
731          */
732         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
733         or      r30,r30,r31
734         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
735         /* Write the linux PTE atomically (setting busy) */
736         stdcx.  r30,0,r6
737         bne-    1b
738         isync
739
740         /* Step 2:
741          *
742          * Insert/Update the HPTE in the hash table. At this point,
743          * r4 (access) is re-useable, we use it for the new HPTE flags
744          */
745
746 BEGIN_FTR_SECTION
747         cmpdi   r9,0                    /* check segment size */
748         bne     3f
749 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
750         /* Calc vpn and put it in r29 */
751         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
752         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
753         or      r29,r28,r29
754
755         /* Calculate hash value for primary slot and store it in r28 */
756         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
757         rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
758         xor     r28,r5,r0
759         b       4f
760
761 3:      /* Calc vpn and put it in r29 */
762         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
763         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
764         or      r29,r28,r29
765
766         /*
767          * calculate hash value for primary slot and
768          * store it in r28 for 1T segment
769          */
770         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
771         clrldi  r5,r5,40                /* vsid & 0xffffff */
772         rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
773         xor     r28,r28,r5
774         xor     r28,r28,r0              /* hash */
775
776         /* Convert linux PTE bits into HW equivalents */
777 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
778         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
779         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
780         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
781         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
782         andc    r0,r30,r0               /* r0 = pte & ~r0 */
783         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
784         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
785
786         /* We eventually do the icache sync here (maybe inline that
787          * code rather than call a C function...)
788          */
789 BEGIN_FTR_SECTION
790         mr      r4,r30
791         mr      r5,r7
792         bl      .hash_page_do_lazy_icache
793 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
794
795         /* At this point, r3 contains new PP bits, save them in
796          * place of "access" in the param area (sic)
797          */
798         std     r3,STK_PARAM(R4)(r1)
799
800         /* Get htab_hash_mask */
801         ld      r4,htab_hash_mask@got(2)
802         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
803
804         /* Check if we may already be in the hashtable, in this case, we
805          * go to out-of-line code to try to modify the HPTE
806          */
807         rldicl. r0,r31,64-12,48
808         bne     ht64_modify_pte
809
810 ht64_insert_pte:
811         /* Clear hpte bits in new pte (we also clear BUSY btw) and
812          * add _PAGE_HPTE_SUB0
813          */
814         lis     r0,_PAGE_HPTEFLAGS@h
815         ori     r0,r0,_PAGE_HPTEFLAGS@l
816         andc    r30,r30,r0
817 #ifdef CONFIG_PPC_64K_PAGES
818         oris    r30,r30,_PAGE_HPTE_SUB0@h
819 #else
820         ori     r30,r30,_PAGE_HASHPTE
821 #endif
822         /* Phyical address in r5 */
823         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
824         sldi    r5,r5,PAGE_SHIFT
825
826         /* Calculate primary group hash */
827         and     r0,r28,r27
828         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
829
830         /* Call ppc_md.hpte_insert */
831         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
832         mr      r4,r29                  /* Retrieve vpn */
833         li      r7,0                    /* !bolted, !secondary */
834         li      r8,MMU_PAGE_64K
835         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
836 _GLOBAL(ht64_call_hpte_insert1)
837         bl      .                       /* patched by htab_finish_init() */
838         cmpdi   0,r3,0
839         bge     ht64_pte_insert_ok      /* Insertion successful */
840         cmpdi   0,r3,-2                 /* Critical failure */
841         beq-    ht64_pte_insert_failure
842
843         /* Now try secondary slot */
844
845         /* Phyical address in r5 */
846         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
847         sldi    r5,r5,PAGE_SHIFT
848
849         /* Calculate secondary group hash */
850         andc    r0,r27,r28
851         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
852
853         /* Call ppc_md.hpte_insert */
854         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
855         mr      r4,r29                  /* Retrieve vpn */
856         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
857         li      r8,MMU_PAGE_64K
858         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
859 _GLOBAL(ht64_call_hpte_insert2)
860         bl      .                       /* patched by htab_finish_init() */
861         cmpdi   0,r3,0
862         bge+    ht64_pte_insert_ok      /* Insertion successful */
863         cmpdi   0,r3,-2                 /* Critical failure */
864         beq-    ht64_pte_insert_failure
865
866         /* Both are full, we need to evict something */
867         mftb    r0
868         /* Pick a random group based on TB */
869         andi.   r0,r0,1
870         mr      r5,r28
871         bne     2f
872         not     r5,r5
873 2:      and     r0,r5,r27
874         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
875         /* Call ppc_md.hpte_remove */
876 _GLOBAL(ht64_call_hpte_remove)
877         bl      .                       /* patched by htab_finish_init() */
878
879         /* Try all again */
880         b       ht64_insert_pte
881
882 ht64_bail_ok:
883         li      r3,0
884         b       ht64_bail
885
886 ht64_pte_insert_ok:
887         /* Insert slot number & secondary bit in PTE */
888         rldimi  r30,r3,12,63-15
889
890         /* Write out the PTE with a normal write
891          * (maybe add eieio may be good still ?)
892          */
893 ht64_write_out_pte:
894         ld      r6,STK_PARAM(R6)(r1)
895         std     r30,0(r6)
896         li      r3, 0
897 ht64_bail:
898         ld      r27,STK_REG(R27)(r1)
899         ld      r28,STK_REG(R28)(r1)
900         ld      r29,STK_REG(R29)(r1)
901         ld      r30,STK_REG(R30)(r1)
902         ld      r31,STK_REG(R31)(r1)
903         addi    r1,r1,STACKFRAMESIZE
904         ld      r0,16(r1)
905         mtlr    r0
906         blr
907
908 ht64_modify_pte:
909         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
910         mr      r4,r3
911         rlwinm  r3,r31,32-12,29,31
912
913         /* Secondary group ? if yes, get a inverted hash value */
914         mr      r5,r28
915         andi.   r0,r31,_PAGE_F_SECOND
916         beq     1f
917         not     r5,r5
918 1:
919         /* Calculate proper slot value for ppc_md.hpte_updatepp */
920         and     r0,r5,r27
921         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
922         add     r3,r0,r3        /* add slot idx */
923
924         /* Call ppc_md.hpte_updatepp */
925         mr      r5,r29                  /* vpn */
926         li      r6,MMU_PAGE_64K
927         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
928         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
929 _GLOBAL(ht64_call_hpte_updatepp)
930         bl      .                       /* patched by htab_finish_init() */
931
932         /* if we failed because typically the HPTE wasn't really here
933          * we try an insertion.
934          */
935         cmpdi   0,r3,-1
936         beq-    ht64_insert_pte
937
938         /* Clear the BUSY bit and Write out the PTE */
939         li      r0,_PAGE_BUSY
940         andc    r30,r30,r0
941         b       ht64_write_out_pte
942
943 ht64_wrong_access:
944         /* Bail out clearing reservation */
945         stdcx.  r31,0,r6
946         li      r3,1
947         b       ht64_bail
948
949 ht64_pte_insert_failure:
950         /* Bail out restoring old PTE */
951         ld      r6,STK_PARAM(R6)(r1)
952         std     r31,0(r6)
953         li      r3,-1
954         b       ht64_bail
955
956
957 #endif /* CONFIG_PPC_HAS_HASH_64K */
958
959
960 /*****************************************************************************
961  *                                                                           *
962  *           Huge pages implementation is in hugetlbpage.c                   *
963  *                                                                           *
964  *****************************************************************************/