powerpc/mm: Fix hash computation function
[~shefty/rdma-dev.git] / arch / powerpc / mm / hash_low_64.S
1 /*
2  * ppc64 MMU hashtable management routines
3  *
4  * (c) Copyright IBM Corp. 2003, 2005
5  *
6  * Maintained by: Benjamin Herrenschmidt
7  *                <benh@kernel.crashing.org>
8  *
9  * This file is covered by the GNU Public Licence v2 as
10  * described in the kernel's COPYING file.
11  */
12
13 #include <asm/reg.h>
14 #include <asm/pgtable.h>
15 #include <asm/mmu.h>
16 #include <asm/page.h>
17 #include <asm/types.h>
18 #include <asm/ppc_asm.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/cputable.h>
21
22         .text
23
24 /*
25  * Stackframe:
26  *              
27  *         +-> Back chain                       (SP + 256)
28  *         |   General register save area       (SP + 112)
29  *         |   Parameter save area              (SP + 48)
30  *         |   TOC save area                    (SP + 40)
31  *         |   link editor doubleword           (SP + 32)
32  *         |   compiler doubleword              (SP + 24)
33  *         |   LR save area                     (SP + 16)
34  *         |   CR save area                     (SP + 8)
35  * SP ---> +-- Back chain                       (SP + 0)
36  */
37
38 #ifndef CONFIG_PPC_64K_PAGES
39
40 /*****************************************************************************
41  *                                                                           *
42  *           4K SW & 4K HW pages implementation                              *
43  *                                                                           *
44  *****************************************************************************/
45
46
47 /*
48  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
49  *               pte_t *ptep, unsigned long trap, int local, int ssize)
50  *
51  * Adds a 4K page to the hash table in a segment of 4K pages only
52  */
53
54 _GLOBAL(__hash_page_4K)
55         mflr    r0
56         std     r0,16(r1)
57         stdu    r1,-STACKFRAMESIZE(r1)
58         /* Save all params that we need after a function call */
59         std     r6,STK_PARAM(R6)(r1)
60         std     r8,STK_PARAM(R8)(r1)
61         std     r9,STK_PARAM(R9)(r1)
62         
63         /* Save non-volatile registers.
64          * r31 will hold "old PTE"
65          * r30 is "new PTE"
66          * r29 is vpn
67          * r28 is a hash value
68          * r27 is hashtab mask (maybe dynamic patched instead ?)
69          */
70         std     r27,STK_REG(R27)(r1)
71         std     r28,STK_REG(R28)(r1)
72         std     r29,STK_REG(R29)(r1)
73         std     r30,STK_REG(R30)(r1)
74         std     r31,STK_REG(R31)(r1)
75         
76         /* Step 1:
77          *
78          * Check permissions, atomically mark the linux PTE busy
79          * and hashed.
80          */ 
81 1:
82         ldarx   r31,0,r6
83         /* Check access rights (access & ~(pte_val(*ptep))) */
84         andc.   r0,r4,r31
85         bne-    htab_wrong_access
86         /* Check if PTE is busy */
87         andi.   r0,r31,_PAGE_BUSY
88         /* If so, just bail out and refault if needed. Someone else
89          * is changing this PTE anyway and might hash it.
90          */
91         bne-    htab_bail_ok
92
93         /* Prepare new PTE value (turn access RW into DIRTY, then
94          * add BUSY,HASHPTE and ACCESSED)
95          */
96         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
97         or      r30,r30,r31
98         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
99         /* Write the linux PTE atomically (setting busy) */
100         stdcx.  r30,0,r6
101         bne-    1b
102         isync
103
104         /* Step 2:
105          *
106          * Insert/Update the HPTE in the hash table. At this point,
107          * r4 (access) is re-useable, we use it for the new HPTE flags
108          */
109
110 BEGIN_FTR_SECTION
111         cmpdi   r9,0                    /* check segment size */
112         bne     3f
113 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
114         /* Calc vpn and put it in r29 */
115         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
116         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
117         or      r29,r28,r29
118         /*
119          * Calculate hash value for primary slot and store it in r28
120          * r3 = va, r5 = vsid
121          * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
122          */
123         rldicl  r0,r3,64-12,48
124         xor     r28,r5,r0               /* hash */
125         b       4f
126
127 3:      /* Calc vpn and put it in r29 */
128         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
129         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
130         or      r29,r28,r29
131
132         /*
133          * calculate hash value for primary slot and
134          * store it in r28 for 1T segment
135          * r3 = va, r5 = vsid
136          */
137         sldi    r28,r5,25               /* vsid << 25 */
138         /* r0 =  (va >> 12) & ((1ul << (40 - 12)) -1) */
139         rldicl  r0,r3,64-12,36
140         xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
141         xor     r28,r28,r0              /* hash */
142
143         /* Convert linux PTE bits into HW equivalents */
144 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
145         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
146         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
147         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
148         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
149         andc    r0,r30,r0               /* r0 = pte & ~r0 */
150         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
151         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
152
153         /* We eventually do the icache sync here (maybe inline that
154          * code rather than call a C function...) 
155          */
156 BEGIN_FTR_SECTION
157         mr      r4,r30
158         mr      r5,r7
159         bl      .hash_page_do_lazy_icache
160 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
161
162         /* At this point, r3 contains new PP bits, save them in
163          * place of "access" in the param area (sic)
164          */
165         std     r3,STK_PARAM(R4)(r1)
166
167         /* Get htab_hash_mask */
168         ld      r4,htab_hash_mask@got(2)
169         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
170
171         /* Check if we may already be in the hashtable, in this case, we
172          * go to out-of-line code to try to modify the HPTE
173          */
174         andi.   r0,r31,_PAGE_HASHPTE
175         bne     htab_modify_pte
176
177 htab_insert_pte:
178         /* Clear hpte bits in new pte (we also clear BUSY btw) and
179          * add _PAGE_HASHPTE
180          */
181         lis     r0,_PAGE_HPTEFLAGS@h
182         ori     r0,r0,_PAGE_HPTEFLAGS@l
183         andc    r30,r30,r0
184         ori     r30,r30,_PAGE_HASHPTE
185
186         /* physical address r5 */
187         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
188         sldi    r5,r5,PAGE_SHIFT
189
190         /* Calculate primary group hash */
191         and     r0,r28,r27
192         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
193
194         /* Call ppc_md.hpte_insert */
195         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
196         mr      r4,r29                  /* Retrieve vpn */
197         li      r7,0                    /* !bolted, !secondary */
198         li      r8,MMU_PAGE_4K          /* page size */
199         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
200 _GLOBAL(htab_call_hpte_insert1)
201         bl      .                       /* Patched by htab_finish_init() */
202         cmpdi   0,r3,0
203         bge     htab_pte_insert_ok      /* Insertion successful */
204         cmpdi   0,r3,-2                 /* Critical failure */
205         beq-    htab_pte_insert_failure
206
207         /* Now try secondary slot */
208         
209         /* physical address r5 */
210         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
211         sldi    r5,r5,PAGE_SHIFT
212
213         /* Calculate secondary group hash */
214         andc    r0,r27,r28
215         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
216         
217         /* Call ppc_md.hpte_insert */
218         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
219         mr      r4,r29                  /* Retrieve vpn */
220         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
221         li      r8,MMU_PAGE_4K          /* page size */
222         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
223 _GLOBAL(htab_call_hpte_insert2)
224         bl      .                       /* Patched by htab_finish_init() */
225         cmpdi   0,r3,0
226         bge+    htab_pte_insert_ok      /* Insertion successful */
227         cmpdi   0,r3,-2                 /* Critical failure */
228         beq-    htab_pte_insert_failure
229
230         /* Both are full, we need to evict something */
231         mftb    r0
232         /* Pick a random group based on TB */
233         andi.   r0,r0,1
234         mr      r5,r28
235         bne     2f
236         not     r5,r5
237 2:      and     r0,r5,r27
238         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */   
239         /* Call ppc_md.hpte_remove */
240 _GLOBAL(htab_call_hpte_remove)
241         bl      .                       /* Patched by htab_finish_init() */
242
243         /* Try all again */
244         b       htab_insert_pte 
245
246 htab_bail_ok:
247         li      r3,0
248         b       htab_bail
249
250 htab_pte_insert_ok:
251         /* Insert slot number & secondary bit in PTE */
252         rldimi  r30,r3,12,63-15
253                 
254         /* Write out the PTE with a normal write
255          * (maybe add eieio may be good still ?)
256          */
257 htab_write_out_pte:
258         ld      r6,STK_PARAM(R6)(r1)
259         std     r30,0(r6)
260         li      r3, 0
261 htab_bail:
262         ld      r27,STK_REG(R27)(r1)
263         ld      r28,STK_REG(R28)(r1)
264         ld      r29,STK_REG(R29)(r1)
265         ld      r30,STK_REG(R30)(r1)
266         ld      r31,STK_REG(R31)(r1)
267         addi    r1,r1,STACKFRAMESIZE
268         ld      r0,16(r1)
269         mtlr    r0
270         blr
271
272 htab_modify_pte:
273         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
274         mr      r4,r3
275         rlwinm  r3,r31,32-12,29,31
276
277         /* Secondary group ? if yes, get a inverted hash value */
278         mr      r5,r28
279         andi.   r0,r31,_PAGE_SECONDARY
280         beq     1f
281         not     r5,r5
282 1:
283         /* Calculate proper slot value for ppc_md.hpte_updatepp */
284         and     r0,r5,r27
285         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
286         add     r3,r0,r3        /* add slot idx */
287
288         /* Call ppc_md.hpte_updatepp */
289         mr      r5,r29                  /* vpn */
290         li      r6,MMU_PAGE_4K          /* page size */
291         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
292         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
293 _GLOBAL(htab_call_hpte_updatepp)
294         bl      .                       /* Patched by htab_finish_init() */
295
296         /* if we failed because typically the HPTE wasn't really here
297          * we try an insertion. 
298          */
299         cmpdi   0,r3,-1
300         beq-    htab_insert_pte
301
302         /* Clear the BUSY bit and Write out the PTE */
303         li      r0,_PAGE_BUSY
304         andc    r30,r30,r0
305         b       htab_write_out_pte
306
307 htab_wrong_access:
308         /* Bail out clearing reservation */
309         stdcx.  r31,0,r6
310         li      r3,1
311         b       htab_bail
312
313 htab_pte_insert_failure:
314         /* Bail out restoring old PTE */
315         ld      r6,STK_PARAM(R6)(r1)
316         std     r31,0(r6)
317         li      r3,-1
318         b       htab_bail
319
320
321 #else /* CONFIG_PPC_64K_PAGES */
322
323
324 /*****************************************************************************
325  *                                                                           *
326  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
327  *                                                                           *
328  *****************************************************************************/
329
330 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
331  *               pte_t *ptep, unsigned long trap, int local, int ssize,
332  *               int subpg_prot)
333  */
334
335 /*
336  * For now, we do NOT implement Admixed pages
337  */
338 _GLOBAL(__hash_page_4K)
339         mflr    r0
340         std     r0,16(r1)
341         stdu    r1,-STACKFRAMESIZE(r1)
342         /* Save all params that we need after a function call */
343         std     r6,STK_PARAM(R6)(r1)
344         std     r8,STK_PARAM(R8)(r1)
345         std     r9,STK_PARAM(R9)(r1)
346
347         /* Save non-volatile registers.
348          * r31 will hold "old PTE"
349          * r30 is "new PTE"
350          * r29 is vpn
351          * r28 is a hash value
352          * r27 is hashtab mask (maybe dynamic patched instead ?)
353          * r26 is the hidx mask
354          * r25 is the index in combo page
355          */
356         std     r25,STK_REG(R25)(r1)
357         std     r26,STK_REG(R26)(r1)
358         std     r27,STK_REG(R27)(r1)
359         std     r28,STK_REG(R28)(r1)
360         std     r29,STK_REG(R29)(r1)
361         std     r30,STK_REG(R30)(r1)
362         std     r31,STK_REG(R31)(r1)
363
364         /* Step 1:
365          *
366          * Check permissions, atomically mark the linux PTE busy
367          * and hashed.
368          */
369 1:
370         ldarx   r31,0,r6
371         /* Check access rights (access & ~(pte_val(*ptep))) */
372         andc.   r0,r4,r31
373         bne-    htab_wrong_access
374         /* Check if PTE is busy */
375         andi.   r0,r31,_PAGE_BUSY
376         /* If so, just bail out and refault if needed. Someone else
377          * is changing this PTE anyway and might hash it.
378          */
379         bne-    htab_bail_ok
380         /* Prepare new PTE value (turn access RW into DIRTY, then
381          * add BUSY and ACCESSED)
382          */
383         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
384         or      r30,r30,r31
385         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
386         oris    r30,r30,_PAGE_COMBO@h
387         /* Write the linux PTE atomically (setting busy) */
388         stdcx.  r30,0,r6
389         bne-    1b
390         isync
391
392         /* Step 2:
393          *
394          * Insert/Update the HPTE in the hash table. At this point,
395          * r4 (access) is re-useable, we use it for the new HPTE flags
396          */
397
398         /* Load the hidx index */
399         rldicl  r25,r3,64-12,60
400
401 BEGIN_FTR_SECTION
402         cmpdi   r9,0                    /* check segment size */
403         bne     3f
404 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
405         /* Calc vpn and put it in r29 */
406         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
407         /*
408          * clrldi r3,r3,64 - SID_SHIFT -->  ea & 0xfffffff
409          * srdi  r28,r3,VPN_SHIFT
410          */
411         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
412         or      r29,r28,r29
413         /*
414          * Calculate hash value for primary slot and store it in r28
415          * r3 = va, r5 = vsid
416          * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
417          */
418         rldicl  r0,r3,64-12,48
419         xor     r28,r5,r0               /* hash */
420         b       4f
421
422 3:      /* Calc vpn and put it in r29 */
423         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
424         /*
425          * clrldi r3,r3,64 - SID_SHIFT_1T -->  ea & 0xffffffffff
426          * srdi r28,r3,VPN_SHIFT
427          */
428         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
429         or      r29,r28,r29
430
431         /*
432          * Calculate hash value for primary slot and
433          * store it in r28  for 1T segment
434          * r3 = va, r5 = vsid
435          */
436         sldi    r28,r5,25               /* vsid << 25 */
437         /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */
438         rldicl  r0,r3,64-12,36
439         xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
440         xor     r28,r28,r0              /* hash */
441
442         /* Convert linux PTE bits into HW equivalents */
443 4:
444 #ifdef CONFIG_PPC_SUBPAGE_PROT
445         andc    r10,r30,r10
446         andi.   r3,r10,0x1fe            /* Get basic set of flags */
447         rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
448 #else
449         andi.   r3,r30,0x1fe            /* Get basic set of flags */
450         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
451 #endif
452         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
453         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
454         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
455         andc    r0,r3,r0                /* r0 = pte & ~r0 */
456         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
457         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
458
459         /* We eventually do the icache sync here (maybe inline that
460          * code rather than call a C function...)
461          */
462 BEGIN_FTR_SECTION
463         mr      r4,r30
464         mr      r5,r7
465         bl      .hash_page_do_lazy_icache
466 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
467
468         /* At this point, r3 contains new PP bits, save them in
469          * place of "access" in the param area (sic)
470          */
471         std     r3,STK_PARAM(R4)(r1)
472
473         /* Get htab_hash_mask */
474         ld      r4,htab_hash_mask@got(2)
475         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
476
477         /* Check if we may already be in the hashtable, in this case, we
478          * go to out-of-line code to try to modify the HPTE. We look for
479          * the bit at (1 >> (index + 32))
480          */
481         rldicl. r0,r31,64-12,48
482         li      r26,0                   /* Default hidx */
483         beq     htab_insert_pte
484
485         /*
486          * Check if the pte was already inserted into the hash table
487          * as a 64k HW page, and invalidate the 64k HPTE if so.
488          */
489         andis.  r0,r31,_PAGE_COMBO@h
490         beq     htab_inval_old_hpte
491
492         ld      r6,STK_PARAM(R6)(r1)
493         ori     r26,r6,0x8000           /* Load the hidx mask */
494         ld      r26,0(r26)
495         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
496         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
497         bne     htab_modify_pte
498
499 htab_insert_pte:
500         /* real page number in r5, PTE RPN value + index */
501         andis.  r0,r31,_PAGE_4K_PFN@h
502         srdi    r5,r31,PTE_RPN_SHIFT
503         bne-    htab_special_pfn
504         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
505         add     r5,r5,r25
506 htab_special_pfn:
507         sldi    r5,r5,HW_PAGE_SHIFT
508
509         /* Calculate primary group hash */
510         and     r0,r28,r27
511         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
512
513         /* Call ppc_md.hpte_insert */
514         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
515         mr      r4,r29                  /* Retrieve vpn */
516         li      r7,0                    /* !bolted, !secondary */
517         li      r8,MMU_PAGE_4K          /* page size */
518         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
519 _GLOBAL(htab_call_hpte_insert1)
520         bl      .                       /* patched by htab_finish_init() */
521         cmpdi   0,r3,0
522         bge     htab_pte_insert_ok      /* Insertion successful */
523         cmpdi   0,r3,-2                 /* Critical failure */
524         beq-    htab_pte_insert_failure
525
526         /* Now try secondary slot */
527
528         /* real page number in r5, PTE RPN value + index */
529         andis.  r0,r31,_PAGE_4K_PFN@h
530         srdi    r5,r31,PTE_RPN_SHIFT
531         bne-    3f
532         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
533         add     r5,r5,r25
534 3:      sldi    r5,r5,HW_PAGE_SHIFT
535
536         /* Calculate secondary group hash */
537         andc    r0,r27,r28
538         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
539
540         /* Call ppc_md.hpte_insert */
541         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
542         mr      r4,r29                  /* Retrieve vpn */
543         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
544         li      r8,MMU_PAGE_4K          /* page size */
545         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
546 _GLOBAL(htab_call_hpte_insert2)
547         bl      .                       /* patched by htab_finish_init() */
548         cmpdi   0,r3,0
549         bge+    htab_pte_insert_ok      /* Insertion successful */
550         cmpdi   0,r3,-2                 /* Critical failure */
551         beq-    htab_pte_insert_failure
552
553         /* Both are full, we need to evict something */
554         mftb    r0
555         /* Pick a random group based on TB */
556         andi.   r0,r0,1
557         mr      r5,r28
558         bne     2f
559         not     r5,r5
560 2:      and     r0,r5,r27
561         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
562         /* Call ppc_md.hpte_remove */
563 _GLOBAL(htab_call_hpte_remove)
564         bl      .                       /* patched by htab_finish_init() */
565
566         /* Try all again */
567         b       htab_insert_pte
568
569         /*
570          * Call out to C code to invalidate an 64k HW HPTE that is
571          * useless now that the segment has been switched to 4k pages.
572          */
573 htab_inval_old_hpte:
574         mr      r3,r29                  /* vpn */
575         mr      r4,r31                  /* PTE.pte */
576         li      r5,0                    /* PTE.hidx */
577         li      r6,MMU_PAGE_64K         /* psize */
578         ld      r7,STK_PARAM(R9)(r1)    /* ssize */
579         ld      r8,STK_PARAM(R8)(r1)    /* local */
580         bl      .flush_hash_page
581         /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
582         lis     r0,_PAGE_HPTE_SUB@h
583         ori     r0,r0,_PAGE_HPTE_SUB@l
584         andc    r30,r30,r0
585         b       htab_insert_pte
586         
587 htab_bail_ok:
588         li      r3,0
589         b       htab_bail
590
591 htab_pte_insert_ok:
592         /* Insert slot number & secondary bit in PTE second half,
593          * clear _PAGE_BUSY and set approriate HPTE slot bit
594          */
595         ld      r6,STK_PARAM(R6)(r1)
596         li      r0,_PAGE_BUSY
597         andc    r30,r30,r0
598         /* HPTE SUB bit */
599         li      r0,1
600         subfic  r5,r25,27               /* Must match bit position in */
601         sld     r0,r0,r5                /* pgtable.h */
602         or      r30,r30,r0
603         /* hindx */
604         sldi    r5,r25,2
605         sld     r3,r3,r5
606         li      r4,0xf
607         sld     r4,r4,r5
608         andc    r26,r26,r4
609         or      r26,r26,r3
610         ori     r5,r6,0x8000
611         std     r26,0(r5)
612         lwsync
613         std     r30,0(r6)
614         li      r3, 0
615 htab_bail:
616         ld      r25,STK_REG(R25)(r1)
617         ld      r26,STK_REG(R26)(r1)
618         ld      r27,STK_REG(R27)(r1)
619         ld      r28,STK_REG(R28)(r1)
620         ld      r29,STK_REG(R29)(r1)
621         ld      r30,STK_REG(R30)(r1)
622         ld      r31,STK_REG(R31)(r1)
623         addi    r1,r1,STACKFRAMESIZE
624         ld      r0,16(r1)
625         mtlr    r0
626         blr
627
628 htab_modify_pte:
629         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
630         mr      r4,r3
631         sldi    r5,r25,2
632         srd     r3,r26,r5
633
634         /* Secondary group ? if yes, get a inverted hash value */
635         mr      r5,r28
636         andi.   r0,r3,0x8 /* page secondary ? */
637         beq     1f
638         not     r5,r5
639 1:      andi.   r3,r3,0x7 /* extract idx alone */
640
641         /* Calculate proper slot value for ppc_md.hpte_updatepp */
642         and     r0,r5,r27
643         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
644         add     r3,r0,r3        /* add slot idx */
645
646         /* Call ppc_md.hpte_updatepp */
647         mr      r5,r29                  /* vpn */
648         li      r6,MMU_PAGE_4K          /* page size */
649         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
650         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
651 _GLOBAL(htab_call_hpte_updatepp)
652         bl      .                       /* patched by htab_finish_init() */
653
654         /* if we failed because typically the HPTE wasn't really here
655          * we try an insertion.
656          */
657         cmpdi   0,r3,-1
658         beq-    htab_insert_pte
659
660         /* Clear the BUSY bit and Write out the PTE */
661         li      r0,_PAGE_BUSY
662         andc    r30,r30,r0
663         ld      r6,STK_PARAM(R6)(r1)
664         std     r30,0(r6)
665         li      r3,0
666         b       htab_bail
667
668 htab_wrong_access:
669         /* Bail out clearing reservation */
670         stdcx.  r31,0,r6
671         li      r3,1
672         b       htab_bail
673
674 htab_pte_insert_failure:
675         /* Bail out restoring old PTE */
676         ld      r6,STK_PARAM(R6)(r1)
677         std     r31,0(r6)
678         li      r3,-1
679         b       htab_bail
680
681 #endif /* CONFIG_PPC_64K_PAGES */
682
683 #ifdef CONFIG_PPC_HAS_HASH_64K
684
685 /*****************************************************************************
686  *                                                                           *
687  *           64K SW & 64K HW in a 64K segment pages implementation           *
688  *                                                                           *
689  *****************************************************************************/
690
691 _GLOBAL(__hash_page_64K)
692         mflr    r0
693         std     r0,16(r1)
694         stdu    r1,-STACKFRAMESIZE(r1)
695         /* Save all params that we need after a function call */
696         std     r6,STK_PARAM(R6)(r1)
697         std     r8,STK_PARAM(R8)(r1)
698         std     r9,STK_PARAM(R9)(r1)
699
700         /* Save non-volatile registers.
701          * r31 will hold "old PTE"
702          * r30 is "new PTE"
703          * r29 is vpn
704          * r28 is a hash value
705          * r27 is hashtab mask (maybe dynamic patched instead ?)
706          */
707         std     r27,STK_REG(R27)(r1)
708         std     r28,STK_REG(R28)(r1)
709         std     r29,STK_REG(R29)(r1)
710         std     r30,STK_REG(R30)(r1)
711         std     r31,STK_REG(R31)(r1)
712
713         /* Step 1:
714          *
715          * Check permissions, atomically mark the linux PTE busy
716          * and hashed.
717          */
718 1:
719         ldarx   r31,0,r6
720         /* Check access rights (access & ~(pte_val(*ptep))) */
721         andc.   r0,r4,r31
722         bne-    ht64_wrong_access
723         /* Check if PTE is busy */
724         andi.   r0,r31,_PAGE_BUSY
725         /* If so, just bail out and refault if needed. Someone else
726          * is changing this PTE anyway and might hash it.
727          */
728         bne-    ht64_bail_ok
729 BEGIN_FTR_SECTION
730         /* Check if PTE has the cache-inhibit bit set */
731         andi.   r0,r31,_PAGE_NO_CACHE
732         /* If so, bail out and refault as a 4k page */
733         bne-    ht64_bail_ok
734 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
735         /* Prepare new PTE value (turn access RW into DIRTY, then
736          * add BUSY and ACCESSED)
737          */
738         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
739         or      r30,r30,r31
740         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
741         /* Write the linux PTE atomically (setting busy) */
742         stdcx.  r30,0,r6
743         bne-    1b
744         isync
745
746         /* Step 2:
747          *
748          * Insert/Update the HPTE in the hash table. At this point,
749          * r4 (access) is re-useable, we use it for the new HPTE flags
750          */
751
752 BEGIN_FTR_SECTION
753         cmpdi   r9,0                    /* check segment size */
754         bne     3f
755 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
756         /* Calc vpn and put it in r29 */
757         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
758         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
759         or      r29,r28,r29
760
761         /* Calculate hash value for primary slot and store it in r28
762          * r3 = va, r5 = vsid
763          * r0 = (va >> 16) & ((1ul << (28 - 16)) -1)
764          */
765         rldicl  r0,r3,64-16,52
766         xor     r28,r5,r0               /* hash */
767         b       4f
768
769 3:      /* Calc vpn and put it in r29 */
770         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
771         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
772         or      r29,r28,r29
773         /*
774          * calculate hash value for primary slot and
775          * store it in r28 for 1T segment
776          * r3 = va, r5 = vsid
777          */
778         sldi    r28,r5,25               /* vsid << 25 */
779         /* r0 = (va >> 16) & ((1ul << (40 - 16)) -1) */
780         rldicl  r0,r3,64-16,40
781         xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
782         xor     r28,r28,r0              /* hash */
783
784         /* Convert linux PTE bits into HW equivalents */
785 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
786         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
787         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
788         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
789         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
790         andc    r0,r30,r0               /* r0 = pte & ~r0 */
791         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
792         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
793
794         /* We eventually do the icache sync here (maybe inline that
795          * code rather than call a C function...)
796          */
797 BEGIN_FTR_SECTION
798         mr      r4,r30
799         mr      r5,r7
800         bl      .hash_page_do_lazy_icache
801 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
802
803         /* At this point, r3 contains new PP bits, save them in
804          * place of "access" in the param area (sic)
805          */
806         std     r3,STK_PARAM(R4)(r1)
807
808         /* Get htab_hash_mask */
809         ld      r4,htab_hash_mask@got(2)
810         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
811
812         /* Check if we may already be in the hashtable, in this case, we
813          * go to out-of-line code to try to modify the HPTE
814          */
815         rldicl. r0,r31,64-12,48
816         bne     ht64_modify_pte
817
818 ht64_insert_pte:
819         /* Clear hpte bits in new pte (we also clear BUSY btw) and
820          * add _PAGE_HPTE_SUB0
821          */
822         lis     r0,_PAGE_HPTEFLAGS@h
823         ori     r0,r0,_PAGE_HPTEFLAGS@l
824         andc    r30,r30,r0
825 #ifdef CONFIG_PPC_64K_PAGES
826         oris    r30,r30,_PAGE_HPTE_SUB0@h
827 #else
828         ori     r30,r30,_PAGE_HASHPTE
829 #endif
830         /* Phyical address in r5 */
831         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
832         sldi    r5,r5,PAGE_SHIFT
833
834         /* Calculate primary group hash */
835         and     r0,r28,r27
836         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
837
838         /* Call ppc_md.hpte_insert */
839         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
840         mr      r4,r29                  /* Retrieve vpn */
841         li      r7,0                    /* !bolted, !secondary */
842         li      r8,MMU_PAGE_64K
843         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
844 _GLOBAL(ht64_call_hpte_insert1)
845         bl      .                       /* patched by htab_finish_init() */
846         cmpdi   0,r3,0
847         bge     ht64_pte_insert_ok      /* Insertion successful */
848         cmpdi   0,r3,-2                 /* Critical failure */
849         beq-    ht64_pte_insert_failure
850
851         /* Now try secondary slot */
852
853         /* Phyical address in r5 */
854         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
855         sldi    r5,r5,PAGE_SHIFT
856
857         /* Calculate secondary group hash */
858         andc    r0,r27,r28
859         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
860
861         /* Call ppc_md.hpte_insert */
862         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
863         mr      r4,r29                  /* Retrieve vpn */
864         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
865         li      r8,MMU_PAGE_64K
866         ld      r9,STK_PARAM(R9)(r1)    /* segment size */
867 _GLOBAL(ht64_call_hpte_insert2)
868         bl      .                       /* patched by htab_finish_init() */
869         cmpdi   0,r3,0
870         bge+    ht64_pte_insert_ok      /* Insertion successful */
871         cmpdi   0,r3,-2                 /* Critical failure */
872         beq-    ht64_pte_insert_failure
873
874         /* Both are full, we need to evict something */
875         mftb    r0
876         /* Pick a random group based on TB */
877         andi.   r0,r0,1
878         mr      r5,r28
879         bne     2f
880         not     r5,r5
881 2:      and     r0,r5,r27
882         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
883         /* Call ppc_md.hpte_remove */
884 _GLOBAL(ht64_call_hpte_remove)
885         bl      .                       /* patched by htab_finish_init() */
886
887         /* Try all again */
888         b       ht64_insert_pte
889
890 ht64_bail_ok:
891         li      r3,0
892         b       ht64_bail
893
894 ht64_pte_insert_ok:
895         /* Insert slot number & secondary bit in PTE */
896         rldimi  r30,r3,12,63-15
897
898         /* Write out the PTE with a normal write
899          * (maybe add eieio may be good still ?)
900          */
901 ht64_write_out_pte:
902         ld      r6,STK_PARAM(R6)(r1)
903         std     r30,0(r6)
904         li      r3, 0
905 ht64_bail:
906         ld      r27,STK_REG(R27)(r1)
907         ld      r28,STK_REG(R28)(r1)
908         ld      r29,STK_REG(R29)(r1)
909         ld      r30,STK_REG(R30)(r1)
910         ld      r31,STK_REG(R31)(r1)
911         addi    r1,r1,STACKFRAMESIZE
912         ld      r0,16(r1)
913         mtlr    r0
914         blr
915
916 ht64_modify_pte:
917         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
918         mr      r4,r3
919         rlwinm  r3,r31,32-12,29,31
920
921         /* Secondary group ? if yes, get a inverted hash value */
922         mr      r5,r28
923         andi.   r0,r31,_PAGE_F_SECOND
924         beq     1f
925         not     r5,r5
926 1:
927         /* Calculate proper slot value for ppc_md.hpte_updatepp */
928         and     r0,r5,r27
929         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
930         add     r3,r0,r3        /* add slot idx */
931
932         /* Call ppc_md.hpte_updatepp */
933         mr      r5,r29                  /* vpn */
934         li      r6,MMU_PAGE_64K
935         ld      r7,STK_PARAM(R9)(r1)    /* segment size */
936         ld      r8,STK_PARAM(R8)(r1)    /* get "local" param */
937 _GLOBAL(ht64_call_hpte_updatepp)
938         bl      .                       /* patched by htab_finish_init() */
939
940         /* if we failed because typically the HPTE wasn't really here
941          * we try an insertion.
942          */
943         cmpdi   0,r3,-1
944         beq-    ht64_insert_pte
945
946         /* Clear the BUSY bit and Write out the PTE */
947         li      r0,_PAGE_BUSY
948         andc    r30,r30,r0
949         b       ht64_write_out_pte
950
951 ht64_wrong_access:
952         /* Bail out clearing reservation */
953         stdcx.  r31,0,r6
954         li      r3,1
955         b       ht64_bail
956
957 ht64_pte_insert_failure:
958         /* Bail out restoring old PTE */
959         ld      r6,STK_PARAM(R6)(r1)
960         std     r31,0(r6)
961         li      r3,-1
962         b       ht64_bail
963
964
965 #endif /* CONFIG_PPC_HAS_HASH_64K */
966
967
968 /*****************************************************************************
969  *                                                                           *
970  *           Huge pages implementation is in hugetlbpage.c                   *
971  *                                                                           *
972  *****************************************************************************/