projects / linux-block.git / blame
| Commit | Line | Data |
|---|---|---|
| 95594cb4 CH |
1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | ||
| 3 | #include <linux/mm.h> | |
| 4 | #include <linux/smp.h> | |
| 31738ede | 5 | #include <linux/sched.h> |
| 95594cb4 | 6 | #include <asm/sbi.h> |
| 3f1e7829 | 7 | #include <asm/mmu_context.h> |
| e9210500 SM |
8 | |
| 9 | static inline void local_flush_tlb_all_asid(unsigned long asid) | |
| 10 | { | |
| 11 | __asm__ __volatile__ ("sfence.vma x0, %0" | |
| 12 | : | |
| 13 | : "r" (asid) | |
| 14 | : "memory"); | |
| 15 | } | |
| 16 | ||
| 17 | static inline void local_flush_tlb_page_asid(unsigned long addr, | |
| 18 | unsigned long asid) | |
| 19 | { | |
| 20 | __asm__ __volatile__ ("sfence.vma %0, %1" | |
| 21 | : | |
| 22 | : "r" (addr), "r" (asid) | |
| 23 | : "memory"); | |
| 24 | } | |
| 95594cb4 CH |
25 | |
| 26 | void flush_tlb_all(void) | |
| 27 | { | |
| 28 | sbi_remote_sfence_vma(NULL, 0, -1); | |
| 29 | } | |
| 30 | ||
| 70c7605c | 31 | static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, |
| c3b2d670 | 32 | unsigned long size, unsigned long stride) |
| 95594cb4 | 33 | { |
| 70c7605c | 34 | struct cpumask *cmask = mm_cpumask(mm); |
| 31738ede | 35 | unsigned int cpuid; |
| 3f1e7829 | 36 | bool broadcast; |
| 95594cb4 | 37 | |
| 6384423f AP |
38 | if (cpumask_empty(cmask)) |
| 39 | return; | |
| 40 | ||
| 31738ede | 41 | cpuid = get_cpu(); |
| 3f1e7829 GR |
42 | /* check if the tlbflush needs to be sent to other CPUs */ |
| 43 | broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; | |
| 44 | if (static_branch_unlikely(&use_asid_allocator)) { | |
| 9a801afd | 45 | unsigned long asid = atomic_long_read(&mm->context.id) & asid_mask; |
| 31738ede | 46 | |
| 3f1e7829 | 47 | if (broadcast) { |
| 26fb751c | 48 | sbi_remote_sfence_vma_asid(cmask, start, size, asid); |
| 3f1e7829 GR |
49 | } else if (size <= stride) { |
| 50 | local_flush_tlb_page_asid(start, asid); | |
| 51 | } else { | |
| 52 | local_flush_tlb_all_asid(asid); | |
| 53 | } | |
| 54 | } else { | |
| 55 | if (broadcast) { | |
| 26fb751c | 56 | sbi_remote_sfence_vma(cmask, start, size); |
| 3f1e7829 | 57 | } else if (size <= stride) { |
| 6efb16b1 | 58 | local_flush_tlb_page(start); |
| 3f1e7829 | 59 | } else { |
| 6efb16b1 | 60 | local_flush_tlb_all(); |
| 3f1e7829 | 61 | } |
| 31738ede AP |
62 | } |
| 63 | ||
| 64 | put_cpu(); | |
| 95594cb4 CH |
65 | } |
| 66 | ||
| 67 | void flush_tlb_mm(struct mm_struct *mm) | |
| 68 | { | |
| 70c7605c | 69 | __sbi_tlb_flush_range(mm, 0, -1, PAGE_SIZE); |
| 95594cb4 CH |
70 | } |
| 71 | ||
| 72 | void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) | |
| 73 | { | |
| 70c7605c | 74 | __sbi_tlb_flush_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); |
| 95594cb4 CH |
75 | } |
| 76 | ||
| 77 | void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, | |
| 78 | unsigned long end) | |
| 79 | { | |
| 70c7605c | 80 | __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PAGE_SIZE); |
| 95594cb4 | 81 | } |
| e88b3331 NS |
82 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 83 | void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, | |
| 84 | unsigned long end) | |
| 85 | { | |
| 70c7605c | 86 | __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PMD_SIZE); |
| e88b3331 NS |
87 | } |
| 88 | #endif |