1 // SPDX-License-Identifier: GPL-2.0
3 * Helpers for IOMMU drivers implementing SVA
5 #include <linux/mmu_context.h>
6 #include <linux/mutex.h>
7 #include <linux/sched/mm.h>
8 #include <linux/iommu.h>
10 #include "iommu-sva.h"
12 static DEFINE_MUTEX(iommu_sva_lock);
14 /* Allocate a PASID for the mm within range (inclusive) */
15 static struct iommu_mm_data *iommu_alloc_mm_data(struct mm_struct *mm, struct device *dev)
17 struct iommu_mm_data *iommu_mm;
20 lockdep_assert_held(&iommu_sva_lock);
22 if (!arch_pgtable_dma_compat(mm))
23 return ERR_PTR(-EBUSY);
25 iommu_mm = mm->iommu_mm;
26 /* Is a PASID already associated with this mm? */
28 if (iommu_mm->pasid >= dev->iommu->max_pasids)
29 return ERR_PTR(-EOVERFLOW);
33 iommu_mm = kzalloc(sizeof(struct iommu_mm_data), GFP_KERNEL);
35 return ERR_PTR(-ENOMEM);
37 pasid = iommu_alloc_global_pasid(dev);
38 if (pasid == IOMMU_PASID_INVALID) {
40 return ERR_PTR(-ENOSPC);
42 iommu_mm->pasid = pasid;
43 INIT_LIST_HEAD(&iommu_mm->sva_domains);
45 * Make sure the write to mm->iommu_mm is not reordered in front of
46 * initialization to iommu_mm fields. If it does, readers may see a
47 * valid iommu_mm with uninitialized values.
49 smp_store_release(&mm->iommu_mm, iommu_mm);
54 * iommu_sva_bind_device() - Bind a process address space to a device
56 * @mm: the mm to bind, caller must hold a reference to mm_users
58 * Create a bond between device and address space, allowing the device to
59 * access the mm using the PASID returned by iommu_sva_get_pasid(). If a
60 * bond already exists between @device and @mm, an additional internal
61 * reference is taken. Caller must call iommu_sva_unbind_device()
62 * to release each reference.
64 * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
65 * initialize the required SVA features.
67 * On error, returns an ERR_PTR value.
69 struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
71 struct iommu_mm_data *iommu_mm;
72 struct iommu_domain *domain;
73 struct iommu_sva *handle;
76 mutex_lock(&iommu_sva_lock);
78 /* Allocate mm->pasid if necessary. */
79 iommu_mm = iommu_alloc_mm_data(mm, dev);
80 if (IS_ERR(iommu_mm)) {
81 ret = PTR_ERR(iommu_mm);
85 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
91 /* Search for an existing domain. */
92 list_for_each_entry(domain, &mm->iommu_mm->sva_domains, next) {
93 ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
100 /* Allocate a new domain and set it on device pasid. */
101 domain = iommu_sva_domain_alloc(dev, mm);
104 goto out_free_handle;
107 ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid);
109 goto out_free_domain;
111 list_add(&domain->next, &mm->iommu_mm->sva_domains);
114 mutex_unlock(&iommu_sva_lock);
116 handle->domain = domain;
120 iommu_domain_free(domain);
124 mutex_unlock(&iommu_sva_lock);
127 EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
130 * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
131 * @handle: the handle returned by iommu_sva_bind_device()
133 * Put reference to a bond between device and address space. The device should
134 * not be issuing any more transaction for this PASID. All outstanding page
135 * requests for this PASID must have been flushed to the IOMMU.
137 void iommu_sva_unbind_device(struct iommu_sva *handle)
139 struct iommu_domain *domain = handle->domain;
140 struct iommu_mm_data *iommu_mm = domain->mm->iommu_mm;
141 struct device *dev = handle->dev;
143 mutex_lock(&iommu_sva_lock);
144 iommu_detach_device_pasid(domain, dev, iommu_mm->pasid);
145 if (--domain->users == 0) {
146 list_del(&domain->next);
147 iommu_domain_free(domain);
149 mutex_unlock(&iommu_sva_lock);
152 EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
154 u32 iommu_sva_get_pasid(struct iommu_sva *handle)
156 struct iommu_domain *domain = handle->domain;
158 return mm_get_enqcmd_pasid(domain->mm);
160 EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
163 * I/O page fault handler for SVA
165 enum iommu_page_response_code
166 iommu_sva_handle_iopf(struct iommu_fault *fault, void *data)
169 struct vm_area_struct *vma;
170 struct mm_struct *mm = data;
171 unsigned int access_flags = 0;
172 unsigned int fault_flags = FAULT_FLAG_REMOTE;
173 struct iommu_fault_page_request *prm = &fault->prm;
174 enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID;
176 if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID))
179 if (!mmget_not_zero(mm))
184 vma = vma_lookup(mm, prm->addr);
189 if (prm->perm & IOMMU_FAULT_PERM_READ)
190 access_flags |= VM_READ;
192 if (prm->perm & IOMMU_FAULT_PERM_WRITE) {
193 access_flags |= VM_WRITE;
194 fault_flags |= FAULT_FLAG_WRITE;
197 if (prm->perm & IOMMU_FAULT_PERM_EXEC) {
198 access_flags |= VM_EXEC;
199 fault_flags |= FAULT_FLAG_INSTRUCTION;
202 if (!(prm->perm & IOMMU_FAULT_PERM_PRIV))
203 fault_flags |= FAULT_FLAG_USER;
205 if (access_flags & ~vma->vm_flags)
209 ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL);
210 status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID :
211 IOMMU_PAGE_RESP_SUCCESS;
214 mmap_read_unlock(mm);
220 void mm_pasid_drop(struct mm_struct *mm)
222 struct iommu_mm_data *iommu_mm = mm->iommu_mm;
227 iommu_free_global_pasid(iommu_mm->pasid);