2 * Copyright (C) 2016, Semihalf
3 * Author: Tomasz Nowicki <tn@semihalf.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * This file implements early detection/parsing of I/O mapping
15 * reported to OS through firmware via I/O Remapping Table (IORT)
16 * IORT document number: ARM DEN 0049A
19 #define pr_fmt(fmt) "ACPI: IORT: " fmt
21 #include <linux/acpi_iort.h>
22 #include <linux/iommu.h>
23 #include <linux/kernel.h>
24 #include <linux/list.h>
25 #include <linux/pci.h>
26 #include <linux/platform_device.h>
27 #include <linux/slab.h>
29 #define IORT_TYPE_MASK(type) (1 << (type))
30 #define IORT_MSI_TYPE (1 << ACPI_IORT_NODE_ITS_GROUP)
31 #define IORT_IOMMU_TYPE ((1 << ACPI_IORT_NODE_SMMU) | \
32 (1 << ACPI_IORT_NODE_SMMU_V3))
34 struct iort_its_msi_chip {
35 struct list_head list;
36 struct fwnode_handle *fw_node;
37 phys_addr_t base_addr;
42 struct list_head list;
43 struct acpi_iort_node *iort_node;
44 struct fwnode_handle *fwnode;
46 static LIST_HEAD(iort_fwnode_list);
47 static DEFINE_SPINLOCK(iort_fwnode_lock);
50 * iort_set_fwnode() - Create iort_fwnode and use it to register
51 * iommu data in the iort_fwnode_list
53 * @node: IORT table node associated with the IOMMU
54 * @fwnode: fwnode associated with the IORT node
56 * Returns: 0 on success
59 static inline int iort_set_fwnode(struct acpi_iort_node *iort_node,
60 struct fwnode_handle *fwnode)
62 struct iort_fwnode *np;
64 np = kzalloc(sizeof(struct iort_fwnode), GFP_ATOMIC);
69 INIT_LIST_HEAD(&np->list);
70 np->iort_node = iort_node;
73 spin_lock(&iort_fwnode_lock);
74 list_add_tail(&np->list, &iort_fwnode_list);
75 spin_unlock(&iort_fwnode_lock);
81 * iort_get_fwnode() - Retrieve fwnode associated with an IORT node
83 * @node: IORT table node to be looked-up
85 * Returns: fwnode_handle pointer on success, NULL on failure
87 static inline struct fwnode_handle *iort_get_fwnode(
88 struct acpi_iort_node *node)
90 struct iort_fwnode *curr;
91 struct fwnode_handle *fwnode = NULL;
93 spin_lock(&iort_fwnode_lock);
94 list_for_each_entry(curr, &iort_fwnode_list, list) {
95 if (curr->iort_node == node) {
96 fwnode = curr->fwnode;
100 spin_unlock(&iort_fwnode_lock);
106 * iort_delete_fwnode() - Delete fwnode associated with an IORT node
108 * @node: IORT table node associated with fwnode to delete
110 static inline void iort_delete_fwnode(struct acpi_iort_node *node)
112 struct iort_fwnode *curr, *tmp;
114 spin_lock(&iort_fwnode_lock);
115 list_for_each_entry_safe(curr, tmp, &iort_fwnode_list, list) {
116 if (curr->iort_node == node) {
117 list_del(&curr->list);
122 spin_unlock(&iort_fwnode_lock);
126 * iort_get_iort_node() - Retrieve iort_node associated with an fwnode
128 * @fwnode: fwnode associated with device to be looked-up
130 * Returns: iort_node pointer on success, NULL on failure
132 static inline struct acpi_iort_node *iort_get_iort_node(
133 struct fwnode_handle *fwnode)
135 struct iort_fwnode *curr;
136 struct acpi_iort_node *iort_node = NULL;
138 spin_lock(&iort_fwnode_lock);
139 list_for_each_entry(curr, &iort_fwnode_list, list) {
140 if (curr->fwnode == fwnode) {
141 iort_node = curr->iort_node;
145 spin_unlock(&iort_fwnode_lock);
150 typedef acpi_status (*iort_find_node_callback)
151 (struct acpi_iort_node *node, void *context);
153 /* Root pointer to the mapped IORT table */
154 static struct acpi_table_header *iort_table;
156 static LIST_HEAD(iort_msi_chip_list);
157 static DEFINE_SPINLOCK(iort_msi_chip_lock);
160 * iort_register_domain_token() - register domain token along with related
161 * ITS ID and base address to the list from where we can get it back later on.
163 * @base: ITS base address.
164 * @fw_node: Domain token.
166 * Returns: 0 on success, -ENOMEM if no memory when allocating list element
168 int iort_register_domain_token(int trans_id, phys_addr_t base,
169 struct fwnode_handle *fw_node)
171 struct iort_its_msi_chip *its_msi_chip;
173 its_msi_chip = kzalloc(sizeof(*its_msi_chip), GFP_KERNEL);
177 its_msi_chip->fw_node = fw_node;
178 its_msi_chip->translation_id = trans_id;
179 its_msi_chip->base_addr = base;
181 spin_lock(&iort_msi_chip_lock);
182 list_add(&its_msi_chip->list, &iort_msi_chip_list);
183 spin_unlock(&iort_msi_chip_lock);
189 * iort_deregister_domain_token() - Deregister domain token based on ITS ID
194 void iort_deregister_domain_token(int trans_id)
196 struct iort_its_msi_chip *its_msi_chip, *t;
198 spin_lock(&iort_msi_chip_lock);
199 list_for_each_entry_safe(its_msi_chip, t, &iort_msi_chip_list, list) {
200 if (its_msi_chip->translation_id == trans_id) {
201 list_del(&its_msi_chip->list);
206 spin_unlock(&iort_msi_chip_lock);
210 * iort_find_domain_token() - Find domain token based on given ITS ID
213 * Returns: domain token when find on the list, NULL otherwise
215 struct fwnode_handle *iort_find_domain_token(int trans_id)
217 struct fwnode_handle *fw_node = NULL;
218 struct iort_its_msi_chip *its_msi_chip;
220 spin_lock(&iort_msi_chip_lock);
221 list_for_each_entry(its_msi_chip, &iort_msi_chip_list, list) {
222 if (its_msi_chip->translation_id == trans_id) {
223 fw_node = its_msi_chip->fw_node;
227 spin_unlock(&iort_msi_chip_lock);
232 static struct acpi_iort_node *iort_scan_node(enum acpi_iort_node_type type,
233 iort_find_node_callback callback,
236 struct acpi_iort_node *iort_node, *iort_end;
237 struct acpi_table_iort *iort;
243 /* Get the first IORT node */
244 iort = (struct acpi_table_iort *)iort_table;
245 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort,
247 iort_end = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
250 for (i = 0; i < iort->node_count; i++) {
251 if (WARN_TAINT(iort_node >= iort_end, TAINT_FIRMWARE_WORKAROUND,
252 "IORT node pointer overflows, bad table!\n"))
255 if (iort_node->type == type &&
256 ACPI_SUCCESS(callback(iort_node, context)))
259 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort_node,
266 static acpi_status iort_match_node_callback(struct acpi_iort_node *node,
269 struct device *dev = context;
270 acpi_status status = AE_NOT_FOUND;
272 if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT) {
273 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
274 struct acpi_device *adev = to_acpi_device_node(dev->fwnode);
275 struct acpi_iort_named_component *ncomp;
280 status = acpi_get_name(adev->handle, ACPI_FULL_PATHNAME, &buf);
281 if (ACPI_FAILURE(status)) {
282 dev_warn(dev, "Can't get device full path name\n");
286 ncomp = (struct acpi_iort_named_component *)node->node_data;
287 status = !strcmp(ncomp->device_name, buf.pointer) ?
288 AE_OK : AE_NOT_FOUND;
289 acpi_os_free(buf.pointer);
290 } else if (node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
291 struct acpi_iort_root_complex *pci_rc;
294 bus = to_pci_bus(dev);
295 pci_rc = (struct acpi_iort_root_complex *)node->node_data;
298 * It is assumed that PCI segment numbers maps one-to-one
299 * with root complexes. Each segment number can represent only
302 status = pci_rc->pci_segment_number == pci_domain_nr(bus) ?
303 AE_OK : AE_NOT_FOUND;
309 static int iort_id_map(struct acpi_iort_id_mapping *map, u8 type, u32 rid_in,
312 /* Single mapping does not care for input id */
313 if (map->flags & ACPI_IORT_ID_SINGLE_MAPPING) {
314 if (type == ACPI_IORT_NODE_NAMED_COMPONENT ||
315 type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
316 *rid_out = map->output_base;
320 pr_warn(FW_BUG "[map %p] SINGLE MAPPING flag not allowed for node type %d, skipping ID map\n",
325 if (rid_in < map->input_base ||
326 (rid_in >= map->input_base + map->id_count))
329 *rid_out = map->output_base + (rid_in - map->input_base);
333 static struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node,
334 u32 *id_out, int index)
336 struct acpi_iort_node *parent;
337 struct acpi_iort_id_mapping *map;
339 if (!node->mapping_offset || !node->mapping_count ||
340 index >= node->mapping_count)
343 map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
344 node->mapping_offset + index * sizeof(*map));
347 if (!map->output_reference) {
348 pr_err(FW_BUG "[node %p type %d] ID map has NULL parent reference\n",
353 parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
354 map->output_reference);
356 if (map->flags & ACPI_IORT_ID_SINGLE_MAPPING) {
357 if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT ||
358 node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX ||
359 node->type == ACPI_IORT_NODE_SMMU_V3 ||
360 node->type == ACPI_IORT_NODE_PMCG) {
361 *id_out = map->output_base;
369 static int iort_get_id_mapping_index(struct acpi_iort_node *node)
371 struct acpi_iort_smmu_v3 *smmu;
373 switch (node->type) {
374 case ACPI_IORT_NODE_SMMU_V3:
376 * SMMUv3 dev ID mapping index was introduced in revision 1
377 * table, not available in revision 0
379 if (node->revision < 1)
382 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
384 * ID mapping index is only ignored if all interrupts are
387 if (smmu->event_gsiv && smmu->pri_gsiv && smmu->gerr_gsiv
391 if (smmu->id_mapping_index >= node->mapping_count) {
392 pr_err(FW_BUG "[node %p type %d] ID mapping index overflows valid mappings\n",
397 return smmu->id_mapping_index;
398 case ACPI_IORT_NODE_PMCG:
405 static struct acpi_iort_node *iort_node_map_id(struct acpi_iort_node *node,
406 u32 id_in, u32 *id_out,
411 /* Parse the ID mapping tree to find specified node type */
413 struct acpi_iort_id_mapping *map;
416 if (IORT_TYPE_MASK(node->type) & type_mask) {
422 if (!node->mapping_offset || !node->mapping_count)
425 map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
426 node->mapping_offset);
429 if (!map->output_reference) {
430 pr_err(FW_BUG "[node %p type %d] ID map has NULL parent reference\n",
436 * Get the special ID mapping index (if any) and skip its
437 * associated ID map to prevent erroneous multi-stage
438 * IORT ID translations.
440 index = iort_get_id_mapping_index(node);
442 /* Do the ID translation */
443 for (i = 0; i < node->mapping_count; i++, map++) {
444 /* if it is special mapping index, skip it */
448 if (!iort_id_map(map, node->type, id, &id))
452 if (i == node->mapping_count)
455 node = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
456 map->output_reference);
460 /* Map input ID to output ID unchanged on mapping failure */
467 static struct acpi_iort_node *iort_node_map_platform_id(
468 struct acpi_iort_node *node, u32 *id_out, u8 type_mask,
471 struct acpi_iort_node *parent;
474 /* step 1: retrieve the initial dev id */
475 parent = iort_node_get_id(node, &id, index);
480 * optional step 2: map the initial dev id if its parent is not
481 * the target type we want, map it again for the use cases such
482 * as NC (named component) -> SMMU -> ITS. If the type is matched,
483 * return the initial dev id and its parent pointer directly.
485 if (!(IORT_TYPE_MASK(parent->type) & type_mask))
486 parent = iort_node_map_id(parent, id, id_out, type_mask);
494 static struct acpi_iort_node *iort_find_dev_node(struct device *dev)
496 struct pci_bus *pbus;
498 if (!dev_is_pci(dev)) {
499 struct acpi_iort_node *node;
501 * scan iort_fwnode_list to see if it's an iort platform
502 * device (such as SMMU, PMCG),its iort node already cached
503 * and associated with fwnode when iort platform devices
506 node = iort_get_iort_node(dev->fwnode);
511 * if not, then it should be a platform device defined in
512 * DSDT/SSDT (with Named Component node in IORT)
514 return iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
515 iort_match_node_callback, dev);
518 /* Find a PCI root bus */
519 pbus = to_pci_dev(dev)->bus;
520 while (!pci_is_root_bus(pbus))
523 return iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
524 iort_match_node_callback, &pbus->dev);
528 * iort_msi_map_rid() - Map a MSI requester ID for a device
529 * @dev: The device for which the mapping is to be done.
530 * @req_id: The device requester ID.
532 * Returns: mapped MSI RID on success, input requester ID otherwise
534 u32 iort_msi_map_rid(struct device *dev, u32 req_id)
536 struct acpi_iort_node *node;
539 node = iort_find_dev_node(dev);
543 iort_node_map_id(node, req_id, &dev_id, IORT_MSI_TYPE);
548 * iort_pmsi_get_dev_id() - Get the device id for a device
549 * @dev: The device for which the mapping is to be done.
550 * @dev_id: The device ID found.
552 * Returns: 0 for successful find a dev id, -ENODEV on error
554 int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id)
557 struct acpi_iort_node *node;
559 node = iort_find_dev_node(dev);
563 index = iort_get_id_mapping_index(node);
564 /* if there is a valid index, go get the dev_id directly */
566 if (iort_node_get_id(node, dev_id, index))
569 for (i = 0; i < node->mapping_count; i++) {
570 if (iort_node_map_platform_id(node, dev_id,
579 static int __maybe_unused iort_find_its_base(u32 its_id, phys_addr_t *base)
581 struct iort_its_msi_chip *its_msi_chip;
584 spin_lock(&iort_msi_chip_lock);
585 list_for_each_entry(its_msi_chip, &iort_msi_chip_list, list) {
586 if (its_msi_chip->translation_id == its_id) {
587 *base = its_msi_chip->base_addr;
592 spin_unlock(&iort_msi_chip_lock);
598 * iort_dev_find_its_id() - Find the ITS identifier for a device
600 * @req_id: Device's requester ID
601 * @idx: Index of the ITS identifier list.
602 * @its_id: ITS identifier.
604 * Returns: 0 on success, appropriate error value otherwise
606 static int iort_dev_find_its_id(struct device *dev, u32 req_id,
607 unsigned int idx, int *its_id)
609 struct acpi_iort_its_group *its;
610 struct acpi_iort_node *node;
612 node = iort_find_dev_node(dev);
616 node = iort_node_map_id(node, req_id, NULL, IORT_MSI_TYPE);
620 /* Move to ITS specific data */
621 its = (struct acpi_iort_its_group *)node->node_data;
622 if (idx > its->its_count) {
623 dev_err(dev, "requested ITS ID index [%d] is greater than available [%d]\n",
624 idx, its->its_count);
628 *its_id = its->identifiers[idx];
633 * iort_get_device_domain() - Find MSI domain related to a device
635 * @req_id: Requester ID for the device.
637 * Returns: the MSI domain for this device, NULL otherwise
639 struct irq_domain *iort_get_device_domain(struct device *dev, u32 req_id)
641 struct fwnode_handle *handle;
644 if (iort_dev_find_its_id(dev, req_id, 0, &its_id))
647 handle = iort_find_domain_token(its_id);
651 return irq_find_matching_fwnode(handle, DOMAIN_BUS_PCI_MSI);
654 static void iort_set_device_domain(struct device *dev,
655 struct acpi_iort_node *node)
657 struct acpi_iort_its_group *its;
658 struct acpi_iort_node *msi_parent;
659 struct acpi_iort_id_mapping *map;
660 struct fwnode_handle *iort_fwnode;
661 struct irq_domain *domain;
664 index = iort_get_id_mapping_index(node);
668 map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
669 node->mapping_offset + index * sizeof(*map));
672 if (!map->output_reference ||
673 !(map->flags & ACPI_IORT_ID_SINGLE_MAPPING)) {
674 pr_err(FW_BUG "[node %p type %d] Invalid MSI mapping\n",
679 msi_parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
680 map->output_reference);
682 if (!msi_parent || msi_parent->type != ACPI_IORT_NODE_ITS_GROUP)
685 /* Move to ITS specific data */
686 its = (struct acpi_iort_its_group *)msi_parent->node_data;
688 iort_fwnode = iort_find_domain_token(its->identifiers[0]);
692 domain = irq_find_matching_fwnode(iort_fwnode, DOMAIN_BUS_PLATFORM_MSI);
694 dev_set_msi_domain(dev, domain);
698 * iort_get_platform_device_domain() - Find MSI domain related to a
700 * @dev: the dev pointer associated with the platform device
702 * Returns: the MSI domain for this device, NULL otherwise
704 static struct irq_domain *iort_get_platform_device_domain(struct device *dev)
706 struct acpi_iort_node *node, *msi_parent = NULL;
707 struct fwnode_handle *iort_fwnode;
708 struct acpi_iort_its_group *its;
711 /* find its associated iort node */
712 node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
713 iort_match_node_callback, dev);
717 /* then find its msi parent node */
718 for (i = 0; i < node->mapping_count; i++) {
719 msi_parent = iort_node_map_platform_id(node, NULL,
728 /* Move to ITS specific data */
729 its = (struct acpi_iort_its_group *)msi_parent->node_data;
731 iort_fwnode = iort_find_domain_token(its->identifiers[0]);
735 return irq_find_matching_fwnode(iort_fwnode, DOMAIN_BUS_PLATFORM_MSI);
738 void acpi_configure_pmsi_domain(struct device *dev)
740 struct irq_domain *msi_domain;
742 msi_domain = iort_get_platform_device_domain(dev);
744 dev_set_msi_domain(dev, msi_domain);
747 static int __maybe_unused __get_pci_rid(struct pci_dev *pdev, u16 alias,
756 #ifdef CONFIG_IOMMU_API
757 static struct acpi_iort_node *iort_get_msi_resv_iommu(struct device *dev)
759 struct acpi_iort_node *iommu;
760 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
762 iommu = iort_get_iort_node(fwspec->iommu_fwnode);
764 if (iommu && (iommu->type == ACPI_IORT_NODE_SMMU_V3)) {
765 struct acpi_iort_smmu_v3 *smmu;
767 smmu = (struct acpi_iort_smmu_v3 *)iommu->node_data;
768 if (smmu->model == ACPI_IORT_SMMU_V3_HISILICON_HI161X)
775 static inline const struct iommu_ops *iort_fwspec_iommu_ops(struct device *dev)
777 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
779 return (fwspec && fwspec->ops) ? fwspec->ops : NULL;
782 static inline int iort_add_device_replay(const struct iommu_ops *ops,
787 if (dev->bus && !device_iommu_mapped(dev))
788 err = iommu_probe_device(dev);
794 * iort_iommu_msi_get_resv_regions - Reserved region driver helper
795 * @dev: Device from iommu_get_resv_regions()
796 * @head: Reserved region list from iommu_get_resv_regions()
798 * Returns: Number of msi reserved regions on success (0 if platform
799 * doesn't require the reservation or no associated msi regions),
800 * appropriate error value otherwise. The ITS interrupt translation
801 * spaces (ITS_base + SZ_64K, SZ_64K) associated with the device
802 * are the msi reserved regions.
804 int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head)
806 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
807 struct acpi_iort_its_group *its;
808 struct acpi_iort_node *iommu_node, *its_node = NULL;
811 iommu_node = iort_get_msi_resv_iommu(dev);
816 * Current logic to reserve ITS regions relies on HW topologies
817 * where a given PCI or named component maps its IDs to only one
818 * ITS group; if a PCI or named component can map its IDs to
819 * different ITS groups through IORT mappings this function has
820 * to be reworked to ensure we reserve regions for all ITS groups
821 * a given PCI or named component may map IDs to.
824 for (i = 0; i < fwspec->num_ids; i++) {
825 its_node = iort_node_map_id(iommu_node,
827 NULL, IORT_MSI_TYPE);
835 /* Move to ITS specific data */
836 its = (struct acpi_iort_its_group *)its_node->node_data;
838 for (i = 0; i < its->its_count; i++) {
841 if (!iort_find_its_base(its->identifiers[i], &base)) {
842 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
843 struct iommu_resv_region *region;
845 region = iommu_alloc_resv_region(base + SZ_64K, SZ_64K,
846 prot, IOMMU_RESV_MSI);
848 list_add_tail(®ion->list, head);
854 return (resv == its->its_count) ? resv : -ENODEV;
857 static inline bool iort_iommu_driver_enabled(u8 type)
860 case ACPI_IORT_NODE_SMMU_V3:
861 return IS_BUILTIN(CONFIG_ARM_SMMU_V3);
862 case ACPI_IORT_NODE_SMMU:
863 return IS_BUILTIN(CONFIG_ARM_SMMU);
865 pr_warn("IORT node type %u does not describe an SMMU\n", type);
870 static int arm_smmu_iort_xlate(struct device *dev, u32 streamid,
871 struct fwnode_handle *fwnode,
872 const struct iommu_ops *ops)
874 int ret = iommu_fwspec_init(dev, fwnode, ops);
877 ret = iommu_fwspec_add_ids(dev, &streamid, 1);
882 static bool iort_pci_rc_supports_ats(struct acpi_iort_node *node)
884 struct acpi_iort_root_complex *pci_rc;
886 pci_rc = (struct acpi_iort_root_complex *)node->node_data;
887 return pci_rc->ats_attribute & ACPI_IORT_ATS_SUPPORTED;
890 static int iort_iommu_xlate(struct device *dev, struct acpi_iort_node *node,
893 const struct iommu_ops *ops;
894 struct fwnode_handle *iort_fwnode;
899 iort_fwnode = iort_get_fwnode(node);
904 * If the ops look-up fails, this means that either
905 * the SMMU drivers have not been probed yet or that
906 * the SMMU drivers are not built in the kernel;
907 * Depending on whether the SMMU drivers are built-in
908 * in the kernel or not, defer the IOMMU configuration
911 ops = iommu_ops_from_fwnode(iort_fwnode);
913 return iort_iommu_driver_enabled(node->type) ?
914 -EPROBE_DEFER : -ENODEV;
916 return arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops);
919 struct iort_pci_alias_info {
921 struct acpi_iort_node *node;
924 static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
926 struct iort_pci_alias_info *info = data;
927 struct acpi_iort_node *parent;
930 parent = iort_node_map_id(info->node, alias, &streamid,
932 return iort_iommu_xlate(info->dev, parent, streamid);
936 * iort_iommu_configure - Set-up IOMMU configuration for a device.
938 * @dev: device to configure
940 * Returns: iommu_ops pointer on configuration success
941 * NULL on configuration failure
943 const struct iommu_ops *iort_iommu_configure(struct device *dev)
945 struct acpi_iort_node *node, *parent;
946 const struct iommu_ops *ops;
951 * If we already translated the fwspec there
952 * is nothing left to do, return the iommu_ops.
954 ops = iort_fwspec_iommu_ops(dev);
958 if (dev_is_pci(dev)) {
959 struct pci_bus *bus = to_pci_dev(dev)->bus;
960 struct iort_pci_alias_info info = { .dev = dev };
962 node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
963 iort_match_node_callback, &bus->dev);
968 err = pci_for_each_dma_alias(to_pci_dev(dev),
969 iort_pci_iommu_init, &info);
971 if (!err && iort_pci_rc_supports_ats(node))
972 dev->iommu_fwspec->flags |= IOMMU_FWSPEC_PCI_RC_ATS;
976 node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
977 iort_match_node_callback, dev);
982 parent = iort_node_map_platform_id(node, &streamid,
987 err = iort_iommu_xlate(dev, parent, streamid);
988 } while (parent && !err);
992 * If we have reason to believe the IOMMU driver missed the initial
993 * add_device callback for dev, replay it to get things in order.
996 ops = iort_fwspec_iommu_ops(dev);
997 err = iort_add_device_replay(ops, dev);
1000 /* Ignore all other errors apart from EPROBE_DEFER */
1001 if (err == -EPROBE_DEFER) {
1004 dev_dbg(dev, "Adding to IOMMU failed: %d\n", err);
1011 static inline const struct iommu_ops *iort_fwspec_iommu_ops(struct device *dev)
1013 static inline int iort_add_device_replay(const struct iommu_ops *ops,
1016 int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head)
1018 const struct iommu_ops *iort_iommu_configure(struct device *dev)
1022 static int nc_dma_get_range(struct device *dev, u64 *size)
1024 struct acpi_iort_node *node;
1025 struct acpi_iort_named_component *ncomp;
1027 node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
1028 iort_match_node_callback, dev);
1032 ncomp = (struct acpi_iort_named_component *)node->node_data;
1034 *size = ncomp->memory_address_limit >= 64 ? U64_MAX :
1035 1ULL<<ncomp->memory_address_limit;
1040 static int rc_dma_get_range(struct device *dev, u64 *size)
1042 struct acpi_iort_node *node;
1043 struct acpi_iort_root_complex *rc;
1044 struct pci_bus *pbus = to_pci_dev(dev)->bus;
1046 node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
1047 iort_match_node_callback, &pbus->dev);
1048 if (!node || node->revision < 1)
1051 rc = (struct acpi_iort_root_complex *)node->node_data;
1053 *size = rc->memory_address_limit >= 64 ? U64_MAX :
1054 1ULL<<rc->memory_address_limit;
1060 * iort_dma_setup() - Set-up device DMA parameters.
1062 * @dev: device to configure
1063 * @dma_addr: device DMA address result pointer
1064 * @size: DMA range size result pointer
1066 void iort_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size)
1068 u64 mask, dmaaddr = 0, size = 0, offset = 0;
1072 * If @dev is expected to be DMA-capable then the bus code that created
1073 * it should have initialised its dma_mask pointer by this point. For
1074 * now, we'll continue the legacy behaviour of coercing it to the
1075 * coherent mask if not, but we'll no longer do so quietly.
1077 if (!dev->dma_mask) {
1078 dev_warn(dev, "DMA mask not set\n");
1079 dev->dma_mask = &dev->coherent_dma_mask;
1082 if (dev->coherent_dma_mask)
1083 size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1);
1087 if (dev_is_pci(dev)) {
1088 ret = acpi_dma_get_range(dev, &dmaaddr, &offset, &size);
1090 ret = rc_dma_get_range(dev, &size);
1092 ret = nc_dma_get_range(dev, &size);
1096 msb = fls64(dmaaddr + size - 1);
1098 * Round-up to the power-of-two mask or set
1099 * the mask to the whole 64-bit address space
1100 * in case the DMA region covers the full
1103 mask = msb == 64 ? U64_MAX : (1ULL << msb) - 1;
1105 * Limit coherent and dma mask based on size
1106 * retrieved from firmware.
1108 dev->bus_dma_mask = mask;
1109 dev->coherent_dma_mask = mask;
1110 *dev->dma_mask = mask;
1113 *dma_addr = dmaaddr;
1116 dev->dma_pfn_offset = PFN_DOWN(offset);
1117 dev_dbg(dev, "dma_pfn_offset(%#08llx)\n", offset);
1120 static void __init acpi_iort_register_irq(int hwirq, const char *name,
1122 struct resource *res)
1124 int irq = acpi_register_gsi(NULL, hwirq, trigger,
1128 pr_err("could not register gsi hwirq %d name [%s]\n", hwirq,
1135 res->flags = IORESOURCE_IRQ;
1139 static int __init arm_smmu_v3_count_resources(struct acpi_iort_node *node)
1141 struct acpi_iort_smmu_v3 *smmu;
1142 /* Always present mem resource */
1145 /* Retrieve SMMUv3 specific data */
1146 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
1148 if (smmu->event_gsiv)
1154 if (smmu->gerr_gsiv)
1157 if (smmu->sync_gsiv)
1163 static bool arm_smmu_v3_is_combined_irq(struct acpi_iort_smmu_v3 *smmu)
1166 * Cavium ThunderX2 implementation doesn't not support unique
1167 * irq line. Use single irq line for all the SMMUv3 interrupts.
1169 if (smmu->model != ACPI_IORT_SMMU_V3_CAVIUM_CN99XX)
1173 * ThunderX2 doesn't support MSIs from the SMMU, so we're checking
1176 return smmu->event_gsiv == smmu->pri_gsiv &&
1177 smmu->event_gsiv == smmu->gerr_gsiv &&
1178 smmu->event_gsiv == smmu->sync_gsiv;
1181 static unsigned long arm_smmu_v3_resource_size(struct acpi_iort_smmu_v3 *smmu)
1184 * Override the size, for Cavium ThunderX2 implementation
1185 * which doesn't support the page 1 SMMU register space.
1187 if (smmu->model == ACPI_IORT_SMMU_V3_CAVIUM_CN99XX)
1193 static void __init arm_smmu_v3_init_resources(struct resource *res,
1194 struct acpi_iort_node *node)
1196 struct acpi_iort_smmu_v3 *smmu;
1199 /* Retrieve SMMUv3 specific data */
1200 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
1202 res[num_res].start = smmu->base_address;
1203 res[num_res].end = smmu->base_address +
1204 arm_smmu_v3_resource_size(smmu) - 1;
1205 res[num_res].flags = IORESOURCE_MEM;
1208 if (arm_smmu_v3_is_combined_irq(smmu)) {
1209 if (smmu->event_gsiv)
1210 acpi_iort_register_irq(smmu->event_gsiv, "combined",
1211 ACPI_EDGE_SENSITIVE,
1215 if (smmu->event_gsiv)
1216 acpi_iort_register_irq(smmu->event_gsiv, "eventq",
1217 ACPI_EDGE_SENSITIVE,
1221 acpi_iort_register_irq(smmu->pri_gsiv, "priq",
1222 ACPI_EDGE_SENSITIVE,
1225 if (smmu->gerr_gsiv)
1226 acpi_iort_register_irq(smmu->gerr_gsiv, "gerror",
1227 ACPI_EDGE_SENSITIVE,
1230 if (smmu->sync_gsiv)
1231 acpi_iort_register_irq(smmu->sync_gsiv, "cmdq-sync",
1232 ACPI_EDGE_SENSITIVE,
1237 static void __init arm_smmu_v3_dma_configure(struct device *dev,
1238 struct acpi_iort_node *node)
1240 struct acpi_iort_smmu_v3 *smmu;
1241 enum dev_dma_attr attr;
1243 /* Retrieve SMMUv3 specific data */
1244 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
1246 attr = (smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE) ?
1247 DEV_DMA_COHERENT : DEV_DMA_NON_COHERENT;
1249 /* We expect the dma masks to be equivalent for all SMMUv3 set-ups */
1250 dev->dma_mask = &dev->coherent_dma_mask;
1252 /* Configure DMA for the page table walker */
1253 acpi_dma_configure(dev, attr);
1256 #if defined(CONFIG_ACPI_NUMA)
1258 * set numa proximity domain for smmuv3 device
1260 static int __init arm_smmu_v3_set_proximity(struct device *dev,
1261 struct acpi_iort_node *node)
1263 struct acpi_iort_smmu_v3 *smmu;
1265 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
1266 if (smmu->flags & ACPI_IORT_SMMU_V3_PXM_VALID) {
1267 int node = acpi_map_pxm_to_node(smmu->pxm);
1269 if (node != NUMA_NO_NODE && !node_online(node))
1272 set_dev_node(dev, node);
1273 pr_info("SMMU-v3[%llx] Mapped to Proximity domain %d\n",
1280 #define arm_smmu_v3_set_proximity NULL
1283 static int __init arm_smmu_count_resources(struct acpi_iort_node *node)
1285 struct acpi_iort_smmu *smmu;
1287 /* Retrieve SMMU specific data */
1288 smmu = (struct acpi_iort_smmu *)node->node_data;
1291 * Only consider the global fault interrupt and ignore the
1292 * configuration access interrupt.
1294 * MMIO address and global fault interrupt resources are always
1295 * present so add them to the context interrupt count as a static
1298 return smmu->context_interrupt_count + 2;
1301 static void __init arm_smmu_init_resources(struct resource *res,
1302 struct acpi_iort_node *node)
1304 struct acpi_iort_smmu *smmu;
1305 int i, hw_irq, trigger, num_res = 0;
1306 u64 *ctx_irq, *glb_irq;
1308 /* Retrieve SMMU specific data */
1309 smmu = (struct acpi_iort_smmu *)node->node_data;
1311 res[num_res].start = smmu->base_address;
1312 res[num_res].end = smmu->base_address + smmu->span - 1;
1313 res[num_res].flags = IORESOURCE_MEM;
1316 glb_irq = ACPI_ADD_PTR(u64, node, smmu->global_interrupt_offset);
1318 hw_irq = IORT_IRQ_MASK(glb_irq[0]);
1319 trigger = IORT_IRQ_TRIGGER_MASK(glb_irq[0]);
1321 acpi_iort_register_irq(hw_irq, "arm-smmu-global", trigger,
1325 ctx_irq = ACPI_ADD_PTR(u64, node, smmu->context_interrupt_offset);
1326 for (i = 0; i < smmu->context_interrupt_count; i++) {
1327 hw_irq = IORT_IRQ_MASK(ctx_irq[i]);
1328 trigger = IORT_IRQ_TRIGGER_MASK(ctx_irq[i]);
1330 acpi_iort_register_irq(hw_irq, "arm-smmu-context", trigger,
1335 static void __init arm_smmu_dma_configure(struct device *dev,
1336 struct acpi_iort_node *node)
1338 struct acpi_iort_smmu *smmu;
1339 enum dev_dma_attr attr;
1341 /* Retrieve SMMU specific data */
1342 smmu = (struct acpi_iort_smmu *)node->node_data;
1344 attr = (smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) ?
1345 DEV_DMA_COHERENT : DEV_DMA_NON_COHERENT;
1347 /* We expect the dma masks to be equivalent for SMMU set-ups */
1348 dev->dma_mask = &dev->coherent_dma_mask;
1350 /* Configure DMA for the page table walker */
1351 acpi_dma_configure(dev, attr);
1354 static int __init arm_smmu_v3_pmcg_count_resources(struct acpi_iort_node *node)
1356 struct acpi_iort_pmcg *pmcg;
1358 /* Retrieve PMCG specific data */
1359 pmcg = (struct acpi_iort_pmcg *)node->node_data;
1362 * There are always 2 memory resources.
1363 * If the overflow_gsiv is present then add that for a total of 3.
1365 return pmcg->overflow_gsiv ? 3 : 2;
1368 static void __init arm_smmu_v3_pmcg_init_resources(struct resource *res,
1369 struct acpi_iort_node *node)
1371 struct acpi_iort_pmcg *pmcg;
1373 /* Retrieve PMCG specific data */
1374 pmcg = (struct acpi_iort_pmcg *)node->node_data;
1376 res[0].start = pmcg->page0_base_address;
1377 res[0].end = pmcg->page0_base_address + SZ_4K - 1;
1378 res[0].flags = IORESOURCE_MEM;
1379 res[1].start = pmcg->page1_base_address;
1380 res[1].end = pmcg->page1_base_address + SZ_4K - 1;
1381 res[1].flags = IORESOURCE_MEM;
1383 if (pmcg->overflow_gsiv)
1384 acpi_iort_register_irq(pmcg->overflow_gsiv, "overflow",
1385 ACPI_EDGE_SENSITIVE, &res[2]);
1388 static struct acpi_platform_list pmcg_plat_info[] __initdata = {
1389 /* HiSilicon Hip08 Platform */
1390 {"HISI ", "HIP08 ", 0, ACPI_SIG_IORT, greater_than_or_equal,
1391 "Erratum #162001800", IORT_SMMU_V3_PMCG_HISI_HIP08},
1395 static int __init arm_smmu_v3_pmcg_add_platdata(struct platform_device *pdev)
1400 idx = acpi_match_platform_list(pmcg_plat_info);
1402 model = pmcg_plat_info[idx].data;
1404 model = IORT_SMMU_V3_PMCG_GENERIC;
1406 return platform_device_add_data(pdev, &model, sizeof(model));
1409 struct iort_dev_config {
1411 int (*dev_init)(struct acpi_iort_node *node);
1412 void (*dev_dma_configure)(struct device *dev,
1413 struct acpi_iort_node *node);
1414 int (*dev_count_resources)(struct acpi_iort_node *node);
1415 void (*dev_init_resources)(struct resource *res,
1416 struct acpi_iort_node *node);
1417 int (*dev_set_proximity)(struct device *dev,
1418 struct acpi_iort_node *node);
1419 int (*dev_add_platdata)(struct platform_device *pdev);
1422 static const struct iort_dev_config iort_arm_smmu_v3_cfg __initconst = {
1423 .name = "arm-smmu-v3",
1424 .dev_dma_configure = arm_smmu_v3_dma_configure,
1425 .dev_count_resources = arm_smmu_v3_count_resources,
1426 .dev_init_resources = arm_smmu_v3_init_resources,
1427 .dev_set_proximity = arm_smmu_v3_set_proximity,
1430 static const struct iort_dev_config iort_arm_smmu_cfg __initconst = {
1432 .dev_dma_configure = arm_smmu_dma_configure,
1433 .dev_count_resources = arm_smmu_count_resources,
1434 .dev_init_resources = arm_smmu_init_resources,
1437 static const struct iort_dev_config iort_arm_smmu_v3_pmcg_cfg __initconst = {
1438 .name = "arm-smmu-v3-pmcg",
1439 .dev_count_resources = arm_smmu_v3_pmcg_count_resources,
1440 .dev_init_resources = arm_smmu_v3_pmcg_init_resources,
1441 .dev_add_platdata = arm_smmu_v3_pmcg_add_platdata,
1444 static __init const struct iort_dev_config *iort_get_dev_cfg(
1445 struct acpi_iort_node *node)
1447 switch (node->type) {
1448 case ACPI_IORT_NODE_SMMU_V3:
1449 return &iort_arm_smmu_v3_cfg;
1450 case ACPI_IORT_NODE_SMMU:
1451 return &iort_arm_smmu_cfg;
1452 case ACPI_IORT_NODE_PMCG:
1453 return &iort_arm_smmu_v3_pmcg_cfg;
1460 * iort_add_platform_device() - Allocate a platform device for IORT node
1461 * @node: Pointer to device ACPI IORT node
1463 * Returns: 0 on success, <0 failure
1465 static int __init iort_add_platform_device(struct acpi_iort_node *node,
1466 const struct iort_dev_config *ops)
1468 struct fwnode_handle *fwnode;
1469 struct platform_device *pdev;
1473 pdev = platform_device_alloc(ops->name, PLATFORM_DEVID_AUTO);
1477 if (ops->dev_set_proximity) {
1478 ret = ops->dev_set_proximity(&pdev->dev, node);
1483 count = ops->dev_count_resources(node);
1485 r = kcalloc(count, sizeof(*r), GFP_KERNEL);
1491 ops->dev_init_resources(r, node);
1493 ret = platform_device_add_resources(pdev, r, count);
1495 * Resources are duplicated in platform_device_add_resources,
1496 * free their allocated memory
1504 * Platform devices based on PMCG nodes uses platform_data to
1505 * pass the hardware model info to the driver. For others, add
1506 * a copy of IORT node pointer to platform_data to be used to
1507 * retrieve IORT data information.
1509 if (ops->dev_add_platdata)
1510 ret = ops->dev_add_platdata(pdev);
1512 ret = platform_device_add_data(pdev, &node, sizeof(node));
1517 fwnode = iort_get_fwnode(node);
1524 pdev->dev.fwnode = fwnode;
1526 if (ops->dev_dma_configure)
1527 ops->dev_dma_configure(&pdev->dev, node);
1529 iort_set_device_domain(&pdev->dev, node);
1531 ret = platform_device_add(pdev);
1533 goto dma_deconfigure;
1538 arch_teardown_dma_ops(&pdev->dev);
1540 platform_device_put(pdev);
1546 static void __init iort_enable_acs(struct acpi_iort_node *iort_node)
1548 static bool acs_enabled __initdata;
1553 if (iort_node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
1554 struct acpi_iort_node *parent;
1555 struct acpi_iort_id_mapping *map;
1558 map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, iort_node,
1559 iort_node->mapping_offset);
1561 for (i = 0; i < iort_node->mapping_count; i++, map++) {
1562 if (!map->output_reference)
1565 parent = ACPI_ADD_PTR(struct acpi_iort_node,
1566 iort_table, map->output_reference);
1568 * If we detect a RC->SMMU mapping, make sure
1569 * we enable ACS on the system.
1571 if ((parent->type == ACPI_IORT_NODE_SMMU) ||
1572 (parent->type == ACPI_IORT_NODE_SMMU_V3)) {
1581 static inline void iort_enable_acs(struct acpi_iort_node *iort_node) { }
1584 static void __init iort_init_platform_devices(void)
1586 struct acpi_iort_node *iort_node, *iort_end;
1587 struct acpi_table_iort *iort;
1588 struct fwnode_handle *fwnode;
1590 const struct iort_dev_config *ops;
1593 * iort_table and iort both point to the start of IORT table, but
1594 * have different struct types
1596 iort = (struct acpi_table_iort *)iort_table;
1598 /* Get the first IORT node */
1599 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort,
1601 iort_end = ACPI_ADD_PTR(struct acpi_iort_node, iort,
1602 iort_table->length);
1604 for (i = 0; i < iort->node_count; i++) {
1605 if (iort_node >= iort_end) {
1606 pr_err("iort node pointer overflows, bad table\n");
1610 iort_enable_acs(iort_node);
1612 ops = iort_get_dev_cfg(iort_node);
1614 fwnode = acpi_alloc_fwnode_static();
1618 iort_set_fwnode(iort_node, fwnode);
1620 ret = iort_add_platform_device(iort_node, ops);
1622 iort_delete_fwnode(iort_node);
1623 acpi_free_fwnode_static(fwnode);
1628 iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort_node,
1633 void __init acpi_iort_init(void)
1637 status = acpi_get_table(ACPI_SIG_IORT, 0, &iort_table);
1638 if (ACPI_FAILURE(status)) {
1639 if (status != AE_NOT_FOUND) {
1640 const char *msg = acpi_format_exception(status);
1642 pr_err("Failed to get table, %s\n", msg);
1648 iort_init_platform_devices();