drivers/irqchip/irq-loongson-eiointc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Loongson Extend I/O Interrupt Controller support
   4  *
   5  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
   6  */
   7
   8 #define pr_fmt(fmt) "eiointc: " fmt
   9
  10 #include <linux/cpuhotplug.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/irq.h>
  13 #include <linux/irqchip.h>
  14 #include <linux/irqdomain.h>
  15 #include <linux/irqchip/chained_irq.h>
  16 #include <linux/kernel.h>
  17 #include <linux/syscore_ops.h>
  18 #include <asm/numa.h>
  19
  20 #define EIOINTC_REG_NODEMAP     0x14a0
  21 #define EIOINTC_REG_IPMAP       0x14c0
  22 #define EIOINTC_REG_ENABLE      0x1600
  23 #define EIOINTC_REG_BOUNCE      0x1680
  24 #define EIOINTC_REG_ISR         0x1800
  25 #define EIOINTC_REG_ROUTE       0x1c00
  26
  27 #define VEC_REG_COUNT           4
  28 #define VEC_COUNT_PER_REG       64
  29 #define VEC_COUNT               (VEC_REG_COUNT * VEC_COUNT_PER_REG)
  30 #define VEC_REG_IDX(irq_id)     ((irq_id) / VEC_COUNT_PER_REG)
  31 #define VEC_REG_BIT(irq_id)     ((irq_id) % VEC_COUNT_PER_REG)
  32 #define EIOINTC_ALL_ENABLE      0xffffffff
  33
  34 #define MAX_EIO_NODES           (NR_CPUS / CORES_PER_EIO_NODE)
  35
  36 static int nr_pics;
  37
  38 struct eiointc_priv {
  39         u32                     node;
  40         u32                     vec_count;
  41         nodemask_t              node_map;
  42         cpumask_t               cpuspan_map;
  43         struct fwnode_handle    *domain_handle;
  44         struct irq_domain       *eiointc_domain;
  45 };
  46
  47 static struct eiointc_priv *eiointc_priv[MAX_IO_PICS];
  48
  49 static void eiointc_enable(void)
  50 {
  51         uint64_t misc;
  52
  53         misc = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
  54         misc |= IOCSR_MISC_FUNC_EXT_IOI_EN;
  55         iocsr_write64(misc, LOONGARCH_IOCSR_MISC_FUNC);
  56 }
  57
  58 static int cpu_to_eio_node(int cpu)
  59 {
  60         return cpu_logical_map(cpu) / CORES_PER_EIO_NODE;
  61 }
  62
  63 #ifdef CONFIG_SMP
  64 static void eiointc_set_irq_route(int pos, unsigned int cpu, unsigned int mnode, nodemask_t *node_map)
  65 {
  66         int i, node, cpu_node, route_node;
  67         unsigned char coremap;
  68         uint32_t pos_off, data, data_byte, data_mask;
  69
  70         pos_off = pos & ~3;
  71         data_byte = pos & 3;
  72         data_mask = ~BIT_MASK(data_byte) & 0xf;
  73
  74         /* Calculate node and coremap of target irq */
  75         cpu_node = cpu_logical_map(cpu) / CORES_PER_EIO_NODE;
  76         coremap = BIT(cpu_logical_map(cpu) % CORES_PER_EIO_NODE);
  77
  78         for_each_online_cpu(i) {
  79                 node = cpu_to_eio_node(i);
  80                 if (!node_isset(node, *node_map))
  81                         continue;
  82
  83                 /* EIO node 0 is in charge of inter-node interrupt dispatch */
  84                 route_node = (node == mnode) ? cpu_node : node;
  85                 data = ((coremap | (route_node << 4)) << (data_byte * 8));
  86                 csr_any_send(EIOINTC_REG_ROUTE + pos_off, data, data_mask, node * CORES_PER_EIO_NODE);
  87         }
  88 }
  89
  90 static DEFINE_RAW_SPINLOCK(affinity_lock);
  91
  92 static int eiointc_set_irq_affinity(struct irq_data *d, const struct cpumask *affinity, bool force)
  93 {
  94         unsigned int cpu;
  95         unsigned long flags;
  96         uint32_t vector, regaddr;
  97         struct eiointc_priv *priv = d->domain->host_data;
  98
  99         raw_spin_lock_irqsave(&affinity_lock, flags);
 100
 101         cpu = cpumask_first_and_and(&priv->cpuspan_map, affinity, cpu_online_mask);
 102         if (cpu >= nr_cpu_ids) {
 103                 raw_spin_unlock_irqrestore(&affinity_lock, flags);
 104                 return -EINVAL;
 105         }
 106
 107         vector = d->hwirq;
 108         regaddr = EIOINTC_REG_ENABLE + ((vector >> 5) << 2);
 109
 110         /* Mask target vector */
 111         csr_any_send(regaddr, EIOINTC_ALL_ENABLE & (~BIT(vector & 0x1F)),
 112                         0x0, priv->node * CORES_PER_EIO_NODE);
 113
 114         /* Set route for target vector */
 115         eiointc_set_irq_route(vector, cpu, priv->node, &priv->node_map);
 116
 117         /* Unmask target vector */
 118         csr_any_send(regaddr, EIOINTC_ALL_ENABLE,
 119                         0x0, priv->node * CORES_PER_EIO_NODE);
 120
 121         irq_data_update_effective_affinity(d, cpumask_of(cpu));
 122
 123         raw_spin_unlock_irqrestore(&affinity_lock, flags);
 124
 125         return IRQ_SET_MASK_OK;
 126 }
 127 #endif
 128
 129 static int eiointc_index(int node)
 130 {
 131         int i;
 132
 133         for (i = 0; i < nr_pics; i++) {
 134                 if (node_isset(node, eiointc_priv[i]->node_map))
 135                         return i;
 136         }
 137
 138         return -1;
 139 }
 140
 141 static int eiointc_router_init(unsigned int cpu)
 142 {
 143         int i, bit;
 144         uint32_t data;
 145         uint32_t node = cpu_to_eio_node(cpu);
 146         int index = eiointc_index(node);
 147
 148         if (index < 0) {
 149                 pr_err("Error: invalid nodemap!\n");
 150                 return -1;
 151         }
 152
 153         if ((cpu_logical_map(cpu) % CORES_PER_EIO_NODE) == 0) {
 154                 eiointc_enable();
 155
 156                 for (i = 0; i < eiointc_priv[0]->vec_count / 32; i++) {
 157                         data = (((1 << (i * 2 + 1)) << 16) | (1 << (i * 2)));
 158                         iocsr_write32(data, EIOINTC_REG_NODEMAP + i * 4);
 159                 }
 160
 161                 for (i = 0; i < eiointc_priv[0]->vec_count / 32 / 4; i++) {
 162                         bit = BIT(1 + index); /* Route to IP[1 + index] */
 163                         data = bit | (bit << 8) | (bit << 16) | (bit << 24);
 164                         iocsr_write32(data, EIOINTC_REG_IPMAP + i * 4);
 165                 }
 166
 167                 for (i = 0; i < eiointc_priv[0]->vec_count / 4; i++) {
 168                         /* Route to Node-0 Core-0 */
 169                         if (index == 0)
 170                                 bit = BIT(cpu_logical_map(0));
 171                         else
 172                                 bit = (eiointc_priv[index]->node << 4) | 1;
 173
 174                         data = bit | (bit << 8) | (bit << 16) | (bit << 24);
 175                         iocsr_write32(data, EIOINTC_REG_ROUTE + i * 4);
 176                 }
 177
 178                 for (i = 0; i < eiointc_priv[0]->vec_count / 32; i++) {
 179                         data = 0xffffffff;
 180                         iocsr_write32(data, EIOINTC_REG_ENABLE + i * 4);
 181                         iocsr_write32(data, EIOINTC_REG_BOUNCE + i * 4);
 182                 }
 183         }
 184
 185         return 0;
 186 }
 187
 188 static void eiointc_irq_dispatch(struct irq_desc *desc)
 189 {
 190         int i;
 191         u64 pending;
 192         bool handled = false;
 193         struct irq_chip *chip = irq_desc_get_chip(desc);
 194         struct eiointc_priv *priv = irq_desc_get_handler_data(desc);
 195
 196         chained_irq_enter(chip, desc);
 197
 198         for (i = 0; i < eiointc_priv[0]->vec_count / VEC_COUNT_PER_REG; i++) {
 199                 pending = iocsr_read64(EIOINTC_REG_ISR + (i << 3));
 200
 201                 /* Skip handling if pending bitmap is zero */
 202                 if (!pending)
 203                         continue;
 204
 205                 /* Clear the IRQs */
 206                 iocsr_write64(pending, EIOINTC_REG_ISR + (i << 3));
 207                 while (pending) {
 208                         int bit = __ffs(pending);
 209                         int irq = bit + VEC_COUNT_PER_REG * i;
 210
 211                         generic_handle_domain_irq(priv->eiointc_domain, irq);
 212                         pending &= ~BIT(bit);
 213                         handled = true;
 214                 }
 215         }
 216
 217         if (!handled)
 218                 spurious_interrupt();
 219
 220         chained_irq_exit(chip, desc);
 221 }
 222
 223 static void eiointc_ack_irq(struct irq_data *d)
 224 {
 225 }
 226
 227 static void eiointc_mask_irq(struct irq_data *d)
 228 {
 229 }
 230
 231 static void eiointc_unmask_irq(struct irq_data *d)
 232 {
 233 }
 234
 235 static struct irq_chip eiointc_irq_chip = {
 236         .name                   = "EIOINTC",
 237         .irq_ack                = eiointc_ack_irq,
 238         .irq_mask               = eiointc_mask_irq,
 239         .irq_unmask             = eiointc_unmask_irq,
 240 #ifdef CONFIG_SMP
 241         .irq_set_affinity       = eiointc_set_irq_affinity,
 242 #endif
 243 };
 244
 245 static int eiointc_domain_alloc(struct irq_domain *domain, unsigned int virq,
 246                                 unsigned int nr_irqs, void *arg)
 247 {
 248         int ret;
 249         unsigned int i, type;
 250         unsigned long hwirq = 0;
 251         struct eiointc_priv *priv = domain->host_data;
 252
 253         ret = irq_domain_translate_onecell(domain, arg, &hwirq, &type);
 254         if (ret)
 255                 return ret;
 256
 257         for (i = 0; i < nr_irqs; i++) {
 258                 irq_domain_set_info(domain, virq + i, hwirq + i, &eiointc_irq_chip,
 259                                         priv, handle_edge_irq, NULL, NULL);
 260         }
 261
 262         return 0;
 263 }
 264
 265 static void eiointc_domain_free(struct irq_domain *domain, unsigned int virq,
 266                                 unsigned int nr_irqs)
 267 {
 268         int i;
 269
 270         for (i = 0; i < nr_irqs; i++) {
 271                 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
 272
 273                 irq_set_handler(virq + i, NULL);
 274                 irq_domain_reset_irq_data(d);
 275         }
 276 }
 277
 278 static const struct irq_domain_ops eiointc_domain_ops = {
 279         .translate      = irq_domain_translate_onecell,
 280         .alloc          = eiointc_domain_alloc,
 281         .free           = eiointc_domain_free,
 282 };
 283
 284 static void acpi_set_vec_parent(int node, struct irq_domain *parent, struct acpi_vector_group *vec_group)
 285 {
 286         int i;
 287
 288         for (i = 0; i < MAX_IO_PICS; i++) {
 289                 if (node == vec_group[i].node) {
 290                         vec_group[i].parent = parent;
 291                         return;
 292                 }
 293         }
 294 }
 295
 296 static struct irq_domain *acpi_get_vec_parent(int node, struct acpi_vector_group *vec_group)
 297 {
 298         int i;
 299
 300         for (i = 0; i < MAX_IO_PICS; i++) {
 301                 if (node == vec_group[i].node)
 302                         return vec_group[i].parent;
 303         }
 304         return NULL;
 305 }
 306
 307 static int eiointc_suspend(void)
 308 {
 309         return 0;
 310 }
 311
 312 static void eiointc_resume(void)
 313 {
 314         eiointc_router_init(0);
 315 }
 316
 317 static struct syscore_ops eiointc_syscore_ops = {
 318         .suspend = eiointc_suspend,
 319         .resume = eiointc_resume,
 320 };
 321
 322 static int __init pch_pic_parse_madt(union acpi_subtable_headers *header,
 323                                         const unsigned long end)
 324 {
 325         struct acpi_madt_bio_pic *pchpic_entry = (struct acpi_madt_bio_pic *)header;
 326         unsigned int node = (pchpic_entry->address >> 44) & 0xf;
 327         struct irq_domain *parent = acpi_get_vec_parent(node, pch_group);
 328
 329         if (parent)
 330                 return pch_pic_acpi_init(parent, pchpic_entry);
 331
 332         return 0;
 333 }
 334
 335 static int __init pch_msi_parse_madt(union acpi_subtable_headers *header,
 336                                         const unsigned long end)
 337 {
 338         struct irq_domain *parent;
 339         struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header;
 340         int node;
 341
 342         if (cpu_has_flatmode)
 343                 node = early_cpu_to_node(eiointc_priv[nr_pics - 1]->node * CORES_PER_EIO_NODE);
 344         else
 345                 node = eiointc_priv[nr_pics - 1]->node;
 346
 347         parent = acpi_get_vec_parent(node, msi_group);
 348
 349         if (parent)
 350                 return pch_msi_acpi_init(parent, pchmsi_entry);
 351
 352         return 0;
 353 }
 354
 355 static int __init acpi_cascade_irqdomain_init(void)
 356 {
 357         int r;
 358
 359         r = acpi_table_parse_madt(ACPI_MADT_TYPE_BIO_PIC, pch_pic_parse_madt, 0);
 360         if (r < 0)
 361                 return r;
 362
 363         r = acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1);
 364         if (r < 0)
 365                 return r;
 366
 367         return 0;
 368 }
 369
 370 static int __init eiointc_init(struct eiointc_priv *priv, int parent_irq,
 371                                u64 node_map)
 372 {
 373         int i;
 374
 375         node_map = node_map ? node_map : -1ULL;
 376         for_each_possible_cpu(i) {
 377                 if (node_map & (1ULL << (cpu_to_eio_node(i)))) {
 378                         node_set(cpu_to_eio_node(i), priv->node_map);
 379                         cpumask_or(&priv->cpuspan_map, &priv->cpuspan_map,
 380                                    cpumask_of(i));
 381                 }
 382         }
 383
 384         priv->eiointc_domain = irq_domain_create_linear(priv->domain_handle,
 385                                                         priv->vec_count,
 386                                                         &eiointc_domain_ops,
 387                                                         priv);
 388         if (!priv->eiointc_domain) {
 389                 pr_err("loongson-extioi: cannot add IRQ domain\n");
 390                 return -ENOMEM;
 391         }
 392
 393         eiointc_priv[nr_pics++] = priv;
 394         eiointc_router_init(0);
 395         irq_set_chained_handler_and_data(parent_irq, eiointc_irq_dispatch, priv);
 396
 397         if (nr_pics == 1) {
 398                 register_syscore_ops(&eiointc_syscore_ops);
 399                 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_LOONGARCH_STARTING,
 400                                           "irqchip/loongarch/intc:starting",
 401                                           eiointc_router_init, NULL);
 402         }
 403
 404         return 0;
 405 }
 406
 407 int __init eiointc_acpi_init(struct irq_domain *parent,
 408                                      struct acpi_madt_eio_pic *acpi_eiointc)
 409 {
 410         int parent_irq, ret;
 411         struct eiointc_priv *priv;
 412         int node;
 413
 414         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 415         if (!priv)
 416                 return -ENOMEM;
 417
 418         priv->domain_handle = irq_domain_alloc_named_id_fwnode("EIOPIC",
 419                                                                acpi_eiointc->node);
 420         if (!priv->domain_handle) {
 421                 pr_err("Unable to allocate domain handle\n");
 422                 goto out_free_priv;
 423         }
 424
 425         priv->vec_count = VEC_COUNT;
 426         priv->node = acpi_eiointc->node;
 427
 428         parent_irq = irq_create_mapping(parent, acpi_eiointc->cascade);
 429
 430         ret = eiointc_init(priv, parent_irq, acpi_eiointc->node_map);
 431         if (ret < 0)
 432                 goto out_free_handle;
 433
 434         if (cpu_has_flatmode)
 435                 node = early_cpu_to_node(acpi_eiointc->node * CORES_PER_EIO_NODE);
 436         else
 437                 node = acpi_eiointc->node;
 438         acpi_set_vec_parent(node, priv->eiointc_domain, pch_group);
 439         acpi_set_vec_parent(node, priv->eiointc_domain, msi_group);
 440
 441         ret = acpi_cascade_irqdomain_init();
 442         if (ret < 0)
 443                 goto out_free_handle;
 444
 445         return ret;
 446
 447 out_free_handle:
 448         irq_domain_free_fwnode(priv->domain_handle);
 449         priv->domain_handle = NULL;
 450 out_free_priv:
 451         kfree(priv);
 452
 453         return -ENOMEM;
 454 }
 455
 456 static int __init eiointc_of_init(struct device_node *of_node,
 457                                   struct device_node *parent)
 458 {
 459         int parent_irq, ret;
 460         struct eiointc_priv *priv;
 461
 462         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 463         if (!priv)
 464                 return -ENOMEM;
 465
 466         parent_irq = irq_of_parse_and_map(of_node, 0);
 467         if (parent_irq <= 0) {
 468                 ret = -ENODEV;
 469                 goto out_free_priv;
 470         }
 471
 472         ret = irq_set_handler_data(parent_irq, priv);
 473         if (ret < 0)
 474                 goto out_free_priv;
 475
 476         /*
 477          * In particular, the number of devices supported by the LS2K0500
 478          * extended I/O interrupt vector is 128.
 479          */
 480         if (of_device_is_compatible(of_node, "loongson,ls2k0500-eiointc"))
 481                 priv->vec_count = 128;
 482         else
 483                 priv->vec_count = VEC_COUNT;
 484
 485         priv->node = 0;
 486         priv->domain_handle = of_node_to_fwnode(of_node);
 487
 488         ret = eiointc_init(priv, parent_irq, 0);
 489         if (ret < 0)
 490                 goto out_free_priv;
 491
 492         return 0;
 493
 494 out_free_priv:
 495         kfree(priv);
 496         return ret;
 497 }
 498
 499 IRQCHIP_DECLARE(loongson_ls2k0500_eiointc, "loongson,ls2k0500-eiointc", eiointc_of_init);
 500 IRQCHIP_DECLARE(loongson_ls2k2000_eiointc, "loongson,ls2k2000-eiointc", eiointc_of_init);