| 1 | /* |
| 2 | * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar |
| 3 | * Copyright (C) 2005-2006, Thomas Gleixner, Russell King |
| 4 | * |
| 5 | * This file contains the interrupt descriptor management code |
| 6 | * |
| 7 | * Detailed information is available in Documentation/DocBook/genericirq |
| 8 | * |
| 9 | */ |
| 10 | #include <linux/irq.h> |
| 11 | #include <linux/slab.h> |
| 12 | #include <linux/export.h> |
| 13 | #include <linux/interrupt.h> |
| 14 | #include <linux/kernel_stat.h> |
| 15 | #include <linux/radix-tree.h> |
| 16 | #include <linux/bitmap.h> |
| 17 | #include <linux/irqdomain.h> |
| 18 | |
| 19 | #include "internals.h" |
| 20 | |
| 21 | /* |
| 22 | * lockdep: we want to handle all irq_desc locks as a single lock-class: |
| 23 | */ |
| 24 | static struct lock_class_key irq_desc_lock_class; |
| 25 | |
| 26 | #if defined(CONFIG_SMP) |
| 27 | static int __init irq_affinity_setup(char *str) |
| 28 | { |
| 29 | zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT); |
| 30 | cpulist_parse(str, irq_default_affinity); |
| 31 | /* |
| 32 | * Set at least the boot cpu. We don't want to end up with |
| 33 | * bugreports caused by random comandline masks |
| 34 | */ |
| 35 | cpumask_set_cpu(smp_processor_id(), irq_default_affinity); |
| 36 | return 1; |
| 37 | } |
| 38 | __setup("irqaffinity=", irq_affinity_setup); |
| 39 | |
| 40 | static void __init init_irq_default_affinity(void) |
| 41 | { |
| 42 | #ifdef CONFIG_CPUMASK_OFFSTACK |
| 43 | if (!irq_default_affinity) |
| 44 | zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT); |
| 45 | #endif |
| 46 | if (cpumask_empty(irq_default_affinity)) |
| 47 | cpumask_setall(irq_default_affinity); |
| 48 | } |
| 49 | #else |
| 50 | static void __init init_irq_default_affinity(void) |
| 51 | { |
| 52 | } |
| 53 | #endif |
| 54 | |
| 55 | #ifdef CONFIG_SMP |
| 56 | static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) |
| 57 | { |
| 58 | if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity, |
| 59 | gfp, node)) |
| 60 | return -ENOMEM; |
| 61 | |
| 62 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 63 | if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) { |
| 64 | free_cpumask_var(desc->irq_common_data.affinity); |
| 65 | return -ENOMEM; |
| 66 | } |
| 67 | #endif |
| 68 | return 0; |
| 69 | } |
| 70 | |
| 71 | static void desc_smp_init(struct irq_desc *desc, int node) |
| 72 | { |
| 73 | cpumask_copy(desc->irq_common_data.affinity, irq_default_affinity); |
| 74 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 75 | cpumask_clear(desc->pending_mask); |
| 76 | #endif |
| 77 | #ifdef CONFIG_NUMA |
| 78 | desc->irq_common_data.node = node; |
| 79 | #endif |
| 80 | } |
| 81 | |
| 82 | #else |
| 83 | static inline int |
| 84 | alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; } |
| 85 | static inline void desc_smp_init(struct irq_desc *desc, int node) { } |
| 86 | #endif |
| 87 | |
| 88 | static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, |
| 89 | struct module *owner) |
| 90 | { |
| 91 | int cpu; |
| 92 | |
| 93 | desc->irq_common_data.handler_data = NULL; |
| 94 | desc->irq_common_data.msi_desc = NULL; |
| 95 | |
| 96 | desc->irq_data.common = &desc->irq_common_data; |
| 97 | desc->irq_data.irq = irq; |
| 98 | desc->irq_data.chip = &no_irq_chip; |
| 99 | desc->irq_data.chip_data = NULL; |
| 100 | irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS); |
| 101 | irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED); |
| 102 | desc->handle_irq = handle_bad_irq; |
| 103 | desc->depth = 1; |
| 104 | desc->irq_count = 0; |
| 105 | desc->irqs_unhandled = 0; |
| 106 | desc->name = NULL; |
| 107 | desc->owner = owner; |
| 108 | for_each_possible_cpu(cpu) |
| 109 | *per_cpu_ptr(desc->kstat_irqs, cpu) = 0; |
| 110 | desc_smp_init(desc, node); |
| 111 | } |
| 112 | |
| 113 | int nr_irqs = NR_IRQS; |
| 114 | EXPORT_SYMBOL_GPL(nr_irqs); |
| 115 | |
| 116 | static DEFINE_MUTEX(sparse_irq_lock); |
| 117 | static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS); |
| 118 | |
| 119 | #ifdef CONFIG_SPARSE_IRQ |
| 120 | |
| 121 | static RADIX_TREE(irq_desc_tree, GFP_KERNEL); |
| 122 | |
| 123 | static void irq_insert_desc(unsigned int irq, struct irq_desc *desc) |
| 124 | { |
| 125 | radix_tree_insert(&irq_desc_tree, irq, desc); |
| 126 | } |
| 127 | |
| 128 | struct irq_desc *irq_to_desc(unsigned int irq) |
| 129 | { |
| 130 | return radix_tree_lookup(&irq_desc_tree, irq); |
| 131 | } |
| 132 | EXPORT_SYMBOL(irq_to_desc); |
| 133 | |
| 134 | static void delete_irq_desc(unsigned int irq) |
| 135 | { |
| 136 | radix_tree_delete(&irq_desc_tree, irq); |
| 137 | } |
| 138 | |
| 139 | #ifdef CONFIG_SMP |
| 140 | static void free_masks(struct irq_desc *desc) |
| 141 | { |
| 142 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 143 | free_cpumask_var(desc->pending_mask); |
| 144 | #endif |
| 145 | free_cpumask_var(desc->irq_common_data.affinity); |
| 146 | } |
| 147 | #else |
| 148 | static inline void free_masks(struct irq_desc *desc) { } |
| 149 | #endif |
| 150 | |
| 151 | void irq_lock_sparse(void) |
| 152 | { |
| 153 | mutex_lock(&sparse_irq_lock); |
| 154 | } |
| 155 | |
| 156 | void irq_unlock_sparse(void) |
| 157 | { |
| 158 | mutex_unlock(&sparse_irq_lock); |
| 159 | } |
| 160 | |
| 161 | static struct irq_desc *alloc_desc(int irq, int node, struct module *owner) |
| 162 | { |
| 163 | struct irq_desc *desc; |
| 164 | gfp_t gfp = GFP_KERNEL; |
| 165 | |
| 166 | desc = kzalloc_node(sizeof(*desc), gfp, node); |
| 167 | if (!desc) |
| 168 | return NULL; |
| 169 | /* allocate based on nr_cpu_ids */ |
| 170 | desc->kstat_irqs = alloc_percpu(unsigned int); |
| 171 | if (!desc->kstat_irqs) |
| 172 | goto err_desc; |
| 173 | |
| 174 | if (alloc_masks(desc, gfp, node)) |
| 175 | goto err_kstat; |
| 176 | |
| 177 | raw_spin_lock_init(&desc->lock); |
| 178 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); |
| 179 | init_rcu_head(&desc->rcu); |
| 180 | |
| 181 | desc_set_defaults(irq, desc, node, owner); |
| 182 | |
| 183 | return desc; |
| 184 | |
| 185 | err_kstat: |
| 186 | free_percpu(desc->kstat_irqs); |
| 187 | err_desc: |
| 188 | kfree(desc); |
| 189 | return NULL; |
| 190 | } |
| 191 | |
| 192 | static void delayed_free_desc(struct rcu_head *rhp) |
| 193 | { |
| 194 | struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu); |
| 195 | |
| 196 | free_masks(desc); |
| 197 | free_percpu(desc->kstat_irqs); |
| 198 | kfree(desc); |
| 199 | } |
| 200 | |
| 201 | static void free_desc(unsigned int irq) |
| 202 | { |
| 203 | struct irq_desc *desc = irq_to_desc(irq); |
| 204 | |
| 205 | unregister_irq_proc(irq, desc); |
| 206 | |
| 207 | /* |
| 208 | * sparse_irq_lock protects also show_interrupts() and |
| 209 | * kstat_irq_usr(). Once we deleted the descriptor from the |
| 210 | * sparse tree we can free it. Access in proc will fail to |
| 211 | * lookup the descriptor. |
| 212 | */ |
| 213 | mutex_lock(&sparse_irq_lock); |
| 214 | delete_irq_desc(irq); |
| 215 | mutex_unlock(&sparse_irq_lock); |
| 216 | |
| 217 | /* |
| 218 | * We free the descriptor, masks and stat fields via RCU. That |
| 219 | * allows demultiplex interrupts to do rcu based management of |
| 220 | * the child interrupts. |
| 221 | */ |
| 222 | call_rcu(&desc->rcu, delayed_free_desc); |
| 223 | } |
| 224 | |
| 225 | static int alloc_descs(unsigned int start, unsigned int cnt, int node, |
| 226 | struct module *owner) |
| 227 | { |
| 228 | struct irq_desc *desc; |
| 229 | int i; |
| 230 | |
| 231 | for (i = 0; i < cnt; i++) { |
| 232 | desc = alloc_desc(start + i, node, owner); |
| 233 | if (!desc) |
| 234 | goto err; |
| 235 | mutex_lock(&sparse_irq_lock); |
| 236 | irq_insert_desc(start + i, desc); |
| 237 | mutex_unlock(&sparse_irq_lock); |
| 238 | } |
| 239 | return start; |
| 240 | |
| 241 | err: |
| 242 | for (i--; i >= 0; i--) |
| 243 | free_desc(start + i); |
| 244 | |
| 245 | mutex_lock(&sparse_irq_lock); |
| 246 | bitmap_clear(allocated_irqs, start, cnt); |
| 247 | mutex_unlock(&sparse_irq_lock); |
| 248 | return -ENOMEM; |
| 249 | } |
| 250 | |
| 251 | static int irq_expand_nr_irqs(unsigned int nr) |
| 252 | { |
| 253 | if (nr > IRQ_BITMAP_BITS) |
| 254 | return -ENOMEM; |
| 255 | nr_irqs = nr; |
| 256 | return 0; |
| 257 | } |
| 258 | |
| 259 | int __init early_irq_init(void) |
| 260 | { |
| 261 | int i, initcnt, node = first_online_node; |
| 262 | struct irq_desc *desc; |
| 263 | |
| 264 | init_irq_default_affinity(); |
| 265 | |
| 266 | /* Let arch update nr_irqs and return the nr of preallocated irqs */ |
| 267 | initcnt = arch_probe_nr_irqs(); |
| 268 | printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt); |
| 269 | |
| 270 | if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS)) |
| 271 | nr_irqs = IRQ_BITMAP_BITS; |
| 272 | |
| 273 | if (WARN_ON(initcnt > IRQ_BITMAP_BITS)) |
| 274 | initcnt = IRQ_BITMAP_BITS; |
| 275 | |
| 276 | if (initcnt > nr_irqs) |
| 277 | nr_irqs = initcnt; |
| 278 | |
| 279 | for (i = 0; i < initcnt; i++) { |
| 280 | desc = alloc_desc(i, node, NULL); |
| 281 | set_bit(i, allocated_irqs); |
| 282 | irq_insert_desc(i, desc); |
| 283 | } |
| 284 | return arch_early_irq_init(); |
| 285 | } |
| 286 | |
| 287 | #else /* !CONFIG_SPARSE_IRQ */ |
| 288 | |
| 289 | struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { |
| 290 | [0 ... NR_IRQS-1] = { |
| 291 | .handle_irq = handle_bad_irq, |
| 292 | .depth = 1, |
| 293 | .lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock), |
| 294 | } |
| 295 | }; |
| 296 | |
| 297 | int __init early_irq_init(void) |
| 298 | { |
| 299 | int count, i, node = first_online_node; |
| 300 | struct irq_desc *desc; |
| 301 | |
| 302 | init_irq_default_affinity(); |
| 303 | |
| 304 | printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS); |
| 305 | |
| 306 | desc = irq_desc; |
| 307 | count = ARRAY_SIZE(irq_desc); |
| 308 | |
| 309 | for (i = 0; i < count; i++) { |
| 310 | desc[i].kstat_irqs = alloc_percpu(unsigned int); |
| 311 | alloc_masks(&desc[i], GFP_KERNEL, node); |
| 312 | raw_spin_lock_init(&desc[i].lock); |
| 313 | lockdep_set_class(&desc[i].lock, &irq_desc_lock_class); |
| 314 | desc_set_defaults(i, &desc[i], node, NULL); |
| 315 | } |
| 316 | return arch_early_irq_init(); |
| 317 | } |
| 318 | |
| 319 | struct irq_desc *irq_to_desc(unsigned int irq) |
| 320 | { |
| 321 | return (irq < NR_IRQS) ? irq_desc + irq : NULL; |
| 322 | } |
| 323 | EXPORT_SYMBOL(irq_to_desc); |
| 324 | |
| 325 | static void free_desc(unsigned int irq) |
| 326 | { |
| 327 | struct irq_desc *desc = irq_to_desc(irq); |
| 328 | unsigned long flags; |
| 329 | |
| 330 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 331 | desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL); |
| 332 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 333 | } |
| 334 | |
| 335 | static inline int alloc_descs(unsigned int start, unsigned int cnt, int node, |
| 336 | struct module *owner) |
| 337 | { |
| 338 | u32 i; |
| 339 | |
| 340 | for (i = 0; i < cnt; i++) { |
| 341 | struct irq_desc *desc = irq_to_desc(start + i); |
| 342 | |
| 343 | desc->owner = owner; |
| 344 | } |
| 345 | return start; |
| 346 | } |
| 347 | |
| 348 | static int irq_expand_nr_irqs(unsigned int nr) |
| 349 | { |
| 350 | return -ENOMEM; |
| 351 | } |
| 352 | |
| 353 | void irq_mark_irq(unsigned int irq) |
| 354 | { |
| 355 | mutex_lock(&sparse_irq_lock); |
| 356 | bitmap_set(allocated_irqs, irq, 1); |
| 357 | mutex_unlock(&sparse_irq_lock); |
| 358 | } |
| 359 | |
| 360 | #ifdef CONFIG_GENERIC_IRQ_LEGACY |
| 361 | void irq_init_desc(unsigned int irq) |
| 362 | { |
| 363 | free_desc(irq); |
| 364 | } |
| 365 | #endif |
| 366 | |
| 367 | #endif /* !CONFIG_SPARSE_IRQ */ |
| 368 | |
| 369 | /** |
| 370 | * generic_handle_irq - Invoke the handler for a particular irq |
| 371 | * @irq: The irq number to handle |
| 372 | * |
| 373 | */ |
| 374 | int generic_handle_irq(unsigned int irq) |
| 375 | { |
| 376 | struct irq_desc *desc = irq_to_desc(irq); |
| 377 | |
| 378 | if (!desc) |
| 379 | return -EINVAL; |
| 380 | generic_handle_irq_desc(desc); |
| 381 | return 0; |
| 382 | } |
| 383 | EXPORT_SYMBOL_GPL(generic_handle_irq); |
| 384 | |
| 385 | #ifdef CONFIG_HANDLE_DOMAIN_IRQ |
| 386 | /** |
| 387 | * __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain |
| 388 | * @domain: The domain where to perform the lookup |
| 389 | * @hwirq: The HW irq number to convert to a logical one |
| 390 | * @lookup: Whether to perform the domain lookup or not |
| 391 | * @regs: Register file coming from the low-level handling code |
| 392 | * |
| 393 | * Returns: 0 on success, or -EINVAL if conversion has failed |
| 394 | */ |
| 395 | int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq, |
| 396 | bool lookup, struct pt_regs *regs) |
| 397 | { |
| 398 | struct pt_regs *old_regs = set_irq_regs(regs); |
| 399 | unsigned int irq = hwirq; |
| 400 | int ret = 0; |
| 401 | |
| 402 | irq_enter(); |
| 403 | |
| 404 | #ifdef CONFIG_IRQ_DOMAIN |
| 405 | if (lookup) |
| 406 | irq = irq_find_mapping(domain, hwirq); |
| 407 | #endif |
| 408 | |
| 409 | /* |
| 410 | * Some hardware gives randomly wrong interrupts. Rather |
| 411 | * than crashing, do something sensible. |
| 412 | */ |
| 413 | if (unlikely(!irq || irq >= nr_irqs)) { |
| 414 | ack_bad_irq(irq); |
| 415 | ret = -EINVAL; |
| 416 | } else { |
| 417 | generic_handle_irq(irq); |
| 418 | } |
| 419 | |
| 420 | irq_exit(); |
| 421 | set_irq_regs(old_regs); |
| 422 | return ret; |
| 423 | } |
| 424 | #endif |
| 425 | |
| 426 | /* Dynamic interrupt handling */ |
| 427 | |
| 428 | /** |
| 429 | * irq_free_descs - free irq descriptors |
| 430 | * @from: Start of descriptor range |
| 431 | * @cnt: Number of consecutive irqs to free |
| 432 | */ |
| 433 | void irq_free_descs(unsigned int from, unsigned int cnt) |
| 434 | { |
| 435 | int i; |
| 436 | |
| 437 | if (from >= nr_irqs || (from + cnt) > nr_irqs) |
| 438 | return; |
| 439 | |
| 440 | for (i = 0; i < cnt; i++) |
| 441 | free_desc(from + i); |
| 442 | |
| 443 | mutex_lock(&sparse_irq_lock); |
| 444 | bitmap_clear(allocated_irqs, from, cnt); |
| 445 | mutex_unlock(&sparse_irq_lock); |
| 446 | } |
| 447 | EXPORT_SYMBOL_GPL(irq_free_descs); |
| 448 | |
| 449 | /** |
| 450 | * irq_alloc_descs - allocate and initialize a range of irq descriptors |
| 451 | * @irq: Allocate for specific irq number if irq >= 0 |
| 452 | * @from: Start the search from this irq number |
| 453 | * @cnt: Number of consecutive irqs to allocate. |
| 454 | * @node: Preferred node on which the irq descriptor should be allocated |
| 455 | * @owner: Owning module (can be NULL) |
| 456 | * |
| 457 | * Returns the first irq number or error code |
| 458 | */ |
| 459 | int __ref |
| 460 | __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node, |
| 461 | struct module *owner) |
| 462 | { |
| 463 | int start, ret; |
| 464 | |
| 465 | if (!cnt) |
| 466 | return -EINVAL; |
| 467 | |
| 468 | if (irq >= 0) { |
| 469 | if (from > irq) |
| 470 | return -EINVAL; |
| 471 | from = irq; |
| 472 | } else { |
| 473 | /* |
| 474 | * For interrupts which are freely allocated the |
| 475 | * architecture can force a lower bound to the @from |
| 476 | * argument. x86 uses this to exclude the GSI space. |
| 477 | */ |
| 478 | from = arch_dynirq_lower_bound(from); |
| 479 | } |
| 480 | |
| 481 | mutex_lock(&sparse_irq_lock); |
| 482 | |
| 483 | start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS, |
| 484 | from, cnt, 0); |
| 485 | ret = -EEXIST; |
| 486 | if (irq >=0 && start != irq) |
| 487 | goto err; |
| 488 | |
| 489 | if (start + cnt > nr_irqs) { |
| 490 | ret = irq_expand_nr_irqs(start + cnt); |
| 491 | if (ret) |
| 492 | goto err; |
| 493 | } |
| 494 | |
| 495 | bitmap_set(allocated_irqs, start, cnt); |
| 496 | mutex_unlock(&sparse_irq_lock); |
| 497 | return alloc_descs(start, cnt, node, owner); |
| 498 | |
| 499 | err: |
| 500 | mutex_unlock(&sparse_irq_lock); |
| 501 | return ret; |
| 502 | } |
| 503 | EXPORT_SYMBOL_GPL(__irq_alloc_descs); |
| 504 | |
| 505 | #ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ |
| 506 | /** |
| 507 | * irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware |
| 508 | * @cnt: number of interrupts to allocate |
| 509 | * @node: node on which to allocate |
| 510 | * |
| 511 | * Returns an interrupt number > 0 or 0, if the allocation fails. |
| 512 | */ |
| 513 | unsigned int irq_alloc_hwirqs(int cnt, int node) |
| 514 | { |
| 515 | int i, irq = __irq_alloc_descs(-1, 0, cnt, node, NULL); |
| 516 | |
| 517 | if (irq < 0) |
| 518 | return 0; |
| 519 | |
| 520 | for (i = irq; cnt > 0; i++, cnt--) { |
| 521 | if (arch_setup_hwirq(i, node)) |
| 522 | goto err; |
| 523 | irq_clear_status_flags(i, _IRQ_NOREQUEST); |
| 524 | } |
| 525 | return irq; |
| 526 | |
| 527 | err: |
| 528 | for (i--; i >= irq; i--) { |
| 529 | irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE); |
| 530 | arch_teardown_hwirq(i); |
| 531 | } |
| 532 | irq_free_descs(irq, cnt); |
| 533 | return 0; |
| 534 | } |
| 535 | EXPORT_SYMBOL_GPL(irq_alloc_hwirqs); |
| 536 | |
| 537 | /** |
| 538 | * irq_free_hwirqs - Free irq descriptor and cleanup the hardware |
| 539 | * @from: Free from irq number |
| 540 | * @cnt: number of interrupts to free |
| 541 | * |
| 542 | */ |
| 543 | void irq_free_hwirqs(unsigned int from, int cnt) |
| 544 | { |
| 545 | int i, j; |
| 546 | |
| 547 | for (i = from, j = cnt; j > 0; i++, j--) { |
| 548 | irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE); |
| 549 | arch_teardown_hwirq(i); |
| 550 | } |
| 551 | irq_free_descs(from, cnt); |
| 552 | } |
| 553 | EXPORT_SYMBOL_GPL(irq_free_hwirqs); |
| 554 | #endif |
| 555 | |
| 556 | /** |
| 557 | * irq_get_next_irq - get next allocated irq number |
| 558 | * @offset: where to start the search |
| 559 | * |
| 560 | * Returns next irq number after offset or nr_irqs if none is found. |
| 561 | */ |
| 562 | unsigned int irq_get_next_irq(unsigned int offset) |
| 563 | { |
| 564 | return find_next_bit(allocated_irqs, nr_irqs, offset); |
| 565 | } |
| 566 | |
| 567 | struct irq_desc * |
| 568 | __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus, |
| 569 | unsigned int check) |
| 570 | { |
| 571 | struct irq_desc *desc = irq_to_desc(irq); |
| 572 | |
| 573 | if (desc) { |
| 574 | if (check & _IRQ_DESC_CHECK) { |
| 575 | if ((check & _IRQ_DESC_PERCPU) && |
| 576 | !irq_settings_is_per_cpu_devid(desc)) |
| 577 | return NULL; |
| 578 | |
| 579 | if (!(check & _IRQ_DESC_PERCPU) && |
| 580 | irq_settings_is_per_cpu_devid(desc)) |
| 581 | return NULL; |
| 582 | } |
| 583 | |
| 584 | if (bus) |
| 585 | chip_bus_lock(desc); |
| 586 | raw_spin_lock_irqsave(&desc->lock, *flags); |
| 587 | } |
| 588 | return desc; |
| 589 | } |
| 590 | |
| 591 | void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus) |
| 592 | { |
| 593 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 594 | if (bus) |
| 595 | chip_bus_sync_unlock(desc); |
| 596 | } |
| 597 | |
| 598 | int irq_set_percpu_devid_partition(unsigned int irq, |
| 599 | const struct cpumask *affinity) |
| 600 | { |
| 601 | struct irq_desc *desc = irq_to_desc(irq); |
| 602 | |
| 603 | if (!desc) |
| 604 | return -EINVAL; |
| 605 | |
| 606 | if (desc->percpu_enabled) |
| 607 | return -EINVAL; |
| 608 | |
| 609 | desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL); |
| 610 | |
| 611 | if (!desc->percpu_enabled) |
| 612 | return -ENOMEM; |
| 613 | |
| 614 | if (affinity) |
| 615 | desc->percpu_affinity = affinity; |
| 616 | else |
| 617 | desc->percpu_affinity = cpu_possible_mask; |
| 618 | |
| 619 | irq_set_percpu_devid_flags(irq); |
| 620 | return 0; |
| 621 | } |
| 622 | |
| 623 | int irq_set_percpu_devid(unsigned int irq) |
| 624 | { |
| 625 | return irq_set_percpu_devid_partition(irq, NULL); |
| 626 | } |
| 627 | |
| 628 | int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity) |
| 629 | { |
| 630 | struct irq_desc *desc = irq_to_desc(irq); |
| 631 | |
| 632 | if (!desc || !desc->percpu_enabled) |
| 633 | return -EINVAL; |
| 634 | |
| 635 | if (affinity) |
| 636 | cpumask_copy(affinity, desc->percpu_affinity); |
| 637 | |
| 638 | return 0; |
| 639 | } |
| 640 | |
| 641 | void kstat_incr_irq_this_cpu(unsigned int irq) |
| 642 | { |
| 643 | kstat_incr_irqs_this_cpu(irq_to_desc(irq)); |
| 644 | } |
| 645 | |
| 646 | /** |
| 647 | * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu |
| 648 | * @irq: The interrupt number |
| 649 | * @cpu: The cpu number |
| 650 | * |
| 651 | * Returns the sum of interrupt counts on @cpu since boot for |
| 652 | * @irq. The caller must ensure that the interrupt is not removed |
| 653 | * concurrently. |
| 654 | */ |
| 655 | unsigned int kstat_irqs_cpu(unsigned int irq, int cpu) |
| 656 | { |
| 657 | struct irq_desc *desc = irq_to_desc(irq); |
| 658 | |
| 659 | return desc && desc->kstat_irqs ? |
| 660 | *per_cpu_ptr(desc->kstat_irqs, cpu) : 0; |
| 661 | } |
| 662 | |
| 663 | /** |
| 664 | * kstat_irqs - Get the statistics for an interrupt |
| 665 | * @irq: The interrupt number |
| 666 | * |
| 667 | * Returns the sum of interrupt counts on all cpus since boot for |
| 668 | * @irq. The caller must ensure that the interrupt is not removed |
| 669 | * concurrently. |
| 670 | */ |
| 671 | unsigned int kstat_irqs(unsigned int irq) |
| 672 | { |
| 673 | struct irq_desc *desc = irq_to_desc(irq); |
| 674 | int cpu; |
| 675 | unsigned int sum = 0; |
| 676 | |
| 677 | if (!desc || !desc->kstat_irqs) |
| 678 | return 0; |
| 679 | for_each_possible_cpu(cpu) |
| 680 | sum += *per_cpu_ptr(desc->kstat_irqs, cpu); |
| 681 | return sum; |
| 682 | } |
| 683 | |
| 684 | /** |
| 685 | * kstat_irqs_usr - Get the statistics for an interrupt |
| 686 | * @irq: The interrupt number |
| 687 | * |
| 688 | * Returns the sum of interrupt counts on all cpus since boot for |
| 689 | * @irq. Contrary to kstat_irqs() this can be called from any |
| 690 | * preemptible context. It's protected against concurrent removal of |
| 691 | * an interrupt descriptor when sparse irqs are enabled. |
| 692 | */ |
| 693 | unsigned int kstat_irqs_usr(unsigned int irq) |
| 694 | { |
| 695 | unsigned int sum; |
| 696 | |
| 697 | irq_lock_sparse(); |
| 698 | sum = kstat_irqs(irq); |
| 699 | irq_unlock_sparse(); |
| 700 | return sum; |
| 701 | } |