Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
d5de8841 JF |
2 | /* |
3 | * Split spinlock implementation out into its own file, so it can be | |
4 | * compiled in a FTRACE-compatible way. | |
5 | */ | |
f2a5fef1 | 6 | #include <linux/kernel.h> |
d5de8841 | 7 | #include <linux/spinlock.h> |
354e7b76 | 8 | #include <linux/slab.h> |
d3132b38 | 9 | #include <linux/atomic.h> |
d5de8841 JF |
10 | |
11 | #include <asm/paravirt.h> | |
e6fd28eb | 12 | #include <asm/qspinlock.h> |
d5de8841 | 13 | |
d5de8841 JF |
14 | #include <xen/events.h> |
15 | ||
16 | #include "xen-ops.h" | |
994025ca | 17 | |
e95e6f17 DV |
18 | static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; |
19 | static DEFINE_PER_CPU(char *, irq_name); | |
d3132b38 | 20 | static DEFINE_PER_CPU(atomic_t, xen_qlock_wait_nest); |
e95e6f17 DV |
21 | static bool xen_pvspin = true; |
22 | ||
e95e6f17 DV |
23 | static void xen_qlock_kick(int cpu) |
24 | { | |
707e59ba RL |
25 | int irq = per_cpu(lock_kicker_irq, cpu); |
26 | ||
27 | /* Don't kick if the target's kicker interrupt is not initialized. */ | |
28 | if (irq == -1) | |
29 | return; | |
30 | ||
e95e6f17 DV |
31 | xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); |
32 | } | |
33 | ||
34 | /* | |
35 | * Halt the current CPU & release it back to the host | |
36 | */ | |
37 | static void xen_qlock_wait(u8 *byte, u8 val) | |
38 | { | |
39 | int irq = __this_cpu_read(lock_kicker_irq); | |
d3132b38 | 40 | atomic_t *nest_cnt = this_cpu_ptr(&xen_qlock_wait_nest); |
e95e6f17 DV |
41 | |
42 | /* If kicker interrupts not initialized yet, just spin */ | |
a8565319 | 43 | if (irq == -1 || in_nmi()) |
e95e6f17 DV |
44 | return; |
45 | ||
d3132b38 JG |
46 | /* Detect reentry. */ |
47 | atomic_inc(nest_cnt); | |
a8565319 | 48 | |
d3132b38 JG |
49 | /* If irq pending already and no nested call clear it. */ |
50 | if (atomic_read(nest_cnt) == 1 && xen_test_irq_pending(irq)) { | |
2ac2a7d4 | 51 | xen_clear_irq_pending(irq); |
a8565319 JG |
52 | } else if (READ_ONCE(*byte) == val) { |
53 | /* Block until irq becomes pending (or a spurious wakeup) */ | |
54 | xen_poll_irq(irq); | |
2ac2a7d4 | 55 | } |
e95e6f17 | 56 | |
d3132b38 | 57 | atomic_dec(nest_cnt); |
e95e6f17 DV |
58 | } |
59 | ||
d5de8841 JF |
60 | static irqreturn_t dummy_handler(int irq, void *dev_id) |
61 | { | |
62 | BUG(); | |
63 | return IRQ_HANDLED; | |
64 | } | |
65 | ||
148f9bb8 | 66 | void xen_init_lock_cpu(int cpu) |
d5de8841 JF |
67 | { |
68 | int irq; | |
354e7b76 | 69 | char *name; |
d5de8841 | 70 | |
090d54bc | 71 | if (!xen_pvspin) |
3310bbed KRW |
72 | return; |
73 | ||
cb91f8f4 | 74 | WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n", |
cb9c6f15 KRW |
75 | cpu, per_cpu(lock_kicker_irq, cpu)); |
76 | ||
d5de8841 JF |
77 | name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); |
78 | irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, | |
79 | cpu, | |
80 | dummy_handler, | |
9d71cee6 | 81 | IRQF_PERCPU|IRQF_NOBALANCING, |
d5de8841 JF |
82 | name, |
83 | NULL); | |
84 | ||
85 | if (irq >= 0) { | |
86 | disable_irq(irq); /* make sure it's never delivered */ | |
87 | per_cpu(lock_kicker_irq, cpu) = irq; | |
354e7b76 | 88 | per_cpu(irq_name, cpu) = name; |
d5de8841 JF |
89 | } |
90 | ||
91 | printk("cpu %d spinlock event irq %d\n", cpu, irq); | |
92 | } | |
93 | ||
d68d82af AN |
94 | void xen_uninit_lock_cpu(int cpu) |
95 | { | |
3310bbed KRW |
96 | if (!xen_pvspin) |
97 | return; | |
98 | ||
d68d82af | 99 | unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL); |
cb9c6f15 | 100 | per_cpu(lock_kicker_irq, cpu) = -1; |
354e7b76 KRW |
101 | kfree(per_cpu(irq_name, cpu)); |
102 | per_cpu(irq_name, cpu) = NULL; | |
d68d82af AN |
103 | } |
104 | ||
3cded417 PZ |
105 | PV_CALLEE_SAVE_REGS_THUNK(xen_vcpu_stolen); |
106 | ||
a945928e KRW |
107 | /* |
108 | * Our init of PV spinlocks is split in two init functions due to us | |
109 | * using paravirt patching and jump labels patching and having to do | |
110 | * all of this before SMP code is invoked. | |
111 | * | |
112 | * The paravirt patching needs to be done _before_ the alternative asm code | |
113 | * is started, otherwise we would not patch the core kernel code. | |
114 | */ | |
d5de8841 JF |
115 | void __init xen_init_spinlocks(void) |
116 | { | |
70dd4998 | 117 | |
47b428d1 WL |
118 | /* Don't need to use pvqspinlock code if there is only 1 vCPU. */ |
119 | if (num_possible_cpus() == 1) | |
120 | xen_pvspin = false; | |
121 | ||
b8fa70b5 JF |
122 | if (!xen_pvspin) { |
123 | printk(KERN_DEBUG "xen: PV spinlocks disabled\n"); | |
090d54bc | 124 | static_branch_disable(&virt_spin_lock_key); |
b8fa70b5 JF |
125 | return; |
126 | } | |
e0fc17a9 | 127 | printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); |
cfd8983f | 128 | |
e95e6f17 | 129 | __pv_init_lock_hash(); |
5c83511b JG |
130 | pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; |
131 | pv_ops.lock.queued_spin_unlock = | |
132 | PV_CALLEE_SAVE(__pv_queued_spin_unlock); | |
133 | pv_ops.lock.wait = xen_qlock_wait; | |
134 | pv_ops.lock.kick = xen_qlock_kick; | |
135 | pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); | |
d5de8841 | 136 | } |
994025ca | 137 | |
b8fa70b5 JF |
138 | static __init int xen_parse_nopvspin(char *arg) |
139 | { | |
140 | xen_pvspin = false; | |
141 | return 0; | |
142 | } | |
143 | early_param("xen_nopvspin", xen_parse_nopvspin); | |
144 |