Commit | Line | Data |
---|---|---|
d5de8841 JF |
1 | /* |
2 | * Split spinlock implementation out into its own file, so it can be | |
3 | * compiled in a FTRACE-compatible way. | |
4 | */ | |
5 | #include <linux/kernel_stat.h> | |
6 | #include <linux/spinlock.h> | |
994025ca JF |
7 | #include <linux/debugfs.h> |
8 | #include <linux/log2.h> | |
5a0e3ad6 | 9 | #include <linux/gfp.h> |
354e7b76 | 10 | #include <linux/slab.h> |
d5de8841 JF |
11 | |
12 | #include <asm/paravirt.h> | |
13 | ||
14 | #include <xen/interface/xen.h> | |
15 | #include <xen/events.h> | |
16 | ||
17 | #include "xen-ops.h" | |
994025ca JF |
18 | #include "debugfs.h" |
19 | ||
e95e6f17 DV |
20 | static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; |
21 | static DEFINE_PER_CPU(char *, irq_name); | |
22 | static bool xen_pvspin = true; | |
23 | ||
e95e6f17 DV |
24 | #include <asm/qspinlock.h> |
25 | ||
26 | static void xen_qlock_kick(int cpu) | |
27 | { | |
707e59ba RL |
28 | int irq = per_cpu(lock_kicker_irq, cpu); |
29 | ||
30 | /* Don't kick if the target's kicker interrupt is not initialized. */ | |
31 | if (irq == -1) | |
32 | return; | |
33 | ||
e95e6f17 DV |
34 | xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); |
35 | } | |
36 | ||
37 | /* | |
38 | * Halt the current CPU & release it back to the host | |
39 | */ | |
40 | static void xen_qlock_wait(u8 *byte, u8 val) | |
41 | { | |
42 | int irq = __this_cpu_read(lock_kicker_irq); | |
43 | ||
44 | /* If kicker interrupts not initialized yet, just spin */ | |
45 | if (irq == -1) | |
46 | return; | |
47 | ||
48 | /* clear pending */ | |
49 | xen_clear_irq_pending(irq); | |
50 | barrier(); | |
51 | ||
52 | /* | |
53 | * We check the byte value after clearing pending IRQ to make sure | |
54 | * that we won't miss a wakeup event because of the clearing. | |
55 | * | |
56 | * The sync_clear_bit() call in xen_clear_irq_pending() is atomic. | |
57 | * So it is effectively a memory barrier for x86. | |
58 | */ | |
59 | if (READ_ONCE(*byte) != val) | |
60 | return; | |
61 | ||
62 | /* | |
63 | * If an interrupt happens here, it will leave the wakeup irq | |
64 | * pending, which will cause xen_poll_irq() to return | |
65 | * immediately. | |
66 | */ | |
67 | ||
68 | /* Block until irq becomes pending (or perhaps a spurious wakeup) */ | |
69 | xen_poll_irq(irq); | |
70 | } | |
71 | ||
d5de8841 JF |
72 | static irqreturn_t dummy_handler(int irq, void *dev_id) |
73 | { | |
74 | BUG(); | |
75 | return IRQ_HANDLED; | |
76 | } | |
77 | ||
148f9bb8 | 78 | void xen_init_lock_cpu(int cpu) |
d5de8841 JF |
79 | { |
80 | int irq; | |
354e7b76 | 81 | char *name; |
d5de8841 | 82 | |
3310bbed KRW |
83 | if (!xen_pvspin) |
84 | return; | |
85 | ||
cb91f8f4 | 86 | WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n", |
cb9c6f15 KRW |
87 | cpu, per_cpu(lock_kicker_irq, cpu)); |
88 | ||
d5de8841 JF |
89 | name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); |
90 | irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, | |
91 | cpu, | |
92 | dummy_handler, | |
9d71cee6 | 93 | IRQF_PERCPU|IRQF_NOBALANCING, |
d5de8841 JF |
94 | name, |
95 | NULL); | |
96 | ||
97 | if (irq >= 0) { | |
98 | disable_irq(irq); /* make sure it's never delivered */ | |
99 | per_cpu(lock_kicker_irq, cpu) = irq; | |
354e7b76 | 100 | per_cpu(irq_name, cpu) = name; |
d5de8841 JF |
101 | } |
102 | ||
103 | printk("cpu %d spinlock event irq %d\n", cpu, irq); | |
104 | } | |
105 | ||
d68d82af AN |
106 | void xen_uninit_lock_cpu(int cpu) |
107 | { | |
3310bbed KRW |
108 | if (!xen_pvspin) |
109 | return; | |
110 | ||
d68d82af | 111 | unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL); |
cb9c6f15 | 112 | per_cpu(lock_kicker_irq, cpu) = -1; |
354e7b76 KRW |
113 | kfree(per_cpu(irq_name, cpu)); |
114 | per_cpu(irq_name, cpu) = NULL; | |
d68d82af AN |
115 | } |
116 | ||
3cded417 PZ |
117 | PV_CALLEE_SAVE_REGS_THUNK(xen_vcpu_stolen); |
118 | ||
a945928e KRW |
119 | /* |
120 | * Our init of PV spinlocks is split in two init functions due to us | |
121 | * using paravirt patching and jump labels patching and having to do | |
122 | * all of this before SMP code is invoked. | |
123 | * | |
124 | * The paravirt patching needs to be done _before_ the alternative asm code | |
125 | * is started, otherwise we would not patch the core kernel code. | |
126 | */ | |
d5de8841 JF |
127 | void __init xen_init_spinlocks(void) |
128 | { | |
70dd4998 | 129 | |
b8fa70b5 JF |
130 | if (!xen_pvspin) { |
131 | printk(KERN_DEBUG "xen: PV spinlocks disabled\n"); | |
132 | return; | |
133 | } | |
e0fc17a9 | 134 | printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); |
cfd8983f | 135 | |
e95e6f17 DV |
136 | __pv_init_lock_hash(); |
137 | pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; | |
138 | pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); | |
139 | pv_lock_ops.wait = xen_qlock_wait; | |
140 | pv_lock_ops.kick = xen_qlock_kick; | |
3cded417 | 141 | pv_lock_ops.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); |
d5de8841 | 142 | } |
994025ca | 143 | |
b8fa70b5 JF |
144 | static __init int xen_parse_nopvspin(char *arg) |
145 | { | |
146 | xen_pvspin = false; | |
147 | return 0; | |
148 | } | |
149 | early_param("xen_nopvspin", xen_parse_nopvspin); | |
150 |