Commit | Line | Data |
---|---|---|
fab97220 HS |
1 | /* |
2 | * IBM eServer eHCA Infiniband device driver for Linux on POWER | |
3 | * | |
4 | * Functions for EQs, NEQs and interrupts | |
5 | * | |
6 | * Authors: Heiko J Schick <schickhj@de.ibm.com> | |
7 | * Khadija Souissi <souissi@de.ibm.com> | |
28db6beb JF |
8 | * Hoang-Nam Nguyen <hnguyen@de.ibm.com> |
9 | * Joachim Fenkes <fenkes@de.ibm.com> | |
fab97220 HS |
10 | * |
11 | * Copyright (c) 2005 IBM Corporation | |
12 | * | |
13 | * All rights reserved. | |
14 | * | |
15 | * This source code is distributed under a dual license of GPL v2.0 and OpenIB | |
16 | * BSD. | |
17 | * | |
18 | * OpenIB BSD License | |
19 | * | |
20 | * Redistribution and use in source and binary forms, with or without | |
21 | * modification, are permitted provided that the following conditions are met: | |
22 | * | |
23 | * Redistributions of source code must retain the above copyright notice, this | |
24 | * list of conditions and the following disclaimer. | |
25 | * | |
26 | * Redistributions in binary form must reproduce the above copyright notice, | |
27 | * this list of conditions and the following disclaimer in the documentation | |
28 | * and/or other materials | |
29 | * provided with the distribution. | |
30 | * | |
31 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |
32 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
33 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
34 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | |
35 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
36 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
37 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
38 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER | |
39 | * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
40 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | |
41 | * POSSIBILITY OF SUCH DAMAGE. | |
42 | */ | |
43 | ||
5a0e3ad6 | 44 | #include <linux/slab.h> |
81942621 | 45 | #include <linux/smpboot.h> |
5a0e3ad6 | 46 | |
fab97220 HS |
47 | #include "ehca_classes.h" |
48 | #include "ehca_irq.h" | |
49 | #include "ehca_iverbs.h" | |
50 | #include "ehca_tools.h" | |
51 | #include "hcp_if.h" | |
52 | #include "hipz_fns.h" | |
7e28db5d | 53 | #include "ipz_pt_fn.h" |
fab97220 | 54 | |
2b94397a HNN |
55 | #define EQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1) |
56 | #define EQE_CQ_QP_NUMBER EHCA_BMASK_IBM( 8, 31) | |
57 | #define EQE_EE_IDENTIFIER EHCA_BMASK_IBM( 2, 7) | |
58 | #define EQE_CQ_NUMBER EHCA_BMASK_IBM( 8, 31) | |
59 | #define EQE_QP_NUMBER EHCA_BMASK_IBM( 8, 31) | |
60 | #define EQE_QP_TOKEN EHCA_BMASK_IBM(32, 63) | |
61 | #define EQE_CQ_TOKEN EHCA_BMASK_IBM(32, 63) | |
62 | ||
63 | #define NEQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1) | |
64 | #define NEQE_EVENT_CODE EHCA_BMASK_IBM( 2, 7) | |
65 | #define NEQE_PORT_NUMBER EHCA_BMASK_IBM( 8, 15) | |
66 | #define NEQE_PORT_AVAILABILITY EHCA_BMASK_IBM(16, 16) | |
67 | #define NEQE_DISRUPTIVE EHCA_BMASK_IBM(16, 16) | |
e57d62a1 | 68 | #define NEQE_SPECIFIC_EVENT EHCA_BMASK_IBM(16, 23) |
2b94397a HNN |
69 | |
70 | #define ERROR_DATA_LENGTH EHCA_BMASK_IBM(52, 63) | |
71 | #define ERROR_DATA_TYPE EHCA_BMASK_IBM( 0, 7) | |
fab97220 | 72 | |
fab97220 HS |
73 | static void queue_comp_task(struct ehca_cq *__cq); |
74 | ||
2b94397a | 75 | static struct ehca_comp_pool *pool; |
fab97220 | 76 | |
fab97220 HS |
77 | static inline void comp_event_callback(struct ehca_cq *cq) |
78 | { | |
79 | if (!cq->ib_cq.comp_handler) | |
80 | return; | |
81 | ||
82 | spin_lock(&cq->cb_lock); | |
83 | cq->ib_cq.comp_handler(&cq->ib_cq, cq->ib_cq.cq_context); | |
84 | spin_unlock(&cq->cb_lock); | |
85 | ||
86 | return; | |
87 | } | |
88 | ||
2b94397a HNN |
89 | static void print_error_data(struct ehca_shca *shca, void *data, |
90 | u64 *rblock, int length) | |
fab97220 HS |
91 | { |
92 | u64 type = EHCA_BMASK_GET(ERROR_DATA_TYPE, rblock[2]); | |
93 | u64 resource = rblock[1]; | |
94 | ||
95 | switch (type) { | |
96 | case 0x1: /* Queue Pair */ | |
97 | { | |
2b94397a | 98 | struct ehca_qp *qp = (struct ehca_qp *)data; |
fab97220 HS |
99 | |
100 | /* only print error data if AER is set */ | |
101 | if (rblock[6] == 0) | |
102 | return; | |
103 | ||
104 | ehca_err(&shca->ib_device, | |
3750f605 | 105 | "QP 0x%x (resource=%llx) has errors.", |
fab97220 HS |
106 | qp->ib_qp.qp_num, resource); |
107 | break; | |
108 | } | |
109 | case 0x4: /* Completion Queue */ | |
110 | { | |
2b94397a | 111 | struct ehca_cq *cq = (struct ehca_cq *)data; |
fab97220 HS |
112 | |
113 | ehca_err(&shca->ib_device, | |
3750f605 | 114 | "CQ 0x%x (resource=%llx) has errors.", |
fab97220 HS |
115 | cq->cq_number, resource); |
116 | break; | |
117 | } | |
118 | default: | |
119 | ehca_err(&shca->ib_device, | |
3750f605 | 120 | "Unknown error type: %llx on %s.", |
fab97220 HS |
121 | type, shca->ib_device.name); |
122 | break; | |
123 | } | |
124 | ||
3750f605 | 125 | ehca_err(&shca->ib_device, "Error data is available: %llx.", resource); |
fab97220 HS |
126 | ehca_err(&shca->ib_device, "EHCA ----- error data begin " |
127 | "---------------------------------------------------"); | |
3750f605 | 128 | ehca_dmp(rblock, length, "resource=%llx", resource); |
fab97220 HS |
129 | ehca_err(&shca->ib_device, "EHCA ----- error data end " |
130 | "----------------------------------------------------"); | |
131 | ||
132 | return; | |
133 | } | |
134 | ||
135 | int ehca_error_data(struct ehca_shca *shca, void *data, | |
136 | u64 resource) | |
137 | { | |
138 | ||
139 | unsigned long ret; | |
140 | u64 *rblock; | |
141 | unsigned long block_count; | |
142 | ||
f2d91361 | 143 | rblock = ehca_alloc_fw_ctrlblock(GFP_ATOMIC); |
fab97220 HS |
144 | if (!rblock) { |
145 | ehca_err(&shca->ib_device, "Cannot allocate rblock memory."); | |
146 | ret = -ENOMEM; | |
147 | goto error_data1; | |
148 | } | |
149 | ||
7e28db5d | 150 | /* rblock must be 4K aligned and should be 4K large */ |
fab97220 HS |
151 | ret = hipz_h_error_data(shca->ipz_hca_handle, |
152 | resource, | |
153 | rblock, | |
154 | &block_count); | |
155 | ||
7e28db5d | 156 | if (ret == H_R_STATE) |
fab97220 | 157 | ehca_err(&shca->ib_device, |
3750f605 | 158 | "No error data is available: %llx.", resource); |
fab97220 HS |
159 | else if (ret == H_SUCCESS) { |
160 | int length; | |
161 | ||
162 | length = EHCA_BMASK_GET(ERROR_DATA_LENGTH, rblock[0]); | |
163 | ||
7e28db5d HNN |
164 | if (length > EHCA_PAGESIZE) |
165 | length = EHCA_PAGESIZE; | |
fab97220 HS |
166 | |
167 | print_error_data(shca, data, rblock, length); | |
7e28db5d | 168 | } else |
fab97220 | 169 | ehca_err(&shca->ib_device, |
3750f605 | 170 | "Error data could not be fetched: %llx", resource); |
fab97220 | 171 | |
7e28db5d | 172 | ehca_free_fw_ctrlblock(rblock); |
fab97220 HS |
173 | |
174 | error_data1: | |
175 | return ret; | |
176 | ||
177 | } | |
178 | ||
5ff70cac JF |
179 | static void dispatch_qp_event(struct ehca_shca *shca, struct ehca_qp *qp, |
180 | enum ib_event_type event_type) | |
fab97220 HS |
181 | { |
182 | struct ib_event event; | |
fab97220 | 183 | |
5b673b71 JF |
184 | /* PATH_MIG without the QP ever having been armed is false alarm */ |
185 | if (event_type == IB_EVENT_PATH_MIG && !qp->mig_armed) | |
186 | return; | |
187 | ||
633a5aed | 188 | event.device = &shca->ib_device; |
5ff70cac | 189 | event.event = event_type; |
fab97220 | 190 | |
633a5aed HNN |
191 | if (qp->ext_type == EQPT_SRQ) { |
192 | if (!qp->ib_srq.event_handler) | |
193 | return; | |
fab97220 | 194 | |
633a5aed HNN |
195 | event.element.srq = &qp->ib_srq; |
196 | qp->ib_srq.event_handler(&event, qp->ib_srq.srq_context); | |
197 | } else { | |
198 | if (!qp->ib_qp.event_handler) | |
199 | return; | |
200 | ||
633a5aed HNN |
201 | event.element.qp = &qp->ib_qp; |
202 | qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); | |
203 | } | |
5ff70cac JF |
204 | } |
205 | ||
206 | static void qp_event_callback(struct ehca_shca *shca, u64 eqe, | |
207 | enum ib_event_type event_type, int fatal) | |
208 | { | |
209 | struct ehca_qp *qp; | |
210 | u32 token = EHCA_BMASK_GET(EQE_QP_TOKEN, eqe); | |
211 | ||
212 | read_lock(&ehca_qp_idr_lock); | |
213 | qp = idr_find(&ehca_qp_idr, token); | |
12137c59 SR |
214 | if (qp) |
215 | atomic_inc(&qp->nr_events); | |
5ff70cac JF |
216 | read_unlock(&ehca_qp_idr_lock); |
217 | ||
218 | if (!qp) | |
219 | return; | |
220 | ||
221 | if (fatal) | |
222 | ehca_error_data(shca, qp, qp->ipz_qp_handle.handle); | |
223 | ||
224 | dispatch_qp_event(shca, qp, fatal && qp->ext_type == EQPT_SRQ ? | |
225 | IB_EVENT_SRQ_ERR : event_type); | |
226 | ||
227 | /* | |
228 | * eHCA only processes one WQE at a time for SRQ base QPs, | |
229 | * so the last WQE has been processed as soon as the QP enters | |
230 | * error state. | |
231 | */ | |
232 | if (fatal && qp->ext_type == EQPT_SRQBASE) | |
233 | dispatch_qp_event(shca, qp, IB_EVENT_QP_LAST_WQE_REACHED); | |
fab97220 | 234 | |
12137c59 SR |
235 | if (atomic_dec_and_test(&qp->nr_events)) |
236 | wake_up(&qp->wait_completion); | |
fab97220 HS |
237 | return; |
238 | } | |
239 | ||
240 | static void cq_event_callback(struct ehca_shca *shca, | |
78d8d5f9 | 241 | u64 eqe) |
fab97220 HS |
242 | { |
243 | struct ehca_cq *cq; | |
fab97220 HS |
244 | u32 token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe); |
245 | ||
26ed687f | 246 | read_lock(&ehca_cq_idr_lock); |
fab97220 | 247 | cq = idr_find(&ehca_cq_idr, token); |
28db6beb JF |
248 | if (cq) |
249 | atomic_inc(&cq->nr_events); | |
26ed687f | 250 | read_unlock(&ehca_cq_idr_lock); |
fab97220 HS |
251 | |
252 | if (!cq) | |
253 | return; | |
254 | ||
255 | ehca_error_data(shca, cq, cq->ipz_cq_handle.handle); | |
256 | ||
28db6beb JF |
257 | if (atomic_dec_and_test(&cq->nr_events)) |
258 | wake_up(&cq->wait_completion); | |
259 | ||
fab97220 HS |
260 | return; |
261 | } | |
262 | ||
263 | static void parse_identifier(struct ehca_shca *shca, u64 eqe) | |
264 | { | |
265 | u8 identifier = EHCA_BMASK_GET(EQE_EE_IDENTIFIER, eqe); | |
266 | ||
267 | switch (identifier) { | |
268 | case 0x02: /* path migrated */ | |
633a5aed | 269 | qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG, 0); |
fab97220 HS |
270 | break; |
271 | case 0x03: /* communication established */ | |
633a5aed | 272 | qp_event_callback(shca, eqe, IB_EVENT_COMM_EST, 0); |
fab97220 HS |
273 | break; |
274 | case 0x04: /* send queue drained */ | |
633a5aed | 275 | qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED, 0); |
fab97220 HS |
276 | break; |
277 | case 0x05: /* QP error */ | |
278 | case 0x06: /* QP error */ | |
633a5aed | 279 | qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL, 1); |
fab97220 HS |
280 | break; |
281 | case 0x07: /* CQ error */ | |
282 | case 0x08: /* CQ error */ | |
283 | cq_event_callback(shca, eqe); | |
284 | break; | |
285 | case 0x09: /* MRMWPTE error */ | |
286 | ehca_err(&shca->ib_device, "MRMWPTE error."); | |
287 | break; | |
288 | case 0x0A: /* port event */ | |
289 | ehca_err(&shca->ib_device, "Port event."); | |
290 | break; | |
291 | case 0x0B: /* MR access error */ | |
292 | ehca_err(&shca->ib_device, "MR access error."); | |
293 | break; | |
294 | case 0x0C: /* EQ error */ | |
295 | ehca_err(&shca->ib_device, "EQ error."); | |
296 | break; | |
297 | case 0x0D: /* P/Q_Key mismatch */ | |
298 | ehca_err(&shca->ib_device, "P/Q_Key mismatch."); | |
299 | break; | |
300 | case 0x10: /* sampling complete */ | |
301 | ehca_err(&shca->ib_device, "Sampling complete."); | |
302 | break; | |
303 | case 0x11: /* unaffiliated access error */ | |
304 | ehca_err(&shca->ib_device, "Unaffiliated access error."); | |
305 | break; | |
e90d0b3d JF |
306 | case 0x12: /* path migrating */ |
307 | ehca_err(&shca->ib_device, "Path migrating."); | |
fab97220 HS |
308 | break; |
309 | case 0x13: /* interface trace stopped */ | |
310 | ehca_err(&shca->ib_device, "Interface trace stopped."); | |
311 | break; | |
312 | case 0x14: /* first error capture info available */ | |
633a5aed HNN |
313 | ehca_info(&shca->ib_device, "First error capture available"); |
314 | break; | |
315 | case 0x15: /* SRQ limit reached */ | |
316 | qp_event_callback(shca, eqe, IB_EVENT_SRQ_LIMIT_REACHED, 0); | |
317 | break; | |
fab97220 HS |
318 | default: |
319 | ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.", | |
320 | identifier, shca->ib_device.name); | |
321 | break; | |
322 | } | |
323 | ||
324 | return; | |
325 | } | |
326 | ||
8705ce5b JF |
327 | static void dispatch_port_event(struct ehca_shca *shca, int port_num, |
328 | enum ib_event_type type, const char *msg) | |
fab97220 HS |
329 | { |
330 | struct ib_event event; | |
8705ce5b JF |
331 | |
332 | ehca_info(&shca->ib_device, "port %d %s.", port_num, msg); | |
333 | event.device = &shca->ib_device; | |
334 | event.event = type; | |
335 | event.element.port_num = port_num; | |
336 | ib_dispatch_event(&event); | |
337 | } | |
338 | ||
339 | static void notify_port_conf_change(struct ehca_shca *shca, int port_num) | |
340 | { | |
341 | struct ehca_sma_attr new_attr; | |
342 | struct ehca_sma_attr *old_attr = &shca->sport[port_num - 1].saved_attr; | |
343 | ||
344 | ehca_query_sma_attr(shca, port_num, &new_attr); | |
345 | ||
346 | if (new_attr.sm_sl != old_attr->sm_sl || | |
347 | new_attr.sm_lid != old_attr->sm_lid) | |
348 | dispatch_port_event(shca, port_num, IB_EVENT_SM_CHANGE, | |
349 | "SM changed"); | |
350 | ||
351 | if (new_attr.lid != old_attr->lid || | |
352 | new_attr.lmc != old_attr->lmc) | |
353 | dispatch_port_event(shca, port_num, IB_EVENT_LID_CHANGE, | |
354 | "LID changed"); | |
355 | ||
356 | if (new_attr.pkey_tbl_len != old_attr->pkey_tbl_len || | |
357 | memcmp(new_attr.pkeys, old_attr->pkeys, | |
358 | sizeof(u16) * new_attr.pkey_tbl_len)) | |
359 | dispatch_port_event(shca, port_num, IB_EVENT_PKEY_CHANGE, | |
360 | "P_Key changed"); | |
361 | ||
362 | *old_attr = new_attr; | |
363 | } | |
364 | ||
fad96ab6 SR |
365 | /* replay modify_qp for sqps -- return 0 if all is well, 1 if AQP1 destroyed */ |
366 | static int replay_modify_qp(struct ehca_sport *sport) | |
367 | { | |
368 | int aqp1_destroyed; | |
369 | unsigned long flags; | |
370 | ||
371 | spin_lock_irqsave(&sport->mod_sqp_lock, flags); | |
372 | ||
373 | aqp1_destroyed = !sport->ibqp_sqp[IB_QPT_GSI]; | |
374 | ||
375 | if (sport->ibqp_sqp[IB_QPT_SMI]) | |
376 | ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_SMI]); | |
377 | if (!aqp1_destroyed) | |
378 | ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_GSI]); | |
379 | ||
380 | spin_unlock_irqrestore(&sport->mod_sqp_lock, flags); | |
381 | ||
382 | return aqp1_destroyed; | |
383 | } | |
384 | ||
8705ce5b JF |
385 | static void parse_ec(struct ehca_shca *shca, u64 eqe) |
386 | { | |
fab97220 HS |
387 | u8 ec = EHCA_BMASK_GET(NEQE_EVENT_CODE, eqe); |
388 | u8 port = EHCA_BMASK_GET(NEQE_PORT_NUMBER, eqe); | |
e57d62a1 | 389 | u8 spec_event; |
bbdd267e | 390 | struct ehca_sport *sport = &shca->sport[port - 1]; |
fab97220 HS |
391 | |
392 | switch (ec) { | |
393 | case 0x30: /* port availability change */ | |
394 | if (EHCA_BMASK_GET(NEQE_PORT_AVAILABILITY, eqe)) { | |
fad96ab6 SR |
395 | /* only replay modify_qp calls in autodetect mode; |
396 | * if AQP1 was destroyed, the port is already down | |
397 | * again and we can drop the event. | |
398 | */ | |
399 | if (ehca_nr_ports < 0) | |
400 | if (replay_modify_qp(sport)) | |
401 | break; | |
bbdd267e HNN |
402 | |
403 | sport->port_state = IB_PORT_ACTIVE; | |
8705ce5b JF |
404 | dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE, |
405 | "is active"); | |
fad96ab6 | 406 | ehca_query_sma_attr(shca, port, &sport->saved_attr); |
fab97220 | 407 | } else { |
bbdd267e | 408 | sport->port_state = IB_PORT_DOWN; |
8705ce5b JF |
409 | dispatch_port_event(shca, port, IB_EVENT_PORT_ERR, |
410 | "is inactive"); | |
fab97220 HS |
411 | } |
412 | break; | |
413 | case 0x31: | |
414 | /* port configuration change | |
415 | * disruptive change is caused by | |
416 | * LID, PKEY or SM change | |
417 | */ | |
8705ce5b JF |
418 | if (EHCA_BMASK_GET(NEQE_DISRUPTIVE, eqe)) { |
419 | ehca_warn(&shca->ib_device, "disruptive port " | |
420 | "%d configuration change", port); | |
421 | ||
bbdd267e | 422 | sport->port_state = IB_PORT_DOWN; |
8705ce5b JF |
423 | dispatch_port_event(shca, port, IB_EVENT_PORT_ERR, |
424 | "is inactive"); | |
425 | ||
bbdd267e | 426 | sport->port_state = IB_PORT_ACTIVE; |
8705ce5b JF |
427 | dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE, |
428 | "is active"); | |
528b03f7 JF |
429 | ehca_query_sma_attr(shca, port, |
430 | &sport->saved_attr); | |
8705ce5b JF |
431 | } else |
432 | notify_port_conf_change(shca, port); | |
fab97220 HS |
433 | break; |
434 | case 0x32: /* adapter malfunction */ | |
435 | ehca_err(&shca->ib_device, "Adapter malfunction."); | |
436 | break; | |
437 | case 0x33: /* trace stopped */ | |
438 | ehca_err(&shca->ib_device, "Traced stopped."); | |
439 | break; | |
e57d62a1 HNN |
440 | case 0x34: /* util async event */ |
441 | spec_event = EHCA_BMASK_GET(NEQE_SPECIFIC_EVENT, eqe); | |
442 | if (spec_event == 0x80) /* client reregister required */ | |
443 | dispatch_port_event(shca, port, | |
444 | IB_EVENT_CLIENT_REREGISTER, | |
445 | "client reregister req."); | |
446 | else | |
447 | ehca_warn(&shca->ib_device, "Unknown util async " | |
448 | "event %x on port %x", spec_event, port); | |
449 | break; | |
fab97220 HS |
450 | default: |
451 | ehca_err(&shca->ib_device, "Unknown event code: %x on %s.", | |
452 | ec, shca->ib_device.name); | |
453 | break; | |
454 | } | |
455 | ||
456 | return; | |
457 | } | |
458 | ||
459 | static inline void reset_eq_pending(struct ehca_cq *cq) | |
460 | { | |
461 | u64 CQx_EP; | |
462 | struct h_galpa gal = cq->galpas.kernel; | |
463 | ||
464 | hipz_galpa_store_cq(gal, cqx_ep, 0x0); | |
465 | CQx_EP = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_ep)); | |
466 | ||
467 | return; | |
468 | } | |
469 | ||
7d12e780 | 470 | irqreturn_t ehca_interrupt_neq(int irq, void *dev_id) |
fab97220 HS |
471 | { |
472 | struct ehca_shca *shca = (struct ehca_shca*)dev_id; | |
473 | ||
474 | tasklet_hi_schedule(&shca->neq.interrupt_task); | |
475 | ||
476 | return IRQ_HANDLED; | |
477 | } | |
478 | ||
479 | void ehca_tasklet_neq(unsigned long data) | |
480 | { | |
481 | struct ehca_shca *shca = (struct ehca_shca*)data; | |
482 | struct ehca_eqe *eqe; | |
483 | u64 ret; | |
484 | ||
5b891a93 | 485 | eqe = ehca_poll_eq(shca, &shca->neq); |
fab97220 HS |
486 | |
487 | while (eqe) { | |
488 | if (!EHCA_BMASK_GET(NEQE_COMPLETION_EVENT, eqe->entry)) | |
489 | parse_ec(shca, eqe->entry); | |
490 | ||
5b891a93 | 491 | eqe = ehca_poll_eq(shca, &shca->neq); |
fab97220 HS |
492 | } |
493 | ||
494 | ret = hipz_h_reset_event(shca->ipz_hca_handle, | |
495 | shca->neq.ipz_eq_handle, 0xFFFFFFFFFFFFFFFFL); | |
496 | ||
497 | if (ret != H_SUCCESS) | |
498 | ehca_err(&shca->ib_device, "Can't clear notification events."); | |
499 | ||
500 | return; | |
501 | } | |
502 | ||
7d12e780 | 503 | irqreturn_t ehca_interrupt_eq(int irq, void *dev_id) |
fab97220 HS |
504 | { |
505 | struct ehca_shca *shca = (struct ehca_shca*)dev_id; | |
506 | ||
507 | tasklet_hi_schedule(&shca->eq.interrupt_task); | |
508 | ||
509 | return IRQ_HANDLED; | |
510 | } | |
511 | ||
fab97220 | 512 | |
78d8d5f9 HNN |
513 | static inline void process_eqe(struct ehca_shca *shca, struct ehca_eqe *eqe) |
514 | { | |
515 | u64 eqe_value; | |
516 | u32 token; | |
78d8d5f9 | 517 | struct ehca_cq *cq; |
31726798 | 518 | |
78d8d5f9 | 519 | eqe_value = eqe->entry; |
3750f605 | 520 | ehca_dbg(&shca->ib_device, "eqe_value=%llx", eqe_value); |
78d8d5f9 | 521 | if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) { |
31726798 | 522 | ehca_dbg(&shca->ib_device, "Got completion event"); |
78d8d5f9 | 523 | token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value); |
26ed687f | 524 | read_lock(&ehca_cq_idr_lock); |
78d8d5f9 | 525 | cq = idr_find(&ehca_cq_idr, token); |
28db6beb JF |
526 | if (cq) |
527 | atomic_inc(&cq->nr_events); | |
26ed687f | 528 | read_unlock(&ehca_cq_idr_lock); |
78d8d5f9 | 529 | if (cq == NULL) { |
78d8d5f9 HNN |
530 | ehca_err(&shca->ib_device, |
531 | "Invalid eqe for non-existing cq token=%x", | |
532 | token); | |
533 | return; | |
534 | } | |
535 | reset_eq_pending(cq); | |
31726798 | 536 | if (ehca_scaling_code) |
4fd30060 | 537 | queue_comp_task(cq); |
31726798 | 538 | else { |
4fd30060 | 539 | comp_event_callback(cq); |
28db6beb | 540 | if (atomic_dec_and_test(&cq->nr_events)) |
31726798 | 541 | wake_up(&cq->wait_completion); |
4fd30060 | 542 | } |
78d8d5f9 | 543 | } else { |
31726798 | 544 | ehca_dbg(&shca->ib_device, "Got non completion event"); |
78d8d5f9 HNN |
545 | parse_identifier(shca, eqe_value); |
546 | } | |
547 | } | |
fab97220 | 548 | |
78d8d5f9 HNN |
549 | void ehca_process_eq(struct ehca_shca *shca, int is_irq) |
550 | { | |
551 | struct ehca_eq *eq = &shca->eq; | |
552 | struct ehca_eqe_cache_entry *eqe_cache = eq->eqe_cache; | |
6f7bc01a | 553 | u64 eqe_value, ret; |
78d8d5f9 HNN |
554 | int eqe_cnt, i; |
555 | int eq_empty = 0; | |
556 | ||
25ef7563 | 557 | spin_lock(&eq->irq_spinlock); |
78d8d5f9 HNN |
558 | if (is_irq) { |
559 | const int max_query_cnt = 100; | |
560 | int query_cnt = 0; | |
561 | int int_state = 1; | |
562 | do { | |
563 | int_state = hipz_h_query_int_state( | |
564 | shca->ipz_hca_handle, eq->ist); | |
565 | query_cnt++; | |
566 | iosync(); | |
567 | } while (int_state && query_cnt < max_query_cnt); | |
568 | if (unlikely((query_cnt == max_query_cnt))) | |
569 | ehca_dbg(&shca->ib_device, "int_state=%x query_cnt=%x", | |
570 | int_state, query_cnt); | |
571 | } | |
fab97220 | 572 | |
78d8d5f9 HNN |
573 | /* read out all eqes */ |
574 | eqe_cnt = 0; | |
575 | do { | |
576 | u32 token; | |
5b891a93 | 577 | eqe_cache[eqe_cnt].eqe = ehca_poll_eq(shca, eq); |
78d8d5f9 HNN |
578 | if (!eqe_cache[eqe_cnt].eqe) |
579 | break; | |
580 | eqe_value = eqe_cache[eqe_cnt].eqe->entry; | |
581 | if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) { | |
582 | token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value); | |
26ed687f | 583 | read_lock(&ehca_cq_idr_lock); |
78d8d5f9 | 584 | eqe_cache[eqe_cnt].cq = idr_find(&ehca_cq_idr, token); |
28db6beb JF |
585 | if (eqe_cache[eqe_cnt].cq) |
586 | atomic_inc(&eqe_cache[eqe_cnt].cq->nr_events); | |
26ed687f | 587 | read_unlock(&ehca_cq_idr_lock); |
78d8d5f9 | 588 | if (!eqe_cache[eqe_cnt].cq) { |
78d8d5f9 HNN |
589 | ehca_err(&shca->ib_device, |
590 | "Invalid eqe for non-existing cq " | |
591 | "token=%x", token); | |
592 | continue; | |
593 | } | |
78d8d5f9 HNN |
594 | } else |
595 | eqe_cache[eqe_cnt].cq = NULL; | |
596 | eqe_cnt++; | |
597 | } while (eqe_cnt < EHCA_EQE_CACHE_SIZE); | |
598 | if (!eqe_cnt) { | |
599 | if (is_irq) | |
600 | ehca_dbg(&shca->ib_device, | |
601 | "No eqe found for irq event"); | |
602 | goto unlock_irq_spinlock; | |
6f7bc01a SR |
603 | } else if (!is_irq) { |
604 | ret = hipz_h_eoi(eq->ist); | |
605 | if (ret != H_SUCCESS) | |
606 | ehca_err(&shca->ib_device, | |
3750f605 | 607 | "bad return code EOI -rc = %lld\n", ret); |
78d8d5f9 | 608 | ehca_dbg(&shca->ib_device, "deadman found %x eqe", eqe_cnt); |
6f7bc01a | 609 | } |
78d8d5f9 HNN |
610 | if (unlikely(eqe_cnt == EHCA_EQE_CACHE_SIZE)) |
611 | ehca_dbg(&shca->ib_device, "too many eqes for one irq event"); | |
612 | /* enable irq for new packets */ | |
613 | for (i = 0; i < eqe_cnt; i++) { | |
614 | if (eq->eqe_cache[i].cq) | |
615 | reset_eq_pending(eq->eqe_cache[i].cq); | |
616 | } | |
617 | /* check eq */ | |
618 | spin_lock(&eq->spinlock); | |
619 | eq_empty = (!ipz_eqit_eq_peek_valid(&shca->eq.ipz_queue)); | |
620 | spin_unlock(&eq->spinlock); | |
621 | /* call completion handler for cached eqes */ | |
622 | for (i = 0; i < eqe_cnt; i++) | |
623 | if (eq->eqe_cache[i].cq) { | |
31726798 | 624 | if (ehca_scaling_code) |
4fd30060 | 625 | queue_comp_task(eq->eqe_cache[i].cq); |
31726798 HNN |
626 | else { |
627 | struct ehca_cq *cq = eq->eqe_cache[i].cq; | |
628 | comp_event_callback(cq); | |
28db6beb | 629 | if (atomic_dec_and_test(&cq->nr_events)) |
31726798 | 630 | wake_up(&cq->wait_completion); |
31726798 | 631 | } |
78d8d5f9 HNN |
632 | } else { |
633 | ehca_dbg(&shca->ib_device, "Got non completion event"); | |
634 | parse_identifier(shca, eq->eqe_cache[i].eqe->entry); | |
fab97220 | 635 | } |
78d8d5f9 HNN |
636 | /* poll eq if not empty */ |
637 | if (eq_empty) | |
638 | goto unlock_irq_spinlock; | |
639 | do { | |
640 | struct ehca_eqe *eqe; | |
5b891a93 | 641 | eqe = ehca_poll_eq(shca, &shca->eq); |
78d8d5f9 HNN |
642 | if (!eqe) |
643 | break; | |
644 | process_eqe(shca, eqe); | |
78d8d5f9 HNN |
645 | } while (1); |
646 | ||
647 | unlock_irq_spinlock: | |
25ef7563 | 648 | spin_unlock(&eq->irq_spinlock); |
78d8d5f9 | 649 | } |
fab97220 | 650 | |
78d8d5f9 HNN |
651 | void ehca_tasklet_eq(unsigned long data) |
652 | { | |
653 | ehca_process_eq((struct ehca_shca*)data, 1); | |
fab97220 HS |
654 | } |
655 | ||
81942621 | 656 | static int find_next_online_cpu(struct ehca_comp_pool *pool) |
fab97220 | 657 | { |
8b16cef3 HNN |
658 | int cpu; |
659 | unsigned long flags; | |
fab97220 | 660 | |
8b16cef3 | 661 | WARN_ON_ONCE(!in_interrupt()); |
4da27d6d | 662 | if (ehca_debug_level >= 3) |
b29179c3 | 663 | ehca_dmp(cpu_online_mask, cpumask_size(), ""); |
fab97220 | 664 | |
8b16cef3 | 665 | spin_lock_irqsave(&pool->last_cpu_lock, flags); |
81942621 TG |
666 | do { |
667 | cpu = cpumask_next(pool->last_cpu, cpu_online_mask); | |
668 | if (cpu >= nr_cpu_ids) | |
669 | cpu = cpumask_first(cpu_online_mask); | |
670 | pool->last_cpu = cpu; | |
c0e12e51 | 671 | } while (!per_cpu_ptr(pool->cpu_comp_tasks, cpu)->active); |
8b16cef3 | 672 | spin_unlock_irqrestore(&pool->last_cpu_lock, flags); |
fab97220 | 673 | |
8b16cef3 | 674 | return cpu; |
fab97220 HS |
675 | } |
676 | ||
677 | static void __queue_comp_task(struct ehca_cq *__cq, | |
81942621 TG |
678 | struct ehca_cpu_comp_task *cct, |
679 | struct task_struct *thread) | |
fab97220 | 680 | { |
8b16cef3 | 681 | unsigned long flags; |
fab97220 | 682 | |
8b16cef3 HNN |
683 | spin_lock_irqsave(&cct->task_lock, flags); |
684 | spin_lock(&__cq->task_lock); | |
fab97220 HS |
685 | |
686 | if (__cq->nr_callbacks == 0) { | |
687 | __cq->nr_callbacks++; | |
688 | list_add_tail(&__cq->entry, &cct->cq_list); | |
689 | cct->cq_jobs++; | |
81942621 | 690 | wake_up_process(thread); |
31726798 | 691 | } else |
fab97220 HS |
692 | __cq->nr_callbacks++; |
693 | ||
8b16cef3 HNN |
694 | spin_unlock(&__cq->task_lock); |
695 | spin_unlock_irqrestore(&cct->task_lock, flags); | |
fab97220 HS |
696 | } |
697 | ||
698 | static void queue_comp_task(struct ehca_cq *__cq) | |
699 | { | |
fab97220 HS |
700 | int cpu_id; |
701 | struct ehca_cpu_comp_task *cct; | |
81942621 | 702 | struct task_struct *thread; |
31726798 HNN |
703 | int cq_jobs; |
704 | unsigned long flags; | |
fab97220 | 705 | |
fab97220 | 706 | cpu_id = find_next_online_cpu(pool); |
fab97220 HS |
707 | BUG_ON(!cpu_online(cpu_id)); |
708 | ||
709 | cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id); | |
bff4a394 | 710 | thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id); |
81942621 | 711 | BUG_ON(!cct || !thread); |
fab97220 | 712 | |
31726798 HNN |
713 | spin_lock_irqsave(&cct->task_lock, flags); |
714 | cq_jobs = cct->cq_jobs; | |
715 | spin_unlock_irqrestore(&cct->task_lock, flags); | |
716 | if (cq_jobs > 0) { | |
fab97220 HS |
717 | cpu_id = find_next_online_cpu(pool); |
718 | cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id); | |
bff4a394 | 719 | thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id); |
81942621 | 720 | BUG_ON(!cct || !thread); |
fab97220 | 721 | } |
81942621 | 722 | __queue_comp_task(__cq, cct, thread); |
fab97220 HS |
723 | } |
724 | ||
2b94397a | 725 | static void run_comp_task(struct ehca_cpu_comp_task *cct) |
fab97220 HS |
726 | { |
727 | struct ehca_cq *cq; | |
fab97220 HS |
728 | |
729 | while (!list_empty(&cct->cq_list)) { | |
730 | cq = list_entry(cct->cq_list.next, struct ehca_cq, entry); | |
81942621 | 731 | spin_unlock_irq(&cct->task_lock); |
fab97220 | 732 | |
28db6beb JF |
733 | comp_event_callback(cq); |
734 | if (atomic_dec_and_test(&cq->nr_events)) | |
31726798 | 735 | wake_up(&cq->wait_completion); |
31726798 | 736 | |
81942621 | 737 | spin_lock_irq(&cct->task_lock); |
8b16cef3 | 738 | spin_lock(&cq->task_lock); |
fab97220 | 739 | cq->nr_callbacks--; |
31726798 | 740 | if (!cq->nr_callbacks) { |
fab97220 HS |
741 | list_del_init(cct->cq_list.next); |
742 | cct->cq_jobs--; | |
743 | } | |
8b16cef3 | 744 | spin_unlock(&cq->task_lock); |
fab97220 | 745 | } |
fab97220 HS |
746 | } |
747 | ||
81942621 | 748 | static void comp_task_park(unsigned int cpu) |
fab97220 | 749 | { |
81942621 TG |
750 | struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu); |
751 | struct ehca_cpu_comp_task *target; | |
752 | struct task_struct *thread; | |
753 | struct ehca_cq *cq, *tmp; | |
754 | LIST_HEAD(list); | |
fab97220 | 755 | |
81942621 TG |
756 | spin_lock_irq(&cct->task_lock); |
757 | cct->cq_jobs = 0; | |
758 | cct->active = 0; | |
759 | list_splice_init(&cct->cq_list, &list); | |
760 | spin_unlock_irq(&cct->task_lock); | |
fab97220 | 761 | |
81942621 TG |
762 | cpu = find_next_online_cpu(pool); |
763 | target = per_cpu_ptr(pool->cpu_comp_tasks, cpu); | |
bff4a394 | 764 | thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu); |
81942621 TG |
765 | spin_lock_irq(&target->task_lock); |
766 | list_for_each_entry_safe(cq, tmp, &list, entry) { | |
767 | list_del(&cq->entry); | |
768 | __queue_comp_task(cq, target, thread); | |
fab97220 | 769 | } |
81942621 | 770 | spin_unlock_irq(&target->task_lock); |
fab97220 HS |
771 | } |
772 | ||
81942621 | 773 | static void comp_task_stop(unsigned int cpu, bool online) |
fab97220 | 774 | { |
81942621 | 775 | struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu); |
fab97220 | 776 | |
81942621 | 777 | spin_lock_irq(&cct->task_lock); |
fab97220 | 778 | cct->cq_jobs = 0; |
81942621 TG |
779 | cct->active = 0; |
780 | WARN_ON(!list_empty(&cct->cq_list)); | |
781 | spin_unlock_irq(&cct->task_lock); | |
fab97220 HS |
782 | } |
783 | ||
81942621 | 784 | static int comp_task_should_run(unsigned int cpu) |
fab97220 HS |
785 | { |
786 | struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu); | |
fab97220 | 787 | |
81942621 | 788 | return cct->cq_jobs; |
fab97220 HS |
789 | } |
790 | ||
bff4a394 | 791 | static void comp_task(unsigned int cpu) |
fab97220 | 792 | { |
81942621 TG |
793 | struct ehca_cpu_comp_task *cct = this_cpu_ptr(pool->cpu_comp_tasks); |
794 | int cql_empty; | |
fab97220 | 795 | |
81942621 TG |
796 | spin_lock_irq(&cct->task_lock); |
797 | cql_empty = list_empty(&cct->cq_list); | |
798 | if (!cql_empty) { | |
799 | __set_current_state(TASK_RUNNING); | |
800 | run_comp_task(cct); | |
fab97220 | 801 | } |
81942621 | 802 | spin_unlock_irq(&cct->task_lock); |
fab97220 | 803 | } |
9faa559c | 804 | |
81942621 TG |
805 | static struct smp_hotplug_thread comp_pool_threads = { |
806 | .thread_should_run = comp_task_should_run, | |
807 | .thread_fn = comp_task, | |
808 | .thread_comm = "ehca_comp/%u", | |
809 | .cleanup = comp_task_stop, | |
810 | .park = comp_task_park, | |
9faa559c | 811 | }; |
fab97220 | 812 | |
fab97220 HS |
813 | int ehca_create_comp_pool(void) |
814 | { | |
81942621 | 815 | int cpu, ret = -ENOMEM; |
fab97220 | 816 | |
4fd30060 HNN |
817 | if (!ehca_scaling_code) |
818 | return 0; | |
819 | ||
fab97220 HS |
820 | pool = kzalloc(sizeof(struct ehca_comp_pool), GFP_KERNEL); |
821 | if (pool == NULL) | |
822 | return -ENOMEM; | |
823 | ||
824 | spin_lock_init(&pool->last_cpu_lock); | |
b29179c3 | 825 | pool->last_cpu = cpumask_any(cpu_online_mask); |
fab97220 HS |
826 | |
827 | pool->cpu_comp_tasks = alloc_percpu(struct ehca_cpu_comp_task); | |
81942621 TG |
828 | if (!pool->cpu_comp_tasks) |
829 | goto out_pool; | |
fab97220 | 830 | |
81942621 TG |
831 | pool->cpu_comp_threads = alloc_percpu(struct task_struct *); |
832 | if (!pool->cpu_comp_threads) | |
833 | goto out_tasks; | |
834 | ||
835 | for_each_present_cpu(cpu) { | |
836 | struct ehca_cpu_comp_task *cct; | |
837 | ||
838 | cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu); | |
839 | spin_lock_init(&cct->task_lock); | |
840 | INIT_LIST_HEAD(&cct->cq_list); | |
fab97220 HS |
841 | } |
842 | ||
81942621 TG |
843 | comp_pool_threads.store = pool->cpu_comp_threads; |
844 | ret = smpboot_register_percpu_thread(&comp_pool_threads); | |
845 | if (ret) | |
846 | goto out_threads; | |
4fd30060 | 847 | |
81942621 TG |
848 | pr_info("eHCA scaling code enabled\n"); |
849 | return ret; | |
fab97220 | 850 | |
81942621 TG |
851 | out_threads: |
852 | free_percpu(pool->cpu_comp_threads); | |
853 | out_tasks: | |
854 | free_percpu(pool->cpu_comp_tasks); | |
855 | out_pool: | |
856 | kfree(pool); | |
857 | return ret; | |
fab97220 HS |
858 | } |
859 | ||
860 | void ehca_destroy_comp_pool(void) | |
861 | { | |
4fd30060 HNN |
862 | if (!ehca_scaling_code) |
863 | return; | |
864 | ||
81942621 | 865 | smpboot_unregister_percpu_thread(&comp_pool_threads); |
259c4ddd | 866 | |
81942621 | 867 | free_percpu(pool->cpu_comp_threads); |
65e5c026 AM |
868 | free_percpu(pool->cpu_comp_tasks); |
869 | kfree(pool); | |
fab97220 | 870 | } |