Commit | Line | Data |
---|---|---|
7a291083 JBT |
1 | /* |
2 | * linux/drivers/net/ehea/ehea_qmr.c | |
3 | * | |
4 | * eHEA ethernet device driver for IBM eServer System p | |
5 | * | |
6 | * (C) Copyright IBM Corp. 2006 | |
7 | * | |
8 | * Authors: | |
9 | * Christoph Raisch <raisch@de.ibm.com> | |
10 | * Jan-Bernd Themann <themann@de.ibm.com> | |
11 | * Thomas Klein <tklein@de.ibm.com> | |
12 | * | |
13 | * | |
14 | * This program is free software; you can redistribute it and/or modify | |
15 | * it under the terms of the GNU General Public License as published by | |
16 | * the Free Software Foundation; either version 2, or (at your option) | |
17 | * any later version. | |
18 | * | |
19 | * This program is distributed in the hope that it will be useful, | |
20 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
21 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
22 | * GNU General Public License for more details. | |
23 | * | |
24 | * You should have received a copy of the GNU General Public License | |
25 | * along with this program; if not, write to the Free Software | |
26 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
27 | */ | |
28 | ||
d7fe0f24 | 29 | #include <linux/mm.h> |
7a291083 JBT |
30 | #include "ehea.h" |
31 | #include "ehea_phyp.h" | |
32 | #include "ehea_qmr.h" | |
33 | ||
48cfb14f | 34 | struct ehea_bmap *ehea_bmap = NULL; |
44c82152 | 35 | |
44c82152 TK |
36 | |
37 | ||
7a291083 JBT |
38 | static void *hw_qpageit_get_inc(struct hw_queue *queue) |
39 | { | |
40 | void *retvalue = hw_qeit_get(queue); | |
41 | ||
42 | queue->current_q_offset += queue->pagesize; | |
43 | if (queue->current_q_offset > queue->queue_length) { | |
44 | queue->current_q_offset -= queue->pagesize; | |
45 | retvalue = NULL; | |
46 | } else if (((u64) retvalue) & (EHEA_PAGESIZE-1)) { | |
47 | ehea_error("not on pageboundary"); | |
48 | retvalue = NULL; | |
49 | } | |
50 | return retvalue; | |
51 | } | |
52 | ||
53 | static int hw_queue_ctor(struct hw_queue *queue, const u32 nr_of_pages, | |
54 | const u32 pagesize, const u32 qe_size) | |
55 | { | |
56 | int pages_per_kpage = PAGE_SIZE / pagesize; | |
57 | int i, k; | |
58 | ||
59 | if ((pagesize > PAGE_SIZE) || (!pages_per_kpage)) { | |
60 | ehea_error("pagesize conflict! kernel pagesize=%d, " | |
61 | "ehea pagesize=%d", (int)PAGE_SIZE, (int)pagesize); | |
62 | return -EINVAL; | |
63 | } | |
64 | ||
65 | queue->queue_length = nr_of_pages * pagesize; | |
f67c6275 | 66 | queue->queue_pages = kmalloc(nr_of_pages * sizeof(void *), GFP_KERNEL); |
7a291083 JBT |
67 | if (!queue->queue_pages) { |
68 | ehea_error("no mem for queue_pages"); | |
69 | return -ENOMEM; | |
70 | } | |
71 | ||
72 | /* | |
73 | * allocate pages for queue: | |
74 | * outer loop allocates whole kernel pages (page aligned) and | |
75 | * inner loop divides a kernel page into smaller hea queue pages | |
76 | */ | |
77 | i = 0; | |
78 | while (i < nr_of_pages) { | |
f67c6275 | 79 | u8 *kpage = (u8 *)get_zeroed_page(GFP_KERNEL); |
7a291083 JBT |
80 | if (!kpage) |
81 | goto out_nomem; | |
82 | for (k = 0; k < pages_per_kpage && i < nr_of_pages; k++) { | |
f67c6275 | 83 | (queue->queue_pages)[i] = (struct ehea_page *)kpage; |
7a291083 JBT |
84 | kpage += pagesize; |
85 | i++; | |
86 | } | |
87 | } | |
88 | ||
89 | queue->current_q_offset = 0; | |
90 | queue->qe_size = qe_size; | |
91 | queue->pagesize = pagesize; | |
92 | queue->toggle_state = 1; | |
93 | ||
94 | return 0; | |
95 | out_nomem: | |
96 | for (i = 0; i < nr_of_pages; i += pages_per_kpage) { | |
97 | if (!(queue->queue_pages)[i]) | |
98 | break; | |
99 | free_page((unsigned long)(queue->queue_pages)[i]); | |
100 | } | |
101 | return -ENOMEM; | |
102 | } | |
103 | ||
104 | static void hw_queue_dtor(struct hw_queue *queue) | |
105 | { | |
106 | int pages_per_kpage = PAGE_SIZE / queue->pagesize; | |
107 | int i, nr_pages; | |
108 | ||
109 | if (!queue || !queue->queue_pages) | |
110 | return; | |
111 | ||
112 | nr_pages = queue->queue_length / queue->pagesize; | |
113 | ||
114 | for (i = 0; i < nr_pages; i += pages_per_kpage) | |
115 | free_page((unsigned long)(queue->queue_pages)[i]); | |
116 | ||
117 | kfree(queue->queue_pages); | |
118 | } | |
119 | ||
120 | struct ehea_cq *ehea_create_cq(struct ehea_adapter *adapter, | |
121 | int nr_of_cqe, u64 eq_handle, u32 cq_token) | |
122 | { | |
123 | struct ehea_cq *cq; | |
124 | struct h_epa epa; | |
125 | u64 *cq_handle_ref, hret, rpage; | |
126 | u32 act_nr_of_entries, act_pages, counter; | |
127 | int ret; | |
128 | void *vpage; | |
129 | ||
130 | cq = kzalloc(sizeof(*cq), GFP_KERNEL); | |
131 | if (!cq) { | |
132 | ehea_error("no mem for cq"); | |
133 | goto out_nomem; | |
134 | } | |
135 | ||
136 | cq->attr.max_nr_of_cqes = nr_of_cqe; | |
137 | cq->attr.cq_token = cq_token; | |
138 | cq->attr.eq_handle = eq_handle; | |
139 | ||
140 | cq->adapter = adapter; | |
141 | ||
142 | cq_handle_ref = &cq->fw_handle; | |
143 | act_nr_of_entries = 0; | |
144 | act_pages = 0; | |
145 | ||
146 | hret = ehea_h_alloc_resource_cq(adapter->handle, &cq->attr, | |
147 | &cq->fw_handle, &cq->epas); | |
148 | if (hret != H_SUCCESS) { | |
149 | ehea_error("alloc_resource_cq failed"); | |
150 | goto out_freemem; | |
151 | } | |
152 | ||
153 | ret = hw_queue_ctor(&cq->hw_queue, cq->attr.nr_pages, | |
154 | EHEA_PAGESIZE, sizeof(struct ehea_cqe)); | |
155 | if (ret) | |
156 | goto out_freeres; | |
157 | ||
158 | for (counter = 0; counter < cq->attr.nr_pages; counter++) { | |
159 | vpage = hw_qpageit_get_inc(&cq->hw_queue); | |
160 | if (!vpage) { | |
161 | ehea_error("hw_qpageit_get_inc failed"); | |
162 | goto out_kill_hwq; | |
163 | } | |
164 | ||
165 | rpage = virt_to_abs(vpage); | |
166 | hret = ehea_h_register_rpage(adapter->handle, | |
167 | 0, EHEA_CQ_REGISTER_ORIG, | |
168 | cq->fw_handle, rpage, 1); | |
169 | if (hret < H_SUCCESS) { | |
170 | ehea_error("register_rpage_cq failed ehea_cq=%p " | |
a1c5a893 | 171 | "hret=%llx counter=%i act_pages=%i", |
7a291083 JBT |
172 | cq, hret, counter, cq->attr.nr_pages); |
173 | goto out_kill_hwq; | |
174 | } | |
175 | ||
176 | if (counter == (cq->attr.nr_pages - 1)) { | |
177 | vpage = hw_qpageit_get_inc(&cq->hw_queue); | |
178 | ||
179 | if ((hret != H_SUCCESS) || (vpage)) { | |
180 | ehea_error("registration of pages not " | |
a1c5a893 | 181 | "complete hret=%llx\n", hret); |
7a291083 JBT |
182 | goto out_kill_hwq; |
183 | } | |
184 | } else { | |
662f44af | 185 | if (hret != H_PAGE_REGISTERED) { |
7a291083 | 186 | ehea_error("CQ: registration of page failed " |
a1c5a893 | 187 | "hret=%llx\n", hret); |
7a291083 JBT |
188 | goto out_kill_hwq; |
189 | } | |
190 | } | |
191 | } | |
192 | ||
193 | hw_qeit_reset(&cq->hw_queue); | |
194 | epa = cq->epas.kernel; | |
195 | ehea_reset_cq_ep(cq); | |
196 | ehea_reset_cq_n1(cq); | |
197 | ||
198 | return cq; | |
199 | ||
200 | out_kill_hwq: | |
201 | hw_queue_dtor(&cq->hw_queue); | |
202 | ||
203 | out_freeres: | |
e542aa6b | 204 | ehea_h_free_resource(adapter->handle, cq->fw_handle, FORCE_FREE); |
7a291083 JBT |
205 | |
206 | out_freemem: | |
207 | kfree(cq); | |
208 | ||
209 | out_nomem: | |
210 | return NULL; | |
211 | } | |
212 | ||
e542aa6b | 213 | u64 ehea_destroy_cq_res(struct ehea_cq *cq, u64 force) |
7a291083 | 214 | { |
e542aa6b JBT |
215 | u64 hret; |
216 | u64 adapter_handle = cq->adapter->handle; | |
217 | ||
d1d25aab | 218 | /* deregister all previous registered pages */ |
e542aa6b JBT |
219 | hret = ehea_h_free_resource(adapter_handle, cq->fw_handle, force); |
220 | if (hret != H_SUCCESS) | |
221 | return hret; | |
222 | ||
223 | hw_queue_dtor(&cq->hw_queue); | |
224 | kfree(cq); | |
225 | ||
226 | return hret; | |
227 | } | |
7a291083 | 228 | |
e542aa6b JBT |
229 | int ehea_destroy_cq(struct ehea_cq *cq) |
230 | { | |
231 | u64 hret; | |
7a291083 JBT |
232 | if (!cq) |
233 | return 0; | |
234 | ||
28721c89 | 235 | hcp_epas_dtor(&cq->epas); |
f67c6275 DM |
236 | hret = ehea_destroy_cq_res(cq, NORMAL_FREE); |
237 | if (hret == H_R_STATE) { | |
e542aa6b JBT |
238 | ehea_error_data(cq->adapter, cq->fw_handle); |
239 | hret = ehea_destroy_cq_res(cq, FORCE_FREE); | |
240 | } | |
1b5135d9 | 241 | |
7a291083 JBT |
242 | if (hret != H_SUCCESS) { |
243 | ehea_error("destroy CQ failed"); | |
244 | return -EIO; | |
245 | } | |
246 | ||
7a291083 JBT |
247 | return 0; |
248 | } | |
249 | ||
250 | struct ehea_eq *ehea_create_eq(struct ehea_adapter *adapter, | |
251 | const enum ehea_eq_type type, | |
252 | const u32 max_nr_of_eqes, const u8 eqe_gen) | |
253 | { | |
254 | int ret, i; | |
255 | u64 hret, rpage; | |
256 | void *vpage; | |
257 | struct ehea_eq *eq; | |
258 | ||
259 | eq = kzalloc(sizeof(*eq), GFP_KERNEL); | |
260 | if (!eq) { | |
261 | ehea_error("no mem for eq"); | |
262 | return NULL; | |
263 | } | |
264 | ||
265 | eq->adapter = adapter; | |
266 | eq->attr.type = type; | |
267 | eq->attr.max_nr_of_eqes = max_nr_of_eqes; | |
268 | eq->attr.eqe_gen = eqe_gen; | |
269 | spin_lock_init(&eq->spinlock); | |
270 | ||
271 | hret = ehea_h_alloc_resource_eq(adapter->handle, | |
272 | &eq->attr, &eq->fw_handle); | |
273 | if (hret != H_SUCCESS) { | |
274 | ehea_error("alloc_resource_eq failed"); | |
275 | goto out_freemem; | |
276 | } | |
277 | ||
278 | ret = hw_queue_ctor(&eq->hw_queue, eq->attr.nr_pages, | |
279 | EHEA_PAGESIZE, sizeof(struct ehea_eqe)); | |
280 | if (ret) { | |
281 | ehea_error("can't allocate eq pages"); | |
282 | goto out_freeres; | |
283 | } | |
284 | ||
285 | for (i = 0; i < eq->attr.nr_pages; i++) { | |
286 | vpage = hw_qpageit_get_inc(&eq->hw_queue); | |
287 | if (!vpage) { | |
288 | ehea_error("hw_qpageit_get_inc failed"); | |
289 | hret = H_RESOURCE; | |
290 | goto out_kill_hwq; | |
291 | } | |
292 | ||
293 | rpage = virt_to_abs(vpage); | |
294 | ||
295 | hret = ehea_h_register_rpage(adapter->handle, 0, | |
296 | EHEA_EQ_REGISTER_ORIG, | |
297 | eq->fw_handle, rpage, 1); | |
298 | ||
299 | if (i == (eq->attr.nr_pages - 1)) { | |
300 | /* last page */ | |
301 | vpage = hw_qpageit_get_inc(&eq->hw_queue); | |
f67c6275 | 302 | if ((hret != H_SUCCESS) || (vpage)) |
7a291083 | 303 | goto out_kill_hwq; |
f67c6275 | 304 | |
7a291083 | 305 | } else { |
662f44af | 306 | if (hret != H_PAGE_REGISTERED) |
7a291083 | 307 | goto out_kill_hwq; |
f67c6275 | 308 | |
7a291083 JBT |
309 | } |
310 | } | |
311 | ||
312 | hw_qeit_reset(&eq->hw_queue); | |
313 | return eq; | |
314 | ||
315 | out_kill_hwq: | |
316 | hw_queue_dtor(&eq->hw_queue); | |
317 | ||
318 | out_freeres: | |
e542aa6b | 319 | ehea_h_free_resource(adapter->handle, eq->fw_handle, FORCE_FREE); |
7a291083 JBT |
320 | |
321 | out_freemem: | |
322 | kfree(eq); | |
323 | return NULL; | |
324 | } | |
325 | ||
326 | struct ehea_eqe *ehea_poll_eq(struct ehea_eq *eq) | |
327 | { | |
328 | struct ehea_eqe *eqe; | |
329 | unsigned long flags; | |
330 | ||
331 | spin_lock_irqsave(&eq->spinlock, flags); | |
f67c6275 | 332 | eqe = (struct ehea_eqe *)hw_eqit_eq_get_inc_valid(&eq->hw_queue); |
7a291083 JBT |
333 | spin_unlock_irqrestore(&eq->spinlock, flags); |
334 | ||
335 | return eqe; | |
336 | } | |
337 | ||
e542aa6b | 338 | u64 ehea_destroy_eq_res(struct ehea_eq *eq, u64 force) |
7a291083 JBT |
339 | { |
340 | u64 hret; | |
341 | unsigned long flags; | |
342 | ||
7a291083 JBT |
343 | spin_lock_irqsave(&eq->spinlock, flags); |
344 | ||
e542aa6b | 345 | hret = ehea_h_free_resource(eq->adapter->handle, eq->fw_handle, force); |
7a291083 JBT |
346 | spin_unlock_irqrestore(&eq->spinlock, flags); |
347 | ||
e542aa6b JBT |
348 | if (hret != H_SUCCESS) |
349 | return hret; | |
7a291083 JBT |
350 | |
351 | hw_queue_dtor(&eq->hw_queue); | |
352 | kfree(eq); | |
353 | ||
e542aa6b JBT |
354 | return hret; |
355 | } | |
356 | ||
357 | int ehea_destroy_eq(struct ehea_eq *eq) | |
358 | { | |
359 | u64 hret; | |
360 | if (!eq) | |
361 | return 0; | |
362 | ||
28721c89 JBT |
363 | hcp_epas_dtor(&eq->epas); |
364 | ||
f67c6275 DM |
365 | hret = ehea_destroy_eq_res(eq, NORMAL_FREE); |
366 | if (hret == H_R_STATE) { | |
e542aa6b JBT |
367 | ehea_error_data(eq->adapter, eq->fw_handle); |
368 | hret = ehea_destroy_eq_res(eq, FORCE_FREE); | |
369 | } | |
370 | ||
371 | if (hret != H_SUCCESS) { | |
372 | ehea_error("destroy EQ failed"); | |
373 | return -EIO; | |
d1d25aab | 374 | } |
e542aa6b | 375 | |
7a291083 JBT |
376 | return 0; |
377 | } | |
378 | ||
379 | /** | |
380 | * allocates memory for a queue and registers pages in phyp | |
381 | */ | |
382 | int ehea_qp_alloc_register(struct ehea_qp *qp, struct hw_queue *hw_queue, | |
383 | int nr_pages, int wqe_size, int act_nr_sges, | |
384 | struct ehea_adapter *adapter, int h_call_q_selector) | |
385 | { | |
386 | u64 hret, rpage; | |
387 | int ret, cnt; | |
388 | void *vpage; | |
389 | ||
390 | ret = hw_queue_ctor(hw_queue, nr_pages, EHEA_PAGESIZE, wqe_size); | |
391 | if (ret) | |
392 | return ret; | |
393 | ||
394 | for (cnt = 0; cnt < nr_pages; cnt++) { | |
395 | vpage = hw_qpageit_get_inc(hw_queue); | |
396 | if (!vpage) { | |
397 | ehea_error("hw_qpageit_get_inc failed"); | |
398 | goto out_kill_hwq; | |
399 | } | |
400 | rpage = virt_to_abs(vpage); | |
401 | hret = ehea_h_register_rpage(adapter->handle, | |
402 | 0, h_call_q_selector, | |
403 | qp->fw_handle, rpage, 1); | |
404 | if (hret < H_SUCCESS) { | |
405 | ehea_error("register_rpage_qp failed"); | |
406 | goto out_kill_hwq; | |
407 | } | |
408 | } | |
409 | hw_qeit_reset(hw_queue); | |
410 | return 0; | |
411 | ||
412 | out_kill_hwq: | |
413 | hw_queue_dtor(hw_queue); | |
414 | return -EIO; | |
415 | } | |
416 | ||
417 | static inline u32 map_wqe_size(u8 wqe_enc_size) | |
418 | { | |
419 | return 128 << wqe_enc_size; | |
420 | } | |
421 | ||
422 | struct ehea_qp *ehea_create_qp(struct ehea_adapter *adapter, | |
423 | u32 pd, struct ehea_qp_init_attr *init_attr) | |
424 | { | |
425 | int ret; | |
426 | u64 hret; | |
427 | struct ehea_qp *qp; | |
428 | u32 wqe_size_in_bytes_sq, wqe_size_in_bytes_rq1; | |
429 | u32 wqe_size_in_bytes_rq2, wqe_size_in_bytes_rq3; | |
430 | ||
431 | ||
432 | qp = kzalloc(sizeof(*qp), GFP_KERNEL); | |
433 | if (!qp) { | |
434 | ehea_error("no mem for qp"); | |
435 | return NULL; | |
436 | } | |
437 | ||
438 | qp->adapter = adapter; | |
439 | ||
440 | hret = ehea_h_alloc_resource_qp(adapter->handle, init_attr, pd, | |
441 | &qp->fw_handle, &qp->epas); | |
442 | if (hret != H_SUCCESS) { | |
443 | ehea_error("ehea_h_alloc_resource_qp failed"); | |
444 | goto out_freemem; | |
445 | } | |
446 | ||
447 | wqe_size_in_bytes_sq = map_wqe_size(init_attr->act_wqe_size_enc_sq); | |
448 | wqe_size_in_bytes_rq1 = map_wqe_size(init_attr->act_wqe_size_enc_rq1); | |
449 | wqe_size_in_bytes_rq2 = map_wqe_size(init_attr->act_wqe_size_enc_rq2); | |
450 | wqe_size_in_bytes_rq3 = map_wqe_size(init_attr->act_wqe_size_enc_rq3); | |
451 | ||
452 | ret = ehea_qp_alloc_register(qp, &qp->hw_squeue, init_attr->nr_sq_pages, | |
453 | wqe_size_in_bytes_sq, | |
454 | init_attr->act_wqe_size_enc_sq, adapter, | |
455 | 0); | |
456 | if (ret) { | |
457 | ehea_error("can't register for sq ret=%x", ret); | |
458 | goto out_freeres; | |
459 | } | |
460 | ||
461 | ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue1, | |
462 | init_attr->nr_rq1_pages, | |
463 | wqe_size_in_bytes_rq1, | |
464 | init_attr->act_wqe_size_enc_rq1, | |
465 | adapter, 1); | |
466 | if (ret) { | |
467 | ehea_error("can't register for rq1 ret=%x", ret); | |
468 | goto out_kill_hwsq; | |
469 | } | |
470 | ||
471 | if (init_attr->rq_count > 1) { | |
472 | ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue2, | |
473 | init_attr->nr_rq2_pages, | |
474 | wqe_size_in_bytes_rq2, | |
475 | init_attr->act_wqe_size_enc_rq2, | |
476 | adapter, 2); | |
477 | if (ret) { | |
478 | ehea_error("can't register for rq2 ret=%x", ret); | |
479 | goto out_kill_hwr1q; | |
480 | } | |
481 | } | |
482 | ||
483 | if (init_attr->rq_count > 2) { | |
484 | ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue3, | |
485 | init_attr->nr_rq3_pages, | |
486 | wqe_size_in_bytes_rq3, | |
487 | init_attr->act_wqe_size_enc_rq3, | |
488 | adapter, 3); | |
489 | if (ret) { | |
490 | ehea_error("can't register for rq3 ret=%x", ret); | |
491 | goto out_kill_hwr2q; | |
492 | } | |
493 | } | |
494 | ||
495 | qp->init_attr = *init_attr; | |
496 | ||
497 | return qp; | |
498 | ||
499 | out_kill_hwr2q: | |
500 | hw_queue_dtor(&qp->hw_rqueue2); | |
501 | ||
502 | out_kill_hwr1q: | |
503 | hw_queue_dtor(&qp->hw_rqueue1); | |
504 | ||
505 | out_kill_hwsq: | |
506 | hw_queue_dtor(&qp->hw_squeue); | |
507 | ||
508 | out_freeres: | |
509 | ehea_h_disable_and_get_hea(adapter->handle, qp->fw_handle); | |
e542aa6b | 510 | ehea_h_free_resource(adapter->handle, qp->fw_handle, FORCE_FREE); |
7a291083 JBT |
511 | |
512 | out_freemem: | |
513 | kfree(qp); | |
514 | return NULL; | |
515 | } | |
516 | ||
e542aa6b | 517 | u64 ehea_destroy_qp_res(struct ehea_qp *qp, u64 force) |
7a291083 | 518 | { |
d1d25aab JBT |
519 | u64 hret; |
520 | struct ehea_qp_init_attr *qp_attr = &qp->init_attr; | |
7a291083 | 521 | |
7a291083 | 522 | |
d1d25aab JBT |
523 | ehea_h_disable_and_get_hea(qp->adapter->handle, qp->fw_handle); |
524 | hret = ehea_h_free_resource(qp->adapter->handle, qp->fw_handle, force); | |
525 | if (hret != H_SUCCESS) | |
526 | return hret; | |
7a291083 | 527 | |
d1d25aab JBT |
528 | hw_queue_dtor(&qp->hw_squeue); |
529 | hw_queue_dtor(&qp->hw_rqueue1); | |
7a291083 | 530 | |
d1d25aab JBT |
531 | if (qp_attr->rq_count > 1) |
532 | hw_queue_dtor(&qp->hw_rqueue2); | |
533 | if (qp_attr->rq_count > 2) | |
534 | hw_queue_dtor(&qp->hw_rqueue3); | |
535 | kfree(qp); | |
7a291083 | 536 | |
d1d25aab | 537 | return hret; |
7a291083 JBT |
538 | } |
539 | ||
e542aa6b JBT |
540 | int ehea_destroy_qp(struct ehea_qp *qp) |
541 | { | |
d1d25aab JBT |
542 | u64 hret; |
543 | if (!qp) | |
544 | return 0; | |
e542aa6b | 545 | |
28721c89 JBT |
546 | hcp_epas_dtor(&qp->epas); |
547 | ||
f67c6275 DM |
548 | hret = ehea_destroy_qp_res(qp, NORMAL_FREE); |
549 | if (hret == H_R_STATE) { | |
d1d25aab JBT |
550 | ehea_error_data(qp->adapter, qp->fw_handle); |
551 | hret = ehea_destroy_qp_res(qp, FORCE_FREE); | |
552 | } | |
e542aa6b | 553 | |
d1d25aab JBT |
554 | if (hret != H_SUCCESS) { |
555 | ehea_error("destroy QP failed"); | |
556 | return -EIO; | |
557 | } | |
e542aa6b | 558 | |
d1d25aab | 559 | return 0; |
e542aa6b JBT |
560 | } |
561 | ||
48cfb14f | 562 | static inline int ehea_calc_index(unsigned long i, unsigned long s) |
44c82152 | 563 | { |
48cfb14f HH |
564 | return (i >> s) & EHEA_INDEX_MASK; |
565 | } | |
44c82152 | 566 | |
48cfb14f HH |
567 | static inline int ehea_init_top_bmap(struct ehea_top_bmap *ehea_top_bmap, |
568 | int dir) | |
569 | { | |
d4f12daf | 570 | if (!ehea_top_bmap->dir[dir]) { |
48cfb14f HH |
571 | ehea_top_bmap->dir[dir] = |
572 | kzalloc(sizeof(struct ehea_dir_bmap), GFP_KERNEL); | |
573 | if (!ehea_top_bmap->dir[dir]) | |
574 | return -ENOMEM; | |
575 | } | |
576 | return 0; | |
577 | } | |
44c82152 | 578 | |
48cfb14f HH |
579 | static inline int ehea_init_bmap(struct ehea_bmap *ehea_bmap, int top, int dir) |
580 | { | |
d4f12daf | 581 | if (!ehea_bmap->top[top]) { |
48cfb14f HH |
582 | ehea_bmap->top[top] = |
583 | kzalloc(sizeof(struct ehea_top_bmap), GFP_KERNEL); | |
584 | if (!ehea_bmap->top[top]) | |
585 | return -ENOMEM; | |
586 | } | |
587 | return ehea_init_top_bmap(ehea_bmap->top[top], dir); | |
588 | } | |
44c82152 | 589 | |
d4f12daf HH |
590 | static DEFINE_MUTEX(ehea_busmap_mutex); |
591 | static unsigned long ehea_mr_len; | |
44c82152 | 592 | |
d4f12daf HH |
593 | #define EHEA_BUSMAP_ADD_SECT 1 |
594 | #define EHEA_BUSMAP_REM_SECT 0 | |
44c82152 | 595 | |
d4f12daf HH |
596 | static void ehea_rebuild_busmap(void) |
597 | { | |
598 | u64 vaddr = EHEA_BUSMAP_START; | |
599 | int top, dir, idx; | |
48cfb14f | 600 | |
d4f12daf HH |
601 | for (top = 0; top < EHEA_MAP_ENTRIES; top++) { |
602 | struct ehea_top_bmap *ehea_top; | |
603 | int valid_dir_entries = 0; | |
48cfb14f | 604 | |
d4f12daf HH |
605 | if (!ehea_bmap->top[top]) |
606 | continue; | |
607 | ehea_top = ehea_bmap->top[top]; | |
608 | for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) { | |
609 | struct ehea_dir_bmap *ehea_dir; | |
610 | int valid_entries = 0; | |
611 | ||
612 | if (!ehea_top->dir[dir]) | |
613 | continue; | |
614 | valid_dir_entries++; | |
615 | ehea_dir = ehea_top->dir[dir]; | |
616 | for (idx = 0; idx < EHEA_MAP_ENTRIES; idx++) { | |
617 | if (!ehea_dir->ent[idx]) | |
618 | continue; | |
619 | valid_entries++; | |
620 | ehea_dir->ent[idx] = vaddr; | |
621 | vaddr += EHEA_SECTSIZE; | |
622 | } | |
623 | if (!valid_entries) { | |
624 | ehea_top->dir[dir] = NULL; | |
625 | kfree(ehea_dir); | |
626 | } | |
627 | } | |
628 | if (!valid_dir_entries) { | |
629 | ehea_bmap->top[top] = NULL; | |
630 | kfree(ehea_top); | |
631 | } | |
632 | } | |
633 | } | |
48cfb14f | 634 | |
3fd09c45 | 635 | static int ehea_update_busmap(unsigned long pfn, unsigned long nr_pages, int add) |
d4f12daf HH |
636 | { |
637 | unsigned long i, start_section, end_section; | |
44c82152 | 638 | |
3fd09c45 TK |
639 | if (!nr_pages) |
640 | return 0; | |
641 | ||
d4f12daf HH |
642 | if (!ehea_bmap) { |
643 | ehea_bmap = kzalloc(sizeof(struct ehea_bmap), GFP_KERNEL); | |
644 | if (!ehea_bmap) | |
645 | return -ENOMEM; | |
44c82152 TK |
646 | } |
647 | ||
d4f12daf | 648 | start_section = (pfn * PAGE_SIZE) / EHEA_SECTSIZE; |
3fd09c45 | 649 | end_section = start_section + ((nr_pages * PAGE_SIZE) / EHEA_SECTSIZE); |
d4f12daf HH |
650 | /* Mark entries as valid or invalid only; address is assigned later */ |
651 | for (i = start_section; i < end_section; i++) { | |
652 | u64 flag; | |
653 | int top = ehea_calc_index(i, EHEA_TOP_INDEX_SHIFT); | |
654 | int dir = ehea_calc_index(i, EHEA_DIR_INDEX_SHIFT); | |
655 | int idx = i & EHEA_INDEX_MASK; | |
c5916cf8 | 656 | |
d4f12daf HH |
657 | if (add) { |
658 | int ret = ehea_init_bmap(ehea_bmap, top, dir); | |
659 | if (ret) | |
660 | return ret; | |
661 | flag = 1; /* valid */ | |
662 | ehea_mr_len += EHEA_SECTSIZE; | |
663 | } else { | |
664 | if (!ehea_bmap->top[top]) | |
665 | continue; | |
666 | if (!ehea_bmap->top[top]->dir[dir]) | |
667 | continue; | |
668 | flag = 0; /* invalid */ | |
669 | ehea_mr_len -= EHEA_SECTSIZE; | |
670 | } | |
48cfb14f | 671 | |
d4f12daf HH |
672 | ehea_bmap->top[top]->dir[dir]->ent[idx] = flag; |
673 | } | |
674 | ehea_rebuild_busmap(); /* Assign contiguous addresses for mr */ | |
44c82152 TK |
675 | return 0; |
676 | } | |
677 | ||
d4f12daf HH |
678 | int ehea_add_sect_bmap(unsigned long pfn, unsigned long nr_pages) |
679 | { | |
680 | int ret; | |
48cfb14f | 681 | |
d4f12daf HH |
682 | mutex_lock(&ehea_busmap_mutex); |
683 | ret = ehea_update_busmap(pfn, nr_pages, EHEA_BUSMAP_ADD_SECT); | |
684 | mutex_unlock(&ehea_busmap_mutex); | |
685 | return ret; | |
686 | } | |
687 | ||
688 | int ehea_rem_sect_bmap(unsigned long pfn, unsigned long nr_pages) | |
689 | { | |
690 | int ret; | |
691 | ||
692 | mutex_lock(&ehea_busmap_mutex); | |
693 | ret = ehea_update_busmap(pfn, nr_pages, EHEA_BUSMAP_REM_SECT); | |
694 | mutex_unlock(&ehea_busmap_mutex); | |
695 | return ret; | |
696 | } | |
697 | ||
3fd09c45 TK |
698 | static int ehea_is_hugepage(unsigned long pfn) |
699 | { | |
700 | int page_order; | |
701 | ||
702 | if (pfn & EHEA_HUGEPAGE_PFN_MASK) | |
703 | return 0; | |
704 | ||
705 | page_order = compound_order(pfn_to_page(pfn)); | |
706 | if (page_order + PAGE_SHIFT != EHEA_HUGEPAGESHIFT) | |
707 | return 0; | |
708 | ||
709 | return 1; | |
710 | } | |
711 | ||
712 | static int ehea_create_busmap_callback(unsigned long initial_pfn, | |
713 | unsigned long total_nr_pages, void *arg) | |
d4f12daf | 714 | { |
3fd09c45 TK |
715 | int ret; |
716 | unsigned long pfn, start_pfn, end_pfn, nr_pages; | |
717 | ||
718 | if ((total_nr_pages * PAGE_SIZE) < EHEA_HUGEPAGE_SIZE) | |
719 | return ehea_update_busmap(initial_pfn, total_nr_pages, | |
720 | EHEA_BUSMAP_ADD_SECT); | |
721 | ||
722 | /* Given chunk is >= 16GB -> check for hugepages */ | |
723 | start_pfn = initial_pfn; | |
724 | end_pfn = initial_pfn + total_nr_pages; | |
725 | pfn = start_pfn; | |
726 | ||
727 | while (pfn < end_pfn) { | |
728 | if (ehea_is_hugepage(pfn)) { | |
729 | /* Add mem found in front of the hugepage */ | |
730 | nr_pages = pfn - start_pfn; | |
731 | ret = ehea_update_busmap(start_pfn, nr_pages, | |
732 | EHEA_BUSMAP_ADD_SECT); | |
733 | if (ret) | |
734 | return ret; | |
735 | ||
736 | /* Skip the hugepage */ | |
737 | pfn += (EHEA_HUGEPAGE_SIZE / PAGE_SIZE); | |
738 | start_pfn = pfn; | |
739 | } else | |
740 | pfn += (EHEA_SECTSIZE / PAGE_SIZE); | |
741 | } | |
742 | ||
743 | /* Add mem found behind the hugepage(s) */ | |
744 | nr_pages = pfn - start_pfn; | |
745 | return ehea_update_busmap(start_pfn, nr_pages, EHEA_BUSMAP_ADD_SECT); | |
d4f12daf | 746 | } |
48cfb14f HH |
747 | |
748 | int ehea_create_busmap(void) | |
749 | { | |
750 | int ret; | |
d4f12daf | 751 | |
48cfb14f HH |
752 | mutex_lock(&ehea_busmap_mutex); |
753 | ehea_mr_len = 0; | |
908eedc6 | 754 | ret = walk_system_ram_range(0, 1ULL << MAX_PHYSMEM_BITS, NULL, |
48cfb14f HH |
755 | ehea_create_busmap_callback); |
756 | mutex_unlock(&ehea_busmap_mutex); | |
757 | return ret; | |
758 | } | |
759 | ||
f67c6275 | 760 | void ehea_destroy_busmap(void) |
44c82152 | 761 | { |
48cfb14f HH |
762 | int top, dir; |
763 | mutex_lock(&ehea_busmap_mutex); | |
764 | if (!ehea_bmap) | |
765 | goto out_destroy; | |
766 | ||
767 | for (top = 0; top < EHEA_MAP_ENTRIES; top++) { | |
768 | if (!ehea_bmap->top[top]) | |
769 | continue; | |
770 | ||
771 | for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) { | |
772 | if (!ehea_bmap->top[top]->dir[dir]) | |
773 | continue; | |
774 | ||
775 | kfree(ehea_bmap->top[top]->dir[dir]); | |
776 | } | |
777 | ||
778 | kfree(ehea_bmap->top[top]); | |
779 | } | |
780 | ||
781 | kfree(ehea_bmap); | |
782 | ehea_bmap = NULL; | |
c5916cf8 | 783 | out_destroy: |
48cfb14f | 784 | mutex_unlock(&ehea_busmap_mutex); |
44c82152 TK |
785 | } |
786 | ||
787 | u64 ehea_map_vaddr(void *caddr) | |
788 | { | |
48cfb14f HH |
789 | int top, dir, idx; |
790 | unsigned long index, offset; | |
791 | ||
792 | if (!ehea_bmap) | |
793 | return EHEA_INVAL_ADDR; | |
794 | ||
795 | index = virt_to_abs(caddr) >> SECTION_SIZE_BITS; | |
796 | top = (index >> EHEA_TOP_INDEX_SHIFT) & EHEA_INDEX_MASK; | |
797 | if (!ehea_bmap->top[top]) | |
798 | return EHEA_INVAL_ADDR; | |
799 | ||
800 | dir = (index >> EHEA_DIR_INDEX_SHIFT) & EHEA_INDEX_MASK; | |
801 | if (!ehea_bmap->top[top]->dir[dir]) | |
802 | return EHEA_INVAL_ADDR; | |
803 | ||
804 | idx = index & EHEA_INDEX_MASK; | |
805 | if (!ehea_bmap->top[top]->dir[dir]->ent[idx]) | |
806 | return EHEA_INVAL_ADDR; | |
807 | ||
808 | offset = (unsigned long)caddr & (EHEA_SECTSIZE - 1); | |
809 | return ehea_bmap->top[top]->dir[dir]->ent[idx] | offset; | |
810 | } | |
811 | ||
812 | static inline void *ehea_calc_sectbase(int top, int dir, int idx) | |
813 | { | |
814 | unsigned long ret = idx; | |
815 | ret |= dir << EHEA_DIR_INDEX_SHIFT; | |
816 | ret |= top << EHEA_TOP_INDEX_SHIFT; | |
817 | return abs_to_virt(ret << SECTION_SIZE_BITS); | |
818 | } | |
819 | ||
820 | static u64 ehea_reg_mr_section(int top, int dir, int idx, u64 *pt, | |
821 | struct ehea_adapter *adapter, | |
822 | struct ehea_mr *mr) | |
823 | { | |
824 | void *pg; | |
825 | u64 j, m, hret; | |
826 | unsigned long k = 0; | |
827 | u64 pt_abs = virt_to_abs(pt); | |
828 | ||
829 | void *sectbase = ehea_calc_sectbase(top, dir, idx); | |
830 | ||
831 | for (j = 0; j < (EHEA_PAGES_PER_SECTION / EHEA_MAX_RPAGE); j++) { | |
832 | ||
833 | for (m = 0; m < EHEA_MAX_RPAGE; m++) { | |
834 | pg = sectbase + ((k++) * EHEA_PAGESIZE); | |
835 | pt[m] = virt_to_abs(pg); | |
836 | } | |
837 | hret = ehea_h_register_rpage_mr(adapter->handle, mr->handle, 0, | |
838 | 0, pt_abs, EHEA_MAX_RPAGE); | |
839 | ||
840 | if ((hret != H_SUCCESS) | |
841 | && (hret != H_PAGE_REGISTERED)) { | |
842 | ehea_h_free_resource(adapter->handle, mr->handle, | |
843 | FORCE_FREE); | |
844 | ehea_error("register_rpage_mr failed"); | |
845 | return hret; | |
846 | } | |
847 | } | |
848 | return hret; | |
849 | } | |
850 | ||
851 | static u64 ehea_reg_mr_sections(int top, int dir, u64 *pt, | |
852 | struct ehea_adapter *adapter, | |
853 | struct ehea_mr *mr) | |
854 | { | |
855 | u64 hret = H_SUCCESS; | |
856 | int idx; | |
857 | ||
858 | for (idx = 0; idx < EHEA_MAP_ENTRIES; idx++) { | |
859 | if (!ehea_bmap->top[top]->dir[dir]->ent[idx]) | |
860 | continue; | |
c5916cf8 | 861 | |
48cfb14f HH |
862 | hret = ehea_reg_mr_section(top, dir, idx, pt, adapter, mr); |
863 | if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED)) | |
c5916cf8 | 864 | return hret; |
48cfb14f HH |
865 | } |
866 | return hret; | |
867 | } | |
868 | ||
869 | static u64 ehea_reg_mr_dir_sections(int top, u64 *pt, | |
870 | struct ehea_adapter *adapter, | |
871 | struct ehea_mr *mr) | |
872 | { | |
873 | u64 hret = H_SUCCESS; | |
874 | int dir; | |
875 | ||
876 | for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) { | |
877 | if (!ehea_bmap->top[top]->dir[dir]) | |
878 | continue; | |
879 | ||
880 | hret = ehea_reg_mr_sections(top, dir, pt, adapter, mr); | |
881 | if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED)) | |
c5916cf8 | 882 | return hret; |
48cfb14f HH |
883 | } |
884 | return hret; | |
44c82152 TK |
885 | } |
886 | ||
e542aa6b | 887 | int ehea_reg_kernel_mr(struct ehea_adapter *adapter, struct ehea_mr *mr) |
7a291083 | 888 | { |
44c82152 | 889 | int ret; |
7a291083 | 890 | u64 *pt; |
48cfb14f | 891 | u64 hret; |
44c82152 | 892 | u32 acc_ctrl = EHEA_MR_ACC_CTRL; |
7a291083 | 893 | |
48cfb14f | 894 | unsigned long top; |
7a291083 | 895 | |
d76e56b4 | 896 | pt = (void *)get_zeroed_page(GFP_KERNEL); |
7a291083 JBT |
897 | if (!pt) { |
898 | ehea_error("no mem"); | |
899 | ret = -ENOMEM; | |
900 | goto out; | |
901 | } | |
7a291083 | 902 | |
48cfb14f HH |
903 | hret = ehea_h_alloc_resource_mr(adapter->handle, EHEA_BUSMAP_START, |
904 | ehea_mr_len, acc_ctrl, adapter->pd, | |
e542aa6b | 905 | &mr->handle, &mr->lkey); |
48cfb14f | 906 | |
7a291083 JBT |
907 | if (hret != H_SUCCESS) { |
908 | ehea_error("alloc_resource_mr failed"); | |
909 | ret = -EIO; | |
910 | goto out; | |
911 | } | |
912 | ||
48cfb14f HH |
913 | if (!ehea_bmap) { |
914 | ehea_h_free_resource(adapter->handle, mr->handle, FORCE_FREE); | |
915 | ehea_error("no busmap available"); | |
916 | ret = -EIO; | |
917 | goto out; | |
918 | } | |
919 | ||
920 | for (top = 0; top < EHEA_MAP_ENTRIES; top++) { | |
921 | if (!ehea_bmap->top[top]) | |
922 | continue; | |
923 | ||
924 | hret = ehea_reg_mr_dir_sections(top, pt, adapter, mr); | |
925 | if((hret != H_PAGE_REGISTERED) && (hret != H_SUCCESS)) | |
926 | break; | |
927 | } | |
7a291083 JBT |
928 | |
929 | if (hret != H_SUCCESS) { | |
44c82152 TK |
930 | ehea_h_free_resource(adapter->handle, mr->handle, FORCE_FREE); |
931 | ehea_error("registering mr failed"); | |
7a291083 JBT |
932 | ret = -EIO; |
933 | goto out; | |
934 | } | |
e542aa6b | 935 | |
44c82152 | 936 | mr->vaddr = EHEA_BUSMAP_START; |
e542aa6b | 937 | mr->adapter = adapter; |
7a291083 JBT |
938 | ret = 0; |
939 | out: | |
d76e56b4 | 940 | free_page((unsigned long)pt); |
7a291083 JBT |
941 | return ret; |
942 | } | |
943 | ||
e542aa6b JBT |
944 | int ehea_rem_mr(struct ehea_mr *mr) |
945 | { | |
946 | u64 hret; | |
947 | ||
948 | if (!mr || !mr->adapter) | |
949 | return -EINVAL; | |
950 | ||
951 | hret = ehea_h_free_resource(mr->adapter->handle, mr->handle, | |
952 | FORCE_FREE); | |
953 | if (hret != H_SUCCESS) { | |
954 | ehea_error("destroy MR failed"); | |
955 | return -EIO; | |
956 | } | |
957 | ||
958 | return 0; | |
959 | } | |
960 | ||
961 | int ehea_gen_smr(struct ehea_adapter *adapter, struct ehea_mr *old_mr, | |
962 | struct ehea_mr *shared_mr) | |
963 | { | |
964 | u64 hret; | |
965 | ||
966 | hret = ehea_h_register_smr(adapter->handle, old_mr->handle, | |
967 | old_mr->vaddr, EHEA_MR_ACC_CTRL, | |
968 | adapter->pd, shared_mr); | |
969 | if (hret != H_SUCCESS) | |
970 | return -EIO; | |
971 | ||
972 | shared_mr->adapter = adapter; | |
973 | ||
974 | return 0; | |
975 | } | |
976 | ||
d2db9eea JBT |
977 | void print_error_data(u64 *data) |
978 | { | |
979 | int length; | |
980 | u64 type = EHEA_BMASK_GET(ERROR_DATA_TYPE, data[2]); | |
981 | u64 resource = data[1]; | |
982 | ||
983 | length = EHEA_BMASK_GET(ERROR_DATA_LENGTH, data[0]); | |
984 | ||
985 | if (length > EHEA_PAGESIZE) | |
986 | length = EHEA_PAGESIZE; | |
987 | ||
988 | if (type == 0x8) /* Queue Pair */ | |
a1c5a893 SR |
989 | ehea_error("QP (resource=%llX) state: AER=0x%llX, AERR=0x%llX, " |
990 | "port=%llX", resource, data[6], data[12], data[22]); | |
d2db9eea | 991 | |
e542aa6b | 992 | if (type == 0x4) /* Completion Queue */ |
a1c5a893 | 993 | ehea_error("CQ (resource=%llX) state: AER=0x%llX", resource, |
e542aa6b JBT |
994 | data[6]); |
995 | ||
996 | if (type == 0x3) /* Event Queue */ | |
a1c5a893 | 997 | ehea_error("EQ (resource=%llX) state: AER=0x%llX", resource, |
e542aa6b JBT |
998 | data[6]); |
999 | ||
d2db9eea JBT |
1000 | ehea_dump(data, length, "error data"); |
1001 | } | |
1002 | ||
1003 | void ehea_error_data(struct ehea_adapter *adapter, u64 res_handle) | |
1004 | { | |
1005 | unsigned long ret; | |
1006 | u64 *rblock; | |
1007 | ||
3faf2693 | 1008 | rblock = (void *)get_zeroed_page(GFP_KERNEL); |
d2db9eea JBT |
1009 | if (!rblock) { |
1010 | ehea_error("Cannot allocate rblock memory."); | |
1011 | return; | |
1012 | } | |
7a291083 | 1013 | |
d2db9eea JBT |
1014 | ret = ehea_h_error_data(adapter->handle, |
1015 | res_handle, | |
1016 | rblock); | |
1017 | ||
1018 | if (ret == H_R_STATE) | |
a1c5a893 | 1019 | ehea_error("No error data is available: %llX.", res_handle); |
d2db9eea JBT |
1020 | else if (ret == H_SUCCESS) |
1021 | print_error_data(rblock); | |
1022 | else | |
a1c5a893 | 1023 | ehea_error("Error data could not be fetched: %llX", res_handle); |
d2db9eea | 1024 | |
3faf2693 | 1025 | free_page((unsigned long)rblock); |
d2db9eea | 1026 | } |