Commit | Line | Data |
---|---|---|
40cb5942 SD |
1 | /* |
2 | * Intel MIC Platform Software Stack (MPSS) | |
3 | * | |
4 | * Copyright(c) 2014 Intel Corporation. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License, version 2, as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License for more details. | |
14 | * | |
15 | * Intel SCIF driver. | |
16 | * | |
17 | */ | |
18 | #include "../bus/scif_bus.h" | |
19 | #include "scif_peer_bus.h" | |
20 | #include "scif_main.h" | |
21 | #include "scif_nodeqp.h" | |
22 | #include "scif_map.h" | |
23 | ||
24 | /* | |
25 | ************************************************************************ | |
26 | * SCIF node Queue Pair (QP) setup flow: | |
27 | * | |
28 | * 1) SCIF driver gets probed with a scif_hw_dev via the scif_hw_bus | |
29 | * 2) scif_setup_qp(..) allocates the local qp and calls | |
30 | * scif_setup_qp_connect(..) which allocates and maps the local | |
31 | * buffer for the inbound QP | |
32 | * 3) The local node updates the device page with the DMA address of the QP | |
33 | * 4) A delayed work is scheduled (qp_dwork) which periodically reads if | |
34 | * the peer node has updated its QP DMA address | |
35 | * 5) Once a valid non zero address is found in the QP DMA address field | |
36 | * in the device page, the local node maps the remote node's QP, | |
37 | * updates its outbound QP and sends a SCIF_INIT message to the peer | |
38 | * 6) The SCIF_INIT message is received by the peer node QP interrupt bottom | |
39 | * half handler by calling scif_init(..) | |
40 | * 7) scif_init(..) registers a new SCIF peer node by calling | |
41 | * scif_peer_register_device(..) which signifies the addition of a new | |
42 | * SCIF node | |
43 | * 8) On the mgmt node, P2P network setup/teardown is initiated if all the | |
44 | * remote nodes are online via scif_p2p_setup(..) | |
45 | * 9) For P2P setup, the host maps the remote nodes' aperture and memory | |
46 | * bars and sends a SCIF_NODE_ADD message to both nodes | |
47 | * 10) As part of scif_nodeadd, both nodes set up their local inbound | |
48 | * QPs and send a SCIF_NODE_ADD_ACK to the mgmt node | |
49 | * 11) As part of scif_node_add_ack(..) the mgmt node forwards the | |
50 | * SCIF_NODE_ADD_ACK to the remote nodes | |
51 | * 12) As part of scif_node_add_ack(..) the remote nodes update their | |
52 | * outbound QPs, make sure they can access memory on the remote node | |
53 | * and then add a new SCIF peer node by calling | |
54 | * scif_peer_register_device(..) which signifies the addition of a new | |
55 | * SCIF node. | |
56 | * 13) The SCIF network is now established across all nodes. | |
57 | * | |
58 | ************************************************************************ | |
59 | * SCIF node QP teardown flow (initiated by non mgmt node): | |
60 | * | |
61 | * 1) SCIF driver gets a remove callback with a scif_hw_dev via the scif_hw_bus | |
62 | * 2) The device page QP DMA address field is updated with 0x0 | |
63 | * 3) A non mgmt node now cleans up all local data structures and sends a | |
64 | * SCIF_EXIT message to the peer and waits for a SCIF_EXIT_ACK | |
65 | * 4) As part of scif_exit(..) handling scif_disconnect_node(..) is called | |
66 | * 5) scif_disconnect_node(..) sends a SCIF_NODE_REMOVE message to all the | |
67 | * peers and waits for a SCIF_NODE_REMOVE_ACK | |
68 | * 6) As part of scif_node_remove(..) a remote node unregisters the peer | |
69 | * node from the SCIF network and sends a SCIF_NODE_REMOVE_ACK | |
70 | * 7) When the mgmt node has received all the SCIF_NODE_REMOVE_ACKs | |
71 | * it sends itself a node remove message whose handling cleans up local | |
72 | * data structures and unregisters the peer node from the SCIF network | |
73 | * 8) The mgmt node sends a SCIF_EXIT_ACK | |
74 | * 9) Upon receipt of the SCIF_EXIT_ACK the node initiating the teardown | |
75 | * completes the SCIF remove routine | |
76 | * 10) The SCIF network is now torn down for the node initiating the | |
77 | * teardown sequence | |
78 | * | |
79 | ************************************************************************ | |
80 | * SCIF node QP teardown flow (initiated by mgmt node): | |
81 | * | |
82 | * 1) SCIF driver gets a remove callback with a scif_hw_dev via the scif_hw_bus | |
83 | * 2) The device page QP DMA address field is updated with 0x0 | |
84 | * 3) The mgmt node calls scif_disconnect_node(..) | |
85 | * 4) scif_disconnect_node(..) sends a SCIF_NODE_REMOVE message to all the peers | |
86 | * and waits for a SCIF_NODE_REMOVE_ACK | |
87 | * 5) As part of scif_node_remove(..) a remote node unregisters the peer | |
88 | * node from the SCIF network and sends a SCIF_NODE_REMOVE_ACK | |
89 | * 6) When the mgmt node has received all the SCIF_NODE_REMOVE_ACKs | |
90 | * it unregisters the peer node from the SCIF network | |
91 | * 7) The mgmt node sends a SCIF_EXIT message and waits for a SCIF_EXIT_ACK. | |
92 | * 8) A non mgmt node upon receipt of a SCIF_EXIT message calls scif_stop(..) | |
93 | * which would clean up local data structures for all SCIF nodes and | |
94 | * then send a SCIF_EXIT_ACK back to the mgmt node | |
95 | * 9) Upon receipt of the SCIF_EXIT_ACK the the mgmt node sends itself a node | |
96 | * remove message whose handling cleans up local data structures and | |
97 | * destroys any P2P mappings. | |
98 | * 10) The SCIF hardware device for which a remove callback was received is now | |
99 | * disconnected from the SCIF network. | |
100 | */ | |
101 | /* | |
102 | * Initializes "local" data structures for the QP. Allocates the QP | |
103 | * ring buffer (rb) and initializes the "in bound" queue. | |
104 | */ | |
105 | int scif_setup_qp_connect(struct scif_qp *qp, dma_addr_t *qp_offset, | |
106 | int local_size, struct scif_dev *scifdev) | |
107 | { | |
108 | void *local_q = NULL; | |
109 | int err = 0; | |
110 | u32 tmp_rd = 0; | |
111 | ||
112 | spin_lock_init(&qp->send_lock); | |
113 | spin_lock_init(&qp->recv_lock); | |
114 | ||
115 | local_q = kzalloc(local_size, GFP_KERNEL); | |
116 | if (!local_q) { | |
117 | err = -ENOMEM; | |
118 | return err; | |
119 | } | |
120 | err = scif_map_single(&qp->local_buf, local_q, scifdev, local_size); | |
121 | if (err) | |
122 | goto kfree; | |
123 | /* | |
124 | * To setup the inbound_q, the buffer lives locally, the read pointer | |
125 | * is remote and the write pointer is local. | |
126 | */ | |
127 | scif_rb_init(&qp->inbound_q, | |
128 | &tmp_rd, | |
129 | &qp->local_write, | |
130 | local_q, get_count_order(local_size)); | |
131 | /* | |
132 | * The read pointer is NULL initially and it is unsafe to use the ring | |
133 | * buffer til this changes! | |
134 | */ | |
135 | qp->inbound_q.read_ptr = NULL; | |
136 | err = scif_map_single(qp_offset, qp, | |
137 | scifdev, sizeof(struct scif_qp)); | |
138 | if (err) | |
139 | goto unmap; | |
140 | qp->local_qp = *qp_offset; | |
141 | return err; | |
142 | unmap: | |
143 | scif_unmap_single(qp->local_buf, scifdev, local_size); | |
144 | qp->local_buf = 0; | |
145 | kfree: | |
146 | kfree(local_q); | |
147 | return err; | |
148 | } | |
149 | ||
150 | /* When the other side has already done it's allocation, this is called */ | |
151 | int scif_setup_qp_accept(struct scif_qp *qp, dma_addr_t *qp_offset, | |
152 | dma_addr_t phys, int local_size, | |
153 | struct scif_dev *scifdev) | |
154 | { | |
155 | void *local_q; | |
156 | void *remote_q; | |
157 | struct scif_qp *remote_qp; | |
158 | int remote_size; | |
159 | int err = 0; | |
160 | ||
161 | spin_lock_init(&qp->send_lock); | |
162 | spin_lock_init(&qp->recv_lock); | |
163 | /* Start by figuring out where we need to point */ | |
164 | remote_qp = scif_ioremap(phys, sizeof(struct scif_qp), scifdev); | |
165 | if (!remote_qp) | |
166 | return -EIO; | |
167 | qp->remote_qp = remote_qp; | |
168 | if (qp->remote_qp->magic != SCIFEP_MAGIC) { | |
169 | err = -EIO; | |
170 | goto iounmap; | |
171 | } | |
172 | qp->remote_buf = remote_qp->local_buf; | |
173 | remote_size = qp->remote_qp->inbound_q.size; | |
174 | remote_q = scif_ioremap(qp->remote_buf, remote_size, scifdev); | |
175 | if (!remote_q) { | |
176 | err = -EIO; | |
177 | goto iounmap; | |
178 | } | |
179 | qp->remote_qp->local_write = 0; | |
180 | /* | |
181 | * To setup the outbound_q, the buffer lives in remote memory, | |
182 | * the read pointer is local, the write pointer is remote | |
183 | */ | |
184 | scif_rb_init(&qp->outbound_q, | |
185 | &qp->local_read, | |
186 | &qp->remote_qp->local_write, | |
187 | remote_q, | |
188 | get_count_order(remote_size)); | |
189 | local_q = kzalloc(local_size, GFP_KERNEL); | |
190 | if (!local_q) { | |
191 | err = -ENOMEM; | |
192 | goto iounmap_1; | |
193 | } | |
194 | err = scif_map_single(&qp->local_buf, local_q, scifdev, local_size); | |
195 | if (err) | |
196 | goto kfree; | |
197 | qp->remote_qp->local_read = 0; | |
198 | /* | |
199 | * To setup the inbound_q, the buffer lives locally, the read pointer | |
200 | * is remote and the write pointer is local | |
201 | */ | |
202 | scif_rb_init(&qp->inbound_q, | |
203 | &qp->remote_qp->local_read, | |
204 | &qp->local_write, | |
205 | local_q, get_count_order(local_size)); | |
206 | err = scif_map_single(qp_offset, qp, scifdev, | |
207 | sizeof(struct scif_qp)); | |
208 | if (err) | |
209 | goto unmap; | |
210 | qp->local_qp = *qp_offset; | |
211 | return err; | |
212 | unmap: | |
213 | scif_unmap_single(qp->local_buf, scifdev, local_size); | |
214 | qp->local_buf = 0; | |
215 | kfree: | |
216 | kfree(local_q); | |
217 | iounmap_1: | |
218 | scif_iounmap(remote_q, remote_size, scifdev); | |
219 | qp->outbound_q.rb_base = NULL; | |
220 | iounmap: | |
221 | scif_iounmap(qp->remote_qp, sizeof(struct scif_qp), scifdev); | |
222 | qp->remote_qp = NULL; | |
223 | return err; | |
224 | } | |
225 | ||
226 | int scif_setup_qp_connect_response(struct scif_dev *scifdev, | |
227 | struct scif_qp *qp, u64 payload) | |
228 | { | |
229 | int err = 0; | |
230 | void *r_buf; | |
231 | int remote_size; | |
232 | phys_addr_t tmp_phys; | |
233 | ||
234 | qp->remote_qp = scif_ioremap(payload, sizeof(struct scif_qp), scifdev); | |
235 | ||
236 | if (!qp->remote_qp) { | |
237 | err = -ENOMEM; | |
238 | goto error; | |
239 | } | |
240 | ||
241 | if (qp->remote_qp->magic != SCIFEP_MAGIC) { | |
242 | dev_err(&scifdev->sdev->dev, | |
243 | "SCIFEP_MAGIC mismatch between self %d remote %d\n", | |
244 | scif_dev[scif_info.nodeid].node, scifdev->node); | |
245 | err = -ENODEV; | |
246 | goto error; | |
247 | } | |
248 | ||
249 | tmp_phys = qp->remote_qp->local_buf; | |
250 | remote_size = qp->remote_qp->inbound_q.size; | |
251 | r_buf = scif_ioremap(tmp_phys, remote_size, scifdev); | |
252 | ||
253 | if (!r_buf) | |
254 | return -EIO; | |
255 | ||
256 | qp->local_read = 0; | |
257 | scif_rb_init(&qp->outbound_q, | |
258 | &qp->local_read, | |
259 | &qp->remote_qp->local_write, | |
260 | r_buf, | |
261 | get_count_order(remote_size)); | |
262 | /* | |
263 | * resetup the inbound_q now that we know where the | |
264 | * inbound_read really is. | |
265 | */ | |
266 | scif_rb_init(&qp->inbound_q, | |
267 | &qp->remote_qp->local_read, | |
268 | &qp->local_write, | |
269 | qp->inbound_q.rb_base, | |
270 | get_count_order(qp->inbound_q.size)); | |
271 | error: | |
272 | return err; | |
273 | } | |
274 | ||
275 | static __always_inline void | |
276 | scif_send_msg_intr(struct scif_dev *scifdev) | |
277 | { | |
278 | struct scif_hw_dev *sdev = scifdev->sdev; | |
279 | ||
280 | if (scifdev_is_p2p(scifdev)) | |
281 | sdev->hw_ops->send_p2p_intr(sdev, scifdev->rdb, &scifdev->mmio); | |
282 | else | |
283 | sdev->hw_ops->send_intr(sdev, scifdev->rdb); | |
284 | } | |
285 | ||
286 | int scif_qp_response(phys_addr_t phys, struct scif_dev *scifdev) | |
287 | { | |
288 | int err = 0; | |
289 | struct scifmsg msg; | |
290 | ||
291 | err = scif_setup_qp_connect_response(scifdev, scifdev->qpairs, phys); | |
292 | if (!err) { | |
293 | /* | |
294 | * Now that everything is setup and mapped, we're ready | |
295 | * to tell the peer about our queue's location | |
296 | */ | |
297 | msg.uop = SCIF_INIT; | |
298 | msg.dst.node = scifdev->node; | |
299 | err = scif_nodeqp_send(scifdev, &msg); | |
300 | } | |
301 | return err; | |
302 | } | |
303 | ||
304 | void scif_send_exit(struct scif_dev *scifdev) | |
305 | { | |
306 | struct scifmsg msg; | |
307 | int ret; | |
308 | ||
309 | scifdev->exit = OP_IN_PROGRESS; | |
310 | msg.uop = SCIF_EXIT; | |
311 | msg.src.node = scif_info.nodeid; | |
312 | msg.dst.node = scifdev->node; | |
313 | ret = scif_nodeqp_send(scifdev, &msg); | |
314 | if (ret) | |
315 | goto done; | |
316 | /* Wait for a SCIF_EXIT_ACK message */ | |
317 | wait_event_timeout(scif_info.exitwq, scifdev->exit == OP_COMPLETED, | |
318 | SCIF_NODE_ALIVE_TIMEOUT); | |
319 | done: | |
320 | scifdev->exit = OP_IDLE; | |
321 | } | |
322 | ||
323 | int scif_setup_qp(struct scif_dev *scifdev) | |
324 | { | |
325 | int err = 0; | |
326 | int local_size; | |
327 | struct scif_qp *qp; | |
328 | ||
329 | local_size = SCIF_NODE_QP_SIZE; | |
330 | ||
331 | qp = kzalloc(sizeof(*qp), GFP_KERNEL); | |
332 | if (!qp) { | |
333 | err = -ENOMEM; | |
334 | return err; | |
335 | } | |
336 | qp->magic = SCIFEP_MAGIC; | |
337 | scifdev->qpairs = qp; | |
338 | err = scif_setup_qp_connect(qp, &scifdev->qp_dma_addr, | |
339 | local_size, scifdev); | |
340 | if (err) | |
341 | goto free_qp; | |
342 | /* | |
343 | * We're as setup as we can be. The inbound_q is setup, w/o a usable | |
344 | * outbound q. When we get a message, the read_ptr will be updated, | |
345 | * and we will pull the message. | |
346 | */ | |
347 | return err; | |
348 | free_qp: | |
349 | kfree(scifdev->qpairs); | |
350 | scifdev->qpairs = NULL; | |
351 | return err; | |
352 | } | |
353 | ||
354 | static void scif_p2p_freesg(struct scatterlist *sg) | |
355 | { | |
356 | kfree(sg); | |
357 | } | |
358 | ||
359 | static struct scatterlist * | |
a6344437 | 360 | scif_p2p_setsg(phys_addr_t pa, int page_size, int page_cnt) |
40cb5942 SD |
361 | { |
362 | struct scatterlist *sg; | |
363 | struct page *page; | |
364 | int i; | |
365 | ||
366 | sg = kcalloc(page_cnt, sizeof(struct scatterlist), GFP_KERNEL); | |
367 | if (!sg) | |
368 | return NULL; | |
369 | sg_init_table(sg, page_cnt); | |
370 | for (i = 0; i < page_cnt; i++) { | |
a6344437 | 371 | page = pfn_to_page(pa >> PAGE_SHIFT); |
40cb5942 | 372 | sg_set_page(&sg[i], page, page_size, 0); |
a6344437 | 373 | pa += page_size; |
40cb5942 SD |
374 | } |
375 | return sg; | |
40cb5942 SD |
376 | } |
377 | ||
378 | /* Init p2p mappings required to access peerdev from scifdev */ | |
379 | static struct scif_p2p_info * | |
380 | scif_init_p2p_info(struct scif_dev *scifdev, struct scif_dev *peerdev) | |
381 | { | |
382 | struct scif_p2p_info *p2p; | |
383 | int num_mmio_pages, num_aper_pages, sg_page_shift, err, num_aper_chunks; | |
384 | struct scif_hw_dev *psdev = peerdev->sdev; | |
385 | struct scif_hw_dev *sdev = scifdev->sdev; | |
386 | ||
387 | num_mmio_pages = psdev->mmio->len >> PAGE_SHIFT; | |
388 | num_aper_pages = psdev->aper->len >> PAGE_SHIFT; | |
389 | ||
390 | p2p = kzalloc(sizeof(*p2p), GFP_KERNEL); | |
391 | if (!p2p) | |
392 | return NULL; | |
a6344437 | 393 | p2p->ppi_sg[SCIF_PPI_MMIO] = scif_p2p_setsg(psdev->mmio->pa, |
40cb5942 SD |
394 | PAGE_SIZE, num_mmio_pages); |
395 | if (!p2p->ppi_sg[SCIF_PPI_MMIO]) | |
396 | goto free_p2p; | |
397 | p2p->sg_nentries[SCIF_PPI_MMIO] = num_mmio_pages; | |
398 | sg_page_shift = get_order(min(psdev->aper->len, (u64)(1 << 30))); | |
399 | num_aper_chunks = num_aper_pages >> (sg_page_shift - PAGE_SHIFT); | |
a6344437 | 400 | p2p->ppi_sg[SCIF_PPI_APER] = scif_p2p_setsg(psdev->aper->pa, |
40cb5942 SD |
401 | 1 << sg_page_shift, |
402 | num_aper_chunks); | |
403 | p2p->sg_nentries[SCIF_PPI_APER] = num_aper_chunks; | |
404 | err = dma_map_sg(&sdev->dev, p2p->ppi_sg[SCIF_PPI_MMIO], | |
405 | num_mmio_pages, PCI_DMA_BIDIRECTIONAL); | |
406 | if (err != num_mmio_pages) | |
407 | goto scif_p2p_free; | |
408 | err = dma_map_sg(&sdev->dev, p2p->ppi_sg[SCIF_PPI_APER], | |
409 | num_aper_chunks, PCI_DMA_BIDIRECTIONAL); | |
410 | if (err != num_aper_chunks) | |
411 | goto dma_unmap; | |
412 | p2p->ppi_da[SCIF_PPI_MMIO] = sg_dma_address(p2p->ppi_sg[SCIF_PPI_MMIO]); | |
413 | p2p->ppi_da[SCIF_PPI_APER] = sg_dma_address(p2p->ppi_sg[SCIF_PPI_APER]); | |
414 | p2p->ppi_len[SCIF_PPI_MMIO] = num_mmio_pages; | |
415 | p2p->ppi_len[SCIF_PPI_APER] = num_aper_pages; | |
416 | p2p->ppi_peer_id = peerdev->node; | |
417 | return p2p; | |
418 | dma_unmap: | |
419 | dma_unmap_sg(&sdev->dev, p2p->ppi_sg[SCIF_PPI_MMIO], | |
420 | p2p->sg_nentries[SCIF_PPI_MMIO], DMA_BIDIRECTIONAL); | |
421 | scif_p2p_free: | |
422 | scif_p2p_freesg(p2p->ppi_sg[SCIF_PPI_MMIO]); | |
423 | scif_p2p_freesg(p2p->ppi_sg[SCIF_PPI_APER]); | |
424 | free_p2p: | |
425 | kfree(p2p); | |
426 | return NULL; | |
427 | } | |
428 | ||
429 | /** | |
430 | * scif_node_connect: Respond to SCIF_NODE_CONNECT interrupt message | |
431 | * @dst: Destination node | |
432 | * | |
433 | * Connect the src and dst node by setting up the p2p connection | |
434 | * between them. Management node here acts like a proxy. | |
435 | */ | |
436 | static void scif_node_connect(struct scif_dev *scifdev, int dst) | |
437 | { | |
438 | struct scif_dev *dev_j = scifdev; | |
439 | struct scif_dev *dev_i = NULL; | |
440 | struct scif_p2p_info *p2p_ij = NULL; /* bus addr for j from i */ | |
441 | struct scif_p2p_info *p2p_ji = NULL; /* bus addr for i from j */ | |
442 | struct scif_p2p_info *p2p; | |
443 | struct list_head *pos, *tmp; | |
444 | struct scifmsg msg; | |
445 | int err; | |
446 | u64 tmppayload; | |
447 | ||
448 | if (dst < 1 || dst > scif_info.maxid) | |
449 | return; | |
450 | ||
451 | dev_i = &scif_dev[dst]; | |
452 | ||
453 | if (!_scifdev_alive(dev_i)) | |
454 | return; | |
455 | /* | |
456 | * If the p2p connection is already setup or in the process of setting | |
457 | * up then just ignore this request. The requested node will get | |
458 | * informed by SCIF_NODE_ADD_ACK or SCIF_NODE_ADD_NACK | |
459 | */ | |
460 | if (!list_empty(&dev_i->p2p)) { | |
461 | list_for_each_safe(pos, tmp, &dev_i->p2p) { | |
462 | p2p = list_entry(pos, struct scif_p2p_info, ppi_list); | |
463 | if (p2p->ppi_peer_id == dev_j->node) | |
464 | return; | |
465 | } | |
466 | } | |
467 | p2p_ij = scif_init_p2p_info(dev_i, dev_j); | |
468 | if (!p2p_ij) | |
469 | return; | |
470 | p2p_ji = scif_init_p2p_info(dev_j, dev_i); | |
471 | if (!p2p_ji) | |
472 | return; | |
473 | list_add_tail(&p2p_ij->ppi_list, &dev_i->p2p); | |
474 | list_add_tail(&p2p_ji->ppi_list, &dev_j->p2p); | |
475 | ||
476 | /* | |
477 | * Send a SCIF_NODE_ADD to dev_i, pass it its bus address | |
478 | * as seen from dev_j | |
479 | */ | |
480 | msg.uop = SCIF_NODE_ADD; | |
481 | msg.src.node = dev_j->node; | |
482 | msg.dst.node = dev_i->node; | |
483 | ||
484 | msg.payload[0] = p2p_ji->ppi_da[SCIF_PPI_APER]; | |
485 | msg.payload[1] = p2p_ij->ppi_da[SCIF_PPI_MMIO]; | |
486 | msg.payload[2] = p2p_ij->ppi_da[SCIF_PPI_APER]; | |
487 | msg.payload[3] = p2p_ij->ppi_len[SCIF_PPI_APER] << PAGE_SHIFT; | |
488 | ||
489 | err = scif_nodeqp_send(dev_i, &msg); | |
490 | if (err) { | |
491 | dev_err(&scifdev->sdev->dev, | |
492 | "%s %d error %d\n", __func__, __LINE__, err); | |
493 | return; | |
494 | } | |
495 | ||
496 | /* Same as above but to dev_j */ | |
497 | msg.uop = SCIF_NODE_ADD; | |
498 | msg.src.node = dev_i->node; | |
499 | msg.dst.node = dev_j->node; | |
500 | ||
501 | tmppayload = msg.payload[0]; | |
502 | msg.payload[0] = msg.payload[2]; | |
503 | msg.payload[2] = tmppayload; | |
504 | msg.payload[1] = p2p_ji->ppi_da[SCIF_PPI_MMIO]; | |
505 | msg.payload[3] = p2p_ji->ppi_len[SCIF_PPI_APER] << PAGE_SHIFT; | |
506 | ||
507 | scif_nodeqp_send(dev_j, &msg); | |
508 | } | |
509 | ||
510 | static void scif_p2p_setup(void) | |
511 | { | |
512 | int i, j; | |
513 | ||
514 | if (!scif_info.p2p_enable) | |
515 | return; | |
516 | ||
517 | for (i = 1; i <= scif_info.maxid; i++) | |
518 | if (!_scifdev_alive(&scif_dev[i])) | |
519 | return; | |
520 | ||
521 | for (i = 1; i <= scif_info.maxid; i++) { | |
522 | for (j = 1; j <= scif_info.maxid; j++) { | |
523 | struct scif_dev *scifdev = &scif_dev[i]; | |
524 | ||
525 | if (i == j) | |
526 | continue; | |
527 | scif_node_connect(scifdev, j); | |
528 | } | |
529 | } | |
530 | } | |
531 | ||
532 | void scif_qp_response_ack(struct work_struct *work) | |
533 | { | |
534 | struct scif_dev *scifdev = container_of(work, struct scif_dev, | |
535 | init_msg_work); | |
536 | struct scif_peer_dev *spdev; | |
537 | ||
538 | /* Drop the INIT message if it has already been received */ | |
539 | if (_scifdev_alive(scifdev)) | |
540 | return; | |
541 | ||
542 | spdev = scif_peer_register_device(scifdev); | |
543 | if (IS_ERR(spdev)) | |
544 | return; | |
545 | ||
546 | if (scif_is_mgmt_node()) { | |
547 | mutex_lock(&scif_info.conflock); | |
548 | scif_p2p_setup(); | |
549 | mutex_unlock(&scif_info.conflock); | |
550 | } | |
551 | } | |
552 | ||
553 | static char *message_types[] = {"BAD", | |
554 | "INIT", | |
555 | "EXIT", | |
556 | "SCIF_EXIT_ACK", | |
557 | "SCIF_NODE_ADD", | |
558 | "SCIF_NODE_ADD_ACK", | |
559 | "SCIF_NODE_ADD_NACK", | |
560 | "REMOVE_NODE", | |
76371c7c NR |
561 | "REMOVE_NODE_ACK", |
562 | "CNCT_REQ", | |
563 | "CNCT_GNT", | |
564 | "CNCT_GNTACK", | |
565 | "CNCT_GNTNACK", | |
566 | "CNCT_REJ", | |
567 | "DISCNCT", | |
fdd9fd5c SD |
568 | "DISCNT_ACK", |
569 | "CLIENT_SENT", | |
570 | "CLIENT_RCVD", | |
571 | "SCIF_GET_NODE_INFO"}; | |
40cb5942 SD |
572 | |
573 | static void | |
574 | scif_display_message(struct scif_dev *scifdev, struct scifmsg *msg, | |
575 | const char *label) | |
576 | { | |
577 | if (!scif_info.en_msg_log) | |
578 | return; | |
579 | if (msg->uop > SCIF_MAX_MSG) { | |
580 | dev_err(&scifdev->sdev->dev, | |
581 | "%s: unknown msg type %d\n", label, msg->uop); | |
582 | return; | |
583 | } | |
584 | dev_info(&scifdev->sdev->dev, | |
585 | "%s: msg type %s, src %d:%d, dest %d:%d payload 0x%llx:0x%llx:0x%llx:0x%llx\n", | |
586 | label, message_types[msg->uop], msg->src.node, msg->src.port, | |
587 | msg->dst.node, msg->dst.port, msg->payload[0], msg->payload[1], | |
588 | msg->payload[2], msg->payload[3]); | |
589 | } | |
590 | ||
591 | int _scif_nodeqp_send(struct scif_dev *scifdev, struct scifmsg *msg) | |
592 | { | |
593 | struct scif_qp *qp = scifdev->qpairs; | |
594 | int err = -ENOMEM, loop_cnt = 0; | |
595 | ||
596 | scif_display_message(scifdev, msg, "Sent"); | |
597 | if (!qp) { | |
598 | err = -EINVAL; | |
599 | goto error; | |
600 | } | |
601 | spin_lock(&qp->send_lock); | |
602 | ||
603 | while ((err = scif_rb_write(&qp->outbound_q, | |
604 | msg, sizeof(struct scifmsg)))) { | |
605 | mdelay(1); | |
606 | #define SCIF_NODEQP_SEND_TO_MSEC (3 * 1000) | |
607 | if (loop_cnt++ > (SCIF_NODEQP_SEND_TO_MSEC)) { | |
608 | err = -ENODEV; | |
609 | break; | |
610 | } | |
611 | } | |
612 | if (!err) | |
613 | scif_rb_commit(&qp->outbound_q); | |
614 | spin_unlock(&qp->send_lock); | |
615 | if (!err) { | |
616 | if (scifdev_self(scifdev)) | |
617 | /* | |
618 | * For loopback we need to emulate an interrupt by | |
619 | * queuing work for the queue handling real node | |
620 | * Qp interrupts. | |
621 | */ | |
622 | queue_work(scifdev->intr_wq, &scifdev->intr_bh); | |
623 | else | |
624 | scif_send_msg_intr(scifdev); | |
625 | } | |
626 | error: | |
627 | if (err) | |
628 | dev_dbg(&scifdev->sdev->dev, | |
629 | "%s %d error %d uop %d\n", | |
630 | __func__, __LINE__, err, msg->uop); | |
631 | return err; | |
632 | } | |
633 | ||
634 | /** | |
635 | * scif_nodeqp_send - Send a message on the node queue pair | |
636 | * @scifdev: Scif Device. | |
637 | * @msg: The message to be sent. | |
638 | */ | |
639 | int scif_nodeqp_send(struct scif_dev *scifdev, struct scifmsg *msg) | |
640 | { | |
641 | int err; | |
642 | struct device *spdev = NULL; | |
643 | ||
644 | if (msg->uop > SCIF_EXIT_ACK) { | |
645 | /* Dont send messages once the exit flow has begun */ | |
646 | if (OP_IDLE != scifdev->exit) | |
647 | return -ENODEV; | |
648 | spdev = scif_get_peer_dev(scifdev); | |
649 | if (IS_ERR(spdev)) { | |
650 | err = PTR_ERR(spdev); | |
651 | return err; | |
652 | } | |
653 | } | |
654 | err = _scif_nodeqp_send(scifdev, msg); | |
655 | if (msg->uop > SCIF_EXIT_ACK) | |
656 | scif_put_peer_dev(spdev); | |
657 | return err; | |
658 | } | |
659 | ||
660 | /* | |
661 | * scif_misc_handler: | |
662 | * | |
663 | * Work queue handler for servicing miscellaneous SCIF tasks. | |
664 | * Examples include: | |
665 | * 1) Cleanup of zombie endpoints. | |
666 | */ | |
667 | void scif_misc_handler(struct work_struct *work) | |
668 | { | |
669 | scif_cleanup_zombie_epd(); | |
670 | } | |
671 | ||
672 | /** | |
673 | * scif_init() - Respond to SCIF_INIT interrupt message | |
674 | * @scifdev: Remote SCIF device node | |
675 | * @msg: Interrupt message | |
676 | */ | |
677 | static __always_inline void | |
678 | scif_init(struct scif_dev *scifdev, struct scifmsg *msg) | |
679 | { | |
680 | /* | |
681 | * Allow the thread waiting for device page updates for the peer QP DMA | |
682 | * address to complete initializing the inbound_q. | |
683 | */ | |
684 | flush_delayed_work(&scifdev->qp_dwork); | |
685 | /* | |
686 | * Delegate the peer device registration to a workqueue, otherwise if | |
687 | * SCIF client probe (called during peer device registration) calls | |
688 | * scif_connect(..), it will block the message processing thread causing | |
689 | * a deadlock. | |
690 | */ | |
691 | schedule_work(&scifdev->init_msg_work); | |
692 | } | |
693 | ||
694 | /** | |
695 | * scif_exit() - Respond to SCIF_EXIT interrupt message | |
696 | * @scifdev: Remote SCIF device node | |
697 | * @msg: Interrupt message | |
698 | * | |
699 | * This function stops the SCIF interface for the node which sent | |
700 | * the SCIF_EXIT message and starts waiting for that node to | |
701 | * resetup the queue pair again. | |
702 | */ | |
703 | static __always_inline void | |
704 | scif_exit(struct scif_dev *scifdev, struct scifmsg *unused) | |
705 | { | |
706 | scifdev->exit_ack_pending = true; | |
707 | if (scif_is_mgmt_node()) | |
708 | scif_disconnect_node(scifdev->node, false); | |
709 | else | |
710 | scif_stop(scifdev); | |
711 | schedule_delayed_work(&scifdev->qp_dwork, | |
712 | msecs_to_jiffies(1000)); | |
713 | } | |
714 | ||
715 | /** | |
716 | * scif_exitack() - Respond to SCIF_EXIT_ACK interrupt message | |
717 | * @scifdev: Remote SCIF device node | |
718 | * @msg: Interrupt message | |
719 | * | |
720 | */ | |
721 | static __always_inline void | |
722 | scif_exit_ack(struct scif_dev *scifdev, struct scifmsg *unused) | |
723 | { | |
724 | scifdev->exit = OP_COMPLETED; | |
725 | wake_up(&scif_info.exitwq); | |
726 | } | |
727 | ||
728 | /** | |
729 | * scif_node_add() - Respond to SCIF_NODE_ADD interrupt message | |
730 | * @scifdev: Remote SCIF device node | |
731 | * @msg: Interrupt message | |
732 | * | |
733 | * When the mgmt node driver has finished initializing a MIC node queue pair it | |
734 | * marks the node as online. It then looks for all currently online MIC cards | |
735 | * and send a SCIF_NODE_ADD message to identify the ID of the new card for | |
736 | * peer to peer initialization | |
737 | * | |
738 | * The local node allocates its incoming queue and sends its address in the | |
739 | * SCIF_NODE_ADD_ACK message back to the mgmt node, the mgmt node "reflects" | |
740 | * this message to the new node | |
741 | */ | |
742 | static __always_inline void | |
743 | scif_node_add(struct scif_dev *scifdev, struct scifmsg *msg) | |
744 | { | |
745 | struct scif_dev *newdev; | |
746 | dma_addr_t qp_offset; | |
747 | int qp_connect; | |
748 | struct scif_hw_dev *sdev; | |
749 | ||
750 | dev_dbg(&scifdev->sdev->dev, | |
751 | "Scifdev %d:%d received NODE_ADD msg for node %d\n", | |
752 | scifdev->node, msg->dst.node, msg->src.node); | |
753 | dev_dbg(&scifdev->sdev->dev, | |
754 | "Remote address for this node's aperture %llx\n", | |
755 | msg->payload[0]); | |
756 | newdev = &scif_dev[msg->src.node]; | |
757 | newdev->node = msg->src.node; | |
758 | newdev->sdev = scif_dev[SCIF_MGMT_NODE].sdev; | |
759 | sdev = newdev->sdev; | |
760 | ||
761 | if (scif_setup_intr_wq(newdev)) { | |
762 | dev_err(&scifdev->sdev->dev, | |
763 | "failed to setup interrupts for %d\n", msg->src.node); | |
764 | goto interrupt_setup_error; | |
765 | } | |
766 | newdev->mmio.va = ioremap_nocache(msg->payload[1], sdev->mmio->len); | |
767 | if (!newdev->mmio.va) { | |
768 | dev_err(&scifdev->sdev->dev, | |
769 | "failed to map mmio for %d\n", msg->src.node); | |
770 | goto mmio_map_error; | |
771 | } | |
772 | newdev->qpairs = kzalloc(sizeof(*newdev->qpairs), GFP_KERNEL); | |
773 | if (!newdev->qpairs) | |
774 | goto qp_alloc_error; | |
775 | /* | |
776 | * Set the base address of the remote node's memory since it gets | |
777 | * added to qp_offset | |
778 | */ | |
779 | newdev->base_addr = msg->payload[0]; | |
780 | ||
781 | qp_connect = scif_setup_qp_connect(newdev->qpairs, &qp_offset, | |
782 | SCIF_NODE_QP_SIZE, newdev); | |
783 | if (qp_connect) { | |
784 | dev_err(&scifdev->sdev->dev, | |
785 | "failed to setup qp_connect %d\n", qp_connect); | |
786 | goto qp_connect_error; | |
787 | } | |
788 | ||
789 | newdev->db = sdev->hw_ops->next_db(sdev); | |
790 | newdev->cookie = sdev->hw_ops->request_irq(sdev, scif_intr_handler, | |
791 | "SCIF_INTR", newdev, | |
792 | newdev->db); | |
793 | if (IS_ERR(newdev->cookie)) | |
794 | goto qp_connect_error; | |
795 | newdev->qpairs->magic = SCIFEP_MAGIC; | |
796 | newdev->qpairs->qp_state = SCIF_QP_OFFLINE; | |
797 | ||
798 | msg->uop = SCIF_NODE_ADD_ACK; | |
799 | msg->dst.node = msg->src.node; | |
800 | msg->src.node = scif_info.nodeid; | |
801 | msg->payload[0] = qp_offset; | |
802 | msg->payload[2] = newdev->db; | |
803 | scif_nodeqp_send(&scif_dev[SCIF_MGMT_NODE], msg); | |
804 | return; | |
805 | qp_connect_error: | |
806 | kfree(newdev->qpairs); | |
807 | newdev->qpairs = NULL; | |
808 | qp_alloc_error: | |
809 | iounmap(newdev->mmio.va); | |
810 | newdev->mmio.va = NULL; | |
811 | mmio_map_error: | |
812 | interrupt_setup_error: | |
813 | dev_err(&scifdev->sdev->dev, | |
814 | "node add failed for node %d\n", msg->src.node); | |
815 | msg->uop = SCIF_NODE_ADD_NACK; | |
816 | msg->dst.node = msg->src.node; | |
817 | msg->src.node = scif_info.nodeid; | |
818 | scif_nodeqp_send(&scif_dev[SCIF_MGMT_NODE], msg); | |
819 | } | |
820 | ||
821 | void scif_poll_qp_state(struct work_struct *work) | |
822 | { | |
823 | #define SCIF_NODE_QP_RETRY 100 | |
824 | #define SCIF_NODE_QP_TIMEOUT 100 | |
825 | struct scif_dev *peerdev = container_of(work, struct scif_dev, | |
826 | p2p_dwork.work); | |
827 | struct scif_qp *qp = &peerdev->qpairs[0]; | |
828 | ||
829 | if (qp->qp_state != SCIF_QP_ONLINE || | |
830 | qp->remote_qp->qp_state != SCIF_QP_ONLINE) { | |
831 | if (peerdev->p2p_retry++ == SCIF_NODE_QP_RETRY) { | |
832 | dev_err(&peerdev->sdev->dev, | |
833 | "Warning: QP check timeout with state %d\n", | |
834 | qp->qp_state); | |
835 | goto timeout; | |
836 | } | |
837 | schedule_delayed_work(&peerdev->p2p_dwork, | |
838 | msecs_to_jiffies(SCIF_NODE_QP_TIMEOUT)); | |
839 | return; | |
840 | } | |
841 | scif_peer_register_device(peerdev); | |
842 | return; | |
843 | timeout: | |
844 | dev_err(&peerdev->sdev->dev, | |
845 | "%s %d remote node %d offline, state = 0x%x\n", | |
846 | __func__, __LINE__, peerdev->node, qp->qp_state); | |
847 | qp->remote_qp->qp_state = SCIF_QP_OFFLINE; | |
848 | scif_cleanup_scifdev(peerdev); | |
849 | } | |
850 | ||
851 | /** | |
852 | * scif_node_add_ack() - Respond to SCIF_NODE_ADD_ACK interrupt message | |
853 | * @scifdev: Remote SCIF device node | |
854 | * @msg: Interrupt message | |
855 | * | |
856 | * After a MIC node receives the SCIF_NODE_ADD_ACK message it send this | |
857 | * message to the mgmt node to confirm the sequence is finished. | |
858 | * | |
859 | */ | |
860 | static __always_inline void | |
861 | scif_node_add_ack(struct scif_dev *scifdev, struct scifmsg *msg) | |
862 | { | |
863 | struct scif_dev *peerdev; | |
864 | struct scif_qp *qp; | |
865 | struct scif_dev *dst_dev = &scif_dev[msg->dst.node]; | |
866 | ||
867 | dev_dbg(&scifdev->sdev->dev, | |
868 | "Scifdev %d received SCIF_NODE_ADD_ACK msg src %d dst %d\n", | |
869 | scifdev->node, msg->src.node, msg->dst.node); | |
870 | dev_dbg(&scifdev->sdev->dev, | |
871 | "payload %llx %llx %llx %llx\n", msg->payload[0], | |
872 | msg->payload[1], msg->payload[2], msg->payload[3]); | |
873 | if (scif_is_mgmt_node()) { | |
874 | /* | |
875 | * the lock serializes with scif_qp_response_ack. The mgmt node | |
876 | * is forwarding the NODE_ADD_ACK message from src to dst we | |
877 | * need to make sure that the dst has already received a | |
878 | * NODE_ADD for src and setup its end of the qp to dst | |
879 | */ | |
880 | mutex_lock(&scif_info.conflock); | |
881 | msg->payload[1] = scif_info.maxid; | |
882 | scif_nodeqp_send(dst_dev, msg); | |
883 | mutex_unlock(&scif_info.conflock); | |
884 | return; | |
885 | } | |
886 | peerdev = &scif_dev[msg->src.node]; | |
887 | peerdev->sdev = scif_dev[SCIF_MGMT_NODE].sdev; | |
888 | peerdev->node = msg->src.node; | |
889 | ||
890 | qp = &peerdev->qpairs[0]; | |
891 | ||
892 | if ((scif_setup_qp_connect_response(peerdev, &peerdev->qpairs[0], | |
893 | msg->payload[0]))) | |
894 | goto local_error; | |
895 | peerdev->rdb = msg->payload[2]; | |
896 | qp->remote_qp->qp_state = SCIF_QP_ONLINE; | |
897 | schedule_delayed_work(&peerdev->p2p_dwork, 0); | |
898 | return; | |
899 | local_error: | |
900 | scif_cleanup_scifdev(peerdev); | |
901 | } | |
902 | ||
903 | /** | |
904 | * scif_node_add_nack: Respond to SCIF_NODE_ADD_NACK interrupt message | |
905 | * @msg: Interrupt message | |
906 | * | |
907 | * SCIF_NODE_ADD failed, so inform the waiting wq. | |
908 | */ | |
909 | static __always_inline void | |
910 | scif_node_add_nack(struct scif_dev *scifdev, struct scifmsg *msg) | |
911 | { | |
912 | if (scif_is_mgmt_node()) { | |
913 | struct scif_dev *dst_dev = &scif_dev[msg->dst.node]; | |
914 | ||
915 | dev_dbg(&scifdev->sdev->dev, | |
916 | "SCIF_NODE_ADD_NACK received from %d\n", scifdev->node); | |
917 | scif_nodeqp_send(dst_dev, msg); | |
918 | } | |
919 | } | |
920 | ||
921 | /* | |
922 | * scif_node_remove: Handle SCIF_NODE_REMOVE message | |
923 | * @msg: Interrupt message | |
924 | * | |
925 | * Handle node removal. | |
926 | */ | |
927 | static __always_inline void | |
928 | scif_node_remove(struct scif_dev *scifdev, struct scifmsg *msg) | |
929 | { | |
930 | int node = msg->payload[0]; | |
931 | struct scif_dev *scdev = &scif_dev[node]; | |
932 | ||
933 | scdev->node_remove_ack_pending = true; | |
934 | scif_handle_remove_node(node); | |
935 | } | |
936 | ||
937 | /* | |
938 | * scif_node_remove_ack: Handle SCIF_NODE_REMOVE_ACK message | |
939 | * @msg: Interrupt message | |
940 | * | |
941 | * The peer has acked a SCIF_NODE_REMOVE message. | |
942 | */ | |
943 | static __always_inline void | |
944 | scif_node_remove_ack(struct scif_dev *scifdev, struct scifmsg *msg) | |
945 | { | |
946 | struct scif_dev *sdev = &scif_dev[msg->payload[0]]; | |
947 | ||
948 | atomic_inc(&sdev->disconn_rescnt); | |
949 | wake_up(&sdev->disconn_wq); | |
950 | } | |
951 | ||
fdd9fd5c SD |
952 | /** |
953 | * scif_get_node_info: Respond to SCIF_GET_NODE_INFO interrupt message | |
954 | * @msg: Interrupt message | |
955 | * | |
956 | * Retrieve node info i.e maxid and total from the mgmt node. | |
957 | */ | |
958 | static __always_inline void | |
959 | scif_get_node_info_resp(struct scif_dev *scifdev, struct scifmsg *msg) | |
960 | { | |
961 | if (scif_is_mgmt_node()) { | |
962 | swap(msg->dst.node, msg->src.node); | |
963 | mutex_lock(&scif_info.conflock); | |
964 | msg->payload[1] = scif_info.maxid; | |
965 | msg->payload[2] = scif_info.total; | |
966 | mutex_unlock(&scif_info.conflock); | |
967 | scif_nodeqp_send(scifdev, msg); | |
968 | } else { | |
969 | struct completion *node_info = | |
970 | (struct completion *)msg->payload[3]; | |
971 | ||
972 | mutex_lock(&scif_info.conflock); | |
973 | scif_info.maxid = msg->payload[1]; | |
974 | scif_info.total = msg->payload[2]; | |
975 | complete_all(node_info); | |
976 | mutex_unlock(&scif_info.conflock); | |
977 | } | |
978 | } | |
979 | ||
40cb5942 SD |
980 | static void |
981 | scif_msg_unknown(struct scif_dev *scifdev, struct scifmsg *msg) | |
982 | { | |
983 | /* Bogus Node Qp Message? */ | |
984 | dev_err(&scifdev->sdev->dev, | |
985 | "Unknown message 0x%xn scifdev->node 0x%x\n", | |
986 | msg->uop, scifdev->node); | |
987 | } | |
988 | ||
989 | static void (*scif_intr_func[SCIF_MAX_MSG + 1]) | |
990 | (struct scif_dev *, struct scifmsg *msg) = { | |
991 | scif_msg_unknown, /* Error */ | |
992 | scif_init, /* SCIF_INIT */ | |
993 | scif_exit, /* SCIF_EXIT */ | |
994 | scif_exit_ack, /* SCIF_EXIT_ACK */ | |
995 | scif_node_add, /* SCIF_NODE_ADD */ | |
996 | scif_node_add_ack, /* SCIF_NODE_ADD_ACK */ | |
997 | scif_node_add_nack, /* SCIF_NODE_ADD_NACK */ | |
998 | scif_node_remove, /* SCIF_NODE_REMOVE */ | |
999 | scif_node_remove_ack, /* SCIF_NODE_REMOVE_ACK */ | |
76371c7c NR |
1000 | scif_cnctreq, /* SCIF_CNCT_REQ */ |
1001 | scif_cnctgnt, /* SCIF_CNCT_GNT */ | |
1002 | scif_cnctgnt_ack, /* SCIF_CNCT_GNTACK */ | |
1003 | scif_cnctgnt_nack, /* SCIF_CNCT_GNTNACK */ | |
1004 | scif_cnctrej, /* SCIF_CNCT_REJ */ | |
1005 | scif_discnct, /* SCIF_DISCNCT */ | |
1006 | scif_discnt_ack, /* SCIF_DISCNT_ACK */ | |
fdd9fd5c SD |
1007 | scif_clientsend, /* SCIF_CLIENT_SENT */ |
1008 | scif_clientrcvd, /* SCIF_CLIENT_RCVD */ | |
1009 | scif_get_node_info_resp,/* SCIF_GET_NODE_INFO */ | |
40cb5942 SD |
1010 | }; |
1011 | ||
1012 | /** | |
1013 | * scif_nodeqp_msg_handler() - Common handler for node messages | |
1014 | * @scifdev: Remote device to respond to | |
1015 | * @qp: Remote memory pointer | |
1016 | * @msg: The message to be handled. | |
1017 | * | |
1018 | * This routine calls the appropriate routine to handle a Node Qp | |
1019 | * message receipt | |
1020 | */ | |
1021 | static int scif_max_msg_id = SCIF_MAX_MSG; | |
1022 | ||
1023 | static void | |
1024 | scif_nodeqp_msg_handler(struct scif_dev *scifdev, | |
1025 | struct scif_qp *qp, struct scifmsg *msg) | |
1026 | { | |
1027 | scif_display_message(scifdev, msg, "Rcvd"); | |
1028 | ||
1029 | if (msg->uop > (u32)scif_max_msg_id) { | |
1030 | /* Bogus Node Qp Message? */ | |
1031 | dev_err(&scifdev->sdev->dev, | |
1032 | "Unknown message 0x%xn scifdev->node 0x%x\n", | |
1033 | msg->uop, scifdev->node); | |
1034 | return; | |
1035 | } | |
1036 | ||
1037 | scif_intr_func[msg->uop](scifdev, msg); | |
1038 | } | |
1039 | ||
1040 | /** | |
1041 | * scif_nodeqp_intrhandler() - Interrupt handler for node messages | |
1042 | * @scifdev: Remote device to respond to | |
1043 | * @qp: Remote memory pointer | |
1044 | * | |
1045 | * This routine is triggered by the interrupt mechanism. It reads | |
1046 | * messages from the node queue RB and calls the Node QP Message handling | |
1047 | * routine. | |
1048 | */ | |
1049 | void scif_nodeqp_intrhandler(struct scif_dev *scifdev, struct scif_qp *qp) | |
1050 | { | |
1051 | struct scifmsg msg; | |
1052 | int read_size; | |
1053 | ||
1054 | do { | |
1055 | read_size = scif_rb_get_next(&qp->inbound_q, &msg, sizeof(msg)); | |
1056 | if (!read_size) | |
1057 | break; | |
1058 | scif_nodeqp_msg_handler(scifdev, qp, &msg); | |
1059 | /* | |
1060 | * The node queue pair is unmapped so skip the read pointer | |
1061 | * update after receipt of a SCIF_EXIT_ACK | |
1062 | */ | |
1063 | if (SCIF_EXIT_ACK == msg.uop) | |
1064 | break; | |
1065 | scif_rb_update_read_ptr(&qp->inbound_q); | |
1066 | } while (1); | |
1067 | } | |
1068 | ||
1069 | /** | |
1070 | * scif_loopb_wq_handler - Loopback Workqueue Handler. | |
1071 | * @work: loop back work | |
1072 | * | |
1073 | * This work queue routine is invoked by the loopback work queue handler. | |
1074 | * It grabs the recv lock, dequeues any available messages from the head | |
1075 | * of the loopback message list, calls the node QP message handler, | |
1076 | * waits for it to return, then frees up this message and dequeues more | |
1077 | * elements of the list if available. | |
1078 | */ | |
1079 | static void scif_loopb_wq_handler(struct work_struct *unused) | |
1080 | { | |
1081 | struct scif_dev *scifdev = scif_info.loopb_dev; | |
1082 | struct scif_qp *qp = scifdev->qpairs; | |
1083 | struct scif_loopb_msg *msg; | |
1084 | ||
1085 | do { | |
1086 | msg = NULL; | |
1087 | spin_lock(&qp->recv_lock); | |
1088 | if (!list_empty(&scif_info.loopb_recv_q)) { | |
1089 | msg = list_first_entry(&scif_info.loopb_recv_q, | |
1090 | struct scif_loopb_msg, | |
1091 | list); | |
1092 | list_del(&msg->list); | |
1093 | } | |
1094 | spin_unlock(&qp->recv_lock); | |
1095 | ||
1096 | if (msg) { | |
1097 | scif_nodeqp_msg_handler(scifdev, qp, &msg->msg); | |
1098 | kfree(msg); | |
1099 | } | |
1100 | } while (msg); | |
1101 | } | |
1102 | ||
1103 | /** | |
1104 | * scif_loopb_msg_handler() - Workqueue handler for loopback messages. | |
1105 | * @scifdev: SCIF device | |
1106 | * @qp: Queue pair. | |
1107 | * | |
1108 | * This work queue routine is triggered when a loopback message is received. | |
1109 | * | |
1110 | * We need special handling for receiving Node Qp messages on a loopback SCIF | |
1111 | * device via two workqueues for receiving messages. | |
1112 | * | |
1113 | * The reason we need the extra workqueue which is not required with *normal* | |
1114 | * non-loopback SCIF devices is the potential classic deadlock described below: | |
1115 | * | |
1116 | * Thread A tries to send a message on a loopback SCIF device and blocks since | |
1117 | * there is no space in the RB while it has the send_lock held or another | |
1118 | * lock called lock X for example. | |
1119 | * | |
1120 | * Thread B: The Loopback Node QP message receive workqueue receives the message | |
1121 | * and tries to send a message (eg an ACK) to the loopback SCIF device. It tries | |
1122 | * to grab the send lock again or lock X and deadlocks with Thread A. The RB | |
1123 | * cannot be drained any further due to this classic deadlock. | |
1124 | * | |
1125 | * In order to avoid deadlocks as mentioned above we have an extra level of | |
1126 | * indirection achieved by having two workqueues. | |
1127 | * 1) The first workqueue whose handler is scif_loopb_msg_handler reads | |
1128 | * messages from the Node QP RB, adds them to a list and queues work for the | |
1129 | * second workqueue. | |
1130 | * | |
1131 | * 2) The second workqueue whose handler is scif_loopb_wq_handler dequeues | |
1132 | * messages from the list, handles them, frees up the memory and dequeues | |
1133 | * more elements from the list if possible. | |
1134 | */ | |
1135 | int | |
1136 | scif_loopb_msg_handler(struct scif_dev *scifdev, struct scif_qp *qp) | |
1137 | { | |
1138 | int read_size; | |
1139 | struct scif_loopb_msg *msg; | |
1140 | ||
1141 | do { | |
1142 | msg = kmalloc(sizeof(*msg), GFP_KERNEL); | |
1143 | if (!msg) | |
1144 | return -ENOMEM; | |
1145 | read_size = scif_rb_get_next(&qp->inbound_q, &msg->msg, | |
1146 | sizeof(struct scifmsg)); | |
1147 | if (read_size != sizeof(struct scifmsg)) { | |
1148 | kfree(msg); | |
1149 | scif_rb_update_read_ptr(&qp->inbound_q); | |
1150 | break; | |
1151 | } | |
1152 | spin_lock(&qp->recv_lock); | |
1153 | list_add_tail(&msg->list, &scif_info.loopb_recv_q); | |
1154 | spin_unlock(&qp->recv_lock); | |
1155 | queue_work(scif_info.loopb_wq, &scif_info.loopb_work); | |
1156 | scif_rb_update_read_ptr(&qp->inbound_q); | |
1157 | } while (read_size == sizeof(struct scifmsg)); | |
1158 | return read_size; | |
1159 | } | |
1160 | ||
1161 | /** | |
1162 | * scif_setup_loopback_qp - One time setup work for Loopback Node Qp. | |
1163 | * @scifdev: SCIF device | |
1164 | * | |
1165 | * Sets up the required loopback workqueues, queue pairs and ring buffers | |
1166 | */ | |
1167 | int scif_setup_loopback_qp(struct scif_dev *scifdev) | |
1168 | { | |
1169 | int err = 0; | |
1170 | void *local_q; | |
1171 | struct scif_qp *qp; | |
1172 | struct scif_peer_dev *spdev; | |
1173 | ||
1174 | err = scif_setup_intr_wq(scifdev); | |
1175 | if (err) | |
1176 | goto exit; | |
1177 | INIT_LIST_HEAD(&scif_info.loopb_recv_q); | |
1178 | snprintf(scif_info.loopb_wqname, sizeof(scif_info.loopb_wqname), | |
1179 | "SCIF LOOPB %d", scifdev->node); | |
1180 | scif_info.loopb_wq = | |
1181 | alloc_ordered_workqueue(scif_info.loopb_wqname, 0); | |
1182 | if (!scif_info.loopb_wq) { | |
1183 | err = -ENOMEM; | |
1184 | goto destroy_intr; | |
1185 | } | |
1186 | INIT_WORK(&scif_info.loopb_work, scif_loopb_wq_handler); | |
1187 | /* Allocate Self Qpair */ | |
1188 | scifdev->qpairs = kzalloc(sizeof(*scifdev->qpairs), GFP_KERNEL); | |
1189 | if (!scifdev->qpairs) { | |
1190 | err = -ENOMEM; | |
1191 | goto destroy_loopb_wq; | |
1192 | } | |
1193 | ||
1194 | qp = scifdev->qpairs; | |
1195 | qp->magic = SCIFEP_MAGIC; | |
1196 | spin_lock_init(&qp->send_lock); | |
1197 | spin_lock_init(&qp->recv_lock); | |
1198 | ||
1199 | local_q = kzalloc(SCIF_NODE_QP_SIZE, GFP_KERNEL); | |
1200 | if (!local_q) { | |
1201 | err = -ENOMEM; | |
1202 | goto free_qpairs; | |
1203 | } | |
1204 | /* | |
1205 | * For loopback the inbound_q and outbound_q are essentially the same | |
1206 | * since the Node sends a message on the loopback interface to the | |
1207 | * outbound_q which is then received on the inbound_q. | |
1208 | */ | |
1209 | scif_rb_init(&qp->outbound_q, | |
1210 | &qp->local_read, | |
1211 | &qp->local_write, | |
1212 | local_q, get_count_order(SCIF_NODE_QP_SIZE)); | |
1213 | ||
1214 | scif_rb_init(&qp->inbound_q, | |
1215 | &qp->local_read, | |
1216 | &qp->local_write, | |
1217 | local_q, get_count_order(SCIF_NODE_QP_SIZE)); | |
1218 | scif_info.nodeid = scifdev->node; | |
1219 | spdev = scif_peer_register_device(scifdev); | |
1220 | if (IS_ERR(spdev)) { | |
1221 | err = PTR_ERR(spdev); | |
1222 | goto free_local_q; | |
1223 | } | |
1224 | scif_info.loopb_dev = scifdev; | |
1225 | return err; | |
1226 | free_local_q: | |
1227 | kfree(local_q); | |
1228 | free_qpairs: | |
1229 | kfree(scifdev->qpairs); | |
1230 | destroy_loopb_wq: | |
1231 | destroy_workqueue(scif_info.loopb_wq); | |
1232 | destroy_intr: | |
1233 | scif_destroy_intr_wq(scifdev); | |
1234 | exit: | |
1235 | return err; | |
1236 | } | |
1237 | ||
1238 | /** | |
1239 | * scif_destroy_loopback_qp - One time uninit work for Loopback Node Qp | |
1240 | * @scifdev: SCIF device | |
1241 | * | |
1242 | * Destroys the workqueues and frees up the Ring Buffer and Queue Pair memory. | |
1243 | */ | |
1244 | int scif_destroy_loopback_qp(struct scif_dev *scifdev) | |
1245 | { | |
1246 | struct scif_peer_dev *spdev; | |
1247 | ||
1248 | rcu_read_lock(); | |
1249 | spdev = rcu_dereference(scifdev->spdev); | |
1250 | rcu_read_unlock(); | |
1251 | if (spdev) | |
1252 | scif_peer_unregister_device(spdev); | |
1253 | destroy_workqueue(scif_info.loopb_wq); | |
1254 | scif_destroy_intr_wq(scifdev); | |
1255 | kfree(scifdev->qpairs->outbound_q.rb_base); | |
1256 | kfree(scifdev->qpairs); | |
1257 | scifdev->sdev = NULL; | |
1258 | scif_info.loopb_dev = NULL; | |
1259 | return 0; | |
1260 | } | |
1261 | ||
1262 | void scif_destroy_p2p(struct scif_dev *scifdev) | |
1263 | { | |
1264 | struct scif_dev *peer_dev; | |
1265 | struct scif_p2p_info *p2p; | |
1266 | struct list_head *pos, *tmp; | |
1267 | int bd; | |
1268 | ||
1269 | mutex_lock(&scif_info.conflock); | |
1270 | /* Free P2P mappings in the given node for all its peer nodes */ | |
1271 | list_for_each_safe(pos, tmp, &scifdev->p2p) { | |
1272 | p2p = list_entry(pos, struct scif_p2p_info, ppi_list); | |
1273 | dma_unmap_sg(&scifdev->sdev->dev, p2p->ppi_sg[SCIF_PPI_MMIO], | |
1274 | p2p->sg_nentries[SCIF_PPI_MMIO], | |
1275 | DMA_BIDIRECTIONAL); | |
1276 | dma_unmap_sg(&scifdev->sdev->dev, p2p->ppi_sg[SCIF_PPI_APER], | |
1277 | p2p->sg_nentries[SCIF_PPI_APER], | |
1278 | DMA_BIDIRECTIONAL); | |
1279 | scif_p2p_freesg(p2p->ppi_sg[SCIF_PPI_MMIO]); | |
1280 | scif_p2p_freesg(p2p->ppi_sg[SCIF_PPI_APER]); | |
1281 | list_del(pos); | |
1282 | kfree(p2p); | |
1283 | } | |
1284 | ||
1285 | /* Free P2P mapping created in the peer nodes for the given node */ | |
1286 | for (bd = SCIF_MGMT_NODE + 1; bd <= scif_info.maxid; bd++) { | |
1287 | peer_dev = &scif_dev[bd]; | |
1288 | list_for_each_safe(pos, tmp, &peer_dev->p2p) { | |
1289 | p2p = list_entry(pos, struct scif_p2p_info, ppi_list); | |
1290 | if (p2p->ppi_peer_id == scifdev->node) { | |
1291 | dma_unmap_sg(&peer_dev->sdev->dev, | |
1292 | p2p->ppi_sg[SCIF_PPI_MMIO], | |
1293 | p2p->sg_nentries[SCIF_PPI_MMIO], | |
1294 | DMA_BIDIRECTIONAL); | |
1295 | dma_unmap_sg(&peer_dev->sdev->dev, | |
1296 | p2p->ppi_sg[SCIF_PPI_APER], | |
1297 | p2p->sg_nentries[SCIF_PPI_APER], | |
1298 | DMA_BIDIRECTIONAL); | |
1299 | scif_p2p_freesg(p2p->ppi_sg[SCIF_PPI_MMIO]); | |
1300 | scif_p2p_freesg(p2p->ppi_sg[SCIF_PPI_APER]); | |
1301 | list_del(pos); | |
1302 | kfree(p2p); | |
1303 | } | |
1304 | } | |
1305 | } | |
1306 | mutex_unlock(&scif_info.conflock); | |
1307 | } |