Commit | Line | Data |
---|---|---|
a5319752 BM |
1 | /* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ |
2 | ||
3 | /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ | |
4 | /* Copyright (c) 2008-2019, IBM Corporation */ | |
5 | ||
6 | #ifndef _SIW_H | |
7 | #define _SIW_H | |
8 | ||
9 | #include <rdma/ib_verbs.h> | |
10 | #include <linux/socket.h> | |
11 | #include <linux/skbuff.h> | |
12 | #include <crypto/hash.h> | |
13 | #include <linux/crc32.h> | |
14 | #include <linux/crc32c.h> | |
15 | ||
16 | #include <rdma/siw-abi.h> | |
17 | #include "iwarp.h" | |
18 | ||
19 | #define SIW_VENDOR_ID 0x626d74 /* ascii 'bmt' for now */ | |
20 | #define SIW_VENDORT_PART_ID 0 | |
21 | #define SIW_MAX_QP (1024 * 100) | |
22 | #define SIW_MAX_QP_WR (1024 * 32) | |
23 | #define SIW_MAX_ORD_QP 128 | |
24 | #define SIW_MAX_IRD_QP 128 | |
25 | #define SIW_MAX_SGE_PBL 256 /* max num sge's for PBL */ | |
26 | #define SIW_MAX_SGE_RD 1 /* iwarp limitation. we could relax */ | |
27 | #define SIW_MAX_CQ (1024 * 100) | |
28 | #define SIW_MAX_CQE (SIW_MAX_QP_WR * 100) | |
29 | #define SIW_MAX_MR (SIW_MAX_QP * 10) | |
30 | #define SIW_MAX_PD SIW_MAX_QP | |
31 | #define SIW_MAX_MW 0 /* to be set if MW's are supported */ | |
32 | #define SIW_MAX_FMR SIW_MAX_MR | |
33 | #define SIW_MAX_SRQ SIW_MAX_QP | |
34 | #define SIW_MAX_SRQ_WR (SIW_MAX_QP_WR * 10) | |
35 | #define SIW_MAX_CONTEXT SIW_MAX_PD | |
36 | ||
37 | /* Min number of bytes for using zero copy transmit */ | |
38 | #define SENDPAGE_THRESH PAGE_SIZE | |
39 | ||
40 | /* Maximum number of frames which can be send in one SQ processing */ | |
41 | #define SQ_USER_MAXBURST 100 | |
42 | ||
43 | /* Maximum number of consecutive IRQ elements which get served | |
44 | * if SQ has pending work. Prevents starving local SQ processing | |
45 | * by serving peer Read Requests. | |
46 | */ | |
47 | #define SIW_IRQ_MAXBURST_SQ_ACTIVE 4 | |
48 | ||
49 | struct siw_dev_cap { | |
50 | int max_qp; | |
51 | int max_qp_wr; | |
52 | int max_ord; /* max. outbound read queue depth */ | |
53 | int max_ird; /* max. inbound read queue depth */ | |
54 | int max_sge; | |
55 | int max_sge_rd; | |
56 | int max_cq; | |
57 | int max_cqe; | |
58 | int max_mr; | |
59 | int max_pd; | |
60 | int max_mw; | |
61 | int max_fmr; | |
62 | int max_srq; | |
63 | int max_srq_wr; | |
64 | int max_srq_sge; | |
65 | }; | |
66 | ||
67 | struct siw_pd { | |
68 | struct ib_pd base_pd; | |
69 | }; | |
70 | ||
71 | struct siw_device { | |
72 | struct ib_device base_dev; | |
73 | struct net_device *netdev; | |
74 | struct siw_dev_cap attrs; | |
75 | ||
76 | u32 vendor_part_id; | |
77 | int numa_node; | |
78 | ||
79 | /* physical port state (only one port per device) */ | |
80 | enum ib_port_state state; | |
81 | ||
82 | spinlock_t lock; | |
83 | ||
84 | struct xarray qp_xa; | |
85 | struct xarray mem_xa; | |
86 | ||
87 | struct list_head cep_list; | |
88 | struct list_head qp_list; | |
89 | ||
90 | /* active objects statistics to enforce limits */ | |
91 | atomic_t num_qp; | |
92 | atomic_t num_cq; | |
93 | atomic_t num_pd; | |
94 | atomic_t num_mr; | |
95 | atomic_t num_srq; | |
96 | atomic_t num_ctx; | |
97 | ||
98 | struct work_struct netdev_down; | |
99 | }; | |
100 | ||
101 | struct siw_uobj { | |
102 | void *addr; | |
103 | u32 size; | |
104 | }; | |
105 | ||
106 | struct siw_ucontext { | |
107 | struct ib_ucontext base_ucontext; | |
108 | struct siw_device *sdev; | |
109 | ||
110 | /* xarray of user mappable objects */ | |
111 | struct xarray xa; | |
112 | u32 uobj_nextkey; | |
113 | }; | |
114 | ||
115 | /* | |
116 | * The RDMA core does not define LOCAL_READ access, which is always | |
117 | * enabled implictely. | |
118 | */ | |
119 | #define IWARP_ACCESS_MASK \ | |
120 | (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | \ | |
121 | IB_ACCESS_REMOTE_READ) | |
122 | ||
123 | /* | |
124 | * siw presentation of user memory registered as source | |
125 | * or target of RDMA operations. | |
126 | */ | |
127 | ||
128 | struct siw_page_chunk { | |
129 | struct page **plist; | |
130 | }; | |
131 | ||
132 | struct siw_umem { | |
133 | struct siw_page_chunk *page_chunk; | |
134 | int num_pages; | |
135 | bool writable; | |
136 | u64 fp_addr; /* First page base address */ | |
137 | struct mm_struct *owning_mm; | |
138 | }; | |
139 | ||
140 | struct siw_pble { | |
141 | u64 addr; /* Address of assigned user buffer */ | |
142 | u64 size; /* Size of this entry */ | |
143 | u64 pbl_off; /* Total offset from start of PBL */ | |
144 | }; | |
145 | ||
146 | struct siw_pbl { | |
147 | unsigned int num_buf; | |
148 | unsigned int max_buf; | |
149 | struct siw_pble pbe[1]; | |
150 | }; | |
151 | ||
152 | struct siw_mr; | |
153 | ||
154 | /* | |
155 | * Generic memory representation for registered siw memory. | |
156 | * Memory lookup always via higher 24 bit of STag (STag index). | |
157 | */ | |
158 | struct siw_mem { | |
159 | struct siw_device *sdev; | |
160 | struct kref ref; | |
161 | u64 va; /* VA of memory */ | |
162 | u64 len; /* length of the memory buffer in bytes */ | |
163 | u32 stag; /* iWarp memory access steering tag */ | |
164 | u8 stag_valid; /* VALID or INVALID */ | |
165 | u8 is_pbl; /* PBL or user space mem */ | |
166 | u8 is_mw; /* Memory Region or Memory Window */ | |
167 | enum ib_access_flags perms; /* local/remote READ & WRITE */ | |
168 | union { | |
169 | struct siw_umem *umem; | |
170 | struct siw_pbl *pbl; | |
171 | void *mem_obj; | |
172 | }; | |
173 | struct ib_pd *pd; | |
174 | }; | |
175 | ||
176 | struct siw_mr { | |
177 | struct ib_mr base_mr; | |
178 | struct siw_mem *mem; | |
179 | struct rcu_head rcu; | |
180 | }; | |
181 | ||
182 | /* | |
183 | * Error codes for local or remote | |
184 | * access to registered memory | |
185 | */ | |
186 | enum siw_access_state { | |
187 | E_ACCESS_OK, | |
188 | E_STAG_INVALID, | |
189 | E_BASE_BOUNDS, | |
190 | E_ACCESS_PERM, | |
191 | E_PD_MISMATCH | |
192 | }; | |
193 | ||
194 | enum siw_wr_state { | |
195 | SIW_WR_IDLE, | |
196 | SIW_WR_QUEUED, /* processing has not started yet */ | |
197 | SIW_WR_INPROGRESS /* initiated processing of the WR */ | |
198 | }; | |
199 | ||
200 | /* The WQE currently being processed (RX or TX) */ | |
201 | struct siw_wqe { | |
202 | /* Copy of applications SQE or RQE */ | |
203 | union { | |
204 | struct siw_sqe sqe; | |
205 | struct siw_rqe rqe; | |
206 | }; | |
207 | struct siw_mem *mem[SIW_MAX_SGE]; /* per sge's resolved mem */ | |
208 | enum siw_wr_state wr_status; | |
209 | enum siw_wc_status wc_status; | |
210 | u32 bytes; /* total bytes to process */ | |
211 | u32 processed; /* bytes processed */ | |
212 | }; | |
213 | ||
214 | struct siw_cq { | |
215 | struct ib_cq base_cq; | |
216 | spinlock_t lock; | |
2c8ccb37 | 217 | struct siw_cq_ctrl *notify; |
a5319752 BM |
218 | struct siw_cqe *queue; |
219 | u32 cq_put; | |
220 | u32 cq_get; | |
221 | u32 num_cqe; | |
222 | bool kernel_verbs; | |
223 | u32 xa_cq_index; /* mmap information for CQE array */ | |
224 | u32 id; /* For debugging only */ | |
225 | }; | |
226 | ||
227 | enum siw_qp_state { | |
228 | SIW_QP_STATE_IDLE, | |
229 | SIW_QP_STATE_RTR, | |
230 | SIW_QP_STATE_RTS, | |
231 | SIW_QP_STATE_CLOSING, | |
232 | SIW_QP_STATE_TERMINATE, | |
233 | SIW_QP_STATE_ERROR, | |
234 | SIW_QP_STATE_COUNT | |
235 | }; | |
236 | ||
237 | enum siw_qp_flags { | |
238 | SIW_RDMA_BIND_ENABLED = (1 << 0), | |
239 | SIW_RDMA_WRITE_ENABLED = (1 << 1), | |
240 | SIW_RDMA_READ_ENABLED = (1 << 2), | |
241 | SIW_SIGNAL_ALL_WR = (1 << 3), | |
242 | SIW_MPA_CRC = (1 << 4), | |
243 | SIW_QP_IN_DESTROY = (1 << 5) | |
244 | }; | |
245 | ||
246 | enum siw_qp_attr_mask { | |
247 | SIW_QP_ATTR_STATE = (1 << 0), | |
248 | SIW_QP_ATTR_ACCESS_FLAGS = (1 << 1), | |
249 | SIW_QP_ATTR_LLP_HANDLE = (1 << 2), | |
250 | SIW_QP_ATTR_ORD = (1 << 3), | |
251 | SIW_QP_ATTR_IRD = (1 << 4), | |
252 | SIW_QP_ATTR_SQ_SIZE = (1 << 5), | |
253 | SIW_QP_ATTR_RQ_SIZE = (1 << 6), | |
254 | SIW_QP_ATTR_MPA = (1 << 7) | |
255 | }; | |
256 | ||
257 | struct siw_srq { | |
258 | struct ib_srq base_srq; | |
259 | spinlock_t lock; | |
260 | u32 max_sge; | |
261 | u32 limit; /* low watermark for async event */ | |
262 | struct siw_rqe *recvq; | |
263 | u32 rq_put; | |
264 | u32 rq_get; | |
265 | u32 num_rqe; /* max # of wqe's allowed */ | |
266 | u32 xa_srq_index; /* mmap information for SRQ array */ | |
267 | char armed; /* inform user if limit hit */ | |
268 | char kernel_verbs; /* '1' if kernel client */ | |
269 | }; | |
270 | ||
271 | struct siw_qp_attrs { | |
272 | enum siw_qp_state state; | |
273 | u32 sq_size; | |
274 | u32 rq_size; | |
275 | u32 orq_size; | |
276 | u32 irq_size; | |
277 | u32 sq_max_sges; | |
278 | u32 rq_max_sges; | |
279 | enum siw_qp_flags flags; | |
280 | ||
281 | struct socket *sk; | |
282 | }; | |
283 | ||
284 | enum siw_tx_ctx { | |
285 | SIW_SEND_HDR, /* start or continue sending HDR */ | |
286 | SIW_SEND_DATA, /* start or continue sending DDP payload */ | |
287 | SIW_SEND_TRAILER, /* start or continue sending TRAILER */ | |
288 | SIW_SEND_SHORT_FPDU/* send whole FPDU hdr|data|trailer at once */ | |
289 | }; | |
290 | ||
291 | enum siw_rx_state { | |
292 | SIW_GET_HDR, /* await new hdr or within hdr */ | |
293 | SIW_GET_DATA_START, /* start of inbound DDP payload */ | |
294 | SIW_GET_DATA_MORE, /* continuation of (misaligned) DDP payload */ | |
295 | SIW_GET_TRAILER/* await new trailer or within trailer */ | |
296 | }; | |
297 | ||
298 | struct siw_rx_stream { | |
299 | struct sk_buff *skb; | |
300 | int skb_new; /* pending unread bytes in skb */ | |
301 | int skb_offset; /* offset in skb */ | |
302 | int skb_copied; /* processed bytes in skb */ | |
303 | ||
304 | union iwarp_hdr hdr; | |
305 | struct mpa_trailer trailer; | |
306 | ||
307 | enum siw_rx_state state; | |
308 | ||
309 | /* | |
310 | * For each FPDU, main RX loop runs through 3 stages: | |
311 | * Receiving protocol headers, placing DDP payload and receiving | |
312 | * trailer information (CRC + possibly padding). | |
313 | * Next two variables keep state on receive status of the | |
314 | * current FPDU part (hdr, data, trailer). | |
315 | */ | |
316 | int fpdu_part_rcvd; /* bytes in pkt part copied */ | |
317 | int fpdu_part_rem; /* bytes in pkt part not seen */ | |
318 | ||
319 | /* | |
320 | * Next expected DDP MSN for each QN + | |
321 | * expected steering tag + | |
322 | * expected DDP tagget offset (all HBO) | |
323 | */ | |
324 | u32 ddp_msn[RDMAP_UNTAGGED_QN_COUNT]; | |
325 | u32 ddp_stag; | |
326 | u64 ddp_to; | |
327 | u32 inval_stag; /* Stag to be invalidated */ | |
328 | ||
329 | struct shash_desc *mpa_crc_hd; | |
330 | u8 rx_suspend : 1; | |
331 | u8 pad : 2; /* # of pad bytes expected */ | |
332 | u8 rdmap_op : 4; /* opcode of current frame */ | |
333 | }; | |
334 | ||
335 | struct siw_rx_fpdu { | |
336 | /* | |
337 | * Local destination memory of inbound RDMA operation. | |
338 | * Valid, according to wqe->wr_status | |
339 | */ | |
340 | struct siw_wqe wqe_active; | |
341 | ||
342 | unsigned int pbl_idx; /* Index into current PBL */ | |
343 | unsigned int sge_idx; /* current sge in rx */ | |
344 | unsigned int sge_off; /* already rcvd in curr. sge */ | |
345 | ||
346 | char first_ddp_seg; /* this is the first DDP seg */ | |
347 | char more_ddp_segs; /* more DDP segs expected */ | |
348 | u8 prev_rdmap_op : 4; /* opcode of prev frame */ | |
349 | }; | |
350 | ||
351 | /* | |
352 | * Shorthands for short packets w/o payload | |
353 | * to be transmitted more efficient. | |
354 | */ | |
355 | struct siw_send_pkt { | |
356 | struct iwarp_send send; | |
357 | __be32 crc; | |
358 | }; | |
359 | ||
360 | struct siw_write_pkt { | |
361 | struct iwarp_rdma_write write; | |
362 | __be32 crc; | |
363 | }; | |
364 | ||
365 | struct siw_rreq_pkt { | |
366 | struct iwarp_rdma_rreq rreq; | |
367 | __be32 crc; | |
368 | }; | |
369 | ||
370 | struct siw_rresp_pkt { | |
371 | struct iwarp_rdma_rresp rresp; | |
372 | __be32 crc; | |
373 | }; | |
374 | ||
375 | struct siw_iwarp_tx { | |
376 | union { | |
377 | union iwarp_hdr hdr; | |
378 | ||
379 | /* Generic part of FPDU header */ | |
380 | struct iwarp_ctrl ctrl; | |
381 | struct iwarp_ctrl_untagged c_untagged; | |
382 | struct iwarp_ctrl_tagged c_tagged; | |
383 | ||
384 | /* FPDU headers */ | |
385 | struct iwarp_rdma_write rwrite; | |
386 | struct iwarp_rdma_rreq rreq; | |
387 | struct iwarp_rdma_rresp rresp; | |
388 | struct iwarp_terminate terminate; | |
389 | struct iwarp_send send; | |
390 | struct iwarp_send_inv send_inv; | |
391 | ||
392 | /* complete short FPDUs */ | |
393 | struct siw_send_pkt send_pkt; | |
394 | struct siw_write_pkt write_pkt; | |
395 | struct siw_rreq_pkt rreq_pkt; | |
396 | struct siw_rresp_pkt rresp_pkt; | |
397 | } pkt; | |
398 | ||
399 | struct mpa_trailer trailer; | |
400 | /* DDP MSN for untagged messages */ | |
401 | u32 ddp_msn[RDMAP_UNTAGGED_QN_COUNT]; | |
402 | ||
403 | enum siw_tx_ctx state; | |
404 | u16 ctrl_len; /* ddp+rdmap hdr */ | |
405 | u16 ctrl_sent; | |
406 | int burst; | |
407 | int bytes_unsent; /* ddp payload bytes */ | |
408 | ||
409 | struct shash_desc *mpa_crc_hd; | |
410 | ||
411 | u8 do_crc : 1; /* do crc for segment */ | |
412 | u8 use_sendpage : 1; /* send w/o copy */ | |
413 | u8 tx_suspend : 1; /* stop sending DDP segs. */ | |
414 | u8 pad : 2; /* # pad in current fpdu */ | |
415 | u8 orq_fence : 1; /* ORQ full or Send fenced */ | |
416 | u8 in_syscall : 1; /* TX out of user context */ | |
417 | u8 zcopy_tx : 1; /* Use TCP_SENDPAGE if possible */ | |
418 | u8 gso_seg_limit; /* Maximum segments for GSO, 0 = unbound */ | |
419 | ||
420 | u16 fpdu_len; /* len of FPDU to tx */ | |
421 | unsigned int tcp_seglen; /* remaining tcp seg space */ | |
422 | ||
423 | struct siw_wqe wqe_active; | |
424 | ||
425 | int pbl_idx; /* Index into current PBL */ | |
426 | int sge_idx; /* current sge in tx */ | |
427 | u32 sge_off; /* already sent in curr. sge */ | |
428 | }; | |
429 | ||
430 | struct siw_qp { | |
431 | struct siw_device *sdev; | |
432 | struct ib_qp *ib_qp; | |
433 | struct kref ref; | |
434 | u32 qp_num; | |
435 | struct list_head devq; | |
436 | int tx_cpu; | |
437 | bool kernel_verbs; | |
438 | struct siw_qp_attrs attrs; | |
439 | ||
440 | struct siw_cep *cep; | |
441 | struct rw_semaphore state_lock; | |
442 | ||
443 | struct ib_pd *pd; | |
444 | struct siw_cq *scq; | |
445 | struct siw_cq *rcq; | |
446 | struct siw_srq *srq; | |
447 | ||
448 | struct siw_iwarp_tx tx_ctx; /* Transmit context */ | |
449 | spinlock_t sq_lock; | |
450 | struct siw_sqe *sendq; /* send queue element array */ | |
451 | uint32_t sq_get; /* consumer index into sq array */ | |
452 | uint32_t sq_put; /* kernel prod. index into sq array */ | |
453 | struct llist_node tx_list; | |
454 | ||
455 | struct siw_sqe *orq; /* outbound read queue element array */ | |
456 | spinlock_t orq_lock; | |
457 | uint32_t orq_get; /* consumer index into orq array */ | |
458 | uint32_t orq_put; /* shared producer index for ORQ */ | |
459 | ||
460 | struct siw_rx_stream rx_stream; | |
461 | struct siw_rx_fpdu *rx_fpdu; | |
462 | struct siw_rx_fpdu rx_tagged; | |
463 | struct siw_rx_fpdu rx_untagged; | |
464 | spinlock_t rq_lock; | |
465 | struct siw_rqe *recvq; /* recv queue element array */ | |
466 | uint32_t rq_get; /* consumer index into rq array */ | |
467 | uint32_t rq_put; /* kernel prod. index into rq array */ | |
468 | ||
469 | struct siw_sqe *irq; /* inbound read queue element array */ | |
470 | uint32_t irq_get; /* consumer index into irq array */ | |
471 | uint32_t irq_put; /* producer index into irq array */ | |
472 | int irq_burst; | |
473 | ||
474 | struct { /* information to be carried in TERMINATE pkt, if valid */ | |
475 | u8 valid; | |
476 | u8 in_tx; | |
477 | u8 layer : 4, etype : 4; | |
478 | u8 ecode; | |
479 | } term_info; | |
480 | u32 xa_sq_index; /* mmap information for SQE array */ | |
481 | u32 xa_rq_index; /* mmap information for RQE array */ | |
482 | struct rcu_head rcu; | |
483 | }; | |
484 | ||
485 | struct siw_base_qp { | |
486 | struct ib_qp base_qp; | |
487 | struct siw_qp *qp; | |
488 | }; | |
489 | ||
490 | /* helper macros */ | |
491 | #define rx_qp(rx) container_of(rx, struct siw_qp, rx_stream) | |
492 | #define tx_qp(tx) container_of(tx, struct siw_qp, tx_ctx) | |
493 | #define tx_wqe(qp) (&(qp)->tx_ctx.wqe_active) | |
494 | #define rx_wqe(rctx) (&(rctx)->wqe_active) | |
495 | #define rx_mem(rctx) ((rctx)->wqe_active.mem[0]) | |
496 | #define tx_type(wqe) ((wqe)->sqe.opcode) | |
497 | #define rx_type(wqe) ((wqe)->rqe.opcode) | |
498 | #define tx_flags(wqe) ((wqe)->sqe.flags) | |
499 | ||
500 | struct iwarp_msg_info { | |
501 | int hdr_len; | |
502 | struct iwarp_ctrl ctrl; | |
503 | int (*rx_data)(struct siw_qp *qp); | |
504 | }; | |
505 | ||
506 | /* Global siw parameters. Currently set in siw_main.c */ | |
507 | extern const bool zcopy_tx; | |
508 | extern const bool try_gso; | |
509 | extern const bool loopback_enabled; | |
510 | extern const bool mpa_crc_required; | |
511 | extern const bool mpa_crc_strict; | |
512 | extern const bool siw_tcp_nagle; | |
513 | extern u_char mpa_version; | |
514 | extern const bool peer_to_peer; | |
515 | extern struct task_struct *siw_tx_thread[]; | |
516 | ||
517 | extern struct crypto_shash *siw_crypto_shash; | |
518 | extern struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1]; | |
519 | ||
520 | /* QP general functions */ | |
521 | int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attr, | |
522 | enum siw_qp_attr_mask mask); | |
523 | int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl); | |
524 | void siw_qp_llp_close(struct siw_qp *qp); | |
525 | void siw_qp_cm_drop(struct siw_qp *qp, int schedule); | |
526 | void siw_send_terminate(struct siw_qp *qp); | |
527 | ||
528 | void siw_qp_get_ref(struct ib_qp *qp); | |
529 | void siw_qp_put_ref(struct ib_qp *qp); | |
530 | int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp); | |
531 | void siw_free_qp(struct kref *ref); | |
532 | ||
533 | void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, | |
534 | u8 etype, u8 ecode, int in_tx); | |
535 | enum ddp_ecode siw_tagged_error(enum siw_access_state state); | |
536 | enum rdmap_ecode siw_rdmap_error(enum siw_access_state state); | |
537 | ||
538 | void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe); | |
539 | int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, | |
540 | enum siw_wc_status status); | |
541 | int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, | |
542 | u32 inval_stag, enum siw_wc_status status); | |
543 | void siw_qp_llp_data_ready(struct sock *sk); | |
544 | void siw_qp_llp_write_space(struct sock *sk); | |
545 | ||
546 | /* QP TX path functions */ | |
547 | int siw_run_sq(void *arg); | |
548 | int siw_qp_sq_process(struct siw_qp *qp); | |
549 | int siw_sq_start(struct siw_qp *qp); | |
550 | int siw_activate_tx(struct siw_qp *qp); | |
551 | void siw_stop_tx_thread(int nr_cpu); | |
552 | int siw_get_tx_cpu(struct siw_device *sdev); | |
553 | void siw_put_tx_cpu(int cpu); | |
554 | ||
555 | /* QP RX path functions */ | |
556 | int siw_proc_send(struct siw_qp *qp); | |
557 | int siw_proc_rreq(struct siw_qp *qp); | |
558 | int siw_proc_rresp(struct siw_qp *qp); | |
559 | int siw_proc_write(struct siw_qp *qp); | |
560 | int siw_proc_terminate(struct siw_qp *qp); | |
561 | ||
562 | int siw_tcp_rx_data(read_descriptor_t *rd_desc, struct sk_buff *skb, | |
563 | unsigned int off, size_t len); | |
564 | ||
565 | static inline void set_rx_fpdu_context(struct siw_qp *qp, u8 opcode) | |
566 | { | |
567 | if (opcode == RDMAP_RDMA_WRITE || opcode == RDMAP_RDMA_READ_RESP) | |
568 | qp->rx_fpdu = &qp->rx_tagged; | |
569 | else | |
570 | qp->rx_fpdu = &qp->rx_untagged; | |
571 | ||
572 | qp->rx_stream.rdmap_op = opcode; | |
573 | } | |
574 | ||
575 | static inline struct siw_ucontext *to_siw_ctx(struct ib_ucontext *base_ctx) | |
576 | { | |
577 | return container_of(base_ctx, struct siw_ucontext, base_ucontext); | |
578 | } | |
579 | ||
580 | static inline struct siw_base_qp *to_siw_base_qp(struct ib_qp *base_qp) | |
581 | { | |
582 | return container_of(base_qp, struct siw_base_qp, base_qp); | |
583 | } | |
584 | ||
585 | static inline struct siw_qp *to_siw_qp(struct ib_qp *base_qp) | |
586 | { | |
587 | return to_siw_base_qp(base_qp)->qp; | |
588 | } | |
589 | ||
590 | static inline struct siw_cq *to_siw_cq(struct ib_cq *base_cq) | |
591 | { | |
592 | return container_of(base_cq, struct siw_cq, base_cq); | |
593 | } | |
594 | ||
595 | static inline struct siw_srq *to_siw_srq(struct ib_srq *base_srq) | |
596 | { | |
597 | return container_of(base_srq, struct siw_srq, base_srq); | |
598 | } | |
599 | ||
600 | static inline struct siw_device *to_siw_dev(struct ib_device *base_dev) | |
601 | { | |
602 | return container_of(base_dev, struct siw_device, base_dev); | |
603 | } | |
604 | ||
605 | static inline struct siw_mr *to_siw_mr(struct ib_mr *base_mr) | |
606 | { | |
607 | return container_of(base_mr, struct siw_mr, base_mr); | |
608 | } | |
609 | ||
610 | static inline struct siw_qp *siw_qp_id2obj(struct siw_device *sdev, int id) | |
611 | { | |
612 | struct siw_qp *qp; | |
613 | ||
614 | rcu_read_lock(); | |
615 | qp = xa_load(&sdev->qp_xa, id); | |
616 | if (likely(qp && kref_get_unless_zero(&qp->ref))) { | |
617 | rcu_read_unlock(); | |
618 | return qp; | |
619 | } | |
620 | rcu_read_unlock(); | |
621 | return NULL; | |
622 | } | |
623 | ||
624 | static inline u32 qp_id(struct siw_qp *qp) | |
625 | { | |
626 | return qp->qp_num; | |
627 | } | |
628 | ||
629 | static inline void siw_qp_get(struct siw_qp *qp) | |
630 | { | |
631 | kref_get(&qp->ref); | |
632 | } | |
633 | ||
634 | static inline void siw_qp_put(struct siw_qp *qp) | |
635 | { | |
636 | kref_put(&qp->ref, siw_free_qp); | |
637 | } | |
638 | ||
639 | static inline int siw_sq_empty(struct siw_qp *qp) | |
640 | { | |
641 | struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; | |
642 | ||
643 | return READ_ONCE(sqe->flags) == 0; | |
644 | } | |
645 | ||
646 | static inline struct siw_sqe *sq_get_next(struct siw_qp *qp) | |
647 | { | |
648 | struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; | |
649 | ||
650 | if (READ_ONCE(sqe->flags) & SIW_WQE_VALID) | |
651 | return sqe; | |
652 | ||
653 | return NULL; | |
654 | } | |
655 | ||
656 | static inline struct siw_sqe *orq_get_current(struct siw_qp *qp) | |
657 | { | |
658 | return &qp->orq[qp->orq_get % qp->attrs.orq_size]; | |
659 | } | |
660 | ||
661 | static inline struct siw_sqe *orq_get_tail(struct siw_qp *qp) | |
662 | { | |
663 | return &qp->orq[qp->orq_put % qp->attrs.orq_size]; | |
664 | } | |
665 | ||
666 | static inline struct siw_sqe *orq_get_free(struct siw_qp *qp) | |
667 | { | |
668 | struct siw_sqe *orq_e = orq_get_tail(qp); | |
669 | ||
670 | if (orq_e && READ_ONCE(orq_e->flags) == 0) | |
671 | return orq_e; | |
672 | ||
673 | return NULL; | |
674 | } | |
675 | ||
676 | static inline int siw_orq_empty(struct siw_qp *qp) | |
677 | { | |
678 | return qp->orq[qp->orq_get % qp->attrs.orq_size].flags == 0 ? 1 : 0; | |
679 | } | |
680 | ||
681 | static inline struct siw_sqe *irq_alloc_free(struct siw_qp *qp) | |
682 | { | |
683 | struct siw_sqe *irq_e = &qp->irq[qp->irq_put % qp->attrs.irq_size]; | |
684 | ||
685 | if (READ_ONCE(irq_e->flags) == 0) { | |
686 | qp->irq_put++; | |
687 | return irq_e; | |
688 | } | |
689 | return NULL; | |
690 | } | |
691 | ||
692 | static inline __wsum siw_csum_update(const void *buff, int len, __wsum sum) | |
693 | { | |
694 | return (__force __wsum)crc32c((__force __u32)sum, buff, len); | |
695 | } | |
696 | ||
697 | static inline __wsum siw_csum_combine(__wsum csum, __wsum csum2, int offset, | |
698 | int len) | |
699 | { | |
700 | return (__force __wsum)__crc32c_le_combine((__force __u32)csum, | |
701 | (__force __u32)csum2, len); | |
702 | } | |
703 | ||
704 | static inline void siw_crc_skb(struct siw_rx_stream *srx, unsigned int len) | |
705 | { | |
706 | const struct skb_checksum_ops siw_cs_ops = { | |
707 | .update = siw_csum_update, | |
708 | .combine = siw_csum_combine, | |
709 | }; | |
710 | __wsum crc = *(u32 *)shash_desc_ctx(srx->mpa_crc_hd); | |
711 | ||
712 | crc = __skb_checksum(srx->skb, srx->skb_offset, len, crc, | |
713 | &siw_cs_ops); | |
714 | *(u32 *)shash_desc_ctx(srx->mpa_crc_hd) = crc; | |
715 | } | |
716 | ||
717 | #define siw_dbg(ibdev, fmt, ...) \ | |
718 | ibdev_dbg(ibdev, "%s: " fmt, __func__, ##__VA_ARGS__) | |
719 | ||
720 | #define siw_dbg_qp(qp, fmt, ...) \ | |
721 | ibdev_dbg(&qp->sdev->base_dev, "QP[%u] %s: " fmt, qp_id(qp), __func__, \ | |
722 | ##__VA_ARGS__) | |
723 | ||
724 | #define siw_dbg_cq(cq, fmt, ...) \ | |
725 | ibdev_dbg(cq->base_cq.device, "CQ[%u] %s: " fmt, cq->id, __func__, \ | |
726 | ##__VA_ARGS__) | |
727 | ||
728 | #define siw_dbg_pd(pd, fmt, ...) \ | |
729 | ibdev_dbg(pd->device, "PD[%u] %s: " fmt, pd->res.id, __func__, \ | |
730 | ##__VA_ARGS__) | |
731 | ||
732 | #define siw_dbg_mem(mem, fmt, ...) \ | |
733 | ibdev_dbg(&mem->sdev->base_dev, \ | |
734 | "MEM[0x%08x] %s: " fmt, mem->stag, __func__, ##__VA_ARGS__) | |
735 | ||
736 | #define siw_dbg_cep(cep, fmt, ...) \ | |
737 | ibdev_dbg(&cep->sdev->base_dev, "CEP[0x%p] %s: " fmt, \ | |
738 | cep, __func__, ##__VA_ARGS__) | |
739 | ||
740 | void siw_cq_flush(struct siw_cq *cq); | |
741 | void siw_sq_flush(struct siw_qp *qp); | |
742 | void siw_rq_flush(struct siw_qp *qp); | |
743 | int siw_reap_cqe(struct siw_cq *cq, struct ib_wc *wc); | |
744 | ||
745 | #endif |