Commit | Line | Data |
---|---|---|
3ec648c6 | 1 | /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ |
a5319752 BM |
2 | |
3 | /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ | |
4 | /* Copyright (c) 2008-2019, IBM Corporation */ | |
5 | ||
6 | #ifndef _SIW_H | |
7 | #define _SIW_H | |
8 | ||
9 | #include <rdma/ib_verbs.h> | |
58fb0b56 | 10 | #include <rdma/restrack.h> |
a5319752 BM |
11 | #include <linux/socket.h> |
12 | #include <linux/skbuff.h> | |
13 | #include <crypto/hash.h> | |
14 | #include <linux/crc32.h> | |
15 | #include <linux/crc32c.h> | |
16 | ||
17 | #include <rdma/siw-abi.h> | |
18 | #include "iwarp.h" | |
19 | ||
20 | #define SIW_VENDOR_ID 0x626d74 /* ascii 'bmt' for now */ | |
21 | #define SIW_VENDORT_PART_ID 0 | |
22 | #define SIW_MAX_QP (1024 * 100) | |
23 | #define SIW_MAX_QP_WR (1024 * 32) | |
24 | #define SIW_MAX_ORD_QP 128 | |
25 | #define SIW_MAX_IRD_QP 128 | |
26 | #define SIW_MAX_SGE_PBL 256 /* max num sge's for PBL */ | |
27 | #define SIW_MAX_SGE_RD 1 /* iwarp limitation. we could relax */ | |
28 | #define SIW_MAX_CQ (1024 * 100) | |
29 | #define SIW_MAX_CQE (SIW_MAX_QP_WR * 100) | |
30 | #define SIW_MAX_MR (SIW_MAX_QP * 10) | |
31 | #define SIW_MAX_PD SIW_MAX_QP | |
32 | #define SIW_MAX_MW 0 /* to be set if MW's are supported */ | |
a5319752 BM |
33 | #define SIW_MAX_SRQ SIW_MAX_QP |
34 | #define SIW_MAX_SRQ_WR (SIW_MAX_QP_WR * 10) | |
35 | #define SIW_MAX_CONTEXT SIW_MAX_PD | |
36 | ||
37 | /* Min number of bytes for using zero copy transmit */ | |
38 | #define SENDPAGE_THRESH PAGE_SIZE | |
39 | ||
40 | /* Maximum number of frames which can be send in one SQ processing */ | |
41 | #define SQ_USER_MAXBURST 100 | |
42 | ||
43 | /* Maximum number of consecutive IRQ elements which get served | |
44 | * if SQ has pending work. Prevents starving local SQ processing | |
45 | * by serving peer Read Requests. | |
46 | */ | |
47 | #define SIW_IRQ_MAXBURST_SQ_ACTIVE 4 | |
48 | ||
49 | struct siw_dev_cap { | |
50 | int max_qp; | |
51 | int max_qp_wr; | |
52 | int max_ord; /* max. outbound read queue depth */ | |
53 | int max_ird; /* max. inbound read queue depth */ | |
54 | int max_sge; | |
55 | int max_sge_rd; | |
56 | int max_cq; | |
57 | int max_cqe; | |
58 | int max_mr; | |
59 | int max_pd; | |
60 | int max_mw; | |
a5319752 BM |
61 | int max_srq; |
62 | int max_srq_wr; | |
63 | int max_srq_sge; | |
64 | }; | |
65 | ||
66 | struct siw_pd { | |
67 | struct ib_pd base_pd; | |
68 | }; | |
69 | ||
70 | struct siw_device { | |
71 | struct ib_device base_dev; | |
72 | struct net_device *netdev; | |
73 | struct siw_dev_cap attrs; | |
74 | ||
75 | u32 vendor_part_id; | |
76 | int numa_node; | |
bad5b6e3 | 77 | char raw_gid[ETH_ALEN]; |
a5319752 BM |
78 | |
79 | /* physical port state (only one port per device) */ | |
80 | enum ib_port_state state; | |
81 | ||
82 | spinlock_t lock; | |
83 | ||
84 | struct xarray qp_xa; | |
85 | struct xarray mem_xa; | |
86 | ||
87 | struct list_head cep_list; | |
88 | struct list_head qp_list; | |
89 | ||
90 | /* active objects statistics to enforce limits */ | |
91 | atomic_t num_qp; | |
92 | atomic_t num_cq; | |
93 | atomic_t num_pd; | |
94 | atomic_t num_mr; | |
95 | atomic_t num_srq; | |
96 | atomic_t num_ctx; | |
97 | ||
98 | struct work_struct netdev_down; | |
99 | }; | |
100 | ||
a5319752 BM |
101 | struct siw_ucontext { |
102 | struct ib_ucontext base_ucontext; | |
103 | struct siw_device *sdev; | |
a5319752 BM |
104 | }; |
105 | ||
106 | /* | |
107 | * The RDMA core does not define LOCAL_READ access, which is always | |
108 | * enabled implictely. | |
109 | */ | |
110 | #define IWARP_ACCESS_MASK \ | |
111 | (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | \ | |
112 | IB_ACCESS_REMOTE_READ) | |
113 | ||
114 | /* | |
115 | * siw presentation of user memory registered as source | |
116 | * or target of RDMA operations. | |
117 | */ | |
118 | ||
119 | struct siw_page_chunk { | |
120 | struct page **plist; | |
121 | }; | |
122 | ||
123 | struct siw_umem { | |
124 | struct siw_page_chunk *page_chunk; | |
125 | int num_pages; | |
126 | bool writable; | |
127 | u64 fp_addr; /* First page base address */ | |
128 | struct mm_struct *owning_mm; | |
129 | }; | |
130 | ||
131 | struct siw_pble { | |
c536277e BM |
132 | dma_addr_t addr; /* Address of assigned buffer */ |
133 | unsigned int size; /* Size of this entry */ | |
134 | unsigned long pbl_off; /* Total offset from start of PBL */ | |
a5319752 BM |
135 | }; |
136 | ||
137 | struct siw_pbl { | |
138 | unsigned int num_buf; | |
139 | unsigned int max_buf; | |
0bc018b7 | 140 | struct siw_pble pbe[] __counted_by(max_buf); |
a5319752 BM |
141 | }; |
142 | ||
a5319752 BM |
143 | /* |
144 | * Generic memory representation for registered siw memory. | |
145 | * Memory lookup always via higher 24 bit of STag (STag index). | |
146 | */ | |
147 | struct siw_mem { | |
148 | struct siw_device *sdev; | |
149 | struct kref ref; | |
150 | u64 va; /* VA of memory */ | |
151 | u64 len; /* length of the memory buffer in bytes */ | |
152 | u32 stag; /* iWarp memory access steering tag */ | |
153 | u8 stag_valid; /* VALID or INVALID */ | |
154 | u8 is_pbl; /* PBL or user space mem */ | |
155 | u8 is_mw; /* Memory Region or Memory Window */ | |
156 | enum ib_access_flags perms; /* local/remote READ & WRITE */ | |
157 | union { | |
158 | struct siw_umem *umem; | |
159 | struct siw_pbl *pbl; | |
160 | void *mem_obj; | |
161 | }; | |
162 | struct ib_pd *pd; | |
163 | }; | |
164 | ||
165 | struct siw_mr { | |
166 | struct ib_mr base_mr; | |
167 | struct siw_mem *mem; | |
168 | struct rcu_head rcu; | |
169 | }; | |
170 | ||
171 | /* | |
172 | * Error codes for local or remote | |
173 | * access to registered memory | |
174 | */ | |
175 | enum siw_access_state { | |
176 | E_ACCESS_OK, | |
177 | E_STAG_INVALID, | |
178 | E_BASE_BOUNDS, | |
179 | E_ACCESS_PERM, | |
180 | E_PD_MISMATCH | |
181 | }; | |
182 | ||
183 | enum siw_wr_state { | |
184 | SIW_WR_IDLE, | |
185 | SIW_WR_QUEUED, /* processing has not started yet */ | |
186 | SIW_WR_INPROGRESS /* initiated processing of the WR */ | |
187 | }; | |
188 | ||
189 | /* The WQE currently being processed (RX or TX) */ | |
190 | struct siw_wqe { | |
191 | /* Copy of applications SQE or RQE */ | |
192 | union { | |
193 | struct siw_sqe sqe; | |
194 | struct siw_rqe rqe; | |
195 | }; | |
196 | struct siw_mem *mem[SIW_MAX_SGE]; /* per sge's resolved mem */ | |
197 | enum siw_wr_state wr_status; | |
198 | enum siw_wc_status wc_status; | |
199 | u32 bytes; /* total bytes to process */ | |
200 | u32 processed; /* bytes processed */ | |
201 | }; | |
202 | ||
203 | struct siw_cq { | |
204 | struct ib_cq base_cq; | |
205 | spinlock_t lock; | |
2c8ccb37 | 206 | struct siw_cq_ctrl *notify; |
a5319752 BM |
207 | struct siw_cqe *queue; |
208 | u32 cq_put; | |
209 | u32 cq_get; | |
210 | u32 num_cqe; | |
11f1a755 | 211 | struct rdma_user_mmap_entry *cq_entry; /* mmap info for CQE array */ |
a5319752 BM |
212 | u32 id; /* For debugging only */ |
213 | }; | |
214 | ||
215 | enum siw_qp_state { | |
216 | SIW_QP_STATE_IDLE, | |
217 | SIW_QP_STATE_RTR, | |
218 | SIW_QP_STATE_RTS, | |
219 | SIW_QP_STATE_CLOSING, | |
220 | SIW_QP_STATE_TERMINATE, | |
221 | SIW_QP_STATE_ERROR, | |
222 | SIW_QP_STATE_COUNT | |
223 | }; | |
224 | ||
225 | enum siw_qp_flags { | |
226 | SIW_RDMA_BIND_ENABLED = (1 << 0), | |
227 | SIW_RDMA_WRITE_ENABLED = (1 << 1), | |
228 | SIW_RDMA_READ_ENABLED = (1 << 2), | |
229 | SIW_SIGNAL_ALL_WR = (1 << 3), | |
230 | SIW_MPA_CRC = (1 << 4), | |
231 | SIW_QP_IN_DESTROY = (1 << 5) | |
232 | }; | |
233 | ||
234 | enum siw_qp_attr_mask { | |
235 | SIW_QP_ATTR_STATE = (1 << 0), | |
236 | SIW_QP_ATTR_ACCESS_FLAGS = (1 << 1), | |
237 | SIW_QP_ATTR_LLP_HANDLE = (1 << 2), | |
238 | SIW_QP_ATTR_ORD = (1 << 3), | |
239 | SIW_QP_ATTR_IRD = (1 << 4), | |
240 | SIW_QP_ATTR_SQ_SIZE = (1 << 5), | |
241 | SIW_QP_ATTR_RQ_SIZE = (1 << 6), | |
242 | SIW_QP_ATTR_MPA = (1 << 7) | |
243 | }; | |
244 | ||
245 | struct siw_srq { | |
246 | struct ib_srq base_srq; | |
247 | spinlock_t lock; | |
248 | u32 max_sge; | |
249 | u32 limit; /* low watermark for async event */ | |
250 | struct siw_rqe *recvq; | |
251 | u32 rq_put; | |
252 | u32 rq_get; | |
253 | u32 num_rqe; /* max # of wqe's allowed */ | |
11f1a755 | 254 | struct rdma_user_mmap_entry *srq_entry; /* mmap info for SRQ array */ |
58fb0b56 BM |
255 | bool armed:1; /* inform user if limit hit */ |
256 | bool is_kernel_res:1; /* true if kernel client */ | |
a5319752 BM |
257 | }; |
258 | ||
259 | struct siw_qp_attrs { | |
260 | enum siw_qp_state state; | |
261 | u32 sq_size; | |
262 | u32 rq_size; | |
263 | u32 orq_size; | |
264 | u32 irq_size; | |
265 | u32 sq_max_sges; | |
266 | u32 rq_max_sges; | |
267 | enum siw_qp_flags flags; | |
268 | ||
269 | struct socket *sk; | |
270 | }; | |
271 | ||
272 | enum siw_tx_ctx { | |
273 | SIW_SEND_HDR, /* start or continue sending HDR */ | |
274 | SIW_SEND_DATA, /* start or continue sending DDP payload */ | |
275 | SIW_SEND_TRAILER, /* start or continue sending TRAILER */ | |
276 | SIW_SEND_SHORT_FPDU/* send whole FPDU hdr|data|trailer at once */ | |
277 | }; | |
278 | ||
279 | enum siw_rx_state { | |
280 | SIW_GET_HDR, /* await new hdr or within hdr */ | |
281 | SIW_GET_DATA_START, /* start of inbound DDP payload */ | |
282 | SIW_GET_DATA_MORE, /* continuation of (misaligned) DDP payload */ | |
283 | SIW_GET_TRAILER/* await new trailer or within trailer */ | |
284 | }; | |
285 | ||
286 | struct siw_rx_stream { | |
287 | struct sk_buff *skb; | |
288 | int skb_new; /* pending unread bytes in skb */ | |
289 | int skb_offset; /* offset in skb */ | |
290 | int skb_copied; /* processed bytes in skb */ | |
291 | ||
292 | union iwarp_hdr hdr; | |
293 | struct mpa_trailer trailer; | |
294 | ||
295 | enum siw_rx_state state; | |
296 | ||
297 | /* | |
298 | * For each FPDU, main RX loop runs through 3 stages: | |
299 | * Receiving protocol headers, placing DDP payload and receiving | |
300 | * trailer information (CRC + possibly padding). | |
301 | * Next two variables keep state on receive status of the | |
302 | * current FPDU part (hdr, data, trailer). | |
303 | */ | |
304 | int fpdu_part_rcvd; /* bytes in pkt part copied */ | |
305 | int fpdu_part_rem; /* bytes in pkt part not seen */ | |
306 | ||
307 | /* | |
308 | * Next expected DDP MSN for each QN + | |
309 | * expected steering tag + | |
310 | * expected DDP tagget offset (all HBO) | |
311 | */ | |
312 | u32 ddp_msn[RDMAP_UNTAGGED_QN_COUNT]; | |
313 | u32 ddp_stag; | |
314 | u64 ddp_to; | |
315 | u32 inval_stag; /* Stag to be invalidated */ | |
316 | ||
317 | struct shash_desc *mpa_crc_hd; | |
318 | u8 rx_suspend : 1; | |
319 | u8 pad : 2; /* # of pad bytes expected */ | |
320 | u8 rdmap_op : 4; /* opcode of current frame */ | |
321 | }; | |
322 | ||
323 | struct siw_rx_fpdu { | |
324 | /* | |
325 | * Local destination memory of inbound RDMA operation. | |
326 | * Valid, according to wqe->wr_status | |
327 | */ | |
328 | struct siw_wqe wqe_active; | |
329 | ||
330 | unsigned int pbl_idx; /* Index into current PBL */ | |
331 | unsigned int sge_idx; /* current sge in rx */ | |
332 | unsigned int sge_off; /* already rcvd in curr. sge */ | |
333 | ||
334 | char first_ddp_seg; /* this is the first DDP seg */ | |
335 | char more_ddp_segs; /* more DDP segs expected */ | |
336 | u8 prev_rdmap_op : 4; /* opcode of prev frame */ | |
337 | }; | |
338 | ||
339 | /* | |
340 | * Shorthands for short packets w/o payload | |
341 | * to be transmitted more efficient. | |
342 | */ | |
343 | struct siw_send_pkt { | |
344 | struct iwarp_send send; | |
345 | __be32 crc; | |
346 | }; | |
347 | ||
348 | struct siw_write_pkt { | |
349 | struct iwarp_rdma_write write; | |
350 | __be32 crc; | |
351 | }; | |
352 | ||
353 | struct siw_rreq_pkt { | |
354 | struct iwarp_rdma_rreq rreq; | |
355 | __be32 crc; | |
356 | }; | |
357 | ||
358 | struct siw_rresp_pkt { | |
359 | struct iwarp_rdma_rresp rresp; | |
360 | __be32 crc; | |
361 | }; | |
362 | ||
363 | struct siw_iwarp_tx { | |
364 | union { | |
365 | union iwarp_hdr hdr; | |
366 | ||
367 | /* Generic part of FPDU header */ | |
368 | struct iwarp_ctrl ctrl; | |
369 | struct iwarp_ctrl_untagged c_untagged; | |
370 | struct iwarp_ctrl_tagged c_tagged; | |
371 | ||
372 | /* FPDU headers */ | |
373 | struct iwarp_rdma_write rwrite; | |
374 | struct iwarp_rdma_rreq rreq; | |
375 | struct iwarp_rdma_rresp rresp; | |
376 | struct iwarp_terminate terminate; | |
377 | struct iwarp_send send; | |
378 | struct iwarp_send_inv send_inv; | |
379 | ||
380 | /* complete short FPDUs */ | |
381 | struct siw_send_pkt send_pkt; | |
382 | struct siw_write_pkt write_pkt; | |
383 | struct siw_rreq_pkt rreq_pkt; | |
384 | struct siw_rresp_pkt rresp_pkt; | |
385 | } pkt; | |
386 | ||
387 | struct mpa_trailer trailer; | |
388 | /* DDP MSN for untagged messages */ | |
389 | u32 ddp_msn[RDMAP_UNTAGGED_QN_COUNT]; | |
390 | ||
391 | enum siw_tx_ctx state; | |
392 | u16 ctrl_len; /* ddp+rdmap hdr */ | |
393 | u16 ctrl_sent; | |
394 | int burst; | |
395 | int bytes_unsent; /* ddp payload bytes */ | |
396 | ||
397 | struct shash_desc *mpa_crc_hd; | |
398 | ||
399 | u8 do_crc : 1; /* do crc for segment */ | |
400 | u8 use_sendpage : 1; /* send w/o copy */ | |
401 | u8 tx_suspend : 1; /* stop sending DDP segs. */ | |
402 | u8 pad : 2; /* # pad in current fpdu */ | |
403 | u8 orq_fence : 1; /* ORQ full or Send fenced */ | |
404 | u8 in_syscall : 1; /* TX out of user context */ | |
405 | u8 zcopy_tx : 1; /* Use TCP_SENDPAGE if possible */ | |
406 | u8 gso_seg_limit; /* Maximum segments for GSO, 0 = unbound */ | |
407 | ||
408 | u16 fpdu_len; /* len of FPDU to tx */ | |
409 | unsigned int tcp_seglen; /* remaining tcp seg space */ | |
410 | ||
411 | struct siw_wqe wqe_active; | |
412 | ||
413 | int pbl_idx; /* Index into current PBL */ | |
414 | int sge_idx; /* current sge in tx */ | |
415 | u32 sge_off; /* already sent in curr. sge */ | |
416 | }; | |
417 | ||
418 | struct siw_qp { | |
58fb0b56 | 419 | struct ib_qp base_qp; |
a5319752 | 420 | struct siw_device *sdev; |
a5319752 | 421 | struct kref ref; |
a3c27880 | 422 | struct completion qp_free; |
a5319752 BM |
423 | struct list_head devq; |
424 | int tx_cpu; | |
a5319752 BM |
425 | struct siw_qp_attrs attrs; |
426 | ||
427 | struct siw_cep *cep; | |
428 | struct rw_semaphore state_lock; | |
429 | ||
430 | struct ib_pd *pd; | |
431 | struct siw_cq *scq; | |
432 | struct siw_cq *rcq; | |
433 | struct siw_srq *srq; | |
434 | ||
435 | struct siw_iwarp_tx tx_ctx; /* Transmit context */ | |
436 | spinlock_t sq_lock; | |
437 | struct siw_sqe *sendq; /* send queue element array */ | |
438 | uint32_t sq_get; /* consumer index into sq array */ | |
439 | uint32_t sq_put; /* kernel prod. index into sq array */ | |
440 | struct llist_node tx_list; | |
441 | ||
442 | struct siw_sqe *orq; /* outbound read queue element array */ | |
443 | spinlock_t orq_lock; | |
444 | uint32_t orq_get; /* consumer index into orq array */ | |
445 | uint32_t orq_put; /* shared producer index for ORQ */ | |
446 | ||
447 | struct siw_rx_stream rx_stream; | |
448 | struct siw_rx_fpdu *rx_fpdu; | |
449 | struct siw_rx_fpdu rx_tagged; | |
450 | struct siw_rx_fpdu rx_untagged; | |
451 | spinlock_t rq_lock; | |
452 | struct siw_rqe *recvq; /* recv queue element array */ | |
453 | uint32_t rq_get; /* consumer index into rq array */ | |
454 | uint32_t rq_put; /* kernel prod. index into rq array */ | |
455 | ||
456 | struct siw_sqe *irq; /* inbound read queue element array */ | |
457 | uint32_t irq_get; /* consumer index into irq array */ | |
458 | uint32_t irq_put; /* producer index into irq array */ | |
459 | int irq_burst; | |
460 | ||
461 | struct { /* information to be carried in TERMINATE pkt, if valid */ | |
462 | u8 valid; | |
463 | u8 in_tx; | |
464 | u8 layer : 4, etype : 4; | |
465 | u8 ecode; | |
466 | } term_info; | |
11f1a755 MK |
467 | struct rdma_user_mmap_entry *sq_entry; /* mmap info for SQE array */ |
468 | struct rdma_user_mmap_entry *rq_entry; /* mmap info for RQE array */ | |
a5319752 BM |
469 | struct rcu_head rcu; |
470 | }; | |
471 | ||
a5319752 BM |
472 | /* helper macros */ |
473 | #define rx_qp(rx) container_of(rx, struct siw_qp, rx_stream) | |
474 | #define tx_qp(tx) container_of(tx, struct siw_qp, tx_ctx) | |
475 | #define tx_wqe(qp) (&(qp)->tx_ctx.wqe_active) | |
476 | #define rx_wqe(rctx) (&(rctx)->wqe_active) | |
477 | #define rx_mem(rctx) ((rctx)->wqe_active.mem[0]) | |
478 | #define tx_type(wqe) ((wqe)->sqe.opcode) | |
479 | #define rx_type(wqe) ((wqe)->rqe.opcode) | |
480 | #define tx_flags(wqe) ((wqe)->sqe.flags) | |
481 | ||
482 | struct iwarp_msg_info { | |
483 | int hdr_len; | |
484 | struct iwarp_ctrl ctrl; | |
485 | int (*rx_data)(struct siw_qp *qp); | |
486 | }; | |
487 | ||
11f1a755 MK |
488 | struct siw_user_mmap_entry { |
489 | struct rdma_user_mmap_entry rdma_entry; | |
490 | void *address; | |
491 | }; | |
492 | ||
a5319752 BM |
493 | /* Global siw parameters. Currently set in siw_main.c */ |
494 | extern const bool zcopy_tx; | |
495 | extern const bool try_gso; | |
496 | extern const bool loopback_enabled; | |
497 | extern const bool mpa_crc_required; | |
498 | extern const bool mpa_crc_strict; | |
499 | extern const bool siw_tcp_nagle; | |
500 | extern u_char mpa_version; | |
501 | extern const bool peer_to_peer; | |
502 | extern struct task_struct *siw_tx_thread[]; | |
503 | ||
504 | extern struct crypto_shash *siw_crypto_shash; | |
505 | extern struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1]; | |
506 | ||
507 | /* QP general functions */ | |
508 | int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attr, | |
509 | enum siw_qp_attr_mask mask); | |
510 | int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl); | |
511 | void siw_qp_llp_close(struct siw_qp *qp); | |
512 | void siw_qp_cm_drop(struct siw_qp *qp, int schedule); | |
513 | void siw_send_terminate(struct siw_qp *qp); | |
514 | ||
515 | void siw_qp_get_ref(struct ib_qp *qp); | |
516 | void siw_qp_put_ref(struct ib_qp *qp); | |
517 | int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp); | |
518 | void siw_free_qp(struct kref *ref); | |
519 | ||
520 | void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, | |
521 | u8 etype, u8 ecode, int in_tx); | |
522 | enum ddp_ecode siw_tagged_error(enum siw_access_state state); | |
523 | enum rdmap_ecode siw_rdmap_error(enum siw_access_state state); | |
524 | ||
525 | void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe); | |
526 | int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, | |
527 | enum siw_wc_status status); | |
528 | int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, | |
529 | u32 inval_stag, enum siw_wc_status status); | |
530 | void siw_qp_llp_data_ready(struct sock *sk); | |
531 | void siw_qp_llp_write_space(struct sock *sk); | |
532 | ||
533 | /* QP TX path functions */ | |
91f36237 BM |
534 | int siw_create_tx_threads(void); |
535 | void siw_stop_tx_threads(void); | |
a5319752 BM |
536 | int siw_run_sq(void *arg); |
537 | int siw_qp_sq_process(struct siw_qp *qp); | |
538 | int siw_sq_start(struct siw_qp *qp); | |
539 | int siw_activate_tx(struct siw_qp *qp); | |
a5319752 BM |
540 | int siw_get_tx_cpu(struct siw_device *sdev); |
541 | void siw_put_tx_cpu(int cpu); | |
542 | ||
543 | /* QP RX path functions */ | |
544 | int siw_proc_send(struct siw_qp *qp); | |
545 | int siw_proc_rreq(struct siw_qp *qp); | |
546 | int siw_proc_rresp(struct siw_qp *qp); | |
547 | int siw_proc_write(struct siw_qp *qp); | |
548 | int siw_proc_terminate(struct siw_qp *qp); | |
549 | ||
550 | int siw_tcp_rx_data(read_descriptor_t *rd_desc, struct sk_buff *skb, | |
551 | unsigned int off, size_t len); | |
552 | ||
553 | static inline void set_rx_fpdu_context(struct siw_qp *qp, u8 opcode) | |
554 | { | |
555 | if (opcode == RDMAP_RDMA_WRITE || opcode == RDMAP_RDMA_READ_RESP) | |
556 | qp->rx_fpdu = &qp->rx_tagged; | |
557 | else | |
558 | qp->rx_fpdu = &qp->rx_untagged; | |
559 | ||
560 | qp->rx_stream.rdmap_op = opcode; | |
561 | } | |
562 | ||
563 | static inline struct siw_ucontext *to_siw_ctx(struct ib_ucontext *base_ctx) | |
564 | { | |
565 | return container_of(base_ctx, struct siw_ucontext, base_ucontext); | |
566 | } | |
567 | ||
a5319752 BM |
568 | static inline struct siw_qp *to_siw_qp(struct ib_qp *base_qp) |
569 | { | |
58fb0b56 | 570 | return container_of(base_qp, struct siw_qp, base_qp); |
a5319752 BM |
571 | } |
572 | ||
573 | static inline struct siw_cq *to_siw_cq(struct ib_cq *base_cq) | |
574 | { | |
575 | return container_of(base_cq, struct siw_cq, base_cq); | |
576 | } | |
577 | ||
578 | static inline struct siw_srq *to_siw_srq(struct ib_srq *base_srq) | |
579 | { | |
580 | return container_of(base_srq, struct siw_srq, base_srq); | |
581 | } | |
582 | ||
583 | static inline struct siw_device *to_siw_dev(struct ib_device *base_dev) | |
584 | { | |
585 | return container_of(base_dev, struct siw_device, base_dev); | |
586 | } | |
587 | ||
588 | static inline struct siw_mr *to_siw_mr(struct ib_mr *base_mr) | |
589 | { | |
590 | return container_of(base_mr, struct siw_mr, base_mr); | |
591 | } | |
592 | ||
11f1a755 MK |
593 | static inline struct siw_user_mmap_entry * |
594 | to_siw_mmap_entry(struct rdma_user_mmap_entry *rdma_mmap) | |
595 | { | |
596 | return container_of(rdma_mmap, struct siw_user_mmap_entry, rdma_entry); | |
597 | } | |
598 | ||
a5319752 BM |
599 | static inline struct siw_qp *siw_qp_id2obj(struct siw_device *sdev, int id) |
600 | { | |
601 | struct siw_qp *qp; | |
602 | ||
603 | rcu_read_lock(); | |
604 | qp = xa_load(&sdev->qp_xa, id); | |
605 | if (likely(qp && kref_get_unless_zero(&qp->ref))) { | |
606 | rcu_read_unlock(); | |
607 | return qp; | |
608 | } | |
609 | rcu_read_unlock(); | |
610 | return NULL; | |
611 | } | |
612 | ||
613 | static inline u32 qp_id(struct siw_qp *qp) | |
614 | { | |
58fb0b56 | 615 | return qp->base_qp.qp_num; |
a5319752 BM |
616 | } |
617 | ||
618 | static inline void siw_qp_get(struct siw_qp *qp) | |
619 | { | |
620 | kref_get(&qp->ref); | |
621 | } | |
622 | ||
623 | static inline void siw_qp_put(struct siw_qp *qp) | |
624 | { | |
625 | kref_put(&qp->ref, siw_free_qp); | |
626 | } | |
627 | ||
628 | static inline int siw_sq_empty(struct siw_qp *qp) | |
629 | { | |
630 | struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; | |
631 | ||
632 | return READ_ONCE(sqe->flags) == 0; | |
633 | } | |
634 | ||
635 | static inline struct siw_sqe *sq_get_next(struct siw_qp *qp) | |
636 | { | |
637 | struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; | |
638 | ||
639 | if (READ_ONCE(sqe->flags) & SIW_WQE_VALID) | |
640 | return sqe; | |
641 | ||
642 | return NULL; | |
643 | } | |
644 | ||
645 | static inline struct siw_sqe *orq_get_current(struct siw_qp *qp) | |
646 | { | |
647 | return &qp->orq[qp->orq_get % qp->attrs.orq_size]; | |
648 | } | |
649 | ||
a5319752 BM |
650 | static inline struct siw_sqe *orq_get_free(struct siw_qp *qp) |
651 | { | |
b43a76f4 | 652 | struct siw_sqe *orq_e = &qp->orq[qp->orq_put % qp->attrs.orq_size]; |
a5319752 | 653 | |
661f3859 | 654 | if (READ_ONCE(orq_e->flags) == 0) |
a5319752 BM |
655 | return orq_e; |
656 | ||
657 | return NULL; | |
658 | } | |
659 | ||
660 | static inline int siw_orq_empty(struct siw_qp *qp) | |
661 | { | |
662 | return qp->orq[qp->orq_get % qp->attrs.orq_size].flags == 0 ? 1 : 0; | |
663 | } | |
664 | ||
665 | static inline struct siw_sqe *irq_alloc_free(struct siw_qp *qp) | |
666 | { | |
667 | struct siw_sqe *irq_e = &qp->irq[qp->irq_put % qp->attrs.irq_size]; | |
668 | ||
669 | if (READ_ONCE(irq_e->flags) == 0) { | |
670 | qp->irq_put++; | |
671 | return irq_e; | |
672 | } | |
673 | return NULL; | |
674 | } | |
675 | ||
676 | static inline __wsum siw_csum_update(const void *buff, int len, __wsum sum) | |
677 | { | |
678 | return (__force __wsum)crc32c((__force __u32)sum, buff, len); | |
679 | } | |
680 | ||
681 | static inline __wsum siw_csum_combine(__wsum csum, __wsum csum2, int offset, | |
682 | int len) | |
683 | { | |
684 | return (__force __wsum)__crc32c_le_combine((__force __u32)csum, | |
685 | (__force __u32)csum2, len); | |
686 | } | |
687 | ||
688 | static inline void siw_crc_skb(struct siw_rx_stream *srx, unsigned int len) | |
689 | { | |
690 | const struct skb_checksum_ops siw_cs_ops = { | |
691 | .update = siw_csum_update, | |
692 | .combine = siw_csum_combine, | |
693 | }; | |
694 | __wsum crc = *(u32 *)shash_desc_ctx(srx->mpa_crc_hd); | |
695 | ||
696 | crc = __skb_checksum(srx->skb, srx->skb_offset, len, crc, | |
697 | &siw_cs_ops); | |
698 | *(u32 *)shash_desc_ctx(srx->mpa_crc_hd) = crc; | |
699 | } | |
700 | ||
701 | #define siw_dbg(ibdev, fmt, ...) \ | |
702 | ibdev_dbg(ibdev, "%s: " fmt, __func__, ##__VA_ARGS__) | |
703 | ||
704 | #define siw_dbg_qp(qp, fmt, ...) \ | |
705 | ibdev_dbg(&qp->sdev->base_dev, "QP[%u] %s: " fmt, qp_id(qp), __func__, \ | |
706 | ##__VA_ARGS__) | |
707 | ||
708 | #define siw_dbg_cq(cq, fmt, ...) \ | |
709 | ibdev_dbg(cq->base_cq.device, "CQ[%u] %s: " fmt, cq->id, __func__, \ | |
710 | ##__VA_ARGS__) | |
711 | ||
712 | #define siw_dbg_pd(pd, fmt, ...) \ | |
713 | ibdev_dbg(pd->device, "PD[%u] %s: " fmt, pd->res.id, __func__, \ | |
714 | ##__VA_ARGS__) | |
715 | ||
716 | #define siw_dbg_mem(mem, fmt, ...) \ | |
717 | ibdev_dbg(&mem->sdev->base_dev, \ | |
718 | "MEM[0x%08x] %s: " fmt, mem->stag, __func__, ##__VA_ARGS__) | |
719 | ||
720 | #define siw_dbg_cep(cep, fmt, ...) \ | |
58fb0b56 | 721 | ibdev_dbg(&cep->sdev->base_dev, "CEP[0x%pK] %s: " fmt, \ |
a5319752 BM |
722 | cep, __func__, ##__VA_ARGS__) |
723 | ||
724 | void siw_cq_flush(struct siw_cq *cq); | |
725 | void siw_sq_flush(struct siw_qp *qp); | |
726 | void siw_rq_flush(struct siw_qp *qp); | |
727 | int siw_reap_cqe(struct siw_cq *cq, struct ib_wc *wc); | |
728 | ||
729 | #endif |