Commit | Line | Data |
---|---|---|
cfdda9d7 SW |
1 | /* |
2 | * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | ||
42b6a949 VP |
33 | #include <linux/module.h> |
34 | #include <linux/moduleparam.h> | |
cfdda9d7 | 35 | #include <rdma/ib_umem.h> |
60063497 | 36 | #include <linux/atomic.h> |
cfdda9d7 SW |
37 | |
38 | #include "iw_cxgb4.h" | |
39 | ||
96bb2706 | 40 | int use_dsgl = 0; |
42b6a949 | 41 | module_param(use_dsgl, int, 0644); |
96bb2706 | 42 | MODULE_PARM_DESC(use_dsgl, "Use DSGL for PBL/FastReg (default=0)"); |
42b6a949 | 43 | |
cfdda9d7 SW |
44 | #define T4_ULPTX_MIN_IO 32 |
45 | #define C4IW_MAX_INLINE_SIZE 96 | |
42b6a949 VP |
46 | #define T4_ULPTX_MAX_DMA 1024 |
47 | #define C4IW_INLINE_THRESHOLD 128 | |
cfdda9d7 | 48 | |
42b6a949 VP |
49 | static int inline_threshold = C4IW_INLINE_THRESHOLD; |
50 | module_param(inline_threshold, int, 0644); | |
51 | MODULE_PARM_DESC(inline_threshold, "inline vs dsgl threshold (default=128)"); | |
52 | ||
53 | static int _c4iw_write_mem_dma_aligned(struct c4iw_rdev *rdev, u32 addr, | |
0e5eca79 | 54 | u32 len, dma_addr_t data, int wait) |
42b6a949 VP |
55 | { |
56 | struct sk_buff *skb; | |
57 | struct ulp_mem_io *req; | |
58 | struct ulptx_sgl *sgl; | |
59 | u8 wr_len; | |
60 | int ret = 0; | |
61 | struct c4iw_wr_wait wr_wait; | |
62 | ||
63 | addr &= 0x7FFFFFF; | |
64 | ||
65 | if (wait) | |
66 | c4iw_init_wr_wait(&wr_wait); | |
67 | wr_len = roundup(sizeof(*req) + sizeof(*sgl), 16); | |
68 | ||
69 | skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL); | |
70 | if (!skb) | |
71 | return -ENOMEM; | |
72 | set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); | |
73 | ||
74 | req = (struct ulp_mem_io *)__skb_put(skb, wr_len); | |
75 | memset(req, 0, wr_len); | |
76 | INIT_ULPTX_WR(req, wr_len, 0, 0); | |
e2ac9628 HS |
77 | req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | |
78 | (wait ? FW_WR_COMPL_F : 0)); | |
298589b1 | 79 | req->wr.wr_lo = wait ? (__force __be64)(unsigned long) &wr_wait : 0L; |
e2ac9628 | 80 | req->wr.wr_mid = cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(wr_len, 16))); |
42b6a949 VP |
81 | req->cmd = cpu_to_be32(ULPTX_CMD(ULP_TX_MEM_WRITE)); |
82 | req->cmd |= cpu_to_be32(V_T5_ULP_MEMIO_ORDER(1)); | |
83 | req->dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN(len>>5)); | |
84 | req->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(req->wr), 16)); | |
85 | req->lock_addr = cpu_to_be32(ULP_MEMIO_ADDR(addr)); | |
86 | ||
87 | sgl = (struct ulptx_sgl *)(req + 1); | |
88 | sgl->cmd_nsge = cpu_to_be32(ULPTX_CMD(ULP_TX_SC_DSGL) | | |
89 | ULPTX_NSGE(1)); | |
90 | sgl->len0 = cpu_to_be32(len); | |
0e5eca79 | 91 | sgl->addr0 = cpu_to_be64(data); |
42b6a949 VP |
92 | |
93 | ret = c4iw_ofld_send(rdev, skb); | |
94 | if (ret) | |
95 | return ret; | |
96 | if (wait) | |
97 | ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); | |
98 | return ret; | |
99 | } | |
100 | ||
101 | static int _c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len, | |
102 | void *data) | |
cfdda9d7 SW |
103 | { |
104 | struct sk_buff *skb; | |
105 | struct ulp_mem_io *req; | |
106 | struct ulptx_idata *sc; | |
107 | u8 wr_len, *to_dp, *from_dp; | |
108 | int copy_len, num_wqe, i, ret = 0; | |
109 | struct c4iw_wr_wait wr_wait; | |
42b6a949 VP |
110 | __be32 cmd = cpu_to_be32(ULPTX_CMD(ULP_TX_MEM_WRITE)); |
111 | ||
112 | if (is_t4(rdev->lldi.adapter_type)) | |
113 | cmd |= cpu_to_be32(ULP_MEMIO_ORDER(1)); | |
114 | else | |
115 | cmd |= cpu_to_be32(V_T5_ULP_MEMIO_IMM(1)); | |
cfdda9d7 SW |
116 | |
117 | addr &= 0x7FFFFFF; | |
118 | PDBG("%s addr 0x%x len %u\n", __func__, addr, len); | |
119 | num_wqe = DIV_ROUND_UP(len, C4IW_MAX_INLINE_SIZE); | |
120 | c4iw_init_wr_wait(&wr_wait); | |
121 | for (i = 0; i < num_wqe; i++) { | |
122 | ||
123 | copy_len = len > C4IW_MAX_INLINE_SIZE ? C4IW_MAX_INLINE_SIZE : | |
124 | len; | |
125 | wr_len = roundup(sizeof *req + sizeof *sc + | |
126 | roundup(copy_len, T4_ULPTX_MIN_IO), 16); | |
127 | ||
d3c814e8 | 128 | skb = alloc_skb(wr_len, GFP_KERNEL); |
cfdda9d7 SW |
129 | if (!skb) |
130 | return -ENOMEM; | |
131 | set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); | |
132 | ||
133 | req = (struct ulp_mem_io *)__skb_put(skb, wr_len); | |
134 | memset(req, 0, wr_len); | |
135 | INIT_ULPTX_WR(req, wr_len, 0, 0); | |
136 | ||
137 | if (i == (num_wqe-1)) { | |
e2ac9628 HS |
138 | req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | |
139 | FW_WR_COMPL_F); | |
c8e081a1 | 140 | req->wr.wr_lo = (__force __be64)(unsigned long) &wr_wait; |
cfdda9d7 | 141 | } else |
e2ac9628 | 142 | req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR)); |
cfdda9d7 | 143 | req->wr.wr_mid = cpu_to_be32( |
e2ac9628 | 144 | FW_WR_LEN16_V(DIV_ROUND_UP(wr_len, 16))); |
cfdda9d7 | 145 | |
42b6a949 | 146 | req->cmd = cmd; |
cfdda9d7 SW |
147 | req->dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN( |
148 | DIV_ROUND_UP(copy_len, T4_ULPTX_MIN_IO))); | |
149 | req->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(req->wr), | |
150 | 16)); | |
151 | req->lock_addr = cpu_to_be32(ULP_MEMIO_ADDR(addr + i * 3)); | |
152 | ||
153 | sc = (struct ulptx_idata *)(req + 1); | |
154 | sc->cmd_more = cpu_to_be32(ULPTX_CMD(ULP_TX_SC_IMM)); | |
155 | sc->len = cpu_to_be32(roundup(copy_len, T4_ULPTX_MIN_IO)); | |
156 | ||
157 | to_dp = (u8 *)(sc + 1); | |
158 | from_dp = (u8 *)data + i * C4IW_MAX_INLINE_SIZE; | |
159 | if (data) | |
160 | memcpy(to_dp, from_dp, copy_len); | |
161 | else | |
162 | memset(to_dp, 0, copy_len); | |
163 | if (copy_len % T4_ULPTX_MIN_IO) | |
164 | memset(to_dp + copy_len, 0, T4_ULPTX_MIN_IO - | |
165 | (copy_len % T4_ULPTX_MIN_IO)); | |
166 | ret = c4iw_ofld_send(rdev, skb); | |
167 | if (ret) | |
168 | return ret; | |
169 | len -= C4IW_MAX_INLINE_SIZE; | |
170 | } | |
171 | ||
aadc4df3 | 172 | ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); |
cfdda9d7 SW |
173 | return ret; |
174 | } | |
175 | ||
c00850dd | 176 | static int _c4iw_write_mem_dma(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data) |
42b6a949 VP |
177 | { |
178 | u32 remain = len; | |
179 | u32 dmalen; | |
180 | int ret = 0; | |
0e5eca79 VP |
181 | dma_addr_t daddr; |
182 | dma_addr_t save; | |
183 | ||
184 | daddr = dma_map_single(&rdev->lldi.pdev->dev, data, len, DMA_TO_DEVICE); | |
185 | if (dma_mapping_error(&rdev->lldi.pdev->dev, daddr)) | |
186 | return -1; | |
187 | save = daddr; | |
42b6a949 VP |
188 | |
189 | while (remain > inline_threshold) { | |
190 | if (remain < T4_ULPTX_MAX_DMA) { | |
191 | if (remain & ~T4_ULPTX_MIN_IO) | |
192 | dmalen = remain & ~(T4_ULPTX_MIN_IO-1); | |
193 | else | |
194 | dmalen = remain; | |
195 | } else | |
196 | dmalen = T4_ULPTX_MAX_DMA; | |
197 | remain -= dmalen; | |
0e5eca79 | 198 | ret = _c4iw_write_mem_dma_aligned(rdev, addr, dmalen, daddr, |
42b6a949 VP |
199 | !remain); |
200 | if (ret) | |
201 | goto out; | |
202 | addr += dmalen >> 5; | |
203 | data += dmalen; | |
0e5eca79 | 204 | daddr += dmalen; |
42b6a949 VP |
205 | } |
206 | if (remain) | |
207 | ret = _c4iw_write_mem_inline(rdev, addr, remain, data); | |
208 | out: | |
0e5eca79 | 209 | dma_unmap_single(&rdev->lldi.pdev->dev, save, len, DMA_TO_DEVICE); |
42b6a949 VP |
210 | return ret; |
211 | } | |
212 | ||
213 | /* | |
214 | * write len bytes of data into addr (32B aligned address) | |
215 | * If data is NULL, clear len byte of memory to zero. | |
216 | */ | |
217 | static int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, | |
218 | void *data) | |
219 | { | |
220 | if (is_t5(rdev->lldi.adapter_type) && use_dsgl) { | |
0e5eca79 VP |
221 | if (len > inline_threshold) { |
222 | if (_c4iw_write_mem_dma(rdev, addr, len, data)) { | |
223 | printk_ratelimited(KERN_WARNING | |
224 | "%s: dma map" | |
225 | " failure (non fatal)\n", | |
226 | pci_name(rdev->lldi.pdev)); | |
227 | return _c4iw_write_mem_inline(rdev, addr, len, | |
228 | data); | |
229 | } else | |
230 | return 0; | |
231 | } else | |
42b6a949 VP |
232 | return _c4iw_write_mem_inline(rdev, addr, len, data); |
233 | } else | |
234 | return _c4iw_write_mem_inline(rdev, addr, len, data); | |
235 | } | |
236 | ||
cfdda9d7 SW |
237 | /* |
238 | * Build and write a TPT entry. | |
239 | * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size, | |
240 | * pbl_size and pbl_addr | |
241 | * OUT: stag index | |
242 | */ | |
243 | static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, | |
244 | u32 *stag, u8 stag_state, u32 pdid, | |
245 | enum fw_ri_stag_type type, enum fw_ri_mem_perms perm, | |
246 | int bind_enabled, u32 zbva, u64 to, | |
247 | u64 len, u8 page_size, u32 pbl_size, u32 pbl_addr) | |
248 | { | |
249 | int err; | |
250 | struct fw_ri_tpte tpt; | |
251 | u32 stag_idx; | |
252 | static atomic_t key; | |
253 | ||
254 | if (c4iw_fatal_error(rdev)) | |
255 | return -EIO; | |
256 | ||
257 | stag_state = stag_state > 0; | |
258 | stag_idx = (*stag) >> 8; | |
259 | ||
260 | if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) { | |
ec3eead2 | 261 | stag_idx = c4iw_get_resource(&rdev->resource.tpt_table); |
98a3e879 SW |
262 | if (!stag_idx) { |
263 | mutex_lock(&rdev->stats.lock); | |
264 | rdev->stats.stag.fail++; | |
265 | mutex_unlock(&rdev->stats.lock); | |
cfdda9d7 | 266 | return -ENOMEM; |
98a3e879 | 267 | } |
8d81ef34 VP |
268 | mutex_lock(&rdev->stats.lock); |
269 | rdev->stats.stag.cur += 32; | |
270 | if (rdev->stats.stag.cur > rdev->stats.stag.max) | |
271 | rdev->stats.stag.max = rdev->stats.stag.cur; | |
272 | mutex_unlock(&rdev->stats.lock); | |
cfdda9d7 SW |
273 | *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff); |
274 | } | |
275 | PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n", | |
276 | __func__, stag_state, type, pdid, stag_idx); | |
277 | ||
278 | /* write TPT entry */ | |
279 | if (reset_tpt_entry) | |
280 | memset(&tpt, 0, sizeof(tpt)); | |
281 | else { | |
282 | tpt.valid_to_pdid = cpu_to_be32(F_FW_RI_TPTE_VALID | | |
283 | V_FW_RI_TPTE_STAGKEY((*stag & M_FW_RI_TPTE_STAGKEY)) | | |
284 | V_FW_RI_TPTE_STAGSTATE(stag_state) | | |
285 | V_FW_RI_TPTE_STAGTYPE(type) | V_FW_RI_TPTE_PDID(pdid)); | |
286 | tpt.locread_to_qpid = cpu_to_be32(V_FW_RI_TPTE_PERM(perm) | | |
287 | (bind_enabled ? F_FW_RI_TPTE_MWBINDEN : 0) | | |
288 | V_FW_RI_TPTE_ADDRTYPE((zbva ? FW_RI_ZERO_BASED_TO : | |
289 | FW_RI_VA_BASED_TO))| | |
290 | V_FW_RI_TPTE_PS(page_size)); | |
291 | tpt.nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32( | |
292 | V_FW_RI_TPTE_PBLADDR(PBL_OFF(rdev, pbl_addr)>>3)); | |
293 | tpt.len_lo = cpu_to_be32((u32)(len & 0xffffffffUL)); | |
294 | tpt.va_hi = cpu_to_be32((u32)(to >> 32)); | |
295 | tpt.va_lo_fbo = cpu_to_be32((u32)(to & 0xffffffffUL)); | |
296 | tpt.dca_mwbcnt_pstag = cpu_to_be32(0); | |
297 | tpt.len_hi = cpu_to_be32((u32)(len >> 32)); | |
298 | } | |
299 | err = write_adapter_mem(rdev, stag_idx + | |
300 | (rdev->lldi.vr->stag.start >> 5), | |
301 | sizeof(tpt), &tpt); | |
302 | ||
8d81ef34 | 303 | if (reset_tpt_entry) { |
ec3eead2 | 304 | c4iw_put_resource(&rdev->resource.tpt_table, stag_idx); |
8d81ef34 VP |
305 | mutex_lock(&rdev->stats.lock); |
306 | rdev->stats.stag.cur -= 32; | |
307 | mutex_unlock(&rdev->stats.lock); | |
308 | } | |
cfdda9d7 SW |
309 | return err; |
310 | } | |
311 | ||
312 | static int write_pbl(struct c4iw_rdev *rdev, __be64 *pbl, | |
313 | u32 pbl_addr, u32 pbl_size) | |
314 | { | |
315 | int err; | |
316 | ||
317 | PDBG("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n", | |
318 | __func__, pbl_addr, rdev->lldi.vr->pbl.start, | |
319 | pbl_size); | |
320 | ||
321 | err = write_adapter_mem(rdev, pbl_addr >> 5, pbl_size << 3, pbl); | |
322 | return err; | |
323 | } | |
324 | ||
325 | static int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size, | |
326 | u32 pbl_addr) | |
327 | { | |
328 | return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, | |
329 | pbl_size, pbl_addr); | |
330 | } | |
331 | ||
332 | static int allocate_window(struct c4iw_rdev *rdev, u32 * stag, u32 pdid) | |
333 | { | |
334 | *stag = T4_STAG_UNSET; | |
335 | return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_MW, 0, 0, 0, | |
336 | 0UL, 0, 0, 0, 0); | |
337 | } | |
338 | ||
339 | static int deallocate_window(struct c4iw_rdev *rdev, u32 stag) | |
340 | { | |
341 | return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0, | |
342 | 0); | |
343 | } | |
344 | ||
345 | static int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid, | |
346 | u32 pbl_size, u32 pbl_addr) | |
347 | { | |
348 | *stag = T4_STAG_UNSET; | |
349 | return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_NSMR, 0, 0, 0, | |
350 | 0UL, 0, 0, pbl_size, pbl_addr); | |
351 | } | |
352 | ||
353 | static int finish_mem_reg(struct c4iw_mr *mhp, u32 stag) | |
354 | { | |
355 | u32 mmid; | |
356 | ||
357 | mhp->attr.state = 1; | |
358 | mhp->attr.stag = stag; | |
359 | mmid = stag >> 8; | |
360 | mhp->ibmr.rkey = mhp->ibmr.lkey = stag; | |
361 | PDBG("%s mmid 0x%x mhp %p\n", __func__, mmid, mhp); | |
362 | return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid); | |
363 | } | |
364 | ||
365 | static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, | |
366 | struct c4iw_mr *mhp, int shift) | |
367 | { | |
368 | u32 stag = T4_STAG_UNSET; | |
369 | int ret; | |
370 | ||
371 | ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, | |
372 | FW_RI_STAG_NSMR, mhp->attr.perms, | |
373 | mhp->attr.mw_bind_enable, mhp->attr.zbva, | |
374 | mhp->attr.va_fbo, mhp->attr.len, shift - 12, | |
375 | mhp->attr.pbl_size, mhp->attr.pbl_addr); | |
376 | if (ret) | |
377 | return ret; | |
378 | ||
379 | ret = finish_mem_reg(mhp, stag); | |
380 | if (ret) | |
381 | dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, | |
382 | mhp->attr.pbl_addr); | |
383 | return ret; | |
384 | } | |
385 | ||
386 | static int reregister_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, | |
387 | struct c4iw_mr *mhp, int shift, int npages) | |
388 | { | |
389 | u32 stag; | |
390 | int ret; | |
391 | ||
392 | if (npages > mhp->attr.pbl_size) | |
393 | return -ENOMEM; | |
394 | ||
395 | stag = mhp->attr.stag; | |
396 | ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, | |
397 | FW_RI_STAG_NSMR, mhp->attr.perms, | |
398 | mhp->attr.mw_bind_enable, mhp->attr.zbva, | |
399 | mhp->attr.va_fbo, mhp->attr.len, shift - 12, | |
400 | mhp->attr.pbl_size, mhp->attr.pbl_addr); | |
401 | if (ret) | |
402 | return ret; | |
403 | ||
404 | ret = finish_mem_reg(mhp, stag); | |
405 | if (ret) | |
406 | dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, | |
407 | mhp->attr.pbl_addr); | |
408 | ||
409 | return ret; | |
410 | } | |
411 | ||
412 | static int alloc_pbl(struct c4iw_mr *mhp, int npages) | |
413 | { | |
414 | mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev, | |
415 | npages << 3); | |
416 | ||
417 | if (!mhp->attr.pbl_addr) | |
418 | return -ENOMEM; | |
419 | ||
420 | mhp->attr.pbl_size = npages; | |
421 | ||
422 | return 0; | |
423 | } | |
424 | ||
425 | static int build_phys_page_list(struct ib_phys_buf *buffer_list, | |
426 | int num_phys_buf, u64 *iova_start, | |
427 | u64 *total_size, int *npages, | |
428 | int *shift, __be64 **page_list) | |
429 | { | |
430 | u64 mask; | |
431 | int i, j, n; | |
432 | ||
433 | mask = 0; | |
434 | *total_size = 0; | |
435 | for (i = 0; i < num_phys_buf; ++i) { | |
436 | if (i != 0 && buffer_list[i].addr & ~PAGE_MASK) | |
437 | return -EINVAL; | |
438 | if (i != 0 && i != num_phys_buf - 1 && | |
439 | (buffer_list[i].size & ~PAGE_MASK)) | |
440 | return -EINVAL; | |
441 | *total_size += buffer_list[i].size; | |
442 | if (i > 0) | |
443 | mask |= buffer_list[i].addr; | |
444 | else | |
445 | mask |= buffer_list[i].addr & PAGE_MASK; | |
446 | if (i != num_phys_buf - 1) | |
447 | mask |= buffer_list[i].addr + buffer_list[i].size; | |
448 | else | |
449 | mask |= (buffer_list[i].addr + buffer_list[i].size + | |
450 | PAGE_SIZE - 1) & PAGE_MASK; | |
451 | } | |
452 | ||
453 | if (*total_size > 0xFFFFFFFFULL) | |
454 | return -ENOMEM; | |
455 | ||
456 | /* Find largest page shift we can use to cover buffers */ | |
457 | for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift)) | |
458 | if ((1ULL << *shift) & mask) | |
459 | break; | |
460 | ||
461 | buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1); | |
462 | buffer_list[0].addr &= ~0ull << *shift; | |
463 | ||
464 | *npages = 0; | |
465 | for (i = 0; i < num_phys_buf; ++i) | |
466 | *npages += (buffer_list[i].size + | |
467 | (1ULL << *shift) - 1) >> *shift; | |
468 | ||
469 | if (!*npages) | |
470 | return -EINVAL; | |
471 | ||
472 | *page_list = kmalloc(sizeof(u64) * *npages, GFP_KERNEL); | |
473 | if (!*page_list) | |
474 | return -ENOMEM; | |
475 | ||
476 | n = 0; | |
477 | for (i = 0; i < num_phys_buf; ++i) | |
478 | for (j = 0; | |
479 | j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift; | |
480 | ++j) | |
481 | (*page_list)[n++] = cpu_to_be64(buffer_list[i].addr + | |
482 | ((u64) j << *shift)); | |
483 | ||
484 | PDBG("%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d\n", | |
485 | __func__, (unsigned long long)*iova_start, | |
486 | (unsigned long long)mask, *shift, (unsigned long long)*total_size, | |
487 | *npages); | |
488 | ||
489 | return 0; | |
490 | ||
491 | } | |
492 | ||
493 | int c4iw_reregister_phys_mem(struct ib_mr *mr, int mr_rereg_mask, | |
494 | struct ib_pd *pd, struct ib_phys_buf *buffer_list, | |
495 | int num_phys_buf, int acc, u64 *iova_start) | |
496 | { | |
497 | ||
498 | struct c4iw_mr mh, *mhp; | |
499 | struct c4iw_pd *php; | |
500 | struct c4iw_dev *rhp; | |
501 | __be64 *page_list = NULL; | |
502 | int shift = 0; | |
503 | u64 total_size; | |
504 | int npages; | |
505 | int ret; | |
506 | ||
507 | PDBG("%s ib_mr %p ib_pd %p\n", __func__, mr, pd); | |
508 | ||
509 | /* There can be no memory windows */ | |
510 | if (atomic_read(&mr->usecnt)) | |
511 | return -EINVAL; | |
512 | ||
513 | mhp = to_c4iw_mr(mr); | |
514 | rhp = mhp->rhp; | |
515 | php = to_c4iw_pd(mr->pd); | |
516 | ||
517 | /* make sure we are on the same adapter */ | |
518 | if (rhp != php->rhp) | |
519 | return -EINVAL; | |
520 | ||
521 | memcpy(&mh, mhp, sizeof *mhp); | |
522 | ||
523 | if (mr_rereg_mask & IB_MR_REREG_PD) | |
524 | php = to_c4iw_pd(pd); | |
525 | if (mr_rereg_mask & IB_MR_REREG_ACCESS) { | |
526 | mh.attr.perms = c4iw_ib_to_tpt_access(acc); | |
527 | mh.attr.mw_bind_enable = (acc & IB_ACCESS_MW_BIND) == | |
528 | IB_ACCESS_MW_BIND; | |
529 | } | |
530 | if (mr_rereg_mask & IB_MR_REREG_TRANS) { | |
531 | ret = build_phys_page_list(buffer_list, num_phys_buf, | |
532 | iova_start, | |
533 | &total_size, &npages, | |
534 | &shift, &page_list); | |
535 | if (ret) | |
536 | return ret; | |
537 | } | |
538 | ||
539 | ret = reregister_mem(rhp, php, &mh, shift, npages); | |
540 | kfree(page_list); | |
541 | if (ret) | |
542 | return ret; | |
543 | if (mr_rereg_mask & IB_MR_REREG_PD) | |
544 | mhp->attr.pdid = php->pdid; | |
545 | if (mr_rereg_mask & IB_MR_REREG_ACCESS) | |
546 | mhp->attr.perms = c4iw_ib_to_tpt_access(acc); | |
547 | if (mr_rereg_mask & IB_MR_REREG_TRANS) { | |
548 | mhp->attr.zbva = 0; | |
549 | mhp->attr.va_fbo = *iova_start; | |
550 | mhp->attr.page_size = shift - 12; | |
551 | mhp->attr.len = (u32) total_size; | |
552 | mhp->attr.pbl_size = npages; | |
553 | } | |
554 | ||
555 | return 0; | |
556 | } | |
557 | ||
558 | struct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd, | |
559 | struct ib_phys_buf *buffer_list, | |
560 | int num_phys_buf, int acc, u64 *iova_start) | |
561 | { | |
562 | __be64 *page_list; | |
563 | int shift; | |
564 | u64 total_size; | |
565 | int npages; | |
566 | struct c4iw_dev *rhp; | |
567 | struct c4iw_pd *php; | |
568 | struct c4iw_mr *mhp; | |
569 | int ret; | |
570 | ||
571 | PDBG("%s ib_pd %p\n", __func__, pd); | |
572 | php = to_c4iw_pd(pd); | |
573 | rhp = php->rhp; | |
574 | ||
575 | mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); | |
576 | if (!mhp) | |
577 | return ERR_PTR(-ENOMEM); | |
578 | ||
579 | mhp->rhp = rhp; | |
580 | ||
581 | /* First check that we have enough alignment */ | |
582 | if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) { | |
583 | ret = -EINVAL; | |
584 | goto err; | |
585 | } | |
586 | ||
587 | if (num_phys_buf > 1 && | |
588 | ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) { | |
589 | ret = -EINVAL; | |
590 | goto err; | |
591 | } | |
592 | ||
593 | ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start, | |
594 | &total_size, &npages, &shift, | |
595 | &page_list); | |
596 | if (ret) | |
597 | goto err; | |
598 | ||
599 | ret = alloc_pbl(mhp, npages); | |
600 | if (ret) { | |
601 | kfree(page_list); | |
32c631f9 | 602 | goto err; |
cfdda9d7 SW |
603 | } |
604 | ||
605 | ret = write_pbl(&mhp->rhp->rdev, page_list, mhp->attr.pbl_addr, | |
606 | npages); | |
607 | kfree(page_list); | |
608 | if (ret) | |
609 | goto err_pbl; | |
610 | ||
611 | mhp->attr.pdid = php->pdid; | |
612 | mhp->attr.zbva = 0; | |
613 | ||
614 | mhp->attr.perms = c4iw_ib_to_tpt_access(acc); | |
615 | mhp->attr.va_fbo = *iova_start; | |
616 | mhp->attr.page_size = shift - 12; | |
617 | ||
618 | mhp->attr.len = (u32) total_size; | |
619 | mhp->attr.pbl_size = npages; | |
620 | ret = register_mem(rhp, php, mhp, shift); | |
621 | if (ret) | |
622 | goto err_pbl; | |
623 | ||
624 | return &mhp->ibmr; | |
625 | ||
626 | err_pbl: | |
627 | c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, | |
628 | mhp->attr.pbl_size << 3); | |
629 | ||
630 | err: | |
631 | kfree(mhp); | |
632 | return ERR_PTR(ret); | |
633 | ||
634 | } | |
635 | ||
636 | struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) | |
637 | { | |
638 | struct c4iw_dev *rhp; | |
639 | struct c4iw_pd *php; | |
640 | struct c4iw_mr *mhp; | |
641 | int ret; | |
642 | u32 stag = T4_STAG_UNSET; | |
643 | ||
644 | PDBG("%s ib_pd %p\n", __func__, pd); | |
645 | php = to_c4iw_pd(pd); | |
646 | rhp = php->rhp; | |
647 | ||
648 | mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); | |
649 | if (!mhp) | |
650 | return ERR_PTR(-ENOMEM); | |
651 | ||
652 | mhp->rhp = rhp; | |
653 | mhp->attr.pdid = php->pdid; | |
654 | mhp->attr.perms = c4iw_ib_to_tpt_access(acc); | |
655 | mhp->attr.mw_bind_enable = (acc&IB_ACCESS_MW_BIND) == IB_ACCESS_MW_BIND; | |
656 | mhp->attr.zbva = 0; | |
657 | mhp->attr.va_fbo = 0; | |
658 | mhp->attr.page_size = 0; | |
659 | mhp->attr.len = ~0UL; | |
660 | mhp->attr.pbl_size = 0; | |
661 | ||
662 | ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, php->pdid, | |
663 | FW_RI_STAG_NSMR, mhp->attr.perms, | |
664 | mhp->attr.mw_bind_enable, 0, 0, ~0UL, 0, 0, 0); | |
665 | if (ret) | |
666 | goto err1; | |
667 | ||
668 | ret = finish_mem_reg(mhp, stag); | |
669 | if (ret) | |
670 | goto err2; | |
671 | return &mhp->ibmr; | |
672 | err2: | |
673 | dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, | |
674 | mhp->attr.pbl_addr); | |
675 | err1: | |
676 | kfree(mhp); | |
677 | return ERR_PTR(ret); | |
678 | } | |
679 | ||
680 | struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, | |
681 | u64 virt, int acc, struct ib_udata *udata) | |
682 | { | |
683 | __be64 *pages; | |
684 | int shift, n, len; | |
eeb8461e | 685 | int i, k, entry; |
cfdda9d7 | 686 | int err = 0; |
eeb8461e | 687 | struct scatterlist *sg; |
cfdda9d7 SW |
688 | struct c4iw_dev *rhp; |
689 | struct c4iw_pd *php; | |
690 | struct c4iw_mr *mhp; | |
691 | ||
692 | PDBG("%s ib_pd %p\n", __func__, pd); | |
693 | ||
694 | if (length == ~0ULL) | |
695 | return ERR_PTR(-EINVAL); | |
696 | ||
697 | if ((length + start) < start) | |
698 | return ERR_PTR(-EINVAL); | |
699 | ||
700 | php = to_c4iw_pd(pd); | |
701 | rhp = php->rhp; | |
702 | mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); | |
703 | if (!mhp) | |
704 | return ERR_PTR(-ENOMEM); | |
705 | ||
706 | mhp->rhp = rhp; | |
707 | ||
708 | mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0); | |
709 | if (IS_ERR(mhp->umem)) { | |
710 | err = PTR_ERR(mhp->umem); | |
711 | kfree(mhp); | |
712 | return ERR_PTR(err); | |
713 | } | |
714 | ||
715 | shift = ffs(mhp->umem->page_size) - 1; | |
716 | ||
eeb8461e | 717 | n = mhp->umem->nmap; |
cfdda9d7 SW |
718 | err = alloc_pbl(mhp, n); |
719 | if (err) | |
720 | goto err; | |
721 | ||
722 | pages = (__be64 *) __get_free_page(GFP_KERNEL); | |
723 | if (!pages) { | |
724 | err = -ENOMEM; | |
725 | goto err_pbl; | |
726 | } | |
727 | ||
728 | i = n = 0; | |
729 | ||
eeb8461e YH |
730 | for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) { |
731 | len = sg_dma_len(sg) >> shift; | |
732 | for (k = 0; k < len; ++k) { | |
733 | pages[i++] = cpu_to_be64(sg_dma_address(sg) + | |
734 | mhp->umem->page_size * k); | |
735 | if (i == PAGE_SIZE / sizeof *pages) { | |
736 | err = write_pbl(&mhp->rhp->rdev, | |
737 | pages, | |
738 | mhp->attr.pbl_addr + (n << 3), i); | |
739 | if (err) | |
740 | goto pbl_done; | |
741 | n += i; | |
742 | i = 0; | |
cfdda9d7 SW |
743 | } |
744 | } | |
eeb8461e | 745 | } |
cfdda9d7 SW |
746 | |
747 | if (i) | |
748 | err = write_pbl(&mhp->rhp->rdev, pages, | |
749 | mhp->attr.pbl_addr + (n << 3), i); | |
750 | ||
751 | pbl_done: | |
752 | free_page((unsigned long) pages); | |
753 | if (err) | |
754 | goto err_pbl; | |
755 | ||
756 | mhp->attr.pdid = php->pdid; | |
757 | mhp->attr.zbva = 0; | |
758 | mhp->attr.perms = c4iw_ib_to_tpt_access(acc); | |
759 | mhp->attr.va_fbo = virt; | |
760 | mhp->attr.page_size = shift - 12; | |
301c2c3f | 761 | mhp->attr.len = length; |
cfdda9d7 SW |
762 | |
763 | err = register_mem(rhp, php, mhp, shift); | |
764 | if (err) | |
765 | goto err_pbl; | |
766 | ||
767 | return &mhp->ibmr; | |
768 | ||
769 | err_pbl: | |
770 | c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, | |
771 | mhp->attr.pbl_size << 3); | |
772 | ||
773 | err: | |
774 | ib_umem_release(mhp->umem); | |
775 | kfree(mhp); | |
776 | return ERR_PTR(err); | |
777 | } | |
778 | ||
7083e42e | 779 | struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) |
cfdda9d7 SW |
780 | { |
781 | struct c4iw_dev *rhp; | |
782 | struct c4iw_pd *php; | |
783 | struct c4iw_mw *mhp; | |
784 | u32 mmid; | |
785 | u32 stag = 0; | |
786 | int ret; | |
787 | ||
7083e42e SM |
788 | if (type != IB_MW_TYPE_1) |
789 | return ERR_PTR(-EINVAL); | |
790 | ||
cfdda9d7 SW |
791 | php = to_c4iw_pd(pd); |
792 | rhp = php->rhp; | |
793 | mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); | |
794 | if (!mhp) | |
795 | return ERR_PTR(-ENOMEM); | |
796 | ret = allocate_window(&rhp->rdev, &stag, php->pdid); | |
797 | if (ret) { | |
798 | kfree(mhp); | |
799 | return ERR_PTR(ret); | |
800 | } | |
801 | mhp->rhp = rhp; | |
802 | mhp->attr.pdid = php->pdid; | |
803 | mhp->attr.type = FW_RI_STAG_MW; | |
804 | mhp->attr.stag = stag; | |
805 | mmid = (stag) >> 8; | |
806 | mhp->ibmw.rkey = stag; | |
807 | if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { | |
808 | deallocate_window(&rhp->rdev, mhp->attr.stag); | |
809 | kfree(mhp); | |
810 | return ERR_PTR(-ENOMEM); | |
811 | } | |
812 | PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag); | |
813 | return &(mhp->ibmw); | |
814 | } | |
815 | ||
816 | int c4iw_dealloc_mw(struct ib_mw *mw) | |
817 | { | |
818 | struct c4iw_dev *rhp; | |
819 | struct c4iw_mw *mhp; | |
820 | u32 mmid; | |
821 | ||
822 | mhp = to_c4iw_mw(mw); | |
823 | rhp = mhp->rhp; | |
824 | mmid = (mw->rkey) >> 8; | |
cfdda9d7 | 825 | remove_handle(rhp, &rhp->mmidr, mmid); |
ec3eead2 | 826 | deallocate_window(&rhp->rdev, mhp->attr.stag); |
cfdda9d7 SW |
827 | kfree(mhp); |
828 | PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp); | |
829 | return 0; | |
830 | } | |
831 | ||
832 | struct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth) | |
833 | { | |
834 | struct c4iw_dev *rhp; | |
835 | struct c4iw_pd *php; | |
836 | struct c4iw_mr *mhp; | |
837 | u32 mmid; | |
838 | u32 stag = 0; | |
839 | int ret = 0; | |
840 | ||
841 | php = to_c4iw_pd(pd); | |
842 | rhp = php->rhp; | |
843 | mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); | |
841dba9a SW |
844 | if (!mhp) { |
845 | ret = -ENOMEM; | |
cfdda9d7 | 846 | goto err; |
841dba9a | 847 | } |
cfdda9d7 SW |
848 | |
849 | mhp->rhp = rhp; | |
850 | ret = alloc_pbl(mhp, pbl_depth); | |
851 | if (ret) | |
852 | goto err1; | |
853 | mhp->attr.pbl_size = pbl_depth; | |
854 | ret = allocate_stag(&rhp->rdev, &stag, php->pdid, | |
855 | mhp->attr.pbl_size, mhp->attr.pbl_addr); | |
856 | if (ret) | |
857 | goto err2; | |
858 | mhp->attr.pdid = php->pdid; | |
859 | mhp->attr.type = FW_RI_STAG_NSMR; | |
860 | mhp->attr.stag = stag; | |
861 | mhp->attr.state = 1; | |
862 | mmid = (stag) >> 8; | |
863 | mhp->ibmr.rkey = mhp->ibmr.lkey = stag; | |
841dba9a SW |
864 | if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { |
865 | ret = -ENOMEM; | |
cfdda9d7 | 866 | goto err3; |
841dba9a | 867 | } |
cfdda9d7 SW |
868 | |
869 | PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag); | |
870 | return &(mhp->ibmr); | |
871 | err3: | |
872 | dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size, | |
873 | mhp->attr.pbl_addr); | |
874 | err2: | |
875 | c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, | |
876 | mhp->attr.pbl_size << 3); | |
877 | err1: | |
878 | kfree(mhp); | |
879 | err: | |
880 | return ERR_PTR(ret); | |
881 | } | |
882 | ||
883 | struct ib_fast_reg_page_list *c4iw_alloc_fastreg_pbl(struct ib_device *device, | |
884 | int page_list_len) | |
885 | { | |
886 | struct c4iw_fr_page_list *c4pl; | |
887 | struct c4iw_dev *dev = to_c4iw_dev(device); | |
888 | dma_addr_t dma_addr; | |
42b6a949 | 889 | int pll_len = roundup(page_list_len * sizeof(u64), 32); |
cfdda9d7 | 890 | |
42b6a949 | 891 | c4pl = kmalloc(sizeof(*c4pl), GFP_KERNEL); |
cfdda9d7 SW |
892 | if (!c4pl) |
893 | return ERR_PTR(-ENOMEM); | |
894 | ||
42b6a949 VP |
895 | c4pl->ibpl.page_list = dma_alloc_coherent(&dev->rdev.lldi.pdev->dev, |
896 | pll_len, &dma_addr, | |
897 | GFP_KERNEL); | |
898 | if (!c4pl->ibpl.page_list) { | |
899 | kfree(c4pl); | |
900 | return ERR_PTR(-ENOMEM); | |
901 | } | |
f38926aa | 902 | dma_unmap_addr_set(c4pl, mapping, dma_addr); |
cfdda9d7 SW |
903 | c4pl->dma_addr = dma_addr; |
904 | c4pl->dev = dev; | |
eda6d1d1 SW |
905 | c4pl->pll_len = pll_len; |
906 | ||
907 | PDBG("%s c4pl %p pll_len %u page_list %p dma_addr %pad\n", | |
908 | __func__, c4pl, c4pl->pll_len, c4pl->ibpl.page_list, | |
909 | &c4pl->dma_addr); | |
cfdda9d7 SW |
910 | |
911 | return &c4pl->ibpl; | |
912 | } | |
913 | ||
914 | void c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list *ibpl) | |
915 | { | |
916 | struct c4iw_fr_page_list *c4pl = to_c4iw_fr_page_list(ibpl); | |
917 | ||
eda6d1d1 SW |
918 | PDBG("%s c4pl %p pll_len %u page_list %p dma_addr %pad\n", |
919 | __func__, c4pl, c4pl->pll_len, c4pl->ibpl.page_list, | |
920 | &c4pl->dma_addr); | |
921 | ||
42b6a949 | 922 | dma_free_coherent(&c4pl->dev->rdev.lldi.pdev->dev, |
eda6d1d1 | 923 | c4pl->pll_len, |
42b6a949 VP |
924 | c4pl->ibpl.page_list, dma_unmap_addr(c4pl, mapping)); |
925 | kfree(c4pl); | |
cfdda9d7 SW |
926 | } |
927 | ||
928 | int c4iw_dereg_mr(struct ib_mr *ib_mr) | |
929 | { | |
930 | struct c4iw_dev *rhp; | |
931 | struct c4iw_mr *mhp; | |
932 | u32 mmid; | |
933 | ||
934 | PDBG("%s ib_mr %p\n", __func__, ib_mr); | |
935 | /* There can be no memory windows */ | |
936 | if (atomic_read(&ib_mr->usecnt)) | |
937 | return -EINVAL; | |
938 | ||
939 | mhp = to_c4iw_mr(ib_mr); | |
940 | rhp = mhp->rhp; | |
941 | mmid = mhp->attr.stag >> 8; | |
ec3eead2 | 942 | remove_handle(rhp, &rhp->mmidr, mmid); |
cfdda9d7 SW |
943 | dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, |
944 | mhp->attr.pbl_addr); | |
945 | if (mhp->attr.pbl_size) | |
946 | c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, | |
947 | mhp->attr.pbl_size << 3); | |
cfdda9d7 SW |
948 | if (mhp->kva) |
949 | kfree((void *) (unsigned long) mhp->kva); | |
950 | if (mhp->umem) | |
951 | ib_umem_release(mhp->umem); | |
952 | PDBG("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp); | |
953 | kfree(mhp); | |
954 | return 0; | |
955 | } |