Commit | Line | Data |
---|---|---|
cef1cce5 | 1 | /* |
87427da5 | 2 | * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved. |
cef1cce5 BS |
3 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | */ | |
33 | ||
5a0e3ad6 TH |
34 | #include <linux/slab.h> |
35 | ||
f7c6a7b5 | 36 | #include <rdma/ib_umem.h> |
cef1cce5 BS |
37 | #include <rdma/ib_pack.h> |
38 | #include <rdma/ib_smi.h> | |
39 | ||
40 | #include "ipath_verbs.h" | |
41 | ||
34b2aafe BS |
42 | /* Fast memory region */ |
43 | struct ipath_fmr { | |
44 | struct ib_fmr ibfmr; | |
45 | u8 page_shift; | |
46 | struct ipath_mregion mr; /* must be last */ | |
47 | }; | |
48 | ||
49 | static inline struct ipath_fmr *to_ifmr(struct ib_fmr *ibfmr) | |
50 | { | |
51 | return container_of(ibfmr, struct ipath_fmr, ibfmr); | |
52 | } | |
53 | ||
cef1cce5 BS |
54 | /** |
55 | * ipath_get_dma_mr - get a DMA memory region | |
56 | * @pd: protection domain for this memory region | |
57 | * @acc: access flags | |
58 | * | |
59 | * Returns the memory region on success, otherwise returns an errno. | |
f2cbb660 RC |
60 | * Note that all DMA addresses should be created via the |
61 | * struct ib_dma_mapping_ops functions (see ipath_dma.c). | |
cef1cce5 BS |
62 | */ |
63 | struct ib_mr *ipath_get_dma_mr(struct ib_pd *pd, int acc) | |
64 | { | |
65 | struct ipath_mr *mr; | |
66 | struct ib_mr *ret; | |
67 | ||
68 | mr = kzalloc(sizeof *mr, GFP_KERNEL); | |
69 | if (!mr) { | |
70 | ret = ERR_PTR(-ENOMEM); | |
71 | goto bail; | |
72 | } | |
73 | ||
74 | mr->mr.access_flags = acc; | |
75 | ret = &mr->ibmr; | |
76 | ||
77 | bail: | |
78 | return ret; | |
79 | } | |
80 | ||
81 | static struct ipath_mr *alloc_mr(int count, | |
82 | struct ipath_lkey_table *lk_table) | |
83 | { | |
84 | struct ipath_mr *mr; | |
85 | int m, i = 0; | |
86 | ||
87 | /* Allocate struct plus pointers to first level page tables. */ | |
88 | m = (count + IPATH_SEGSZ - 1) / IPATH_SEGSZ; | |
89 | mr = kmalloc(sizeof *mr + m * sizeof mr->mr.map[0], GFP_KERNEL); | |
90 | if (!mr) | |
91 | goto done; | |
92 | ||
93 | /* Allocate first level page tables. */ | |
94 | for (; i < m; i++) { | |
95 | mr->mr.map[i] = kmalloc(sizeof *mr->mr.map[0], GFP_KERNEL); | |
96 | if (!mr->mr.map[i]) | |
97 | goto bail; | |
98 | } | |
99 | mr->mr.mapsz = m; | |
100 | ||
cef1cce5 BS |
101 | if (!ipath_alloc_lkey(lk_table, &mr->mr)) |
102 | goto bail; | |
103 | mr->ibmr.rkey = mr->ibmr.lkey = mr->mr.lkey; | |
104 | ||
105 | goto done; | |
106 | ||
107 | bail: | |
108 | while (i) { | |
109 | i--; | |
110 | kfree(mr->mr.map[i]); | |
111 | } | |
112 | kfree(mr); | |
113 | mr = NULL; | |
114 | ||
115 | done: | |
116 | return mr; | |
117 | } | |
118 | ||
cef1cce5 BS |
119 | /** |
120 | * ipath_reg_user_mr - register a userspace memory region | |
121 | * @pd: protection domain for this memory region | |
f7c6a7b5 RD |
122 | * @start: starting userspace address |
123 | * @length: length of region to register | |
124 | * @virt_addr: virtual address to use (from HCA's point of view) | |
cef1cce5 BS |
125 | * @mr_access_flags: access flags for this memory region |
126 | * @udata: unused by the InfiniPath driver | |
127 | * | |
128 | * Returns the memory region on success, otherwise returns an errno. | |
129 | */ | |
f7c6a7b5 RD |
130 | struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, |
131 | u64 virt_addr, int mr_access_flags, | |
132 | struct ib_udata *udata) | |
cef1cce5 BS |
133 | { |
134 | struct ipath_mr *mr; | |
f7c6a7b5 | 135 | struct ib_umem *umem; |
eeb8461e YH |
136 | int n, m, entry; |
137 | struct scatterlist *sg; | |
cef1cce5 BS |
138 | struct ib_mr *ret; |
139 | ||
f7c6a7b5 | 140 | if (length == 0) { |
4a45b7d4 BS |
141 | ret = ERR_PTR(-EINVAL); |
142 | goto bail; | |
143 | } | |
144 | ||
cb9fbc5c AK |
145 | umem = ib_umem_get(pd->uobject->context, start, length, |
146 | mr_access_flags, 0); | |
f7c6a7b5 RD |
147 | if (IS_ERR(umem)) |
148 | return (void *) umem; | |
149 | ||
eeb8461e | 150 | n = umem->nmap; |
cef1cce5 BS |
151 | mr = alloc_mr(n, &to_idev(pd->device)->lk_table); |
152 | if (!mr) { | |
153 | ret = ERR_PTR(-ENOMEM); | |
f7c6a7b5 | 154 | ib_umem_release(umem); |
cef1cce5 BS |
155 | goto bail; |
156 | } | |
157 | ||
6a553af2 | 158 | mr->mr.pd = pd; |
f7c6a7b5 RD |
159 | mr->mr.user_base = start; |
160 | mr->mr.iova = virt_addr; | |
161 | mr->mr.length = length; | |
406f9e5f | 162 | mr->mr.offset = ib_umem_offset(umem); |
cef1cce5 BS |
163 | mr->mr.access_flags = mr_access_flags; |
164 | mr->mr.max_segs = n; | |
f7c6a7b5 | 165 | mr->umem = umem; |
cef1cce5 BS |
166 | |
167 | m = 0; | |
168 | n = 0; | |
eeb8461e YH |
169 | for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { |
170 | void *vaddr; | |
171 | ||
172 | vaddr = page_address(sg_page(sg)); | |
173 | if (!vaddr) { | |
174 | ret = ERR_PTR(-EINVAL); | |
175 | goto bail; | |
176 | } | |
177 | mr->mr.map[m]->segs[n].vaddr = vaddr; | |
178 | mr->mr.map[m]->segs[n].length = umem->page_size; | |
179 | n++; | |
180 | if (n == IPATH_SEGSZ) { | |
181 | m++; | |
182 | n = 0; | |
cef1cce5 BS |
183 | } |
184 | } | |
185 | ret = &mr->ibmr; | |
186 | ||
187 | bail: | |
188 | return ret; | |
189 | } | |
190 | ||
191 | /** | |
192 | * ipath_dereg_mr - unregister and free a memory region | |
193 | * @ibmr: the memory region to free | |
194 | * | |
195 | * Returns 0 on success. | |
196 | * | |
197 | * Note that this is called to free MRs created by ipath_get_dma_mr() | |
198 | * or ipath_reg_user_mr(). | |
199 | */ | |
200 | int ipath_dereg_mr(struct ib_mr *ibmr) | |
201 | { | |
202 | struct ipath_mr *mr = to_imr(ibmr); | |
203 | int i; | |
204 | ||
205 | ipath_free_lkey(&to_idev(ibmr->device)->lk_table, ibmr->lkey); | |
206 | i = mr->mr.mapsz; | |
207 | while (i) { | |
208 | i--; | |
209 | kfree(mr->mr.map[i]); | |
210 | } | |
f7c6a7b5 RD |
211 | |
212 | if (mr->umem) | |
213 | ib_umem_release(mr->umem); | |
214 | ||
cef1cce5 BS |
215 | kfree(mr); |
216 | return 0; | |
217 | } | |
218 | ||
219 | /** | |
220 | * ipath_alloc_fmr - allocate a fast memory region | |
221 | * @pd: the protection domain for this memory region | |
222 | * @mr_access_flags: access flags for this memory region | |
223 | * @fmr_attr: fast memory region attributes | |
224 | * | |
225 | * Returns the memory region on success, otherwise returns an errno. | |
226 | */ | |
227 | struct ib_fmr *ipath_alloc_fmr(struct ib_pd *pd, int mr_access_flags, | |
228 | struct ib_fmr_attr *fmr_attr) | |
229 | { | |
230 | struct ipath_fmr *fmr; | |
231 | int m, i = 0; | |
232 | struct ib_fmr *ret; | |
233 | ||
234 | /* Allocate struct plus pointers to first level page tables. */ | |
235 | m = (fmr_attr->max_pages + IPATH_SEGSZ - 1) / IPATH_SEGSZ; | |
236 | fmr = kmalloc(sizeof *fmr + m * sizeof fmr->mr.map[0], GFP_KERNEL); | |
237 | if (!fmr) | |
238 | goto bail; | |
239 | ||
240 | /* Allocate first level page tables. */ | |
241 | for (; i < m; i++) { | |
242 | fmr->mr.map[i] = kmalloc(sizeof *fmr->mr.map[0], | |
243 | GFP_KERNEL); | |
244 | if (!fmr->mr.map[i]) | |
245 | goto bail; | |
246 | } | |
247 | fmr->mr.mapsz = m; | |
248 | ||
249 | /* | |
250 | * ib_alloc_fmr() will initialize fmr->ibfmr except for lkey & | |
251 | * rkey. | |
252 | */ | |
253 | if (!ipath_alloc_lkey(&to_idev(pd->device)->lk_table, &fmr->mr)) | |
254 | goto bail; | |
255 | fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mr.lkey; | |
256 | /* | |
257 | * Resources are allocated but no valid mapping (RKEY can't be | |
258 | * used). | |
259 | */ | |
6a553af2 | 260 | fmr->mr.pd = pd; |
cef1cce5 BS |
261 | fmr->mr.user_base = 0; |
262 | fmr->mr.iova = 0; | |
263 | fmr->mr.length = 0; | |
264 | fmr->mr.offset = 0; | |
265 | fmr->mr.access_flags = mr_access_flags; | |
266 | fmr->mr.max_segs = fmr_attr->max_pages; | |
267 | fmr->page_shift = fmr_attr->page_shift; | |
268 | ||
269 | ret = &fmr->ibfmr; | |
270 | goto done; | |
271 | ||
272 | bail: | |
273 | while (i) | |
274 | kfree(fmr->mr.map[--i]); | |
275 | kfree(fmr); | |
276 | ret = ERR_PTR(-ENOMEM); | |
277 | ||
278 | done: | |
279 | return ret; | |
280 | } | |
281 | ||
282 | /** | |
283 | * ipath_map_phys_fmr - set up a fast memory region | |
284 | * @ibmfr: the fast memory region to set up | |
285 | * @page_list: the list of pages to associate with the fast memory region | |
286 | * @list_len: the number of pages to associate with the fast memory region | |
287 | * @iova: the virtual address of the start of the fast memory region | |
288 | * | |
289 | * This may be called from interrupt context. | |
290 | */ | |
291 | ||
292 | int ipath_map_phys_fmr(struct ib_fmr *ibfmr, u64 * page_list, | |
293 | int list_len, u64 iova) | |
294 | { | |
295 | struct ipath_fmr *fmr = to_ifmr(ibfmr); | |
296 | struct ipath_lkey_table *rkt; | |
297 | unsigned long flags; | |
298 | int m, n, i; | |
299 | u32 ps; | |
300 | int ret; | |
301 | ||
302 | if (list_len > fmr->mr.max_segs) { | |
303 | ret = -EINVAL; | |
304 | goto bail; | |
305 | } | |
306 | rkt = &to_idev(ibfmr->device)->lk_table; | |
307 | spin_lock_irqsave(&rkt->lock, flags); | |
308 | fmr->mr.user_base = iova; | |
309 | fmr->mr.iova = iova; | |
310 | ps = 1 << fmr->page_shift; | |
311 | fmr->mr.length = list_len * ps; | |
312 | m = 0; | |
313 | n = 0; | |
314 | ps = 1 << fmr->page_shift; | |
315 | for (i = 0; i < list_len; i++) { | |
f2cbb660 | 316 | fmr->mr.map[m]->segs[n].vaddr = (void *) page_list[i]; |
cef1cce5 BS |
317 | fmr->mr.map[m]->segs[n].length = ps; |
318 | if (++n == IPATH_SEGSZ) { | |
319 | m++; | |
320 | n = 0; | |
321 | } | |
322 | } | |
323 | spin_unlock_irqrestore(&rkt->lock, flags); | |
324 | ret = 0; | |
325 | ||
326 | bail: | |
327 | return ret; | |
328 | } | |
329 | ||
330 | /** | |
331 | * ipath_unmap_fmr - unmap fast memory regions | |
332 | * @fmr_list: the list of fast memory regions to unmap | |
333 | * | |
334 | * Returns 0 on success. | |
335 | */ | |
336 | int ipath_unmap_fmr(struct list_head *fmr_list) | |
337 | { | |
338 | struct ipath_fmr *fmr; | |
339 | struct ipath_lkey_table *rkt; | |
340 | unsigned long flags; | |
341 | ||
342 | list_for_each_entry(fmr, fmr_list, ibfmr.list) { | |
343 | rkt = &to_idev(fmr->ibfmr.device)->lk_table; | |
344 | spin_lock_irqsave(&rkt->lock, flags); | |
345 | fmr->mr.user_base = 0; | |
346 | fmr->mr.iova = 0; | |
347 | fmr->mr.length = 0; | |
348 | spin_unlock_irqrestore(&rkt->lock, flags); | |
349 | } | |
350 | return 0; | |
351 | } | |
352 | ||
353 | /** | |
354 | * ipath_dealloc_fmr - deallocate a fast memory region | |
355 | * @ibfmr: the fast memory region to deallocate | |
356 | * | |
357 | * Returns 0 on success. | |
358 | */ | |
359 | int ipath_dealloc_fmr(struct ib_fmr *ibfmr) | |
360 | { | |
361 | struct ipath_fmr *fmr = to_ifmr(ibfmr); | |
362 | int i; | |
363 | ||
364 | ipath_free_lkey(&to_idev(ibfmr->device)->lk_table, ibfmr->lkey); | |
365 | i = fmr->mr.mapsz; | |
366 | while (i) | |
367 | kfree(fmr->mr.map[--i]); | |
368 | kfree(fmr); | |
369 | return 0; | |
370 | } |