Commit | Line | Data |
---|---|---|
77241056 MM |
1 | /* |
2 | * | |
3 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
4 | * redistributing this file, you may do so under either license. | |
5 | * | |
6 | * GPL LICENSE SUMMARY | |
7 | * | |
8 | * Copyright(c) 2015 Intel Corporation. | |
9 | * | |
10 | * This program is free software; you can redistribute it and/or modify | |
11 | * it under the terms of version 2 of the GNU General Public License as | |
12 | * published by the Free Software Foundation. | |
13 | * | |
14 | * This program is distributed in the hope that it will be useful, but | |
15 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
17 | * General Public License for more details. | |
18 | * | |
19 | * BSD LICENSE | |
20 | * | |
21 | * Copyright(c) 2015 Intel Corporation. | |
22 | * | |
23 | * Redistribution and use in source and binary forms, with or without | |
24 | * modification, are permitted provided that the following conditions | |
25 | * are met: | |
26 | * | |
27 | * - Redistributions of source code must retain the above copyright | |
28 | * notice, this list of conditions and the following disclaimer. | |
29 | * - Redistributions in binary form must reproduce the above copyright | |
30 | * notice, this list of conditions and the following disclaimer in | |
31 | * the documentation and/or other materials provided with the | |
32 | * distribution. | |
33 | * - Neither the name of Intel Corporation nor the names of its | |
34 | * contributors may be used to endorse or promote products derived | |
35 | * from this software without specific prior written permission. | |
36 | * | |
37 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
38 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
39 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
40 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
41 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
42 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
43 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
44 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
45 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
46 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
47 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
48 | * | |
49 | */ | |
50 | ||
51 | #include <linux/err.h> | |
52 | #include <linux/slab.h> | |
53 | #include <linux/vmalloc.h> | |
54 | ||
55 | #include "verbs.h" | |
56 | ||
57 | /** | |
58 | * hfi1_post_srq_receive - post a receive on a shared receive queue | |
59 | * @ibsrq: the SRQ to post the receive on | |
60 | * @wr: the list of work requests to post | |
61 | * @bad_wr: A pointer to the first WR to cause a problem is put here | |
62 | * | |
63 | * This may be called from interrupt context. | |
64 | */ | |
65 | int hfi1_post_srq_receive(struct ib_srq *ibsrq, struct ib_recv_wr *wr, | |
66 | struct ib_recv_wr **bad_wr) | |
67 | { | |
68 | struct hfi1_srq *srq = to_isrq(ibsrq); | |
69 | struct hfi1_rwq *wq; | |
70 | unsigned long flags; | |
71 | int ret; | |
72 | ||
73 | for (; wr; wr = wr->next) { | |
74 | struct hfi1_rwqe *wqe; | |
75 | u32 next; | |
76 | int i; | |
77 | ||
78 | if ((unsigned) wr->num_sge > srq->rq.max_sge) { | |
79 | *bad_wr = wr; | |
80 | ret = -EINVAL; | |
81 | goto bail; | |
82 | } | |
83 | ||
84 | spin_lock_irqsave(&srq->rq.lock, flags); | |
85 | wq = srq->rq.wq; | |
86 | next = wq->head + 1; | |
87 | if (next >= srq->rq.size) | |
88 | next = 0; | |
89 | if (next == wq->tail) { | |
90 | spin_unlock_irqrestore(&srq->rq.lock, flags); | |
91 | *bad_wr = wr; | |
92 | ret = -ENOMEM; | |
93 | goto bail; | |
94 | } | |
95 | ||
96 | wqe = get_rwqe_ptr(&srq->rq, wq->head); | |
97 | wqe->wr_id = wr->wr_id; | |
98 | wqe->num_sge = wr->num_sge; | |
99 | for (i = 0; i < wr->num_sge; i++) | |
100 | wqe->sg_list[i] = wr->sg_list[i]; | |
101 | /* Make sure queue entry is written before the head index. */ | |
102 | smp_wmb(); | |
103 | wq->head = next; | |
104 | spin_unlock_irqrestore(&srq->rq.lock, flags); | |
105 | } | |
106 | ret = 0; | |
107 | ||
108 | bail: | |
109 | return ret; | |
110 | } | |
111 | ||
112 | /** | |
113 | * hfi1_create_srq - create a shared receive queue | |
114 | * @ibpd: the protection domain of the SRQ to create | |
115 | * @srq_init_attr: the attributes of the SRQ | |
116 | * @udata: data from libibverbs when creating a user SRQ | |
117 | */ | |
118 | struct ib_srq *hfi1_create_srq(struct ib_pd *ibpd, | |
119 | struct ib_srq_init_attr *srq_init_attr, | |
120 | struct ib_udata *udata) | |
121 | { | |
122 | struct hfi1_ibdev *dev = to_idev(ibpd->device); | |
123 | struct hfi1_srq *srq; | |
124 | u32 sz; | |
125 | struct ib_srq *ret; | |
126 | ||
127 | if (srq_init_attr->srq_type != IB_SRQT_BASIC) { | |
128 | ret = ERR_PTR(-ENOSYS); | |
129 | goto done; | |
130 | } | |
131 | ||
132 | if (srq_init_attr->attr.max_sge == 0 || | |
133 | srq_init_attr->attr.max_sge > hfi1_max_srq_sges || | |
134 | srq_init_attr->attr.max_wr == 0 || | |
135 | srq_init_attr->attr.max_wr > hfi1_max_srq_wrs) { | |
136 | ret = ERR_PTR(-EINVAL); | |
137 | goto done; | |
138 | } | |
139 | ||
140 | srq = kmalloc(sizeof(*srq), GFP_KERNEL); | |
141 | if (!srq) { | |
142 | ret = ERR_PTR(-ENOMEM); | |
143 | goto done; | |
144 | } | |
145 | ||
146 | /* | |
147 | * Need to use vmalloc() if we want to support large #s of entries. | |
148 | */ | |
149 | srq->rq.size = srq_init_attr->attr.max_wr + 1; | |
150 | srq->rq.max_sge = srq_init_attr->attr.max_sge; | |
151 | sz = sizeof(struct ib_sge) * srq->rq.max_sge + | |
152 | sizeof(struct hfi1_rwqe); | |
153 | srq->rq.wq = vmalloc_user(sizeof(struct hfi1_rwq) + srq->rq.size * sz); | |
154 | if (!srq->rq.wq) { | |
155 | ret = ERR_PTR(-ENOMEM); | |
156 | goto bail_srq; | |
157 | } | |
158 | ||
159 | /* | |
160 | * Return the address of the RWQ as the offset to mmap. | |
161 | * See hfi1_mmap() for details. | |
162 | */ | |
163 | if (udata && udata->outlen >= sizeof(__u64)) { | |
164 | int err; | |
165 | u32 s = sizeof(struct hfi1_rwq) + srq->rq.size * sz; | |
166 | ||
167 | srq->ip = | |
168 | hfi1_create_mmap_info(dev, s, ibpd->uobject->context, | |
169 | srq->rq.wq); | |
170 | if (!srq->ip) { | |
171 | ret = ERR_PTR(-ENOMEM); | |
172 | goto bail_wq; | |
173 | } | |
174 | ||
175 | err = ib_copy_to_udata(udata, &srq->ip->offset, | |
176 | sizeof(srq->ip->offset)); | |
177 | if (err) { | |
178 | ret = ERR_PTR(err); | |
179 | goto bail_ip; | |
180 | } | |
181 | } else | |
182 | srq->ip = NULL; | |
183 | ||
184 | /* | |
185 | * ib_create_srq() will initialize srq->ibsrq. | |
186 | */ | |
187 | spin_lock_init(&srq->rq.lock); | |
188 | srq->rq.wq->head = 0; | |
189 | srq->rq.wq->tail = 0; | |
190 | srq->limit = srq_init_attr->attr.srq_limit; | |
191 | ||
192 | spin_lock(&dev->n_srqs_lock); | |
193 | if (dev->n_srqs_allocated == hfi1_max_srqs) { | |
194 | spin_unlock(&dev->n_srqs_lock); | |
195 | ret = ERR_PTR(-ENOMEM); | |
196 | goto bail_ip; | |
197 | } | |
198 | ||
199 | dev->n_srqs_allocated++; | |
200 | spin_unlock(&dev->n_srqs_lock); | |
201 | ||
202 | if (srq->ip) { | |
203 | spin_lock_irq(&dev->pending_lock); | |
204 | list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps); | |
205 | spin_unlock_irq(&dev->pending_lock); | |
206 | } | |
207 | ||
208 | ret = &srq->ibsrq; | |
209 | goto done; | |
210 | ||
211 | bail_ip: | |
212 | kfree(srq->ip); | |
213 | bail_wq: | |
214 | vfree(srq->rq.wq); | |
215 | bail_srq: | |
216 | kfree(srq); | |
217 | done: | |
218 | return ret; | |
219 | } | |
220 | ||
221 | /** | |
222 | * hfi1_modify_srq - modify a shared receive queue | |
223 | * @ibsrq: the SRQ to modify | |
224 | * @attr: the new attributes of the SRQ | |
225 | * @attr_mask: indicates which attributes to modify | |
226 | * @udata: user data for libibverbs.so | |
227 | */ | |
228 | int hfi1_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, | |
229 | enum ib_srq_attr_mask attr_mask, | |
230 | struct ib_udata *udata) | |
231 | { | |
232 | struct hfi1_srq *srq = to_isrq(ibsrq); | |
233 | struct hfi1_rwq *wq; | |
234 | int ret = 0; | |
235 | ||
236 | if (attr_mask & IB_SRQ_MAX_WR) { | |
237 | struct hfi1_rwq *owq; | |
238 | struct hfi1_rwqe *p; | |
239 | u32 sz, size, n, head, tail; | |
240 | ||
241 | /* Check that the requested sizes are below the limits. */ | |
242 | if ((attr->max_wr > hfi1_max_srq_wrs) || | |
243 | ((attr_mask & IB_SRQ_LIMIT) ? | |
244 | attr->srq_limit : srq->limit) > attr->max_wr) { | |
245 | ret = -EINVAL; | |
246 | goto bail; | |
247 | } | |
248 | ||
249 | sz = sizeof(struct hfi1_rwqe) + | |
250 | srq->rq.max_sge * sizeof(struct ib_sge); | |
251 | size = attr->max_wr + 1; | |
252 | wq = vmalloc_user(sizeof(struct hfi1_rwq) + size * sz); | |
253 | if (!wq) { | |
254 | ret = -ENOMEM; | |
255 | goto bail; | |
256 | } | |
257 | ||
258 | /* Check that we can write the offset to mmap. */ | |
259 | if (udata && udata->inlen >= sizeof(__u64)) { | |
260 | __u64 offset_addr; | |
261 | __u64 offset = 0; | |
262 | ||
263 | ret = ib_copy_from_udata(&offset_addr, udata, | |
264 | sizeof(offset_addr)); | |
265 | if (ret) | |
266 | goto bail_free; | |
267 | udata->outbuf = | |
268 | (void __user *) (unsigned long) offset_addr; | |
269 | ret = ib_copy_to_udata(udata, &offset, | |
270 | sizeof(offset)); | |
271 | if (ret) | |
272 | goto bail_free; | |
273 | } | |
274 | ||
275 | spin_lock_irq(&srq->rq.lock); | |
276 | /* | |
277 | * validate head and tail pointer values and compute | |
278 | * the number of remaining WQEs. | |
279 | */ | |
280 | owq = srq->rq.wq; | |
281 | head = owq->head; | |
282 | tail = owq->tail; | |
283 | if (head >= srq->rq.size || tail >= srq->rq.size) { | |
284 | ret = -EINVAL; | |
285 | goto bail_unlock; | |
286 | } | |
287 | n = head; | |
288 | if (n < tail) | |
289 | n += srq->rq.size - tail; | |
290 | else | |
291 | n -= tail; | |
292 | if (size <= n) { | |
293 | ret = -EINVAL; | |
294 | goto bail_unlock; | |
295 | } | |
296 | n = 0; | |
297 | p = wq->wq; | |
298 | while (tail != head) { | |
299 | struct hfi1_rwqe *wqe; | |
300 | int i; | |
301 | ||
302 | wqe = get_rwqe_ptr(&srq->rq, tail); | |
303 | p->wr_id = wqe->wr_id; | |
304 | p->num_sge = wqe->num_sge; | |
305 | for (i = 0; i < wqe->num_sge; i++) | |
306 | p->sg_list[i] = wqe->sg_list[i]; | |
307 | n++; | |
308 | p = (struct hfi1_rwqe *)((char *)p + sz); | |
309 | if (++tail >= srq->rq.size) | |
310 | tail = 0; | |
311 | } | |
312 | srq->rq.wq = wq; | |
313 | srq->rq.size = size; | |
314 | wq->head = n; | |
315 | wq->tail = 0; | |
316 | if (attr_mask & IB_SRQ_LIMIT) | |
317 | srq->limit = attr->srq_limit; | |
318 | spin_unlock_irq(&srq->rq.lock); | |
319 | ||
320 | vfree(owq); | |
321 | ||
322 | if (srq->ip) { | |
323 | struct hfi1_mmap_info *ip = srq->ip; | |
324 | struct hfi1_ibdev *dev = to_idev(srq->ibsrq.device); | |
325 | u32 s = sizeof(struct hfi1_rwq) + size * sz; | |
326 | ||
327 | hfi1_update_mmap_info(dev, ip, s, wq); | |
328 | ||
329 | /* | |
330 | * Return the offset to mmap. | |
331 | * See hfi1_mmap() for details. | |
332 | */ | |
333 | if (udata && udata->inlen >= sizeof(__u64)) { | |
334 | ret = ib_copy_to_udata(udata, &ip->offset, | |
335 | sizeof(ip->offset)); | |
336 | if (ret) | |
337 | goto bail; | |
338 | } | |
339 | ||
340 | /* | |
341 | * Put user mapping info onto the pending list | |
342 | * unless it already is on the list. | |
343 | */ | |
344 | spin_lock_irq(&dev->pending_lock); | |
345 | if (list_empty(&ip->pending_mmaps)) | |
346 | list_add(&ip->pending_mmaps, | |
347 | &dev->pending_mmaps); | |
348 | spin_unlock_irq(&dev->pending_lock); | |
349 | } | |
350 | } else if (attr_mask & IB_SRQ_LIMIT) { | |
351 | spin_lock_irq(&srq->rq.lock); | |
352 | if (attr->srq_limit >= srq->rq.size) | |
353 | ret = -EINVAL; | |
354 | else | |
355 | srq->limit = attr->srq_limit; | |
356 | spin_unlock_irq(&srq->rq.lock); | |
357 | } | |
358 | goto bail; | |
359 | ||
360 | bail_unlock: | |
361 | spin_unlock_irq(&srq->rq.lock); | |
362 | bail_free: | |
363 | vfree(wq); | |
364 | bail: | |
365 | return ret; | |
366 | } | |
367 | ||
368 | int hfi1_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) | |
369 | { | |
370 | struct hfi1_srq *srq = to_isrq(ibsrq); | |
371 | ||
372 | attr->max_wr = srq->rq.size - 1; | |
373 | attr->max_sge = srq->rq.max_sge; | |
374 | attr->srq_limit = srq->limit; | |
375 | return 0; | |
376 | } | |
377 | ||
378 | /** | |
379 | * hfi1_destroy_srq - destroy a shared receive queue | |
380 | * @ibsrq: the SRQ to destroy | |
381 | */ | |
382 | int hfi1_destroy_srq(struct ib_srq *ibsrq) | |
383 | { | |
384 | struct hfi1_srq *srq = to_isrq(ibsrq); | |
385 | struct hfi1_ibdev *dev = to_idev(ibsrq->device); | |
386 | ||
387 | spin_lock(&dev->n_srqs_lock); | |
388 | dev->n_srqs_allocated--; | |
389 | spin_unlock(&dev->n_srqs_lock); | |
390 | if (srq->ip) | |
391 | kref_put(&srq->ip->ref, hfi1_release_mmap_info); | |
392 | else | |
393 | vfree(srq->rq.wq); | |
394 | kfree(srq); | |
395 | ||
396 | return 0; | |
397 | } |