Commit | Line | Data |
---|---|---|
e126ba97 | 1 | /* |
302bdf68 | 2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
e126ba97 EC |
3 | * |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | ||
adec640e | 33 | #include <linux/highmem.h> |
e126ba97 EC |
34 | #include <linux/kernel.h> |
35 | #include <linux/module.h> | |
fc50db98 | 36 | #include <linux/delay.h> |
e126ba97 EC |
37 | #include <linux/mlx5/driver.h> |
38 | #include <linux/mlx5/cmd.h> | |
39 | #include "mlx5_core.h" | |
40 | ||
41 | enum { | |
42 | MLX5_PAGES_CANT_GIVE = 0, | |
43 | MLX5_PAGES_GIVE = 1, | |
44 | MLX5_PAGES_TAKE = 2 | |
45 | }; | |
46 | ||
47 | struct mlx5_pages_req { | |
48 | struct mlx5_core_dev *dev; | |
f241e749 | 49 | u16 func_id; |
0a324f31 | 50 | s32 npages; |
e126ba97 EC |
51 | struct work_struct work; |
52 | }; | |
53 | ||
54 | struct fw_page { | |
bf0bf77f EC |
55 | struct rb_node rb_node; |
56 | u64 addr; | |
57 | struct page *page; | |
58 | u16 func_id; | |
59 | unsigned long bitmask; | |
60 | struct list_head list; | |
61 | unsigned free_count; | |
e126ba97 EC |
62 | }; |
63 | ||
dabed0e6 EC |
64 | enum { |
65 | MAX_RECLAIM_TIME_MSECS = 5000, | |
fc50db98 | 66 | MAX_RECLAIM_VFS_PAGES_TIME_MSECS = 2 * 1000 * 60, |
dabed0e6 EC |
67 | }; |
68 | ||
bf0bf77f EC |
69 | enum { |
70 | MLX5_MAX_RECLAIM_TIME_MILI = 5000, | |
05bdb2ab | 71 | MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE, |
bf0bf77f EC |
72 | }; |
73 | ||
e126ba97 EC |
74 | static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id) |
75 | { | |
76 | struct rb_root *root = &dev->priv.page_root; | |
77 | struct rb_node **new = &root->rb_node; | |
78 | struct rb_node *parent = NULL; | |
79 | struct fw_page *nfp; | |
80 | struct fw_page *tfp; | |
bf0bf77f | 81 | int i; |
e126ba97 EC |
82 | |
83 | while (*new) { | |
84 | parent = *new; | |
85 | tfp = rb_entry(parent, struct fw_page, rb_node); | |
86 | if (tfp->addr < addr) | |
87 | new = &parent->rb_left; | |
88 | else if (tfp->addr > addr) | |
89 | new = &parent->rb_right; | |
90 | else | |
91 | return -EEXIST; | |
92 | } | |
93 | ||
bf0bf77f | 94 | nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); |
e126ba97 EC |
95 | if (!nfp) |
96 | return -ENOMEM; | |
97 | ||
98 | nfp->addr = addr; | |
99 | nfp->page = page; | |
100 | nfp->func_id = func_id; | |
bf0bf77f EC |
101 | nfp->free_count = MLX5_NUM_4K_IN_PAGE; |
102 | for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++) | |
103 | set_bit(i, &nfp->bitmask); | |
e126ba97 EC |
104 | |
105 | rb_link_node(&nfp->rb_node, parent, new); | |
106 | rb_insert_color(&nfp->rb_node, root); | |
bf0bf77f | 107 | list_add(&nfp->list, &dev->priv.free_list); |
e126ba97 EC |
108 | |
109 | return 0; | |
110 | } | |
111 | ||
bf0bf77f | 112 | static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr) |
e126ba97 EC |
113 | { |
114 | struct rb_root *root = &dev->priv.page_root; | |
115 | struct rb_node *tmp = root->rb_node; | |
bf0bf77f | 116 | struct fw_page *result = NULL; |
e126ba97 EC |
117 | struct fw_page *tfp; |
118 | ||
119 | while (tmp) { | |
120 | tfp = rb_entry(tmp, struct fw_page, rb_node); | |
121 | if (tfp->addr < addr) { | |
122 | tmp = tmp->rb_left; | |
123 | } else if (tfp->addr > addr) { | |
124 | tmp = tmp->rb_right; | |
125 | } else { | |
bf0bf77f | 126 | result = tfp; |
e126ba97 EC |
127 | break; |
128 | } | |
129 | } | |
130 | ||
131 | return result; | |
132 | } | |
133 | ||
134 | static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, | |
0a324f31 | 135 | s32 *npages, int boot) |
e126ba97 | 136 | { |
a533ed5e SM |
137 | u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {0}; |
138 | u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {0}; | |
e126ba97 EC |
139 | int err; |
140 | ||
a533ed5e SM |
141 | MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); |
142 | MLX5_SET(query_pages_in, in, op_mod, boot ? | |
143 | MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES : | |
144 | MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES); | |
0a324f31 | 145 | |
a533ed5e | 146 | err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); |
e126ba97 EC |
147 | if (err) |
148 | return err; | |
149 | ||
a533ed5e SM |
150 | *npages = MLX5_GET(query_pages_out, out, num_pages); |
151 | *func_id = MLX5_GET(query_pages_out, out, function_id); | |
e126ba97 EC |
152 | |
153 | return err; | |
154 | } | |
155 | ||
bf0bf77f EC |
156 | static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr) |
157 | { | |
158 | struct fw_page *fp; | |
159 | unsigned n; | |
160 | ||
24e42754 | 161 | if (list_empty(&dev->priv.free_list)) |
bf0bf77f | 162 | return -ENOMEM; |
bf0bf77f EC |
163 | |
164 | fp = list_entry(dev->priv.free_list.next, struct fw_page, list); | |
165 | n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask)); | |
166 | if (n >= MLX5_NUM_4K_IN_PAGE) { | |
167 | mlx5_core_warn(dev, "alloc 4k bug\n"); | |
168 | return -ENOENT; | |
169 | } | |
170 | clear_bit(n, &fp->bitmask); | |
171 | fp->free_count--; | |
172 | if (!fp->free_count) | |
173 | list_del(&fp->list); | |
174 | ||
05bdb2ab | 175 | *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE; |
bf0bf77f EC |
176 | |
177 | return 0; | |
178 | } | |
179 | ||
59d2d18c HL |
180 | #define MLX5_U64_4K_PAGE_MASK ((~(u64)0U) << PAGE_SHIFT) |
181 | ||
bf0bf77f EC |
182 | static void free_4k(struct mlx5_core_dev *dev, u64 addr) |
183 | { | |
184 | struct fw_page *fwp; | |
185 | int n; | |
186 | ||
59d2d18c | 187 | fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK); |
bf0bf77f EC |
188 | if (!fwp) { |
189 | mlx5_core_warn(dev, "page not found\n"); | |
190 | return; | |
191 | } | |
192 | ||
59d2d18c | 193 | n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; |
bf0bf77f EC |
194 | fwp->free_count++; |
195 | set_bit(n, &fwp->bitmask); | |
196 | if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) { | |
197 | rb_erase(&fwp->rb_node, &dev->priv.page_root); | |
2b136d02 EC |
198 | if (fwp->free_count != 1) |
199 | list_del(&fwp->list); | |
59d2d18c HL |
200 | dma_unmap_page(&dev->pdev->dev, addr & MLX5_U64_4K_PAGE_MASK, |
201 | PAGE_SIZE, DMA_BIDIRECTIONAL); | |
bf0bf77f EC |
202 | __free_page(fwp->page); |
203 | kfree(fwp); | |
204 | } else if (fwp->free_count == 1) { | |
205 | list_add(&fwp->list, &dev->priv.free_list); | |
206 | } | |
207 | } | |
208 | ||
209 | static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) | |
210 | { | |
211 | struct page *page; | |
212 | u64 addr; | |
213 | int err; | |
ad189106 | 214 | int nid = dev_to_node(&dev->pdev->dev); |
bf0bf77f | 215 | |
ad189106 | 216 | page = alloc_pages_node(nid, GFP_HIGHUSER, 0); |
bf0bf77f EC |
217 | if (!page) { |
218 | mlx5_core_warn(dev, "failed to allocate page\n"); | |
219 | return -ENOMEM; | |
220 | } | |
221 | addr = dma_map_page(&dev->pdev->dev, page, 0, | |
222 | PAGE_SIZE, DMA_BIDIRECTIONAL); | |
223 | if (dma_mapping_error(&dev->pdev->dev, addr)) { | |
224 | mlx5_core_warn(dev, "failed dma mapping page\n"); | |
225 | err = -ENOMEM; | |
226 | goto out_alloc; | |
227 | } | |
228 | err = insert_page(dev, addr, page, func_id); | |
229 | if (err) { | |
230 | mlx5_core_err(dev, "failed to track allocated page\n"); | |
231 | goto out_mapping; | |
232 | } | |
233 | ||
234 | return 0; | |
235 | ||
236 | out_mapping: | |
237 | dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); | |
238 | ||
239 | out_alloc: | |
240 | __free_page(page); | |
241 | ||
242 | return err; | |
243 | } | |
a8ffe63e EC |
244 | |
245 | static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id) | |
246 | { | |
a533ed5e SM |
247 | u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; |
248 | u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; | |
a8ffe63e EC |
249 | int err; |
250 | ||
a533ed5e SM |
251 | MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); |
252 | MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_CANT_GIVE); | |
253 | MLX5_SET(manage_pages_in, in, function_id, func_id); | |
c4f287c4 | 254 | |
a533ed5e | 255 | err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); |
a8ffe63e | 256 | if (err) |
a533ed5e SM |
257 | mlx5_core_warn(dev, "page notify failed func_id(%d) err(%d)\n", |
258 | func_id, err); | |
a8ffe63e EC |
259 | } |
260 | ||
e126ba97 EC |
261 | static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, |
262 | int notify_fail) | |
263 | { | |
a533ed5e SM |
264 | u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; |
265 | int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); | |
e126ba97 EC |
266 | u64 addr; |
267 | int err; | |
a533ed5e | 268 | u32 *in; |
e126ba97 EC |
269 | int i; |
270 | ||
a533ed5e | 271 | inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]); |
e126ba97 EC |
272 | in = mlx5_vzalloc(inlen); |
273 | if (!in) { | |
a8ffe63e | 274 | err = -ENOMEM; |
e126ba97 | 275 | mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); |
a8ffe63e | 276 | goto out_free; |
e126ba97 | 277 | } |
e126ba97 EC |
278 | |
279 | for (i = 0; i < npages; i++) { | |
bf0bf77f EC |
280 | retry: |
281 | err = alloc_4k(dev, &addr); | |
e126ba97 | 282 | if (err) { |
bf0bf77f EC |
283 | if (err == -ENOMEM) |
284 | err = alloc_system_page(dev, func_id); | |
285 | if (err) | |
286 | goto out_4k; | |
287 | ||
288 | goto retry; | |
e126ba97 | 289 | } |
a533ed5e | 290 | MLX5_SET64(manage_pages_in, in, pas[i], addr); |
e126ba97 EC |
291 | } |
292 | ||
a533ed5e SM |
293 | MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); |
294 | MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE); | |
295 | MLX5_SET(manage_pages_in, in, function_id, func_id); | |
296 | MLX5_SET(manage_pages_in, in, input_num_entries, npages); | |
297 | ||
298 | err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); | |
e126ba97 | 299 | if (err) { |
1a91de28 JP |
300 | mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", |
301 | func_id, npages, err); | |
a8ffe63e | 302 | goto out_4k; |
e126ba97 | 303 | } |
e126ba97 | 304 | |
fc50db98 EC |
305 | dev->priv.fw_pages += npages; |
306 | if (func_id) | |
307 | dev->priv.vfs_pages += npages; | |
308 | ||
e126ba97 EC |
309 | mlx5_core_dbg(dev, "err %d\n", err); |
310 | ||
a8ffe63e EC |
311 | kvfree(in); |
312 | return 0; | |
952f5f6e | 313 | |
bf0bf77f EC |
314 | out_4k: |
315 | for (i--; i >= 0; i--) | |
a533ed5e | 316 | free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i])); |
e126ba97 | 317 | out_free: |
479163f4 | 318 | kvfree(in); |
a8ffe63e EC |
319 | if (notify_fail) |
320 | page_notify_fail(dev, func_id); | |
e126ba97 EC |
321 | return err; |
322 | } | |
323 | ||
5adff6a0 | 324 | static int reclaim_pages_cmd(struct mlx5_core_dev *dev, |
a533ed5e | 325 | u32 *in, int in_size, u32 *out, int out_size) |
5adff6a0 DJ |
326 | { |
327 | struct fw_page *fwp; | |
328 | struct rb_node *p; | |
329 | u32 npages; | |
330 | u32 i = 0; | |
331 | ||
c4f287c4 SM |
332 | if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) |
333 | return mlx5_cmd_exec(dev, in, in_size, out, out_size); | |
a533ed5e SM |
334 | |
335 | /* No hard feelings, we want our pages back! */ | |
336 | npages = MLX5_GET(manage_pages_in, in, input_num_entries); | |
5adff6a0 DJ |
337 | |
338 | p = rb_first(&dev->priv.page_root); | |
339 | while (p && i < npages) { | |
340 | fwp = rb_entry(p, struct fw_page, rb_node); | |
a533ed5e | 341 | MLX5_SET64(manage_pages_out, out, pas[i], fwp->addr); |
5adff6a0 DJ |
342 | p = rb_next(p); |
343 | i++; | |
344 | } | |
345 | ||
a533ed5e | 346 | MLX5_SET(manage_pages_out, out, output_num_entries, i); |
5adff6a0 DJ |
347 | return 0; |
348 | } | |
349 | ||
e126ba97 EC |
350 | static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, |
351 | int *nclaimed) | |
352 | { | |
a533ed5e SM |
353 | int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); |
354 | u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; | |
e126ba97 | 355 | int num_claimed; |
a533ed5e | 356 | u32 *out; |
e126ba97 EC |
357 | int err; |
358 | int i; | |
359 | ||
dabed0e6 EC |
360 | if (nclaimed) |
361 | *nclaimed = 0; | |
362 | ||
a533ed5e | 363 | outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); |
e126ba97 EC |
364 | out = mlx5_vzalloc(outlen); |
365 | if (!out) | |
366 | return -ENOMEM; | |
367 | ||
a533ed5e SM |
368 | MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); |
369 | MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE); | |
370 | MLX5_SET(manage_pages_in, in, function_id, func_id); | |
371 | MLX5_SET(manage_pages_in, in, input_num_entries, npages); | |
372 | ||
e126ba97 | 373 | mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); |
a533ed5e | 374 | err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen); |
e126ba97 | 375 | if (err) { |
5adff6a0 | 376 | mlx5_core_err(dev, "failed reclaiming pages: err %d\n", err); |
e126ba97 EC |
377 | goto out_free; |
378 | } | |
379 | ||
a533ed5e | 380 | num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); |
fc50db98 EC |
381 | if (num_claimed > npages) { |
382 | mlx5_core_warn(dev, "fw returned %d, driver asked %d => corruption\n", | |
383 | num_claimed, npages); | |
384 | err = -EINVAL; | |
385 | goto out_free; | |
386 | } | |
e126ba97 | 387 | |
a533ed5e SM |
388 | for (i = 0; i < num_claimed; i++) |
389 | free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i])); | |
390 | ||
5adff6a0 DJ |
391 | |
392 | if (nclaimed) | |
393 | *nclaimed = num_claimed; | |
394 | ||
fc50db98 EC |
395 | dev->priv.fw_pages -= num_claimed; |
396 | if (func_id) | |
397 | dev->priv.vfs_pages -= num_claimed; | |
e126ba97 EC |
398 | |
399 | out_free: | |
479163f4 | 400 | kvfree(out); |
e126ba97 EC |
401 | return err; |
402 | } | |
403 | ||
404 | static void pages_work_handler(struct work_struct *work) | |
405 | { | |
406 | struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); | |
407 | struct mlx5_core_dev *dev = req->dev; | |
408 | int err = 0; | |
409 | ||
410 | if (req->npages < 0) | |
411 | err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); | |
412 | else if (req->npages > 0) | |
413 | err = give_pages(dev, req->func_id, req->npages, 1); | |
414 | ||
415 | if (err) | |
1a91de28 JP |
416 | mlx5_core_warn(dev, "%s fail %d\n", |
417 | req->npages < 0 ? "reclaim" : "give", err); | |
e126ba97 EC |
418 | |
419 | kfree(req); | |
420 | } | |
421 | ||
422 | void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, | |
0a324f31 | 423 | s32 npages) |
e126ba97 EC |
424 | { |
425 | struct mlx5_pages_req *req; | |
426 | ||
427 | req = kzalloc(sizeof(*req), GFP_ATOMIC); | |
428 | if (!req) { | |
429 | mlx5_core_warn(dev, "failed to allocate pages request\n"); | |
430 | return; | |
431 | } | |
432 | ||
433 | req->dev = dev; | |
434 | req->func_id = func_id; | |
435 | req->npages = npages; | |
436 | INIT_WORK(&req->work, pages_work_handler); | |
437 | queue_work(dev->priv.pg_wq, &req->work); | |
438 | } | |
439 | ||
cd23b14b | 440 | int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) |
e126ba97 | 441 | { |
e126ba97 | 442 | u16 uninitialized_var(func_id); |
0a324f31 | 443 | s32 uninitialized_var(npages); |
e126ba97 EC |
444 | int err; |
445 | ||
0a324f31 | 446 | err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); |
e126ba97 EC |
447 | if (err) |
448 | return err; | |
449 | ||
0a324f31 ML |
450 | mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", |
451 | npages, boot ? "boot" : "init", func_id); | |
e126ba97 | 452 | |
0a324f31 | 453 | return give_pages(dev, func_id, npages, 0); |
e126ba97 EC |
454 | } |
455 | ||
4e3d677b ML |
456 | enum { |
457 | MLX5_BLKS_FOR_RECLAIM_PAGES = 12 | |
458 | }; | |
459 | ||
e126ba97 EC |
460 | static int optimal_reclaimed_pages(void) |
461 | { | |
462 | struct mlx5_cmd_prot_block *block; | |
463 | struct mlx5_cmd_layout *lay; | |
464 | int ret; | |
465 | ||
4e3d677b | 466 | ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - |
a533ed5e SM |
467 | MLX5_ST_SZ_BYTES(manage_pages_out)) / |
468 | MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); | |
e126ba97 EC |
469 | |
470 | return ret; | |
471 | } | |
472 | ||
473 | int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) | |
474 | { | |
dabed0e6 | 475 | unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); |
e126ba97 EC |
476 | struct fw_page *fwp; |
477 | struct rb_node *p; | |
dabed0e6 | 478 | int nclaimed = 0; |
89d44f0a | 479 | int err = 0; |
e126ba97 EC |
480 | |
481 | do { | |
482 | p = rb_first(&dev->priv.page_root); | |
483 | if (p) { | |
484 | fwp = rb_entry(p, struct fw_page, rb_node); | |
5adff6a0 DJ |
485 | err = reclaim_pages(dev, fwp->func_id, |
486 | optimal_reclaimed_pages(), | |
487 | &nclaimed); | |
488 | ||
e126ba97 | 489 | if (err) { |
1a91de28 JP |
490 | mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", |
491 | err); | |
e126ba97 EC |
492 | return err; |
493 | } | |
dabed0e6 EC |
494 | if (nclaimed) |
495 | end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); | |
e126ba97 EC |
496 | } |
497 | if (time_after(jiffies, end)) { | |
498 | mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); | |
499 | break; | |
500 | } | |
501 | } while (p); | |
502 | ||
5adff6a0 DJ |
503 | WARN(dev->priv.fw_pages, |
504 | "FW pages counter is %d after reclaiming all pages\n", | |
505 | dev->priv.fw_pages); | |
506 | WARN(dev->priv.vfs_pages, | |
507 | "VFs FW pages counter is %d after reclaiming all pages\n", | |
508 | dev->priv.vfs_pages); | |
509 | ||
e126ba97 EC |
510 | return 0; |
511 | } | |
512 | ||
513 | void mlx5_pagealloc_init(struct mlx5_core_dev *dev) | |
514 | { | |
515 | dev->priv.page_root = RB_ROOT; | |
bf0bf77f | 516 | INIT_LIST_HEAD(&dev->priv.free_list); |
e126ba97 EC |
517 | } |
518 | ||
519 | void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) | |
520 | { | |
521 | /* nothing */ | |
522 | } | |
523 | ||
524 | int mlx5_pagealloc_start(struct mlx5_core_dev *dev) | |
525 | { | |
526 | dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); | |
527 | if (!dev->priv.pg_wq) | |
528 | return -ENOMEM; | |
529 | ||
530 | return 0; | |
531 | } | |
532 | ||
533 | void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) | |
534 | { | |
535 | destroy_workqueue(dev->priv.pg_wq); | |
536 | } | |
fc50db98 EC |
537 | |
538 | int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev) | |
539 | { | |
540 | unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_VFS_PAGES_TIME_MSECS); | |
541 | int prev_vfs_pages = dev->priv.vfs_pages; | |
542 | ||
543 | mlx5_core_dbg(dev, "Waiting for %d pages from %s\n", prev_vfs_pages, | |
544 | dev->priv.name); | |
545 | while (dev->priv.vfs_pages) { | |
546 | if (time_after(jiffies, end)) { | |
547 | mlx5_core_warn(dev, "aborting while there are %d pending pages\n", dev->priv.vfs_pages); | |
548 | return -ETIMEDOUT; | |
549 | } | |
550 | if (dev->priv.vfs_pages < prev_vfs_pages) { | |
551 | end = jiffies + msecs_to_jiffies(MAX_RECLAIM_VFS_PAGES_TIME_MSECS); | |
552 | prev_vfs_pages = dev->priv.vfs_pages; | |
553 | } | |
554 | msleep(50); | |
555 | } | |
556 | ||
557 | mlx5_core_dbg(dev, "All pages received from %s\n", dev->priv.name); | |
558 | return 0; | |
559 | } |