projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dmaengine: idxd: Add shared workqueue support
[linux-2.6-block.git] / drivers / dma / idxd / device.c
CommitLineData
bfe1d560
DJ		 1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/pci.h>
7#include <linux/io-64-nonatomic-lo-hi.h>
8f47d1a5 8#include <linux/dmaengine.h>
4548a6ad
DJ		 9#include <linux/irq.h>
10#include <linux/msi.h>
bfe1d560 11#include <uapi/linux/idxd.h>
8f47d1a5 12#include "../dmaengine.h"
bfe1d560
DJ		 13#include "idxd.h"
14#include "registers.h"
15
0d5c10b4
DJ		 16static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
17 u32 *status);
bfe1d560
DJ		 18
19/* Interrupt control bits */
4548a6ad 20void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)
bfe1d560 21{
4548a6ad 22 struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector);
bfe1d560 23
4548a6ad 24 pci_msi_mask_irq(data);
bfe1d560
DJ		 25}
26
27void idxd_mask_msix_vectors(struct idxd_device *idxd)
28{
29 struct pci_dev *pdev = idxd->pdev;
30 int msixcnt = pci_msix_vec_count(pdev);
4548a6ad 31 int i;
bfe1d560 32
4548a6ad
DJ		 33 for (i = 0; i < msixcnt; i++)
34 idxd_mask_msix_vector(idxd, i);
bfe1d560
DJ		 35}
36
4548a6ad 37void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id)
bfe1d560 38{
4548a6ad 39 struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector);
bfe1d560 40
4548a6ad 41 pci_msi_unmask_irq(data);
bfe1d560
DJ		 42}
43
44void idxd_unmask_error_interrupts(struct idxd_device *idxd)
45{
46 union genctrl_reg genctrl;
47
48 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
49 genctrl.softerr_int_en = 1;
50 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
51}
52
53void idxd_mask_error_interrupts(struct idxd_device *idxd)
54{
55 union genctrl_reg genctrl;
56
57 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
58 genctrl.softerr_int_en = 0;
59 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
60}
61
62static void free_hw_descs(struct idxd_wq *wq)
63{
64 int i;
65
66 for (i = 0; i < wq->num_descs; i++)
67 kfree(wq->hw_descs[i]);
68
69 kfree(wq->hw_descs);
70}
71
72static int alloc_hw_descs(struct idxd_wq *wq, int num)
73{
74 struct device *dev = &wq->idxd->pdev->dev;
75 int i;
76 int node = dev_to_node(dev);
77
78 wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),
79 GFP_KERNEL, node);
80 if (!wq->hw_descs)
81 return -ENOMEM;
82
83 for (i = 0; i < num; i++) {
84 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),
85 GFP_KERNEL, node);
86 if (!wq->hw_descs[i]) {
87 free_hw_descs(wq);
88 return -ENOMEM;
89 }
90 }
91
92 return 0;
93}
94
95static void free_descs(struct idxd_wq *wq)
96{
97 int i;
98
99 for (i = 0; i < wq->num_descs; i++)
100 kfree(wq->descs[i]);
101
102 kfree(wq->descs);
103}
104
105static int alloc_descs(struct idxd_wq *wq, int num)
106{
107 struct device *dev = &wq->idxd->pdev->dev;
108 int i;
109 int node = dev_to_node(dev);
110
111 wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),
112 GFP_KERNEL, node);
113 if (!wq->descs)
114 return -ENOMEM;
115
116 for (i = 0; i < num; i++) {
117 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),
118 GFP_KERNEL, node);
119 if (!wq->descs[i]) {
120 free_descs(wq);
121 return -ENOMEM;
122 }
123 }
124
125 return 0;
126}
127
128/* WQ control bits */
129int idxd_wq_alloc_resources(struct idxd_wq *wq)
130{
131 struct idxd_device *idxd = wq->idxd;
bfe1d560
DJ		 132 struct device *dev = &idxd->pdev->dev;
133 int rc, num_descs, i;
134
c52ca478
DJ		 135 if (wq->type != IDXD_WQT_KERNEL)
136 return 0;
137
0705107f
DJ		 138 wq->num_descs = wq->size;
139 num_descs = wq->size;
bfe1d560
DJ		 140
141 rc = alloc_hw_descs(wq, num_descs);
142 if (rc < 0)
143 return rc;
144
145 wq->compls_size = num_descs * sizeof(struct dsa_completion_record);
146 wq->compls = dma_alloc_coherent(dev, wq->compls_size,
147 &wq->compls_addr, GFP_KERNEL);
148 if (!wq->compls) {
149 rc = -ENOMEM;
150 goto fail_alloc_compls;
151 }
152
153 rc = alloc_descs(wq, num_descs);
154 if (rc < 0)
155 goto fail_alloc_descs;
156
0705107f
DJ		 157 rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL,
158 dev_to_node(dev));
bfe1d560
DJ		 159 if (rc < 0)
160 goto fail_sbitmap_init;
161
162 for (i = 0; i < num_descs; i++) {
163 struct idxd_desc *desc = wq->descs[i];
164
165 desc->hw = wq->hw_descs[i];
166 desc->completion = &wq->compls[i];
167 desc->compl_dma = wq->compls_addr +
168 sizeof(struct dsa_completion_record) * i;
169 desc->id = i;
170 desc->wq = wq;
0705107f 171 desc->cpu = -1;
8f47d1a5
DJ		 172 dma_async_tx_descriptor_init(&desc->txd, &wq->dma_chan);
173 desc->txd.tx_submit = idxd_dma_tx_submit;
bfe1d560
DJ		 174 }
175
176 return 0;
177
178 fail_sbitmap_init:
179 free_descs(wq);
180 fail_alloc_descs:
181 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
182 fail_alloc_compls:
183 free_hw_descs(wq);
184 return rc;
185}
186
187void idxd_wq_free_resources(struct idxd_wq *wq)
188{
189 struct device *dev = &wq->idxd->pdev->dev;
190
c52ca478
DJ		 191 if (wq->type != IDXD_WQT_KERNEL)
192 return;
193
bfe1d560
DJ		 194 free_hw_descs(wq);
195 free_descs(wq);
196 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
0705107f 197 sbitmap_queue_free(&wq->sbq);
bfe1d560
DJ		 198}
199
200int idxd_wq_enable(struct idxd_wq *wq)
201{
202 struct idxd_device *idxd = wq->idxd;
203 struct device *dev = &idxd->pdev->dev;
204 u32 status;
bfe1d560
DJ		 205
206 if (wq->state == IDXD_WQ_ENABLED) {
207 dev_dbg(dev, "WQ %d already enabled\n", wq->id);
208 return -ENXIO;
209 }
210
0d5c10b4 211 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);
bfe1d560
DJ		 212
213 if (status != IDXD_CMDSTS_SUCCESS &&
214 status != IDXD_CMDSTS_ERR_WQ_ENABLED) {
215 dev_dbg(dev, "WQ enable failed: %#x\n", status);
216 return -ENXIO;
217 }
218
219 wq->state = IDXD_WQ_ENABLED;
220 dev_dbg(dev, "WQ %d enabled\n", wq->id);
221 return 0;
222}
223
224int idxd_wq_disable(struct idxd_wq *wq)
225{
226 struct idxd_device *idxd = wq->idxd;
227 struct device *dev = &idxd->pdev->dev;
228 u32 status, operand;
bfe1d560 229
bfe1d560
DJ		 230 dev_dbg(dev, "Disabling WQ %d\n", wq->id);
231
232 if (wq->state != IDXD_WQ_ENABLED) {
233 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
234 return 0;
235 }
236
237 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
0d5c10b4 238 idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_WQ, operand, &status);
bfe1d560
DJ		 239
240 if (status != IDXD_CMDSTS_SUCCESS) {
241 dev_dbg(dev, "WQ disable failed: %#x\n", status);
242 return -ENXIO;
243 }
244
245 wq->state = IDXD_WQ_DISABLED;
246 dev_dbg(dev, "WQ %d disabled\n", wq->id);
247 return 0;
248}
249
0d5c10b4
DJ		 250void idxd_wq_drain(struct idxd_wq *wq)
251{
252 struct idxd_device *idxd = wq->idxd;
253 struct device *dev = &idxd->pdev->dev;
254 u32 operand;
255
256 if (wq->state != IDXD_WQ_ENABLED) {
257 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
258 return;
259 }
260
261 dev_dbg(dev, "Draining WQ %d\n", wq->id);
262 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
263 idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_WQ, operand, NULL);
264}
265
c52ca478
DJ		 266int idxd_wq_map_portal(struct idxd_wq *wq)
267{
268 struct idxd_device *idxd = wq->idxd;
269 struct pci_dev *pdev = idxd->pdev;
270 struct device *dev = &pdev->dev;
271 resource_size_t start;
272
273 start = pci_resource_start(pdev, IDXD_WQ_BAR);
274 start = start + wq->id * IDXD_PORTAL_SIZE;
275
8e50d392
DJ		 276 wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
277 if (!wq->portal)
c52ca478 278 return -ENOMEM;
c52ca478
DJ		 279
280 return 0;
281}
282
283void idxd_wq_unmap_portal(struct idxd_wq *wq)
284{
285 struct device *dev = &wq->idxd->pdev->dev;
286
8e50d392
DJ		 287 devm_iounmap(dev, wq->portal);
288}
289
290int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
291{
292 struct idxd_device *idxd = wq->idxd;
293 int rc;
294 union wqcfg wqcfg;
295 unsigned int offset;
296 unsigned long flags;
297
298 rc = idxd_wq_disable(wq);
299 if (rc < 0)
300 return rc;
301
302 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
303 spin_lock_irqsave(&idxd->dev_lock, flags);
304 wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
305 wqcfg.pasid_en = 1;
306 wqcfg.pasid = pasid;
307 iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
308 spin_unlock_irqrestore(&idxd->dev_lock, flags);
309
310 rc = idxd_wq_enable(wq);
311 if (rc < 0)
312 return rc;
313
314 return 0;
315}
316
317int idxd_wq_disable_pasid(struct idxd_wq *wq)
318{
319 struct idxd_device *idxd = wq->idxd;
320 int rc;
321 union wqcfg wqcfg;
322 unsigned int offset;
323 unsigned long flags;
324
325 rc = idxd_wq_disable(wq);
326 if (rc < 0)
327 return rc;
328
329 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
330 spin_lock_irqsave(&idxd->dev_lock, flags);
331 wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
332 wqcfg.pasid_en = 0;
333 wqcfg.pasid = 0;
334 iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
335 spin_unlock_irqrestore(&idxd->dev_lock, flags);
336
337 rc = idxd_wq_enable(wq);
338 if (rc < 0)
339 return rc;
340
341 return 0;
c52ca478
DJ		 342}
343
da32b28c
DJ		 344void idxd_wq_disable_cleanup(struct idxd_wq *wq)
345{
346 struct idxd_device *idxd = wq->idxd;
347 struct device *dev = &idxd->pdev->dev;
348 int i, wq_offset;
349
350 lockdep_assert_held(&idxd->dev_lock);
d98793b5 351 memset(wq->wqcfg, 0, idxd->wqcfg_size);
da32b28c
DJ		 352 wq->type = IDXD_WQT_NONE;
353 wq->size = 0;
354 wq->group = NULL;
355 wq->threshold = 0;
356 wq->priority = 0;
357 clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
358 memset(wq->name, 0, WQ_NAME_SIZE);
359
d98793b5
DJ		 360 for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
361 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
da32b28c
DJ		 362 iowrite32(0, idxd->reg_base + wq_offset);
363 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n",
364 wq->id, i, wq_offset,
365 ioread32(idxd->reg_base + wq_offset));
366 }
367}
368
bfe1d560
DJ		 369/* Device control bits */
370static inline bool idxd_is_enabled(struct idxd_device *idxd)
371{
372 union gensts_reg gensts;
373
374 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
375
376 if (gensts.state == IDXD_DEVICE_STATE_ENABLED)
377 return true;
378 return false;
379}
380
0d5c10b4
DJ		 381/*
382 * This is function is only used for reset during probe and will
383 * poll for completion. Once the device is setup with interrupts,
384 * all commands will be done via interrupt completion.
385 */
386void idxd_device_init_reset(struct idxd_device *idxd)
bfe1d560 387{
0d5c10b4
DJ		 388 struct device *dev = &idxd->pdev->dev;
389 union idxd_command_reg cmd;
390 unsigned long flags;
bfe1d560 391
0d5c10b4
DJ		 392 memset(&cmd, 0, sizeof(cmd));
393 cmd.cmd = IDXD_CMD_RESET_DEVICE;
394 dev_dbg(dev, "%s: sending reset for init.\n", __func__);
395 spin_lock_irqsave(&idxd->dev_lock, flags);
396 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
bfe1d560 397
0d5c10b4
DJ		 398 while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) &
399 IDXD_CMDSTS_ACTIVE)
400 cpu_relax();
401 spin_unlock_irqrestore(&idxd->dev_lock, flags);
bfe1d560
DJ		 402}
403
0d5c10b4
DJ		 404static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
405 u32 *status)
bfe1d560
DJ		 406{
407 union idxd_command_reg cmd;
0d5c10b4
DJ		 408 DECLARE_COMPLETION_ONSTACK(done);
409 unsigned long flags;
bfe1d560
DJ		 410
411 memset(&cmd, 0, sizeof(cmd));
412 cmd.cmd = cmd_code;
413 cmd.operand = operand;
0d5c10b4
DJ		 414 cmd.int_req = 1;
415
416 spin_lock_irqsave(&idxd->dev_lock, flags);
417 wait_event_lock_irq(idxd->cmd_waitq,
418 !test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags),
419 idxd->dev_lock);
420
bfe1d560
DJ		 421 dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",
422 __func__, cmd_code, operand);
0d5c10b4 423
ff18de55 424 idxd->cmd_status = 0;
0d5c10b4
DJ		 425 __set_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
426 idxd->cmd_done = &done;
bfe1d560
DJ		 427 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
428
0d5c10b4
DJ		 429 /*
430 * After command submitted, release lock and go to sleep until
431 * the command completes via interrupt.
432 */
433 spin_unlock_irqrestore(&idxd->dev_lock, flags);
434 wait_for_completion(&done);
435 spin_lock_irqsave(&idxd->dev_lock, flags);
ff18de55 436 if (status) {
0d5c10b4 437 *status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
ff18de55
DJ		 438 idxd->cmd_status = *status & GENMASK(7, 0);
439 }
440
0d5c10b4
DJ		 441 __clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
442 /* Wake up other pending commands */
443 wake_up(&idxd->cmd_waitq);
444 spin_unlock_irqrestore(&idxd->dev_lock, flags);
bfe1d560
DJ		 445}
446
447int idxd_device_enable(struct idxd_device *idxd)
448{
449 struct device *dev = &idxd->pdev->dev;
bfe1d560
DJ		 450 u32 status;
451
bfe1d560
DJ		 452 if (idxd_is_enabled(idxd)) {
453 dev_dbg(dev, "Device already enabled\n");
454 return -ENXIO;
455 }
456
0d5c10b4 457 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_DEVICE, 0, &status);
bfe1d560
DJ		 458
459 /* If the command is successful or if the device was enabled */
460 if (status != IDXD_CMDSTS_SUCCESS &&
461 status != IDXD_CMDSTS_ERR_DEV_ENABLED) {
462 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
463 return -ENXIO;
464 }
465
466 idxd->state = IDXD_DEV_ENABLED;
467 return 0;
468}
469
df841b17
DJ		 470void idxd_device_wqs_clear_state(struct idxd_device *idxd)
471{
472 int i;
473
474 lockdep_assert_held(&idxd->dev_lock);
475
476 for (i = 0; i < idxd->max_wqs; i++) {
477 struct idxd_wq *wq = &idxd->wqs[i];
478
479 if (wq->state == IDXD_WQ_ENABLED) {
480 idxd_wq_disable_cleanup(wq);
481 wq->state = IDXD_WQ_DISABLED;
482 }
483 }
484}
485
bfe1d560
DJ		 486int idxd_device_disable(struct idxd_device *idxd)
487{
488 struct device *dev = &idxd->pdev->dev;
bfe1d560 489 u32 status;
df841b17 490 unsigned long flags;
bfe1d560 491
bfe1d560
DJ		 492 if (!idxd_is_enabled(idxd)) {
493 dev_dbg(dev, "Device is not enabled\n");
494 return 0;
495 }
496
0d5c10b4 497 idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_DEVICE, 0, &status);
bfe1d560
DJ		 498
499 /* If the command is successful or if the device was disabled */
500 if (status != IDXD_CMDSTS_SUCCESS &&
501 !(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) {
502 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
0d5c10b4 503 return -ENXIO;
bfe1d560
DJ		 504 }
505
df841b17
DJ		 506 spin_lock_irqsave(&idxd->dev_lock, flags);
507 idxd_device_wqs_clear_state(idxd);
bfe1d560 508 idxd->state = IDXD_DEV_CONF_READY;
df841b17 509 spin_unlock_irqrestore(&idxd->dev_lock, flags);
bfe1d560
DJ		 510 return 0;
511}
512
0d5c10b4 513void idxd_device_reset(struct idxd_device *idxd)
bfe1d560 514{
df841b17
DJ		 515 unsigned long flags;
516
0d5c10b4 517 idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);
df841b17
DJ		 518 spin_lock_irqsave(&idxd->dev_lock, flags);
519 idxd_device_wqs_clear_state(idxd);
520 idxd->state = IDXD_DEV_CONF_READY;
521 spin_unlock_irqrestore(&idxd->dev_lock, flags);
bfe1d560
DJ		 522}
523
8e50d392
DJ		 524void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid)
525{
526 struct device *dev = &idxd->pdev->dev;
527 u32 operand;
528
529 operand = pasid;
530 dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_DRAIN_PASID, operand);
531 idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_PASID, operand, NULL);
532 dev_dbg(dev, "pasid %d drained\n", pasid);
533}
534
bfe1d560
DJ		 535/* Device configuration bits */
536static void idxd_group_config_write(struct idxd_group *group)
537{
538 struct idxd_device *idxd = group->idxd;
539 struct device *dev = &idxd->pdev->dev;
540 int i;
541 u32 grpcfg_offset;
542
543 dev_dbg(dev, "Writing group %d cfg registers\n", group->id);
544
545 /* setup GRPWQCFG */
546 for (i = 0; i < 4; i++) {
547 grpcfg_offset = idxd->grpcfg_offset +
548 group->id * 64 + i * sizeof(u64);
549 iowrite64(group->grpcfg.wqs[i],
550 idxd->reg_base + grpcfg_offset);
551 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
552 group->id, i, grpcfg_offset,
553 ioread64(idxd->reg_base + grpcfg_offset));
554 }
555
556 /* setup GRPENGCFG */
557 grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32;
558 iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);
559 dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
560 grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));
561
562 /* setup GRPFLAGS */
563 grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40;
564 iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
565 dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
566 group->id, grpcfg_offset,
567 ioread32(idxd->reg_base + grpcfg_offset));
568}
569
570static int idxd_groups_config_write(struct idxd_device *idxd)
571
572{
573 union gencfg_reg reg;
574 int i;
575 struct device *dev = &idxd->pdev->dev;
576
577 /* Setup bandwidth token limit */
578 if (idxd->token_limit) {
579 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
580 reg.token_limit = idxd->token_limit;
581 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
582 }
583
584 dev_dbg(dev, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET,
585 ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET));
586
587 for (i = 0; i < idxd->max_groups; i++) {
588 struct idxd_group *group = &idxd->groups[i];
589
590 idxd_group_config_write(group);
591 }
592
593 return 0;
594}
595
596static int idxd_wq_config_write(struct idxd_wq *wq)
597{
598 struct idxd_device *idxd = wq->idxd;
599 struct device *dev = &idxd->pdev->dev;
600 u32 wq_offset;
601 int i;
602
603 if (!wq->group)
604 return 0;
605
d98793b5 606 memset(wq->wqcfg, 0, idxd->wqcfg_size);
bfe1d560
DJ		 607
608 /* byte 0-3 */
d98793b5 609 wq->wqcfg->wq_size = wq->size;
bfe1d560
DJ		 610
611 if (wq->size == 0) {
612 dev_warn(dev, "Incorrect work queue size: 0\n");
613 return -EINVAL;
614 }
615
616 /* bytes 4-7 */
d98793b5 617 wq->wqcfg->wq_thresh = wq->threshold;
bfe1d560
DJ		 618
619 /* byte 8-11 */
d98793b5 620 wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL);
8e50d392
DJ		 621 if (wq_dedicated(wq))
622 wq->wqcfg->mode = 1;
623
624 if (device_pasid_enabled(idxd)) {
625 wq->wqcfg->pasid_en = 1;
626 if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq))
627 wq->wqcfg->pasid = idxd->pasid;
628 }
629
d98793b5 630 wq->wqcfg->priority = wq->priority;
bfe1d560 631
8e50d392
DJ		 632 if (idxd->hw.gen_cap.block_on_fault &&
633 test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags))
634 wq->wqcfg->bof = 1;
635
bfe1d560 636 /* bytes 12-15 */
d98793b5
DJ		 637 wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);
638 wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size);
bfe1d560
DJ		 639
640 dev_dbg(dev, "WQ %d CFGs\n", wq->id);
d98793b5
DJ		 641 for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
642 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
643 iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset);
bfe1d560
DJ		 644 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n",
645 wq->id, i, wq_offset,
646 ioread32(idxd->reg_base + wq_offset));
647 }
648
649 return 0;
650}
651
652static int idxd_wqs_config_write(struct idxd_device *idxd)
653{
654 int i, rc;
655
656 for (i = 0; i < idxd->max_wqs; i++) {
657 struct idxd_wq *wq = &idxd->wqs[i];
658
659 rc = idxd_wq_config_write(wq);
660 if (rc < 0)
661 return rc;
662 }
663
664 return 0;
665}
666
667static void idxd_group_flags_setup(struct idxd_device *idxd)
668{
669 int i;
670
671 /* TC-A 0 and TC-B 1 should be defaults */
672 for (i = 0; i < idxd->max_groups; i++) {
673 struct idxd_group *group = &idxd->groups[i];
674
675 if (group->tc_a == -1)
a1fcaf07 676 group->tc_a = group->grpcfg.flags.tc_a = 0;
bfe1d560
DJ		 677 else
678 group->grpcfg.flags.tc_a = group->tc_a;
679 if (group->tc_b == -1)
a1fcaf07 680 group->tc_b = group->grpcfg.flags.tc_b = 1;
bfe1d560
DJ		 681 else
682 group->grpcfg.flags.tc_b = group->tc_b;
683 group->grpcfg.flags.use_token_limit = group->use_token_limit;
684 group->grpcfg.flags.tokens_reserved = group->tokens_reserved;
685 if (group->tokens_allowed)
686 group->grpcfg.flags.tokens_allowed =
687 group->tokens_allowed;
688 else
689 group->grpcfg.flags.tokens_allowed = idxd->max_tokens;
690 }
691}
692
693static int idxd_engines_setup(struct idxd_device *idxd)
694{
695 int i, engines = 0;
696 struct idxd_engine *eng;
697 struct idxd_group *group;
698
699 for (i = 0; i < idxd->max_groups; i++) {
700 group = &idxd->groups[i];
701 group->grpcfg.engines = 0;
702 }
703
704 for (i = 0; i < idxd->max_engines; i++) {
705 eng = &idxd->engines[i];
706 group = eng->group;
707
708 if (!group)
709 continue;
710
711 group->grpcfg.engines |= BIT(eng->id);
712 engines++;
713 }
714
715 if (!engines)
716 return -EINVAL;
717
718 return 0;
719}
720
721static int idxd_wqs_setup(struct idxd_device *idxd)
722{
723 struct idxd_wq *wq;
724 struct idxd_group *group;
725 int i, j, configured = 0;
726 struct device *dev = &idxd->pdev->dev;
727
728 for (i = 0; i < idxd->max_groups; i++) {
729 group = &idxd->groups[i];
730 for (j = 0; j < 4; j++)
731 group->grpcfg.wqs[j] = 0;
732 }
733
734 for (i = 0; i < idxd->max_wqs; i++) {
735 wq = &idxd->wqs[i];
736 group = wq->group;
737
738 if (!wq->group)
739 continue;
740 if (!wq->size)
741 continue;
742
8e50d392
DJ		 743 if (wq_shared(wq) && !device_swq_supported(idxd)) {
744 dev_warn(dev, "No shared wq support but configured.\n");
bfe1d560
DJ		 745 return -EINVAL;
746 }
747
748 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);
749 configured++;
750 }
751
752 if (configured == 0)
753 return -EINVAL;
754
755 return 0;
756}
757
758int idxd_device_config(struct idxd_device *idxd)
759{
760 int rc;
761
762 lockdep_assert_held(&idxd->dev_lock);
763 rc = idxd_wqs_setup(idxd);
764 if (rc < 0)
765 return rc;
766
767 rc = idxd_engines_setup(idxd);
768 if (rc < 0)
769 return rc;
770
771 idxd_group_flags_setup(idxd);
772
773 rc = idxd_wqs_config_write(idxd);
774 if (rc < 0)
775 return rc;
776
777 rc = idxd_groups_config_write(idxd);
778 if (rc < 0)
779 return rc;
780
781 return 0;
782}
Linux 6.x block layer and io_uring tree(s)