projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dmaengine: idxd: Separate user and kernel pasid enabling
[linux-block.git] / drivers / dma / idxd / device.c
CommitLineData
bfe1d560
DJ		 1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/pci.h>
7#include <linux/io-64-nonatomic-lo-hi.h>
8f47d1a5 8#include <linux/dmaengine.h>
4548a6ad
DJ		 9#include <linux/irq.h>
10#include <linux/msi.h>
bfe1d560 11#include <uapi/linux/idxd.h>
8f47d1a5 12#include "../dmaengine.h"
bfe1d560
DJ		 13#include "idxd.h"
14#include "registers.h"
15
0d5c10b4
DJ		 16static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
17 u32 *status);
0dcfe41e
DJ		 18static void idxd_device_wqs_clear_state(struct idxd_device *idxd);
19static void idxd_wq_disable_cleanup(struct idxd_wq *wq);
bfe1d560
DJ		 20
21/* Interrupt control bits */
bfe1d560
DJ		 22void idxd_unmask_error_interrupts(struct idxd_device *idxd)
23{
24 union genctrl_reg genctrl;
25
26 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
27 genctrl.softerr_int_en = 1;
5b0c68c4 28 genctrl.halt_int_en = 1;
bfe1d560
DJ		 29 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
30}
31
32void idxd_mask_error_interrupts(struct idxd_device *idxd)
33{
34 union genctrl_reg genctrl;
35
36 genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
37 genctrl.softerr_int_en = 0;
5b0c68c4 38 genctrl.halt_int_en = 0;
bfe1d560
DJ		 39 iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
40}
41
42static void free_hw_descs(struct idxd_wq *wq)
43{
44 int i;
45
46 for (i = 0; i < wq->num_descs; i++)
47 kfree(wq->hw_descs[i]);
48
49 kfree(wq->hw_descs);
50}
51
52static int alloc_hw_descs(struct idxd_wq *wq, int num)
53{
54 struct device *dev = &wq->idxd->pdev->dev;
55 int i;
56 int node = dev_to_node(dev);
57
58 wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),
59 GFP_KERNEL, node);
60 if (!wq->hw_descs)
61 return -ENOMEM;
62
63 for (i = 0; i < num; i++) {
64 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),
65 GFP_KERNEL, node);
66 if (!wq->hw_descs[i]) {
67 free_hw_descs(wq);
68 return -ENOMEM;
69 }
70 }
71
72 return 0;
73}
74
75static void free_descs(struct idxd_wq *wq)
76{
77 int i;
78
79 for (i = 0; i < wq->num_descs; i++)
80 kfree(wq->descs[i]);
81
82 kfree(wq->descs);
83}
84
85static int alloc_descs(struct idxd_wq *wq, int num)
86{
87 struct device *dev = &wq->idxd->pdev->dev;
88 int i;
89 int node = dev_to_node(dev);
90
91 wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),
92 GFP_KERNEL, node);
93 if (!wq->descs)
94 return -ENOMEM;
95
96 for (i = 0; i < num; i++) {
97 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),
98 GFP_KERNEL, node);
99 if (!wq->descs[i]) {
100 free_descs(wq);
101 return -ENOMEM;
102 }
103 }
104
105 return 0;
106}
107
108/* WQ control bits */
109int idxd_wq_alloc_resources(struct idxd_wq *wq)
110{
111 struct idxd_device *idxd = wq->idxd;
bfe1d560
DJ		 112 struct device *dev = &idxd->pdev->dev;
113 int rc, num_descs, i;
114
c52ca478
DJ		 115 if (wq->type != IDXD_WQT_KERNEL)
116 return 0;
117
9806eb5c
DJ		 118 num_descs = wq_dedicated(wq) ? wq->size : wq->threshold;
119 wq->num_descs = num_descs;
bfe1d560
DJ		 120
121 rc = alloc_hw_descs(wq, num_descs);
122 if (rc < 0)
123 return rc;
124
2efe58cf
DJ		 125 wq->compls_size = num_descs * idxd->data->compl_size;
126 wq->compls = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr, GFP_KERNEL);
127 if (!wq->compls) {
bfe1d560
DJ		 128 rc = -ENOMEM;
129 goto fail_alloc_compls;
130 }
131
132 rc = alloc_descs(wq, num_descs);
133 if (rc < 0)
134 goto fail_alloc_descs;
135
0705107f
DJ		 136 rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL,
137 dev_to_node(dev));
bfe1d560
DJ		 138 if (rc < 0)
139 goto fail_sbitmap_init;
140
141 for (i = 0; i < num_descs; i++) {
142 struct idxd_desc *desc = wq->descs[i];
143
144 desc->hw = wq->hw_descs[i];
435b512d 145 if (idxd->data->type == IDXD_TYPE_DSA)
f25b4638 146 desc->completion = &wq->compls[i];
435b512d 147 else if (idxd->data->type == IDXD_TYPE_IAX)
f25b4638 148 desc->iax_completion = &wq->iax_compls[i];
435b512d 149 desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i;
bfe1d560
DJ		 150 desc->id = i;
151 desc->wq = wq;
0705107f 152 desc->cpu = -1;
bfe1d560
DJ		 153 }
154
155 return 0;
156
157 fail_sbitmap_init:
158 free_descs(wq);
159 fail_alloc_descs:
2efe58cf 160 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
bfe1d560
DJ		 161 fail_alloc_compls:
162 free_hw_descs(wq);
163 return rc;
164}
165
166void idxd_wq_free_resources(struct idxd_wq *wq)
167{
168 struct device *dev = &wq->idxd->pdev->dev;
169
c52ca478
DJ		 170 if (wq->type != IDXD_WQT_KERNEL)
171 return;
172
bfe1d560
DJ		 173 free_hw_descs(wq);
174 free_descs(wq);
2efe58cf 175 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
0705107f 176 sbitmap_queue_free(&wq->sbq);
bfe1d560
DJ		 177}
178
179int idxd_wq_enable(struct idxd_wq *wq)
180{
181 struct idxd_device *idxd = wq->idxd;
182 struct device *dev = &idxd->pdev->dev;
183 u32 status;
bfe1d560
DJ		 184
185 if (wq->state == IDXD_WQ_ENABLED) {
186 dev_dbg(dev, "WQ %d already enabled\n", wq->id);
187 return -ENXIO;
188 }
189
0d5c10b4 190 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);
bfe1d560
DJ		 191
192 if (status != IDXD_CMDSTS_SUCCESS &&
193 status != IDXD_CMDSTS_ERR_WQ_ENABLED) {
194 dev_dbg(dev, "WQ enable failed: %#x\n", status);
195 return -ENXIO;
196 }
197
198 wq->state = IDXD_WQ_ENABLED;
199 dev_dbg(dev, "WQ %d enabled\n", wq->id);
200 return 0;
201}
202
0dcfe41e 203int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)
bfe1d560
DJ		 204{
205 struct idxd_device *idxd = wq->idxd;
206 struct device *dev = &idxd->pdev->dev;
207 u32 status, operand;
bfe1d560 208
bfe1d560
DJ		 209 dev_dbg(dev, "Disabling WQ %d\n", wq->id);
210
211 if (wq->state != IDXD_WQ_ENABLED) {
212 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
213 return 0;
214 }
215
216 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
0d5c10b4 217 idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_WQ, operand, &status);
bfe1d560
DJ		 218
219 if (status != IDXD_CMDSTS_SUCCESS) {
220 dev_dbg(dev, "WQ disable failed: %#x\n", status);
221 return -ENXIO;
222 }
223
0dcfe41e
DJ		 224 if (reset_config)
225 idxd_wq_disable_cleanup(wq);
bfe1d560
DJ		 226 wq->state = IDXD_WQ_DISABLED;
227 dev_dbg(dev, "WQ %d disabled\n", wq->id);
228 return 0;
229}
230
0d5c10b4
DJ		 231void idxd_wq_drain(struct idxd_wq *wq)
232{
233 struct idxd_device *idxd = wq->idxd;
234 struct device *dev = &idxd->pdev->dev;
235 u32 operand;
236
237 if (wq->state != IDXD_WQ_ENABLED) {
238 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
239 return;
240 }
241
242 dev_dbg(dev, "Draining WQ %d\n", wq->id);
243 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
244 idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_WQ, operand, NULL);
245}
246
ea9aadc0
DJ		 247void idxd_wq_reset(struct idxd_wq *wq)
248{
249 struct idxd_device *idxd = wq->idxd;
250 struct device *dev = &idxd->pdev->dev;
251 u32 operand;
252
253 if (wq->state != IDXD_WQ_ENABLED) {
254 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
255 return;
256 }
257
258 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
259 idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL);
0dcfe41e 260 idxd_wq_disable_cleanup(wq);
ea9aadc0
DJ		 261 wq->state = IDXD_WQ_DISABLED;
262}
263
c52ca478
DJ		 264int idxd_wq_map_portal(struct idxd_wq *wq)
265{
266 struct idxd_device *idxd = wq->idxd;
267 struct pci_dev *pdev = idxd->pdev;
268 struct device *dev = &pdev->dev;
269 resource_size_t start;
270
271 start = pci_resource_start(pdev, IDXD_WQ_BAR);
8326be9f 272 start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED);
c52ca478 273
8e50d392
DJ		 274 wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
275 if (!wq->portal)
c52ca478 276 return -ENOMEM;
c52ca478
DJ		 277
278 return 0;
279}
280
281void idxd_wq_unmap_portal(struct idxd_wq *wq)
282{
283 struct device *dev = &wq->idxd->pdev->dev;
284
8e50d392 285 devm_iounmap(dev, wq->portal);
5b0c68c4 286 wq->portal = NULL;
a9c17152 287 wq->portal_offset = 0;
5b0c68c4
DJ		 288}
289
290void idxd_wqs_unmap_portal(struct idxd_device *idxd)
291{
292 int i;
293
294 for (i = 0; i < idxd->max_wqs; i++) {
295 struct idxd_wq *wq = idxd->wqs[i];
296
297 if (wq->portal)
298 idxd_wq_unmap_portal(wq);
299 }
8e50d392
DJ		 300}
301
3157dd0a 302static void __idxd_wq_set_priv_locked(struct idxd_wq *wq, int priv)
8e50d392
DJ		 303{
304 struct idxd_device *idxd = wq->idxd;
8e50d392
DJ		 305 union wqcfg wqcfg;
306 unsigned int offset;
8e50d392 307
3157dd0a
DJ		 308 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PRIVL_IDX);
309 spin_lock(&idxd->dev_lock);
310 wqcfg.bits[WQCFG_PRIVL_IDX] = ioread32(idxd->reg_base + offset);
311 wqcfg.priv = priv;
312 wq->wqcfg->bits[WQCFG_PRIVL_IDX] = wqcfg.bits[WQCFG_PRIVL_IDX];
313 iowrite32(wqcfg.bits[WQCFG_PRIVL_IDX], idxd->reg_base + offset);
314 spin_unlock(&idxd->dev_lock);
315}
316
317static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid)
318{
319 struct idxd_device *idxd = wq->idxd;
320 union wqcfg wqcfg;
321 unsigned int offset;
8e50d392
DJ		 322
323 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
cf84a4b9 324 spin_lock(&idxd->dev_lock);
8e50d392
DJ		 325 wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
326 wqcfg.pasid_en = 1;
327 wqcfg.pasid = pasid;
3157dd0a 328 wq->wqcfg->bits[WQCFG_PASID_IDX] = wqcfg.bits[WQCFG_PASID_IDX];
8e50d392 329 iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
cf84a4b9 330 spin_unlock(&idxd->dev_lock);
3157dd0a
DJ		 331}
332
333int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
334{
335 int rc;
336
337 rc = idxd_wq_disable(wq, false);
338 if (rc < 0)
339 return rc;
340
341 __idxd_wq_set_pasid_locked(wq, pasid);
8e50d392
DJ		 342
343 rc = idxd_wq_enable(wq);
344 if (rc < 0)
345 return rc;
346
347 return 0;
348}
349
350int idxd_wq_disable_pasid(struct idxd_wq *wq)
351{
352 struct idxd_device *idxd = wq->idxd;
353 int rc;
354 union wqcfg wqcfg;
355 unsigned int offset;
8e50d392 356
0dcfe41e 357 rc = idxd_wq_disable(wq, false);
8e50d392
DJ		 358 if (rc < 0)
359 return rc;
360
361 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
cf84a4b9 362 spin_lock(&idxd->dev_lock);
8e50d392
DJ		 363 wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
364 wqcfg.pasid_en = 0;
365 wqcfg.pasid = 0;
366 iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
cf84a4b9 367 spin_unlock(&idxd->dev_lock);
8e50d392
DJ		 368
369 rc = idxd_wq_enable(wq);
370 if (rc < 0)
371 return rc;
372
373 return 0;
c52ca478
DJ		 374}
375
0dcfe41e 376static void idxd_wq_disable_cleanup(struct idxd_wq *wq)
da32b28c
DJ		 377{
378 struct idxd_device *idxd = wq->idxd;
da32b28c 379
0dcfe41e 380 lockdep_assert_held(&wq->wq_lock);
484f910e 381 memset(wq->wqcfg, 0, idxd->wqcfg_size);
da32b28c 382 wq->type = IDXD_WQT_NONE;
da32b28c
DJ		 383 wq->threshold = 0;
384 wq->priority = 0;
92de5fa2 385 wq->ats_dis = 0;
7930d855 386 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;
da32b28c 387 clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
bd2f4ae5 388 clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
da32b28c 389 memset(wq->name, 0, WQ_NAME_SIZE);
92452a72
DJ		 390 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;
391 wq->max_batch_size = WQ_DEFAULT_MAX_BATCH;
da32b28c
DJ		 392}
393
0f225705
DJ		 394static void idxd_wq_device_reset_cleanup(struct idxd_wq *wq)
395{
396 lockdep_assert_held(&wq->wq_lock);
397
398 idxd_wq_disable_cleanup(wq);
399 wq->size = 0;
400 wq->group = NULL;
401}
402
93a40a6d
DJ		 403static void idxd_wq_ref_release(struct percpu_ref *ref)
404{
405 struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active);
406
407 complete(&wq->wq_dead);
408}
409
410int idxd_wq_init_percpu_ref(struct idxd_wq *wq)
411{
412 int rc;
413
414 memset(&wq->wq_active, 0, sizeof(wq->wq_active));
56fc39f5
DJ		 415 rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release,
416 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
93a40a6d
DJ		 417 if (rc < 0)
418 return rc;
419 reinit_completion(&wq->wq_dead);
56fc39f5 420 reinit_completion(&wq->wq_resurrect);
93a40a6d
DJ		 421 return 0;
422}
423
bd5970a0 424void __idxd_wq_quiesce(struct idxd_wq *wq)
93a40a6d 425{
bd5970a0 426 lockdep_assert_held(&wq->wq_lock);
56fc39f5 427 reinit_completion(&wq->wq_resurrect);
93a40a6d 428 percpu_ref_kill(&wq->wq_active);
56fc39f5 429 complete_all(&wq->wq_resurrect);
93a40a6d 430 wait_for_completion(&wq->wq_dead);
93a40a6d
DJ		 431}
432
bd5970a0
DJ		 433void idxd_wq_quiesce(struct idxd_wq *wq)
434{
435 mutex_lock(&wq->wq_lock);
436 __idxd_wq_quiesce(wq);
437 mutex_unlock(&wq->wq_lock);
438}
439
bfe1d560
DJ		 440/* Device control bits */
441static inline bool idxd_is_enabled(struct idxd_device *idxd)
442{
443 union gensts_reg gensts;
444
445 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
446
447 if (gensts.state == IDXD_DEVICE_STATE_ENABLED)
448 return true;
449 return false;
450}
451
89e3becd
DJ		 452static inline bool idxd_device_is_halted(struct idxd_device *idxd)
453{
454 union gensts_reg gensts;
455
456 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
457
458 return (gensts.state == IDXD_DEVICE_STATE_HALT);
459}
460
0d5c10b4
DJ		 461/*
462 * This is function is only used for reset during probe and will
463 * poll for completion. Once the device is setup with interrupts,
464 * all commands will be done via interrupt completion.
465 */
89e3becd 466int idxd_device_init_reset(struct idxd_device *idxd)
bfe1d560 467{
0d5c10b4
DJ		 468 struct device *dev = &idxd->pdev->dev;
469 union idxd_command_reg cmd;
bfe1d560 470
89e3becd
DJ		 471 if (idxd_device_is_halted(idxd)) {
472 dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
473 return -ENXIO;
474 }
475
0d5c10b4
DJ		 476 memset(&cmd, 0, sizeof(cmd));
477 cmd.cmd = IDXD_CMD_RESET_DEVICE;
478 dev_dbg(dev, "%s: sending reset for init.\n", __func__);
f9f4082d 479 spin_lock(&idxd->cmd_lock);
0d5c10b4 480 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
bfe1d560 481
0d5c10b4
DJ		 482 while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) &
483 IDXD_CMDSTS_ACTIVE)
484 cpu_relax();
f9f4082d 485 spin_unlock(&idxd->cmd_lock);
89e3becd 486 return 0;
bfe1d560
DJ		 487}
488
0d5c10b4
DJ		 489static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
490 u32 *status)
bfe1d560
DJ		 491{
492 union idxd_command_reg cmd;
0d5c10b4 493 DECLARE_COMPLETION_ONSTACK(done);
53499d1f 494 u32 stat;
bfe1d560 495
89e3becd
DJ		 496 if (idxd_device_is_halted(idxd)) {
497 dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
28ac8e03
CIK		 498 if (status)
499 *status = IDXD_CMDSTS_HW_ERR;
89e3becd
DJ		 500 return;
501 }
502
bfe1d560
DJ		 503 memset(&cmd, 0, sizeof(cmd));
504 cmd.cmd = cmd_code;
505 cmd.operand = operand;
0d5c10b4
DJ		 506 cmd.int_req = 1;
507
f9f4082d 508 spin_lock(&idxd->cmd_lock);
0d5c10b4
DJ		 509 wait_event_lock_irq(idxd->cmd_waitq,
510 !test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags),
53b2ee7f 511 idxd->cmd_lock);
0d5c10b4 512
bfe1d560
DJ		 513 dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",
514 __func__, cmd_code, operand);
0d5c10b4 515
ff18de55 516 idxd->cmd_status = 0;
0d5c10b4
DJ		 517 __set_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
518 idxd->cmd_done = &done;
bfe1d560
DJ		 519 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
520
0d5c10b4
DJ		 521 /*
522 * After command submitted, release lock and go to sleep until
523 * the command completes via interrupt.
524 */
f9f4082d 525 spin_unlock(&idxd->cmd_lock);
0d5c10b4 526 wait_for_completion(&done);
53499d1f 527 stat = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
f9f4082d 528 spin_lock(&idxd->cmd_lock);
53499d1f
DJ		 529 if (status)
530 *status = stat;
531 idxd->cmd_status = stat & GENMASK(7, 0);
ff18de55 532
0d5c10b4
DJ		 533 __clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
534 /* Wake up other pending commands */
535 wake_up(&idxd->cmd_waitq);
f9f4082d 536 spin_unlock(&idxd->cmd_lock);
bfe1d560
DJ		 537}
538
539int idxd_device_enable(struct idxd_device *idxd)
540{
541 struct device *dev = &idxd->pdev->dev;
bfe1d560
DJ		 542 u32 status;
543
bfe1d560
DJ		 544 if (idxd_is_enabled(idxd)) {
545 dev_dbg(dev, "Device already enabled\n");
546 return -ENXIO;
547 }
548
0d5c10b4 549 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_DEVICE, 0, &status);
bfe1d560
DJ		 550
551 /* If the command is successful or if the device was enabled */
552 if (status != IDXD_CMDSTS_SUCCESS &&
553 status != IDXD_CMDSTS_ERR_DEV_ENABLED) {
554 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
555 return -ENXIO;
556 }
557
558 idxd->state = IDXD_DEV_ENABLED;
559 return 0;
560}
561
562int idxd_device_disable(struct idxd_device *idxd)
563{
564 struct device *dev = &idxd->pdev->dev;
bfe1d560
DJ		 565 u32 status;
566
bfe1d560
DJ		 567 if (!idxd_is_enabled(idxd)) {
568 dev_dbg(dev, "Device is not enabled\n");
569 return 0;
570 }
571
0d5c10b4 572 idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_DEVICE, 0, &status);
bfe1d560
DJ		 573
574 /* If the command is successful or if the device was disabled */
575 if (status != IDXD_CMDSTS_SUCCESS &&
576 !(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) {
577 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
0d5c10b4 578 return -ENXIO;
bfe1d560
DJ		 579 }
580
cf84a4b9 581 spin_lock(&idxd->dev_lock);
0dcfe41e 582 idxd_device_clear_state(idxd);
f52058ae 583 idxd->state = IDXD_DEV_DISABLED;
cf84a4b9 584 spin_unlock(&idxd->dev_lock);
bfe1d560
DJ		 585 return 0;
586}
587
0d5c10b4 588void idxd_device_reset(struct idxd_device *idxd)
bfe1d560 589{
0d5c10b4 590 idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);
cf84a4b9 591 spin_lock(&idxd->dev_lock);
0dcfe41e 592 idxd_device_clear_state(idxd);
f52058ae 593 idxd->state = IDXD_DEV_DISABLED;
e530a9f3 594 idxd_unmask_error_interrupts(idxd);
cf84a4b9 595 spin_unlock(&idxd->dev_lock);
bfe1d560
DJ		 596}
597
8e50d392
DJ		 598void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid)
599{
600 struct device *dev = &idxd->pdev->dev;
601 u32 operand;
602
603 operand = pasid;
604 dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_DRAIN_PASID, operand);
605 idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_PASID, operand, NULL);
606 dev_dbg(dev, "pasid %d drained\n", pasid);
607}
608
eb15e715
DJ		 609int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
610 enum idxd_interrupt_type irq_type)
611{
612 struct device *dev = &idxd->pdev->dev;
613 u32 operand, status;
614
615 if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)))
616 return -EOPNOTSUPP;
617
618 dev_dbg(dev, "get int handle, idx %d\n", idx);
619
620 operand = idx & GENMASK(15, 0);
621 if (irq_type == IDXD_IRQ_IMS)
622 operand |= CMD_INT_HANDLE_IMS;
623
624 dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_REQUEST_INT_HANDLE, operand);
625
626 idxd_cmd_exec(idxd, IDXD_CMD_REQUEST_INT_HANDLE, operand, &status);
627
628 if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) {
629 dev_dbg(dev, "request int handle failed: %#x\n", status);
630 return -ENXIO;
631 }
632
633 *handle = (status >> IDXD_CMDSTS_RES_SHIFT) & GENMASK(15, 0);
634
635 dev_dbg(dev, "int handle acquired: %u\n", *handle);
636 return 0;
637}
638
639int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
640 enum idxd_interrupt_type irq_type)
641{
642 struct device *dev = &idxd->pdev->dev;
643 u32 operand, status;
644 union idxd_command_reg cmd;
eb15e715
DJ		 645
646 if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)))
647 return -EOPNOTSUPP;
648
649 dev_dbg(dev, "release int handle, handle %d\n", handle);
650
651 memset(&cmd, 0, sizeof(cmd));
652 operand = handle & GENMASK(15, 0);
653
654 if (irq_type == IDXD_IRQ_IMS)
655 operand |= CMD_INT_HANDLE_IMS;
656
657 cmd.cmd = IDXD_CMD_RELEASE_INT_HANDLE;
658 cmd.operand = operand;
659
660 dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE, operand);
661
f9f4082d 662 spin_lock(&idxd->cmd_lock);
eb15e715
DJ		 663 iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
664
665 while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE)
666 cpu_relax();
667 status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
f9f4082d 668 spin_unlock(&idxd->cmd_lock);
eb15e715
DJ		 669
670 if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) {
671 dev_dbg(dev, "release int handle failed: %#x\n", status);
672 return -ENXIO;
673 }
674
675 dev_dbg(dev, "int handle released.\n");
676 return 0;
677}
678
bfe1d560 679/* Device configuration bits */
0dcfe41e
DJ		 680static void idxd_engines_clear_state(struct idxd_device *idxd)
681{
682 struct idxd_engine *engine;
683 int i;
684
685 lockdep_assert_held(&idxd->dev_lock);
686 for (i = 0; i < idxd->max_engines; i++) {
687 engine = idxd->engines[i];
688 engine->group = NULL;
689 }
690}
691
692static void idxd_groups_clear_state(struct idxd_device *idxd)
693{
694 struct idxd_group *group;
695 int i;
696
697 lockdep_assert_held(&idxd->dev_lock);
698 for (i = 0; i < idxd->max_groups; i++) {
699 group = idxd->groups[i];
700 memset(&group->grpcfg, 0, sizeof(group->grpcfg));
701 group->num_engines = 0;
702 group->num_wqs = 0;
7ed6f1b8
DJ		 703 group->use_rdbuf_limit = false;
704 group->rdbufs_allowed = 0;
705 group->rdbufs_reserved = 0;
ea7c8f59
DJ		 706 if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) {
707 group->tc_a = 1;
708 group->tc_b = 1;
709 } else {
710 group->tc_a = -1;
711 group->tc_b = -1;
712 }
0dcfe41e
DJ		 713 }
714}
715
716static void idxd_device_wqs_clear_state(struct idxd_device *idxd)
717{
718 int i;
719
720 lockdep_assert_held(&idxd->dev_lock);
721 for (i = 0; i < idxd->max_wqs; i++) {
722 struct idxd_wq *wq = idxd->wqs[i];
723
724 if (wq->state == IDXD_WQ_ENABLED) {
725 idxd_wq_disable_cleanup(wq);
0f225705 726 idxd_wq_device_reset_cleanup(wq);
0dcfe41e
DJ		 727 wq->state = IDXD_WQ_DISABLED;
728 }
729 }
730}
731
732void idxd_device_clear_state(struct idxd_device *idxd)
733{
734 idxd_groups_clear_state(idxd);
735 idxd_engines_clear_state(idxd);
736 idxd_device_wqs_clear_state(idxd);
737}
738
bfe1d560
DJ		 739static void idxd_group_config_write(struct idxd_group *group)
740{
741 struct idxd_device *idxd = group->idxd;
742 struct device *dev = &idxd->pdev->dev;
743 int i;
744 u32 grpcfg_offset;
745
746 dev_dbg(dev, "Writing group %d cfg registers\n", group->id);
747
748 /* setup GRPWQCFG */
5a712701
DJ		 749 for (i = 0; i < GRPWQCFG_STRIDES; i++) {
750 grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i);
751 iowrite64(group->grpcfg.wqs[i], idxd->reg_base + grpcfg_offset);
bfe1d560
DJ		 752 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
753 group->id, i, grpcfg_offset,
754 ioread64(idxd->reg_base + grpcfg_offset));
755 }
756
757 /* setup GRPENGCFG */
5a712701 758 grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id);
bfe1d560
DJ		 759 iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);
760 dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
761 grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));
762
763 /* setup GRPFLAGS */
5a712701 764 grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id);
bfe1d560
DJ		 765 iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
766 dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
767 group->id, grpcfg_offset,
768 ioread32(idxd->reg_base + grpcfg_offset));
769}
770
771static int idxd_groups_config_write(struct idxd_device *idxd)
772
773{
774 union gencfg_reg reg;
775 int i;
776 struct device *dev = &idxd->pdev->dev;
777
7ed6f1b8
DJ		 778 /* Setup bandwidth rdbuf limit */
779 if (idxd->hw.gen_cap.config_en && idxd->rdbuf_limit) {
bfe1d560 780 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
7ed6f1b8 781 reg.rdbuf_limit = idxd->rdbuf_limit;
bfe1d560
DJ		 782 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
783 }
784
785 dev_dbg(dev, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET,
786 ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET));
787
788 for (i = 0; i < idxd->max_groups; i++) {
defe49f9 789 struct idxd_group *group = idxd->groups[i];
bfe1d560
DJ		 790
791 idxd_group_config_write(group);
792 }
793
794 return 0;
795}
796
d8071323
DJ		 797static bool idxd_device_pasid_priv_enabled(struct idxd_device *idxd)
798{
799 struct pci_dev *pdev = idxd->pdev;
800
801 if (pdev->pasid_enabled && (pdev->pasid_features & PCI_PASID_CAP_PRIV))
802 return true;
803 return false;
804}
805
bfe1d560
DJ		 806static int idxd_wq_config_write(struct idxd_wq *wq)
807{
808 struct idxd_device *idxd = wq->idxd;
809 struct device *dev = &idxd->pdev->dev;
810 u32 wq_offset;
811 int i;
812
813 if (!wq->group)
814 return 0;
815
ea9aadc0
DJ		 816 /*
817 * Instead of memset the entire shadow copy of WQCFG, copy from the hardware after
818 * wq reset. This will copy back the sticky values that are present on some devices.
819 */
820 for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
821 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
3157dd0a 822 wq->wqcfg->bits[i] |= ioread32(idxd->reg_base + wq_offset);
ea9aadc0 823 }
bfe1d560 824
92452a72
DJ		 825 if (wq->size == 0 && wq->type != IDXD_WQT_NONE)
826 wq->size = WQ_DEFAULT_QUEUE_DEPTH;
827
bfe1d560 828 /* byte 0-3 */
484f910e 829 wq->wqcfg->wq_size = wq->size;
bfe1d560 830
bfe1d560 831 /* bytes 4-7 */
484f910e 832 wq->wqcfg->wq_thresh = wq->threshold;
bfe1d560
DJ		 833
834 /* byte 8-11 */
8e50d392
DJ		 835 if (wq_dedicated(wq))
836 wq->wqcfg->mode = 1;
837
d8071323 838 /*
3157dd0a 839 * The WQ priv bit is set depending on the WQ type. priv = 1 if the
d8071323
DJ		 840 * WQ type is kernel to indicate privileged access. This setting only
841 * matters for dedicated WQ. According to the DSA spec:
842 * If the WQ is in dedicated mode, WQ PASID Enable is 1, and the
843 * Privileged Mode Enable field of the PCI Express PASID capability
844 * is 0, this field must be 0.
845 *
846 * In the case of a dedicated kernel WQ that is not able to support
847 * the PASID cap, then the configuration will be rejected.
848 */
d8071323
DJ		 849 if (wq_dedicated(wq) && wq->wqcfg->pasid_en &&
850 !idxd_device_pasid_priv_enabled(idxd) &&
851 wq->type == IDXD_WQT_KERNEL) {
852 idxd->cmd_status = IDXD_SCMD_WQ_NO_PRIV;
853 return -EOPNOTSUPP;
854 }
855
484f910e 856 wq->wqcfg->priority = wq->priority;
bfe1d560 857
8e50d392
DJ		 858 if (idxd->hw.gen_cap.block_on_fault &&
859 test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags))
860 wq->wqcfg->bof = 1;
861
92de5fa2
DJ		 862 if (idxd->hw.wq_cap.wq_ats_support)
863 wq->wqcfg->wq_ats_disable = wq->ats_dis;
864
bfe1d560 865 /* bytes 12-15 */
484f910e
DJ		 866 wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);
867 wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size);
bfe1d560
DJ		 868
869 dev_dbg(dev, "WQ %d CFGs\n", wq->id);
484f910e
DJ		 870 for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
871 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
872 iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset);
bfe1d560
DJ		 873 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n",
874 wq->id, i, wq_offset,
875 ioread32(idxd->reg_base + wq_offset));
876 }
877
878 return 0;
879}
880
881static int idxd_wqs_config_write(struct idxd_device *idxd)
882{
883 int i, rc;
884
885 for (i = 0; i < idxd->max_wqs; i++) {
7c5dd23e 886 struct idxd_wq *wq = idxd->wqs[i];
bfe1d560
DJ		 887
888 rc = idxd_wq_config_write(wq);
889 if (rc < 0)
890 return rc;
891 }
892
893 return 0;
894}
895
896static void idxd_group_flags_setup(struct idxd_device *idxd)
897{
898 int i;
899
900 /* TC-A 0 and TC-B 1 should be defaults */
901 for (i = 0; i < idxd->max_groups; i++) {
defe49f9 902 struct idxd_group *group = idxd->groups[i];
bfe1d560
DJ		 903
904 if (group->tc_a == -1)
a1fcaf07 905 group->tc_a = group->grpcfg.flags.tc_a = 0;
bfe1d560
DJ		 906 else
907 group->grpcfg.flags.tc_a = group->tc_a;
908 if (group->tc_b == -1)
a1fcaf07 909 group->tc_b = group->grpcfg.flags.tc_b = 1;
bfe1d560
DJ		 910 else
911 group->grpcfg.flags.tc_b = group->tc_b;
7ed6f1b8
DJ		 912 group->grpcfg.flags.use_rdbuf_limit = group->use_rdbuf_limit;
913 group->grpcfg.flags.rdbufs_reserved = group->rdbufs_reserved;
914 if (group->rdbufs_allowed)
915 group->grpcfg.flags.rdbufs_allowed = group->rdbufs_allowed;
bfe1d560 916 else
7ed6f1b8 917 group->grpcfg.flags.rdbufs_allowed = idxd->max_rdbufs;
bfe1d560
DJ		 918 }
919}
920
921static int idxd_engines_setup(struct idxd_device *idxd)
922{
923 int i, engines = 0;
924 struct idxd_engine *eng;
925 struct idxd_group *group;
926
927 for (i = 0; i < idxd->max_groups; i++) {
defe49f9 928 group = idxd->groups[i];
bfe1d560
DJ		 929 group->grpcfg.engines = 0;
930 }
931
932 for (i = 0; i < idxd->max_engines; i++) {
75b91130 933 eng = idxd->engines[i];
bfe1d560
DJ		 934 group = eng->group;
935
936 if (!group)
937 continue;
938
939 group->grpcfg.engines |= BIT(eng->id);
940 engines++;
941 }
942
943 if (!engines)
944 return -EINVAL;
945
946 return 0;
947}
948
949static int idxd_wqs_setup(struct idxd_device *idxd)
950{
951 struct idxd_wq *wq;
952 struct idxd_group *group;
953 int i, j, configured = 0;
954 struct device *dev = &idxd->pdev->dev;
955
956 for (i = 0; i < idxd->max_groups; i++) {
defe49f9 957 group = idxd->groups[i];
bfe1d560
DJ		 958 for (j = 0; j < 4; j++)
959 group->grpcfg.wqs[j] = 0;
960 }
961
962 for (i = 0; i < idxd->max_wqs; i++) {
7c5dd23e 963 wq = idxd->wqs[i];
bfe1d560
DJ		 964 group = wq->group;
965
966 if (!wq->group)
967 continue;
bfe1d560 968
42a1b738 969 if (wq_shared(wq) && !wq_shared_supported(wq)) {
125d1037 970 idxd->cmd_status = IDXD_SCMD_WQ_NO_SWQ_SUPPORT;
8e50d392 971 dev_warn(dev, "No shared wq support but configured.\n");
bfe1d560
DJ		 972 return -EINVAL;
973 }
974
975 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);
976 configured++;
977 }
978
125d1037
DJ		 979 if (configured == 0) {
980 idxd->cmd_status = IDXD_SCMD_WQ_NONE_CONFIGURED;
bfe1d560 981 return -EINVAL;
125d1037 982 }
bfe1d560
DJ		 983
984 return 0;
985}
986
987int idxd_device_config(struct idxd_device *idxd)
988{
989 int rc;
990
991 lockdep_assert_held(&idxd->dev_lock);
992 rc = idxd_wqs_setup(idxd);
993 if (rc < 0)
994 return rc;
995
996 rc = idxd_engines_setup(idxd);
997 if (rc < 0)
998 return rc;
999
1000 idxd_group_flags_setup(idxd);
1001
1002 rc = idxd_wqs_config_write(idxd);
1003 if (rc < 0)
1004 return rc;
1005
1006 rc = idxd_groups_config_write(idxd);
1007 if (rc < 0)
1008 return rc;
1009
1010 return 0;
1011}
8c66bbdc
DJ		 1012
1013static int idxd_wq_load_config(struct idxd_wq *wq)
1014{
1015 struct idxd_device *idxd = wq->idxd;
1016 struct device *dev = &idxd->pdev->dev;
1017 int wqcfg_offset;
1018 int i;
1019
1020 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0);
1021 memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size);
1022
1023 wq->size = wq->wqcfg->wq_size;
1024 wq->threshold = wq->wqcfg->wq_thresh;
8c66bbdc
DJ		 1025
1026 /* The driver does not support shared WQ mode in read-only config yet */
1027 if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en)
1028 return -EOPNOTSUPP;
1029
1030 set_bit(WQ_FLAG_DEDICATED, &wq->flags);
1031
1032 wq->priority = wq->wqcfg->priority;
1033
23084545
DJ		 1034 wq->max_xfer_bytes = 1ULL << wq->wqcfg->max_xfer_shift;
1035 wq->max_batch_size = 1ULL << wq->wqcfg->max_batch_shift;
1036
8c66bbdc
DJ		 1037 for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
1038 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i);
1039 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]);
1040 }
1041
1042 return 0;
1043}
1044
1045static void idxd_group_load_config(struct idxd_group *group)
1046{
1047 struct idxd_device *idxd = group->idxd;
1048 struct device *dev = &idxd->pdev->dev;
1049 int i, j, grpcfg_offset;
1050
1051 /*
1052 * Load WQS bit fields
1053 * Iterate through all 256 bits 64 bits at a time
1054 */
1055 for (i = 0; i < GRPWQCFG_STRIDES; i++) {
1056 struct idxd_wq *wq;
1057
1058 grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i);
1059 group->grpcfg.wqs[i] = ioread64(idxd->reg_base + grpcfg_offset);
1060 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
1061 group->id, i, grpcfg_offset, group->grpcfg.wqs[i]);
1062
1063 if (i * 64 >= idxd->max_wqs)
1064 break;
1065
1066 /* Iterate through all 64 bits and check for wq set */
1067 for (j = 0; j < 64; j++) {
1068 int id = i * 64 + j;
1069
1070 /* No need to check beyond max wqs */
1071 if (id >= idxd->max_wqs)
1072 break;
1073
1074 /* Set group assignment for wq if wq bit is set */
1075 if (group->grpcfg.wqs[i] & BIT(j)) {
1076 wq = idxd->wqs[id];
1077 wq->group = group;
1078 }
1079 }
1080 }
1081
1082 grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id);
1083 group->grpcfg.engines = ioread64(idxd->reg_base + grpcfg_offset);
1084 dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
1085 grpcfg_offset, group->grpcfg.engines);
1086
1087 /* Iterate through all 64 bits to check engines set */
1088 for (i = 0; i < 64; i++) {
1089 if (i >= idxd->max_engines)
1090 break;
1091
1092 if (group->grpcfg.engines & BIT(i)) {
1093 struct idxd_engine *engine = idxd->engines[i];
1094
1095 engine->group = group;
1096 }
1097 }
1098
1099 grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id);
1100 group->grpcfg.flags.bits = ioread32(idxd->reg_base + grpcfg_offset);
1101 dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
1102 group->id, grpcfg_offset, group->grpcfg.flags.bits);
1103}
1104
1105int idxd_device_load_config(struct idxd_device *idxd)
1106{
1107 union gencfg_reg reg;
1108 int i, rc;
1109
1110 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
7ed6f1b8 1111 idxd->rdbuf_limit = reg.rdbuf_limit;
8c66bbdc
DJ		 1112
1113 for (i = 0; i < idxd->max_groups; i++) {
1114 struct idxd_group *group = idxd->groups[i];
1115
1116 idxd_group_load_config(group);
1117 }
1118
1119 for (i = 0; i < idxd->max_wqs; i++) {
1120 struct idxd_wq *wq = idxd->wqs[i];
1121
1122 rc = idxd_wq_load_config(wq);
1123 if (rc < 0)
1124 return rc;
1125 }
1126
1127 return 0;
1128}
1f2bb403 1129
403a2e23
DJ		 1130static void idxd_flush_pending_descs(struct idxd_irq_entry *ie)
1131{
1132 struct idxd_desc *desc, *itr;
1133 struct llist_node *head;
1134 LIST_HEAD(flist);
1135 enum idxd_complete_type ctype;
1136
1137 spin_lock(&ie->list_lock);
1138 head = llist_del_all(&ie->pending_llist);
1139 if (head) {
1140 llist_for_each_entry_safe(desc, itr, head, llnode)
1141 list_add_tail(&desc->list, &ie->work_list);
1142 }
1143
1144 list_for_each_entry_safe(desc, itr, &ie->work_list, list)
1145 list_move_tail(&desc->list, &flist);
1146 spin_unlock(&ie->list_lock);
1147
1148 list_for_each_entry_safe(desc, itr, &flist, list) {
1149 list_del(&desc->list);
1150 ctype = desc->completion->status ? IDXD_COMPLETE_NORMAL : IDXD_COMPLETE_ABORT;
1151 idxd_dma_complete_txd(desc, ctype, true);
1152 }
1153}
1154
1155static void idxd_device_set_perm_entry(struct idxd_device *idxd,
1156 struct idxd_irq_entry *ie)
1157{
1158 union msix_perm mperm;
1159
1160 if (ie->pasid == INVALID_IOASID)
1161 return;
1162
1163 mperm.bits = 0;
1164 mperm.pasid = ie->pasid;
1165 mperm.pasid_en = 1;
1166 iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + ie->id * 8);
1167}
1168
1169static void idxd_device_clear_perm_entry(struct idxd_device *idxd,
1170 struct idxd_irq_entry *ie)
1171{
1172 iowrite32(0, idxd->reg_base + idxd->msix_perm_offset + ie->id * 8);
1173}
1174
1175void idxd_wq_free_irq(struct idxd_wq *wq)
1176{
1177 struct idxd_device *idxd = wq->idxd;
1178 struct idxd_irq_entry *ie = &wq->ie;
1179
1180 synchronize_irq(ie->vector);
1181 free_irq(ie->vector, ie);
1182 idxd_flush_pending_descs(ie);
1183 if (idxd->request_int_handles)
1184 idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX);
1185 idxd_device_clear_perm_entry(idxd, ie);
1186 ie->vector = -1;
1187 ie->int_handle = INVALID_INT_HANDLE;
1188 ie->pasid = INVALID_IOASID;
1189}
1190
1191int idxd_wq_request_irq(struct idxd_wq *wq)
1192{
1193 struct idxd_device *idxd = wq->idxd;
1194 struct pci_dev *pdev = idxd->pdev;
1195 struct device *dev = &pdev->dev;
1196 struct idxd_irq_entry *ie;
1197 int rc;
1198
63c14ae6
DJ		 1199 if (wq->type != IDXD_WQT_KERNEL)
1200 return 0;
1201
403a2e23
DJ		 1202 ie = &wq->ie;
1203 ie->vector = pci_irq_vector(pdev, ie->id);
1204 ie->pasid = device_pasid_enabled(idxd) ? idxd->pasid : INVALID_IOASID;
1205 idxd_device_set_perm_entry(idxd, ie);
1206
1207 rc = request_threaded_irq(ie->vector, NULL, idxd_wq_thread, 0, "idxd-portal", ie);
1208 if (rc < 0) {
1209 dev_err(dev, "Failed to request irq %d.\n", ie->vector);
1210 goto err_irq;
1211 }
1212
1213 if (idxd->request_int_handles) {
1214 rc = idxd_device_request_int_handle(idxd, ie->id, &ie->int_handle,
1215 IDXD_IRQ_MSIX);
1216 if (rc < 0)
1217 goto err_int_handle;
1218 } else {
1219 ie->int_handle = ie->id;
1220 }
1221
1222 return 0;
1223
1224err_int_handle:
1225 ie->int_handle = INVALID_INT_HANDLE;
1226 free_irq(ie->vector, ie);
1227err_irq:
1228 idxd_device_clear_perm_entry(idxd, ie);
1229 ie->pasid = INVALID_IOASID;
1230 return rc;
1231}
1232
63c14ae6 1233int drv_enable_wq(struct idxd_wq *wq)
1f2bb403
DJ		 1234{
1235 struct idxd_device *idxd = wq->idxd;
1236 struct device *dev = &idxd->pdev->dev;
1f2bb403
DJ		 1237 int rc = -ENXIO;
1238
1239 lockdep_assert_held(&wq->wq_lock);
1240
125d1037
DJ		 1241 if (idxd->state != IDXD_DEV_ENABLED) {
1242 idxd->cmd_status = IDXD_SCMD_DEV_NOT_ENABLED;
1f2bb403 1243 goto err;
125d1037 1244 }
1f2bb403
DJ		 1245
1246 if (wq->state != IDXD_WQ_DISABLED) {
1247 dev_dbg(dev, "wq %d already enabled.\n", wq->id);
125d1037 1248 idxd->cmd_status = IDXD_SCMD_WQ_ENABLED;
1f2bb403
DJ		 1249 rc = -EBUSY;
1250 goto err;
1251 }
1252
1253 if (!wq->group) {
1254 dev_dbg(dev, "wq %d not attached to group.\n", wq->id);
125d1037 1255 idxd->cmd_status = IDXD_SCMD_WQ_NO_GRP;
1f2bb403
DJ		 1256 goto err;
1257 }
1258
1259 if (strlen(wq->name) == 0) {
125d1037 1260 idxd->cmd_status = IDXD_SCMD_WQ_NO_NAME;
1f2bb403
DJ		 1261 dev_dbg(dev, "wq %d name not set.\n", wq->id);
1262 goto err;
1263 }
1264
1265 /* Shared WQ checks */
1266 if (wq_shared(wq)) {
42a1b738 1267 if (!wq_shared_supported(wq)) {
125d1037 1268 idxd->cmd_status = IDXD_SCMD_WQ_NO_SVM;
1f2bb403
DJ		 1269 dev_dbg(dev, "PASID not enabled and shared wq.\n");
1270 goto err;
1271 }
1272 /*
1273 * Shared wq with the threshold set to 0 means the user
1274 * did not set the threshold or transitioned from a
1275 * dedicated wq but did not set threshold. A value
1276 * of 0 would effectively disable the shared wq. The
1277 * driver does not allow a value of 0 to be set for
1278 * threshold via sysfs.
1279 */
1280 if (wq->threshold == 0) {
125d1037 1281 idxd->cmd_status = IDXD_SCMD_WQ_NO_THRESH;
1f2bb403
DJ		 1282 dev_dbg(dev, "Shared wq and threshold 0.\n");
1283 goto err;
1284 }
1285 }
1286
3157dd0a
DJ		 1287 /*
1288 * In the event that the WQ is configurable for pasid and priv bits.
1289 * For kernel wq, the driver should setup the pasid, pasid_en, and priv bit.
1290 * However, for non-kernel wq, the driver should only set the pasid_en bit for
1291 * shared wq. A dedicated wq that is not 'kernel' type will configure pasid and
1292 * pasid_en later on so there is no need to setup.
1293 */
1294 if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
1295 int priv = 0;
1296
42a1b738 1297 if (wq_pasid_enabled(wq)) {
3157dd0a
DJ		 1298 if (is_idxd_wq_kernel(wq) || wq_shared(wq)) {
1299 u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0;
1300
1301 __idxd_wq_set_pasid_locked(wq, pasid);
1302 }
1303 }
1304
1305 if (is_idxd_wq_kernel(wq))
1306 priv = 1;
1307 __idxd_wq_set_priv_locked(wq, priv);
1308 }
1309
0cda4f69 1310 rc = 0;
cf84a4b9 1311 spin_lock(&idxd->dev_lock);
1f2bb403
DJ		 1312 if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1313 rc = idxd_device_config(idxd);
cf84a4b9 1314 spin_unlock(&idxd->dev_lock);
1f2bb403
DJ		 1315 if (rc < 0) {
1316 dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc);
1317 goto err;
1318 }
1319
1320 rc = idxd_wq_enable(wq);
1321 if (rc < 0) {
1322 dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc);
1323 goto err;
1324 }
1325
1326 rc = idxd_wq_map_portal(wq);
1327 if (rc < 0) {
125d1037 1328 idxd->cmd_status = IDXD_SCMD_WQ_PORTAL_ERR;
1f2bb403
DJ		 1329 dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc);
1330 goto err_map_portal;
1331 }
1332
1333 wq->client_count = 0;
63c14ae6
DJ		 1334
1335 rc = idxd_wq_request_irq(wq);
1336 if (rc < 0) {
1337 idxd->cmd_status = IDXD_SCMD_WQ_IRQ_ERR;
1338 dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc);
1339 goto err_irq;
1340 }
1341
1342 rc = idxd_wq_alloc_resources(wq);
1343 if (rc < 0) {
1344 idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR;
1345 dev_dbg(dev, "WQ resource alloc failed\n");
1346 goto err_res_alloc;
1347 }
1348
1349 rc = idxd_wq_init_percpu_ref(wq);
1350 if (rc < 0) {
1351 idxd->cmd_status = IDXD_SCMD_PERCPU_ERR;
1352 dev_dbg(dev, "percpu_ref setup failed\n");
1353 goto err_ref;
1354 }
1355
1f2bb403
DJ		 1356 return 0;
1357
63c14ae6
DJ		 1358err_ref:
1359 idxd_wq_free_resources(wq);
1360err_res_alloc:
1361 idxd_wq_free_irq(wq);
1362err_irq:
1363 idxd_wq_unmap_portal(wq);
1f2bb403
DJ		 1364err_map_portal:
1365 rc = idxd_wq_disable(wq, false);
1366 if (rc < 0)
1367 dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq)));
1368err:
1369 return rc;
1370}
1371
63c14ae6 1372void drv_disable_wq(struct idxd_wq *wq)
69e4f8be
DJ		 1373{
1374 struct idxd_device *idxd = wq->idxd;
1375 struct device *dev = &idxd->pdev->dev;
1376
1377 lockdep_assert_held(&wq->wq_lock);
1378
69e4f8be
DJ		 1379 if (idxd_wq_refcount(wq))
1380 dev_warn(dev, "Clients has claim on wq %d: %d\n",
1381 wq->id, idxd_wq_refcount(wq));
1382
63c14ae6 1383 idxd_wq_free_resources(wq);
69e4f8be 1384 idxd_wq_unmap_portal(wq);
69e4f8be
DJ		 1385 idxd_wq_drain(wq);
1386 idxd_wq_reset(wq);
63c14ae6
DJ		 1387 percpu_ref_exit(&wq->wq_active);
1388 idxd_wq_free_irq(wq);
1389 wq->type = IDXD_WQT_NONE;
69e4f8be 1390 wq->client_count = 0;
69e4f8be
DJ		 1391}
1392
bd42805b
DJ		 1393int idxd_device_drv_probe(struct idxd_dev *idxd_dev)
1394{
1395 struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev);
bd42805b
DJ		 1396 int rc = 0;
1397
1398 /*
1399 * Device should be in disabled state for the idxd_drv to load. If it's in
1400 * enabled state, then the device was altered outside of driver's control.
1401 * If the state is in halted state, then we don't want to proceed.
1402 */
125d1037
DJ		 1403 if (idxd->state != IDXD_DEV_DISABLED) {
1404 idxd->cmd_status = IDXD_SCMD_DEV_ENABLED;
bd42805b 1405 return -ENXIO;
125d1037 1406 }
bd42805b
DJ		 1407
1408 /* Device configuration */
cf84a4b9 1409 spin_lock(&idxd->dev_lock);
bd42805b
DJ		 1410 if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1411 rc = idxd_device_config(idxd);
cf84a4b9 1412 spin_unlock(&idxd->dev_lock);
bd42805b
DJ		 1413 if (rc < 0)
1414 return -ENXIO;
1415
1416 /* Start device */
1417 rc = idxd_device_enable(idxd);
1418 if (rc < 0)
1419 return rc;
1420
1421 /* Setup DMA device without channels */
1422 rc = idxd_register_dma_device(idxd);
1423 if (rc < 0) {
1424 idxd_device_disable(idxd);
125d1037 1425 idxd->cmd_status = IDXD_SCMD_DEV_DMA_ERR;
bd42805b
DJ		 1426 return rc;
1427 }
1428
125d1037 1429 idxd->cmd_status = 0;
bd42805b
DJ		 1430 return 0;
1431}
745e92a6
DJ		 1432
1433void idxd_device_drv_remove(struct idxd_dev *idxd_dev)
1434{
1435 struct device *dev = &idxd_dev->conf_dev;
1436 struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev);
1437 int i;
1438
1439 for (i = 0; i < idxd->max_wqs; i++) {
1440 struct idxd_wq *wq = idxd->wqs[i];
1441 struct device *wq_dev = wq_confdev(wq);
1442
1443 if (wq->state == IDXD_WQ_DISABLED)
1444 continue;
1445 dev_warn(dev, "Active wq %d on disable %s.\n", i, dev_name(wq_dev));
1446 device_release_driver(wq_dev);
1447 }
1448
1449 idxd_unregister_dma_device(idxd);
1450 idxd_device_disable(idxd);
1451 if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1452 idxd_device_reset(idxd);
1453}
034b3290
DJ		 1454
1455static enum idxd_dev_type dev_types[] = {
1456 IDXD_DEV_DSA,
1457 IDXD_DEV_IAX,
1458 IDXD_DEV_NONE,
1459};
1460
1461struct idxd_device_driver idxd_drv = {
1462 .type = dev_types,
1463 .probe = idxd_device_drv_probe,
1464 .remove = idxd_device_drv_remove,
1465 .name = "idxd",
1466};
6e7f3ee9 1467EXPORT_SYMBOL_GPL(idxd_drv);
Linux 6.x block layer and io_uring tree(s)