struct nvkm_chid *chid;
struct nvkm_chid *cgid;
+ struct list_head runqs;
+
DECLARE_BITMAP(mask, NVKM_FIFO_CHID_NR);
int nr;
struct list_head chan;
nvkm-y += nvkm/engine/fifo/base.o
nvkm-y += nvkm/engine/fifo/chan.o
nvkm-y += nvkm/engine/fifo/chid.o
+nvkm-y += nvkm/engine/fifo/runq.o
nvkm-y += nvkm/engine/fifo/nv04.o
nvkm-y += nvkm/engine/fifo/nv10.o
#include "priv.h"
#include "chan.h"
#include "chid.h"
+#include "runq.h"
#include <core/gpuobj.h>
#include <subdev/mc.h>
nvkm_fifo_oneinit(struct nvkm_engine *engine)
{
struct nvkm_fifo *fifo = nvkm_fifo(engine);
- int ret;
+ int ret, nr, i;
/* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */
if (fifo->func->chid_nr) {
return ret;
}
+ /* Create runqueues for each PBDMA. */
+ if (fifo->func->runq_nr) {
+ for (nr = fifo->func->runq_nr(fifo), i = 0; i < nr; i++) {
+ if (!nvkm_runq_new(fifo, i))
+ return -ENOMEM;
+ }
+ }
+
if (fifo->func->oneinit)
return fifo->func->oneinit(fifo);
nvkm_fifo_dtor(struct nvkm_engine *engine)
{
struct nvkm_fifo *fifo = nvkm_fifo(engine);
+ struct nvkm_runq *runq, *rtmp;
void *data = fifo;
+ list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head)
+ nvkm_runq_del(runq);
+
nvkm_chid_unref(&fifo->cgid);
nvkm_chid_unref(&fifo->chid);
int ret, nr;
fifo->func = func;
+ INIT_LIST_HEAD(&fifo->runqs);
spin_lock_init(&fifo->lock);
mutex_init(&fifo->mutex);
*/
#include "chan.h"
#include "chid.h"
+#include "runq.h"
#include "gf100.h"
#include "changf100.h"
nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat);
}
+static const struct nvkm_runq_func
+gf100_runq = {
+};
+
void
gf100_fifo_runlist_commit(struct gf100_fifo *fifo)
{
nvkm_wr32(device, 0x002628, 0x00000001); /* ENGINE_INTR_EN */
}
+int
+gf100_fifo_runq_nr(struct nvkm_fifo *fifo)
+{
+ struct nvkm_device *device = fifo->engine.subdev.device;
+ u32 save;
+
+ /* Determine number of PBDMAs by checking valid enable bits. */
+ save = nvkm_mask(device, 0x000204, 0xffffffff, 0xffffffff);
+ save = nvkm_mask(device, 0x000204, 0xffffffff, save);
+ return hweight32(save);
+}
+
int
gf100_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr)
{
struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
int ret;
- /* Determine number of PBDMAs by checking valid enable bits. */
- nvkm_wr32(device, 0x002204, 0xffffffff);
- fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x002204));
- nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr);
-
+ fifo->pbdma_nr = fifo->base.func->runq_nr(&fifo->base);
ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000,
false, &fifo->runlist.mem[0]);
.oneinit = gf100_fifo_oneinit,
.chid_nr = nv50_fifo_chid_nr,
.chid_ctor = gf100_fifo_chid_ctor,
+ .runq_nr = gf100_fifo_runq_nr,
.init = gf100_fifo_init,
.fini = gf100_fifo_fini,
.intr = gf100_fifo_intr,
.id_engine = gf100_fifo_id_engine,
.uevent_init = gf100_fifo_uevent_init,
.uevent_fini = gf100_fifo_uevent_fini,
+ .runq = &gf100_runq,
.cgrp = {{ }, &nv04_cgrp },
.chan = {{ 0, 0, FERMI_CHANNEL_GPFIFO }, &gf100_chan, .oclass = &gf100_fifo_gpfifo_oclass },
};
*/
#include "chan.h"
#include "chid.h"
+#include "runq.h"
#include "gk104.h"
#include "cgrp.h"
{}
};
+const struct nvkm_runq_func
+gk104_runq = {
+};
+
void
gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl,
struct nvkm_memory *mem, int nr)
nvkm_wr32(device, 0x000204, (1 << fifo->pbdma_nr) - 1);
}
-int
-gk104_fifo_pbdma_nr(struct gk104_fifo *fifo)
-{
- struct nvkm_device *device = fifo->base.engine.subdev.device;
- /* Determine number of PBDMAs by checking valid enable bits. */
- nvkm_wr32(device, 0x000204, 0xffffffff);
- return hweight32(nvkm_rd32(device, 0x000204));
-}
-
const struct gk104_fifo_pbdma_func
gk104_fifo_pbdma = {
- .nr = gk104_fifo_pbdma_nr,
.init = gk104_fifo_pbdma_init,
};
int pbid, ret, i, j;
u32 *map;
- fifo->pbdma_nr = fifo->func->pbdma->nr(fifo);
- nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr);
+ fifo->pbdma_nr = fifo->func->runq_nr(&fifo->base);
/* Read PBDMA->runlist(s) mapping from HW. */
if (!(map = kcalloc(fifo->pbdma_nr, sizeof(*map), GFP_KERNEL)))
.oneinit = gk104_fifo_oneinit,
.chid_nr = gk104_fifo_chid_nr,
.chid_ctor = gf100_fifo_chid_ctor,
+ .runq_nr = gf100_fifo_runq_nr,
.info = gk104_fifo_info,
.init = gk104_fifo_init,
.fini = gk104_fifo_fini,
.recover_chan = gk104_fifo_recover_chan,
.runlist = &gk104_fifo_runlist,
.pbdma = &gk104_fifo_pbdma,
+ .runq = &gk104_runq,
.cgrp = {{ }, &nv04_cgrp },
.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new },
};
void gk104_fifo_uevent_init(struct nvkm_fifo *fifo);
extern const struct gk104_fifo_pbdma_func gk104_fifo_pbdma;
-int gk104_fifo_pbdma_nr(struct gk104_fifo *);
void gk104_fifo_pbdma_init(struct gk104_fifo *);
extern const struct nvkm_enum gk104_fifo_fault_access[];
extern const struct nvkm_enum gk104_fifo_fault_engine[];
extern const struct gk104_fifo_runlist_func gm107_fifo_runlist;
extern const struct gk104_fifo_pbdma_func gm200_fifo_pbdma;
-int gm200_fifo_pbdma_nr(struct gk104_fifo *);
extern const struct nvkm_enum gp100_fifo_fault_engine[];
.oneinit = gk104_fifo_oneinit,
.chid_nr = gk104_fifo_chid_nr,
.chid_ctor = gk110_fifo_chid_ctor,
+ .runq_nr = gf100_fifo_runq_nr,
.info = gk104_fifo_info,
.init = gk104_fifo_init,
.fini = gk104_fifo_fini,
.recover_chan = gk104_fifo_recover_chan,
.runlist = &gk110_fifo_runlist,
.pbdma = &gk104_fifo_pbdma,
+ .runq = &gk104_runq,
.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp },
.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
};
*
* Authors: Ben Skeggs
*/
+#include "runq.h"
#include "gk104.h"
#include "changk104.h"
const struct gk104_fifo_pbdma_func
gk208_fifo_pbdma = {
- .nr = gk104_fifo_pbdma_nr,
.init = gk104_fifo_pbdma_init,
.init_timeout = gk208_fifo_pbdma_init_timeout,
};
+const struct nvkm_runq_func
+gk208_runq = {
+};
+
static int
gk208_fifo_chid_nr(struct nvkm_fifo *fifo)
{
.oneinit = gk104_fifo_oneinit,
.chid_nr = gk208_fifo_chid_nr,
.chid_ctor = gk110_fifo_chid_ctor,
+ .runq_nr = gf100_fifo_runq_nr,
.info = gk104_fifo_info,
.init = gk104_fifo_init,
.fini = gk104_fifo_fini,
.recover_chan = gk104_fifo_recover_chan,
.runlist = &gk110_fifo_runlist,
.pbdma = &gk208_fifo_pbdma,
+ .runq = &gk208_runq,
.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp },
.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
};
.oneinit = gk104_fifo_oneinit,
.chid_nr = nv50_fifo_chid_nr,
.chid_ctor = gk110_fifo_chid_ctor,
+ .runq_nr = gf100_fifo_runq_nr,
.info = gk104_fifo_info,
.init = gk104_fifo_init,
.fini = gk104_fifo_fini,
.recover_chan = gk104_fifo_recover_chan,
.runlist = &gk110_fifo_runlist,
.pbdma = &gk208_fifo_pbdma,
+ .runq = &gk208_runq,
.cgrp = {{ }, &gk110_cgrp },
.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
};
.oneinit = gk104_fifo_oneinit,
.chid_nr = gm107_fifo_chid_nr,
.chid_ctor = gk110_fifo_chid_ctor,
+ .runq_nr = gf100_fifo_runq_nr,
.info = gk104_fifo_info,
.init = gk104_fifo_init,
.fini = gk104_fifo_fini,
.recover_chan = gk104_fifo_recover_chan,
.runlist = &gm107_fifo_runlist,
.pbdma = &gk208_fifo_pbdma,
+ .runq = &gk208_runq,
.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp },
.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
};
#include <nvif/class.h>
int
-gm200_fifo_pbdma_nr(struct gk104_fifo *fifo)
+gm200_fifo_runq_nr(struct nvkm_fifo *fifo)
{
- struct nvkm_device *device = fifo->base.engine.subdev.device;
- return nvkm_rd32(device, 0x002004) & 0x000000ff;
+ return nvkm_rd32(fifo->engine.subdev.device, 0x002004) & 0x000000ff;
}
const struct gk104_fifo_pbdma_func
gm200_fifo_pbdma = {
- .nr = gm200_fifo_pbdma_nr,
.init = gk104_fifo_pbdma_init,
.init_timeout = gk208_fifo_pbdma_init_timeout,
};
.oneinit = gk104_fifo_oneinit,
.chid_nr = gm200_fifo_chid_nr,
.chid_ctor = gk110_fifo_chid_ctor,
+ .runq_nr = gm200_fifo_runq_nr,
.info = gk104_fifo_info,
.init = gk104_fifo_init,
.fini = gk104_fifo_fini,
.recover_chan = gk104_fifo_recover_chan,
.runlist = &gm107_fifo_runlist,
.pbdma = &gm200_fifo_pbdma,
+ .runq = &gk208_runq,
.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp },
.chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
};
.oneinit = gk104_fifo_oneinit,
.chid_nr = gm200_fifo_chid_nr,
.chid_ctor = gk110_fifo_chid_ctor,
+ .runq_nr = gm200_fifo_runq_nr,
.info = gk104_fifo_info,
.init = gk104_fifo_init,
.fini = gk104_fifo_fini,
.recover_chan = gk104_fifo_recover_chan,
.runlist = &gm107_fifo_runlist,
.pbdma = &gm200_fifo_pbdma,
+ .runq = &gk208_runq,
.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true },
.chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
};
*/
#include "chan.h"
#include "cgrp.h"
+#include "runq.h"
#include "gk104.h"
#include "changk104.h"
gv100_chan = {
};
+const struct nvkm_runq_func
+gv100_runq = {
+};
+
void
gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan,
struct nvkm_memory *memory, u32 offset)
.oneinit = gk104_fifo_oneinit,
.chid_nr = gm200_fifo_chid_nr,
.chid_ctor = gk110_fifo_chid_ctor,
+ .runq_nr = gm200_fifo_runq_nr,
.info = gk104_fifo_info,
.init = gk104_fifo_init,
.fini = gk104_fifo_fini,
.recover_chan = gk104_fifo_recover_chan,
.runlist = &gv100_fifo_runlist,
.pbdma = &gm200_fifo_pbdma,
+ .runq = &gv100_runq,
.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true },
.chan = {{ 0, 0, VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new },
};
int (*oneinit)(struct nvkm_fifo *);
int (*chid_nr)(struct nvkm_fifo *);
int (*chid_ctor)(struct nvkm_fifo *, int nr);
+ int (*runq_nr)(struct nvkm_fifo *);
int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data);
void (*init)(struct nvkm_fifo *);
} *runlist;
const struct gk104_fifo_pbdma_func {
- int (*nr)(struct gk104_fifo *);
void (*init)(struct gk104_fifo *);
void (*init_timeout)(struct gk104_fifo *);
} *pbdma;
+ const struct nvkm_runq_func *runq;
+
struct nvkm_fifo_func_cgrp {
struct nvkm_sclass user;
const struct nvkm_cgrp_func *func;
extern const struct nvkm_chan_func g84_chan;
int gf100_fifo_chid_ctor(struct nvkm_fifo *, int);
+int gf100_fifo_runq_nr(struct nvkm_fifo *);
void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
int gk104_fifo_chid_nr(struct nvkm_fifo *);
void gk104_fifo_recover_chan(struct nvkm_fifo *, int);
int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *);
struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int);
+extern const struct nvkm_runq_func gk104_runq;
int gk110_fifo_chid_ctor(struct nvkm_fifo *, int);
extern const struct nvkm_cgrp_func gk110_cgrp;
extern const struct nvkm_chan_func gk110_chan;
+extern const struct nvkm_runq_func gk208_runq;
+
void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault;
extern const struct nvkm_chan_func gm107_chan;
int gm200_fifo_chid_nr(struct nvkm_fifo *);
+int gm200_fifo_runq_nr(struct nvkm_fifo *);
void gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
+extern const struct nvkm_runq_func gv100_runq;
+
extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault;
int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *,
--- /dev/null
+/*
+ * Copyright 2021 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "runq.h"
+#include "priv.h"
+
+void
+nvkm_runq_del(struct nvkm_runq *runq)
+{
+ list_del(&runq->head);
+ kfree(runq);
+}
+
+struct nvkm_runq *
+nvkm_runq_new(struct nvkm_fifo *fifo, int pbid)
+{
+ struct nvkm_runq *runq;
+
+ if (!(runq = kzalloc(sizeof(*runq), GFP_KERNEL)))
+ return NULL;
+
+ runq->func = fifo->func->runq;
+ runq->fifo = fifo;
+ runq->id = pbid;
+ list_add_tail(&runq->head, &fifo->runqs);
+ return runq;
+}
--- /dev/null
+/* SPDX-License-Identifier: MIT */
+#ifndef __NVKM_RUNQ_H__
+#define __NVKM_RUNQ_H__
+#include <core/os.h>
+
+struct nvkm_runq {
+ const struct nvkm_runq_func {
+ } *func;
+ struct nvkm_fifo *fifo;
+ int id;
+
+ struct list_head head;
+};
+
+struct nvkm_runq *nvkm_runq_new(struct nvkm_fifo *, int pbid);
+void nvkm_runq_del(struct nvkm_runq *);
+
+#define nvkm_runq_foreach_cond(runq,fifo,cond) nvkm_list_foreach(runq, &(fifo)->runqs, head, (cond))
+
+#define RUNQ_PRINT(r,l,p,f,a...) \
+ nvkm_printk__(&(r)->fifo->engine.subdev, NV_DBG_##l, p, "PBDMA%d:"f, (r)->id, ##a)
+#define RUNQ_ERROR(r,f,a...) RUNQ_PRINT((r), ERROR, err, " "f"\n", ##a)
+#define RUNQ_DEBUG(r,f,a...) RUNQ_PRINT((r), DEBUG, info, " "f"\n", ##a)
+#endif
static const struct gk104_fifo_pbdma_func
tu102_fifo_pbdma = {
- .nr = gm200_fifo_pbdma_nr,
.init = tu102_fifo_pbdma_init,
.init_timeout = gk208_fifo_pbdma_init_timeout,
};
.oneinit = gk104_fifo_oneinit,
.chid_nr = gm200_fifo_chid_nr,
.chid_ctor = gk110_fifo_chid_ctor,
+ .runq_nr = gm200_fifo_runq_nr,
.info = gk104_fifo_info,
.init = gk104_fifo_init,
.fini = gk104_fifo_fini,
.recover_chan = tu102_fifo_recover_chan,
.runlist = &tu102_fifo_runlist,
.pbdma = &tu102_fifo_pbdma,
+ .runq = &gv100_runq,
.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true },
.chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new },
};
projects / linux-block.git / commitdiff