Commit | Line | Data |
---|---|---|
9952f691 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
6579324a TB |
2 | /* |
3 | * Tegra host1x Channel | |
4 | * | |
5 | * Copyright (c) 2010-2013, NVIDIA Corporation. | |
6579324a TB |
6 | */ |
7 | ||
e1e90644 | 8 | #include <linux/host1x.h> |
de5469c2 | 9 | #include <linux/iommu.h> |
6579324a | 10 | #include <linux/slab.h> |
e1e90644 | 11 | |
6579324a TB |
12 | #include <trace/events/host1x.h> |
13 | ||
fc3be3e8 TR |
14 | #include "../channel.h" |
15 | #include "../dev.h" | |
16 | #include "../intr.h" | |
17 | #include "../job.h" | |
6579324a | 18 | |
6579324a TB |
19 | #define TRACE_MAX_LENGTH 128U |
20 | ||
6236451d TB |
21 | static void trace_write_gather(struct host1x_cdma *cdma, struct host1x_bo *bo, |
22 | u32 offset, u32 words) | |
23 | { | |
b40d02bf | 24 | struct device *dev = cdma_to_channel(cdma)->dev; |
6236451d TB |
25 | void *mem = NULL; |
26 | ||
27 | if (host1x_debug_trace_cmdbuf) | |
28 | mem = host1x_bo_mmap(bo); | |
29 | ||
30 | if (mem) { | |
31 | u32 i; | |
32 | /* | |
33 | * Write in batches of 128 as there seems to be a limit | |
34 | * of how much you can output to ftrace at once. | |
35 | */ | |
36 | for (i = 0; i < words; i += TRACE_MAX_LENGTH) { | |
b40d02bf | 37 | u32 num_words = min(words - i, TRACE_MAX_LENGTH); |
0b8070d1 | 38 | |
b40d02bf TR |
39 | offset += i * sizeof(u32); |
40 | ||
41 | trace_host1x_cdma_push_gather(dev_name(dev), bo, | |
42 | num_words, offset, | |
43 | mem); | |
6236451d | 44 | } |
b40d02bf | 45 | |
6236451d TB |
46 | host1x_bo_munmap(bo, mem); |
47 | } | |
48 | } | |
49 | ||
6579324a TB |
50 | static void submit_gathers(struct host1x_job *job) |
51 | { | |
52 | struct host1x_cdma *cdma = &job->channel->cdma; | |
67a82dbc TR |
53 | #if HOST1X_HW < 6 |
54 | struct device *dev = job->channel->dev; | |
55 | #endif | |
6579324a TB |
56 | unsigned int i; |
57 | ||
58 | for (i = 0; i < job->num_gathers; i++) { | |
59 | struct host1x_job_gather *g = &job->gathers[i]; | |
67a82dbc TR |
60 | dma_addr_t addr = g->base + g->offset; |
61 | u32 op2, op3; | |
62 | ||
63 | op2 = lower_32_bits(addr); | |
64 | op3 = upper_32_bits(addr); | |
0b8070d1 | 65 | |
67a82dbc TR |
66 | trace_write_gather(cdma, g->bo, g->offset, g->words); |
67 | ||
68 | if (op3 != 0) { | |
69 | #if HOST1X_HW >= 6 | |
70 | u32 op1 = host1x_opcode_gather_wide(g->words); | |
71 | u32 op4 = HOST1X_OPCODE_NOP; | |
72 | ||
73 | host1x_cdma_push_wide(cdma, op1, op2, op3, op4); | |
74 | #else | |
75 | dev_err(dev, "invalid gather for push buffer %pad\n", | |
76 | &addr); | |
77 | continue; | |
78 | #endif | |
79 | } else { | |
80 | u32 op1 = host1x_opcode_gather(g->words); | |
81 | ||
82 | host1x_cdma_push(cdma, op1, op2); | |
83 | } | |
6579324a TB |
84 | } |
85 | } | |
86 | ||
f5a954fe AM |
87 | static inline void synchronize_syncpt_base(struct host1x_job *job) |
88 | { | |
89 | struct host1x *host = dev_get_drvdata(job->channel->dev->parent); | |
90 | struct host1x_syncpt *sp = host->syncpt + job->syncpt_id; | |
5c0d8d38 TR |
91 | unsigned int id; |
92 | u32 value; | |
f5a954fe AM |
93 | |
94 | value = host1x_syncpt_read_max(sp); | |
95 | id = sp->base->id; | |
96 | ||
97 | host1x_cdma_push(&job->channel->cdma, | |
98 | host1x_opcode_setclass(HOST1X_CLASS_HOST1X, | |
99 | HOST1X_UCLASS_LOAD_SYNCPT_BASE, 1), | |
100 | HOST1X_UCLASS_LOAD_SYNCPT_BASE_BASE_INDX_F(id) | | |
101 | HOST1X_UCLASS_LOAD_SYNCPT_BASE_VALUE_F(value)); | |
102 | } | |
103 | ||
de5469c2 TR |
104 | static void host1x_channel_set_streamid(struct host1x_channel *channel) |
105 | { | |
8bbad1ba AB |
106 | #if HOST1X_HW >= 6 |
107 | u32 sid = 0x7f; | |
108 | #ifdef CONFIG_IOMMU_API | |
de5469c2 | 109 | struct iommu_fwspec *spec = dev_iommu_fwspec_get(channel->dev->parent); |
8bbad1ba AB |
110 | if (spec) |
111 | sid = spec->ids[0] & 0xffff; | |
112 | #endif | |
de5469c2 TR |
113 | |
114 | host1x_ch_writel(channel, sid, HOST1X_CHANNEL_SMMU_STREAMID); | |
115 | #endif | |
116 | } | |
117 | ||
6579324a TB |
118 | static int channel_submit(struct host1x_job *job) |
119 | { | |
120 | struct host1x_channel *ch = job->channel; | |
121 | struct host1x_syncpt *sp; | |
122 | u32 user_syncpt_incrs = job->syncpt_incrs; | |
123 | u32 prev_max = 0; | |
124 | u32 syncval; | |
125 | int err; | |
126 | struct host1x_waitlist *completed_waiter = NULL; | |
127 | struct host1x *host = dev_get_drvdata(ch->dev->parent); | |
128 | ||
129 | sp = host->syncpt + job->syncpt_id; | |
130 | trace_host1x_channel_submit(dev_name(ch->dev), | |
131 | job->num_gathers, job->num_relocs, | |
24c94e16 | 132 | job->syncpt_id, job->syncpt_incrs); |
6579324a TB |
133 | |
134 | /* before error checks, return current max */ | |
135 | prev_max = job->syncpt_end = host1x_syncpt_read_max(sp); | |
136 | ||
137 | /* get submit lock */ | |
138 | err = mutex_lock_interruptible(&ch->submitlock); | |
139 | if (err) | |
140 | goto error; | |
141 | ||
142 | completed_waiter = kzalloc(sizeof(*completed_waiter), GFP_KERNEL); | |
143 | if (!completed_waiter) { | |
144 | mutex_unlock(&ch->submitlock); | |
145 | err = -ENOMEM; | |
146 | goto error; | |
147 | } | |
148 | ||
de5469c2 TR |
149 | host1x_channel_set_streamid(ch); |
150 | ||
6579324a TB |
151 | /* begin a CDMA submit */ |
152 | err = host1x_cdma_begin(&ch->cdma, job); | |
153 | if (err) { | |
154 | mutex_unlock(&ch->submitlock); | |
155 | goto error; | |
156 | } | |
157 | ||
158 | if (job->serialize) { | |
159 | /* | |
160 | * Force serialization by inserting a host wait for the | |
161 | * previous job to finish before this one can commence. | |
162 | */ | |
163 | host1x_cdma_push(&ch->cdma, | |
164 | host1x_opcode_setclass(HOST1X_CLASS_HOST1X, | |
165 | host1x_uclass_wait_syncpt_r(), 1), | |
166 | host1x_class_host_wait_syncpt(job->syncpt_id, | |
167 | host1x_syncpt_read_max(sp))); | |
168 | } | |
169 | ||
f5a954fe AM |
170 | /* Synchronize base register to allow using it for relative waiting */ |
171 | if (sp->base) | |
172 | synchronize_syncpt_base(job); | |
173 | ||
6579324a TB |
174 | syncval = host1x_syncpt_incr_max(sp, user_syncpt_incrs); |
175 | ||
c3f52220 MP |
176 | host1x_hw_syncpt_assign_to_channel(host, sp, ch); |
177 | ||
6579324a TB |
178 | job->syncpt_end = syncval; |
179 | ||
180 | /* add a setclass for modules that require it */ | |
181 | if (job->class) | |
182 | host1x_cdma_push(&ch->cdma, | |
183 | host1x_opcode_setclass(job->class, 0, 0), | |
184 | HOST1X_OPCODE_NOP); | |
185 | ||
186 | submit_gathers(job); | |
187 | ||
188 | /* end CDMA submit & stash pinned hMems into sync queue */ | |
189 | host1x_cdma_end(&ch->cdma, job); | |
190 | ||
191 | trace_host1x_channel_submitted(dev_name(ch->dev), prev_max, syncval); | |
192 | ||
193 | /* schedule a submit complete interrupt */ | |
ac330f45 | 194 | err = host1x_intr_add_action(host, sp, syncval, |
6579324a TB |
195 | HOST1X_INTR_ACTION_SUBMIT_COMPLETE, ch, |
196 | completed_waiter, NULL); | |
197 | completed_waiter = NULL; | |
198 | WARN(err, "Failed to set submit complete interrupt"); | |
199 | ||
200 | mutex_unlock(&ch->submitlock); | |
201 | ||
202 | return 0; | |
203 | ||
204 | error: | |
205 | kfree(completed_waiter); | |
206 | return err; | |
207 | } | |
208 | ||
2316f29f MP |
209 | static void enable_gather_filter(struct host1x *host, |
210 | struct host1x_channel *ch) | |
211 | { | |
212 | #if HOST1X_HW >= 6 | |
213 | u32 val; | |
214 | ||
215 | if (!host->hv_regs) | |
216 | return; | |
217 | ||
218 | val = host1x_hypervisor_readl( | |
219 | host, HOST1X_HV_CH_KERNEL_FILTER_GBUFFER(ch->id / 32)); | |
220 | val |= BIT(ch->id % 32); | |
221 | host1x_hypervisor_writel( | |
222 | host, val, HOST1X_HV_CH_KERNEL_FILTER_GBUFFER(ch->id / 32)); | |
223 | #elif HOST1X_HW >= 4 | |
224 | host1x_ch_writel(ch, | |
225 | HOST1X_CHANNEL_CHANNELCTRL_KERNEL_FILTER_GBUFFER(1), | |
226 | HOST1X_CHANNEL_CHANNELCTRL); | |
227 | #endif | |
228 | } | |
229 | ||
6579324a TB |
230 | static int host1x_channel_init(struct host1x_channel *ch, struct host1x *dev, |
231 | unsigned int index) | |
232 | { | |
b7c61d51 TR |
233 | #if HOST1X_HW < 6 |
234 | ch->regs = dev->regs + index * 0x4000; | |
235 | #else | |
236 | ch->regs = dev->regs + index * 0x100; | |
237 | #endif | |
2316f29f | 238 | enable_gather_filter(dev, ch); |
6579324a TB |
239 | return 0; |
240 | } | |
241 | ||
242 | static const struct host1x_channel_ops host1x_channel_ops = { | |
243 | .init = host1x_channel_init, | |
244 | .submit = channel_submit, | |
245 | }; |