projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Create GEN specific write MMIO
[linux-2.6-block.git] / drivers / gpu / drm / i915 / intel_uncore.c
CommitLineData
907b28c5
CW		 1/*
2 * Copyright © 2013 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include "i915_drv.h"
25#include "intel_drv.h"
26
27#define FORCEWAKE_ACK_TIMEOUT_MS 2
28
6af5d92f
CW		 29#define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__))
30#define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__))
31
32#define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__))
33#define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__))
34
35#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
36#define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__))
37
38#define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__))
39#define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__))
40
41#define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__)
42
43
907b28c5
CW		 44static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
45{
46 u32 gt_thread_status_mask;
47
48 if (IS_HASWELL(dev_priv->dev))
49 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
50 else
51 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
52
53 /* w/a for a sporadic read returning 0 by waiting for the GT
54 * thread to wake up.
55 */
6af5d92f 56 if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
907b28c5
CW		 57 DRM_ERROR("GT thread status wait timed out\n");
58}
59
60static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
61{
6af5d92f
CW		 62 __raw_i915_write32(dev_priv, FORCEWAKE, 0);
63 /* something from same cacheline, but !FORCEWAKE */
64 __raw_posting_read(dev_priv, ECOBUS);
907b28c5
CW		 65}
66
67static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
68{
6af5d92f 69 if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
907b28c5
CW		 70 FORCEWAKE_ACK_TIMEOUT_MS))
71 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
72
6af5d92f
CW		 73 __raw_i915_write32(dev_priv, FORCEWAKE, 1);
74 /* something from same cacheline, but !FORCEWAKE */
75 __raw_posting_read(dev_priv, ECOBUS);
907b28c5 76
6af5d92f 77 if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1),
907b28c5
CW		 78 FORCEWAKE_ACK_TIMEOUT_MS))
79 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
80
81 /* WaRsForcewakeWaitTC0:snb */
82 __gen6_gt_wait_for_thread_c0(dev_priv);
83}
84
85static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
86{
6af5d92f 87 __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
907b28c5 88 /* something from same cacheline, but !FORCEWAKE_MT */
6af5d92f 89 __raw_posting_read(dev_priv, ECOBUS);
907b28c5
CW		 90}
91
92static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
93{
94 u32 forcewake_ack;
95
96 if (IS_HASWELL(dev_priv->dev))
97 forcewake_ack = FORCEWAKE_ACK_HSW;
98 else
99 forcewake_ack = FORCEWAKE_MT_ACK;
100
6af5d92f 101 if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0,
907b28c5
CW		 102 FORCEWAKE_ACK_TIMEOUT_MS))
103 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
104
6af5d92f
CW		 105 __raw_i915_write32(dev_priv, FORCEWAKE_MT,
106 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
907b28c5 107 /* something from same cacheline, but !FORCEWAKE_MT */
6af5d92f 108 __raw_posting_read(dev_priv, ECOBUS);
907b28c5 109
6af5d92f 110 if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL),
907b28c5
CW		 111 FORCEWAKE_ACK_TIMEOUT_MS))
112 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
113
114 /* WaRsForcewakeWaitTC0:ivb,hsw */
115 __gen6_gt_wait_for_thread_c0(dev_priv);
116}
117
118static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
119{
120 u32 gtfifodbg;
6af5d92f
CW		 121
122 gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
907b28c5
CW		 123 if (WARN(gtfifodbg & GT_FIFO_CPU_ERROR_MASK,
124 "MMIO read or write has been dropped %x\n", gtfifodbg))
6af5d92f 125 __raw_i915_write32(dev_priv, GTFIFODBG, GT_FIFO_CPU_ERROR_MASK);
907b28c5
CW		 126}
127
128static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
129{
6af5d92f 130 __raw_i915_write32(dev_priv, FORCEWAKE, 0);
907b28c5 131 /* something from same cacheline, but !FORCEWAKE */
6af5d92f 132 __raw_posting_read(dev_priv, ECOBUS);
907b28c5
CW		 133 gen6_gt_check_fifodbg(dev_priv);
134}
135
136static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
137{
6af5d92f
CW		 138 __raw_i915_write32(dev_priv, FORCEWAKE_MT,
139 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
907b28c5 140 /* something from same cacheline, but !FORCEWAKE_MT */
6af5d92f 141 __raw_posting_read(dev_priv, ECOBUS);
907b28c5
CW		 142 gen6_gt_check_fifodbg(dev_priv);
143}
144
145static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
146{
147 int ret = 0;
148
149 if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
150 int loop = 500;
6af5d92f 151 u32 fifo = __raw_i915_read32(dev_priv, GT_FIFO_FREE_ENTRIES);
907b28c5
CW		 152 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
153 udelay(10);
6af5d92f 154 fifo = __raw_i915_read32(dev_priv, GT_FIFO_FREE_ENTRIES);
907b28c5
CW		 155 }
156 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
157 ++ret;
158 dev_priv->uncore.fifo_count = fifo;
159 }
160 dev_priv->uncore.fifo_count--;
161
162 return ret;
163}
164
165static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
166{
6af5d92f
CW		 167 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
168 _MASKED_BIT_DISABLE(0xffff));
907b28c5 169 /* something from same cacheline, but !FORCEWAKE_VLV */
6af5d92f 170 __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
907b28c5
CW		 171}
172
173static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
174{
6af5d92f 175 if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL) == 0,
907b28c5
CW		 176 FORCEWAKE_ACK_TIMEOUT_MS))
177 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
178
6af5d92f
CW		 179 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
180 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
181 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
907b28c5
CW		 182 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
183
6af5d92f 184 if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL),
907b28c5
CW		 185 FORCEWAKE_ACK_TIMEOUT_MS))
186 DRM_ERROR("Timed out waiting for GT to ack forcewake request.\n");
187
6af5d92f 188 if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK_MEDIA_VLV) &
907b28c5
CW		 189 FORCEWAKE_KERNEL),
190 FORCEWAKE_ACK_TIMEOUT_MS))
191 DRM_ERROR("Timed out waiting for media to ack forcewake request.\n");
192
193 /* WaRsForcewakeWaitTC0:vlv */
194 __gen6_gt_wait_for_thread_c0(dev_priv);
195}
196
197static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
198{
6af5d92f
CW		 199 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
200 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
201 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
907b28c5
CW		 202 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
203 /* The below doubles as a POSTING_READ */
204 gen6_gt_check_fifodbg(dev_priv);
205}
206
aec347ab
CW		 207static void gen6_force_wake_work(struct work_struct *work)
208{
209 struct drm_i915_private *dev_priv =
210 container_of(work, typeof(*dev_priv), uncore.force_wake_work.work);
211 unsigned long irqflags;
212
213 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
214 if (--dev_priv->uncore.forcewake_count == 0)
215 dev_priv->uncore.funcs.force_wake_put(dev_priv);
216 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
217}
218
907b28c5
CW		 219void intel_uncore_early_sanitize(struct drm_device *dev)
220{
221 struct drm_i915_private *dev_priv = dev->dev_private;
222
223 if (HAS_FPGA_DBG_UNCLAIMED(dev))
6af5d92f 224 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
18ce3994
BW		 225
226 if (IS_HASWELL(dev) &&
227 (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) {
228 /* The docs do not explain exactly how the calculation can be
229 * made. It is somewhat guessable, but for now, it's always
230 * 128MB.
231 * NB: We can't write IDICR yet because we do not have gt funcs
232 * set up */
233 dev_priv->ellc_size = 128;
234 DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size);
235 }
907b28c5
CW		 236}
237
521198a2 238static void intel_uncore_forcewake_reset(struct drm_device *dev)
907b28c5
CW		 239{
240 struct drm_i915_private *dev_priv = dev->dev_private;
241
242 if (IS_VALLEYVIEW(dev)) {
243 vlv_force_wake_reset(dev_priv);
244 } else if (INTEL_INFO(dev)->gen >= 6) {
245 __gen6_gt_force_wake_reset(dev_priv);
246 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
247 __gen6_gt_force_wake_mt_reset(dev_priv);
248 }
521198a2
MK		 249}
250
251void intel_uncore_sanitize(struct drm_device *dev)
252{
02f4c9e0
CML		 253 struct drm_i915_private *dev_priv = dev->dev_private;
254 u32 reg_val;
255
521198a2 256 intel_uncore_forcewake_reset(dev);
907b28c5
CW		 257
258 /* BIOS often leaves RC6 enabled, but disable it for hw init */
259 intel_disable_gt_powersave(dev);
02f4c9e0
CML		 260
261 /* Turn off power gate, require especially for the BIOS less system */
262 if (IS_VALLEYVIEW(dev)) {
263
264 mutex_lock(&dev_priv->rps.hw_lock);
265 reg_val = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS);
266
267 if (reg_val & (RENDER_PWRGT | MEDIA_PWRGT | DISP2D_PWRGT))
268 vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, 0x0);
269
270 mutex_unlock(&dev_priv->rps.hw_lock);
271
272 }
907b28c5
CW		 273}
274
275/*
276 * Generally this is called implicitly by the register read function. However,
277 * if some sequence requires the GT to not power down then this function should
278 * be called at the beginning of the sequence followed by a call to
279 * gen6_gt_force_wake_put() at the end of the sequence.
280 */
281void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
282{
283 unsigned long irqflags;
284
285 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
286 if (dev_priv->uncore.forcewake_count++ == 0)
287 dev_priv->uncore.funcs.force_wake_get(dev_priv);
288 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
289}
290
291/*
292 * see gen6_gt_force_wake_get()
293 */
294void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
295{
296 unsigned long irqflags;
297
298 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
aec347ab
CW		 299 if (--dev_priv->uncore.forcewake_count == 0) {
300 dev_priv->uncore.forcewake_count++;
301 mod_delayed_work(dev_priv->wq,
302 &dev_priv->uncore.force_wake_work,
303 1);
304 }
907b28c5
CW		 305 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
306}
307
308/* We give fast paths for the really cool registers */
309#define NEEDS_FORCE_WAKE(dev_priv, reg) \
310 ((HAS_FORCE_WAKE((dev_priv)->dev)) && \
311 ((reg) < 0x40000) && \
312 ((reg) != FORCEWAKE))
313
314static void
315ilk_dummy_write(struct drm_i915_private *dev_priv)
316{
317 /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
318 * the chip from rc6 before touching it for real. MI_MODE is masked,
319 * hence harmless to write 0 into. */
6af5d92f 320 __raw_i915_write32(dev_priv, MI_MODE, 0);
907b28c5
CW		 321}
322
323static void
324hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg)
325{
326 if (HAS_FPGA_DBG_UNCLAIMED(dev_priv->dev) &&
6af5d92f 327 (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
907b28c5
CW		 328 DRM_ERROR("Unknown unclaimed register before writing to %x\n",
329 reg);
6af5d92f 330 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
907b28c5
CW		 331 }
332}
333
334static void
335hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg)
336{
337 if (HAS_FPGA_DBG_UNCLAIMED(dev_priv->dev) &&
6af5d92f 338 (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
907b28c5 339 DRM_ERROR("Unclaimed write to %x\n", reg);
6af5d92f 340 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
907b28c5
CW		 341 }
342}
343
5d738795
BW		 344#define REG_READ_HEADER(x) \
345 unsigned long irqflags; \
346 u##x val = 0; \
347 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
348
349#define REG_READ_FOOTER \
350 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
351 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
352 return val
353
3967018e 354#define __gen4_read(x) \
0b274481 355static u##x \
3967018e
BW		 356gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
357 REG_READ_HEADER(x); \
358 val = __raw_i915_read##x(dev_priv, reg); \
359 REG_READ_FOOTER; \
360}
361
362#define __gen5_read(x) \
363static u##x \
364gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
365 REG_READ_HEADER(x); \
366 ilk_dummy_write(dev_priv); \
367 val = __raw_i915_read##x(dev_priv, reg); \
368 REG_READ_FOOTER; \
369}
370
371#define __gen6_read(x) \
372static u##x \
373gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
5d738795 374 REG_READ_HEADER(x); \
907b28c5
CW		 375 if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
376 if (dev_priv->uncore.forcewake_count == 0) \
377 dev_priv->uncore.funcs.force_wake_get(dev_priv); \
6af5d92f 378 val = __raw_i915_read##x(dev_priv, reg); \
907b28c5
CW		 379 if (dev_priv->uncore.forcewake_count == 0) \
380 dev_priv->uncore.funcs.force_wake_put(dev_priv); \
381 } else { \
6af5d92f 382 val = __raw_i915_read##x(dev_priv, reg); \
907b28c5 383 } \
5d738795 384 REG_READ_FOOTER; \
907b28c5
CW		 385}
386
3967018e
BW		 387__gen6_read(8)
388__gen6_read(16)
389__gen6_read(32)
390__gen6_read(64)
391__gen5_read(8)
392__gen5_read(16)
393__gen5_read(32)
394__gen5_read(64)
395__gen4_read(8)
396__gen4_read(16)
397__gen4_read(32)
398__gen4_read(64)
399
400#undef __gen6_read
401#undef __gen5_read
402#undef __gen4_read
5d738795
BW		 403#undef REG_READ_FOOTER
404#undef REG_READ_HEADER
405
406#define REG_WRITE_HEADER \
407 unsigned long irqflags; \
408 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
409 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
907b28c5 410
4032ef43 411#define __gen4_write(x) \
0b274481 412static void \
4032ef43
BW		 413gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
414 REG_WRITE_HEADER; \
415 __raw_i915_write##x(dev_priv, reg, val); \
416 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
417}
418
419#define __gen5_write(x) \
420static void \
421gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
422 REG_WRITE_HEADER; \
423 ilk_dummy_write(dev_priv); \
424 __raw_i915_write##x(dev_priv, reg, val); \
425 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
426}
427
428#define __gen6_write(x) \
429static void \
430gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
431 u32 __fifo_ret = 0; \
432 REG_WRITE_HEADER; \
433 if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
434 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
435 } \
436 __raw_i915_write##x(dev_priv, reg, val); \
437 if (unlikely(__fifo_ret)) { \
438 gen6_gt_check_fifodbg(dev_priv); \
439 } \
440 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
441}
442
443#define __hsw_write(x) \
444static void \
445hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
907b28c5 446 u32 __fifo_ret = 0; \
5d738795 447 REG_WRITE_HEADER; \
907b28c5
CW		 448 if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
449 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
450 } \
907b28c5 451 hsw_unclaimed_reg_clear(dev_priv, reg); \
6af5d92f 452 __raw_i915_write##x(dev_priv, reg, val); \
907b28c5
CW		 453 if (unlikely(__fifo_ret)) { \
454 gen6_gt_check_fifodbg(dev_priv); \
455 } \
456 hsw_unclaimed_reg_check(dev_priv, reg); \
457 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
458}
3967018e 459
4032ef43
BW		 460__hsw_write(8)
461__hsw_write(16)
462__hsw_write(32)
463__hsw_write(64)
464__gen6_write(8)
465__gen6_write(16)
466__gen6_write(32)
467__gen6_write(64)
468__gen5_write(8)
469__gen5_write(16)
470__gen5_write(32)
471__gen5_write(64)
472__gen4_write(8)
473__gen4_write(16)
474__gen4_write(32)
475__gen4_write(64)
476
477#undef __hsw_write
478#undef __gen6_write
479#undef __gen5_write
480#undef __gen4_write
5d738795 481#undef REG_WRITE_HEADER
907b28c5 482
0b274481
BW		 483void intel_uncore_init(struct drm_device *dev)
484{
485 struct drm_i915_private *dev_priv = dev->dev_private;
486
487 INIT_DELAYED_WORK(&dev_priv->uncore.force_wake_work,
488 gen6_force_wake_work);
489
490 if (IS_VALLEYVIEW(dev)) {
491 dev_priv->uncore.funcs.force_wake_get = vlv_force_wake_get;
492 dev_priv->uncore.funcs.force_wake_put = vlv_force_wake_put;
493 } else if (IS_HASWELL(dev)) {
494 dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get;
495 dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put;
496 } else if (IS_IVYBRIDGE(dev)) {
497 u32 ecobus;
498
499 /* IVB configs may use multi-threaded forcewake */
500
501 /* A small trick here - if the bios hasn't configured
502 * MT forcewake, and if the device is in RC6, then
503 * force_wake_mt_get will not wake the device and the
504 * ECOBUS read will return zero. Which will be
505 * (correctly) interpreted by the test below as MT
506 * forcewake being disabled.
507 */
508 mutex_lock(&dev->struct_mutex);
509 __gen6_gt_force_wake_mt_get(dev_priv);
510 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
511 __gen6_gt_force_wake_mt_put(dev_priv);
512 mutex_unlock(&dev->struct_mutex);
513
514 if (ecobus & FORCEWAKE_MT_ENABLE) {
515 dev_priv->uncore.funcs.force_wake_get =
516 __gen6_gt_force_wake_mt_get;
517 dev_priv->uncore.funcs.force_wake_put =
518 __gen6_gt_force_wake_mt_put;
519 } else {
520 DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
521 DRM_INFO("when using vblank-synced partial screen updates.\n");
522 dev_priv->uncore.funcs.force_wake_get =
523 __gen6_gt_force_wake_get;
524 dev_priv->uncore.funcs.force_wake_put =
525 __gen6_gt_force_wake_put;
526 }
527 } else if (IS_GEN6(dev)) {
528 dev_priv->uncore.funcs.force_wake_get =
529 __gen6_gt_force_wake_get;
530 dev_priv->uncore.funcs.force_wake_put =
531 __gen6_gt_force_wake_put;
532 }
533
3967018e
BW		 534 switch (INTEL_INFO(dev)->gen) {
535 case 7:
536 case 6:
4032ef43
BW		 537 if (IS_HASWELL(dev)) {
538 dev_priv->uncore.funcs.mmio_writeb = hsw_write8;
539 dev_priv->uncore.funcs.mmio_writew = hsw_write16;
540 dev_priv->uncore.funcs.mmio_writel = hsw_write32;
541 dev_priv->uncore.funcs.mmio_writeq = hsw_write64;
542 } else {
543 dev_priv->uncore.funcs.mmio_writeb = gen6_write8;
544 dev_priv->uncore.funcs.mmio_writew = gen6_write16;
545 dev_priv->uncore.funcs.mmio_writel = gen6_write32;
546 dev_priv->uncore.funcs.mmio_writeq = gen6_write64;
547 }
3967018e
BW		 548 dev_priv->uncore.funcs.mmio_readb = gen6_read8;
549 dev_priv->uncore.funcs.mmio_readw = gen6_read16;
550 dev_priv->uncore.funcs.mmio_readl = gen6_read32;
551 dev_priv->uncore.funcs.mmio_readq = gen6_read64;
552 break;
553 case 5:
4032ef43
BW		 554 dev_priv->uncore.funcs.mmio_writeb = gen5_write8;
555 dev_priv->uncore.funcs.mmio_writew = gen5_write16;
556 dev_priv->uncore.funcs.mmio_writel = gen5_write32;
557 dev_priv->uncore.funcs.mmio_writeq = gen5_write64;
3967018e
BW		 558 dev_priv->uncore.funcs.mmio_readb = gen5_read8;
559 dev_priv->uncore.funcs.mmio_readw = gen5_read16;
560 dev_priv->uncore.funcs.mmio_readl = gen5_read32;
561 dev_priv->uncore.funcs.mmio_readq = gen5_read64;
562 break;
563 case 4:
564 case 3:
565 case 2:
4032ef43
BW		 566 dev_priv->uncore.funcs.mmio_writeb = gen4_write8;
567 dev_priv->uncore.funcs.mmio_writew = gen4_write16;
568 dev_priv->uncore.funcs.mmio_writel = gen4_write32;
569 dev_priv->uncore.funcs.mmio_writeq = gen4_write64;
3967018e
BW		 570 dev_priv->uncore.funcs.mmio_readb = gen4_read8;
571 dev_priv->uncore.funcs.mmio_readw = gen4_read16;
572 dev_priv->uncore.funcs.mmio_readl = gen4_read32;
573 dev_priv->uncore.funcs.mmio_readq = gen4_read64;
574 break;
575 }
0b274481
BW		 576}
577
578void intel_uncore_fini(struct drm_device *dev)
579{
580 struct drm_i915_private *dev_priv = dev->dev_private;
581
582 flush_delayed_work(&dev_priv->uncore.force_wake_work);
583
584 /* Paranoia: make sure we have disabled everything before we exit. */
585 intel_uncore_sanitize(dev);
586}
587
907b28c5
CW		 588static const struct register_whitelist {
589 uint64_t offset;
590 uint32_t size;
591 uint32_t gen_bitmask; /* support gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
592} whitelist[] = {
593 { RING_TIMESTAMP(RENDER_RING_BASE), 8, 0xF0 },
594};
595
596int i915_reg_read_ioctl(struct drm_device *dev,
597 void *data, struct drm_file *file)
598{
599 struct drm_i915_private *dev_priv = dev->dev_private;
600 struct drm_i915_reg_read *reg = data;
601 struct register_whitelist const *entry = whitelist;
602 int i;
603
604 for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
605 if (entry->offset == reg->offset &&
606 (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
607 break;
608 }
609
610 if (i == ARRAY_SIZE(whitelist))
611 return -EINVAL;
612
613 switch (entry->size) {
614 case 8:
615 reg->val = I915_READ64(reg->offset);
616 break;
617 case 4:
618 reg->val = I915_READ(reg->offset);
619 break;
620 case 2:
621 reg->val = I915_READ16(reg->offset);
622 break;
623 case 1:
624 reg->val = I915_READ8(reg->offset);
625 break;
626 default:
627 WARN_ON(1);
628 return -EINVAL;
629 }
630
631 return 0;
632}
633
907b28c5
CW		 634static int i965_reset_complete(struct drm_device *dev)
635{
636 u8 gdrst;
637 pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
638 return (gdrst & GRDOM_RESET_ENABLE) == 0;
639}
640
641static int i965_do_reset(struct drm_device *dev)
642{
643 int ret;
644
645 /*
646 * Set the domains we want to reset (GRDOM/bits 2 and 3) as
647 * well as the reset bit (GR/bit 0). Setting the GR bit
648 * triggers the reset; when done, the hardware will clear it.
649 */
650 pci_write_config_byte(dev->pdev, I965_GDRST,
651 GRDOM_RENDER | GRDOM_RESET_ENABLE);
652 ret = wait_for(i965_reset_complete(dev), 500);
653 if (ret)
654 return ret;
655
656 /* We can't reset render&media without also resetting display ... */
657 pci_write_config_byte(dev->pdev, I965_GDRST,
658 GRDOM_MEDIA | GRDOM_RESET_ENABLE);
659
660 ret = wait_for(i965_reset_complete(dev), 500);
661 if (ret)
662 return ret;
663
664 pci_write_config_byte(dev->pdev, I965_GDRST, 0);
665
666 return 0;
667}
668
669static int ironlake_do_reset(struct drm_device *dev)
670{
671 struct drm_i915_private *dev_priv = dev->dev_private;
672 u32 gdrst;
673 int ret;
674
675 gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
676 gdrst &= ~GRDOM_MASK;
677 I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
678 gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE);
679 ret = wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
680 if (ret)
681 return ret;
682
683 /* We can't reset render&media without also resetting display ... */
684 gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
685 gdrst &= ~GRDOM_MASK;
686 I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
687 gdrst | GRDOM_MEDIA | GRDOM_RESET_ENABLE);
688 return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
689}
690
691static int gen6_do_reset(struct drm_device *dev)
692{
693 struct drm_i915_private *dev_priv = dev->dev_private;
694 int ret;
695 unsigned long irqflags;
696
697 /* Hold uncore.lock across reset to prevent any register access
698 * with forcewake not set correctly
699 */
700 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
701
702 /* Reset the chip */
703
704 /* GEN6_GDRST is not in the gt power well, no need to check
705 * for fifo space for the write or forcewake the chip for
706 * the read
707 */
6af5d92f 708 __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL);
907b28c5
CW		 709
710 /* Spin waiting for the device to ack the reset request */
6af5d92f 711 ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
907b28c5 712
521198a2
MK		 713 intel_uncore_forcewake_reset(dev);
714
907b28c5
CW		 715 /* If reset with a user forcewake, try to restore, otherwise turn it off */
716 if (dev_priv->uncore.forcewake_count)
717 dev_priv->uncore.funcs.force_wake_get(dev_priv);
718 else
719 dev_priv->uncore.funcs.force_wake_put(dev_priv);
720
721 /* Restore fifo count */
6af5d92f 722 dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GT_FIFO_FREE_ENTRIES);
907b28c5
CW		 723
724 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
725 return ret;
726}
727
728int intel_gpu_reset(struct drm_device *dev)
729{
730 switch (INTEL_INFO(dev)->gen) {
731 case 7:
732 case 6: return gen6_do_reset(dev);
733 case 5: return ironlake_do_reset(dev);
734 case 4: return i965_do_reset(dev);
907b28c5
CW		 735 default: return -ENODEV;
736 }
737}
738
739void intel_uncore_clear_errors(struct drm_device *dev)
740{
741 struct drm_i915_private *dev_priv = dev->dev_private;
742
6af5d92f 743 /* XXX needs spinlock around caller's grouping */
907b28c5 744 if (HAS_FPGA_DBG_UNCLAIMED(dev))
6af5d92f 745 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
907b28c5
CW		 746}
747
748void intel_uncore_check_errors(struct drm_device *dev)
749{
750 struct drm_i915_private *dev_priv = dev->dev_private;
751
752 if (HAS_FPGA_DBG_UNCLAIMED(dev) &&
6af5d92f 753 (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
907b28c5 754 DRM_ERROR("Unclaimed register before interrupt\n");
6af5d92f 755 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
907b28c5
CW		 756 }
757}
Linux 6.x block layer and io_uring tree(s)