[linux-block.git] / drivers / gpu / drm / i915 / i915_gem_tiling.c

/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

#include <linux/string.h>
#include <linux/bitops.h>
#include <drm/drmP.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"

/**
 * DOC: buffer object tiling
 *
 * i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace
 * interface to declare fence register requirements.
 *
 * In principle GEM doesn't care at all about the internal data layout of an
 * object, and hence it also doesn't care about tiling or swizzling. There's two
 * exceptions:
 *
 * - For X and Y tiling the hardware provides detilers for CPU access, so called
 *   fences. Since there's only a limited amount of them the kernel must manage
 *   these, and therefore userspace must tell the kernel the object tiling if it
 *   wants to use fences for detiling.
 * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which
 *   depends upon the physical page frame number. When swapping such objects the
 *   page frame number might change and the kernel must be able to fix this up
 *   and hence now the tiling. Note that on a subset of platforms with
 *   asymmetric memory channel population the swizzling pattern changes in an
 *   unknown way, and for those the kernel simply forbids swapping completely.
 *
 * Since neither of this applies for new tiling layouts on modern platforms like
 * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled.
 * Anything else can be handled in userspace entirely without the kernel's
 * invovlement.
 */

/**
 * i915_gem_fence_size - required global GTT size for a fence
 * @i915: i915 device
 * @size: object size
 * @tiling: tiling mode
 * @stride: tiling stride
 *
 * Return the required global GTT size for a fence (view of a tiled object),
 * taking into account potential fence register mapping.
 */
u32 i915_gem_fence_size(struct drm_i915_private *i915,
			u32 size, unsigned int tiling, unsigned int stride)
{
	u32 ggtt_size;

	GEM_BUG_ON(!size);

	if (tiling == I915_TILING_NONE)
		return size;

	GEM_BUG_ON(!stride);

	if (INTEL_GEN(i915) >= 4) {
		stride *= i915_gem_tile_height(tiling);
		GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE));
		return roundup(size, stride);
	}

	/* Previous chips need a power-of-two fence region when tiling */
	if (IS_GEN3(i915))
		ggtt_size = 1024*1024;
	else
		ggtt_size = 512*1024;

	while (ggtt_size < size)
		ggtt_size <<= 1;

	return ggtt_size;
}

/**
 * i915_gem_fence_alignment - required global GTT alignment for a fence
 * @i915: i915 device
 * @size: object size
 * @tiling: tiling mode
 * @stride: tiling stride
 *
 * Return the required global GTT alignment for a fence (a view of a tiled
 * object), taking into account potential fence register mapping.
 */
u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size,
			     unsigned int tiling, unsigned int stride)
{
	GEM_BUG_ON(!size);

	/*
	 * Minimum alignment is 4k (GTT page size), but might be greater
	 * if a fence register is needed for the object.
	 */
	if (tiling == I915_TILING_NONE)
		return I915_GTT_MIN_ALIGNMENT;

	if (INTEL_GEN(i915) >= 4)
		return I965_FENCE_PAGE;

	/*
	 * Previous chips need to be aligned to the size of the smallest
	 * fence register that can contain the object.
	 */
	return i915_gem_fence_size(i915, size, tiling, stride);
}

/* Check pitch constriants for all chips & tiling formats */
static bool
i915_tiling_ok(struct drm_i915_gem_object *obj,
	       unsigned int tiling, unsigned int stride)
{
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	unsigned int tile_width;

	/* Linear is always fine */
	if (tiling == I915_TILING_NONE)
		return true;

	if (tiling > I915_TILING_LAST)
		return false;

	/* check maximum stride & object size */
	/* i965+ stores the end address of the gtt mapping in the fence
	 * reg, so dont bother to check the size */
	if (INTEL_GEN(i915) >= 7) {
		if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
			return false;
	} else if (INTEL_GEN(i915) >= 4) {
		if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
			return false;
	} else {
		if (stride > 8192)
			return false;

		if (!is_power_of_2(stride))
			return false;
	}

	if (IS_GEN2(i915) ||
	    (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915)))
		tile_width = 128;
	else
		tile_width = 512;

	if (!stride || !IS_ALIGNED(stride, tile_width))
		return false;

	return true;
}

static bool i915_vma_fence_prepare(struct i915_vma *vma,
				   int tiling_mode, unsigned int stride)
{
	struct drm_i915_private *i915 = vma->vm->i915;
	u32 size, alignment;

	if (!i915_vma_is_map_and_fenceable(vma))
		return true;

	size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride);
	if (vma->node.size < size)
		return false;

	alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride);
	if (!IS_ALIGNED(vma->node.start, alignment))
		return false;

	return true;
}

/* Make the current GTT allocation valid for the change in tiling. */
static int
i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
			      int tiling_mode, unsigned int stride)
{
	struct i915_vma *vma;
	int ret;

	if (tiling_mode == I915_TILING_NONE)
		return 0;

	list_for_each_entry(vma, &obj->vma_list, obj_link) {
		if (!i915_vma_is_ggtt(vma))
			break;

		if (i915_vma_fence_prepare(vma, tiling_mode, stride))
			continue;

		ret = i915_vma_unbind(vma);
		if (ret)
			return ret;
	}

	return 0;
}

int
i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
			   unsigned int tiling, unsigned int stride)
{
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	struct i915_vma *vma;
	int err;

	/* Make sure we don't cross-contaminate obj->tiling_and_stride */
	BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);

	GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
	GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
	lockdep_assert_held(&i915->drm.struct_mutex);

	if ((tiling | stride) == obj->tiling_and_stride)
		return 0;

	if (i915_gem_object_is_framebuffer(obj))
		return -EBUSY;

	/* We need to rebind the object if its current allocation
	 * no longer meets the alignment restrictions for its new
	 * tiling mode. Otherwise we can just leave it alone, but
	 * need to ensure that any fence register is updated before
	 * the next fenced (either through the GTT or by the BLT unit
	 * on older GPUs) access.
	 *
	 * After updating the tiling parameters, we then flag whether
	 * we need to update an associated fence register. Note this
	 * has to also include the unfenced register the GPU uses
	 * whilst executing a fenced command for an untiled object.
	 */

	err = i915_gem_object_fence_prepare(obj, tiling, stride);
	if (err)
		return err;

	i915_gem_object_lock(obj);
	if (i915_gem_object_is_framebuffer(obj)) {
		i915_gem_object_unlock(obj);
		return -EBUSY;
	}

	/* If the memory has unknown (i.e. varying) swizzling, we pin the
	 * pages to prevent them being swapped out and causing corruption
	 * due to the change in swizzling.
	 */
	mutex_lock(&obj->mm.lock);
	if (obj->mm.pages &&
	    obj->mm.madv == I915_MADV_WILLNEED &&
	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
		if (tiling == I915_TILING_NONE) {
			GEM_BUG_ON(!obj->mm.quirked);
			__i915_gem_object_unpin_pages(obj);
			obj->mm.quirked = false;
		}
		if (!i915_gem_object_is_tiled(obj)) {
			GEM_BUG_ON(!obj->mm.quirked);
			__i915_gem_object_pin_pages(obj);
			obj->mm.quirked = true;
		}
	}
	mutex_unlock(&obj->mm.lock);

	list_for_each_entry(vma, &obj->vma_list, obj_link) {
		if (!i915_vma_is_ggtt(vma))
			break;

		vma->fence_size =
			i915_gem_fence_size(i915, vma->size, tiling, stride);
		vma->fence_alignment =
			i915_gem_fence_alignment(i915,
						 vma->size, tiling, stride);

		if (vma->fence)
			vma->fence->dirty = true;
	}

	obj->tiling_and_stride = tiling | stride;
	i915_gem_object_unlock(obj);

	/* Force the fence to be reacquired for GTT access */
	i915_gem_release_mmap(obj);

	/* Try to preallocate memory required to save swizzling on put-pages */
	if (i915_gem_object_needs_bit17_swizzle(obj)) {
		if (!obj->bit_17) {
			obj->bit_17 = kcalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT),
					      sizeof(long), GFP_KERNEL);
		}
	} else {
		kfree(obj->bit_17);
		obj->bit_17 = NULL;
	}

	return 0;
}

/**
 * i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode
 * @dev: DRM device
 * @data: data pointer for the ioctl
 * @file: DRM file for the ioctl call
 *
 * Sets the tiling mode of an object, returning the required swizzling of
 * bit 6 of addresses in the object.
 *
 * Called by the user via ioctl.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int
i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
			  struct drm_file *file)
{
	struct drm_i915_gem_set_tiling *args = data;
	struct drm_i915_gem_object *obj;
	int err;

	obj = i915_gem_object_lookup(file, args->handle);
	if (!obj)
		return -ENOENT;

	if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
		err = -EINVAL;
		goto err;
	}

	if (args->tiling_mode == I915_TILING_NONE) {
		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
		args->stride = 0;
	} else {
		if (args->tiling_mode == I915_TILING_X)
			args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_x;
		else
			args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_y;

		/* Hide bit 17 swizzling from the user.  This prevents old Mesa
		 * from aborting the application on sw fallbacks to bit 17,
		 * and we use the pread/pwrite bit17 paths to swizzle for it.
		 * If there was a user that was relying on the swizzle
		 * information for drm_intel_bo_map()ed reads/writes this would
		 * break it, but we don't have any of those.
		 */
		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
			args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
			args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;

		/* If we can't handle the swizzling, make it untiled. */
		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
			args->tiling_mode = I915_TILING_NONE;
			args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
			args->stride = 0;
		}
	}

	err = mutex_lock_interruptible(&dev->struct_mutex);
	if (err)
		goto err;

	err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
	mutex_unlock(&dev->struct_mutex);

	/* We have to maintain this existing ABI... */
	args->stride = i915_gem_object_get_stride(obj);
	args->tiling_mode = i915_gem_object_get_tiling(obj);

err:
	i915_gem_object_put(obj);
	return err;
}

/**
 * i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode
 * @dev: DRM device
 * @data: data pointer for the ioctl
 * @file: DRM file for the ioctl call
 *
 * Returns the current tiling mode and required bit 6 swizzling for the object.
 *
 * Called by the user via ioctl.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int
i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
			  struct drm_file *file)
{
	struct drm_i915_gem_get_tiling *args = data;
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct drm_i915_gem_object *obj;
	int err = -ENOENT;

	rcu_read_lock();
	obj = i915_gem_object_lookup_rcu(file, args->handle);
	if (obj) {
		args->tiling_mode =
			READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
		err = 0;
	}
	rcu_read_unlock();
	if (unlikely(err))
		return err;

	switch (args->tiling_mode) {
	case I915_TILING_X:
		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
		break;
	case I915_TILING_Y:
		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
		break;
	default:
	case I915_TILING_NONE:
		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
		break;
	}

	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
	if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
		args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
	else
		args->phys_swizzle_mode = args->swizzle_mode;
	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
		args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
		args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;

	return 0;
}
Commit	Line	Data
673a394b EA	1	/*
	2	* Copyright © 2008 Intel Corporation
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice (including the next
	12	* paragraph) shall be included in all copies or substantial portions of the
	13	* Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	21	* IN THE SOFTWARE.
	22	*
	23	* Authors:
	24	* Eric Anholt <eric@anholt.net>
	25	*
	26	*/
	27
760285e7 DH	28	#include <linux/string.h>
	29	#include <linux/bitops.h>
	30	#include <drm/drmP.h>
	31	#include <drm/i915_drm.h>
673a394b EA	32	#include "i915_drv.h"
673a394b EA	33
3271dca4 DV	34	/**
3271dca4 DV	35	* DOC: buffer object tiling
673a394b	36	*
111dbcab CW	37	* i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace
111dbcab CW	38	* interface to declare fence register requirements.
673a394b	39	*
3271dca4 DV	40	* In principle GEM doesn't care at all about the internal data layout of an
	41	* object, and hence it also doesn't care about tiling or swizzling. There's two
	42	* exceptions:
673a394b	43	*
3271dca4 DV	44	* - For X and Y tiling the hardware provides detilers for CPU access, so called
	45	* fences. Since there's only a limited amount of them the kernel must manage
	46	* these, and therefore userspace must tell the kernel the object tiling if it
	47	* wants to use fences for detiling.
	48	* - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which
	49	* depends upon the physical page frame number. When swapping such objects the
	50	* page frame number might change and the kernel must be able to fix this up
	51	* and hence now the tiling. Note that on a subset of platforms with
	52	* asymmetric memory channel population the swizzling pattern changes in an
	53	* unknown way, and for those the kernel simply forbids swapping completely.
673a394b	54	*
3271dca4 DV	55	* Since neither of this applies for new tiling layouts on modern platforms like
	56	* W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled.
	57	* Anything else can be handled in userspace entirely without the kernel's
	58	* invovlement.
673a394b EA	59	*/
673a394b EA	60
91d4e0aa CW	61	/**
	62	* i915_gem_fence_size - required global GTT size for a fence
	63	* @i915: i915 device
	64	* @size: object size
	65	* @tiling: tiling mode
	66	* @stride: tiling stride
	67	*
	68	* Return the required global GTT size for a fence (view of a tiled object),
	69	* taking into account potential fence register mapping.
	70	*/
	71	u32 i915_gem_fence_size(struct drm_i915_private *i915,
	72	u32 size, unsigned int tiling, unsigned int stride)
	73	{
	74	u32 ggtt_size;
	75
	76	GEM_BUG_ON(!size);
	77
	78	if (tiling == I915_TILING_NONE)
	79	return size;
	80
	81	GEM_BUG_ON(!stride);
	82
	83	if (INTEL_GEN(i915) >= 4) {
	84	stride *= i915_gem_tile_height(tiling);
f51455d4	85	GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE));
91d4e0aa CW	86	return roundup(size, stride);
	87	}
	88
	89	/* Previous chips need a power-of-two fence region when tiling */
	90	if (IS_GEN3(i915))
	91	ggtt_size = 1024*1024;
	92	else
	93	ggtt_size = 512*1024;
	94
	95	while (ggtt_size < size)
	96	ggtt_size <<= 1;
	97
	98	return ggtt_size;
	99	}
	100
	101	/**
	102	* i915_gem_fence_alignment - required global GTT alignment for a fence
	103	* @i915: i915 device
	104	* @size: object size
	105	* @tiling: tiling mode
	106	* @stride: tiling stride
	107	*
	108	* Return the required global GTT alignment for a fence (a view of a tiled
	109	* object), taking into account potential fence register mapping.
	110	*/
	111	u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size,
	112	unsigned int tiling, unsigned int stride)
	113	{
	114	GEM_BUG_ON(!size);
	115
	116	/*
	117	* Minimum alignment is 4k (GTT page size), but might be greater
	118	* if a fence register is needed for the object.
	119	*/
f51455d4 CW	120	if (tiling == I915_TILING_NONE)
	121	return I915_GTT_MIN_ALIGNMENT;
	122
	123	if (INTEL_GEN(i915) >= 4)
	124	return I965_FENCE_PAGE;
91d4e0aa CW	125
	126	/*
	127	* Previous chips need to be aligned to the size of the smallest
	128	* fence register that can contain the object.
	129	*/
	130	return i915_gem_fence_size(i915, size, tiling, stride);
	131	}
	132
0f973f27	133	/* Check pitch constriants for all chips & tiling formats */
a00b10c3	134	static bool
957870f9 CW	135	i915_tiling_ok(struct drm_i915_gem_object *obj,
957870f9 CW	136	unsigned int tiling, unsigned int stride)
0f973f27	137	{
957870f9 CW	138	struct drm_i915_private *i915 = to_i915(obj->base.dev);
957870f9 CW	139	unsigned int tile_width;
0f973f27 JB	140
0f973f27 JB	141	/* Linear is always fine */
957870f9	142	if (tiling == I915_TILING_NONE)
0f973f27 JB	143	return true;
0f973f27 JB	144
957870f9	145	if (tiling > I915_TILING_LAST)
deeb1519 CW	146	return false;
deeb1519 CW	147
8d7773a3	148	/* check maximum stride & object size */
3a062478 VS	149	/* i965+ stores the end address of the gtt mapping in the fence
3a062478 VS	150	* reg, so dont bother to check the size */
957870f9	151	if (INTEL_GEN(i915) >= 7) {
3a062478 VS	152	if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
3a062478 VS	153	return false;
957870f9	154	} else if (INTEL_GEN(i915) >= 4) {
8d7773a3 DV	155	if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
8d7773a3 DV	156	return false;
a6c45cf0	157	} else {
c36a2a6d	158	if (stride > 8192)
8d7773a3	159	return false;
e76a16de	160
a3a1e533 CW	161	if (!is_power_of_2(stride))
a3a1e533 CW	162	return false;
8d7773a3 DV	163	}
8d7773a3 DV	164
957870f9 CW	165	if (IS_GEN2(i915) \|\|
	166	(tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915)))
	167	tile_width = 128;
	168	else
	169	tile_width = 512;
	170
52da22e7	171	if (!stride \|\| !IS_ALIGNED(stride, tile_width))
fe48d8de VS	172	return false;
fe48d8de VS	173
a3a1e533	174	return true;
0f973f27 JB	175	}
0f973f27 JB	176
5b30694b CW	177	static bool i915_vma_fence_prepare(struct i915_vma *vma,
5b30694b CW	178	int tiling_mode, unsigned int stride)
49ef5294	179	{
944397f0 CW	180	struct drm_i915_private *i915 = vma->vm->i915;
944397f0 CW	181	u32 size, alignment;
49ef5294 CW	182
	183	if (!i915_vma_is_map_and_fenceable(vma))
	184	return true;
	185
91d4e0aa	186	size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride);
49ef5294 CW	187	if (vma->node.size < size)
	188	return false;
	189
91d4e0aa	190	alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride);
f51455d4	191	if (!IS_ALIGNED(vma->node.start, alignment))
49ef5294 CW	192	return false;
	193
	194	return true;
	195	}
	196
f23eda8c CW	197	/* Make the current GTT allocation valid for the change in tiling. */
f23eda8c CW	198	static int
5b30694b CW	199	i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
5b30694b CW	200	int tiling_mode, unsigned int stride)
52dc7d32	201	{
f23eda8c	202	struct i915_vma *vma;
49ef5294	203	int ret;
52dc7d32 CW	204
52dc7d32 CW	205	if (tiling_mode == I915_TILING_NONE)
f23eda8c	206	return 0;
52dc7d32	207
49ef5294	208	list_for_each_entry(vma, &obj->vma_list, obj_link) {
944397f0 CW	209	if (!i915_vma_is_ggtt(vma))
	210	break;
	211
5b30694b	212	if (i915_vma_fence_prepare(vma, tiling_mode, stride))
49ef5294	213	continue;
a6c45cf0	214
49ef5294 CW	215	ret = i915_vma_unbind(vma);
	216	if (ret)
	217	return ret;
df153158 CW	218	}
df153158 CW	219
f23eda8c	220	return 0;
52dc7d32 CW	221	}
52dc7d32 CW	222
957870f9 CW	223	int
	224	i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
	225	unsigned int tiling, unsigned int stride)
	226	{
	227	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	228	struct i915_vma *vma;
	229	int err;
	230
	231	/* Make sure we don't cross-contaminate obj->tiling_and_stride */
	232	BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);
	233
	234	GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
	235	GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
	236	lockdep_assert_held(&i915->drm.struct_mutex);
	237
	238	if ((tiling \| stride) == obj->tiling_and_stride)
	239	return 0;
	240
dd689287	241	if (i915_gem_object_is_framebuffer(obj))
957870f9 CW	242	return -EBUSY;
	243
	244	/* We need to rebind the object if its current allocation
	245	* no longer meets the alignment restrictions for its new
	246	* tiling mode. Otherwise we can just leave it alone, but
	247	* need to ensure that any fence register is updated before
	248	* the next fenced (either through the GTT or by the BLT unit
	249	* on older GPUs) access.
	250	*
	251	* After updating the tiling parameters, we then flag whether
	252	* we need to update an associated fence register. Note this
	253	* has to also include the unfenced register the GPU uses
	254	* whilst executing a fenced command for an untiled object.
	255	*/
	256
	257	err = i915_gem_object_fence_prepare(obj, tiling, stride);
	258	if (err)
	259	return err;
	260
dd689287 CW	261	i915_gem_object_lock(obj);
	262	if (i915_gem_object_is_framebuffer(obj)) {
	263	i915_gem_object_unlock(obj);
	264	return -EBUSY;
	265	}
	266
957870f9 CW	267	/* If the memory has unknown (i.e. varying) swizzling, we pin the
	268	* pages to prevent them being swapped out and causing corruption
	269	* due to the change in swizzling.
	270	*/
	271	mutex_lock(&obj->mm.lock);
	272	if (obj->mm.pages &&
	273	obj->mm.madv == I915_MADV_WILLNEED &&
	274	i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
	275	if (tiling == I915_TILING_NONE) {
	276	GEM_BUG_ON(!obj->mm.quirked);
	277	__i915_gem_object_unpin_pages(obj);
	278	obj->mm.quirked = false;
	279	}
	280	if (!i915_gem_object_is_tiled(obj)) {
	281	GEM_BUG_ON(!obj->mm.quirked);
	282	__i915_gem_object_pin_pages(obj);
	283	obj->mm.quirked = true;
	284	}
	285	}
	286	mutex_unlock(&obj->mm.lock);
	287
	288	list_for_each_entry(vma, &obj->vma_list, obj_link) {
	289	if (!i915_vma_is_ggtt(vma))
	290	break;
	291
	292	vma->fence_size =
	293	i915_gem_fence_size(i915, vma->size, tiling, stride);
	294	vma->fence_alignment =
	295	i915_gem_fence_alignment(i915,
	296	vma->size, tiling, stride);
	297
	298	if (vma->fence)
	299	vma->fence->dirty = true;
	300	}
	301
	302	obj->tiling_and_stride = tiling \| stride;
dd689287	303	i915_gem_object_unlock(obj);
957870f9 CW	304
	305	/* Force the fence to be reacquired for GTT access */
	306	i915_gem_release_mmap(obj);
	307
	308	/* Try to preallocate memory required to save swizzling on put-pages */
	309	if (i915_gem_object_needs_bit17_swizzle(obj)) {
	310	if (!obj->bit_17) {
	311	obj->bit_17 = kcalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT),
	312	sizeof(long), GFP_KERNEL);
	313	}
	314	} else {
	315	kfree(obj->bit_17);
	316	obj->bit_17 = NULL;
	317	}
	318
	319	return 0;
	320	}
	321
673a394b	322	/**
111dbcab	323	* i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode
3271dca4 DV	324	* @dev: DRM device
	325	* @data: data pointer for the ioctl
	326	* @file: DRM file for the ioctl call
	327	*
673a394b EA	328	* Sets the tiling mode of an object, returning the required swizzling of
673a394b EA	329	* bit 6 of addresses in the object.
3271dca4 DV	330	*
	331	* Called by the user via ioctl.
	332	*
	333	* Returns:
	334	* Zero on success, negative errno on failure.
673a394b EA	335	*/
673a394b EA	336	int
111dbcab CW	337	i915_gem_set_tiling_ioctl(struct drm_device dev, void data,
111dbcab CW	338	struct drm_file *file)
673a394b EA	339	{
673a394b EA	340	struct drm_i915_gem_set_tiling *args = data;
05394f39	341	struct drm_i915_gem_object *obj;
957870f9	342	int err;
3e510a8e	343
03ac0642 CW	344	obj = i915_gem_object_lookup(file, args->handle);
03ac0642 CW	345	if (!obj)
bf79cb91	346	return -ENOENT;
673a394b	347
957870f9 CW	348	if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
957870f9 CW	349	err = -EINVAL;
6c31a614	350	goto err;
31770bd4 DV	351	}
31770bd4 DV	352
673a394b	353	if (args->tiling_mode == I915_TILING_NONE) {
673a394b	354	args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
52dc7d32	355	args->stride = 0;
673a394b EA	356	} else {
673a394b EA	357	if (args->tiling_mode == I915_TILING_X)
957870f9	358	args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_x;
673a394b	359	else
957870f9	360	args->swizzle_mode = to_i915(dev)->mm.bit_6_swizzle_y;
280b713b EA	361
	362	/* Hide bit 17 swizzling from the user. This prevents old Mesa
	363	* from aborting the application on sw fallbacks to bit 17,
	364	* and we use the pread/pwrite bit17 paths to swizzle for it.
	365	* If there was a user that was relying on the swizzle
	366	* information for drm_intel_bo_map()ed reads/writes this would
	367	* break it, but we don't have any of those.
	368	*/
	369	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
	370	args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
	371	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
	372	args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
	373
673a394b EA	374	/* If we can't handle the swizzling, make it untiled. */
	375	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
	376	args->tiling_mode = I915_TILING_NONE;
	377	args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
52dc7d32	378	args->stride = 0;
673a394b EA	379	}
673a394b EA	380	}
0f973f27	381
957870f9 CW	382	err = mutex_lock_interruptible(&dev->struct_mutex);
	383	if (err)
	384	goto err;
467cffba	385
957870f9 CW	386	err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
	387	mutex_unlock(&dev->struct_mutex);
	388
	389	/* We have to maintain this existing ABI... */
3e510a8e CW	390	args->stride = i915_gem_object_get_stride(obj);
3e510a8e CW	391	args->tiling_mode = i915_gem_object_get_tiling(obj);
e9b73c67	392
6c31a614	393	err:
f8c417cd	394	i915_gem_object_put(obj);
f23eda8c	395	return err;
673a394b EA	396	}
	397
	398	/**
111dbcab	399	* i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode
3271dca4 DV	400	* @dev: DRM device
	401	* @data: data pointer for the ioctl
	402	* @file: DRM file for the ioctl call
	403	*
673a394b	404	* Returns the current tiling mode and required bit 6 swizzling for the object.
3271dca4 DV	405	*
	406	* Called by the user via ioctl.
	407	*
	408	* Returns:
	409	* Zero on success, negative errno on failure.
673a394b EA	410	*/
673a394b EA	411	int
111dbcab CW	412	i915_gem_get_tiling_ioctl(struct drm_device dev, void data,
111dbcab CW	413	struct drm_file *file)
673a394b EA	414	{
673a394b EA	415	struct drm_i915_gem_get_tiling *args = data;
fac5e23e	416	struct drm_i915_private *dev_priv = to_i915(dev);
05394f39	417	struct drm_i915_gem_object *obj;
fbbd37b3 CW	418	int err = -ENOENT;
	419
	420	rcu_read_lock();
	421	obj = i915_gem_object_lookup_rcu(file, args->handle);
	422	if (obj) {
	423	args->tiling_mode =
	424	READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
	425	err = 0;
	426	}
	427	rcu_read_unlock();
	428	if (unlikely(err))
	429	return err;
673a394b	430
9ad36761	431	switch (args->tiling_mode) {
673a394b EA	432	case I915_TILING_X:
	433	args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
	434	break;
	435	case I915_TILING_Y:
	436	args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
	437	break;
fbbd37b3	438	default:
673a394b EA	439	case I915_TILING_NONE:
	440	args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
	441	break;
673a394b EA	442	}
673a394b EA	443
280b713b	444	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
5eb3e5a5 CW	445	if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
	446	args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
	447	else
	448	args->phys_swizzle_mode = args->swizzle_mode;
280b713b EA	449	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
	450	args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
	451	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
	452	args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
	453
673a394b EA	454	return 0;
673a394b EA	455	}