Commit | Line | Data |
---|---|---|
920cf419 | 1 | /* |
10be98a7 | 2 | * SPDX-License-Identifier: MIT |
920cf419 | 3 | * |
10be98a7 | 4 | * Copyright © 2014-2016 Intel Corporation |
920cf419 CW |
5 | */ |
6 | ||
10be98a7 CW |
7 | #include <linux/scatterlist.h> |
8 | #include <linux/slab.h> | |
9 | #include <linux/swiotlb.h> | |
10 | ||
920cf419 | 11 | #include "i915_drv.h" |
10be98a7 CW |
12 | #include "i915_gem.h" |
13 | #include "i915_gem_object.h" | |
37d63f8f | 14 | #include "i915_scatterlist.h" |
10be98a7 | 15 | #include "i915_utils.h" |
920cf419 CW |
16 | |
17 | #define QUIET (__GFP_NORETRY | __GFP_NOWARN) | |
ee2202d7 | 18 | #define MAYFAIL (__GFP_RETRY_MAYFAIL | __GFP_NOWARN) |
920cf419 | 19 | |
920cf419 CW |
20 | static void internal_free_pages(struct sg_table *st) |
21 | { | |
22 | struct scatterlist *sg; | |
23 | ||
4703b047 CW |
24 | for (sg = st->sgl; sg; sg = __sg_next(sg)) { |
25 | if (sg_page(sg)) | |
26 | __free_pages(sg_page(sg), get_order(sg->length)); | |
27 | } | |
920cf419 CW |
28 | |
29 | sg_free_table(st); | |
30 | kfree(st); | |
31 | } | |
32 | ||
b91b09ee | 33 | static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) |
920cf419 CW |
34 | { |
35 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
920cf419 CW |
36 | struct sg_table *st; |
37 | struct scatterlist *sg; | |
84e8978e | 38 | unsigned int sg_page_sizes; |
bb96dcf5 | 39 | unsigned int npages; |
920cf419 CW |
40 | int max_order; |
41 | gfp_t gfp; | |
42 | ||
920cf419 CW |
43 | max_order = MAX_ORDER; |
44 | #ifdef CONFIG_SWIOTLB | |
5584f1b1 JG |
45 | if (swiotlb_nr_tbl()) { |
46 | unsigned int max_segment; | |
47 | ||
48 | max_segment = swiotlb_max_segment(); | |
49 | if (max_segment) { | |
50 | max_segment = max_t(unsigned int, max_segment, | |
51 | PAGE_SIZE) >> PAGE_SHIFT; | |
52 | max_order = min(max_order, ilog2(max_segment)); | |
53 | } | |
54 | } | |
920cf419 CW |
55 | #endif |
56 | ||
57 | gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; | |
c0f86832 | 58 | if (IS_I965GM(i915) || IS_I965G(i915)) { |
920cf419 CW |
59 | /* 965gm cannot relocate objects above 4GiB. */ |
60 | gfp &= ~__GFP_HIGHMEM; | |
61 | gfp |= __GFP_DMA32; | |
62 | } | |
63 | ||
bb96dcf5 CW |
64 | create_st: |
65 | st = kmalloc(sizeof(*st), GFP_KERNEL); | |
66 | if (!st) | |
b91b09ee | 67 | return -ENOMEM; |
bb96dcf5 CW |
68 | |
69 | npages = obj->base.size / PAGE_SIZE; | |
70 | if (sg_alloc_table(st, npages, GFP_KERNEL)) { | |
71 | kfree(st); | |
b91b09ee | 72 | return -ENOMEM; |
bb96dcf5 CW |
73 | } |
74 | ||
75 | sg = st->sgl; | |
76 | st->nents = 0; | |
84e8978e | 77 | sg_page_sizes = 0; |
bb96dcf5 | 78 | |
920cf419 CW |
79 | do { |
80 | int order = min(fls(npages) - 1, max_order); | |
81 | struct page *page; | |
82 | ||
83 | do { | |
ee2202d7 CW |
84 | page = alloc_pages(gfp | (order ? QUIET : MAYFAIL), |
85 | order); | |
920cf419 CW |
86 | if (page) |
87 | break; | |
88 | if (!order--) | |
89 | goto err; | |
90 | ||
91 | /* Limit subsequent allocations as well */ | |
92 | max_order = order; | |
93 | } while (1); | |
94 | ||
95 | sg_set_page(sg, page, PAGE_SIZE << order, 0); | |
84e8978e | 96 | sg_page_sizes |= PAGE_SIZE << order; |
920cf419 CW |
97 | st->nents++; |
98 | ||
99 | npages -= 1 << order; | |
100 | if (!npages) { | |
101 | sg_mark_end(sg); | |
102 | break; | |
103 | } | |
104 | ||
105 | sg = __sg_next(sg); | |
106 | } while (1); | |
920cf419 | 107 | |
bb96dcf5 CW |
108 | if (i915_gem_gtt_prepare_pages(obj, st)) { |
109 | /* Failed to dma-map try again with single page sg segments */ | |
110 | if (get_order(st->sgl->length)) { | |
111 | internal_free_pages(st); | |
112 | max_order = 0; | |
113 | goto create_st; | |
114 | } | |
920cf419 | 115 | goto err; |
bb96dcf5 | 116 | } |
920cf419 | 117 | |
84e8978e | 118 | __i915_gem_object_set_pages(obj, st, sg_page_sizes); |
b91b09ee MA |
119 | |
120 | return 0; | |
920cf419 CW |
121 | |
122 | err: | |
4703b047 | 123 | sg_set_page(sg, NULL, 0, 0); |
920cf419 CW |
124 | sg_mark_end(sg); |
125 | internal_free_pages(st); | |
b91b09ee MA |
126 | |
127 | return -ENOMEM; | |
920cf419 CW |
128 | } |
129 | ||
03ac84f1 CW |
130 | static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj, |
131 | struct sg_table *pages) | |
920cf419 | 132 | { |
03ac84f1 CW |
133 | i915_gem_gtt_finish_pages(obj, pages); |
134 | internal_free_pages(pages); | |
920cf419 | 135 | |
a4f5ea64 | 136 | obj->mm.dirty = false; |
920cf419 CW |
137 | } |
138 | ||
139 | static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { | |
7d192daa | 140 | .name = "i915_gem_object_internal", |
3599a91c TU |
141 | .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | |
142 | I915_GEM_OBJECT_IS_SHRINKABLE, | |
920cf419 CW |
143 | .get_pages = i915_gem_object_get_pages_internal, |
144 | .put_pages = i915_gem_object_put_pages_internal, | |
145 | }; | |
146 | ||
147 | /** | |
40f46095 CW |
148 | * i915_gem_object_create_internal: create an object with volatile pages |
149 | * @i915: the i915 device | |
150 | * @size: the size in bytes of backing storage to allocate for the object | |
151 | * | |
920cf419 CW |
152 | * Creates a new object that wraps some internal memory for private use. |
153 | * This object is not backed by swappable storage, and as such its contents | |
154 | * are volatile and only valid whilst pinned. If the object is reaped by the | |
155 | * shrinker, its pages and data will be discarded. Equally, it is not a full | |
156 | * GEM object and so not valid for access from userspace. This makes it useful | |
157 | * for hardware interfaces like ringbuffers (which are pinned from the time | |
158 | * the request is written to the time the hardware stops accessing it), but | |
159 | * not for contexts (which need to be preserved when not active for later | |
160 | * reuse). Note that it is not cleared upon allocation. | |
161 | */ | |
162 | struct drm_i915_gem_object * | |
163 | i915_gem_object_create_internal(struct drm_i915_private *i915, | |
fcd46e53 | 164 | phys_addr_t size) |
920cf419 | 165 | { |
7867d709 | 166 | static struct lock_class_key lock_class; |
920cf419 | 167 | struct drm_i915_gem_object *obj; |
b8f55be6 | 168 | unsigned int cache_level; |
920cf419 | 169 | |
fcd46e53 | 170 | GEM_BUG_ON(!size); |
bf6b2030 | 171 | GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); |
fcd46e53 CW |
172 | |
173 | if (overflows_type(size, obj->base.size)) | |
174 | return ERR_PTR(-E2BIG); | |
175 | ||
13f1bfd3 | 176 | obj = i915_gem_object_alloc(); |
920cf419 CW |
177 | if (!obj) |
178 | return ERR_PTR(-ENOMEM); | |
179 | ||
180 | drm_gem_private_object_init(&i915->drm, &obj->base, size); | |
7867d709 | 181 | i915_gem_object_init(obj, &i915_gem_object_internal_ops, &lock_class); |
920cf419 | 182 | |
7c98501a MA |
183 | /* |
184 | * Mark the object as volatile, such that the pages are marked as | |
185 | * dontneed whilst they are still pinned. As soon as they are unpinned | |
186 | * they are allowed to be reaped by the shrinker, and the caller is | |
187 | * expected to repopulate - the contents of this object are only valid | |
188 | * whilst active and pinned. | |
189 | */ | |
190 | i915_gem_object_set_volatile(obj); | |
191 | ||
c0a51fd0 CK |
192 | obj->read_domains = I915_GEM_DOMAIN_CPU; |
193 | obj->write_domain = I915_GEM_DOMAIN_CPU; | |
b8f55be6 CW |
194 | |
195 | cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; | |
196 | i915_gem_object_set_cache_coherency(obj, cache_level); | |
920cf419 CW |
197 | |
198 | return obj; | |
199 | } |