Commit | Line | Data |
---|---|---|
920d2b5e BS |
1 | /* |
2 | * Copyright 2017 Red Hat Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | #include "uvmm.h" | |
23 | #include "umem.h" | |
24 | #include "ummu.h" | |
25 | ||
26 | #include <core/client.h> | |
27 | #include <core/memory.h> | |
28 | ||
29 | #include <nvif/if000c.h> | |
30 | #include <nvif/unpack.h> | |
31 | ||
32 | static const struct nvkm_object_func nvkm_uvmm; | |
33 | struct nvkm_vmm * | |
34 | nvkm_uvmm_search(struct nvkm_client *client, u64 handle) | |
35 | { | |
36 | struct nvkm_object *object; | |
37 | ||
38 | object = nvkm_object_search(client, handle, &nvkm_uvmm); | |
39 | if (IS_ERR(object)) | |
40 | return (void *)object; | |
41 | ||
42 | return nvkm_uvmm(object)->vmm; | |
43 | } | |
44 | ||
a5ff307f BS |
45 | static int |
46 | nvkm_uvmm_mthd_pfnclr(struct nvkm_uvmm *uvmm, void *argv, u32 argc) | |
47 | { | |
48 | struct nvkm_client *client = uvmm->object.client; | |
49 | union { | |
50 | struct nvif_vmm_pfnclr_v0 v0; | |
51 | } *args = argv; | |
52 | struct nvkm_vmm *vmm = uvmm->vmm; | |
53 | int ret = -ENOSYS; | |
54 | u64 addr, size; | |
55 | ||
56 | if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { | |
57 | addr = args->v0.addr; | |
58 | size = args->v0.size; | |
59 | } else | |
60 | return ret; | |
61 | ||
62 | if (!client->super) | |
63 | return -ENOENT; | |
64 | ||
65 | if (size) { | |
66 | mutex_lock(&vmm->mutex); | |
67 | ret = nvkm_vmm_pfn_unmap(vmm, addr, size); | |
68 | mutex_unlock(&vmm->mutex); | |
69 | } | |
70 | ||
71 | return ret; | |
72 | } | |
73 | ||
74 | static int | |
75 | nvkm_uvmm_mthd_pfnmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) | |
76 | { | |
77 | struct nvkm_client *client = uvmm->object.client; | |
78 | union { | |
79 | struct nvif_vmm_pfnmap_v0 v0; | |
80 | } *args = argv; | |
81 | struct nvkm_vmm *vmm = uvmm->vmm; | |
82 | int ret = -ENOSYS; | |
83 | u64 addr, size, *phys; | |
84 | u8 page; | |
85 | ||
86 | if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { | |
87 | page = args->v0.page; | |
88 | addr = args->v0.addr; | |
89 | size = args->v0.size; | |
90 | phys = args->v0.phys; | |
91 | if (argc != (size >> page) * sizeof(args->v0.phys[0])) | |
92 | return -EINVAL; | |
93 | } else | |
94 | return ret; | |
95 | ||
96 | if (!client->super) | |
97 | return -ENOENT; | |
98 | ||
99 | if (size) { | |
100 | mutex_lock(&vmm->mutex); | |
101 | ret = nvkm_vmm_pfn_map(vmm, page, addr, size, phys); | |
102 | mutex_unlock(&vmm->mutex); | |
103 | } | |
104 | ||
105 | return ret; | |
106 | } | |
107 | ||
920d2b5e BS |
108 | static int |
109 | nvkm_uvmm_mthd_unmap(struct nvkm_uvmm *uvmm, void *argv, u32 argc) | |
110 | { | |
111 | struct nvkm_client *client = uvmm->object.client; | |
112 | union { | |
113 | struct nvif_vmm_unmap_v0 v0; | |
114 | } *args = argv; | |
115 | struct nvkm_vmm *vmm = uvmm->vmm; | |
116 | struct nvkm_vma *vma; | |
117 | int ret = -ENOSYS; | |
118 | u64 addr; | |
119 | ||
120 | if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { | |
121 | addr = args->v0.addr; | |
122 | } else | |
123 | return ret; | |
124 | ||
125 | mutex_lock(&vmm->mutex); | |
126 | vma = nvkm_vmm_node_search(vmm, addr); | |
127 | if (ret = -ENOENT, !vma || vma->addr != addr) { | |
128 | VMM_DEBUG(vmm, "lookup %016llx: %016llx", | |
129 | addr, vma ? vma->addr : ~0ULL); | |
130 | goto done; | |
131 | } | |
132 | ||
133 | if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { | |
134 | VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, | |
135 | vma->user, !client->super, vma->busy); | |
136 | goto done; | |
137 | } | |
138 | ||
139 | if (ret = -EINVAL, !vma->memory) { | |
140 | VMM_DEBUG(vmm, "unmapped"); | |
141 | goto done; | |
142 | } | |
143 | ||
a5ff307f | 144 | nvkm_vmm_unmap_locked(vmm, vma, false); |
920d2b5e BS |
145 | ret = 0; |
146 | done: | |
147 | mutex_unlock(&vmm->mutex); | |
148 | return ret; | |
149 | } | |
150 | ||
151 | static int | |
152 | nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc) | |
153 | { | |
154 | struct nvkm_client *client = uvmm->object.client; | |
155 | union { | |
156 | struct nvif_vmm_map_v0 v0; | |
157 | } *args = argv; | |
158 | u64 addr, size, handle, offset; | |
159 | struct nvkm_vmm *vmm = uvmm->vmm; | |
160 | struct nvkm_vma *vma; | |
161 | struct nvkm_memory *memory; | |
162 | int ret = -ENOSYS; | |
163 | ||
164 | if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, true))) { | |
165 | addr = args->v0.addr; | |
166 | size = args->v0.size; | |
167 | handle = args->v0.memory; | |
168 | offset = args->v0.offset; | |
169 | } else | |
170 | return ret; | |
171 | ||
94ee54dc CB |
172 | memory = nvkm_umem_search(client, handle); |
173 | if (IS_ERR(memory)) { | |
920d2b5e BS |
174 | VMM_DEBUG(vmm, "memory %016llx %ld\n", handle, PTR_ERR(memory)); |
175 | return PTR_ERR(memory); | |
176 | } | |
177 | ||
178 | mutex_lock(&vmm->mutex); | |
179 | if (ret = -ENOENT, !(vma = nvkm_vmm_node_search(vmm, addr))) { | |
180 | VMM_DEBUG(vmm, "lookup %016llx", addr); | |
181 | goto fail; | |
182 | } | |
183 | ||
184 | if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { | |
185 | VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, | |
186 | vma->user, !client->super, vma->busy); | |
187 | goto fail; | |
188 | } | |
189 | ||
a5ff307f BS |
190 | if (ret = -EINVAL, vma->mapped && !vma->memory) { |
191 | VMM_DEBUG(vmm, "pfnmap %016llx", addr); | |
192 | goto fail; | |
193 | } | |
194 | ||
920d2b5e BS |
195 | if (ret = -EINVAL, vma->addr != addr || vma->size != size) { |
196 | if (addr + size > vma->addr + vma->size || vma->memory || | |
197 | (vma->refd == NVKM_VMA_PAGE_NONE && !vma->mapref)) { | |
198 | VMM_DEBUG(vmm, "split %d %d %d " | |
199 | "%016llx %016llx %016llx %016llx", | |
200 | !!vma->memory, vma->refd, vma->mapref, | |
201 | addr, size, vma->addr, (u64)vma->size); | |
202 | goto fail; | |
203 | } | |
204 | ||
729eba33 BS |
205 | vma = nvkm_vmm_node_split(vmm, vma, addr, size); |
206 | if (!vma) { | |
207 | ret = -ENOMEM; | |
208 | goto fail; | |
920d2b5e BS |
209 | } |
210 | } | |
211 | vma->busy = true; | |
212 | mutex_unlock(&vmm->mutex); | |
213 | ||
214 | ret = nvkm_memory_map(memory, offset, vmm, vma, argv, argc); | |
215 | if (ret == 0) { | |
216 | /* Successful map will clear vma->busy. */ | |
217 | nvkm_memory_unref(&memory); | |
218 | return 0; | |
219 | } | |
220 | ||
221 | mutex_lock(&vmm->mutex); | |
222 | vma->busy = false; | |
223 | nvkm_vmm_unmap_region(vmm, vma); | |
224 | fail: | |
225 | mutex_unlock(&vmm->mutex); | |
226 | nvkm_memory_unref(&memory); | |
227 | return ret; | |
228 | } | |
229 | ||
230 | static int | |
231 | nvkm_uvmm_mthd_put(struct nvkm_uvmm *uvmm, void *argv, u32 argc) | |
232 | { | |
233 | struct nvkm_client *client = uvmm->object.client; | |
234 | union { | |
235 | struct nvif_vmm_put_v0 v0; | |
236 | } *args = argv; | |
237 | struct nvkm_vmm *vmm = uvmm->vmm; | |
238 | struct nvkm_vma *vma; | |
239 | int ret = -ENOSYS; | |
240 | u64 addr; | |
241 | ||
242 | if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { | |
243 | addr = args->v0.addr; | |
244 | } else | |
245 | return ret; | |
246 | ||
247 | mutex_lock(&vmm->mutex); | |
248 | vma = nvkm_vmm_node_search(vmm, args->v0.addr); | |
249 | if (ret = -ENOENT, !vma || vma->addr != addr || vma->part) { | |
250 | VMM_DEBUG(vmm, "lookup %016llx: %016llx %d", addr, | |
251 | vma ? vma->addr : ~0ULL, vma ? vma->part : 0); | |
252 | goto done; | |
253 | } | |
254 | ||
255 | if (ret = -ENOENT, (!vma->user && !client->super) || vma->busy) { | |
256 | VMM_DEBUG(vmm, "denied %016llx: %d %d %d", addr, | |
257 | vma->user, !client->super, vma->busy); | |
258 | goto done; | |
259 | } | |
260 | ||
261 | nvkm_vmm_put_locked(vmm, vma); | |
262 | ret = 0; | |
263 | done: | |
264 | mutex_unlock(&vmm->mutex); | |
265 | return ret; | |
266 | } | |
267 | ||
268 | static int | |
269 | nvkm_uvmm_mthd_get(struct nvkm_uvmm *uvmm, void *argv, u32 argc) | |
270 | { | |
271 | struct nvkm_client *client = uvmm->object.client; | |
272 | union { | |
273 | struct nvif_vmm_get_v0 v0; | |
274 | } *args = argv; | |
275 | struct nvkm_vmm *vmm = uvmm->vmm; | |
276 | struct nvkm_vma *vma; | |
277 | int ret = -ENOSYS; | |
278 | bool getref, mapref, sparse; | |
279 | u8 page, align; | |
280 | u64 size; | |
281 | ||
282 | if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { | |
283 | getref = args->v0.type == NVIF_VMM_GET_V0_PTES; | |
284 | mapref = args->v0.type == NVIF_VMM_GET_V0_ADDR; | |
285 | sparse = args->v0.sparse; | |
286 | page = args->v0.page; | |
287 | align = args->v0.align; | |
288 | size = args->v0.size; | |
289 | } else | |
290 | return ret; | |
291 | ||
292 | mutex_lock(&vmm->mutex); | |
293 | ret = nvkm_vmm_get_locked(vmm, getref, mapref, sparse, | |
294 | page, align, size, &vma); | |
295 | mutex_unlock(&vmm->mutex); | |
296 | if (ret) | |
297 | return ret; | |
298 | ||
299 | args->v0.addr = vma->addr; | |
300 | vma->user = !client->super; | |
301 | return ret; | |
302 | } | |
303 | ||
304 | static int | |
305 | nvkm_uvmm_mthd_page(struct nvkm_uvmm *uvmm, void *argv, u32 argc) | |
306 | { | |
307 | union { | |
308 | struct nvif_vmm_page_v0 v0; | |
309 | } *args = argv; | |
310 | const struct nvkm_vmm_page *page; | |
311 | int ret = -ENOSYS; | |
312 | u8 type, index, nr; | |
313 | ||
314 | page = uvmm->vmm->func->page; | |
315 | for (nr = 0; page[nr].shift; nr++); | |
316 | ||
317 | if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) { | |
318 | if ((index = args->v0.index) >= nr) | |
319 | return -EINVAL; | |
320 | type = page[index].type; | |
321 | args->v0.shift = page[index].shift; | |
322 | args->v0.sparse = !!(type & NVKM_VMM_PAGE_SPARSE); | |
323 | args->v0.vram = !!(type & NVKM_VMM_PAGE_VRAM); | |
324 | args->v0.host = !!(type & NVKM_VMM_PAGE_HOST); | |
325 | args->v0.comp = !!(type & NVKM_VMM_PAGE_COMP); | |
326 | } else | |
327 | return -ENOSYS; | |
328 | ||
329 | return 0; | |
330 | } | |
331 | ||
332 | static int | |
333 | nvkm_uvmm_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) | |
334 | { | |
335 | struct nvkm_uvmm *uvmm = nvkm_uvmm(object); | |
336 | switch (mthd) { | |
337 | case NVIF_VMM_V0_PAGE : return nvkm_uvmm_mthd_page (uvmm, argv, argc); | |
338 | case NVIF_VMM_V0_GET : return nvkm_uvmm_mthd_get (uvmm, argv, argc); | |
339 | case NVIF_VMM_V0_PUT : return nvkm_uvmm_mthd_put (uvmm, argv, argc); | |
340 | case NVIF_VMM_V0_MAP : return nvkm_uvmm_mthd_map (uvmm, argv, argc); | |
341 | case NVIF_VMM_V0_UNMAP : return nvkm_uvmm_mthd_unmap (uvmm, argv, argc); | |
a5ff307f BS |
342 | case NVIF_VMM_V0_PFNMAP: return nvkm_uvmm_mthd_pfnmap(uvmm, argv, argc); |
343 | case NVIF_VMM_V0_PFNCLR: return nvkm_uvmm_mthd_pfnclr(uvmm, argv, argc); | |
71871aa6 BS |
344 | case NVIF_VMM_V0_MTHD(0x00) ... NVIF_VMM_V0_MTHD(0x7f): |
345 | if (uvmm->vmm->func->mthd) { | |
346 | return uvmm->vmm->func->mthd(uvmm->vmm, | |
347 | uvmm->object.client, | |
348 | mthd, argv, argc); | |
349 | } | |
350 | break; | |
920d2b5e BS |
351 | default: |
352 | break; | |
353 | } | |
354 | return -EINVAL; | |
355 | } | |
356 | ||
357 | static void * | |
358 | nvkm_uvmm_dtor(struct nvkm_object *object) | |
359 | { | |
360 | struct nvkm_uvmm *uvmm = nvkm_uvmm(object); | |
361 | nvkm_vmm_unref(&uvmm->vmm); | |
362 | return uvmm; | |
363 | } | |
364 | ||
365 | static const struct nvkm_object_func | |
366 | nvkm_uvmm = { | |
367 | .dtor = nvkm_uvmm_dtor, | |
368 | .mthd = nvkm_uvmm_mthd, | |
369 | }; | |
370 | ||
371 | int | |
372 | nvkm_uvmm_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, | |
373 | struct nvkm_object **pobject) | |
374 | { | |
375 | struct nvkm_mmu *mmu = nvkm_ummu(oclass->parent)->mmu; | |
376 | const bool more = oclass->base.maxver >= 0; | |
377 | union { | |
378 | struct nvif_vmm_v0 v0; | |
379 | } *args = argv; | |
380 | const struct nvkm_vmm_page *page; | |
381 | struct nvkm_uvmm *uvmm; | |
382 | int ret = -ENOSYS; | |
383 | u64 addr, size; | |
2606f291 | 384 | bool managed; |
920d2b5e BS |
385 | |
386 | if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, more))) { | |
2606f291 | 387 | managed = args->v0.managed != 0; |
920d2b5e BS |
388 | addr = args->v0.addr; |
389 | size = args->v0.size; | |
390 | } else | |
391 | return ret; | |
392 | ||
393 | if (!(uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL))) | |
394 | return -ENOMEM; | |
395 | nvkm_object_ctor(&nvkm_uvmm, oclass, &uvmm->object); | |
396 | *pobject = &uvmm->object; | |
397 | ||
398 | if (!mmu->vmm) { | |
2606f291 | 399 | ret = mmu->func->vmm.ctor(mmu, managed, addr, size, argv, argc, |
920d2b5e BS |
400 | NULL, "user", &uvmm->vmm); |
401 | if (ret) | |
402 | return ret; | |
403 | ||
404 | uvmm->vmm->debug = max(uvmm->vmm->debug, oclass->client->debug); | |
405 | } else { | |
406 | if (size) | |
407 | return -EINVAL; | |
408 | ||
409 | uvmm->vmm = nvkm_vmm_ref(mmu->vmm); | |
410 | } | |
411 | ||
412 | page = uvmm->vmm->func->page; | |
413 | args->v0.page_nr = 0; | |
414 | while (page && (page++)->shift) | |
415 | args->v0.page_nr++; | |
416 | args->v0.addr = uvmm->vmm->start; | |
417 | args->v0.size = uvmm->vmm->limit; | |
418 | return 0; | |
419 | } |