Commit | Line | Data |
---|---|---|
0b1de5d5 CW |
1 | /* |
2 | * Copyright © 2016 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
25 | #include <linux/kernel.h> | |
26 | #include <asm/fpu/api.h> | |
27 | ||
28 | #include "i915_drv.h" | |
29 | ||
21aea5cc | 30 | static DEFINE_STATIC_KEY_FALSE(has_movntdqa); |
0b1de5d5 CW |
31 | |
32 | #ifdef CONFIG_AS_MOVNTDQA | |
33 | static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) | |
34 | { | |
35 | kernel_fpu_begin(); | |
36 | ||
37 | len >>= 4; | |
38 | while (len >= 4) { | |
39 | asm("movntdqa (%0), %%xmm0\n" | |
40 | "movntdqa 16(%0), %%xmm1\n" | |
41 | "movntdqa 32(%0), %%xmm2\n" | |
42 | "movntdqa 48(%0), %%xmm3\n" | |
43 | "movaps %%xmm0, (%1)\n" | |
44 | "movaps %%xmm1, 16(%1)\n" | |
45 | "movaps %%xmm2, 32(%1)\n" | |
46 | "movaps %%xmm3, 48(%1)\n" | |
47 | :: "r" (src), "r" (dst) : "memory"); | |
48 | src += 64; | |
49 | dst += 64; | |
50 | len -= 4; | |
51 | } | |
52 | while (len--) { | |
53 | asm("movntdqa (%0), %%xmm0\n" | |
54 | "movaps %%xmm0, (%1)\n" | |
55 | :: "r" (src), "r" (dst) : "memory"); | |
56 | src += 16; | |
57 | dst += 16; | |
58 | } | |
59 | ||
60 | kernel_fpu_end(); | |
61 | } | |
62 | #endif | |
63 | ||
64 | /** | |
65 | * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC | |
66 | * @dst: destination pointer | |
67 | * @src: source pointer | |
68 | * @len: how many bytes to copy | |
69 | * | |
70 | * i915_memcpy_from_wc copies @len bytes from @src to @dst using | |
71 | * non-temporal instructions where available. Note that all arguments | |
72 | * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple | |
73 | * of 16. | |
74 | * | |
75 | * To test whether accelerated reads from WC are supported, use | |
76 | * i915_memcpy_from_wc(NULL, NULL, 0); | |
77 | * | |
78 | * Returns true if the copy was successful, false if the preconditions | |
79 | * are not met. | |
80 | */ | |
81 | bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len) | |
82 | { | |
83 | if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15)) | |
84 | return false; | |
85 | ||
86 | #ifdef CONFIG_AS_MOVNTDQA | |
87 | if (static_branch_likely(&has_movntdqa)) { | |
88 | if (likely(len)) | |
89 | __memcpy_ntdqa(dst, src, len); | |
90 | return true; | |
91 | } | |
92 | #endif | |
93 | ||
94 | return false; | |
95 | } | |
96 | ||
97 | void i915_memcpy_init_early(struct drm_i915_private *dev_priv) | |
98 | { | |
99 | if (static_cpu_has(X86_FEATURE_XMM4_1)) | |
100 | static_branch_enable(&has_movntdqa); | |
101 | } |