Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * linux/arch/i386/kernel/i387.c | |
3 | * | |
4 | * Copyright (C) 1994 Linus Torvalds | |
5 | * | |
6 | * Pentium III FXSR, SSE support | |
7 | * General FPU state handling cleanups | |
8 | * Gareth Hughes <gareth@valinux.com>, May 2000 | |
9 | */ | |
10 | ||
11 | #include <linux/config.h> | |
12 | #include <linux/sched.h> | |
13 | #include <asm/processor.h> | |
14 | #include <asm/i387.h> | |
15 | #include <asm/math_emu.h> | |
16 | #include <asm/sigcontext.h> | |
17 | #include <asm/user.h> | |
18 | #include <asm/ptrace.h> | |
19 | #include <asm/uaccess.h> | |
20 | ||
21 | #ifdef CONFIG_MATH_EMULATION | |
22 | #define HAVE_HWFP (boot_cpu_data.hard_math) | |
23 | #else | |
24 | #define HAVE_HWFP 1 | |
25 | #endif | |
26 | ||
27 | static unsigned long mxcsr_feature_mask = 0xffffffff; | |
28 | ||
29 | void mxcsr_feature_mask_init(void) | |
30 | { | |
31 | unsigned long mask = 0; | |
32 | clts(); | |
33 | if (cpu_has_fxsr) { | |
34 | memset(¤t->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct)); | |
35 | asm volatile("fxsave %0" : : "m" (current->thread.i387.fxsave)); | |
36 | mask = current->thread.i387.fxsave.mxcsr_mask; | |
37 | if (mask == 0) mask = 0x0000ffbf; | |
38 | } | |
39 | mxcsr_feature_mask &= mask; | |
40 | stts(); | |
41 | } | |
42 | ||
43 | /* | |
44 | * The _current_ task is using the FPU for the first time | |
45 | * so initialize it and set the mxcsr to its default | |
46 | * value at reset if we support XMM instructions and then | |
47 | * remeber the current task has used the FPU. | |
48 | */ | |
49 | void init_fpu(struct task_struct *tsk) | |
50 | { | |
51 | if (cpu_has_fxsr) { | |
52 | memset(&tsk->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct)); | |
53 | tsk->thread.i387.fxsave.cwd = 0x37f; | |
54 | if (cpu_has_xmm) | |
55 | tsk->thread.i387.fxsave.mxcsr = 0x1f80; | |
56 | } else { | |
57 | memset(&tsk->thread.i387.fsave, 0, sizeof(struct i387_fsave_struct)); | |
58 | tsk->thread.i387.fsave.cwd = 0xffff037fu; | |
59 | tsk->thread.i387.fsave.swd = 0xffff0000u; | |
60 | tsk->thread.i387.fsave.twd = 0xffffffffu; | |
61 | tsk->thread.i387.fsave.fos = 0xffff0000u; | |
62 | } | |
63 | /* only the device not available exception or ptrace can call init_fpu */ | |
64 | set_stopped_child_used_math(tsk); | |
65 | } | |
66 | ||
67 | /* | |
68 | * FPU lazy state save handling. | |
69 | */ | |
70 | ||
71 | void kernel_fpu_begin(void) | |
72 | { | |
73 | struct thread_info *thread = current_thread_info(); | |
74 | ||
75 | preempt_disable(); | |
76 | if (thread->status & TS_USEDFPU) { | |
77 | __save_init_fpu(thread->task); | |
78 | return; | |
79 | } | |
80 | clts(); | |
81 | } | |
82 | ||
83 | void restore_fpu( struct task_struct *tsk ) | |
84 | { | |
85 | if ( cpu_has_fxsr ) { | |
86 | asm volatile( "fxrstor %0" | |
87 | : : "m" (tsk->thread.i387.fxsave) ); | |
88 | } else { | |
89 | asm volatile( "frstor %0" | |
90 | : : "m" (tsk->thread.i387.fsave) ); | |
91 | } | |
92 | } | |
93 | ||
94 | /* | |
95 | * FPU tag word conversions. | |
96 | */ | |
97 | ||
98 | static inline unsigned short twd_i387_to_fxsr( unsigned short twd ) | |
99 | { | |
100 | unsigned int tmp; /* to avoid 16 bit prefixes in the code */ | |
101 | ||
102 | /* Transform each pair of bits into 01 (valid) or 00 (empty) */ | |
103 | tmp = ~twd; | |
104 | tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */ | |
105 | /* and move the valid bits to the lower byte. */ | |
106 | tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */ | |
107 | tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */ | |
108 | tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */ | |
109 | return tmp; | |
110 | } | |
111 | ||
112 | static inline unsigned long twd_fxsr_to_i387( struct i387_fxsave_struct *fxsave ) | |
113 | { | |
114 | struct _fpxreg *st = NULL; | |
115 | unsigned long tos = (fxsave->swd >> 11) & 7; | |
116 | unsigned long twd = (unsigned long) fxsave->twd; | |
117 | unsigned long tag; | |
118 | unsigned long ret = 0xffff0000u; | |
119 | int i; | |
120 | ||
121 | #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16); | |
122 | ||
123 | for ( i = 0 ; i < 8 ; i++ ) { | |
124 | if ( twd & 0x1 ) { | |
125 | st = FPREG_ADDR( fxsave, (i - tos) & 7 ); | |
126 | ||
127 | switch ( st->exponent & 0x7fff ) { | |
128 | case 0x7fff: | |
129 | tag = 2; /* Special */ | |
130 | break; | |
131 | case 0x0000: | |
132 | if ( !st->significand[0] && | |
133 | !st->significand[1] && | |
134 | !st->significand[2] && | |
135 | !st->significand[3] ) { | |
136 | tag = 1; /* Zero */ | |
137 | } else { | |
138 | tag = 2; /* Special */ | |
139 | } | |
140 | break; | |
141 | default: | |
142 | if ( st->significand[3] & 0x8000 ) { | |
143 | tag = 0; /* Valid */ | |
144 | } else { | |
145 | tag = 2; /* Special */ | |
146 | } | |
147 | break; | |
148 | } | |
149 | } else { | |
150 | tag = 3; /* Empty */ | |
151 | } | |
152 | ret |= (tag << (2 * i)); | |
153 | twd = twd >> 1; | |
154 | } | |
155 | return ret; | |
156 | } | |
157 | ||
158 | /* | |
159 | * FPU state interaction. | |
160 | */ | |
161 | ||
162 | unsigned short get_fpu_cwd( struct task_struct *tsk ) | |
163 | { | |
164 | if ( cpu_has_fxsr ) { | |
165 | return tsk->thread.i387.fxsave.cwd; | |
166 | } else { | |
167 | return (unsigned short)tsk->thread.i387.fsave.cwd; | |
168 | } | |
169 | } | |
170 | ||
171 | unsigned short get_fpu_swd( struct task_struct *tsk ) | |
172 | { | |
173 | if ( cpu_has_fxsr ) { | |
174 | return tsk->thread.i387.fxsave.swd; | |
175 | } else { | |
176 | return (unsigned short)tsk->thread.i387.fsave.swd; | |
177 | } | |
178 | } | |
179 | ||
180 | #if 0 | |
181 | unsigned short get_fpu_twd( struct task_struct *tsk ) | |
182 | { | |
183 | if ( cpu_has_fxsr ) { | |
184 | return tsk->thread.i387.fxsave.twd; | |
185 | } else { | |
186 | return (unsigned short)tsk->thread.i387.fsave.twd; | |
187 | } | |
188 | } | |
189 | #endif /* 0 */ | |
190 | ||
191 | unsigned short get_fpu_mxcsr( struct task_struct *tsk ) | |
192 | { | |
193 | if ( cpu_has_xmm ) { | |
194 | return tsk->thread.i387.fxsave.mxcsr; | |
195 | } else { | |
196 | return 0x1f80; | |
197 | } | |
198 | } | |
199 | ||
200 | #if 0 | |
201 | ||
202 | void set_fpu_cwd( struct task_struct *tsk, unsigned short cwd ) | |
203 | { | |
204 | if ( cpu_has_fxsr ) { | |
205 | tsk->thread.i387.fxsave.cwd = cwd; | |
206 | } else { | |
207 | tsk->thread.i387.fsave.cwd = ((long)cwd | 0xffff0000u); | |
208 | } | |
209 | } | |
210 | ||
211 | void set_fpu_swd( struct task_struct *tsk, unsigned short swd ) | |
212 | { | |
213 | if ( cpu_has_fxsr ) { | |
214 | tsk->thread.i387.fxsave.swd = swd; | |
215 | } else { | |
216 | tsk->thread.i387.fsave.swd = ((long)swd | 0xffff0000u); | |
217 | } | |
218 | } | |
219 | ||
220 | void set_fpu_twd( struct task_struct *tsk, unsigned short twd ) | |
221 | { | |
222 | if ( cpu_has_fxsr ) { | |
223 | tsk->thread.i387.fxsave.twd = twd_i387_to_fxsr(twd); | |
224 | } else { | |
225 | tsk->thread.i387.fsave.twd = ((long)twd | 0xffff0000u); | |
226 | } | |
227 | } | |
228 | ||
229 | #endif /* 0 */ | |
230 | ||
231 | /* | |
232 | * FXSR floating point environment conversions. | |
233 | */ | |
234 | ||
235 | static int convert_fxsr_to_user( struct _fpstate __user *buf, | |
236 | struct i387_fxsave_struct *fxsave ) | |
237 | { | |
238 | unsigned long env[7]; | |
239 | struct _fpreg __user *to; | |
240 | struct _fpxreg *from; | |
241 | int i; | |
242 | ||
243 | env[0] = (unsigned long)fxsave->cwd | 0xffff0000ul; | |
244 | env[1] = (unsigned long)fxsave->swd | 0xffff0000ul; | |
245 | env[2] = twd_fxsr_to_i387(fxsave); | |
246 | env[3] = fxsave->fip; | |
247 | env[4] = fxsave->fcs | ((unsigned long)fxsave->fop << 16); | |
248 | env[5] = fxsave->foo; | |
249 | env[6] = fxsave->fos; | |
250 | ||
251 | if ( __copy_to_user( buf, env, 7 * sizeof(unsigned long) ) ) | |
252 | return 1; | |
253 | ||
254 | to = &buf->_st[0]; | |
255 | from = (struct _fpxreg *) &fxsave->st_space[0]; | |
256 | for ( i = 0 ; i < 8 ; i++, to++, from++ ) { | |
257 | unsigned long __user *t = (unsigned long __user *)to; | |
258 | unsigned long *f = (unsigned long *)from; | |
259 | ||
260 | if (__put_user(*f, t) || | |
261 | __put_user(*(f + 1), t + 1) || | |
262 | __put_user(from->exponent, &to->exponent)) | |
263 | return 1; | |
264 | } | |
265 | return 0; | |
266 | } | |
267 | ||
268 | static int convert_fxsr_from_user( struct i387_fxsave_struct *fxsave, | |
269 | struct _fpstate __user *buf ) | |
270 | { | |
271 | unsigned long env[7]; | |
272 | struct _fpxreg *to; | |
273 | struct _fpreg __user *from; | |
274 | int i; | |
275 | ||
276 | if ( __copy_from_user( env, buf, 7 * sizeof(long) ) ) | |
277 | return 1; | |
278 | ||
279 | fxsave->cwd = (unsigned short)(env[0] & 0xffff); | |
280 | fxsave->swd = (unsigned short)(env[1] & 0xffff); | |
281 | fxsave->twd = twd_i387_to_fxsr((unsigned short)(env[2] & 0xffff)); | |
282 | fxsave->fip = env[3]; | |
283 | fxsave->fop = (unsigned short)((env[4] & 0xffff0000ul) >> 16); | |
284 | fxsave->fcs = (env[4] & 0xffff); | |
285 | fxsave->foo = env[5]; | |
286 | fxsave->fos = env[6]; | |
287 | ||
288 | to = (struct _fpxreg *) &fxsave->st_space[0]; | |
289 | from = &buf->_st[0]; | |
290 | for ( i = 0 ; i < 8 ; i++, to++, from++ ) { | |
291 | unsigned long *t = (unsigned long *)to; | |
292 | unsigned long __user *f = (unsigned long __user *)from; | |
293 | ||
294 | if (__get_user(*t, f) || | |
295 | __get_user(*(t + 1), f + 1) || | |
296 | __get_user(to->exponent, &from->exponent)) | |
297 | return 1; | |
298 | } | |
299 | return 0; | |
300 | } | |
301 | ||
302 | /* | |
303 | * Signal frame handlers. | |
304 | */ | |
305 | ||
306 | static inline int save_i387_fsave( struct _fpstate __user *buf ) | |
307 | { | |
308 | struct task_struct *tsk = current; | |
309 | ||
310 | unlazy_fpu( tsk ); | |
311 | tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd; | |
312 | if ( __copy_to_user( buf, &tsk->thread.i387.fsave, | |
313 | sizeof(struct i387_fsave_struct) ) ) | |
314 | return -1; | |
315 | return 1; | |
316 | } | |
317 | ||
318 | static int save_i387_fxsave( struct _fpstate __user *buf ) | |
319 | { | |
320 | struct task_struct *tsk = current; | |
321 | int err = 0; | |
322 | ||
323 | unlazy_fpu( tsk ); | |
324 | ||
325 | if ( convert_fxsr_to_user( buf, &tsk->thread.i387.fxsave ) ) | |
326 | return -1; | |
327 | ||
328 | err |= __put_user( tsk->thread.i387.fxsave.swd, &buf->status ); | |
329 | err |= __put_user( X86_FXSR_MAGIC, &buf->magic ); | |
330 | if ( err ) | |
331 | return -1; | |
332 | ||
333 | if ( __copy_to_user( &buf->_fxsr_env[0], &tsk->thread.i387.fxsave, | |
334 | sizeof(struct i387_fxsave_struct) ) ) | |
335 | return -1; | |
336 | return 1; | |
337 | } | |
338 | ||
339 | int save_i387( struct _fpstate __user *buf ) | |
340 | { | |
341 | if ( !used_math() ) | |
342 | return 0; | |
343 | ||
344 | /* This will cause a "finit" to be triggered by the next | |
345 | * attempted FPU operation by the 'current' process. | |
346 | */ | |
347 | clear_used_math(); | |
348 | ||
349 | if ( HAVE_HWFP ) { | |
350 | if ( cpu_has_fxsr ) { | |
351 | return save_i387_fxsave( buf ); | |
352 | } else { | |
353 | return save_i387_fsave( buf ); | |
354 | } | |
355 | } else { | |
356 | return save_i387_soft( ¤t->thread.i387.soft, buf ); | |
357 | } | |
358 | } | |
359 | ||
360 | static inline int restore_i387_fsave( struct _fpstate __user *buf ) | |
361 | { | |
362 | struct task_struct *tsk = current; | |
363 | clear_fpu( tsk ); | |
364 | return __copy_from_user( &tsk->thread.i387.fsave, buf, | |
365 | sizeof(struct i387_fsave_struct) ); | |
366 | } | |
367 | ||
368 | static int restore_i387_fxsave( struct _fpstate __user *buf ) | |
369 | { | |
370 | int err; | |
371 | struct task_struct *tsk = current; | |
372 | clear_fpu( tsk ); | |
373 | err = __copy_from_user( &tsk->thread.i387.fxsave, &buf->_fxsr_env[0], | |
374 | sizeof(struct i387_fxsave_struct) ); | |
375 | /* mxcsr reserved bits must be masked to zero for security reasons */ | |
376 | tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; | |
377 | return err ? 1 : convert_fxsr_from_user( &tsk->thread.i387.fxsave, buf ); | |
378 | } | |
379 | ||
380 | int restore_i387( struct _fpstate __user *buf ) | |
381 | { | |
382 | int err; | |
383 | ||
384 | if ( HAVE_HWFP ) { | |
385 | if ( cpu_has_fxsr ) { | |
386 | err = restore_i387_fxsave( buf ); | |
387 | } else { | |
388 | err = restore_i387_fsave( buf ); | |
389 | } | |
390 | } else { | |
391 | err = restore_i387_soft( ¤t->thread.i387.soft, buf ); | |
392 | } | |
393 | set_used_math(); | |
394 | return err; | |
395 | } | |
396 | ||
397 | /* | |
398 | * ptrace request handlers. | |
399 | */ | |
400 | ||
401 | static inline int get_fpregs_fsave( struct user_i387_struct __user *buf, | |
402 | struct task_struct *tsk ) | |
403 | { | |
404 | return __copy_to_user( buf, &tsk->thread.i387.fsave, | |
405 | sizeof(struct user_i387_struct) ); | |
406 | } | |
407 | ||
408 | static inline int get_fpregs_fxsave( struct user_i387_struct __user *buf, | |
409 | struct task_struct *tsk ) | |
410 | { | |
411 | return convert_fxsr_to_user( (struct _fpstate __user *)buf, | |
412 | &tsk->thread.i387.fxsave ); | |
413 | } | |
414 | ||
415 | int get_fpregs( struct user_i387_struct __user *buf, struct task_struct *tsk ) | |
416 | { | |
417 | if ( HAVE_HWFP ) { | |
418 | if ( cpu_has_fxsr ) { | |
419 | return get_fpregs_fxsave( buf, tsk ); | |
420 | } else { | |
421 | return get_fpregs_fsave( buf, tsk ); | |
422 | } | |
423 | } else { | |
424 | return save_i387_soft( &tsk->thread.i387.soft, | |
425 | (struct _fpstate __user *)buf ); | |
426 | } | |
427 | } | |
428 | ||
429 | static inline int set_fpregs_fsave( struct task_struct *tsk, | |
430 | struct user_i387_struct __user *buf ) | |
431 | { | |
432 | return __copy_from_user( &tsk->thread.i387.fsave, buf, | |
433 | sizeof(struct user_i387_struct) ); | |
434 | } | |
435 | ||
436 | static inline int set_fpregs_fxsave( struct task_struct *tsk, | |
437 | struct user_i387_struct __user *buf ) | |
438 | { | |
439 | return convert_fxsr_from_user( &tsk->thread.i387.fxsave, | |
440 | (struct _fpstate __user *)buf ); | |
441 | } | |
442 | ||
443 | int set_fpregs( struct task_struct *tsk, struct user_i387_struct __user *buf ) | |
444 | { | |
445 | if ( HAVE_HWFP ) { | |
446 | if ( cpu_has_fxsr ) { | |
447 | return set_fpregs_fxsave( tsk, buf ); | |
448 | } else { | |
449 | return set_fpregs_fsave( tsk, buf ); | |
450 | } | |
451 | } else { | |
452 | return restore_i387_soft( &tsk->thread.i387.soft, | |
453 | (struct _fpstate __user *)buf ); | |
454 | } | |
455 | } | |
456 | ||
457 | int get_fpxregs( struct user_fxsr_struct __user *buf, struct task_struct *tsk ) | |
458 | { | |
459 | if ( cpu_has_fxsr ) { | |
460 | if (__copy_to_user( buf, &tsk->thread.i387.fxsave, | |
461 | sizeof(struct user_fxsr_struct) )) | |
462 | return -EFAULT; | |
463 | return 0; | |
464 | } else { | |
465 | return -EIO; | |
466 | } | |
467 | } | |
468 | ||
469 | int set_fpxregs( struct task_struct *tsk, struct user_fxsr_struct __user *buf ) | |
470 | { | |
471 | int ret = 0; | |
472 | ||
473 | if ( cpu_has_fxsr ) { | |
474 | if (__copy_from_user( &tsk->thread.i387.fxsave, buf, | |
475 | sizeof(struct user_fxsr_struct) )) | |
476 | ret = -EFAULT; | |
477 | /* mxcsr reserved bits must be masked to zero for security reasons */ | |
478 | tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; | |
479 | } else { | |
480 | ret = -EIO; | |
481 | } | |
482 | return ret; | |
483 | } | |
484 | ||
485 | /* | |
486 | * FPU state for core dumps. | |
487 | */ | |
488 | ||
489 | static inline void copy_fpu_fsave( struct task_struct *tsk, | |
490 | struct user_i387_struct *fpu ) | |
491 | { | |
492 | memcpy( fpu, &tsk->thread.i387.fsave, | |
493 | sizeof(struct user_i387_struct) ); | |
494 | } | |
495 | ||
496 | static inline void copy_fpu_fxsave( struct task_struct *tsk, | |
497 | struct user_i387_struct *fpu ) | |
498 | { | |
499 | unsigned short *to; | |
500 | unsigned short *from; | |
501 | int i; | |
502 | ||
503 | memcpy( fpu, &tsk->thread.i387.fxsave, 7 * sizeof(long) ); | |
504 | ||
505 | to = (unsigned short *)&fpu->st_space[0]; | |
506 | from = (unsigned short *)&tsk->thread.i387.fxsave.st_space[0]; | |
507 | for ( i = 0 ; i < 8 ; i++, to += 5, from += 8 ) { | |
508 | memcpy( to, from, 5 * sizeof(unsigned short) ); | |
509 | } | |
510 | } | |
511 | ||
512 | int dump_fpu( struct pt_regs *regs, struct user_i387_struct *fpu ) | |
513 | { | |
514 | int fpvalid; | |
515 | struct task_struct *tsk = current; | |
516 | ||
517 | fpvalid = !!used_math(); | |
518 | if ( fpvalid ) { | |
519 | unlazy_fpu( tsk ); | |
520 | if ( cpu_has_fxsr ) { | |
521 | copy_fpu_fxsave( tsk, fpu ); | |
522 | } else { | |
523 | copy_fpu_fsave( tsk, fpu ); | |
524 | } | |
525 | } | |
526 | ||
527 | return fpvalid; | |
528 | } | |
529 | ||
530 | int dump_task_fpu(struct task_struct *tsk, struct user_i387_struct *fpu) | |
531 | { | |
532 | int fpvalid = !!tsk_used_math(tsk); | |
533 | ||
534 | if (fpvalid) { | |
535 | if (tsk == current) | |
536 | unlazy_fpu(tsk); | |
537 | if (cpu_has_fxsr) | |
538 | copy_fpu_fxsave(tsk, fpu); | |
539 | else | |
540 | copy_fpu_fsave(tsk, fpu); | |
541 | } | |
542 | return fpvalid; | |
543 | } | |
544 | ||
545 | int dump_task_extended_fpu(struct task_struct *tsk, struct user_fxsr_struct *fpu) | |
546 | { | |
547 | int fpvalid = tsk_used_math(tsk) && cpu_has_fxsr; | |
548 | ||
549 | if (fpvalid) { | |
550 | if (tsk == current) | |
551 | unlazy_fpu(tsk); | |
552 | memcpy(fpu, &tsk->thread.i387.fxsave, sizeof(*fpu)); | |
553 | } | |
554 | return fpvalid; | |
555 | } |