x86/debug: Remove perpetually broken, unmaintainable dwarf annotations

[linux-2.6-block.git] / arch / x86 / lib / copy_user_64.S

diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
index fa997dfaef242fa9abdb28c20658a939caf72697..982ce34f4a9bf66011fc2652b45466d9c2b276f9 100644 (file)
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -7,7 +7,6 @@
  */
 
 #include <linux/linkage.h>
-#include <asm/dwarf2.h>
 #include <asm/current.h>
 #include <asm/asm-offsets.h>
 #include <asm/thread_info.h>
@@ -16,33 +15,8 @@
 #include <asm/asm.h>
 #include <asm/smap.h>
 
-       .macro ALIGN_DESTINATION
-       /* check for bad alignment of destination */
-       movl %edi,%ecx
-       andl $7,%ecx
-       jz 102f                         /* already aligned */
-       subl $8,%ecx
-       negl %ecx
-       subl %ecx,%edx
-100:   movb (%rsi),%al
-101:   movb %al,(%rdi)
-       incq %rsi
-       incq %rdi
-       decl %ecx
-       jnz 100b
-102:
-       .section .fixup,"ax"
-103:   addl %ecx,%edx                  /* ecx is zerorest also */
-       jmp copy_user_handle_tail
-       .previous
-
-       _ASM_EXTABLE(100b,103b)
-       _ASM_EXTABLE(101b,103b)
-       .endm
-
 /* Standard copy_to_user with segment limit checking */
 ENTRY(_copy_to_user)
-       CFI_STARTPROC
        GET_THREAD_INFO(%rax)
        movq %rdi,%rcx
        addq %rdx,%rcx
@@ -54,12 +28,10 @@ ENTRY(_copy_to_user)
                      X86_FEATURE_REP_GOOD,                     \
                      "jmp copy_user_enhanced_fast_string",     \
                      X86_FEATURE_ERMS
-       CFI_ENDPROC
 ENDPROC(_copy_to_user)
 
 /* Standard copy_from_user with segment limit checking */
 ENTRY(_copy_from_user)
-       CFI_STARTPROC
        GET_THREAD_INFO(%rax)
        movq %rsi,%rcx
        addq %rdx,%rcx
@@ -71,14 +43,12 @@ ENTRY(_copy_from_user)
                      X86_FEATURE_REP_GOOD,                     \
                      "jmp copy_user_enhanced_fast_string",     \
                      X86_FEATURE_ERMS
-       CFI_ENDPROC
 ENDPROC(_copy_from_user)
 
        .section .fixup,"ax"
        /* must zero dest */
 ENTRY(bad_from_user)
 bad_from_user:
-       CFI_STARTPROC
        movl %edx,%ecx
        xorl %eax,%eax
        rep
@@ -86,7 +56,6 @@ bad_from_user:
 bad_to_user:
        movl %edx,%eax
        ret
-       CFI_ENDPROC
 ENDPROC(bad_from_user)
        .previous
 
@@ -104,7 +73,6 @@ ENDPROC(bad_from_user)
  * eax uncopied bytes or 0 if successful.
  */
 ENTRY(copy_user_generic_unrolled)
-       CFI_STARTPROC
        ASM_STAC
        cmpl $8,%edx
        jb 20f          /* less then 8 bytes, go to byte copy loop */
@@ -186,7 +154,6 @@ ENTRY(copy_user_generic_unrolled)
        _ASM_EXTABLE(19b,40b)
        _ASM_EXTABLE(21b,50b)
        _ASM_EXTABLE(22b,50b)
-       CFI_ENDPROC
 ENDPROC(copy_user_generic_unrolled)
 
 /* Some CPUs run faster using the string copy instructions.
@@ -208,7 +175,6 @@ ENDPROC(copy_user_generic_unrolled)
  * eax uncopied bytes or 0 if successful.
  */
 ENTRY(copy_user_generic_string)
-       CFI_STARTPROC
        ASM_STAC
        cmpl $8,%edx
        jb 2f           /* less than 8 bytes, go to byte copy loop */
@@ -233,7 +199,6 @@ ENTRY(copy_user_generic_string)
 
        _ASM_EXTABLE(1b,11b)
        _ASM_EXTABLE(3b,12b)
-       CFI_ENDPROC
 ENDPROC(copy_user_generic_string)
 
 /*
@@ -249,7 +214,6 @@ ENDPROC(copy_user_generic_string)
  * eax uncopied bytes or 0 if successful.
  */
 ENTRY(copy_user_enhanced_fast_string)
-       CFI_STARTPROC
        ASM_STAC
        movl %edx,%ecx
 1:     rep
@@ -264,5 +228,94 @@ ENTRY(copy_user_enhanced_fast_string)
        .previous
 
        _ASM_EXTABLE(1b,12b)
-       CFI_ENDPROC
 ENDPROC(copy_user_enhanced_fast_string)
+
+/*
+ * copy_user_nocache - Uncached memory copy with exception handling
+ * This will force destination/source out of cache for more performance.
+ */
+ENTRY(__copy_user_nocache)
+       ASM_STAC
+       cmpl $8,%edx
+       jb 20f          /* less then 8 bytes, go to byte copy loop */
+       ALIGN_DESTINATION
+       movl %edx,%ecx
+       andl $63,%edx
+       shrl $6,%ecx
+       jz 17f
+1:     movq (%rsi),%r8
+2:     movq 1*8(%rsi),%r9
+3:     movq 2*8(%rsi),%r10
+4:     movq 3*8(%rsi),%r11
+5:     movnti %r8,(%rdi)
+6:     movnti %r9,1*8(%rdi)
+7:     movnti %r10,2*8(%rdi)
+8:     movnti %r11,3*8(%rdi)
+9:     movq 4*8(%rsi),%r8
+10:    movq 5*8(%rsi),%r9
+11:    movq 6*8(%rsi),%r10
+12:    movq 7*8(%rsi),%r11
+13:    movnti %r8,4*8(%rdi)
+14:    movnti %r9,5*8(%rdi)
+15:    movnti %r10,6*8(%rdi)
+16:    movnti %r11,7*8(%rdi)
+       leaq 64(%rsi),%rsi
+       leaq 64(%rdi),%rdi
+       decl %ecx
+       jnz 1b
+17:    movl %edx,%ecx
+       andl $7,%edx
+       shrl $3,%ecx
+       jz 20f
+18:    movq (%rsi),%r8
+19:    movnti %r8,(%rdi)
+       leaq 8(%rsi),%rsi
+       leaq 8(%rdi),%rdi
+       decl %ecx
+       jnz 18b
+20:    andl %edx,%edx
+       jz 23f
+       movl %edx,%ecx
+21:    movb (%rsi),%al
+22:    movb %al,(%rdi)
+       incq %rsi
+       incq %rdi
+       decl %ecx
+       jnz 21b
+23:    xorl %eax,%eax
+       ASM_CLAC
+       sfence
+       ret
+
+       .section .fixup,"ax"
+30:    shll $6,%ecx
+       addl %ecx,%edx
+       jmp 60f
+40:    lea (%rdx,%rcx,8),%rdx
+       jmp 60f
+50:    movl %ecx,%edx
+60:    sfence
+       jmp copy_user_handle_tail
+       .previous
+
+       _ASM_EXTABLE(1b,30b)
+       _ASM_EXTABLE(2b,30b)
+       _ASM_EXTABLE(3b,30b)
+       _ASM_EXTABLE(4b,30b)
+       _ASM_EXTABLE(5b,30b)
+       _ASM_EXTABLE(6b,30b)
+       _ASM_EXTABLE(7b,30b)
+       _ASM_EXTABLE(8b,30b)
+       _ASM_EXTABLE(9b,30b)
+       _ASM_EXTABLE(10b,30b)
+       _ASM_EXTABLE(11b,30b)
+       _ASM_EXTABLE(12b,30b)
+       _ASM_EXTABLE(13b,30b)
+       _ASM_EXTABLE(14b,30b)
+       _ASM_EXTABLE(15b,30b)
+       _ASM_EXTABLE(16b,30b)
+       _ASM_EXTABLE(18b,40b)
+       _ASM_EXTABLE(19b,40b)
+       _ASM_EXTABLE(21b,50b)
+       _ASM_EXTABLE(22b,50b)
+ENDPROC(__copy_user_nocache)