arch/arc/include/asm/uaccess.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   4  *
   5  * vineetg: June 2010
   6  *    -__clear_user( ) called multiple times during elf load was byte loop
   7  *    converted to do as much word clear as possible.
   8  *
   9  * vineetg: Dec 2009
  10  *    -Hand crafted constant propagation for "constant" copy sizes
  11  *    -stock kernel shrunk by 33K at -O3
  12  *
  13  * vineetg: Sept 2009
  14  *    -Added option to (UN)inline copy_(to|from)_user to reduce code sz
  15  *    -kernel shrunk by 200K even at -O3 (gcc 4.2.1)
  16  *    -Enabled when doing -Os
  17  *
  18  * Amit Bhor, Sameer Dhavale: Codito Technologies 2004
  19  */
  20
  21 #ifndef _ASM_ARC_UACCESS_H
  22 #define _ASM_ARC_UACCESS_H
  23
  24 #include <linux/string.h>       /* for generic string functions */
  25
  26 /*********** Single byte/hword/word copies ******************/
  27
  28 #define __get_user_fn(sz, u, k)                                 \
  29 ({                                                              \
  30         long __ret = 0; /* success by default */        \
  31         switch (sz) {                                           \
  32         case 1: __arc_get_user_one(*(k), u, "ldb", __ret); break;       \
  33         case 2: __arc_get_user_one(*(k), u, "ldw", __ret); break;       \
  34         case 4: __arc_get_user_one(*(k), u, "ld", __ret);  break;       \
  35         case 8: __arc_get_user_one_64(*(k), u, __ret);     break;       \
  36         }                                                       \
  37         __ret;                                                  \
  38 })
  39
  40 /*
  41  * Returns 0 on success, -EFAULT if not.
  42  * @ret already contains 0 - given that errors will be less likely
  43  * (hence +r asm constraint below).
  44  * In case of error, fixup code will make it -EFAULT
  45  */
  46 #define __arc_get_user_one(dst, src, op, ret)   \
  47         __asm__ __volatile__(                   \
  48         "1:     "op"    %1,[%2]\n"              \
  49         "2:     ;nop\n"                         \
  50         "       .section .fixup, \"ax\"\n"      \
  51         "       .align 4\n"                     \
  52         "3:     # return -EFAULT\n"             \
  53         "       mov %0, %3\n"                   \
  54         "       # zero out dst ptr\n"           \
  55         "       mov %1,  0\n"                   \
  56         "       j   2b\n"                       \
  57         "       .previous\n"                    \
  58         "       .section __ex_table, \"a\"\n"   \
  59         "       .align 4\n"                     \
  60         "       .word 1b,3b\n"                  \
  61         "       .previous\n"                    \
  62                                                 \
  63         : "+r" (ret), "=r" (dst)                \
  64         : "r" (src), "ir" (-EFAULT))
  65
  66 #define __arc_get_user_one_64(dst, src, ret)    \
  67         __asm__ __volatile__(                   \
  68         "1:     ld   %1,[%2]\n"                 \
  69         "4:     ld  %R1,[%2, 4]\n"              \
  70         "2:     ;nop\n"                         \
  71         "       .section .fixup, \"ax\"\n"      \
  72         "       .align 4\n"                     \
  73         "3:     # return -EFAULT\n"             \
  74         "       mov %0, %3\n"                   \
  75         "       # zero out dst ptr\n"           \
  76         "       mov %1,  0\n"                   \
  77         "       mov %R1, 0\n"                   \
  78         "       j   2b\n"                       \
  79         "       .previous\n"                    \
  80         "       .section __ex_table, \"a\"\n"   \
  81         "       .align 4\n"                     \
  82         "       .word 1b,3b\n"                  \
  83         "       .word 4b,3b\n"                  \
  84         "       .previous\n"                    \
  85                                                 \
  86         : "+r" (ret), "=r" (dst)                \
  87         : "r" (src), "ir" (-EFAULT))
  88
  89 #define __put_user_fn(sz, u, k)                                 \
  90 ({                                                              \
  91         long __ret = 0; /* success by default */        \
  92         switch (sz) {                                           \
  93         case 1: __arc_put_user_one(*(k), u, "stb", __ret); break;       \
  94         case 2: __arc_put_user_one(*(k), u, "stw", __ret); break;       \
  95         case 4: __arc_put_user_one(*(k), u, "st", __ret);  break;       \
  96         case 8: __arc_put_user_one_64(*(k), u, __ret);     break;       \
  97         }                                                       \
  98         __ret;                                                  \
  99 })
 100
 101 #define __arc_put_user_one(src, dst, op, ret)   \
 102         __asm__ __volatile__(                   \
 103         "1:     "op"    %1,[%2]\n"              \
 104         "2:     ;nop\n"                         \
 105         "       .section .fixup, \"ax\"\n"      \
 106         "       .align 4\n"                     \
 107         "3:     mov %0, %3\n"                   \
 108         "       j   2b\n"                       \
 109         "       .previous\n"                    \
 110         "       .section __ex_table, \"a\"\n"   \
 111         "       .align 4\n"                     \
 112         "       .word 1b,3b\n"                  \
 113         "       .previous\n"                    \
 114                                                 \
 115         : "+r" (ret)                            \
 116         : "r" (src), "r" (dst), "ir" (-EFAULT))
 117
 118 #define __arc_put_user_one_64(src, dst, ret)    \
 119         __asm__ __volatile__(                   \
 120         "1:     st   %1,[%2]\n"                 \
 121         "4:     st  %R1,[%2, 4]\n"              \
 122         "2:     ;nop\n"                         \
 123         "       .section .fixup, \"ax\"\n"      \
 124         "       .align 4\n"                     \
 125         "3:     mov %0, %3\n"                   \
 126         "       j   2b\n"                       \
 127         "       .previous\n"                    \
 128         "       .section __ex_table, \"a\"\n"   \
 129         "       .align 4\n"                     \
 130         "       .word 1b,3b\n"                  \
 131         "       .word 4b,3b\n"                  \
 132         "       .previous\n"                    \
 133                                                 \
 134         : "+r" (ret)                            \
 135         : "r" (src), "r" (dst), "ir" (-EFAULT))
 136
 137
 138 static inline unsigned long
 139 raw_copy_from_user(void *to, const void __user *from, unsigned long n)
 140 {
 141         long res = 0;
 142         char val;
 143         unsigned long tmp1, tmp2, tmp3, tmp4;
 144         unsigned long orig_n = n;
 145
 146         if (n == 0)
 147                 return 0;
 148
 149         /* fallback for unaligned access when hardware doesn't support */
 150         if (!IS_ENABLED(CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS) &&
 151              (((unsigned long)to & 0x3) || ((unsigned long)from & 0x3))) {
 152
 153                 unsigned char tmp;
 154
 155                 __asm__ __volatile__ (
 156                 "       mov.f   lp_count, %0            \n"
 157                 "       lpnz 2f                         \n"
 158                 "1:     ldb.ab  %1, [%3, 1]             \n"
 159                 "       stb.ab  %1, [%2, 1]             \n"
 160                 "       sub     %0,%0,1                 \n"
 161                 "2:     ;nop                            \n"
 162                 "       .section .fixup, \"ax\"         \n"
 163                 "       .align 4                        \n"
 164                 "3:     j   2b                          \n"
 165                 "       .previous                       \n"
 166                 "       .section __ex_table, \"a\"      \n"
 167                 "       .align 4                        \n"
 168                 "       .word   1b, 3b                  \n"
 169                 "       .previous                       \n"
 170
 171                 : "+r" (n),
 172                 /*
 173                  * Note as an '&' earlyclobber operand to make sure the
 174                  * temporary register inside the loop is not the same as
 175                  *  FROM or TO.
 176                 */
 177                   "=&r" (tmp), "+r" (to), "+r" (from)
 178                 :
 179                 : "lp_count", "memory");
 180
 181                 return n;
 182         }
 183
 184         /*
 185          * Hand-crafted constant propagation to reduce code sz of the
 186          * laddered copy 16x,8,4,2,1
 187          */
 188         if (__builtin_constant_p(orig_n)) {
 189                 res = orig_n;
 190
 191                 if (orig_n / 16) {
 192                         orig_n = orig_n % 16;
 193
 194                         __asm__ __volatile__(
 195                         "       lsr   lp_count, %7,4            \n"
 196                         "       lp    3f                        \n"
 197                         "1:     ld.ab   %3, [%2, 4]             \n"
 198                         "11:    ld.ab   %4, [%2, 4]             \n"
 199                         "12:    ld.ab   %5, [%2, 4]             \n"
 200                         "13:    ld.ab   %6, [%2, 4]             \n"
 201                         "       st.ab   %3, [%1, 4]             \n"
 202                         "       st.ab   %4, [%1, 4]             \n"
 203                         "       st.ab   %5, [%1, 4]             \n"
 204                         "       st.ab   %6, [%1, 4]             \n"
 205                         "       sub     %0,%0,16                \n"
 206                         "3:     ;nop                            \n"
 207                         "       .section .fixup, \"ax\"         \n"
 208                         "       .align 4                        \n"
 209                         "4:     j   3b                          \n"
 210                         "       .previous                       \n"
 211                         "       .section __ex_table, \"a\"      \n"
 212                         "       .align 4                        \n"
 213                         "       .word   1b, 4b                  \n"
 214                         "       .word   11b,4b                  \n"
 215                         "       .word   12b,4b                  \n"
 216                         "       .word   13b,4b                  \n"
 217                         "       .previous                       \n"
 218                         : "+r" (res), "+r"(to), "+r"(from),
 219                           "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
 220                         : "ir"(n)
 221                         : "lp_count", "memory");
 222                 }
 223                 if (orig_n / 8) {
 224                         orig_n = orig_n % 8;
 225
 226                         __asm__ __volatile__(
 227                         "14:    ld.ab   %3, [%2,4]              \n"
 228                         "15:    ld.ab   %4, [%2,4]              \n"
 229                         "       st.ab   %3, [%1,4]              \n"
 230                         "       st.ab   %4, [%1,4]              \n"
 231                         "       sub     %0,%0,8                 \n"
 232                         "31:    ;nop                            \n"
 233                         "       .section .fixup, \"ax\"         \n"
 234                         "       .align 4                        \n"
 235                         "4:     j   31b                         \n"
 236                         "       .previous                       \n"
 237                         "       .section __ex_table, \"a\"      \n"
 238                         "       .align 4                        \n"
 239                         "       .word   14b,4b                  \n"
 240                         "       .word   15b,4b                  \n"
 241                         "       .previous                       \n"
 242                         : "+r" (res), "+r"(to), "+r"(from),
 243                           "=r"(tmp1), "=r"(tmp2)
 244                         :
 245                         : "memory");
 246                 }
 247                 if (orig_n / 4) {
 248                         orig_n = orig_n % 4;
 249
 250                         __asm__ __volatile__(
 251                         "16:    ld.ab   %3, [%2,4]              \n"
 252                         "       st.ab   %3, [%1,4]              \n"
 253                         "       sub     %0,%0,4                 \n"
 254                         "32:    ;nop                            \n"
 255                         "       .section .fixup, \"ax\"         \n"
 256                         "       .align 4                        \n"
 257                         "4:     j   32b                         \n"
 258                         "       .previous                       \n"
 259                         "       .section __ex_table, \"a\"      \n"
 260                         "       .align 4                        \n"
 261                         "       .word   16b,4b                  \n"
 262                         "       .previous                       \n"
 263                         : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
 264                         :
 265                         : "memory");
 266                 }
 267                 if (orig_n / 2) {
 268                         orig_n = orig_n % 2;
 269
 270                         __asm__ __volatile__(
 271                         "17:    ldw.ab   %3, [%2,2]             \n"
 272                         "       stw.ab   %3, [%1,2]             \n"
 273                         "       sub      %0,%0,2                \n"
 274                         "33:    ;nop                            \n"
 275                         "       .section .fixup, \"ax\"         \n"
 276                         "       .align 4                        \n"
 277                         "4:     j   33b                         \n"
 278                         "       .previous                       \n"
 279                         "       .section __ex_table, \"a\"      \n"
 280                         "       .align 4                        \n"
 281                         "       .word   17b,4b                  \n"
 282                         "       .previous                       \n"
 283                         : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
 284                         :
 285                         : "memory");
 286                 }
 287                 if (orig_n & 1) {
 288                         __asm__ __volatile__(
 289                         "18:    ldb.ab   %3, [%2,2]             \n"
 290                         "       stb.ab   %3, [%1,2]             \n"
 291                         "       sub      %0,%0,1                \n"
 292                         "34:    ; nop                           \n"
 293                         "       .section .fixup, \"ax\"         \n"
 294                         "       .align 4                        \n"
 295                         "4:     j   34b                         \n"
 296                         "       .previous                       \n"
 297                         "       .section __ex_table, \"a\"      \n"
 298                         "       .align 4                        \n"
 299                         "       .word   18b,4b                  \n"
 300                         "       .previous                       \n"
 301                         : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
 302                         :
 303                         : "memory");
 304                 }
 305         } else {  /* n is NOT constant, so laddered copy of 16x,8,4,2,1  */
 306
 307                 __asm__ __volatile__(
 308                 "       mov %0,%3                       \n"
 309                 "       lsr.f   lp_count, %3,4          \n"  /* 16x bytes */
 310                 "       lpnz    3f                      \n"
 311                 "1:     ld.ab   %5, [%2, 4]             \n"
 312                 "11:    ld.ab   %6, [%2, 4]             \n"
 313                 "12:    ld.ab   %7, [%2, 4]             \n"
 314                 "13:    ld.ab   %8, [%2, 4]             \n"
 315                 "       st.ab   %5, [%1, 4]             \n"
 316                 "       st.ab   %6, [%1, 4]             \n"
 317                 "       st.ab   %7, [%1, 4]             \n"
 318                 "       st.ab   %8, [%1, 4]             \n"
 319                 "       sub     %0,%0,16                \n"
 320                 "3:     and.f   %3,%3,0xf               \n"  /* stragglers */
 321                 "       bz      34f                     \n"
 322                 "       bbit0   %3,3,31f                \n"  /* 8 bytes left */
 323                 "14:    ld.ab   %5, [%2,4]              \n"
 324                 "15:    ld.ab   %6, [%2,4]              \n"
 325                 "       st.ab   %5, [%1,4]              \n"
 326                 "       st.ab   %6, [%1,4]              \n"
 327                 "       sub.f   %0,%0,8                 \n"
 328                 "31:    bbit0   %3,2,32f                \n"  /* 4 bytes left */
 329                 "16:    ld.ab   %5, [%2,4]              \n"
 330                 "       st.ab   %5, [%1,4]              \n"
 331                 "       sub.f   %0,%0,4                 \n"
 332                 "32:    bbit0   %3,1,33f                \n"  /* 2 bytes left */
 333                 "17:    ldw.ab  %5, [%2,2]              \n"
 334                 "       stw.ab  %5, [%1,2]              \n"
 335                 "       sub.f   %0,%0,2                 \n"
 336                 "33:    bbit0   %3,0,34f                \n"
 337                 "18:    ldb.ab  %5, [%2,1]              \n"  /* 1 byte left */
 338                 "       stb.ab  %5, [%1,1]              \n"
 339                 "       sub.f   %0,%0,1                 \n"
 340                 "34:    ;nop                            \n"
 341                 "       .section .fixup, \"ax\"         \n"
 342                 "       .align 4                        \n"
 343                 "4:     j   34b                         \n"
 344                 "       .previous                       \n"
 345                 "       .section __ex_table, \"a\"      \n"
 346                 "       .align 4                        \n"
 347                 "       .word   1b, 4b                  \n"
 348                 "       .word   11b,4b                  \n"
 349                 "       .word   12b,4b                  \n"
 350                 "       .word   13b,4b                  \n"
 351                 "       .word   14b,4b                  \n"
 352                 "       .word   15b,4b                  \n"
 353                 "       .word   16b,4b                  \n"
 354                 "       .word   17b,4b                  \n"
 355                 "       .word   18b,4b                  \n"
 356                 "       .previous                       \n"
 357                 : "=r" (res), "+r"(to), "+r"(from), "+r"(n), "=r"(val),
 358                   "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
 359                 :
 360                 : "lp_count", "memory");
 361         }
 362
 363         return res;
 364 }
 365
 366 static inline unsigned long
 367 raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 368 {
 369         long res = 0;
 370         char val;
 371         unsigned long tmp1, tmp2, tmp3, tmp4;
 372         unsigned long orig_n = n;
 373
 374         if (n == 0)
 375                 return 0;
 376
 377         /* fallback for unaligned access when hardware doesn't support */
 378         if (!IS_ENABLED(CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS) &&
 379              (((unsigned long)to & 0x3) || ((unsigned long)from & 0x3))) {
 380
 381                 unsigned char tmp;
 382
 383                 __asm__ __volatile__(
 384                 "       mov.f   lp_count, %0            \n"
 385                 "       lpnz 3f                         \n"
 386                 "       ldb.ab  %1, [%3, 1]             \n"
 387                 "1:     stb.ab  %1, [%2, 1]             \n"
 388                 "       sub     %0, %0, 1               \n"
 389                 "3:     ;nop                            \n"
 390                 "       .section .fixup, \"ax\"         \n"
 391                 "       .align 4                        \n"
 392                 "4:     j   3b                          \n"
 393                 "       .previous                       \n"
 394                 "       .section __ex_table, \"a\"      \n"
 395                 "       .align 4                        \n"
 396                 "       .word   1b, 4b                  \n"
 397                 "       .previous                       \n"
 398
 399                 : "+r" (n),
 400                 /* Note as an '&' earlyclobber operand to make sure the
 401                  * temporary register inside the loop is not the same as
 402                  * FROM or TO.
 403                  */
 404                   "=&r" (tmp), "+r" (to), "+r" (from)
 405                 :
 406                 : "lp_count", "memory");
 407
 408                 return n;
 409         }
 410
 411         if (__builtin_constant_p(orig_n)) {
 412                 res = orig_n;
 413
 414                 if (orig_n / 16) {
 415                         orig_n = orig_n % 16;
 416
 417                         __asm__ __volatile__(
 418                         "       lsr lp_count, %7,4              \n"
 419                         "       lp  3f                          \n"
 420                         "       ld.ab %3, [%2, 4]               \n"
 421                         "       ld.ab %4, [%2, 4]               \n"
 422                         "       ld.ab %5, [%2, 4]               \n"
 423                         "       ld.ab %6, [%2, 4]               \n"
 424                         "1:     st.ab %3, [%1, 4]               \n"
 425                         "11:    st.ab %4, [%1, 4]               \n"
 426                         "12:    st.ab %5, [%1, 4]               \n"
 427                         "13:    st.ab %6, [%1, 4]               \n"
 428                         "       sub   %0, %0, 16                \n"
 429                         "3:;nop                                 \n"
 430                         "       .section .fixup, \"ax\"         \n"
 431                         "       .align 4                        \n"
 432                         "4:     j   3b                          \n"
 433                         "       .previous                       \n"
 434                         "       .section __ex_table, \"a\"      \n"
 435                         "       .align 4                        \n"
 436                         "       .word   1b, 4b                  \n"
 437                         "       .word   11b,4b                  \n"
 438                         "       .word   12b,4b                  \n"
 439                         "       .word   13b,4b                  \n"
 440                         "       .previous                       \n"
 441                         : "+r" (res), "+r"(to), "+r"(from),
 442                           "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
 443                         : "ir"(n)
 444                         : "lp_count", "memory");
 445                 }
 446                 if (orig_n / 8) {
 447                         orig_n = orig_n % 8;
 448
 449                         __asm__ __volatile__(
 450                         "       ld.ab   %3, [%2,4]              \n"
 451                         "       ld.ab   %4, [%2,4]              \n"
 452                         "14:    st.ab   %3, [%1,4]              \n"
 453                         "15:    st.ab   %4, [%1,4]              \n"
 454                         "       sub     %0, %0, 8               \n"
 455                         "31:;nop                                \n"
 456                         "       .section .fixup, \"ax\"         \n"
 457                         "       .align 4                        \n"
 458                         "4:     j   31b                         \n"
 459                         "       .previous                       \n"
 460                         "       .section __ex_table, \"a\"      \n"
 461                         "       .align 4                        \n"
 462                         "       .word   14b,4b                  \n"
 463                         "       .word   15b,4b                  \n"
 464                         "       .previous                       \n"
 465                         : "+r" (res), "+r"(to), "+r"(from),
 466                           "=r"(tmp1), "=r"(tmp2)
 467                         :
 468                         : "memory");
 469                 }
 470                 if (orig_n / 4) {
 471                         orig_n = orig_n % 4;
 472
 473                         __asm__ __volatile__(
 474                         "       ld.ab   %3, [%2,4]              \n"
 475                         "16:    st.ab   %3, [%1,4]              \n"
 476                         "       sub     %0, %0, 4               \n"
 477                         "32:;nop                                \n"
 478                         "       .section .fixup, \"ax\"         \n"
 479                         "       .align 4                        \n"
 480                         "4:     j   32b                         \n"
 481                         "       .previous                       \n"
 482                         "       .section __ex_table, \"a\"      \n"
 483                         "       .align 4                        \n"
 484                         "       .word   16b,4b                  \n"
 485                         "       .previous                       \n"
 486                         : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
 487                         :
 488                         : "memory");
 489                 }
 490                 if (orig_n / 2) {
 491                         orig_n = orig_n % 2;
 492
 493                         __asm__ __volatile__(
 494                         "       ldw.ab    %3, [%2,2]            \n"
 495                         "17:    stw.ab    %3, [%1,2]            \n"
 496                         "       sub       %0, %0, 2             \n"
 497                         "33:;nop                                \n"
 498                         "       .section .fixup, \"ax\"         \n"
 499                         "       .align 4                        \n"
 500                         "4:     j   33b                         \n"
 501                         "       .previous                       \n"
 502                         "       .section __ex_table, \"a\"      \n"
 503                         "       .align 4                        \n"
 504                         "       .word   17b,4b                  \n"
 505                         "       .previous                       \n"
 506                         : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
 507                         :
 508                         : "memory");
 509                 }
 510                 if (orig_n & 1) {
 511                         __asm__ __volatile__(
 512                         "       ldb.ab  %3, [%2,1]              \n"
 513                         "18:    stb.ab  %3, [%1,1]              \n"
 514                         "       sub     %0, %0, 1               \n"
 515                         "34:    ;nop                            \n"
 516                         "       .section .fixup, \"ax\"         \n"
 517                         "       .align 4                        \n"
 518                         "4:     j   34b                         \n"
 519                         "       .previous                       \n"
 520                         "       .section __ex_table, \"a\"      \n"
 521                         "       .align 4                        \n"
 522                         "       .word   18b,4b                  \n"
 523                         "       .previous                       \n"
 524                         : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
 525                         :
 526                         : "memory");
 527                 }
 528         } else {  /* n is NOT constant, so laddered copy of 16x,8,4,2,1  */
 529
 530                 __asm__ __volatile__(
 531                 "       mov   %0,%3                     \n"
 532                 "       lsr.f lp_count, %3,4            \n"  /* 16x bytes */
 533                 "       lpnz  3f                        \n"
 534                 "       ld.ab %5, [%2, 4]               \n"
 535                 "       ld.ab %6, [%2, 4]               \n"
 536                 "       ld.ab %7, [%2, 4]               \n"
 537                 "       ld.ab %8, [%2, 4]               \n"
 538                 "1:     st.ab %5, [%1, 4]               \n"
 539                 "11:    st.ab %6, [%1, 4]               \n"
 540                 "12:    st.ab %7, [%1, 4]               \n"
 541                 "13:    st.ab %8, [%1, 4]               \n"
 542                 "       sub   %0, %0, 16                \n"
 543                 "3:     and.f %3,%3,0xf                 \n" /* stragglers */
 544                 "       bz 34f                          \n"
 545                 "       bbit0   %3,3,31f                \n" /* 8 bytes left */
 546                 "       ld.ab   %5, [%2,4]              \n"
 547                 "       ld.ab   %6, [%2,4]              \n"
 548                 "14:    st.ab   %5, [%1,4]              \n"
 549                 "15:    st.ab   %6, [%1,4]              \n"
 550                 "       sub.f   %0, %0, 8               \n"
 551                 "31:    bbit0   %3,2,32f                \n"  /* 4 bytes left */
 552                 "       ld.ab   %5, [%2,4]              \n"
 553                 "16:    st.ab   %5, [%1,4]              \n"
 554                 "       sub.f   %0, %0, 4               \n"
 555                 "32:    bbit0 %3,1,33f                  \n"  /* 2 bytes left */
 556                 "       ldw.ab    %5, [%2,2]            \n"
 557                 "17:    stw.ab    %5, [%1,2]            \n"
 558                 "       sub.f %0, %0, 2                 \n"
 559                 "33:    bbit0 %3,0,34f                  \n"
 560                 "       ldb.ab    %5, [%2,1]            \n"  /* 1 byte left */
 561                 "18:    stb.ab  %5, [%1,1]              \n"
 562                 "       sub.f %0, %0, 1                 \n"
 563                 "34:    ;nop                            \n"
 564                 "       .section .fixup, \"ax\"         \n"
 565                 "       .align 4                        \n"
 566                 "4:     j   34b                         \n"
 567                 "       .previous                       \n"
 568                 "       .section __ex_table, \"a\"      \n"
 569                 "       .align 4                        \n"
 570                 "       .word   1b, 4b                  \n"
 571                 "       .word   11b,4b                  \n"
 572                 "       .word   12b,4b                  \n"
 573                 "       .word   13b,4b                  \n"
 574                 "       .word   14b,4b                  \n"
 575                 "       .word   15b,4b                  \n"
 576                 "       .word   16b,4b                  \n"
 577                 "       .word   17b,4b                  \n"
 578                 "       .word   18b,4b                  \n"
 579                 "       .previous                       \n"
 580                 : "=r" (res), "+r"(to), "+r"(from), "+r"(n), "=r"(val),
 581                   "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
 582                 :
 583                 : "lp_count", "memory");
 584         }
 585
 586         return res;
 587 }
 588
 589 static inline unsigned long __clear_user(void __user *to, unsigned long n)
 590 {
 591         long res = n;
 592         unsigned char *d_char = to;
 593
 594         __asm__ __volatile__(
 595         "       bbit0   %0, 0, 1f               \n"
 596         "75:    stb.ab  %2, [%0,1]              \n"
 597         "       sub %1, %1, 1                   \n"
 598         "1:     bbit0   %0, 1, 2f               \n"
 599         "76:    stw.ab  %2, [%0,2]              \n"
 600         "       sub %1, %1, 2                   \n"
 601         "2:     asr.f   lp_count, %1, 2         \n"
 602         "       lpnz    3f                      \n"
 603         "77:    st.ab   %2, [%0,4]              \n"
 604         "       sub %1, %1, 4                   \n"
 605         "3:     bbit0   %1, 1, 4f               \n"
 606         "78:    stw.ab  %2, [%0,2]              \n"
 607         "       sub %1, %1, 2                   \n"
 608         "4:     bbit0   %1, 0, 5f               \n"
 609         "79:    stb.ab  %2, [%0,1]              \n"
 610         "       sub %1, %1, 1                   \n"
 611         "5:                                     \n"
 612         "       .section .fixup, \"ax\"         \n"
 613         "       .align 4                        \n"
 614         "3:     j   5b                          \n"
 615         "       .previous                       \n"
 616         "       .section __ex_table, \"a\"      \n"
 617         "       .align 4                        \n"
 618         "       .word   75b, 3b                 \n"
 619         "       .word   76b, 3b                 \n"
 620         "       .word   77b, 3b                 \n"
 621         "       .word   78b, 3b                 \n"
 622         "       .word   79b, 3b                 \n"
 623         "       .previous                       \n"
 624         : "+r"(d_char), "+r"(res)
 625         : "i"(0)
 626         : "lp_count", "memory");
 627
 628         return res;
 629 }
 630
 631 #define INLINE_COPY_TO_USER
 632 #define INLINE_COPY_FROM_USER
 633
 634 #define __clear_user                    __clear_user
 635
 636 #include <asm-generic/uaccess.h>
 637
 638 #endif