crc/crc32c-intel.c

   1 #include <inttypes.h>
   2 #include <string.h>
   3 #include <unistd.h>
   4 #include <stdlib.h>
   5 #include <signal.h>
   6 #include <sys/types.h>
   7 #include <sys/wait.h>
   8 #include "crc32c.h"
   9
  10 /*
  11  * Based on a posting to lkml by Austin Zhang <austin.zhang@intel.com>
  12  *
  13  * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
  14  * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
  15  * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
  16  * http://www.intel.com/products/processor/manuals/
  17  * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
  18  * Volume 2A: Instruction Set Reference, A-M
  19  */
  20
  21 bool crc32c_intel_available = false;
  22
  23 #ifdef ARCH_HAVE_SSE4_2
  24
  25 #if BITS_PER_LONG == 64
  26 #define REX_PRE "0x48, "
  27 #define SCALE_F 8
  28 #else
  29 #define REX_PRE
  30 #define SCALE_F 4
  31 #endif
  32
  33 static bool crc32c_probed;
  34
  35 static uint32_t crc32c_intel_le_hw_byte(uint32_t crc, unsigned char const *data,
  36                                         unsigned long length)
  37 {
  38         while (length--) {
  39                 __asm__ __volatile__(
  40                         ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
  41                         :"=S"(crc)
  42                         :"0"(crc), "c"(*data)
  43                 );
  44                 data++;
  45         }
  46
  47         return crc;
  48 }
  49
  50 /*
  51  * Steps through buffer one byte at at time, calculates reflected
  52  * crc using table.
  53  */
  54 uint32_t crc32c_intel(unsigned char const *data, unsigned long length)
  55 {
  56         unsigned int iquotient = length / SCALE_F;
  57         unsigned int iremainder = length % SCALE_F;
  58 #if BITS_PER_LONG == 64
  59         uint64_t *ptmp = (uint64_t *) data;
  60 #else
  61         uint32_t *ptmp = (uint32_t *) data;
  62 #endif
  63         uint32_t crc = ~0;
  64
  65         while (iquotient--) {
  66                 __asm__ __volatile__(
  67                         ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
  68                         :"=S"(crc)
  69                         :"0"(crc), "c"(*ptmp)
  70                 );
  71                 ptmp++;
  72         }
  73
  74         if (iremainder)
  75                 crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp,
  76                                  iremainder);
  77
  78         return crc;
  79 }
  80
  81 void crc32c_intel_probe(void)
  82 {
  83         if (!crc32c_probed) {
  84                 unsigned int eax, ebx, ecx = 0, edx;
  85
  86                 eax = 1;
  87
  88                 do_cpuid(&eax, &ebx, &ecx, &edx);
  89                 crc32c_intel_available = (ecx & (1 << 20)) != 0;
  90                 crc32c_probed = true;
  91         }
  92 }
  93
  94 #endif /* ARCH_HAVE_SSE */