[fio.git] / crc / crc32c-intel.c

#include <inttypes.h>
#include <string.h>
#include <unistd.h>
#include <stdlib.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/wait.h>
#include "crc32c.h"

/*
 * Based on a posting to lkml by Austin Zhang <austin.zhang@intel.com>
 *
 * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
 * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
 * http://www.intel.com/products/processor/manuals/
 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
 * Volume 2A: Instruction Set Reference, A-M
 */

int crc32c_intel_available = 0;

#ifdef ARCH_HAVE_SSE4_2

#if BITS_PER_LONG == 64
#define REX_PRE "0x48, "
#define SCALE_F 8
#else
#define REX_PRE
#define SCALE_F 4
#endif

static int crc32c_probed;

static uint32_t crc32c_intel_le_hw_byte(uint32_t crc, unsigned char const *data,
					unsigned long length)
{
	while (length--) {
		__asm__ __volatile__(
			".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
			:"=S"(crc)
			:"0"(crc), "c"(*data)
		);
		data++;
	}

	return crc;
}

/*
 * Steps through buffer one byte at at time, calculates reflected 
 * crc using table.
 */
uint32_t crc32c_intel(unsigned char const *data, unsigned long length)
{
	unsigned int iquotient = length / SCALE_F;
	unsigned int iremainder = length % SCALE_F;
#if BITS_PER_LONG == 64
	uint64_t *ptmp = (uint64_t *) data;
#else
	uint32_t *ptmp = (uint32_t *) data;
#endif
	uint32_t crc = ~0;

	while (iquotient--) {
		__asm__ __volatile__(
			".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
			:"=S"(crc)
			:"0"(crc), "c"(*ptmp)
		);
		ptmp++;
	}

	if (iremainder)
		crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp,
				 iremainder);

	return crc;
}

void crc32c_intel_probe(void)
{
	if (!crc32c_probed) {
		unsigned int eax, ebx, ecx = 0, edx;

		eax = 1;

		do_cpuid(&eax, &ebx, &ecx, &edx);
		crc32c_intel_available = (ecx & (1 << 20)) != 0;
		crc32c_probed = 1;
	}
}

#endif /* ARCH_HAVE_SSE */
Commit	Line	Data
3845591f	1	#include <inttypes.h>
5d7c5d34 JA	2	#include <string.h>
	3	#include <unistd.h>
	4	#include <stdlib.h>
	5	#include <signal.h>
	6	#include <sys/types.h>
	7	#include <sys/wait.h>
419484b9	8	#include "crc32c.h"
3845591f JA	9
	10	/*
	11	* Based on a posting to lkml by Austin Zhang <austin.zhang@intel.com>
	12	*
	13	* Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
	14	* CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
	15	* CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
	16	* http://www.intel.com/products/processor/manuals/
	17	* Intel(R) 64 and IA-32 Architectures Software Developer's Manual
	18	* Volume 2A: Instruction Set Reference, A-M
	19	*/
	20
e3aaafc4 JA	21	int crc32c_intel_available = 0;
e3aaafc4 JA	22
2f68124f	23	#ifdef ARCH_HAVE_SSE4_2
419484b9	24
3845591f JA	25	#if BITS_PER_LONG == 64
	26	#define REX_PRE "0x48, "
	27	#define SCALE_F 8
	28	#else
	29	#define REX_PRE
	30	#define SCALE_F 4
	31	#endif
	32
e3aaafc4 JA	33	static int crc32c_probed;
e3aaafc4 JA	34
cc62ea70 JA	35	static uint32_t crc32c_intel_le_hw_byte(uint32_t crc, unsigned char const *data,
cc62ea70 JA	36	unsigned long length)
3845591f JA	37	{
	38	while (length--) {
	39	__asm__ __volatile__(
	40	".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
	41	:"=S"(crc)
	42	:"0"(crc), "c"(*data)
	43	);
	44	data++;
	45	}
	46
	47	return crc;
	48	}
	49
	50	/*
	51	* Steps through buffer one byte at at time, calculates reflected
	52	* crc using table.
	53	*/
	54	uint32_t crc32c_intel(unsigned char const *data, unsigned long length)
	55	{
	56	unsigned int iquotient = length / SCALE_F;
	57	unsigned int iremainder = length % SCALE_F;
	58	#if BITS_PER_LONG == 64
	59	uint64_t ptmp = (uint64_t ) data;
	60	#else
	61	uint32_t ptmp = (uint32_t ) data;
	62	#endif
	63	uint32_t crc = ~0;
	64
	65	while (iquotient--) {
	66	__asm__ __volatile__(
	67	".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
	68	:"=S"(crc)
	69	:"0"(crc), "c"(*ptmp)
	70	);
	71	ptmp++;
	72	}
	73
	74	if (iremainder)
	75	crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp,
	76	iremainder);
	77
	78	return crc;
	79	}
419484b9	80
e3aaafc4	81	void crc32c_intel_probe(void)
5d7c5d34	82	{
e3aaafc4	83	if (!crc32c_probed) {
267339ff	84	unsigned int eax, ebx, ecx = 0, edx;
419484b9	85
e3aaafc4	86	eax = 1;
e0ab5f97	87
e3aaafc4 JA	88	do_cpuid(&eax, &ebx, &ecx, &edx);
	89	crc32c_intel_available = (ecx & (1 << 20)) != 0;
	90	crc32c_probed = 1;
	91	}
5d7c5d34 JA	92	}
	93
	94	#endif /* ARCH_HAVE_SSE */