[linux-2.6-block.git] / arch / x86 / include / asm / string_64.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_STRING_64_H
#define _ASM_X86_STRING_64_H

#ifdef __KERNEL__
#include <linux/jump_label.h>

/* Written 2002 by Andi Kleen */

/* Even with __builtin_ the compiler may decide to use the out of line
   function. */

#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY)
#include <linux/kmsan_string.h>
#endif

#define __HAVE_ARCH_MEMCPY 1
#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY)
#undef memcpy
#define memcpy __msan_memcpy
#else
extern void *memcpy(void *to, const void *from, size_t len);
#endif
extern void *__memcpy(void *to, const void *from, size_t len);

#define __HAVE_ARCH_MEMSET
#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY)
extern void *__msan_memset(void *s, int c, size_t n);
#undef memset
#define memset __msan_memset
#else
void *memset(void *s, int c, size_t n);
#endif
void *__memset(void *s, int c, size_t n);

#define __HAVE_ARCH_MEMSET16
static inline void *memset16(uint16_t *s, uint16_t v, size_t n)
{
	long d0, d1;
	asm volatile("rep\n\t"
		     "stosw"
		     : "=&c" (d0), "=&D" (d1)
		     : "a" (v), "1" (s), "0" (n)
		     : "memory");
	return s;
}

#define __HAVE_ARCH_MEMSET32
static inline void *memset32(uint32_t *s, uint32_t v, size_t n)
{
	long d0, d1;
	asm volatile("rep\n\t"
		     "stosl"
		     : "=&c" (d0), "=&D" (d1)
		     : "a" (v), "1" (s), "0" (n)
		     : "memory");
	return s;
}

#define __HAVE_ARCH_MEMSET64
static inline void *memset64(uint64_t *s, uint64_t v, size_t n)
{
	long d0, d1;
	asm volatile("rep\n\t"
		     "stosq"
		     : "=&c" (d0), "=&D" (d1)
		     : "a" (v), "1" (s), "0" (n)
		     : "memory");
	return s;
}

#define __HAVE_ARCH_MEMMOVE
#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY)
#undef memmove
void *__msan_memmove(void *dest, const void *src, size_t len);
#define memmove __msan_memmove
#else
void *memmove(void *dest, const void *src, size_t count);
#endif
void *__memmove(void *dest, const void *src, size_t count);

int memcmp(const void *cs, const void *ct, size_t count);
size_t strlen(const char *s);
char *strcpy(char *dest, const char *src);
char *strcat(char *dest, const char *src);
int strcmp(const char *cs, const char *ct);

#if (defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__))
/*
 * For files that not instrumented (e.g. mm/slub.c) we
 * should use not instrumented version of mem* functions.
 */

#undef memcpy
#define memcpy(dst, src, len) __memcpy(dst, src, len)
#undef memmove
#define memmove(dst, src, len) __memmove(dst, src, len)
#undef memset
#define memset(s, c, n) __memset(s, c, n)

#ifndef __NO_FORTIFY
#define __NO_FORTIFY /* FORTIFY_SOURCE uses __builtin_memcpy, etc. */
#endif

#endif

#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
#define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
void __memcpy_flushcache(void *dst, const void *src, size_t cnt);
static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t cnt)
{
	if (__builtin_constant_p(cnt)) {
		switch (cnt) {
			case 4:
				asm ("movntil %1, %0" : "=m"(*(u32 *)dst) : "r"(*(u32 *)src));
				return;
			case 8:
				asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src));
				return;
			case 16:
				asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src));
				asm ("movntiq %1, %0" : "=m"(*(u64 *)(dst + 8)) : "r"(*(u64 *)(src + 8)));
				return;
		}
	}
	__memcpy_flushcache(dst, src, cnt);
}
#endif

#endif /* __KERNEL__ */

#endif /* _ASM_X86_STRING_64_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1965aae3 PA	2	#ifndef _ASM_X86_STRING_64_H
1965aae3 PA	3	#define _ASM_X86_STRING_64_H
1da177e4 LT	4
1da177e4 LT	5	#ifdef __KERNEL__
3637efb0	6	#include <linux/jump_label.h>
1da177e4	7
953b2f1e	8	/* Written 2002 by Andi Kleen */
1da177e4	9
1da177e4 LT	10	/* Even with __builtin_ the compiler may decide to use the out of line
	11	function. */
	12
78a498c3 AP	13	#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY)
	14	#include <linux/kmsan_string.h>
	15	#endif
	16
1da177e4	17	#define __HAVE_ARCH_MEMCPY 1
78a498c3	18	#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY)
ff901d80	19	#undef memcpy
ff901d80 AP	20	#define memcpy __msan_memcpy
ff901d80 AP	21	#else
a75ca545	22	extern void memcpy(void to, const void *from, size_t len);
ff901d80	23	#endif
393f203f AR	24	extern void __memcpy(void to, const void *from, size_t len);
393f203f AR	25
1da177e4	26	#define __HAVE_ARCH_MEMSET
78a498c3	27	#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY)
ff901d80 AP	28	extern void __msan_memset(void s, int c, size_t n);
	29	#undef memset
	30	#define memset __msan_memset
	31	#else
6edfba1b	32	void memset(void s, int c, size_t n);
ff901d80	33	#endif
393f203f	34	void __memset(void s, int c, size_t n);
1da177e4	35
4c512485 MW	36	#define __HAVE_ARCH_MEMSET16
	37	static inline void memset16(uint16_t s, uint16_t v, size_t n)
	38	{
	39	long d0, d1;
	40	asm volatile("rep\n\t"
	41	"stosw"
	42	: "=&c" (d0), "=&D" (d1)
	43	: "a" (v), "1" (s), "0" (n)
	44	: "memory");
	45	return s;
	46	}
	47
	48	#define __HAVE_ARCH_MEMSET32
	49	static inline void memset32(uint32_t s, uint32_t v, size_t n)
	50	{
	51	long d0, d1;
	52	asm volatile("rep\n\t"
	53	"stosl"
	54	: "=&c" (d0), "=&D" (d1)
	55	: "a" (v), "1" (s), "0" (n)
	56	: "memory");
	57	return s;
	58	}
	59
	60	#define __HAVE_ARCH_MEMSET64
	61	static inline void memset64(uint64_t s, uint64_t v, size_t n)
	62	{
	63	long d0, d1;
	64	asm volatile("rep\n\t"
	65	"stosq"
	66	: "=&c" (d0), "=&D" (d1)
	67	: "a" (v), "1" (s), "0" (n)
	68	: "memory");
	69	return s;
	70	}
	71
1da177e4	72	#define __HAVE_ARCH_MEMMOVE
78a498c3	73	#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY)
ff901d80 AP	74	#undef memmove
	75	void __msan_memmove(void dest, const void *src, size_t len);
	76	#define memmove __msan_memmove
	77	#else
953b2f1e	78	void memmove(void dest, const void *src, size_t count);
ff901d80	79	#endif
393f203f	80	void __memmove(void dest, const void *src, size_t count);
1da177e4	81
953b2f1e JP	82	int memcmp(const void cs, const void ct, size_t count);
	83	size_t strlen(const char *s);
	84	char strcpy(char dest, const char *src);
	85	char strcat(char dest, const char *src);
	86	int strcmp(const char cs, const char ct);
1da177e4	87
ff901d80	88	#if (defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__))
393f203f AR	89	/*
	90	* For files that not instrumented (e.g. mm/slub.c) we
	91	* should use not instrumented version of mem* functions.
	92	*/
	93
	94	#undef memcpy
	95	#define memcpy(dst, src, len) __memcpy(dst, src, len)
ff901d80	96	#undef memmove
393f203f	97	#define memmove(dst, src, len) __memmove(dst, src, len)
ff901d80	98	#undef memset
393f203f	99	#define memset(s, c, n) __memset(s, c, n)
6974f0c4 DM	100
	101	#ifndef __NO_FORTIFY
	102	#define __NO_FORTIFY /* FORTIFY_SOURCE uses __builtin_memcpy, etc. */
	103	#endif
	104
393f203f AR	105	#endif
393f203f AR	106
0aed55af DW	107	#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
0aed55af DW	108	#define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
02101c45 MP	109	void __memcpy_flushcache(void dst, const void src, size_t cnt);
	110	static __always_inline void memcpy_flushcache(void dst, const void src, size_t cnt)
	111	{
	112	if (__builtin_constant_p(cnt)) {
	113	switch (cnt) {
	114	case 4:
	115	asm ("movntil %1, %0" : "=m"((u32 )dst) : "r"((u32 )src));
	116	return;
	117	case 8:
	118	asm ("movntiq %1, %0" : "=m"((u64 )dst) : "r"((u64 )src));
	119	return;
	120	case 16:
	121	asm ("movntiq %1, %0" : "=m"((u64 )dst) : "r"((u64 )src));
	122	asm ("movntiq %1, %0" : "=m"((u64 )(dst + 8)) : "r"((u64 )(src + 8)));
	123	return;
	124	}
	125	}
	126	__memcpy_flushcache(dst, src, cnt);
	127	}
0aed55af DW	128	#endif
0aed55af DW	129
1da177e4 LT	130	#endif /* __KERNEL__ */
1da177e4 LT	131
1965aae3	132	#endif /* _ASM_X86_STRING_64_H */