[linux-block.git] / lib / iomap.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Implement the default iomap interfaces
 *
 * (C) Copyright 2004 Linus Torvalds
 */
#include <linux/pci.h>
#include <linux/io.h>
#include <linux/kmsan-checks.h>

#include <linux/export.h>

/*
 * Read/write from/to an (offsettable) iomem cookie. It might be a PIO
 * access or a MMIO access, these functions don't care. The info is
 * encoded in the hardware mapping set up by the mapping functions
 * (or the cookie itself, depending on implementation and hw).
 *
 * The generic routines don't assume any hardware mappings, and just
 * encode the PIO/MMIO as part of the cookie. They coldly assume that
 * the MMIO IO mappings are not in the low address range.
 *
 * Architectures for which this is not true can't use this generic
 * implementation and should do their own copy.
 */

#ifndef HAVE_ARCH_PIO_SIZE
/*
 * We encode the physical PIO addresses (0-0xffff) into the
 * pointer by offsetting them with a constant (0x10000) and
 * assuming that all the low addresses are always PIO. That means
 * we can do some sanity checks on the low bits, and don't
 * need to just take things for granted.
 */
#define PIO_OFFSET	0x10000UL
#define PIO_MASK	0x0ffffUL
#define PIO_RESERVED	0x40000UL
#endif

static void bad_io_access(unsigned long port, const char *access)
{
	static int count = 10;
	if (count) {
		count--;
		WARN(1, KERN_ERR "Bad IO access at port %#lx (%s)\n", port, access);
	}
}

/*
 * Ugly macros are a way of life.
 */
#define IO_COND(addr, is_pio, is_mmio) do {			\
	unsigned long port = (unsigned long __force)addr;	\
	if (port >= PIO_RESERVED) {				\
		is_mmio;					\
	} else if (port > PIO_OFFSET) {				\
		port &= PIO_MASK;				\
		is_pio;						\
	} else							\
		bad_io_access(port, #is_pio );			\
} while (0)

#ifndef pio_read16be
#define pio_read16be(port) swab16(inw(port))
#define pio_read32be(port) swab32(inl(port))
#endif

#ifndef mmio_read16be
#define mmio_read16be(addr) swab16(readw(addr))
#define mmio_read32be(addr) swab32(readl(addr))
#define mmio_read64be(addr) swab64(readq(addr))
#endif

/*
 * Here and below, we apply __no_kmsan_checks to functions reading data from
 * hardware, to ensure that KMSAN marks their return values as initialized.
 */
__no_kmsan_checks
unsigned int ioread8(const void __iomem *addr)
{
	IO_COND(addr, return inb(port), return readb(addr));
	return 0xff;
}
__no_kmsan_checks
unsigned int ioread16(const void __iomem *addr)
{
	IO_COND(addr, return inw(port), return readw(addr));
	return 0xffff;
}
__no_kmsan_checks
unsigned int ioread16be(const void __iomem *addr)
{
	IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr));
	return 0xffff;
}
__no_kmsan_checks
unsigned int ioread32(const void __iomem *addr)
{
	IO_COND(addr, return inl(port), return readl(addr));
	return 0xffffffff;
}
__no_kmsan_checks
unsigned int ioread32be(const void __iomem *addr)
{
	IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr));
	return 0xffffffff;
}
EXPORT_SYMBOL(ioread8);
EXPORT_SYMBOL(ioread16);
EXPORT_SYMBOL(ioread16be);
EXPORT_SYMBOL(ioread32);
EXPORT_SYMBOL(ioread32be);

#ifdef readq
static u64 pio_read64_lo_hi(unsigned long port)
{
	u64 lo, hi;

	lo = inl(port);
	hi = inl(port + sizeof(u32));

	return lo | (hi << 32);
}

static u64 pio_read64_hi_lo(unsigned long port)
{
	u64 lo, hi;

	hi = inl(port + sizeof(u32));
	lo = inl(port);

	return lo | (hi << 32);
}

static u64 pio_read64be_lo_hi(unsigned long port)
{
	u64 lo, hi;

	lo = pio_read32be(port + sizeof(u32));
	hi = pio_read32be(port);

	return lo | (hi << 32);
}

static u64 pio_read64be_hi_lo(unsigned long port)
{
	u64 lo, hi;

	hi = pio_read32be(port);
	lo = pio_read32be(port + sizeof(u32));

	return lo | (hi << 32);
}

__no_kmsan_checks
u64 ioread64_lo_hi(const void __iomem *addr)
{
	IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr));
	return 0xffffffffffffffffULL;
}

__no_kmsan_checks
u64 ioread64_hi_lo(const void __iomem *addr)
{
	IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr));
	return 0xffffffffffffffffULL;
}

__no_kmsan_checks
u64 ioread64be_lo_hi(const void __iomem *addr)
{
	IO_COND(addr, return pio_read64be_lo_hi(port),
		return mmio_read64be(addr));
	return 0xffffffffffffffffULL;
}

__no_kmsan_checks
u64 ioread64be_hi_lo(const void __iomem *addr)
{
	IO_COND(addr, return pio_read64be_hi_lo(port),
		return mmio_read64be(addr));
	return 0xffffffffffffffffULL;
}

EXPORT_SYMBOL(ioread64_lo_hi);
EXPORT_SYMBOL(ioread64_hi_lo);
EXPORT_SYMBOL(ioread64be_lo_hi);
EXPORT_SYMBOL(ioread64be_hi_lo);

#endif /* readq */

#ifndef pio_write16be
#define pio_write16be(val,port) outw(swab16(val),port)
#define pio_write32be(val,port) outl(swab32(val),port)
#endif

#ifndef mmio_write16be
#define mmio_write16be(val,port) writew(swab16(val),port)
#define mmio_write32be(val,port) writel(swab32(val),port)
#define mmio_write64be(val,port) writeq(swab64(val),port)
#endif

void iowrite8(u8 val, void __iomem *addr)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(&val, sizeof(val));
	IO_COND(addr, outb(val,port), writeb(val, addr));
}
void iowrite16(u16 val, void __iomem *addr)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(&val, sizeof(val));
	IO_COND(addr, outw(val,port), writew(val, addr));
}
void iowrite16be(u16 val, void __iomem *addr)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(&val, sizeof(val));
	IO_COND(addr, pio_write16be(val,port), mmio_write16be(val, addr));
}
void iowrite32(u32 val, void __iomem *addr)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(&val, sizeof(val));
	IO_COND(addr, outl(val,port), writel(val, addr));
}
void iowrite32be(u32 val, void __iomem *addr)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(&val, sizeof(val));
	IO_COND(addr, pio_write32be(val,port), mmio_write32be(val, addr));
}
EXPORT_SYMBOL(iowrite8);
EXPORT_SYMBOL(iowrite16);
EXPORT_SYMBOL(iowrite16be);
EXPORT_SYMBOL(iowrite32);
EXPORT_SYMBOL(iowrite32be);

#ifdef writeq
static void pio_write64_lo_hi(u64 val, unsigned long port)
{
	outl(val, port);
	outl(val >> 32, port + sizeof(u32));
}

static void pio_write64_hi_lo(u64 val, unsigned long port)
{
	outl(val >> 32, port + sizeof(u32));
	outl(val, port);
}

static void pio_write64be_lo_hi(u64 val, unsigned long port)
{
	pio_write32be(val, port + sizeof(u32));
	pio_write32be(val >> 32, port);
}

static void pio_write64be_hi_lo(u64 val, unsigned long port)
{
	pio_write32be(val >> 32, port);
	pio_write32be(val, port + sizeof(u32));
}

void iowrite64_lo_hi(u64 val, void __iomem *addr)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(&val, sizeof(val));
	IO_COND(addr, pio_write64_lo_hi(val, port),
		writeq(val, addr));
}

void iowrite64_hi_lo(u64 val, void __iomem *addr)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(&val, sizeof(val));
	IO_COND(addr, pio_write64_hi_lo(val, port),
		writeq(val, addr));
}

void iowrite64be_lo_hi(u64 val, void __iomem *addr)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(&val, sizeof(val));
	IO_COND(addr, pio_write64be_lo_hi(val, port),
		mmio_write64be(val, addr));
}

void iowrite64be_hi_lo(u64 val, void __iomem *addr)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(&val, sizeof(val));
	IO_COND(addr, pio_write64be_hi_lo(val, port),
		mmio_write64be(val, addr));
}

EXPORT_SYMBOL(iowrite64_lo_hi);
EXPORT_SYMBOL(iowrite64_hi_lo);
EXPORT_SYMBOL(iowrite64be_lo_hi);
EXPORT_SYMBOL(iowrite64be_hi_lo);

#endif /* readq */

/*
 * These are the "repeat MMIO read/write" functions.
 * Note the "__raw" accesses, since we don't want to
 * convert to CPU byte order. We write in "IO byte
 * order" (we also don't have IO barriers).
 */
#ifndef mmio_insb
static inline void mmio_insb(const void __iomem *addr, u8 *dst, int count)
{
	while (--count >= 0) {
		u8 data = __raw_readb(addr);
		*dst = data;
		dst++;
	}
}
static inline void mmio_insw(const void __iomem *addr, u16 *dst, int count)
{
	while (--count >= 0) {
		u16 data = __raw_readw(addr);
		*dst = data;
		dst++;
	}
}
static inline void mmio_insl(const void __iomem *addr, u32 *dst, int count)
{
	while (--count >= 0) {
		u32 data = __raw_readl(addr);
		*dst = data;
		dst++;
	}
}
#endif

#ifndef mmio_outsb
static inline void mmio_outsb(void __iomem *addr, const u8 *src, int count)
{
	while (--count >= 0) {
		__raw_writeb(*src, addr);
		src++;
	}
}
static inline void mmio_outsw(void __iomem *addr, const u16 *src, int count)
{
	while (--count >= 0) {
		__raw_writew(*src, addr);
		src++;
	}
}
static inline void mmio_outsl(void __iomem *addr, const u32 *src, int count)
{
	while (--count >= 0) {
		__raw_writel(*src, addr);
		src++;
	}
}
#endif

void ioread8_rep(const void __iomem *addr, void *dst, unsigned long count)
{
	IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count));
	/* KMSAN must treat values read from devices as initialized. */
	kmsan_unpoison_memory(dst, count);
}
void ioread16_rep(const void __iomem *addr, void *dst, unsigned long count)
{
	IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count));
	/* KMSAN must treat values read from devices as initialized. */
	kmsan_unpoison_memory(dst, count * 2);
}
void ioread32_rep(const void __iomem *addr, void *dst, unsigned long count)
{
	IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count));
	/* KMSAN must treat values read from devices as initialized. */
	kmsan_unpoison_memory(dst, count * 4);
}
EXPORT_SYMBOL(ioread8_rep);
EXPORT_SYMBOL(ioread16_rep);
EXPORT_SYMBOL(ioread32_rep);

void iowrite8_rep(void __iomem *addr, const void *src, unsigned long count)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(src, count);
	IO_COND(addr, outsb(port, src, count), mmio_outsb(addr, src, count));
}
void iowrite16_rep(void __iomem *addr, const void *src, unsigned long count)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(src, count * 2);
	IO_COND(addr, outsw(port, src, count), mmio_outsw(addr, src, count));
}
void iowrite32_rep(void __iomem *addr, const void *src, unsigned long count)
{
	/* Make sure uninitialized memory isn't copied to devices. */
	kmsan_check_memory(src, count * 4);
	IO_COND(addr, outsl(port, src,count), mmio_outsl(addr, src, count));
}
EXPORT_SYMBOL(iowrite8_rep);
EXPORT_SYMBOL(iowrite16_rep);
EXPORT_SYMBOL(iowrite32_rep);

#ifdef CONFIG_HAS_IOPORT_MAP
/* Create a virtual mapping cookie for an IO port range */
void __iomem *ioport_map(unsigned long port, unsigned int nr)
{
	if (port > PIO_MASK)
		return NULL;
	return (void __iomem *) (unsigned long) (port + PIO_OFFSET);
}

void ioport_unmap(void __iomem *addr)
{
	/* Nothing to do */
}
EXPORT_SYMBOL(ioport_map);
EXPORT_SYMBOL(ioport_unmap);
#endif /* CONFIG_HAS_IOPORT_MAP */

#ifdef CONFIG_PCI
/* Hide the details if this is a MMIO or PIO address space and just do what
 * you expect in the correct way. */
void pci_iounmap(struct pci_dev *dev, void __iomem * addr)
{
	IO_COND(addr, /* nothing */, iounmap(addr));
}
EXPORT_SYMBOL(pci_iounmap);
#endif /* CONFIG_PCI */
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
1da177e4 LT	2	/*
	3	* Implement the default iomap interfaces
	4	*
	5	* (C) Copyright 2004 Linus Torvalds
	6	*/
	7	#include <linux/pci.h>
9ac7849e	8	#include <linux/io.h>
a28a4d47	9	#include <linux/kmsan-checks.h>
9ac7849e	10
8bc3bcc9	11	#include <linux/export.h>
1da177e4 LT	12
	13	/*
	14	* Read/write from/to an (offsettable) iomem cookie. It might be a PIO
	15	* access or a MMIO access, these functions don't care. The info is
	16	* encoded in the hardware mapping set up by the mapping functions
	17	* (or the cookie itself, depending on implementation and hw).
	18	*
	19	* The generic routines don't assume any hardware mappings, and just
	20	* encode the PIO/MMIO as part of the cookie. They coldly assume that
	21	* the MMIO IO mappings are not in the low address range.
	22	*
	23	* Architectures for which this is not true can't use this generic
	24	* implementation and should do their own copy.
	25	*/
	26
	27	#ifndef HAVE_ARCH_PIO_SIZE
	28	/*
	29	* We encode the physical PIO addresses (0-0xffff) into the
	30	* pointer by offsetting them with a constant (0x10000) and
	31	* assuming that all the low addresses are always PIO. That means
	32	* we can do some sanity checks on the low bits, and don't
	33	* need to just take things for granted.
	34	*/
	35	#define PIO_OFFSET 0x10000UL
	36	#define PIO_MASK 0x0ffffUL
	37	#define PIO_RESERVED 0x40000UL
	38	#endif
	39
6cbf0c70 LT	40	static void bad_io_access(unsigned long port, const char *access)
	41	{
	42	static int count = 10;
	43	if (count) {
	44	count--;
5cd2b459	45	WARN(1, KERN_ERR "Bad IO access at port %#lx (%s)\n", port, access);
6cbf0c70 LT	46	}
	47	}
	48
1da177e4 LT	49	/*
	50	* Ugly macros are a way of life.
	51	*/
1da177e4 LT	52	#define IO_COND(addr, is_pio, is_mmio) do { \
1da177e4 LT	53	unsigned long port = (unsigned long __force)addr; \
6cbf0c70 LT	54	if (port >= PIO_RESERVED) { \
	55	is_mmio; \
	56	} else if (port > PIO_OFFSET) { \
1da177e4 LT	57	port &= PIO_MASK; \
1da177e4 LT	58	is_pio; \
6cbf0c70 LT	59	} else \
6cbf0c70 LT	60	bad_io_access(port, #is_pio ); \
1da177e4 LT	61	} while (0)
1da177e4 LT	62
34ba8a5c LT	63	#ifndef pio_read16be
	64	#define pio_read16be(port) swab16(inw(port))
	65	#define pio_read32be(port) swab32(inl(port))
	66	#endif
	67
	68	#ifndef mmio_read16be
aecc787c LG	69	#define mmio_read16be(addr) swab16(readw(addr))
aecc787c LG	70	#define mmio_read32be(addr) swab32(readl(addr))
79bf0cbd	71	#define mmio_read64be(addr) swab64(readq(addr))
34ba8a5c LT	72	#endif
34ba8a5c LT	73
a28a4d47 AP	74	/*
	75	* Here and below, we apply __no_kmsan_checks to functions reading data from
	76	* hardware, to ensure that KMSAN marks their return values as initialized.
	77	*/
	78	__no_kmsan_checks
8f28ca6b	79	unsigned int ioread8(const void __iomem *addr)
1da177e4 LT	80	{
1da177e4 LT	81	IO_COND(addr, return inb(port), return readb(addr));
6cbf0c70	82	return 0xff;
1da177e4	83	}
a28a4d47	84	__no_kmsan_checks
8f28ca6b	85	unsigned int ioread16(const void __iomem *addr)
1da177e4 LT	86	{
1da177e4 LT	87	IO_COND(addr, return inw(port), return readw(addr));
6cbf0c70	88	return 0xffff;
1da177e4	89	}
a28a4d47	90	__no_kmsan_checks
8f28ca6b	91	unsigned int ioread16be(const void __iomem *addr)
dae409a2	92	{
34ba8a5c	93	IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr));
6cbf0c70	94	return 0xffff;
dae409a2	95	}
a28a4d47	96	__no_kmsan_checks
8f28ca6b	97	unsigned int ioread32(const void __iomem *addr)
1da177e4 LT	98	{
1da177e4 LT	99	IO_COND(addr, return inl(port), return readl(addr));
6cbf0c70	100	return 0xffffffff;
1da177e4	101	}
a28a4d47	102	__no_kmsan_checks
8f28ca6b	103	unsigned int ioread32be(const void __iomem *addr)
dae409a2	104	{
34ba8a5c	105	IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr));
6cbf0c70	106	return 0xffffffff;
dae409a2	107	}
1da177e4 LT	108	EXPORT_SYMBOL(ioread8);
1da177e4 LT	109	EXPORT_SYMBOL(ioread16);
dae409a2	110	EXPORT_SYMBOL(ioread16be);
1da177e4	111	EXPORT_SYMBOL(ioread32);
dae409a2	112	EXPORT_SYMBOL(ioread32be);
1da177e4	113
79bf0cbd LG	114	#ifdef readq
	115	static u64 pio_read64_lo_hi(unsigned long port)
	116	{
	117	u64 lo, hi;
	118
	119	lo = inl(port);
	120	hi = inl(port + sizeof(u32));
	121
	122	return lo \| (hi << 32);
	123	}
	124
	125	static u64 pio_read64_hi_lo(unsigned long port)
	126	{
	127	u64 lo, hi;
	128
	129	hi = inl(port + sizeof(u32));
	130	lo = inl(port);
	131
	132	return lo \| (hi << 32);
	133	}
	134
	135	static u64 pio_read64be_lo_hi(unsigned long port)
	136	{
	137	u64 lo, hi;
	138
	139	lo = pio_read32be(port + sizeof(u32));
	140	hi = pio_read32be(port);
	141
	142	return lo \| (hi << 32);
	143	}
	144
	145	static u64 pio_read64be_hi_lo(unsigned long port)
	146	{
	147	u64 lo, hi;
	148
	149	hi = pio_read32be(port);
	150	lo = pio_read32be(port + sizeof(u32));
	151
	152	return lo \| (hi << 32);
	153	}
	154
a28a4d47	155	__no_kmsan_checks
8f28ca6b	156	u64 ioread64_lo_hi(const void __iomem *addr)
79bf0cbd LG	157	{
	158	IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr));
	159	return 0xffffffffffffffffULL;
	160	}
	161
a28a4d47	162	__no_kmsan_checks
8f28ca6b	163	u64 ioread64_hi_lo(const void __iomem *addr)
79bf0cbd LG	164	{
	165	IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr));
	166	return 0xffffffffffffffffULL;
	167	}
	168
a28a4d47	169	__no_kmsan_checks
8f28ca6b	170	u64 ioread64be_lo_hi(const void __iomem *addr)
79bf0cbd LG	171	{
	172	IO_COND(addr, return pio_read64be_lo_hi(port),
	173	return mmio_read64be(addr));
	174	return 0xffffffffffffffffULL;
	175	}
	176
a28a4d47	177	__no_kmsan_checks
8f28ca6b	178	u64 ioread64be_hi_lo(const void __iomem *addr)
79bf0cbd LG	179	{
	180	IO_COND(addr, return pio_read64be_hi_lo(port),
	181	return mmio_read64be(addr));
	182	return 0xffffffffffffffffULL;
	183	}
	184
	185	EXPORT_SYMBOL(ioread64_lo_hi);
	186	EXPORT_SYMBOL(ioread64_hi_lo);
	187	EXPORT_SYMBOL(ioread64be_lo_hi);
	188	EXPORT_SYMBOL(ioread64be_hi_lo);
	189
	190	#endif /* readq */
	191
34ba8a5c LT	192	#ifndef pio_write16be
	193	#define pio_write16be(val,port) outw(swab16(val),port)
	194	#define pio_write32be(val,port) outl(swab32(val),port)
	195	#endif
	196
	197	#ifndef mmio_write16be
aecc787c LG	198	#define mmio_write16be(val,port) writew(swab16(val),port)
aecc787c LG	199	#define mmio_write32be(val,port) writel(swab32(val),port)
79bf0cbd	200	#define mmio_write64be(val,port) writeq(swab64(val),port)
34ba8a5c LT	201	#endif
34ba8a5c LT	202
9f741cb8	203	void iowrite8(u8 val, void __iomem *addr)
1da177e4	204	{
a28a4d47 AP	205	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	206	kmsan_check_memory(&val, sizeof(val));
1da177e4 LT	207	IO_COND(addr, outb(val,port), writeb(val, addr));
1da177e4 LT	208	}
9f741cb8	209	void iowrite16(u16 val, void __iomem *addr)
1da177e4	210	{
a28a4d47 AP	211	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	212	kmsan_check_memory(&val, sizeof(val));
1da177e4 LT	213	IO_COND(addr, outw(val,port), writew(val, addr));
1da177e4 LT	214	}
9f741cb8	215	void iowrite16be(u16 val, void __iomem *addr)
dae409a2	216	{
a28a4d47 AP	217	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	218	kmsan_check_memory(&val, sizeof(val));
34ba8a5c	219	IO_COND(addr, pio_write16be(val,port), mmio_write16be(val, addr));
dae409a2	220	}
9f741cb8	221	void iowrite32(u32 val, void __iomem *addr)
1da177e4	222	{
a28a4d47 AP	223	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	224	kmsan_check_memory(&val, sizeof(val));
1da177e4 LT	225	IO_COND(addr, outl(val,port), writel(val, addr));
1da177e4 LT	226	}
9f741cb8	227	void iowrite32be(u32 val, void __iomem *addr)
dae409a2	228	{
a28a4d47 AP	229	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	230	kmsan_check_memory(&val, sizeof(val));
34ba8a5c	231	IO_COND(addr, pio_write32be(val,port), mmio_write32be(val, addr));
dae409a2	232	}
1da177e4 LT	233	EXPORT_SYMBOL(iowrite8);
1da177e4 LT	234	EXPORT_SYMBOL(iowrite16);
dae409a2	235	EXPORT_SYMBOL(iowrite16be);
1da177e4	236	EXPORT_SYMBOL(iowrite32);
dae409a2	237	EXPORT_SYMBOL(iowrite32be);
1da177e4	238
79bf0cbd LG	239	#ifdef writeq
	240	static void pio_write64_lo_hi(u64 val, unsigned long port)
	241	{
	242	outl(val, port);
	243	outl(val >> 32, port + sizeof(u32));
	244	}
	245
	246	static void pio_write64_hi_lo(u64 val, unsigned long port)
	247	{
	248	outl(val >> 32, port + sizeof(u32));
	249	outl(val, port);
	250	}
	251
	252	static void pio_write64be_lo_hi(u64 val, unsigned long port)
	253	{
	254	pio_write32be(val, port + sizeof(u32));
	255	pio_write32be(val >> 32, port);
	256	}
	257
	258	static void pio_write64be_hi_lo(u64 val, unsigned long port)
	259	{
	260	pio_write32be(val >> 32, port);
	261	pio_write32be(val, port + sizeof(u32));
	262	}
	263
	264	void iowrite64_lo_hi(u64 val, void __iomem *addr)
	265	{
a28a4d47 AP	266	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	267	kmsan_check_memory(&val, sizeof(val));
79bf0cbd LG	268	IO_COND(addr, pio_write64_lo_hi(val, port),
	269	writeq(val, addr));
	270	}
	271
	272	void iowrite64_hi_lo(u64 val, void __iomem *addr)
	273	{
a28a4d47 AP	274	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	275	kmsan_check_memory(&val, sizeof(val));
79bf0cbd LG	276	IO_COND(addr, pio_write64_hi_lo(val, port),
	277	writeq(val, addr));
	278	}
	279
	280	void iowrite64be_lo_hi(u64 val, void __iomem *addr)
	281	{
a28a4d47 AP	282	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	283	kmsan_check_memory(&val, sizeof(val));
79bf0cbd LG	284	IO_COND(addr, pio_write64be_lo_hi(val, port),
	285	mmio_write64be(val, addr));
	286	}
	287
	288	void iowrite64be_hi_lo(u64 val, void __iomem *addr)
	289	{
a28a4d47 AP	290	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	291	kmsan_check_memory(&val, sizeof(val));
79bf0cbd LG	292	IO_COND(addr, pio_write64be_hi_lo(val, port),
	293	mmio_write64be(val, addr));
	294	}
	295
	296	EXPORT_SYMBOL(iowrite64_lo_hi);
	297	EXPORT_SYMBOL(iowrite64_hi_lo);
	298	EXPORT_SYMBOL(iowrite64be_lo_hi);
	299	EXPORT_SYMBOL(iowrite64be_hi_lo);
	300
	301	#endif /* readq */
	302
1da177e4 LT	303	/*
	304	* These are the "repeat MMIO read/write" functions.
	305	* Note the "__raw" accesses, since we don't want to
	306	* convert to CPU byte order. We write in "IO byte
	307	* order" (we also don't have IO barriers).
	308	*/
34ba8a5c	309	#ifndef mmio_insb
8f28ca6b	310	static inline void mmio_insb(const void __iomem addr, u8 dst, int count)
1da177e4 LT	311	{
	312	while (--count >= 0) {
	313	u8 data = __raw_readb(addr);
	314	*dst = data;
	315	dst++;
	316	}
	317	}
8f28ca6b	318	static inline void mmio_insw(const void __iomem addr, u16 dst, int count)
1da177e4 LT	319	{
	320	while (--count >= 0) {
	321	u16 data = __raw_readw(addr);
	322	*dst = data;
	323	dst++;
	324	}
	325	}
8f28ca6b	326	static inline void mmio_insl(const void __iomem addr, u32 dst, int count)
1da177e4 LT	327	{
	328	while (--count >= 0) {
	329	u32 data = __raw_readl(addr);
	330	*dst = data;
	331	dst++;
	332	}
	333	}
34ba8a5c	334	#endif
1da177e4	335
34ba8a5c	336	#ifndef mmio_outsb
1da177e4 LT	337	static inline void mmio_outsb(void __iomem addr, const u8 src, int count)
	338	{
	339	while (--count >= 0) {
	340	__raw_writeb(*src, addr);
	341	src++;
	342	}
	343	}
	344	static inline void mmio_outsw(void __iomem addr, const u16 src, int count)
	345	{
	346	while (--count >= 0) {
	347	__raw_writew(*src, addr);
	348	src++;
	349	}
	350	}
	351	static inline void mmio_outsl(void __iomem addr, const u32 src, int count)
	352	{
	353	while (--count >= 0) {
	354	__raw_writel(*src, addr);
	355	src++;
	356	}
	357	}
34ba8a5c	358	#endif
1da177e4	359
8f28ca6b	360	void ioread8_rep(const void __iomem addr, void dst, unsigned long count)
1da177e4 LT	361	{
1da177e4 LT	362	IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count));
a28a4d47 AP	363	/* KMSAN must treat values read from devices as initialized. */
a28a4d47 AP	364	kmsan_unpoison_memory(dst, count);
1da177e4	365	}
8f28ca6b	366	void ioread16_rep(const void __iomem addr, void dst, unsigned long count)
1da177e4 LT	367	{
1da177e4 LT	368	IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count));
a28a4d47 AP	369	/* KMSAN must treat values read from devices as initialized. */
a28a4d47 AP	370	kmsan_unpoison_memory(dst, count * 2);
1da177e4	371	}
8f28ca6b	372	void ioread32_rep(const void __iomem addr, void dst, unsigned long count)
1da177e4 LT	373	{
1da177e4 LT	374	IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count));
a28a4d47 AP	375	/* KMSAN must treat values read from devices as initialized. */
a28a4d47 AP	376	kmsan_unpoison_memory(dst, count * 4);
1da177e4 LT	377	}
	378	EXPORT_SYMBOL(ioread8_rep);
	379	EXPORT_SYMBOL(ioread16_rep);
	380	EXPORT_SYMBOL(ioread32_rep);
	381
9f741cb8	382	void iowrite8_rep(void __iomem addr, const void src, unsigned long count)
1da177e4	383	{
a28a4d47 AP	384	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	385	kmsan_check_memory(src, count);
1da177e4 LT	386	IO_COND(addr, outsb(port, src, count), mmio_outsb(addr, src, count));
1da177e4 LT	387	}
9f741cb8	388	void iowrite16_rep(void __iomem addr, const void src, unsigned long count)
1da177e4	389	{
a28a4d47 AP	390	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	391	kmsan_check_memory(src, count * 2);
1da177e4 LT	392	IO_COND(addr, outsw(port, src, count), mmio_outsw(addr, src, count));
1da177e4 LT	393	}
9f741cb8	394	void iowrite32_rep(void __iomem addr, const void src, unsigned long count)
1da177e4	395	{
a28a4d47 AP	396	/* Make sure uninitialized memory isn't copied to devices. */
a28a4d47 AP	397	kmsan_check_memory(src, count * 4);
1da177e4 LT	398	IO_COND(addr, outsl(port, src,count), mmio_outsl(addr, src, count));
	399	}
	400	EXPORT_SYMBOL(iowrite8_rep);
	401	EXPORT_SYMBOL(iowrite16_rep);
	402	EXPORT_SYMBOL(iowrite32_rep);
	403
ce816fa8	404	#ifdef CONFIG_HAS_IOPORT_MAP
1da177e4 LT	405	/* Create a virtual mapping cookie for an IO port range */
	406	void __iomem *ioport_map(unsigned long port, unsigned int nr)
	407	{
	408	if (port > PIO_MASK)
	409	return NULL;
	410	return (void __iomem *) (unsigned long) (port + PIO_OFFSET);
	411	}
	412
	413	void ioport_unmap(void __iomem *addr)
	414	{
	415	/* Nothing to do */
	416	}
	417	EXPORT_SYMBOL(ioport_map);
	418	EXPORT_SYMBOL(ioport_unmap);
ce816fa8	419	#endif /* CONFIG_HAS_IOPORT_MAP */
1da177e4	420
82ed223c	421	#ifdef CONFIG_PCI
66eab4df MT	422	/* Hide the details if this is a MMIO or PIO address space and just do what
66eab4df MT	423	* you expect in the correct way. */
1da177e4 LT	424	void pci_iounmap(struct pci_dev dev, void __iomem addr)
	425	{
	426	IO_COND(addr, /* nothing */, iounmap(addr));
	427	}
1da177e4	428	EXPORT_SYMBOL(pci_iounmap);
82ed223c	429	#endif /* CONFIG_PCI */