2 * Re-map IO memory to kernel address space so that we can access it.
3 * This is needed for high PCI addresses that aren't mapped in the
4 * 640k-1MB IO memory area on PC's
6 * (C) Copyright 1995 1996 Linus Torvalds
9 #include <linux/bootmem.h>
10 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
16 #include <asm/cacheflush.h>
18 #include <asm/fixmap.h>
19 #include <asm/pgtable.h>
20 #include <asm/tlbflush.h>
21 #include <asm/pgalloc.h>
26 unsigned long __phys_addr(unsigned long x)
28 if (x >= __START_KERNEL_map)
29 return x - __START_KERNEL_map + phys_base;
30 return x - PAGE_OFFSET;
32 EXPORT_SYMBOL(__phys_addr);
34 static inline int phys_addr_valid(unsigned long addr)
36 return addr < (1UL << boot_cpu_data.x86_phys_bits);
41 static inline int phys_addr_valid(unsigned long addr)
48 int page_is_ram(unsigned long pagenr)
50 unsigned long addr, end;
54 * A special case is the first 4Kb of memory;
55 * This is a BIOS owned area, not kernel ram, but generally
56 * not listed as such in the E820 table.
62 * Second special case: Some BIOSen report the PC BIOS
63 * area (640->1Mb) as ram even though it is not.
65 if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
66 pagenr < (BIOS_END >> PAGE_SHIFT))
69 for (i = 0; i < e820.nr_map; i++) {
73 if (e820.map[i].type != E820_RAM)
75 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
76 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
79 if ((pagenr >= addr) && (pagenr < end))
86 * Fix up the linear direct mapping of the kernel to avoid cache attribute
89 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
90 unsigned long prot_val)
92 unsigned long nrpages = size >> PAGE_SHIFT;
98 err = _set_memory_uc(vaddr, nrpages);
101 err = _set_memory_wc(vaddr, nrpages);
104 err = _set_memory_wb(vaddr, nrpages);
112 * Remap an arbitrary physical address space into the kernel virtual
113 * address space. Needed when the kernel wants to access high addresses
116 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
117 * have to convert them into an offset in a page-aligned mapping, but the
118 * caller shouldn't need to know that small detail.
120 static void __iomem *__ioremap(resource_size_t phys_addr, unsigned long size,
121 unsigned long prot_val)
123 unsigned long pfn, offset, last_addr, vaddr;
124 struct vm_struct *area;
125 unsigned long new_prot_val;
128 /* Don't allow wraparound or zero size */
129 last_addr = phys_addr + size - 1;
130 if (!size || last_addr < phys_addr)
133 if (!phys_addr_valid(phys_addr)) {
134 printk(KERN_WARNING "ioremap: invalid physical address %lx\n",
141 * Don't remap the low PCI/ISA area, it's always mapped..
143 if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
144 return (__force void __iomem *)phys_to_virt(phys_addr);
147 * Don't allow anybody to remap normal RAM that we're using..
149 for (pfn = phys_addr >> PAGE_SHIFT;
150 (pfn << PAGE_SHIFT) < last_addr; pfn++) {
152 int is_ram = page_is_ram(pfn);
154 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
156 WARN_ON_ONCE(is_ram);
160 * Mappings have to be page-aligned
162 offset = phys_addr & ~PAGE_MASK;
163 phys_addr &= PAGE_MASK;
164 size = PAGE_ALIGN(last_addr+1) - phys_addr;
166 if (reserve_memtype(phys_addr, phys_addr + size,
167 prot_val, &new_prot_val)) {
169 * Do not fallback to certain memory types with certain
171 * - request is uncached, return cannot be write-back
172 * - request is uncached, return cannot be write-combine
173 * - request is write-combine, return cannot be write-back
175 if ((prot_val == _PAGE_CACHE_UC &&
176 (new_prot_val == _PAGE_CACHE_WB ||
177 new_prot_val == _PAGE_CACHE_WC)) ||
178 (prot_val == _PAGE_CACHE_WC &&
179 new_prot_val == _PAGE_CACHE_WB)) {
180 free_memtype(phys_addr, phys_addr + size);
183 prot_val = new_prot_val;
189 prot = PAGE_KERNEL_NOCACHE;
192 prot = PAGE_KERNEL_WC;
202 area = get_vm_area(size, VM_IOREMAP);
205 area->phys_addr = phys_addr;
206 vaddr = (unsigned long) area->addr;
207 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
208 free_memtype(phys_addr, phys_addr + size);
213 if (ioremap_change_attr(vaddr, size, prot_val) < 0) {
214 free_memtype(phys_addr, phys_addr + size);
219 return (void __iomem *) (vaddr + offset);
223 * ioremap_nocache - map bus memory into CPU space
224 * @offset: bus address of the memory
225 * @size: size of the resource to map
227 * ioremap_nocache performs a platform specific sequence of operations to
228 * make bus memory CPU accessible via the readb/readw/readl/writeb/
229 * writew/writel functions and the other mmio helpers. The returned
230 * address is not guaranteed to be usable directly as a virtual
233 * This version of ioremap ensures that the memory is marked uncachable
234 * on the CPU as well as honouring existing caching rules from things like
235 * the PCI bus. Note that there are other caches and buffers on many
236 * busses. In particular driver authors should read up on PCI writes
238 * It's useful if some control registers are in such an area and
239 * write combining or read caching is not desirable:
241 * Must be freed with iounmap.
243 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
245 return __ioremap(phys_addr, size, _PAGE_CACHE_UC);
247 EXPORT_SYMBOL(ioremap_nocache);
250 * ioremap_wc - map memory into CPU space write combined
251 * @offset: bus address of the memory
252 * @size: size of the resource to map
254 * This version of ioremap ensures that the memory is marked write combining.
255 * Write combining allows faster writes to some hardware devices.
257 * Must be freed with iounmap.
259 void __iomem *ioremap_wc(unsigned long phys_addr, unsigned long size)
262 return __ioremap(phys_addr, size, _PAGE_CACHE_WC);
264 return ioremap_nocache(phys_addr, size);
266 EXPORT_SYMBOL(ioremap_wc);
268 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
270 return __ioremap(phys_addr, size, _PAGE_CACHE_WB);
272 EXPORT_SYMBOL(ioremap_cache);
275 * iounmap - Free a IO remapping
276 * @addr: virtual address from ioremap_*
278 * Caller must ensure there is only one unmapping for the same pointer.
280 void iounmap(volatile void __iomem *addr)
282 struct vm_struct *p, *o;
284 if ((void __force *)addr <= high_memory)
288 * __ioremap special-cases the PCI/ISA range by not instantiating a
289 * vm_area and by simply returning an address into the kernel mapping
290 * of ISA space. So handle that here.
292 if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
293 addr < phys_to_virt(ISA_END_ADDRESS))
296 addr = (volatile void __iomem *)
297 (PAGE_MASK & (unsigned long __force)addr);
299 /* Use the vm area unlocked, assuming the caller
300 ensures there isn't another iounmap for the same address
301 in parallel. Reuse of the virtual address is prevented by
302 leaving it in the global lists until we're done with it.
303 cpa takes care of the direct mappings. */
304 read_lock(&vmlist_lock);
305 for (p = vmlist; p; p = p->next) {
309 read_unlock(&vmlist_lock);
312 printk(KERN_ERR "iounmap: bad address %p\n", addr);
317 free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
319 /* Finally remove it */
320 o = remove_vm_area((void *)addr);
321 BUG_ON(p != o || o == NULL);
324 EXPORT_SYMBOL(iounmap);
328 int __initdata early_ioremap_debug;
330 static int __init early_ioremap_debug_setup(char *str)
332 early_ioremap_debug = 1;
336 early_param("early_ioremap_debug", early_ioremap_debug_setup);
338 static __initdata int after_paging_init;
339 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)]
340 __section(.bss.page_aligned);
342 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
344 /* Don't assume we're using swapper_pg_dir at this point */
345 pgd_t *base = __va(read_cr3());
346 pgd_t *pgd = &base[pgd_index(addr)];
347 pud_t *pud = pud_offset(pgd, addr);
348 pmd_t *pmd = pmd_offset(pud, addr);
353 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
355 return &bm_pte[pte_index(addr)];
358 void __init early_ioremap_init(void)
362 if (early_ioremap_debug)
363 printk(KERN_INFO "early_ioremap_init()\n");
365 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
366 memset(bm_pte, 0, sizeof(bm_pte));
367 pmd_populate_kernel(&init_mm, pmd, bm_pte);
370 * The boot-ioremap range spans multiple pmds, for which
371 * we are not prepared:
373 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
375 printk(KERN_WARNING "pmd %p != %p\n",
376 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
377 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
378 fix_to_virt(FIX_BTMAP_BEGIN));
379 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
380 fix_to_virt(FIX_BTMAP_END));
382 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
383 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
388 void __init early_ioremap_clear(void)
392 if (early_ioremap_debug)
393 printk(KERN_INFO "early_ioremap_clear()\n");
395 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
397 paravirt_release_pt(__pa(bm_pte) >> PAGE_SHIFT);
401 void __init early_ioremap_reset(void)
403 enum fixed_addresses idx;
404 unsigned long addr, phys;
407 after_paging_init = 1;
408 for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
409 addr = fix_to_virt(idx);
410 pte = early_ioremap_pte(addr);
411 if (pte_present(*pte)) {
412 phys = pte_val(*pte) & PAGE_MASK;
413 set_fixmap(idx, phys);
418 static void __init __early_set_fixmap(enum fixed_addresses idx,
419 unsigned long phys, pgprot_t flags)
421 unsigned long addr = __fix_to_virt(idx);
424 if (idx >= __end_of_fixed_addresses) {
428 pte = early_ioremap_pte(addr);
429 if (pgprot_val(flags))
430 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
432 pte_clear(NULL, addr, pte);
433 __flush_tlb_one(addr);
436 static inline void __init early_set_fixmap(enum fixed_addresses idx,
439 if (after_paging_init)
440 set_fixmap(idx, phys);
442 __early_set_fixmap(idx, phys, PAGE_KERNEL);
445 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
447 if (after_paging_init)
450 __early_set_fixmap(idx, 0, __pgprot(0));
454 int __initdata early_ioremap_nested;
456 static int __init check_early_ioremap_leak(void)
458 if (!early_ioremap_nested)
462 "Debug warning: early ioremap leak of %d areas detected.\n",
463 early_ioremap_nested);
465 "please boot with early_ioremap_debug and report the dmesg.\n");
470 late_initcall(check_early_ioremap_leak);
472 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
474 unsigned long offset, last_addr;
475 unsigned int nrpages, nesting;
476 enum fixed_addresses idx0, idx;
478 WARN_ON(system_state != SYSTEM_BOOTING);
480 nesting = early_ioremap_nested;
481 if (early_ioremap_debug) {
482 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
483 phys_addr, size, nesting);
487 /* Don't allow wraparound or zero size */
488 last_addr = phys_addr + size - 1;
489 if (!size || last_addr < phys_addr) {
494 if (nesting >= FIX_BTMAPS_NESTING) {
498 early_ioremap_nested++;
500 * Mappings have to be page-aligned
502 offset = phys_addr & ~PAGE_MASK;
503 phys_addr &= PAGE_MASK;
504 size = PAGE_ALIGN(last_addr) - phys_addr;
507 * Mappings have to fit in the FIX_BTMAP area.
509 nrpages = size >> PAGE_SHIFT;
510 if (nrpages > NR_FIX_BTMAPS) {
518 idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
520 while (nrpages > 0) {
521 early_set_fixmap(idx, phys_addr);
522 phys_addr += PAGE_SIZE;
526 if (early_ioremap_debug)
527 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
529 return (void *) (offset + fix_to_virt(idx0));
532 void __init early_iounmap(void *addr, unsigned long size)
534 unsigned long virt_addr;
535 unsigned long offset;
536 unsigned int nrpages;
537 enum fixed_addresses idx;
538 unsigned int nesting;
540 nesting = --early_ioremap_nested;
541 WARN_ON(nesting < 0);
543 if (early_ioremap_debug) {
544 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
549 virt_addr = (unsigned long)addr;
550 if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
554 offset = virt_addr & ~PAGE_MASK;
555 nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
557 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
558 while (nrpages > 0) {
559 early_clear_fixmap(idx);
565 void __this_fixmap_does_not_exist(void)
570 #endif /* CONFIG_X86_32 */