Linux 2.6.36-rc3
[linux-2.6-block.git] / arch / x86 / kernel / cpu / mtrr / generic.c
CommitLineData
a1a499a3
JSR
1/*
2 * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
3 * because MTRRs can span upto 40 bits (36bits on most modern x86)
4 */
5#define DEBUG
6
7#include <linux/module.h>
1da177e4 8#include <linux/init.h>
a1a499a3 9#include <linux/io.h>
1da177e4 10#include <linux/mm.h>
a1a499a3 11
7ebad705 12#include <asm/processor-flags.h>
a1a499a3 13#include <asm/cpufeature.h>
1da177e4 14#include <asm/tlbflush.h>
a1a499a3
JSR
15#include <asm/system.h>
16#include <asm/mtrr.h>
17#include <asm/msr.h>
2e5d9c85 18#include <asm/pat.h>
a1a499a3 19
1da177e4
LT
20#include "mtrr.h"
21
de938c51 22struct fixed_range_block {
a1a499a3
JSR
23 int base_msr; /* start address of an MTRR block */
24 int ranges; /* number of MTRRs in this block */
de938c51
BK
25};
26
27static struct fixed_range_block fixed_range_blocks[] = {
a1a499a3
JSR
28 { MSR_MTRRfix64K_00000, 1 }, /* one 64k MTRR */
29 { MSR_MTRRfix16K_80000, 2 }, /* two 16k MTRRs */
30 { MSR_MTRRfix4K_C0000, 8 }, /* eight 4k MTRRs */
de938c51
BK
31 {}
32};
33
1da177e4 34static unsigned long smp_changes_mask;
2e5d9c85 35static int mtrr_state_set;
95ffa243 36u64 mtrr_tom2;
1da177e4 37
a1a499a3 38struct mtrr_state_type mtrr_state;
932d27a7
SY
39EXPORT_SYMBOL_GPL(mtrr_state);
40
a1a499a3 41/*
3ff42da5
AH
42 * BIOS is expected to clear MtrrFixDramModEn bit, see for example
43 * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
44 * Opteron Processors" (26094 Rev. 3.30 February 2006), section
45 * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set
46 * to 1 during BIOS initalization of the fixed MTRRs, then cleared to
47 * 0 for operation."
48 */
49static inline void k8_check_syscfg_dram_mod_en(void)
50{
51 u32 lo, hi;
52
53 if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) &&
54 (boot_cpu_data.x86 >= 0x0f)))
55 return;
56
57 rdmsr(MSR_K8_SYSCFG, lo, hi);
58 if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) {
59 printk(KERN_ERR FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]"
60 " not cleared by BIOS, clearing this bit\n",
61 smp_processor_id());
62 lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
63 mtrr_wrmsr(MSR_K8_SYSCFG, lo, hi);
64 }
65}
66
2e5d9c85 67/*
68 * Returns the effective MTRR type for the region
69 * Error returns:
70 * - 0xFE - when the range is "not entirely covered" by _any_ var range MTRR
71 * - 0xFF - when MTRR is not enabled
72 */
73u8 mtrr_type_lookup(u64 start, u64 end)
74{
75 int i;
76 u64 base, mask;
77 u8 prev_match, curr_match;
78
79 if (!mtrr_state_set)
80 return 0xFF;
81
82 if (!mtrr_state.enabled)
83 return 0xFF;
84
85 /* Make end inclusive end, instead of exclusive */
86 end--;
87
88 /* Look in fixed ranges. Just return the type as per start */
89 if (mtrr_state.have_fixed && (start < 0x100000)) {
90 int idx;
91
92 if (start < 0x80000) {
93 idx = 0;
94 idx += (start >> 16);
95 return mtrr_state.fixed_ranges[idx];
96 } else if (start < 0xC0000) {
97 idx = 1 * 8;
98 idx += ((start - 0x80000) >> 14);
99 return mtrr_state.fixed_ranges[idx];
100 } else if (start < 0x1000000) {
101 idx = 3 * 8;
102 idx += ((start - 0xC0000) >> 12);
103 return mtrr_state.fixed_ranges[idx];
104 }
105 }
106
107 /*
108 * Look in variable ranges
109 * Look of multiple ranges matching this address and pick type
110 * as per MTRR precedence
111 */
a1a499a3 112 if (!(mtrr_state.enabled & 2))
2e5d9c85 113 return mtrr_state.def_type;
2e5d9c85 114
115 prev_match = 0xFF;
116 for (i = 0; i < num_var_ranges; ++i) {
117 unsigned short start_state, end_state;
118
119 if (!(mtrr_state.var_ranges[i].mask_lo & (1 << 11)))
120 continue;
121
122 base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) +
123 (mtrr_state.var_ranges[i].base_lo & PAGE_MASK);
124 mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) +
125 (mtrr_state.var_ranges[i].mask_lo & PAGE_MASK);
126
127 start_state = ((start & mask) == (base & mask));
128 end_state = ((end & mask) == (base & mask));
129 if (start_state != end_state)
130 return 0xFE;
131
a1a499a3 132 if ((start & mask) != (base & mask))
2e5d9c85 133 continue;
2e5d9c85 134
135 curr_match = mtrr_state.var_ranges[i].base_lo & 0xff;
136 if (prev_match == 0xFF) {
137 prev_match = curr_match;
138 continue;
139 }
140
141 if (prev_match == MTRR_TYPE_UNCACHABLE ||
142 curr_match == MTRR_TYPE_UNCACHABLE) {
143 return MTRR_TYPE_UNCACHABLE;
144 }
145
146 if ((prev_match == MTRR_TYPE_WRBACK &&
147 curr_match == MTRR_TYPE_WRTHROUGH) ||
148 (prev_match == MTRR_TYPE_WRTHROUGH &&
149 curr_match == MTRR_TYPE_WRBACK)) {
150 prev_match = MTRR_TYPE_WRTHROUGH;
151 curr_match = MTRR_TYPE_WRTHROUGH;
152 }
153
a1a499a3 154 if (prev_match != curr_match)
2e5d9c85 155 return MTRR_TYPE_UNCACHABLE;
2e5d9c85 156 }
157
95ffa243
YL
158 if (mtrr_tom2) {
159 if (start >= (1ULL<<32) && (end < mtrr_tom2))
35605a10
YL
160 return MTRR_TYPE_WRBACK;
161 }
162
2e5d9c85 163 if (prev_match != 0xFF)
164 return prev_match;
165
166 return mtrr_state.def_type;
167}
168
a1a499a3 169/* Get the MSR pair relating to a var range */
bf8c4817 170static void
1da177e4
LT
171get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
172{
173 rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
174 rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
175}
176
a1a499a3 177/* Fill the MSR pair relating to a var range */
95ffa243
YL
178void fill_mtrr_var_range(unsigned int index,
179 u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi)
180{
181 struct mtrr_var_range *vr;
182
183 vr = mtrr_state.var_ranges;
184
185 vr[index].base_lo = base_lo;
186 vr[index].base_hi = base_hi;
187 vr[index].mask_lo = mask_lo;
188 vr[index].mask_hi = mask_hi;
189}
190
a1a499a3 191static void get_fixed_ranges(mtrr_type *frs)
1da177e4 192{
a1a499a3 193 unsigned int *p = (unsigned int *)frs;
1da177e4
LT
194 int i;
195
3ff42da5
AH
196 k8_check_syscfg_dram_mod_en();
197
a036c7a3 198 rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]);
1da177e4
LT
199
200 for (i = 0; i < 2; i++)
7d9d55e4 201 rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]);
1da177e4 202 for (i = 0; i < 8; i++)
ba5673ff 203 rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]);
1da177e4
LT
204}
205
2b3b4835
BK
206void mtrr_save_fixed_ranges(void *info)
207{
84288ad8
AM
208 if (cpu_has_mtrr)
209 get_fixed_ranges(mtrr_state.fixed_ranges);
2b3b4835
BK
210}
211
d4c90e37
YL
212static unsigned __initdata last_fixed_start;
213static unsigned __initdata last_fixed_end;
214static mtrr_type __initdata last_fixed_type;
215
216static void __init print_fixed_last(void)
217{
218 if (!last_fixed_end)
219 return;
220
a1a499a3
JSR
221 pr_debug(" %05X-%05X %s\n", last_fixed_start,
222 last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
d4c90e37
YL
223
224 last_fixed_end = 0;
225}
226
227static void __init update_fixed_last(unsigned base, unsigned end,
a1a499a3 228 mtrr_type type)
d4c90e37
YL
229{
230 last_fixed_start = base;
231 last_fixed_end = end;
232 last_fixed_type = type;
233}
234
a1a499a3
JSR
235static void __init
236print_fixed(unsigned base, unsigned step, const mtrr_type *types)
365bff80
JB
237{
238 unsigned i;
239
d4c90e37
YL
240 for (i = 0; i < 8; ++i, ++types, base += step) {
241 if (last_fixed_end == 0) {
242 update_fixed_last(base, base + step, *types);
243 continue;
244 }
245 if (last_fixed_end == base && last_fixed_type == *types) {
246 last_fixed_end = base + step;
247 continue;
248 }
249 /* new segments: gap or different type */
250 print_fixed_last();
251 update_fixed_last(base, base + step, *types);
252 }
365bff80
JB
253}
254
2e5d9c85 255static void prepare_set(void);
256static void post_set(void);
257
8ad97905
YL
258static void __init print_mtrr_state(void)
259{
260 unsigned int i;
261 int high_width;
262
a1a499a3
JSR
263 pr_debug("MTRR default type: %s\n",
264 mtrr_attrib_to_str(mtrr_state.def_type));
8ad97905 265 if (mtrr_state.have_fixed) {
a1a499a3
JSR
266 pr_debug("MTRR fixed ranges %sabled:\n",
267 mtrr_state.enabled & 1 ? "en" : "dis");
8ad97905
YL
268 print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
269 for (i = 0; i < 2; ++i)
a1a499a3
JSR
270 print_fixed(0x80000 + i * 0x20000, 0x04000,
271 mtrr_state.fixed_ranges + (i + 1) * 8);
8ad97905 272 for (i = 0; i < 8; ++i)
a1a499a3
JSR
273 print_fixed(0xC0000 + i * 0x08000, 0x01000,
274 mtrr_state.fixed_ranges + (i + 3) * 8);
d4c90e37
YL
275
276 /* tail */
277 print_fixed_last();
8ad97905 278 }
a1a499a3
JSR
279 pr_debug("MTRR variable ranges %sabled:\n",
280 mtrr_state.enabled & 2 ? "en" : "dis");
917a0153
YL
281 if (size_or_mask & 0xffffffffUL)
282 high_width = ffs(size_or_mask & 0xffffffffUL) - 1;
283 else
284 high_width = ffs(size_or_mask>>32) + 32 - 1;
285 high_width = (high_width - (32 - PAGE_SHIFT) + 3) / 4;
a1a499a3 286
8ad97905
YL
287 for (i = 0; i < num_var_ranges; ++i) {
288 if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
a1a499a3
JSR
289 pr_debug(" %u base %0*X%05X000 mask %0*X%05X000 %s\n",
290 i,
291 high_width,
292 mtrr_state.var_ranges[i].base_hi,
293 mtrr_state.var_ranges[i].base_lo >> 12,
294 high_width,
295 mtrr_state.var_ranges[i].mask_hi,
296 mtrr_state.var_ranges[i].mask_lo >> 12,
297 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
8ad97905 298 else
a1a499a3 299 pr_debug(" %u disabled\n", i);
8ad97905 300 }
a1a499a3
JSR
301 if (mtrr_tom2)
302 pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20);
8ad97905
YL
303}
304
a1a499a3 305/* Grab all of the MTRR state for this CPU into *state */
9ef231a4 306void __init get_mtrr_state(void)
1da177e4 307{
1da177e4 308 struct mtrr_var_range *vrs;
2e5d9c85 309 unsigned long flags;
a1a499a3
JSR
310 unsigned lo, dummy;
311 unsigned int i;
1da177e4 312
1da177e4
LT
313 vrs = mtrr_state.var_ranges;
314
d9bcc01d 315 rdmsr(MSR_MTRRcap, lo, dummy);
365bff80
JB
316 mtrr_state.have_fixed = (lo >> 8) & 1;
317
1da177e4
LT
318 for (i = 0; i < num_var_ranges; i++)
319 get_mtrr_var_range(i, &vrs[i]);
365bff80
JB
320 if (mtrr_state.have_fixed)
321 get_fixed_ranges(mtrr_state.fixed_ranges);
1da177e4 322
52650257 323 rdmsr(MSR_MTRRdefType, lo, dummy);
1da177e4
LT
324 mtrr_state.def_type = (lo & 0xff);
325 mtrr_state.enabled = (lo & 0xc00) >> 10;
365bff80 326
35605a10 327 if (amd_special_default_mtrr()) {
0da72a4a 328 unsigned low, high;
a1a499a3 329
35605a10 330 /* TOP_MEM2 */
0da72a4a 331 rdmsr(MSR_K8_TOP_MEM2, low, high);
95ffa243
YL
332 mtrr_tom2 = high;
333 mtrr_tom2 <<= 32;
334 mtrr_tom2 |= low;
8004dd96 335 mtrr_tom2 &= 0xffffff800000ULL;
35605a10 336 }
8ad97905
YL
337
338 print_mtrr_state();
339
2e5d9c85 340 mtrr_state_set = 1;
341
342 /* PAT setup for BP. We need to go through sync steps here */
343 local_irq_save(flags);
344 prepare_set();
345
346 pat_init();
347
348 post_set();
349 local_irq_restore(flags);
1da177e4
LT
350}
351
a1a499a3 352/* Some BIOS's are messed up and don't set all MTRRs the same! */
1da177e4
LT
353void __init mtrr_state_warn(void)
354{
355 unsigned long mask = smp_changes_mask;
356
357 if (!mask)
358 return;
359 if (mask & MTRR_CHANGE_MASK_FIXED)
a1a499a3 360 pr_warning("mtrr: your CPUs had inconsistent fixed MTRR settings\n");
1da177e4 361 if (mask & MTRR_CHANGE_MASK_VARIABLE)
a1a499a3 362 pr_warning("mtrr: your CPUs had inconsistent variable MTRR settings\n");
1da177e4 363 if (mask & MTRR_CHANGE_MASK_DEFTYPE)
a1a499a3
JSR
364 pr_warning("mtrr: your CPUs had inconsistent MTRRdefType settings\n");
365
1da177e4
LT
366 printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
367 printk(KERN_INFO "mtrr: corrected configuration.\n");
368}
369
a1a499a3
JSR
370/*
371 * Doesn't attempt to pass an error out to MTRR users
372 * because it's quite complicated in some cases and probably not
373 * worth it because the best error handling is to ignore it.
374 */
1da177e4
LT
375void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
376{
a1a499a3 377 if (wrmsr_safe(msr, a, b) < 0) {
1da177e4
LT
378 printk(KERN_ERR
379 "MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
380 smp_processor_id(), msr, a, b);
a1a499a3 381 }
1da177e4
LT
382}
383
de938c51 384/**
a1a499a3
JSR
385 * set_fixed_range - checks & updates a fixed-range MTRR if it
386 * differs from the value it should have
1d3381eb
RD
387 * @msr: MSR address of the MTTR which should be checked and updated
388 * @changed: pointer which indicates whether the MTRR needed to be changed
389 * @msrwords: pointer to the MSR values which the MSR should have
de938c51 390 */
2d2ee8de 391static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
de938c51
BK
392{
393 unsigned lo, hi;
394
395 rdmsr(msr, lo, hi);
396
397 if (lo != msrwords[0] || hi != msrwords[1]) {
de938c51 398 mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
2d2ee8de 399 *changed = true;
de938c51
BK
400 }
401}
402
1d3381eb
RD
403/**
404 * generic_get_free_region - Get a free MTRR.
405 * @base: The starting (base) address of the region.
406 * @size: The size (in bytes) of the region.
407 * @replace_reg: mtrr index to be replaced; set to invalid value if none.
408 *
409 * Returns: The index of the region on success, else negative on error.
410 */
a1a499a3
JSR
411int
412generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
1da177e4 413{
365bff80 414 unsigned long lbase, lsize;
a1a499a3
JSR
415 mtrr_type ltype;
416 int i, max;
1da177e4
LT
417
418 max = num_var_ranges;
365bff80
JB
419 if (replace_reg >= 0 && replace_reg < max)
420 return replace_reg;
a1a499a3 421
1da177e4
LT
422 for (i = 0; i < max; ++i) {
423 mtrr_if->get(i, &lbase, &lsize, &ltype);
424 if (lsize == 0)
425 return i;
426 }
a1a499a3 427
1da177e4
LT
428 return -ENOSPC;
429}
430
408b664a 431static void generic_get_mtrr(unsigned int reg, unsigned long *base,
365bff80 432 unsigned long *size, mtrr_type *type)
1da177e4
LT
433{
434 unsigned int mask_lo, mask_hi, base_lo, base_hi;
38cc1c3d 435 unsigned int tmp, hi;
1da177e4 436
8ad97905
YL
437 /*
438 * get_mtrr doesn't need to update mtrr_state, also it could be called
439 * from any cpu, so try to print it out directly.
440 */
fa10ba64 441 get_cpu();
63516ef6 442
1da177e4 443 rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
8ad97905 444
1da177e4 445 if ((mask_lo & 0x800) == 0) {
a1a499a3 446 /* Invalid (i.e. free) range */
1da177e4
LT
447 *base = 0;
448 *size = 0;
449 *type = 0;
63516ef6 450 goto out_put_cpu;
1da177e4
LT
451 }
452
453 rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);
454
63516ef6 455 /* Work out the shifted address mask: */
38cc1c3d
YL
456 tmp = mask_hi << (32 - PAGE_SHIFT) | mask_lo >> PAGE_SHIFT;
457 mask_lo = size_or_mask | tmp;
63516ef6
YL
458
459 /* Expand tmp with high bits to all 1s: */
38cc1c3d
YL
460 hi = fls(tmp);
461 if (hi > 0) {
462 tmp |= ~((1<<(hi - 1)) - 1);
463
464 if (tmp != mask_lo) {
942fa3b6 465 printk(KERN_WARNING "mtrr: your BIOS has configured an incorrect mask, fixing it.\n");
38cc1c3d
YL
466 mask_lo = tmp;
467 }
468 }
1da177e4 469
63516ef6
YL
470 /*
471 * This works correctly if size is a power of two, i.e. a
472 * contiguous range:
473 */
1da177e4
LT
474 *size = -mask_lo;
475 *base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
476 *type = base_lo & 0xff;
8ad97905 477
63516ef6
YL
478out_put_cpu:
479 put_cpu();
1da177e4
LT
480}
481
de938c51 482/**
a1a499a3
JSR
483 * set_fixed_ranges - checks & updates the fixed-range MTRRs if they
484 * differ from the saved set
1d3381eb 485 * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges()
de938c51 486 */
a1a499a3 487static int set_fixed_ranges(mtrr_type *frs)
1da177e4 488{
a1a499a3 489 unsigned long long *saved = (unsigned long long *)frs;
2d2ee8de 490 bool changed = false;
a1a499a3 491 int block = -1, range;
1da177e4 492
3ff42da5
AH
493 k8_check_syscfg_dram_mod_en();
494
a1a499a3
JSR
495 while (fixed_range_blocks[++block].ranges) {
496 for (range = 0; range < fixed_range_blocks[block].ranges; range++)
497 set_fixed_range(fixed_range_blocks[block].base_msr + range,
498 &changed, (unsigned int *)saved++);
499 }
1da177e4 500
1da177e4
LT
501 return changed;
502}
503
a1a499a3
JSR
504/*
505 * Set the MSR pair relating to a var range.
506 * Returns true if changes are made.
507 */
2d2ee8de 508static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
1da177e4
LT
509{
510 unsigned int lo, hi;
2d2ee8de 511 bool changed = false;
1da177e4
LT
512
513 rdmsr(MTRRphysBase_MSR(index), lo, hi);
514 if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
cf94b62f
SS
515 || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
516 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
a1a499a3 517
1da177e4 518 mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
2d2ee8de 519 changed = true;
1da177e4
LT
520 }
521
522 rdmsr(MTRRphysMask_MSR(index), lo, hi);
523
524 if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
cf94b62f
SS
525 || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
526 (hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
1da177e4 527 mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
2d2ee8de 528 changed = true;
1da177e4
LT
529 }
530 return changed;
531}
532
365bff80
JB
533static u32 deftype_lo, deftype_hi;
534
1d3381eb
RD
535/**
536 * set_mtrr_state - Set the MTRR state for this CPU.
537 *
538 * NOTE: The CPU must already be in a safe state for MTRR changes.
539 * RETURNS: 0 if no changes made, else a mask indicating what was changed.
540 */
365bff80 541static unsigned long set_mtrr_state(void)
1da177e4 542{
1da177e4 543 unsigned long change_mask = 0;
a1a499a3 544 unsigned int i;
1da177e4 545
a1a499a3 546 for (i = 0; i < num_var_ranges; i++) {
1da177e4
LT
547 if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
548 change_mask |= MTRR_CHANGE_MASK_VARIABLE;
a1a499a3 549 }
1da177e4 550
365bff80 551 if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
1da177e4
LT
552 change_mask |= MTRR_CHANGE_MASK_FIXED;
553
a1a499a3
JSR
554 /*
555 * Set_mtrr_restore restores the old value of MTRRdefType,
556 * so to set it we fiddle with the saved value:
557 */
1da177e4
LT
558 if ((deftype_lo & 0xff) != mtrr_state.def_type
559 || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
a1a499a3
JSR
560
561 deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type |
562 (mtrr_state.enabled << 10);
1da177e4
LT
563 change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
564 }
565
566 return change_mask;
567}
568
569
a1a499a3 570static unsigned long cr4;
40d6753e 571static DEFINE_RAW_SPINLOCK(set_atomicity_lock);
1da177e4
LT
572
573/*
a1a499a3
JSR
574 * Since we are disabling the cache don't allow any interrupts,
575 * they would run extremely slow and would only increase the pain.
576 *
577 * The caller must ensure that local interrupts are disabled and
578 * are reenabled after post_set() has been called.
1da177e4 579 */
182daa55 580static void prepare_set(void) __acquires(set_atomicity_lock)
1da177e4
LT
581{
582 unsigned long cr0;
583
a1a499a3
JSR
584 /*
585 * Note that this is not ideal
586 * since the cache is only flushed/disabled for this CPU while the
587 * MTRRs are changed, but changing this requires more invasive
588 * changes to the way the kernel boots
589 */
1da177e4 590
40d6753e 591 raw_spin_lock(&set_atomicity_lock);
1da177e4 592
a1a499a3 593 /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
7ebad705 594 cr0 = read_cr0() | X86_CR0_CD;
1da177e4
LT
595 write_cr0(cr0);
596 wbinvd();
597
a1a499a3
JSR
598 /* Save value of CR4 and clear Page Global Enable (bit 7) */
599 if (cpu_has_pge) {
1da177e4
LT
600 cr4 = read_cr4();
601 write_cr4(cr4 & ~X86_CR4_PGE);
602 }
603
604 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
605 __flush_tlb();
606
a1a499a3 607 /* Save MTRR state */
52650257 608 rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
1da177e4 609
a1a499a3 610 /* Disable MTRRs, and set the default type to uncached */
52650257 611 mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
1da177e4
LT
612}
613
182daa55 614static void post_set(void) __releases(set_atomicity_lock)
1da177e4 615{
a1a499a3 616 /* Flush TLBs (no need to flush caches - they are disabled) */
1da177e4
LT
617 __flush_tlb();
618
619 /* Intel (P6) standard MTRRs */
52650257 620 mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
a1a499a3
JSR
621
622 /* Enable caches */
1da177e4
LT
623 write_cr0(read_cr0() & 0xbfffffff);
624
a1a499a3
JSR
625 /* Restore value of CR4 */
626 if (cpu_has_pge)
1da177e4 627 write_cr4(cr4);
40d6753e 628 raw_spin_unlock(&set_atomicity_lock);
1da177e4
LT
629}
630
631static void generic_set_all(void)
632{
633 unsigned long mask, count;
634 unsigned long flags;
635
636 local_irq_save(flags);
637 prepare_set();
638
639 /* Actually set the state */
365bff80 640 mask = set_mtrr_state();
1da177e4 641
2e5d9c85 642 /* also set PAT */
643 pat_init();
644
1da177e4
LT
645 post_set();
646 local_irq_restore(flags);
647
a1a499a3 648 /* Use the atomic bitops to update the global mask */
1da177e4
LT
649 for (count = 0; count < sizeof mask * 8; ++count) {
650 if (mask & 0x01)
651 set_bit(count, &smp_changes_mask);
652 mask >>= 1;
653 }
a1a499a3 654
1da177e4
LT
655}
656
a1a499a3
JSR
657/**
658 * generic_set_mtrr - set variable MTRR register on the local CPU.
659 *
660 * @reg: The register to set.
661 * @base: The base address of the region.
662 * @size: The size of the region. If this is 0 the region is disabled.
663 * @type: The type of the region.
664 *
665 * Returns nothing.
666 */
1da177e4
LT
667static void generic_set_mtrr(unsigned int reg, unsigned long base,
668 unsigned long size, mtrr_type type)
1da177e4
LT
669{
670 unsigned long flags;
3b520b23
SL
671 struct mtrr_var_range *vr;
672
673 vr = &mtrr_state.var_ranges[reg];
1da177e4
LT
674
675 local_irq_save(flags);
676 prepare_set();
677
678 if (size == 0) {
a1a499a3
JSR
679 /*
680 * The invalid bit is kept in the mask, so we simply
681 * clear the relevant mask register to disable a range.
682 */
1da177e4 683 mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
3b520b23 684 memset(vr, 0, sizeof(struct mtrr_var_range));
1da177e4 685 } else {
3b520b23
SL
686 vr->base_lo = base << PAGE_SHIFT | type;
687 vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
688 vr->mask_lo = -size << PAGE_SHIFT | 0x800;
689 vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);
690
691 mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
692 mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
1da177e4
LT
693 }
694
695 post_set();
696 local_irq_restore(flags);
697}
698
a1a499a3
JSR
699int generic_validate_add_page(unsigned long base, unsigned long size,
700 unsigned int type)
1da177e4
LT
701{
702 unsigned long lbase, last;
703
a1a499a3
JSR
704 /*
705 * For Intel PPro stepping <= 7
706 * must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF
707 */
1da177e4
LT
708 if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
709 boot_cpu_data.x86_model == 1 &&
710 boot_cpu_data.x86_mask <= 7) {
711 if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
a1a499a3 712 pr_warning("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
1da177e4
LT
713 return -EINVAL;
714 }
9b483417 715 if (!(base + size < 0x70000 || base > 0x7003F) &&
1da177e4
LT
716 (type == MTRR_TYPE_WRCOMB
717 || type == MTRR_TYPE_WRBACK)) {
a1a499a3 718 pr_warning("mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
1da177e4
LT
719 return -EINVAL;
720 }
721 }
722
a1a499a3
JSR
723 /*
724 * Check upper bits of base and last are equal and lower bits are 0
725 * for base and 1 for last
726 */
1da177e4
LT
727 last = base + size - 1;
728 for (lbase = base; !(lbase & 1) && (last & 1);
a1a499a3
JSR
729 lbase = lbase >> 1, last = last >> 1)
730 ;
1da177e4 731 if (lbase != last) {
a1a499a3 732 pr_warning("mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", base, size);
1da177e4
LT
733 return -EINVAL;
734 }
735 return 0;
736}
737
1da177e4
LT
738static int generic_have_wrcomb(void)
739{
740 unsigned long config, dummy;
d9bcc01d 741 rdmsr(MSR_MTRRcap, config, dummy);
a1a499a3 742 return config & (1 << 10);
1da177e4
LT
743}
744
745int positive_have_wrcomb(void)
746{
747 return 1;
748}
749
a1a499a3
JSR
750/*
751 * Generic structure...
1da177e4 752 */
3b9cfc0a 753const struct mtrr_ops generic_mtrr_ops = {
a1a499a3
JSR
754 .use_intel_if = 1,
755 .set_all = generic_set_all,
756 .get = generic_get_mtrr,
757 .get_free_region = generic_get_free_region,
758 .set = generic_set_mtrr,
759 .validate_add_page = generic_validate_add_page,
760 .have_wrcomb = generic_have_wrcomb,
1da177e4 761};