drivers/acpi/processor_throttling.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * processor_throttling.c - Throttling submodule of the ACPI processor driver
   4  *
   5  *  Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com>
   6  *  Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
   7  *  Copyright (C) 2004       Dominik Brodowski <linux@brodo.de>
   8  *  Copyright (C) 2004  Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
   9  *                      - Added processor hotplug support
  10  */
  11
  12 #define pr_fmt(fmt) "ACPI: " fmt
  13
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/slab.h>
  17 #include <linux/init.h>
  18 #include <linux/sched.h>
  19 #include <linux/cpufreq.h>
  20 #include <linux/acpi.h>
  21 #include <linux/uaccess.h>
  22 #include <acpi/processor.h>
  23 #include <asm/io.h>
  24 #ifdef CONFIG_X86
  25 #include <asm/msr.h>
  26 #endif
  27
  28 /* ignore_tpc:
  29  *  0 -> acpi processor driver doesn't ignore _TPC values
  30  *  1 -> acpi processor driver ignores _TPC values
  31  */
  32 static int ignore_tpc;
  33 module_param(ignore_tpc, int, 0644);
  34 MODULE_PARM_DESC(ignore_tpc, "Disable broken BIOS _TPC throttling support");
  35
  36 struct throttling_tstate {
  37         unsigned int cpu;               /* cpu nr */
  38         int target_state;               /* target T-state */
  39 };
  40
  41 struct acpi_processor_throttling_arg {
  42         struct acpi_processor *pr;
  43         int target_state;
  44         bool force;
  45 };
  46
  47 #define THROTTLING_PRECHANGE       (1)
  48 #define THROTTLING_POSTCHANGE      (2)
  49
  50 static int acpi_processor_get_throttling(struct acpi_processor *pr);
  51 static int __acpi_processor_set_throttling(struct acpi_processor *pr,
  52                                            int state, bool force, bool direct);
  53
  54 static int acpi_processor_update_tsd_coord(void)
  55 {
  56         int count_target;
  57         int retval = 0;
  58         unsigned int i, j;
  59         cpumask_var_t covered_cpus;
  60         struct acpi_processor *pr, *match_pr;
  61         struct acpi_tsd_package *pdomain, *match_pdomain;
  62         struct acpi_processor_throttling *pthrottling, *match_pthrottling;
  63
  64         if (!zalloc_cpumask_var(&covered_cpus, GFP_KERNEL))
  65                 return -ENOMEM;
  66
  67         /*
  68          * Now that we have _TSD data from all CPUs, lets setup T-state
  69          * coordination between all CPUs.
  70          */
  71         for_each_possible_cpu(i) {
  72                 pr = per_cpu(processors, i);
  73                 if (!pr)
  74                         continue;
  75
  76                 /* Basic validity check for domain info */
  77                 pthrottling = &(pr->throttling);
  78
  79                 /*
  80                  * If tsd package for one cpu is invalid, the coordination
  81                  * among all CPUs is thought as invalid.
  82                  * Maybe it is ugly.
  83                  */
  84                 if (!pthrottling->tsd_valid_flag) {
  85                         retval = -EINVAL;
  86                         break;
  87                 }
  88         }
  89         if (retval)
  90                 goto err_ret;
  91
  92         for_each_possible_cpu(i) {
  93                 pr = per_cpu(processors, i);
  94                 if (!pr)
  95                         continue;
  96
  97                 if (cpumask_test_cpu(i, covered_cpus))
  98                         continue;
  99                 pthrottling = &pr->throttling;
 100
 101                 pdomain = &(pthrottling->domain_info);
 102                 cpumask_set_cpu(i, pthrottling->shared_cpu_map);
 103                 cpumask_set_cpu(i, covered_cpus);
 104                 /*
 105                  * If the number of processor in the TSD domain is 1, it is
 106                  * unnecessary to parse the coordination for this CPU.
 107                  */
 108                 if (pdomain->num_processors <= 1)
 109                         continue;
 110
 111                 /* Validate the Domain info */
 112                 count_target = pdomain->num_processors;
 113
 114                 for_each_possible_cpu(j) {
 115                         if (i == j)
 116                                 continue;
 117
 118                         match_pr = per_cpu(processors, j);
 119                         if (!match_pr)
 120                                 continue;
 121
 122                         match_pthrottling = &(match_pr->throttling);
 123                         match_pdomain = &(match_pthrottling->domain_info);
 124                         if (match_pdomain->domain != pdomain->domain)
 125                                 continue;
 126
 127                         /* Here i and j are in the same domain.
 128                          * If two TSD packages have the same domain, they
 129                          * should have the same num_porcessors and
 130                          * coordination type. Otherwise it will be regarded
 131                          * as illegal.
 132                          */
 133                         if (match_pdomain->num_processors != count_target) {
 134                                 retval = -EINVAL;
 135                                 goto err_ret;
 136                         }
 137
 138                         if (pdomain->coord_type != match_pdomain->coord_type) {
 139                                 retval = -EINVAL;
 140                                 goto err_ret;
 141                         }
 142
 143                         cpumask_set_cpu(j, covered_cpus);
 144                         cpumask_set_cpu(j, pthrottling->shared_cpu_map);
 145                 }
 146                 for_each_possible_cpu(j) {
 147                         if (i == j)
 148                                 continue;
 149
 150                         match_pr = per_cpu(processors, j);
 151                         if (!match_pr)
 152                                 continue;
 153
 154                         match_pthrottling = &(match_pr->throttling);
 155                         match_pdomain = &(match_pthrottling->domain_info);
 156                         if (match_pdomain->domain != pdomain->domain)
 157                                 continue;
 158
 159                         /*
 160                          * If some CPUS have the same domain, they
 161                          * will have the same shared_cpu_map.
 162                          */
 163                         cpumask_copy(match_pthrottling->shared_cpu_map,
 164                                      pthrottling->shared_cpu_map);
 165                 }
 166         }
 167
 168 err_ret:
 169         free_cpumask_var(covered_cpus);
 170
 171         for_each_possible_cpu(i) {
 172                 pr = per_cpu(processors, i);
 173                 if (!pr)
 174                         continue;
 175
 176                 /*
 177                  * Assume no coordination on any error parsing domain info.
 178                  * The coordination type will be forced as SW_ALL.
 179                  */
 180                 if (retval) {
 181                         pthrottling = &(pr->throttling);
 182                         cpumask_clear(pthrottling->shared_cpu_map);
 183                         cpumask_set_cpu(i, pthrottling->shared_cpu_map);
 184                         pthrottling->shared_type = DOMAIN_COORD_TYPE_SW_ALL;
 185                 }
 186         }
 187
 188         return retval;
 189 }
 190
 191 /*
 192  * Update the T-state coordination after the _TSD
 193  * data for all cpus is obtained.
 194  */
 195 void acpi_processor_throttling_init(void)
 196 {
 197         if (acpi_processor_update_tsd_coord())
 198                 pr_debug("Assume no T-state coordination\n");
 199 }
 200
 201 static int acpi_processor_throttling_notifier(unsigned long event, void *data)
 202 {
 203         struct throttling_tstate *p_tstate = data;
 204         struct acpi_processor *pr;
 205         unsigned int cpu;
 206         int target_state;
 207         struct acpi_processor_limit *p_limit;
 208         struct acpi_processor_throttling *p_throttling;
 209
 210         cpu = p_tstate->cpu;
 211         pr = per_cpu(processors, cpu);
 212         if (!pr) {
 213                 pr_debug("Invalid pr pointer\n");
 214                 return 0;
 215         }
 216         if (!pr->flags.throttling) {
 217                 acpi_handle_debug(pr->handle,
 218                                   "Throttling control unsupported on CPU %d\n",
 219                                   cpu);
 220                 return 0;
 221         }
 222         target_state = p_tstate->target_state;
 223         p_throttling = &(pr->throttling);
 224         switch (event) {
 225         case THROTTLING_PRECHANGE:
 226                 /*
 227                  * Prechange event is used to choose one proper t-state,
 228                  * which meets the limits of thermal, user and _TPC.
 229                  */
 230                 p_limit = &pr->limit;
 231                 if (p_limit->thermal.tx > target_state)
 232                         target_state = p_limit->thermal.tx;
 233                 if (p_limit->user.tx > target_state)
 234                         target_state = p_limit->user.tx;
 235                 if (pr->throttling_platform_limit > target_state)
 236                         target_state = pr->throttling_platform_limit;
 237                 if (target_state >= p_throttling->state_count) {
 238                         pr_warn("Exceed the limit of T-state \n");
 239                         target_state = p_throttling->state_count - 1;
 240                 }
 241                 p_tstate->target_state = target_state;
 242                 acpi_handle_debug(pr->handle,
 243                                   "PreChange Event: target T-state of CPU %d is T%d\n",
 244                                   cpu, target_state);
 245                 break;
 246         case THROTTLING_POSTCHANGE:
 247                 /*
 248                  * Postchange event is only used to update the
 249                  * T-state flag of acpi_processor_throttling.
 250                  */
 251                 p_throttling->state = target_state;
 252                 acpi_handle_debug(pr->handle,
 253                                   "PostChange Event: CPU %d is switched to T%d\n",
 254                                   cpu, target_state);
 255                 break;
 256         default:
 257                 pr_warn("Unsupported Throttling notifier event\n");
 258                 break;
 259         }
 260
 261         return 0;
 262 }
 263
 264 /*
 265  * _TPC - Throttling Present Capabilities
 266  */
 267 static int acpi_processor_get_platform_limit(struct acpi_processor *pr)
 268 {
 269         acpi_status status = 0;
 270         unsigned long long tpc = 0;
 271
 272         if (!pr)
 273                 return -EINVAL;
 274
 275         if (ignore_tpc)
 276                 goto end;
 277
 278         status = acpi_evaluate_integer(pr->handle, "_TPC", NULL, &tpc);
 279         if (ACPI_FAILURE(status)) {
 280                 if (status != AE_NOT_FOUND)
 281                         acpi_evaluation_failure_warn(pr->handle, "_TPC", status);
 282
 283                 return -ENODEV;
 284         }
 285
 286 end:
 287         pr->throttling_platform_limit = (int)tpc;
 288         return 0;
 289 }
 290
 291 int acpi_processor_tstate_has_changed(struct acpi_processor *pr)
 292 {
 293         int result = 0;
 294         int throttling_limit;
 295         int current_state;
 296         struct acpi_processor_limit *limit;
 297         int target_state;
 298
 299         if (ignore_tpc)
 300                 return 0;
 301
 302         result = acpi_processor_get_platform_limit(pr);
 303         if (result) {
 304                 /* Throttling Limit is unsupported */
 305                 return result;
 306         }
 307
 308         throttling_limit = pr->throttling_platform_limit;
 309         if (throttling_limit >= pr->throttling.state_count) {
 310                 /* Uncorrect Throttling Limit */
 311                 return -EINVAL;
 312         }
 313
 314         current_state = pr->throttling.state;
 315         if (current_state > throttling_limit) {
 316                 /*
 317                  * The current state can meet the requirement of
 318                  * _TPC limit. But it is reasonable that OSPM changes
 319                  * t-states from high to low for better performance.
 320                  * Of course the limit condition of thermal
 321                  * and user should be considered.
 322                  */
 323                 limit = &pr->limit;
 324                 target_state = throttling_limit;
 325                 if (limit->thermal.tx > target_state)
 326                         target_state = limit->thermal.tx;
 327                 if (limit->user.tx > target_state)
 328                         target_state = limit->user.tx;
 329         } else if (current_state == throttling_limit) {
 330                 /*
 331                  * Unnecessary to change the throttling state
 332                  */
 333                 return 0;
 334         } else {
 335                 /*
 336                  * If the current state is lower than the limit of _TPC, it
 337                  * will be forced to switch to the throttling state defined
 338                  * by throttling_platfor_limit.
 339                  * Because the previous state meets with the limit condition
 340                  * of thermal and user, it is unnecessary to check it again.
 341                  */
 342                 target_state = throttling_limit;
 343         }
 344         return acpi_processor_set_throttling(pr, target_state, false);
 345 }
 346
 347 /*
 348  * This function is used to reevaluate whether the T-state is valid
 349  * after one CPU is onlined/offlined.
 350  * It is noted that it won't reevaluate the following properties for
 351  * the T-state.
 352  *      1. Control method.
 353  *      2. the number of supported T-state
 354  *      3. TSD domain
 355  */
 356 void acpi_processor_reevaluate_tstate(struct acpi_processor *pr,
 357                                         bool is_dead)
 358 {
 359         int result = 0;
 360
 361         if (is_dead) {
 362                 /* When one CPU is offline, the T-state throttling
 363                  * will be invalidated.
 364                  */
 365                 pr->flags.throttling = 0;
 366                 return;
 367         }
 368         /* the following is to recheck whether the T-state is valid for
 369          * the online CPU
 370          */
 371         if (!pr->throttling.state_count) {
 372                 /* If the number of T-state is invalid, it is
 373                  * invalidated.
 374                  */
 375                 pr->flags.throttling = 0;
 376                 return;
 377         }
 378         pr->flags.throttling = 1;
 379
 380         /* Disable throttling (if enabled).  We'll let subsequent
 381          * policy (e.g.thermal) decide to lower performance if it
 382          * so chooses, but for now we'll crank up the speed.
 383          */
 384
 385         result = acpi_processor_get_throttling(pr);
 386         if (result)
 387                 goto end;
 388
 389         if (pr->throttling.state) {
 390                 result = acpi_processor_set_throttling(pr, 0, false);
 391                 if (result)
 392                         goto end;
 393         }
 394
 395 end:
 396         if (result)
 397                 pr->flags.throttling = 0;
 398 }
 399 /*
 400  * _PTC - Processor Throttling Control (and status) register location
 401  */
 402 static int acpi_processor_get_throttling_control(struct acpi_processor *pr)
 403 {
 404         int result = 0;
 405         acpi_status status = 0;
 406         struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 407         union acpi_object *ptc = NULL;
 408         union acpi_object obj;
 409         struct acpi_processor_throttling *throttling;
 410
 411         status = acpi_evaluate_object(pr->handle, "_PTC", NULL, &buffer);
 412         if (ACPI_FAILURE(status)) {
 413                 if (status != AE_NOT_FOUND)
 414                         acpi_evaluation_failure_warn(pr->handle, "_PTC", status);
 415
 416                 return -ENODEV;
 417         }
 418
 419         ptc = (union acpi_object *)buffer.pointer;
 420         if (!ptc || (ptc->type != ACPI_TYPE_PACKAGE)
 421             || (ptc->package.count != 2)) {
 422                 pr_err("Invalid _PTC data\n");
 423                 result = -EFAULT;
 424                 goto end;
 425         }
 426
 427         /*
 428          * control_register
 429          */
 430
 431         obj = ptc->package.elements[0];
 432
 433         if ((obj.type != ACPI_TYPE_BUFFER)
 434             || (obj.buffer.length < sizeof(struct acpi_ptc_register))
 435             || (obj.buffer.pointer == NULL)) {
 436                 pr_err("Invalid _PTC data (control_register)\n");
 437                 result = -EFAULT;
 438                 goto end;
 439         }
 440         memcpy(&pr->throttling.control_register, obj.buffer.pointer,
 441                sizeof(struct acpi_ptc_register));
 442
 443         /*
 444          * status_register
 445          */
 446
 447         obj = ptc->package.elements[1];
 448
 449         if ((obj.type != ACPI_TYPE_BUFFER)
 450             || (obj.buffer.length < sizeof(struct acpi_ptc_register))
 451             || (obj.buffer.pointer == NULL)) {
 452                 pr_err("Invalid _PTC data (status_register)\n");
 453                 result = -EFAULT;
 454                 goto end;
 455         }
 456
 457         memcpy(&pr->throttling.status_register, obj.buffer.pointer,
 458                sizeof(struct acpi_ptc_register));
 459
 460         throttling = &pr->throttling;
 461
 462         if ((throttling->control_register.bit_width +
 463                 throttling->control_register.bit_offset) > 32) {
 464                 pr_err("Invalid _PTC control register\n");
 465                 result = -EFAULT;
 466                 goto end;
 467         }
 468
 469         if ((throttling->status_register.bit_width +
 470                 throttling->status_register.bit_offset) > 32) {
 471                 pr_err("Invalid _PTC status register\n");
 472                 result = -EFAULT;
 473                 goto end;
 474         }
 475
 476 end:
 477         kfree(buffer.pointer);
 478
 479         return result;
 480 }
 481
 482 /*
 483  * _TSS - Throttling Supported States
 484  */
 485 static int acpi_processor_get_throttling_states(struct acpi_processor *pr)
 486 {
 487         int result = 0;
 488         acpi_status status = AE_OK;
 489         struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 490         struct acpi_buffer format = { sizeof("NNNNN"), "NNNNN" };
 491         struct acpi_buffer state = { 0, NULL };
 492         union acpi_object *tss = NULL;
 493         int i;
 494
 495         status = acpi_evaluate_object(pr->handle, "_TSS", NULL, &buffer);
 496         if (ACPI_FAILURE(status)) {
 497                 if (status != AE_NOT_FOUND)
 498                         acpi_evaluation_failure_warn(pr->handle, "_TSS", status);
 499
 500                 return -ENODEV;
 501         }
 502
 503         tss = buffer.pointer;
 504         if (!tss || (tss->type != ACPI_TYPE_PACKAGE)) {
 505                 pr_err("Invalid _TSS data\n");
 506                 result = -EFAULT;
 507                 goto end;
 508         }
 509
 510         acpi_handle_debug(pr->handle, "Found %d throttling states\n",
 511                           tss->package.count);
 512
 513         pr->throttling.state_count = tss->package.count;
 514         pr->throttling.states_tss =
 515             kmalloc_array(tss->package.count,
 516                           sizeof(struct acpi_processor_tx_tss),
 517                           GFP_KERNEL);
 518         if (!pr->throttling.states_tss) {
 519                 result = -ENOMEM;
 520                 goto end;
 521         }
 522
 523         for (i = 0; i < pr->throttling.state_count; i++) {
 524
 525                 struct acpi_processor_tx_tss *tx =
 526                     (struct acpi_processor_tx_tss *)&(pr->throttling.
 527                                                       states_tss[i]);
 528
 529                 state.length = sizeof(struct acpi_processor_tx_tss);
 530                 state.pointer = tx;
 531
 532                 acpi_handle_debug(pr->handle, "Extracting state %d\n", i);
 533
 534                 status = acpi_extract_package(&(tss->package.elements[i]),
 535                                               &format, &state);
 536                 if (ACPI_FAILURE(status)) {
 537                         acpi_handle_warn(pr->handle, "Invalid _TSS data: %s\n",
 538                                          acpi_format_exception(status));
 539                         result = -EFAULT;
 540                         kfree(pr->throttling.states_tss);
 541                         goto end;
 542                 }
 543
 544                 if (!tx->freqpercentage) {
 545                         pr_err("Invalid _TSS data: freq is zero\n");
 546                         result = -EFAULT;
 547                         kfree(pr->throttling.states_tss);
 548                         goto end;
 549                 }
 550         }
 551
 552 end:
 553         kfree(buffer.pointer);
 554
 555         return result;
 556 }
 557
 558 /*
 559  * _TSD - T-State Dependencies
 560  */
 561 static int acpi_processor_get_tsd(struct acpi_processor *pr)
 562 {
 563         int result = 0;
 564         acpi_status status = AE_OK;
 565         struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 566         struct acpi_buffer format = { sizeof("NNNNN"), "NNNNN" };
 567         struct acpi_buffer state = { 0, NULL };
 568         union acpi_object *tsd = NULL;
 569         struct acpi_tsd_package *pdomain;
 570         struct acpi_processor_throttling *pthrottling;
 571
 572         pthrottling = &pr->throttling;
 573         pthrottling->tsd_valid_flag = 0;
 574
 575         status = acpi_evaluate_object(pr->handle, "_TSD", NULL, &buffer);
 576         if (ACPI_FAILURE(status)) {
 577                 if (status != AE_NOT_FOUND)
 578                         acpi_evaluation_failure_warn(pr->handle, "_TSD", status);
 579
 580                 return -ENODEV;
 581         }
 582
 583         tsd = buffer.pointer;
 584         if (!tsd || (tsd->type != ACPI_TYPE_PACKAGE)) {
 585                 pr_err("Invalid _TSD data\n");
 586                 result = -EFAULT;
 587                 goto end;
 588         }
 589
 590         if (tsd->package.count != 1) {
 591                 pr_err("Invalid _TSD data\n");
 592                 result = -EFAULT;
 593                 goto end;
 594         }
 595
 596         pdomain = &(pr->throttling.domain_info);
 597
 598         state.length = sizeof(struct acpi_tsd_package);
 599         state.pointer = pdomain;
 600
 601         status = acpi_extract_package(&(tsd->package.elements[0]),
 602                                       &format, &state);
 603         if (ACPI_FAILURE(status)) {
 604                 pr_err("Invalid _TSD data\n");
 605                 result = -EFAULT;
 606                 goto end;
 607         }
 608
 609         if (pdomain->num_entries != ACPI_TSD_REV0_ENTRIES) {
 610                 pr_err("Unknown _TSD:num_entries\n");
 611                 result = -EFAULT;
 612                 goto end;
 613         }
 614
 615         if (pdomain->revision != ACPI_TSD_REV0_REVISION) {
 616                 pr_err("Unknown _TSD:revision\n");
 617                 result = -EFAULT;
 618                 goto end;
 619         }
 620
 621         pthrottling = &pr->throttling;
 622         pthrottling->tsd_valid_flag = 1;
 623         pthrottling->shared_type = pdomain->coord_type;
 624         cpumask_set_cpu(pr->id, pthrottling->shared_cpu_map);
 625         /*
 626          * If the coordination type is not defined in ACPI spec,
 627          * the tsd_valid_flag will be clear and coordination type
 628          * will be forecd as DOMAIN_COORD_TYPE_SW_ALL.
 629          */
 630         if (pdomain->coord_type != DOMAIN_COORD_TYPE_SW_ALL &&
 631                 pdomain->coord_type != DOMAIN_COORD_TYPE_SW_ANY &&
 632                 pdomain->coord_type != DOMAIN_COORD_TYPE_HW_ALL) {
 633                 pthrottling->tsd_valid_flag = 0;
 634                 pthrottling->shared_type = DOMAIN_COORD_TYPE_SW_ALL;
 635         }
 636
 637 end:
 638         kfree(buffer.pointer);
 639         return result;
 640 }
 641
 642 /* --------------------------------------------------------------------------
 643                               Throttling Control
 644    -------------------------------------------------------------------------- */
 645 static int acpi_processor_get_throttling_fadt(struct acpi_processor *pr)
 646 {
 647         int state = 0;
 648         u32 value = 0;
 649         u32 duty_mask = 0;
 650         u32 duty_value = 0;
 651
 652         if (!pr)
 653                 return -EINVAL;
 654
 655         if (!pr->flags.throttling)
 656                 return -ENODEV;
 657
 658         /*
 659          * We don't care about error returns - we just try to mark
 660          * these reserved so that nobody else is confused into thinking
 661          * that this region might be unused..
 662          *
 663          * (In particular, allocating the IO range for Cardbus)
 664          */
 665         request_region(pr->throttling.address, 6, "ACPI CPU throttle");
 666
 667         pr->throttling.state = 0;
 668
 669         duty_mask = pr->throttling.state_count - 1;
 670
 671         duty_mask <<= pr->throttling.duty_offset;
 672
 673         local_irq_disable();
 674
 675         value = inl(pr->throttling.address);
 676
 677         /*
 678          * Compute the current throttling state when throttling is enabled
 679          * (bit 4 is on).
 680          */
 681         if (value & 0x10) {
 682                 duty_value = value & duty_mask;
 683                 duty_value >>= pr->throttling.duty_offset;
 684
 685                 if (duty_value)
 686                         state = pr->throttling.state_count - duty_value;
 687         }
 688
 689         pr->throttling.state = state;
 690
 691         local_irq_enable();
 692
 693         acpi_handle_debug(pr->handle,
 694                           "Throttling state is T%d (%d%% throttling applied)\n",
 695                           state, pr->throttling.states[state].performance);
 696
 697         return 0;
 698 }
 699
 700 #ifdef CONFIG_X86
 701 static int acpi_throttling_rdmsr(u64 *value)
 702 {
 703         u64 msr_high, msr_low;
 704         u64 msr = 0;
 705         int ret = -1;
 706
 707         if ((this_cpu_read(cpu_info.x86_vendor) != X86_VENDOR_INTEL) ||
 708                 !this_cpu_has(X86_FEATURE_ACPI)) {
 709                 pr_err("HARDWARE addr space,NOT supported yet\n");
 710         } else {
 711                 msr_low = 0;
 712                 msr_high = 0;
 713                 rdmsr_safe(MSR_IA32_THERM_CONTROL,
 714                         (u32 *)&msr_low, (u32 *) &msr_high);
 715                 msr = (msr_high << 32) | msr_low;
 716                 *value = (u64) msr;
 717                 ret = 0;
 718         }
 719         return ret;
 720 }
 721
 722 static int acpi_throttling_wrmsr(u64 value)
 723 {
 724         int ret = -1;
 725         u64 msr;
 726
 727         if ((this_cpu_read(cpu_info.x86_vendor) != X86_VENDOR_INTEL) ||
 728                 !this_cpu_has(X86_FEATURE_ACPI)) {
 729                 pr_err("HARDWARE addr space,NOT supported yet\n");
 730         } else {
 731                 msr = value;
 732                 wrmsr_safe(MSR_IA32_THERM_CONTROL,
 733                         msr & 0xffffffff, msr >> 32);
 734                 ret = 0;
 735         }
 736         return ret;
 737 }
 738 #else
 739 static int acpi_throttling_rdmsr(u64 *value)
 740 {
 741         pr_err("HARDWARE addr space,NOT supported yet\n");
 742         return -1;
 743 }
 744
 745 static int acpi_throttling_wrmsr(u64 value)
 746 {
 747         pr_err("HARDWARE addr space,NOT supported yet\n");
 748         return -1;
 749 }
 750 #endif
 751
 752 static int acpi_read_throttling_status(struct acpi_processor *pr,
 753                                         u64 *value)
 754 {
 755         u32 bit_width, bit_offset;
 756         u32 ptc_value;
 757         u64 ptc_mask;
 758         struct acpi_processor_throttling *throttling;
 759         int ret = -1;
 760
 761         throttling = &pr->throttling;
 762         switch (throttling->status_register.space_id) {
 763         case ACPI_ADR_SPACE_SYSTEM_IO:
 764                 bit_width = throttling->status_register.bit_width;
 765                 bit_offset = throttling->status_register.bit_offset;
 766
 767                 acpi_os_read_port((acpi_io_address) throttling->status_register.
 768                                   address, &ptc_value,
 769                                   (u32) (bit_width + bit_offset));
 770                 ptc_mask = (1 << bit_width) - 1;
 771                 *value = (u64) ((ptc_value >> bit_offset) & ptc_mask);
 772                 ret = 0;
 773                 break;
 774         case ACPI_ADR_SPACE_FIXED_HARDWARE:
 775                 ret = acpi_throttling_rdmsr(value);
 776                 break;
 777         default:
 778                 pr_err("Unknown addr space %d\n",
 779                        (u32) (throttling->status_register.space_id));
 780         }
 781         return ret;
 782 }
 783
 784 static int acpi_write_throttling_state(struct acpi_processor *pr,
 785                                 u64 value)
 786 {
 787         u32 bit_width, bit_offset;
 788         u64 ptc_value;
 789         u64 ptc_mask;
 790         struct acpi_processor_throttling *throttling;
 791         int ret = -1;
 792
 793         throttling = &pr->throttling;
 794         switch (throttling->control_register.space_id) {
 795         case ACPI_ADR_SPACE_SYSTEM_IO:
 796                 bit_width = throttling->control_register.bit_width;
 797                 bit_offset = throttling->control_register.bit_offset;
 798                 ptc_mask = (1 << bit_width) - 1;
 799                 ptc_value = value & ptc_mask;
 800
 801                 acpi_os_write_port((acpi_io_address) throttling->
 802                                         control_register.address,
 803                                         (u32) (ptc_value << bit_offset),
 804                                         (u32) (bit_width + bit_offset));
 805                 ret = 0;
 806                 break;
 807         case ACPI_ADR_SPACE_FIXED_HARDWARE:
 808                 ret = acpi_throttling_wrmsr(value);
 809                 break;
 810         default:
 811                 pr_err("Unknown addr space %d\n",
 812                        (u32) (throttling->control_register.space_id));
 813         }
 814         return ret;
 815 }
 816
 817 static int acpi_get_throttling_state(struct acpi_processor *pr,
 818                                 u64 value)
 819 {
 820         int i;
 821
 822         for (i = 0; i < pr->throttling.state_count; i++) {
 823                 struct acpi_processor_tx_tss *tx =
 824                     (struct acpi_processor_tx_tss *)&(pr->throttling.
 825                                                       states_tss[i]);
 826                 if (tx->control == value)
 827                         return i;
 828         }
 829         return -1;
 830 }
 831
 832 static int acpi_get_throttling_value(struct acpi_processor *pr,
 833                         int state, u64 *value)
 834 {
 835         int ret = -1;
 836
 837         if (state >= 0 && state <= pr->throttling.state_count) {
 838                 struct acpi_processor_tx_tss *tx =
 839                     (struct acpi_processor_tx_tss *)&(pr->throttling.
 840                                                       states_tss[state]);
 841                 *value = tx->control;
 842                 ret = 0;
 843         }
 844         return ret;
 845 }
 846
 847 static int acpi_processor_get_throttling_ptc(struct acpi_processor *pr)
 848 {
 849         int state = 0;
 850         int ret;
 851         u64 value;
 852
 853         if (!pr)
 854                 return -EINVAL;
 855
 856         if (!pr->flags.throttling)
 857                 return -ENODEV;
 858
 859         pr->throttling.state = 0;
 860
 861         value = 0;
 862         ret = acpi_read_throttling_status(pr, &value);
 863         if (ret >= 0) {
 864                 state = acpi_get_throttling_state(pr, value);
 865                 if (state == -1) {
 866                         acpi_handle_debug(pr->handle,
 867                                           "Invalid throttling state, reset\n");
 868                         state = 0;
 869                         ret = __acpi_processor_set_throttling(pr, state, true,
 870                                                               true);
 871                         if (ret)
 872                                 return ret;
 873                 }
 874                 pr->throttling.state = state;
 875         }
 876
 877         return 0;
 878 }
 879
 880 static long __acpi_processor_get_throttling(void *data)
 881 {
 882         struct acpi_processor *pr = data;
 883
 884         return pr->throttling.acpi_processor_get_throttling(pr);
 885 }
 886
 887 static int acpi_processor_get_throttling(struct acpi_processor *pr)
 888 {
 889         if (!pr)
 890                 return -EINVAL;
 891
 892         if (!pr->flags.throttling)
 893                 return -ENODEV;
 894
 895         /*
 896          * This is either called from the CPU hotplug callback of
 897          * processor_driver or via the ACPI probe function. In the latter
 898          * case the CPU is not guaranteed to be online. Both call sites are
 899          * protected against CPU hotplug.
 900          */
 901         if (!cpu_online(pr->id))
 902                 return -ENODEV;
 903
 904         return call_on_cpu(pr->id, __acpi_processor_get_throttling, pr, false);
 905 }
 906
 907 static int acpi_processor_get_fadt_info(struct acpi_processor *pr)
 908 {
 909         int i, step;
 910
 911         if (!pr->throttling.address) {
 912                 acpi_handle_debug(pr->handle, "No throttling register\n");
 913                 return -EINVAL;
 914         } else if (!pr->throttling.duty_width) {
 915                 acpi_handle_debug(pr->handle, "No throttling states\n");
 916                 return -EINVAL;
 917         }
 918         /* TBD: Support duty_cycle values that span bit 4. */
 919         else if ((pr->throttling.duty_offset + pr->throttling.duty_width) > 4) {
 920                 pr_warn("duty_cycle spans bit 4\n");
 921                 return -EINVAL;
 922         }
 923
 924         pr->throttling.state_count = 1 << acpi_gbl_FADT.duty_width;
 925
 926         /*
 927          * Compute state values. Note that throttling displays a linear power
 928          * performance relationship (at 50% performance the CPU will consume
 929          * 50% power).  Values are in 1/10th of a percent to preserve accuracy.
 930          */
 931
 932         step = (1000 / pr->throttling.state_count);
 933
 934         for (i = 0; i < pr->throttling.state_count; i++) {
 935                 pr->throttling.states[i].performance = 1000 - step * i;
 936                 pr->throttling.states[i].power = 1000 - step * i;
 937         }
 938         return 0;
 939 }
 940
 941 static int acpi_processor_set_throttling_fadt(struct acpi_processor *pr,
 942                                               int state, bool force)
 943 {
 944         u32 value = 0;
 945         u32 duty_mask = 0;
 946         u32 duty_value = 0;
 947
 948         if (!pr)
 949                 return -EINVAL;
 950
 951         if ((state < 0) || (state > (pr->throttling.state_count - 1)))
 952                 return -EINVAL;
 953
 954         if (!pr->flags.throttling)
 955                 return -ENODEV;
 956
 957         if (!force && (state == pr->throttling.state))
 958                 return 0;
 959
 960         if (state < pr->throttling_platform_limit)
 961                 return -EPERM;
 962         /*
 963          * Calculate the duty_value and duty_mask.
 964          */
 965         if (state) {
 966                 duty_value = pr->throttling.state_count - state;
 967
 968                 duty_value <<= pr->throttling.duty_offset;
 969
 970                 /* Used to clear all duty_value bits */
 971                 duty_mask = pr->throttling.state_count - 1;
 972
 973                 duty_mask <<= acpi_gbl_FADT.duty_offset;
 974                 duty_mask = ~duty_mask;
 975         }
 976
 977         local_irq_disable();
 978
 979         /*
 980          * Disable throttling by writing a 0 to bit 4.  Note that we must
 981          * turn it off before you can change the duty_value.
 982          */
 983         value = inl(pr->throttling.address);
 984         if (value & 0x10) {
 985                 value &= 0xFFFFFFEF;
 986                 outl(value, pr->throttling.address);
 987         }
 988
 989         /*
 990          * Write the new duty_value and then enable throttling.  Note
 991          * that a state value of 0 leaves throttling disabled.
 992          */
 993         if (state) {
 994                 value &= duty_mask;
 995                 value |= duty_value;
 996                 outl(value, pr->throttling.address);
 997
 998                 value |= 0x00000010;
 999                 outl(value, pr->throttling.address);
1000         }
1001
1002         pr->throttling.state = state;
1003
1004         local_irq_enable();
1005
1006         acpi_handle_debug(pr->handle,
1007                           "Throttling state set to T%d (%d%%)\n", state,
1008                           (pr->throttling.states[state].performance ? pr->
1009                            throttling.states[state].performance / 10 : 0));
1010
1011         return 0;
1012 }
1013
1014 static int acpi_processor_set_throttling_ptc(struct acpi_processor *pr,
1015                                              int state, bool force)
1016 {
1017         int ret;
1018         u64 value;
1019
1020         if (!pr)
1021                 return -EINVAL;
1022
1023         if ((state < 0) || (state > (pr->throttling.state_count - 1)))
1024                 return -EINVAL;
1025
1026         if (!pr->flags.throttling)
1027                 return -ENODEV;
1028
1029         if (!force && (state == pr->throttling.state))
1030                 return 0;
1031
1032         if (state < pr->throttling_platform_limit)
1033                 return -EPERM;
1034
1035         value = 0;
1036         ret = acpi_get_throttling_value(pr, state, &value);
1037         if (ret >= 0) {
1038                 acpi_write_throttling_state(pr, value);
1039                 pr->throttling.state = state;
1040         }
1041
1042         return 0;
1043 }
1044
1045 static long acpi_processor_throttling_fn(void *data)
1046 {
1047         struct acpi_processor_throttling_arg *arg = data;
1048         struct acpi_processor *pr = arg->pr;
1049
1050         return pr->throttling.acpi_processor_set_throttling(pr,
1051                         arg->target_state, arg->force);
1052 }
1053
1054 static int __acpi_processor_set_throttling(struct acpi_processor *pr,
1055                                            int state, bool force, bool direct)
1056 {
1057         int ret = 0;
1058         unsigned int i;
1059         struct acpi_processor *match_pr;
1060         struct acpi_processor_throttling *p_throttling;
1061         struct acpi_processor_throttling_arg arg;
1062         struct throttling_tstate t_state;
1063
1064         if (!pr)
1065                 return -EINVAL;
1066
1067         if (!pr->flags.throttling)
1068                 return -ENODEV;
1069
1070         if ((state < 0) || (state > (pr->throttling.state_count - 1)))
1071                 return -EINVAL;
1072
1073         if (cpu_is_offline(pr->id)) {
1074                 /*
1075                  * the cpu pointed by pr->id is offline. Unnecessary to change
1076                  * the throttling state any more.
1077                  */
1078                 return -ENODEV;
1079         }
1080
1081         t_state.target_state = state;
1082         p_throttling = &(pr->throttling);
1083
1084         /*
1085          * The throttling notifier will be called for every
1086          * affected cpu in order to get one proper T-state.
1087          * The notifier event is THROTTLING_PRECHANGE.
1088          */
1089         for_each_cpu_and(i, cpu_online_mask, p_throttling->shared_cpu_map) {
1090                 t_state.cpu = i;
1091                 acpi_processor_throttling_notifier(THROTTLING_PRECHANGE,
1092                                                         &t_state);
1093         }
1094         /*
1095          * The function of acpi_processor_set_throttling will be called
1096          * to switch T-state. If the coordination type is SW_ALL or HW_ALL,
1097          * it is necessary to call it for every affected cpu. Otherwise
1098          * it can be called only for the cpu pointed by pr.
1099          */
1100         if (p_throttling->shared_type == DOMAIN_COORD_TYPE_SW_ANY) {
1101                 arg.pr = pr;
1102                 arg.target_state = state;
1103                 arg.force = force;
1104                 ret = call_on_cpu(pr->id, acpi_processor_throttling_fn, &arg,
1105                                   direct);
1106         } else {
1107                 /*
1108                  * When the T-state coordination is SW_ALL or HW_ALL,
1109                  * it is necessary to set T-state for every affected
1110                  * cpus.
1111                  */
1112                 for_each_cpu_and(i, cpu_online_mask,
1113                     p_throttling->shared_cpu_map) {
1114                         match_pr = per_cpu(processors, i);
1115                         /*
1116                          * If the pointer is invalid, we will report the
1117                          * error message and continue.
1118                          */
1119                         if (!match_pr) {
1120                                 acpi_handle_debug(pr->handle,
1121                                         "Invalid Pointer for CPU %d\n", i);
1122                                 continue;
1123                         }
1124                         /*
1125                          * If the throttling control is unsupported on CPU i,
1126                          * we will report the error message and continue.
1127                          */
1128                         if (!match_pr->flags.throttling) {
1129                                 acpi_handle_debug(pr->handle,
1130                                         "Throttling Control unsupported on CPU %d\n", i);
1131                                 continue;
1132                         }
1133
1134                         arg.pr = match_pr;
1135                         arg.target_state = state;
1136                         arg.force = force;
1137                         ret = call_on_cpu(pr->id, acpi_processor_throttling_fn,
1138                                           &arg, direct);
1139                 }
1140         }
1141         /*
1142          * After the set_throttling is called, the
1143          * throttling notifier is called for every
1144          * affected cpu to update the T-states.
1145          * The notifier event is THROTTLING_POSTCHANGE
1146          */
1147         for_each_cpu_and(i, cpu_online_mask, p_throttling->shared_cpu_map) {
1148                 t_state.cpu = i;
1149                 acpi_processor_throttling_notifier(THROTTLING_POSTCHANGE,
1150                                                         &t_state);
1151         }
1152
1153         return ret;
1154 }
1155
1156 int acpi_processor_set_throttling(struct acpi_processor *pr, int state,
1157                                   bool force)
1158 {
1159         return __acpi_processor_set_throttling(pr, state, force, false);
1160 }
1161
1162 int acpi_processor_get_throttling_info(struct acpi_processor *pr)
1163 {
1164         int result = 0;
1165         struct acpi_processor_throttling *pthrottling;
1166
1167         acpi_handle_debug(pr->handle,
1168                           "pblk_address[0x%08x] duty_offset[%d] duty_width[%d]\n",
1169                           pr->throttling.address,
1170                           pr->throttling.duty_offset,
1171                           pr->throttling.duty_width);
1172
1173         /*
1174          * Evaluate _PTC, _TSS and _TPC
1175          * They must all be present or none of them can be used.
1176          */
1177         if (acpi_processor_get_throttling_control(pr) ||
1178                 acpi_processor_get_throttling_states(pr) ||
1179                 acpi_processor_get_platform_limit(pr)) {
1180                 pr->throttling.acpi_processor_get_throttling =
1181                     &acpi_processor_get_throttling_fadt;
1182                 pr->throttling.acpi_processor_set_throttling =
1183                     &acpi_processor_set_throttling_fadt;
1184                 if (acpi_processor_get_fadt_info(pr))
1185                         return 0;
1186         } else {
1187                 pr->throttling.acpi_processor_get_throttling =
1188                     &acpi_processor_get_throttling_ptc;
1189                 pr->throttling.acpi_processor_set_throttling =
1190                     &acpi_processor_set_throttling_ptc;
1191         }
1192
1193         /*
1194          * If TSD package for one CPU can't be parsed successfully, it means
1195          * that this CPU will have no coordination with other CPUs.
1196          */
1197         if (acpi_processor_get_tsd(pr)) {
1198                 pthrottling = &pr->throttling;
1199                 pthrottling->tsd_valid_flag = 0;
1200                 cpumask_set_cpu(pr->id, pthrottling->shared_cpu_map);
1201                 pthrottling->shared_type = DOMAIN_COORD_TYPE_SW_ALL;
1202         }
1203
1204         /*
1205          * PIIX4 Errata: We don't support throttling on the original PIIX4.
1206          * This shouldn't be an issue as few (if any) mobile systems ever
1207          * used this part.
1208          */
1209         if (errata.piix4.throttle) {
1210                 acpi_handle_debug(pr->handle,
1211                                   "Throttling not supported on PIIX4 A- or B-step\n");
1212                 return 0;
1213         }
1214
1215         acpi_handle_debug(pr->handle, "Found %d throttling states\n",
1216                           pr->throttling.state_count);
1217
1218         pr->flags.throttling = 1;
1219
1220         /*
1221          * Disable throttling (if enabled).  We'll let subsequent policy (e.g.
1222          * thermal) decide to lower performance if it so chooses, but for now
1223          * we'll crank up the speed.
1224          */
1225
1226         result = acpi_processor_get_throttling(pr);
1227         if (result)
1228                 goto end;
1229
1230         if (pr->throttling.state) {
1231                 acpi_handle_debug(pr->handle,
1232                                   "Disabling throttling (was T%d)\n",
1233                                   pr->throttling.state);
1234                 result = acpi_processor_set_throttling(pr, 0, false);
1235                 if (result)
1236                         goto end;
1237         }
1238
1239 end:
1240         if (result)
1241                 pr->flags.throttling = 0;
1242
1243         return result;
1244 }
1245