drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * uncore-frquency-tpmi: Uncore frequency scaling using TPMI
   4  *
   5  * Copyright (c) 2023, Intel Corporation.
   6  * All Rights Reserved.
   7  *
   8  * The hardware interface to read/write is basically substitution of
   9  * MSR 0x620 and 0x621.
  10  * There are specific MMIO offset and bits to get/set minimum and
  11  * maximum uncore ratio, similar to MSRs.
  12  * The scope of the uncore MSRs was package scope. But TPMI allows
  13  * new gen CPUs to have multiple uncore controls at uncore-cluster
  14  * level. Each package can have multiple power domains which further
  15  * can have multiple clusters.
  16  * Here number of power domains = number of resources in this aux
  17  * device. There are offsets and bits to discover number of clusters
  18  * and offset for each cluster level controls.
  19  *
  20  */
  21
  22 #include <linux/auxiliary_bus.h>
  23 #include <linux/bitfield.h>
  24 #include <linux/bits.h>
  25 #include <linux/io.h>
  26 #include <linux/module.h>
  27 #include <linux/intel_tpmi.h>
  28
  29 #include "uncore-frequency-common.h"
  30
  31 #define UNCORE_MAJOR_VERSION            0
  32 #define UNCORE_MINOR_VERSION            1
  33 #define UNCORE_HEADER_INDEX             0
  34 #define UNCORE_FABRIC_CLUSTER_OFFSET    8
  35
  36 /* status + control + adv_ctl1 + adv_ctl2 */
  37 #define UNCORE_FABRIC_CLUSTER_SIZE      (4 * 8)
  38
  39 #define UNCORE_STATUS_INDEX             0
  40 #define UNCORE_CONTROL_INDEX            8
  41
  42 #define UNCORE_FREQ_KHZ_MULTIPLIER      100000
  43
  44 struct tpmi_uncore_struct;
  45
  46 /* Information for each cluster */
  47 struct tpmi_uncore_cluster_info {
  48         bool root_domain;
  49         u8 __iomem *cluster_base;
  50         struct uncore_data uncore_data;
  51         struct tpmi_uncore_struct *uncore_root;
  52 };
  53
  54 /* Information for each power domain */
  55 struct tpmi_uncore_power_domain_info {
  56         u8 __iomem *uncore_base;
  57         int ufs_header_ver;
  58         int cluster_count;
  59         struct tpmi_uncore_cluster_info *cluster_infos;
  60 };
  61
  62 /* Information for all power domains in a package */
  63 struct tpmi_uncore_struct {
  64         int power_domain_count;
  65         int max_ratio;
  66         int min_ratio;
  67         struct tpmi_uncore_power_domain_info *pd_info;
  68         struct tpmi_uncore_cluster_info root_cluster;
  69         bool write_blocked;
  70 };
  71
  72 #define UNCORE_GENMASK_MIN_RATIO        GENMASK_ULL(21, 15)
  73 #define UNCORE_GENMASK_MAX_RATIO        GENMASK_ULL(14, 8)
  74 #define UNCORE_GENMASK_CURRENT_RATIO    GENMASK_ULL(6, 0)
  75
  76 /* Helper function to read MMIO offset for max/min control frequency */
  77 static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info,
  78                              unsigned int *min, unsigned int *max)
  79 {
  80         u64 control;
  81
  82         control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
  83         *max = FIELD_GET(UNCORE_GENMASK_MAX_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
  84         *min = FIELD_GET(UNCORE_GENMASK_MIN_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
  85 }
  86
  87 #define UNCORE_MAX_RATIO        FIELD_MAX(UNCORE_GENMASK_MAX_RATIO)
  88
  89 /* Callback for sysfs read for max/min frequencies. Called under mutex locks */
  90 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min,
  91                                     unsigned int *max)
  92 {
  93         struct tpmi_uncore_cluster_info *cluster_info;
  94
  95         cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
  96
  97         if (cluster_info->root_domain) {
  98                 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
  99                 int i, _min = 0, _max = 0;
 100
 101                 *min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER;
 102                 *max = 0;
 103
 104                 /*
 105                  * Get the max/min by looking at each cluster. Get the lowest
 106                  * min and highest max.
 107                  */
 108                 for (i = 0; i < uncore_root->power_domain_count; ++i) {
 109                         int j;
 110
 111                         for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) {
 112                                 read_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
 113                                                   &_min, &_max);
 114                                 if (*min > _min)
 115                                         *min = _min;
 116                                 if (*max < _max)
 117                                         *max = _max;
 118                         }
 119                 }
 120                 return 0;
 121         }
 122
 123         read_control_freq(cluster_info, min, max);
 124
 125         return 0;
 126 }
 127
 128 /* Helper function to write MMIO offset for max/min control frequency */
 129 static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input,
 130                               unsigned int min_max)
 131 {
 132         u64 control;
 133
 134         control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
 135
 136         if (min_max) {
 137                 control &= ~UNCORE_GENMASK_MAX_RATIO;
 138                 control |= FIELD_PREP(UNCORE_GENMASK_MAX_RATIO, input);
 139         } else {
 140                 control &= ~UNCORE_GENMASK_MIN_RATIO;
 141                 control |= FIELD_PREP(UNCORE_GENMASK_MIN_RATIO, input);
 142         }
 143
 144         writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX));
 145 }
 146
 147 /* Callback for sysfs write for max/min frequencies. Called under mutex locks */
 148 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
 149                                      unsigned int min_max)
 150 {
 151         struct tpmi_uncore_cluster_info *cluster_info;
 152         struct tpmi_uncore_struct *uncore_root;
 153
 154         input /= UNCORE_FREQ_KHZ_MULTIPLIER;
 155         if (!input || input > UNCORE_MAX_RATIO)
 156                 return -EINVAL;
 157
 158         cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
 159         uncore_root = cluster_info->uncore_root;
 160
 161         if (uncore_root->write_blocked)
 162                 return -EPERM;
 163
 164         /* Update each cluster in a package */
 165         if (cluster_info->root_domain) {
 166                 struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
 167                 int i;
 168
 169                 for (i = 0; i < uncore_root->power_domain_count; ++i) {
 170                         int j;
 171
 172                         for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j)
 173                                 write_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
 174                                                   input, min_max);
 175                 }
 176
 177                 if (min_max)
 178                         uncore_root->max_ratio = input;
 179                 else
 180                         uncore_root->min_ratio = input;
 181
 182                 return 0;
 183         }
 184
 185         if (min_max && uncore_root->max_ratio && uncore_root->max_ratio < input)
 186                 return -EINVAL;
 187
 188         if (!min_max && uncore_root->min_ratio && uncore_root->min_ratio > input)
 189                 return -EINVAL;
 190
 191         write_control_freq(cluster_info, input, min_max);
 192
 193         return 0;
 194 }
 195
 196 /* Callback for sysfs read for the current uncore frequency. Called under mutex locks */
 197 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
 198 {
 199         struct tpmi_uncore_cluster_info *cluster_info;
 200         u64 status;
 201
 202         cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
 203         if (cluster_info->root_domain)
 204                 return -ENODATA;
 205
 206         status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX);
 207         *freq = FIELD_GET(UNCORE_GENMASK_CURRENT_RATIO, status) * UNCORE_FREQ_KHZ_MULTIPLIER;
 208
 209         return 0;
 210 }
 211
 212 static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore)
 213 {
 214         int i;
 215
 216         for (i = 0; i < tpmi_uncore->power_domain_count; ++i) {
 217                 struct tpmi_uncore_power_domain_info *pd_info;
 218                 int j;
 219
 220                 pd_info = &tpmi_uncore->pd_info[i];
 221                 if (!pd_info->uncore_base)
 222                         continue;
 223
 224                 for (j = 0; j < pd_info->cluster_count; ++j) {
 225                         struct tpmi_uncore_cluster_info *cluster_info;
 226
 227                         cluster_info = &pd_info->cluster_infos[j];
 228                         uncore_freq_remove_die_entry(&cluster_info->uncore_data);
 229                 }
 230         }
 231 }
 232
 233 #define UNCORE_VERSION_MASK                     GENMASK_ULL(7, 0)
 234 #define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK     GENMASK_ULL(15, 8)
 235 #define UNCORE_CLUSTER_OFF_MASK                 GENMASK_ULL(7, 0)
 236 #define UNCORE_MAX_CLUSTER_PER_DOMAIN           8
 237
 238 static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id)
 239 {
 240         bool read_blocked = 0, write_blocked = 0;
 241         struct intel_tpmi_plat_info *plat_info;
 242         struct tpmi_uncore_struct *tpmi_uncore;
 243         int ret, i, pkg = 0;
 244         int num_resources;
 245
 246         ret = tpmi_get_feature_status(auxdev, TPMI_ID_UNCORE, &read_blocked, &write_blocked);
 247         if (ret)
 248                 dev_info(&auxdev->dev, "Can't read feature status: ignoring blocked status\n");
 249
 250         if (read_blocked) {
 251                 dev_info(&auxdev->dev, "Firmware has blocked reads, exiting\n");
 252                 return -ENODEV;
 253         }
 254
 255         /* Get number of power domains, which is equal to number of resources */
 256         num_resources = tpmi_get_resource_count(auxdev);
 257         if (!num_resources)
 258                 return -EINVAL;
 259
 260         /* Register callbacks to uncore core */
 261         ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq,
 262                                       uncore_read_freq);
 263         if (ret)
 264                 return ret;
 265
 266         /* Allocate uncore instance per package */
 267         tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL);
 268         if (!tpmi_uncore) {
 269                 ret = -ENOMEM;
 270                 goto err_rem_common;
 271         }
 272
 273         /* Allocate memory for all power domains in a package */
 274         tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources,
 275                                             sizeof(*tpmi_uncore->pd_info),
 276                                             GFP_KERNEL);
 277         if (!tpmi_uncore->pd_info) {
 278                 ret = -ENOMEM;
 279                 goto err_rem_common;
 280         }
 281
 282         tpmi_uncore->power_domain_count = num_resources;
 283         tpmi_uncore->write_blocked = write_blocked;
 284
 285         /* Get the package ID from the TPMI core */
 286         plat_info = tpmi_get_platform_data(auxdev);
 287         if (plat_info)
 288                 pkg = plat_info->package_id;
 289         else
 290                 dev_info(&auxdev->dev, "Platform information is NULL\n");
 291
 292         for (i = 0; i < num_resources; ++i) {
 293                 struct tpmi_uncore_power_domain_info *pd_info;
 294                 struct resource *res;
 295                 u64 cluster_offset;
 296                 u8 cluster_mask;
 297                 int mask, j;
 298                 u64 header;
 299
 300                 res = tpmi_get_resource_at_index(auxdev, i);
 301                 if (!res)
 302                         continue;
 303
 304                 pd_info = &tpmi_uncore->pd_info[i];
 305
 306                 pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res);
 307                 if (IS_ERR(pd_info->uncore_base)) {
 308                         ret = PTR_ERR(pd_info->uncore_base);
 309                         /*
 310                          * Set to NULL so that clean up can still remove other
 311                          * entries already created if any by
 312                          * remove_cluster_entries()
 313                          */
 314                         pd_info->uncore_base = NULL;
 315                         goto remove_clusters;
 316                 }
 317
 318                 /* Check for version and skip this resource if there is mismatch */
 319                 header = readq(pd_info->uncore_base);
 320                 pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK;
 321
 322                 if (pd_info->ufs_header_ver == TPMI_VERSION_INVALID)
 323                         continue;
 324
 325                 if (TPMI_MAJOR_VERSION(pd_info->ufs_header_ver) != UNCORE_MAJOR_VERSION) {
 326                         dev_err(&auxdev->dev, "Uncore: Unsupported major version:%lx\n",
 327                                 TPMI_MAJOR_VERSION(pd_info->ufs_header_ver));
 328                         ret = -ENODEV;
 329                         goto remove_clusters;
 330                 }
 331
 332                 if (TPMI_MINOR_VERSION(pd_info->ufs_header_ver) != UNCORE_MINOR_VERSION)
 333                         dev_info(&auxdev->dev, "Uncore: Ignore: Unsupported minor version:%lx\n",
 334                                  TPMI_MINOR_VERSION(pd_info->ufs_header_ver));
 335
 336                 /* Get Cluster ID Mask */
 337                 cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header);
 338                 if (!cluster_mask) {
 339                         dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask);
 340                         continue;
 341                 }
 342
 343                 /* Find out number of clusters in this resource */
 344                 pd_info->cluster_count = hweight8(cluster_mask);
 345
 346                 pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count,
 347                                                       sizeof(struct tpmi_uncore_cluster_info),
 348                                                       GFP_KERNEL);
 349                 if (!pd_info->cluster_infos) {
 350                         ret = -ENOMEM;
 351                         goto remove_clusters;
 352                 }
 353                 /*
 354                  * Each byte in the register point to status and control
 355                  * registers belonging to cluster id 0-8.
 356                  */
 357                 cluster_offset = readq(pd_info->uncore_base +
 358                                         UNCORE_FABRIC_CLUSTER_OFFSET);
 359
 360                 for (j = 0; j < pd_info->cluster_count; ++j) {
 361                         struct tpmi_uncore_cluster_info *cluster_info;
 362
 363                         /* Get the offset for this cluster */
 364                         mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK);
 365                         /* Offset in QWORD, so change to bytes */
 366                         mask <<= 3;
 367
 368                         cluster_info = &pd_info->cluster_infos[j];
 369
 370                         cluster_info->cluster_base = pd_info->uncore_base + mask;
 371
 372                         cluster_info->uncore_data.package_id = pkg;
 373                         /* There are no dies like Cascade Lake */
 374                         cluster_info->uncore_data.die_id = 0;
 375                         cluster_info->uncore_data.domain_id = i;
 376                         cluster_info->uncore_data.cluster_id = j;
 377
 378                         cluster_info->uncore_root = tpmi_uncore;
 379
 380                         ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0);
 381                         if (ret) {
 382                                 cluster_info->cluster_base = NULL;
 383                                 goto remove_clusters;
 384                         }
 385                         /* Point to next cluster offset */
 386                         cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN;
 387                 }
 388         }
 389
 390         auxiliary_set_drvdata(auxdev, tpmi_uncore);
 391
 392         tpmi_uncore->root_cluster.root_domain = true;
 393         tpmi_uncore->root_cluster.uncore_root = tpmi_uncore;
 394
 395         tpmi_uncore->root_cluster.uncore_data.package_id = pkg;
 396         tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID;
 397         ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0);
 398         if (ret)
 399                 goto remove_clusters;
 400
 401         return 0;
 402
 403 remove_clusters:
 404         remove_cluster_entries(tpmi_uncore);
 405 err_rem_common:
 406         uncore_freq_common_exit();
 407
 408         return ret;
 409 }
 410
 411 static void uncore_remove(struct auxiliary_device *auxdev)
 412 {
 413         struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev);
 414
 415         uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data);
 416         remove_cluster_entries(tpmi_uncore);
 417
 418         uncore_freq_common_exit();
 419 }
 420
 421 static const struct auxiliary_device_id intel_uncore_id_table[] = {
 422         { .name = "intel_vsec.tpmi-uncore" },
 423         {}
 424 };
 425 MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table);
 426
 427 static struct auxiliary_driver intel_uncore_aux_driver = {
 428         .id_table       = intel_uncore_id_table,
 429         .remove         = uncore_remove,
 430         .probe          = uncore_probe,
 431 };
 432
 433 module_auxiliary_driver(intel_uncore_aux_driver);
 434
 435 MODULE_IMPORT_NS(INTEL_TPMI);
 436 MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY);
 437 MODULE_DESCRIPTION("Intel TPMI UFS Driver");
 438 MODULE_LICENSE("GPL");