drivers/memory/tegra/tegra20.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012 NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 #include <linux/bitfield.h>
   7 #include <linux/delay.h>
   8 #include <linux/device.h>
   9 #include <linux/mutex.h>
  10 #include <linux/of.h>
  11 #include <linux/slab.h>
  12 #include <linux/string.h>
  13
  14 #include <dt-bindings/memory/tegra20-mc.h>
  15
  16 #include "mc.h"
  17
  18 #define MC_STAT_CONTROL                         0x90
  19 #define MC_STAT_EMC_CLOCK_LIMIT                 0xa0
  20 #define MC_STAT_EMC_CLOCKS                      0xa4
  21 #define MC_STAT_EMC_CONTROL_0                   0xa8
  22 #define MC_STAT_EMC_CONTROL_1                   0xac
  23 #define MC_STAT_EMC_COUNT_0                     0xb8
  24 #define MC_STAT_EMC_COUNT_1                     0xbc
  25
  26 #define MC_STAT_CONTROL_CLIENT_ID               GENMASK(13,  8)
  27 #define MC_STAT_CONTROL_EVENT                   GENMASK(23, 16)
  28 #define MC_STAT_CONTROL_PRI_EVENT               GENMASK(25, 24)
  29 #define MC_STAT_CONTROL_FILTER_CLIENT_ENABLE    GENMASK(26, 26)
  30 #define MC_STAT_CONTROL_FILTER_PRI              GENMASK(29, 28)
  31
  32 #define MC_STAT_CONTROL_PRI_EVENT_HP            0
  33 #define MC_STAT_CONTROL_PRI_EVENT_TM            1
  34 #define MC_STAT_CONTROL_PRI_EVENT_BW            2
  35
  36 #define MC_STAT_CONTROL_FILTER_PRI_DISABLE      0
  37 #define MC_STAT_CONTROL_FILTER_PRI_NO           1
  38 #define MC_STAT_CONTROL_FILTER_PRI_YES          2
  39
  40 #define MC_STAT_CONTROL_EVENT_QUALIFIED         0
  41 #define MC_STAT_CONTROL_EVENT_ANY_READ          1
  42 #define MC_STAT_CONTROL_EVENT_ANY_WRITE         2
  43 #define MC_STAT_CONTROL_EVENT_RD_WR_CHANGE      3
  44 #define MC_STAT_CONTROL_EVENT_SUCCESSIVE        4
  45 #define MC_STAT_CONTROL_EVENT_ARB_BANK_AA       5
  46 #define MC_STAT_CONTROL_EVENT_ARB_BANK_BB       6
  47 #define MC_STAT_CONTROL_EVENT_PAGE_MISS         7
  48 #define MC_STAT_CONTROL_EVENT_AUTO_PRECHARGE    8
  49
  50 #define EMC_GATHER_RST                          (0 << 8)
  51 #define EMC_GATHER_CLEAR                        (1 << 8)
  52 #define EMC_GATHER_DISABLE                      (2 << 8)
  53 #define EMC_GATHER_ENABLE                       (3 << 8)
  54
  55 #define MC_STAT_SAMPLE_TIME_USEC                16000
  56
  57 /* we store collected statistics as a fixed point values */
  58 #define MC_FX_FRAC_SCALE                        100
  59
  60 static DEFINE_MUTEX(tegra20_mc_stat_lock);
  61
  62 struct tegra20_mc_stat_gather {
  63         unsigned int pri_filter;
  64         unsigned int pri_event;
  65         unsigned int result;
  66         unsigned int client;
  67         unsigned int event;
  68         bool client_enb;
  69 };
  70
  71 struct tegra20_mc_stat {
  72         struct tegra20_mc_stat_gather gather0;
  73         struct tegra20_mc_stat_gather gather1;
  74         unsigned int sample_time_usec;
  75         const struct tegra_mc *mc;
  76 };
  77
  78 struct tegra20_mc_client_stat {
  79         unsigned int events;
  80         unsigned int arb_high_prio;
  81         unsigned int arb_timeout;
  82         unsigned int arb_bandwidth;
  83         unsigned int rd_wr_change;
  84         unsigned int successive;
  85         unsigned int page_miss;
  86         unsigned int auto_precharge;
  87         unsigned int arb_bank_aa;
  88         unsigned int arb_bank_bb;
  89 };
  90
  91 static const struct tegra_mc_client tegra20_mc_clients[] = {
  92         {
  93                 .id = 0x00,
  94                 .name = "display0a",
  95         }, {
  96                 .id = 0x01,
  97                 .name = "display0ab",
  98         }, {
  99                 .id = 0x02,
 100                 .name = "display0b",
 101         }, {
 102                 .id = 0x03,
 103                 .name = "display0bb",
 104         }, {
 105                 .id = 0x04,
 106                 .name = "display0c",
 107         }, {
 108                 .id = 0x05,
 109                 .name = "display0cb",
 110         }, {
 111                 .id = 0x06,
 112                 .name = "display1b",
 113         }, {
 114                 .id = 0x07,
 115                 .name = "display1bb",
 116         }, {
 117                 .id = 0x08,
 118                 .name = "eppup",
 119         }, {
 120                 .id = 0x09,
 121                 .name = "g2pr",
 122         }, {
 123                 .id = 0x0a,
 124                 .name = "g2sr",
 125         }, {
 126                 .id = 0x0b,
 127                 .name = "mpeunifbr",
 128         }, {
 129                 .id = 0x0c,
 130                 .name = "viruv",
 131         }, {
 132                 .id = 0x0d,
 133                 .name = "avpcarm7r",
 134         }, {
 135                 .id = 0x0e,
 136                 .name = "displayhc",
 137         }, {
 138                 .id = 0x0f,
 139                 .name = "displayhcb",
 140         }, {
 141                 .id = 0x10,
 142                 .name = "fdcdrd",
 143         }, {
 144                 .id = 0x11,
 145                 .name = "g2dr",
 146         }, {
 147                 .id = 0x12,
 148                 .name = "host1xdmar",
 149         }, {
 150                 .id = 0x13,
 151                 .name = "host1xr",
 152         }, {
 153                 .id = 0x14,
 154                 .name = "idxsrd",
 155         }, {
 156                 .id = 0x15,
 157                 .name = "mpcorer",
 158         }, {
 159                 .id = 0x16,
 160                 .name = "mpe_ipred",
 161         }, {
 162                 .id = 0x17,
 163                 .name = "mpeamemrd",
 164         }, {
 165                 .id = 0x18,
 166                 .name = "mpecsrd",
 167         }, {
 168                 .id = 0x19,
 169                 .name = "ppcsahbdmar",
 170         }, {
 171                 .id = 0x1a,
 172                 .name = "ppcsahbslvr",
 173         }, {
 174                 .id = 0x1b,
 175                 .name = "texsrd",
 176         }, {
 177                 .id = 0x1c,
 178                 .name = "vdebsevr",
 179         }, {
 180                 .id = 0x1d,
 181                 .name = "vdember",
 182         }, {
 183                 .id = 0x1e,
 184                 .name = "vdemcer",
 185         }, {
 186                 .id = 0x1f,
 187                 .name = "vdetper",
 188         }, {
 189                 .id = 0x20,
 190                 .name = "eppu",
 191         }, {
 192                 .id = 0x21,
 193                 .name = "eppv",
 194         }, {
 195                 .id = 0x22,
 196                 .name = "eppy",
 197         }, {
 198                 .id = 0x23,
 199                 .name = "mpeunifbw",
 200         }, {
 201                 .id = 0x24,
 202                 .name = "viwsb",
 203         }, {
 204                 .id = 0x25,
 205                 .name = "viwu",
 206         }, {
 207                 .id = 0x26,
 208                 .name = "viwv",
 209         }, {
 210                 .id = 0x27,
 211                 .name = "viwy",
 212         }, {
 213                 .id = 0x28,
 214                 .name = "g2dw",
 215         }, {
 216                 .id = 0x29,
 217                 .name = "avpcarm7w",
 218         }, {
 219                 .id = 0x2a,
 220                 .name = "fdcdwr",
 221         }, {
 222                 .id = 0x2b,
 223                 .name = "host1xw",
 224         }, {
 225                 .id = 0x2c,
 226                 .name = "ispw",
 227         }, {
 228                 .id = 0x2d,
 229                 .name = "mpcorew",
 230         }, {
 231                 .id = 0x2e,
 232                 .name = "mpecswr",
 233         }, {
 234                 .id = 0x2f,
 235                 .name = "ppcsahbdmaw",
 236         }, {
 237                 .id = 0x30,
 238                 .name = "ppcsahbslvw",
 239         }, {
 240                 .id = 0x31,
 241                 .name = "vdebsevw",
 242         }, {
 243                 .id = 0x32,
 244                 .name = "vdembew",
 245         }, {
 246                 .id = 0x33,
 247                 .name = "vdetpmw",
 248         },
 249 };
 250
 251 #define TEGRA20_MC_RESET(_name, _control, _status, _reset, _bit)        \
 252         {                                                               \
 253                 .name = #_name,                                         \
 254                 .id = TEGRA20_MC_RESET_##_name,                         \
 255                 .control = _control,                                    \
 256                 .status = _status,                                      \
 257                 .reset = _reset,                                        \
 258                 .bit = _bit,                                            \
 259         }
 260
 261 static const struct tegra_mc_reset tegra20_mc_resets[] = {
 262         TEGRA20_MC_RESET(AVPC,   0x100, 0x140, 0x104,  0),
 263         TEGRA20_MC_RESET(DC,     0x100, 0x144, 0x104,  1),
 264         TEGRA20_MC_RESET(DCB,    0x100, 0x148, 0x104,  2),
 265         TEGRA20_MC_RESET(EPP,    0x100, 0x14c, 0x104,  3),
 266         TEGRA20_MC_RESET(2D,     0x100, 0x150, 0x104,  4),
 267         TEGRA20_MC_RESET(HC,     0x100, 0x154, 0x104,  5),
 268         TEGRA20_MC_RESET(ISP,    0x100, 0x158, 0x104,  6),
 269         TEGRA20_MC_RESET(MPCORE, 0x100, 0x15c, 0x104,  7),
 270         TEGRA20_MC_RESET(MPEA,   0x100, 0x160, 0x104,  8),
 271         TEGRA20_MC_RESET(MPEB,   0x100, 0x164, 0x104,  9),
 272         TEGRA20_MC_RESET(MPEC,   0x100, 0x168, 0x104, 10),
 273         TEGRA20_MC_RESET(3D,     0x100, 0x16c, 0x104, 11),
 274         TEGRA20_MC_RESET(PPCS,   0x100, 0x170, 0x104, 12),
 275         TEGRA20_MC_RESET(VDE,    0x100, 0x174, 0x104, 13),
 276         TEGRA20_MC_RESET(VI,     0x100, 0x178, 0x104, 14),
 277 };
 278
 279 static int tegra20_mc_hotreset_assert(struct tegra_mc *mc,
 280                                       const struct tegra_mc_reset *rst)
 281 {
 282         unsigned long flags;
 283         u32 value;
 284
 285         spin_lock_irqsave(&mc->lock, flags);
 286
 287         value = mc_readl(mc, rst->reset);
 288         mc_writel(mc, value & ~BIT(rst->bit), rst->reset);
 289
 290         spin_unlock_irqrestore(&mc->lock, flags);
 291
 292         return 0;
 293 }
 294
 295 static int tegra20_mc_hotreset_deassert(struct tegra_mc *mc,
 296                                         const struct tegra_mc_reset *rst)
 297 {
 298         unsigned long flags;
 299         u32 value;
 300
 301         spin_lock_irqsave(&mc->lock, flags);
 302
 303         value = mc_readl(mc, rst->reset);
 304         mc_writel(mc, value | BIT(rst->bit), rst->reset);
 305
 306         spin_unlock_irqrestore(&mc->lock, flags);
 307
 308         return 0;
 309 }
 310
 311 static int tegra20_mc_block_dma(struct tegra_mc *mc,
 312                                 const struct tegra_mc_reset *rst)
 313 {
 314         unsigned long flags;
 315         u32 value;
 316
 317         spin_lock_irqsave(&mc->lock, flags);
 318
 319         value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
 320         mc_writel(mc, value, rst->control);
 321
 322         spin_unlock_irqrestore(&mc->lock, flags);
 323
 324         return 0;
 325 }
 326
 327 static bool tegra20_mc_dma_idling(struct tegra_mc *mc,
 328                                   const struct tegra_mc_reset *rst)
 329 {
 330         return mc_readl(mc, rst->status) == 0;
 331 }
 332
 333 static int tegra20_mc_reset_status(struct tegra_mc *mc,
 334                                    const struct tegra_mc_reset *rst)
 335 {
 336         return (mc_readl(mc, rst->reset) & BIT(rst->bit)) == 0;
 337 }
 338
 339 static int tegra20_mc_unblock_dma(struct tegra_mc *mc,
 340                                   const struct tegra_mc_reset *rst)
 341 {
 342         unsigned long flags;
 343         u32 value;
 344
 345         spin_lock_irqsave(&mc->lock, flags);
 346
 347         value = mc_readl(mc, rst->control) | BIT(rst->bit);
 348         mc_writel(mc, value, rst->control);
 349
 350         spin_unlock_irqrestore(&mc->lock, flags);
 351
 352         return 0;
 353 }
 354
 355 static const struct tegra_mc_reset_ops tegra20_mc_reset_ops = {
 356         .hotreset_assert = tegra20_mc_hotreset_assert,
 357         .hotreset_deassert = tegra20_mc_hotreset_deassert,
 358         .block_dma = tegra20_mc_block_dma,
 359         .dma_idling = tegra20_mc_dma_idling,
 360         .unblock_dma = tegra20_mc_unblock_dma,
 361         .reset_status = tegra20_mc_reset_status,
 362 };
 363
 364 static int tegra20_mc_icc_set(struct icc_node *src, struct icc_node *dst)
 365 {
 366         /*
 367          * It should be possible to tune arbitration knobs here, but the
 368          * default values are known to work well on all devices. Hence
 369          * nothing to do here so far.
 370          */
 371         return 0;
 372 }
 373
 374 static int tegra20_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw,
 375                                    u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
 376 {
 377         /*
 378          * ISO clients need to reserve extra bandwidth up-front because
 379          * there could be high bandwidth pressure during initial filling
 380          * of the client's FIFO buffers.  Secondly, we need to take into
 381          * account impurities of the memory subsystem.
 382          */
 383         if (tag & TEGRA_MC_ICC_TAG_ISO)
 384                 peak_bw = tegra_mc_scale_percents(peak_bw, 300);
 385
 386         *agg_avg += avg_bw;
 387         *agg_peak = max(*agg_peak, peak_bw);
 388
 389         return 0;
 390 }
 391
 392 static struct icc_node_data *
 393 tegra20_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data)
 394 {
 395         struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
 396         unsigned int i, idx = spec->args[0];
 397         struct icc_node_data *ndata;
 398         struct icc_node *node;
 399
 400         list_for_each_entry(node, &mc->provider.nodes, node_list) {
 401                 if (node->id != idx)
 402                         continue;
 403
 404                 ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);
 405                 if (!ndata)
 406                         return ERR_PTR(-ENOMEM);
 407
 408                 ndata->node = node;
 409
 410                 /* these clients are isochronous by default */
 411                 if (strstarts(node->name, "display") ||
 412                     strstarts(node->name, "vi"))
 413                         ndata->tag = TEGRA_MC_ICC_TAG_ISO;
 414                 else
 415                         ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT;
 416
 417                 return ndata;
 418         }
 419
 420         for (i = 0; i < mc->soc->num_clients; i++) {
 421                 if (mc->soc->clients[i].id == idx)
 422                         return ERR_PTR(-EPROBE_DEFER);
 423         }
 424
 425         dev_err(mc->dev, "invalid ICC client ID %u\n", idx);
 426
 427         return ERR_PTR(-EINVAL);
 428 }
 429
 430 static const struct tegra_mc_icc_ops tegra20_mc_icc_ops = {
 431         .xlate_extended = tegra20_mc_of_icc_xlate_extended,
 432         .aggregate = tegra20_mc_icc_aggreate,
 433         .set = tegra20_mc_icc_set,
 434 };
 435
 436 static u32 tegra20_mc_stat_gather_control(const struct tegra20_mc_stat_gather *g)
 437 {
 438         u32 control;
 439
 440         control  = FIELD_PREP(MC_STAT_CONTROL_EVENT, g->event);
 441         control |= FIELD_PREP(MC_STAT_CONTROL_CLIENT_ID, g->client);
 442         control |= FIELD_PREP(MC_STAT_CONTROL_PRI_EVENT, g->pri_event);
 443         control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_PRI, g->pri_filter);
 444         control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_CLIENT_ENABLE, g->client_enb);
 445
 446         return control;
 447 }
 448
 449 static void tegra20_mc_stat_gather(struct tegra20_mc_stat *stat)
 450 {
 451         u32 clocks, count0, count1, control_0, control_1;
 452         const struct tegra_mc *mc = stat->mc;
 453
 454         control_0 = tegra20_mc_stat_gather_control(&stat->gather0);
 455         control_1 = tegra20_mc_stat_gather_control(&stat->gather1);
 456
 457         /*
 458          * Reset statistic gathers state, select statistics collection mode
 459          * and set clocks counter saturation limit to maximum.
 460          */
 461         mc_writel(mc, 0x00000000, MC_STAT_CONTROL);
 462         mc_writel(mc,  control_0, MC_STAT_EMC_CONTROL_0);
 463         mc_writel(mc,  control_1, MC_STAT_EMC_CONTROL_1);
 464         mc_writel(mc, 0xffffffff, MC_STAT_EMC_CLOCK_LIMIT);
 465
 466         mc_writel(mc, EMC_GATHER_ENABLE, MC_STAT_CONTROL);
 467         fsleep(stat->sample_time_usec);
 468         mc_writel(mc, EMC_GATHER_DISABLE, MC_STAT_CONTROL);
 469
 470         count0 = mc_readl(mc, MC_STAT_EMC_COUNT_0);
 471         count1 = mc_readl(mc, MC_STAT_EMC_COUNT_1);
 472         clocks = mc_readl(mc, MC_STAT_EMC_CLOCKS);
 473         clocks = max(clocks / 100 / MC_FX_FRAC_SCALE, 1u);
 474
 475         stat->gather0.result = DIV_ROUND_UP(count0, clocks);
 476         stat->gather1.result = DIV_ROUND_UP(count1, clocks);
 477 }
 478
 479 static void tegra20_mc_stat_events(const struct tegra_mc *mc,
 480                                    const struct tegra_mc_client *client0,
 481                                    const struct tegra_mc_client *client1,
 482                                    unsigned int pri_filter,
 483                                    unsigned int pri_event,
 484                                    unsigned int event,
 485                                    unsigned int *result0,
 486                                    unsigned int *result1)
 487 {
 488         struct tegra20_mc_stat stat = {};
 489
 490         stat.gather0.client = client0 ? client0->id : 0;
 491         stat.gather0.pri_filter = pri_filter;
 492         stat.gather0.client_enb = !!client0;
 493         stat.gather0.pri_event = pri_event;
 494         stat.gather0.event = event;
 495
 496         stat.gather1.client = client1 ? client1->id : 0;
 497         stat.gather1.pri_filter = pri_filter;
 498         stat.gather1.client_enb = !!client1;
 499         stat.gather1.pri_event = pri_event;
 500         stat.gather1.event = event;
 501
 502         stat.sample_time_usec = MC_STAT_SAMPLE_TIME_USEC;
 503         stat.mc = mc;
 504
 505         tegra20_mc_stat_gather(&stat);
 506
 507         *result0 = stat.gather0.result;
 508         *result1 = stat.gather1.result;
 509 }
 510
 511 static void tegra20_mc_collect_stats(const struct tegra_mc *mc,
 512                                      struct tegra20_mc_client_stat *stats)
 513 {
 514         const struct tegra_mc_client *client0, *client1;
 515         unsigned int i;
 516
 517         /* collect memory controller utilization percent for each client */
 518         for (i = 0; i < mc->soc->num_clients; i += 2) {
 519                 client0 = &mc->soc->clients[i];
 520                 client1 = &mc->soc->clients[i + 1];
 521
 522                 if (i + 1 == mc->soc->num_clients)
 523                         client1 = NULL;
 524
 525                 tegra20_mc_stat_events(mc, client0, client1,
 526                                        MC_STAT_CONTROL_FILTER_PRI_DISABLE,
 527                                        MC_STAT_CONTROL_PRI_EVENT_HP,
 528                                        MC_STAT_CONTROL_EVENT_QUALIFIED,
 529                                        &stats[i + 0].events,
 530                                        &stats[i + 1].events);
 531         }
 532
 533         /* collect more info from active clients */
 534         for (i = 0; i < mc->soc->num_clients; i++) {
 535                 unsigned int clienta, clientb = mc->soc->num_clients;
 536
 537                 for (client0 = NULL; i < mc->soc->num_clients; i++) {
 538                         if (stats[i].events) {
 539                                 client0 = &mc->soc->clients[i];
 540                                 clienta = i++;
 541                                 break;
 542                         }
 543                 }
 544
 545                 for (client1 = NULL; i < mc->soc->num_clients; i++) {
 546                         if (stats[i].events) {
 547                                 client1 = &mc->soc->clients[i];
 548                                 clientb = i;
 549                                 break;
 550                         }
 551                 }
 552
 553                 if (!client0 && !client1)
 554                         break;
 555
 556                 tegra20_mc_stat_events(mc, client0, client1,
 557                                        MC_STAT_CONTROL_FILTER_PRI_YES,
 558                                        MC_STAT_CONTROL_PRI_EVENT_HP,
 559                                        MC_STAT_CONTROL_EVENT_QUALIFIED,
 560                                        &stats[clienta].arb_high_prio,
 561                                        &stats[clientb].arb_high_prio);
 562
 563                 tegra20_mc_stat_events(mc, client0, client1,
 564                                        MC_STAT_CONTROL_FILTER_PRI_YES,
 565                                        MC_STAT_CONTROL_PRI_EVENT_TM,
 566                                        MC_STAT_CONTROL_EVENT_QUALIFIED,
 567                                        &stats[clienta].arb_timeout,
 568                                        &stats[clientb].arb_timeout);
 569
 570                 tegra20_mc_stat_events(mc, client0, client1,
 571                                        MC_STAT_CONTROL_FILTER_PRI_YES,
 572                                        MC_STAT_CONTROL_PRI_EVENT_BW,
 573                                        MC_STAT_CONTROL_EVENT_QUALIFIED,
 574                                        &stats[clienta].arb_bandwidth,
 575                                        &stats[clientb].arb_bandwidth);
 576
 577                 tegra20_mc_stat_events(mc, client0, client1,
 578                                        MC_STAT_CONTROL_FILTER_PRI_DISABLE,
 579                                        MC_STAT_CONTROL_PRI_EVENT_HP,
 580                                        MC_STAT_CONTROL_EVENT_RD_WR_CHANGE,
 581                                        &stats[clienta].rd_wr_change,
 582                                        &stats[clientb].rd_wr_change);
 583
 584                 tegra20_mc_stat_events(mc, client0, client1,
 585                                        MC_STAT_CONTROL_FILTER_PRI_DISABLE,
 586                                        MC_STAT_CONTROL_PRI_EVENT_HP,
 587                                        MC_STAT_CONTROL_EVENT_SUCCESSIVE,
 588                                        &stats[clienta].successive,
 589                                        &stats[clientb].successive);
 590
 591                 tegra20_mc_stat_events(mc, client0, client1,
 592                                        MC_STAT_CONTROL_FILTER_PRI_DISABLE,
 593                                        MC_STAT_CONTROL_PRI_EVENT_HP,
 594                                        MC_STAT_CONTROL_EVENT_PAGE_MISS,
 595                                        &stats[clienta].page_miss,
 596                                        &stats[clientb].page_miss);
 597         }
 598 }
 599
 600 static void tegra20_mc_printf_percents(struct seq_file *s,
 601                                        const char *fmt,
 602                                        unsigned int percents_fx)
 603 {
 604         char percents_str[8];
 605
 606         snprintf(percents_str, ARRAY_SIZE(percents_str), "%3u.%02u%%",
 607                  percents_fx / MC_FX_FRAC_SCALE, percents_fx % MC_FX_FRAC_SCALE);
 608
 609         seq_printf(s, fmt, percents_str);
 610 }
 611
 612 static int tegra20_mc_stats_show(struct seq_file *s, void *unused)
 613 {
 614         const struct tegra_mc *mc = dev_get_drvdata(s->private);
 615         struct tegra20_mc_client_stat *stats;
 616         unsigned int i;
 617
 618         stats = kcalloc(mc->soc->num_clients + 1, sizeof(*stats), GFP_KERNEL);
 619         if (!stats)
 620                 return -ENOMEM;
 621
 622         mutex_lock(&tegra20_mc_stat_lock);
 623
 624         tegra20_mc_collect_stats(mc, stats);
 625
 626         mutex_unlock(&tegra20_mc_stat_lock);
 627
 628         seq_puts(s, "Memory client   Events   Timeout   High priority   Bandwidth ARB   RW change   Successive   Page miss\n");
 629         seq_puts(s, "-----------------------------------------------------------------------------------------------------\n");
 630
 631         for (i = 0; i < mc->soc->num_clients; i++) {
 632                 seq_printf(s, "%-14s  ", mc->soc->clients[i].name);
 633
 634                 /* An event is generated when client performs R/W request. */
 635                 tegra20_mc_printf_percents(s,  "%-9s", stats[i].events);
 636
 637                 /*
 638                  * An event is generated based on the timeout (TM) signal
 639                  * accompanying a request for arbitration.
 640                  */
 641                 tegra20_mc_printf_percents(s, "%-10s", stats[i].arb_timeout);
 642
 643                 /*
 644                  * An event is generated based on the high-priority (HP) signal
 645                  * accompanying a request for arbitration.
 646                  */
 647                 tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_high_prio);
 648
 649                 /*
 650                  * An event is generated based on the bandwidth (BW) signal
 651                  * accompanying a request for arbitration.
 652                  */
 653                 tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_bandwidth);
 654
 655                 /*
 656                  * An event is generated when the memory controller switches
 657                  * between making a read request to making a write request.
 658                  */
 659                 tegra20_mc_printf_percents(s, "%-12s", stats[i].rd_wr_change);
 660
 661                 /*
 662                  * An even generated when the chosen client has wins arbitration
 663                  * when it was also the winner at the previous request.  If a
 664                  * client makes N requests in a row that are honored, SUCCESSIVE
 665                  * will be counted (N-1) times.  Large values for this event
 666                  * imply that if we were patient enough, all of those requests
 667                  * could have been coalesced.
 668                  */
 669                 tegra20_mc_printf_percents(s, "%-13s", stats[i].successive);
 670
 671                 /*
 672                  * An event is generated when the memory controller detects a
 673                  * page miss for the current request.
 674                  */
 675                 tegra20_mc_printf_percents(s, "%-12s\n", stats[i].page_miss);
 676         }
 677
 678         kfree(stats);
 679
 680         return 0;
 681 }
 682
 683 static int tegra20_mc_probe(struct tegra_mc *mc)
 684 {
 685         debugfs_create_devm_seqfile(mc->dev, "stats", mc->debugfs.root,
 686                                     tegra20_mc_stats_show);
 687
 688         return 0;
 689 }
 690
 691 static int tegra20_mc_suspend(struct tegra_mc *mc)
 692 {
 693         int err;
 694
 695         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
 696                 err = tegra_gart_suspend(mc->gart);
 697                 if (err < 0)
 698                         return err;
 699         }
 700
 701         return 0;
 702 }
 703
 704 static int tegra20_mc_resume(struct tegra_mc *mc)
 705 {
 706         int err;
 707
 708         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
 709                 err = tegra_gart_resume(mc->gart);
 710                 if (err < 0)
 711                         return err;
 712         }
 713
 714         return 0;
 715 }
 716
 717 static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
 718 {
 719         struct tegra_mc *mc = data;
 720         unsigned long status;
 721         unsigned int bit;
 722
 723         /* mask all interrupts to avoid flooding */
 724         status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 725         if (!status)
 726                 return IRQ_NONE;
 727
 728         for_each_set_bit(bit, &status, 32) {
 729                 const char *error = tegra_mc_status_names[bit];
 730                 const char *direction = "read", *secure = "";
 731                 const char *client, *desc;
 732                 phys_addr_t addr;
 733                 u32 value, reg;
 734                 u8 id, type;
 735
 736                 switch (BIT(bit)) {
 737                 case MC_INT_DECERR_EMEM:
 738                         reg = MC_DECERR_EMEM_OTHERS_STATUS;
 739                         value = mc_readl(mc, reg);
 740
 741                         id = value & mc->soc->client_id_mask;
 742                         desc = tegra_mc_error_names[2];
 743
 744                         if (value & BIT(31))
 745                                 direction = "write";
 746                         break;
 747
 748                 case MC_INT_INVALID_GART_PAGE:
 749                         reg = MC_GART_ERROR_REQ;
 750                         value = mc_readl(mc, reg);
 751
 752                         id = (value >> 1) & mc->soc->client_id_mask;
 753                         desc = tegra_mc_error_names[2];
 754
 755                         if (value & BIT(0))
 756                                 direction = "write";
 757                         break;
 758
 759                 case MC_INT_SECURITY_VIOLATION:
 760                         reg = MC_SECURITY_VIOLATION_STATUS;
 761                         value = mc_readl(mc, reg);
 762
 763                         id = value & mc->soc->client_id_mask;
 764                         type = (value & BIT(30)) ? 4 : 3;
 765                         desc = tegra_mc_error_names[type];
 766                         secure = "secure ";
 767
 768                         if (value & BIT(31))
 769                                 direction = "write";
 770                         break;
 771
 772                 default:
 773                         continue;
 774                 }
 775
 776                 client = mc->soc->clients[id].name;
 777                 addr = mc_readl(mc, reg + sizeof(u32));
 778
 779                 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
 780                                     client, secure, direction, &addr, error,
 781                                     desc);
 782         }
 783
 784         /* clear interrupts */
 785         mc_writel(mc, status, MC_INTSTATUS);
 786
 787         return IRQ_HANDLED;
 788 }
 789
 790 static const struct tegra_mc_ops tegra20_mc_ops = {
 791         .probe = tegra20_mc_probe,
 792         .suspend = tegra20_mc_suspend,
 793         .resume = tegra20_mc_resume,
 794         .handle_irq = tegra20_mc_handle_irq,
 795 };
 796
 797 const struct tegra_mc_soc tegra20_mc_soc = {
 798         .clients = tegra20_mc_clients,
 799         .num_clients = ARRAY_SIZE(tegra20_mc_clients),
 800         .num_address_bits = 32,
 801         .client_id_mask = 0x3f,
 802         .intmask = MC_INT_SECURITY_VIOLATION | MC_INT_INVALID_GART_PAGE |
 803                    MC_INT_DECERR_EMEM,
 804         .reset_ops = &tegra20_mc_reset_ops,
 805         .resets = tegra20_mc_resets,
 806         .num_resets = ARRAY_SIZE(tegra20_mc_resets),
 807         .icc_ops = &tegra20_mc_icc_ops,
 808         .ops = &tegra20_mc_ops,
 809 };