arch/s390/kernel/perf_pai_crypto.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Performance event support - Processor Activity Instrumentation Facility
   4  *
   5  *  Copyright IBM Corp. 2022
   6  *  Author(s): Thomas Richter <tmricht@linux.ibm.com>
   7  */
   8 #define KMSG_COMPONENT  "pai_crypto"
   9 #define pr_fmt(fmt)     KMSG_COMPONENT ": " fmt
  10
  11 #include <linux/kernel.h>
  12 #include <linux/kernel_stat.h>
  13 #include <linux/percpu.h>
  14 #include <linux/notifier.h>
  15 #include <linux/init.h>
  16 #include <linux/export.h>
  17 #include <linux/io.h>
  18 #include <linux/perf_event.h>
  19
  20 #include <asm/ctl_reg.h>
  21 #include <asm/pai.h>
  22 #include <asm/debug.h>
  23
  24 static debug_info_t *cfm_dbg;
  25 static unsigned int paicrypt_cnt;       /* Size of the mapped counter sets */
  26                                         /* extracted with QPACI instruction */
  27
  28 DEFINE_STATIC_KEY_FALSE(pai_key);
  29
  30 struct pai_userdata {
  31         u16 num;
  32         u64 value;
  33 } __packed;
  34
  35 struct paicrypt_map {
  36         unsigned long *page;            /* Page for CPU to store counters */
  37         struct pai_userdata *save;      /* Page to store no-zero counters */
  38         unsigned int active_events;     /* # of PAI crypto users */
  39         refcount_t refcnt;              /* Reference count mapped buffers */
  40         enum paievt_mode mode;          /* Type of event */
  41         struct perf_event *event;       /* Perf event for sampling */
  42 };
  43
  44 static DEFINE_PER_CPU(struct paicrypt_map, paicrypt_map);
  45
  46 /* Release the PMU if event is the last perf event */
  47 static DEFINE_MUTEX(pai_reserve_mutex);
  48
  49 /* Adjust usage counters and remove allocated memory when all users are
  50  * gone.
  51  */
  52 static void paicrypt_event_destroy(struct perf_event *event)
  53 {
  54         struct paicrypt_map *cpump = per_cpu_ptr(&paicrypt_map, event->cpu);
  55
  56         cpump->event = NULL;
  57         static_branch_dec(&pai_key);
  58         mutex_lock(&pai_reserve_mutex);
  59         debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d"
  60                             " mode %d refcnt %u\n", __func__,
  61                             event->attr.config, event->cpu,
  62                             cpump->active_events, cpump->mode,
  63                             refcount_read(&cpump->refcnt));
  64         if (refcount_dec_and_test(&cpump->refcnt)) {
  65                 debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n",
  66                                     __func__, (unsigned long)cpump->page,
  67                                     cpump->save);
  68                 free_page((unsigned long)cpump->page);
  69                 cpump->page = NULL;
  70                 kvfree(cpump->save);
  71                 cpump->save = NULL;
  72                 cpump->mode = PAI_MODE_NONE;
  73         }
  74         mutex_unlock(&pai_reserve_mutex);
  75 }
  76
  77 static u64 paicrypt_getctr(struct paicrypt_map *cpump, int nr, bool kernel)
  78 {
  79         if (kernel)
  80                 nr += PAI_CRYPTO_MAXCTR;
  81         return cpump->page[nr];
  82 }
  83
  84 /* Read the counter values. Return value from location in CMP. For event
  85  * CRYPTO_ALL sum up all events.
  86  */
  87 static u64 paicrypt_getdata(struct perf_event *event, bool kernel)
  88 {
  89         struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
  90         u64 sum = 0;
  91         int i;
  92
  93         if (event->attr.config != PAI_CRYPTO_BASE) {
  94                 return paicrypt_getctr(cpump,
  95                                        event->attr.config - PAI_CRYPTO_BASE,
  96                                        kernel);
  97         }
  98
  99         for (i = 1; i <= paicrypt_cnt; i++) {
 100                 u64 val = paicrypt_getctr(cpump, i, kernel);
 101
 102                 if (!val)
 103                         continue;
 104                 sum += val;
 105         }
 106         return sum;
 107 }
 108
 109 static u64 paicrypt_getall(struct perf_event *event)
 110 {
 111         u64 sum = 0;
 112
 113         if (!event->attr.exclude_kernel)
 114                 sum += paicrypt_getdata(event, true);
 115         if (!event->attr.exclude_user)
 116                 sum += paicrypt_getdata(event, false);
 117
 118         return sum;
 119 }
 120
 121 /* Used to avoid races in checking concurrent access of counting and
 122  * sampling for crypto events
 123  *
 124  * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is
 125  * allowed and when this event is running, no counting event is allowed.
 126  * Several counting events are allowed in parallel, but no sampling event
 127  * is allowed while one (or more) counting events are running.
 128  *
 129  * This function is called in process context and it is save to block.
 130  * When the event initialization functions fails, no other call back will
 131  * be invoked.
 132  *
 133  * Allocate the memory for the event.
 134  */
 135 static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump)
 136 {
 137         int rc = 0;
 138
 139         mutex_lock(&pai_reserve_mutex);
 140         if (a->sample_period) {         /* Sampling requested */
 141                 if (cpump->mode != PAI_MODE_NONE)
 142                         rc = -EBUSY;    /* ... sampling/counting active */
 143         } else {                        /* Counting requested */
 144                 if (cpump->mode == PAI_MODE_SAMPLING)
 145                         rc = -EBUSY;    /* ... and sampling active */
 146         }
 147         if (rc)
 148                 goto unlock;
 149
 150         /* Allocate memory for counter page and counter extraction.
 151          * Only the first counting event has to allocate a page.
 152          */
 153         if (cpump->page) {
 154                 refcount_inc(&cpump->refcnt);
 155                 goto unlock;
 156         }
 157
 158         rc = -ENOMEM;
 159         cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 160         if (!cpump->page)
 161                 goto unlock;
 162         cpump->save = kvmalloc_array(paicrypt_cnt + 1,
 163                                      sizeof(struct pai_userdata), GFP_KERNEL);
 164         if (!cpump->save) {
 165                 free_page((unsigned long)cpump->page);
 166                 cpump->page = NULL;
 167                 goto unlock;
 168         }
 169         rc = 0;
 170         refcount_set(&cpump->refcnt, 1);
 171
 172 unlock:
 173         /* If rc is non-zero, do not set mode and reference count */
 174         if (!rc) {
 175                 cpump->mode = a->sample_period ? PAI_MODE_SAMPLING
 176                                                : PAI_MODE_COUNTING;
 177         }
 178         debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d"
 179                             " mode %d refcnt %u page %#lx save %p rc %d\n",
 180                             __func__, a->sample_period, cpump->active_events,
 181                             cpump->mode, refcount_read(&cpump->refcnt),
 182                             (unsigned long)cpump->page, cpump->save, rc);
 183         mutex_unlock(&pai_reserve_mutex);
 184         return rc;
 185 }
 186
 187 /* Might be called on different CPU than the one the event is intended for. */
 188 static int paicrypt_event_init(struct perf_event *event)
 189 {
 190         struct perf_event_attr *a = &event->attr;
 191         struct paicrypt_map *cpump;
 192         int rc;
 193
 194         /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */
 195         if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)
 196                 return -ENOENT;
 197         /* PAI crypto event must be in valid range */
 198         if (a->config < PAI_CRYPTO_BASE ||
 199             a->config > PAI_CRYPTO_BASE + paicrypt_cnt)
 200                 return -EINVAL;
 201         /* Allow only CPU wide operation, no process context for now. */
 202         if (event->hw.target || event->cpu == -1)
 203                 return -ENOENT;
 204         /* Allow only CRYPTO_ALL for sampling. */
 205         if (a->sample_period && a->config != PAI_CRYPTO_BASE)
 206                 return -EINVAL;
 207
 208         cpump = per_cpu_ptr(&paicrypt_map, event->cpu);
 209         rc = paicrypt_busy(a, cpump);
 210         if (rc)
 211                 return rc;
 212
 213         /* Event initialization sets last_tag to 0. When later on the events
 214          * are deleted and re-added, do not reset the event count value to zero.
 215          * Events are added, deleted and re-added when 2 or more events
 216          * are active at the same time.
 217          */
 218         event->hw.last_tag = 0;
 219         cpump->event = event;
 220         event->destroy = paicrypt_event_destroy;
 221
 222         if (a->sample_period) {
 223                 a->sample_period = 1;
 224                 a->freq = 0;
 225                 /* Register for paicrypt_sched_task() to be called */
 226                 event->attach_state |= PERF_ATTACH_SCHED_CB;
 227                 /* Add raw data which contain the memory mapped counters */
 228                 a->sample_type |= PERF_SAMPLE_RAW;
 229                 /* Turn off inheritance */
 230                 a->inherit = 0;
 231         }
 232
 233         static_branch_inc(&pai_key);
 234         return 0;
 235 }
 236
 237 static void paicrypt_read(struct perf_event *event)
 238 {
 239         u64 prev, new, delta;
 240
 241         prev = local64_read(&event->hw.prev_count);
 242         new = paicrypt_getall(event);
 243         local64_set(&event->hw.prev_count, new);
 244         delta = (prev <= new) ? new - prev
 245                               : (-1ULL - prev) + new + 1;        /* overflow */
 246         local64_add(delta, &event->count);
 247 }
 248
 249 static void paicrypt_start(struct perf_event *event, int flags)
 250 {
 251         u64 sum;
 252
 253         if (!event->hw.last_tag) {
 254                 event->hw.last_tag = 1;
 255                 sum = paicrypt_getall(event);           /* Get current value */
 256                 local64_set(&event->count, 0);
 257                 local64_set(&event->hw.prev_count, sum);
 258         }
 259 }
 260
 261 static int paicrypt_add(struct perf_event *event, int flags)
 262 {
 263         struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
 264         unsigned long ccd;
 265
 266         if (++cpump->active_events == 1) {
 267                 ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET;
 268                 WRITE_ONCE(S390_lowcore.ccd, ccd);
 269                 __ctl_set_bit(0, 50);
 270         }
 271         cpump->event = event;
 272         if (flags & PERF_EF_START && !event->attr.sample_period) {
 273                 /* Only counting needs initial counter value */
 274                 paicrypt_start(event, PERF_EF_RELOAD);
 275         }
 276         event->hw.state = 0;
 277         if (event->attr.sample_period)
 278                 perf_sched_cb_inc(event->pmu);
 279         return 0;
 280 }
 281
 282 static void paicrypt_stop(struct perf_event *event, int flags)
 283 {
 284         paicrypt_read(event);
 285         event->hw.state = PERF_HES_STOPPED;
 286 }
 287
 288 static void paicrypt_del(struct perf_event *event, int flags)
 289 {
 290         struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
 291
 292         if (event->attr.sample_period)
 293                 perf_sched_cb_dec(event->pmu);
 294         if (!event->attr.sample_period)
 295                 /* Only counting needs to read counter */
 296                 paicrypt_stop(event, PERF_EF_UPDATE);
 297         if (--cpump->active_events == 0) {
 298                 __ctl_clear_bit(0, 50);
 299                 WRITE_ONCE(S390_lowcore.ccd, 0);
 300         }
 301 }
 302
 303 /* Create raw data and save it in buffer. Returns number of bytes copied.
 304  * Saves only positive counter entries of the form
 305  * 2 bytes: Number of counter
 306  * 8 bytes: Value of counter
 307  */
 308 static size_t paicrypt_copy(struct pai_userdata *userdata,
 309                             struct paicrypt_map *cpump,
 310                             bool exclude_user, bool exclude_kernel)
 311 {
 312         int i, outidx = 0;
 313
 314         for (i = 1; i <= paicrypt_cnt; i++) {
 315                 u64 val = 0;
 316
 317                 if (!exclude_kernel)
 318                         val += paicrypt_getctr(cpump, i, true);
 319                 if (!exclude_user)
 320                         val += paicrypt_getctr(cpump, i, false);
 321                 if (val) {
 322                         userdata[outidx].num = i;
 323                         userdata[outidx].value = val;
 324                         outidx++;
 325                 }
 326         }
 327         return outidx * sizeof(struct pai_userdata);
 328 }
 329
 330 static int paicrypt_push_sample(void)
 331 {
 332         struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
 333         struct perf_event *event = cpump->event;
 334         struct perf_sample_data data;
 335         struct perf_raw_record raw;
 336         struct pt_regs regs;
 337         size_t rawsize;
 338         int overflow;
 339
 340         if (!cpump->event)              /* No event active */
 341                 return 0;
 342         rawsize = paicrypt_copy(cpump->save, cpump,
 343                                 cpump->event->attr.exclude_user,
 344                                 cpump->event->attr.exclude_kernel);
 345         if (!rawsize)                   /* No incremented counters */
 346                 return 0;
 347
 348         /* Setup perf sample */
 349         memset(&regs, 0, sizeof(regs));
 350         memset(&raw, 0, sizeof(raw));
 351         memset(&data, 0, sizeof(data));
 352         perf_sample_data_init(&data, 0, event->hw.last_period);
 353         if (event->attr.sample_type & PERF_SAMPLE_TID) {
 354                 data.tid_entry.pid = task_tgid_nr(current);
 355                 data.tid_entry.tid = task_pid_nr(current);
 356         }
 357         if (event->attr.sample_type & PERF_SAMPLE_TIME)
 358                 data.time = event->clock();
 359         if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
 360                 data.id = event->id;
 361         if (event->attr.sample_type & PERF_SAMPLE_CPU) {
 362                 data.cpu_entry.cpu = smp_processor_id();
 363                 data.cpu_entry.reserved = 0;
 364         }
 365         if (event->attr.sample_type & PERF_SAMPLE_RAW) {
 366                 raw.frag.size = rawsize;
 367                 raw.frag.data = cpump->save;
 368                 perf_sample_save_raw_data(&data, &raw);
 369         }
 370
 371         overflow = perf_event_overflow(event, &data, &regs);
 372         perf_event_update_userpage(event);
 373         /* Clear lowcore page after read */
 374         memset(cpump->page, 0, PAGE_SIZE);
 375         return overflow;
 376 }
 377
 378 /* Called on schedule-in and schedule-out. No access to event structure,
 379  * but for sampling only event CRYPTO_ALL is allowed.
 380  */
 381 static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
 382 {
 383         /* We started with a clean page on event installation. So read out
 384          * results on schedule_out and if page was dirty, clear values.
 385          */
 386         if (!sched_in)
 387                 paicrypt_push_sample();
 388 }
 389
 390 /* Attribute definitions for paicrypt interface. As with other CPU
 391  * Measurement Facilities, there is one attribute per mapped counter.
 392  * The number of mapped counters may vary per machine generation. Use
 393  * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction
 394  * to determine the number of mapped counters. The instructions returns
 395  * a positive number, which is the highest number of supported counters.
 396  * All counters less than this number are also supported, there are no
 397  * holes. A returned number of zero means no support for mapped counters.
 398  *
 399  * The identification of the counter is a unique number. The chosen range
 400  * is 0x1000 + offset in mapped kernel page.
 401  * All CPU Measurement Facility counters identifiers must be unique and
 402  * the numbers from 0 to 496 are already used for the CPU Measurement
 403  * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already
 404  * used for the CPU Measurement Sampling facility.
 405  */
 406 PMU_FORMAT_ATTR(event, "config:0-63");
 407
 408 static struct attribute *paicrypt_format_attr[] = {
 409         &format_attr_event.attr,
 410         NULL,
 411 };
 412
 413 static struct attribute_group paicrypt_events_group = {
 414         .name = "events",
 415         .attrs = NULL                   /* Filled in attr_event_init() */
 416 };
 417
 418 static struct attribute_group paicrypt_format_group = {
 419         .name = "format",
 420         .attrs = paicrypt_format_attr,
 421 };
 422
 423 static const struct attribute_group *paicrypt_attr_groups[] = {
 424         &paicrypt_events_group,
 425         &paicrypt_format_group,
 426         NULL,
 427 };
 428
 429 /* Performance monitoring unit for mapped counters */
 430 static struct pmu paicrypt = {
 431         .task_ctx_nr  = perf_invalid_context,
 432         .event_init   = paicrypt_event_init,
 433         .add          = paicrypt_add,
 434         .del          = paicrypt_del,
 435         .start        = paicrypt_start,
 436         .stop         = paicrypt_stop,
 437         .read         = paicrypt_read,
 438         .sched_task   = paicrypt_sched_task,
 439         .attr_groups  = paicrypt_attr_groups
 440 };
 441
 442 /* List of symbolic PAI counter names. */
 443 static const char * const paicrypt_ctrnames[] = {
 444         [0] = "CRYPTO_ALL",
 445         [1] = "KM_DEA",
 446         [2] = "KM_TDEA_128",
 447         [3] = "KM_TDEA_192",
 448         [4] = "KM_ENCRYPTED_DEA",
 449         [5] = "KM_ENCRYPTED_TDEA_128",
 450         [6] = "KM_ENCRYPTED_TDEA_192",
 451         [7] = "KM_AES_128",
 452         [8] = "KM_AES_192",
 453         [9] = "KM_AES_256",
 454         [10] = "KM_ENCRYPTED_AES_128",
 455         [11] = "KM_ENCRYPTED_AES_192",
 456         [12] = "KM_ENCRYPTED_AES_256",
 457         [13] = "KM_XTS_AES_128",
 458         [14] = "KM_XTS_AES_256",
 459         [15] = "KM_XTS_ENCRYPTED_AES_128",
 460         [16] = "KM_XTS_ENCRYPTED_AES_256",
 461         [17] = "KMC_DEA",
 462         [18] = "KMC_TDEA_128",
 463         [19] = "KMC_TDEA_192",
 464         [20] = "KMC_ENCRYPTED_DEA",
 465         [21] = "KMC_ENCRYPTED_TDEA_128",
 466         [22] = "KMC_ENCRYPTED_TDEA_192",
 467         [23] = "KMC_AES_128",
 468         [24] = "KMC_AES_192",
 469         [25] = "KMC_AES_256",
 470         [26] = "KMC_ENCRYPTED_AES_128",
 471         [27] = "KMC_ENCRYPTED_AES_192",
 472         [28] = "KMC_ENCRYPTED_AES_256",
 473         [29] = "KMC_PRNG",
 474         [30] = "KMA_GCM_AES_128",
 475         [31] = "KMA_GCM_AES_192",
 476         [32] = "KMA_GCM_AES_256",
 477         [33] = "KMA_GCM_ENCRYPTED_AES_128",
 478         [34] = "KMA_GCM_ENCRYPTED_AES_192",
 479         [35] = "KMA_GCM_ENCRYPTED_AES_256",
 480         [36] = "KMF_DEA",
 481         [37] = "KMF_TDEA_128",
 482         [38] = "KMF_TDEA_192",
 483         [39] = "KMF_ENCRYPTED_DEA",
 484         [40] = "KMF_ENCRYPTED_TDEA_128",
 485         [41] = "KMF_ENCRYPTED_TDEA_192",
 486         [42] = "KMF_AES_128",
 487         [43] = "KMF_AES_192",
 488         [44] = "KMF_AES_256",
 489         [45] = "KMF_ENCRYPTED_AES_128",
 490         [46] = "KMF_ENCRYPTED_AES_192",
 491         [47] = "KMF_ENCRYPTED_AES_256",
 492         [48] = "KMCTR_DEA",
 493         [49] = "KMCTR_TDEA_128",
 494         [50] = "KMCTR_TDEA_192",
 495         [51] = "KMCTR_ENCRYPTED_DEA",
 496         [52] = "KMCTR_ENCRYPTED_TDEA_128",
 497         [53] = "KMCTR_ENCRYPTED_TDEA_192",
 498         [54] = "KMCTR_AES_128",
 499         [55] = "KMCTR_AES_192",
 500         [56] = "KMCTR_AES_256",
 501         [57] = "KMCTR_ENCRYPTED_AES_128",
 502         [58] = "KMCTR_ENCRYPTED_AES_192",
 503         [59] = "KMCTR_ENCRYPTED_AES_256",
 504         [60] = "KMO_DEA",
 505         [61] = "KMO_TDEA_128",
 506         [62] = "KMO_TDEA_192",
 507         [63] = "KMO_ENCRYPTED_DEA",
 508         [64] = "KMO_ENCRYPTED_TDEA_128",
 509         [65] = "KMO_ENCRYPTED_TDEA_192",
 510         [66] = "KMO_AES_128",
 511         [67] = "KMO_AES_192",
 512         [68] = "KMO_AES_256",
 513         [69] = "KMO_ENCRYPTED_AES_128",
 514         [70] = "KMO_ENCRYPTED_AES_192",
 515         [71] = "KMO_ENCRYPTED_AES_256",
 516         [72] = "KIMD_SHA_1",
 517         [73] = "KIMD_SHA_256",
 518         [74] = "KIMD_SHA_512",
 519         [75] = "KIMD_SHA3_224",
 520         [76] = "KIMD_SHA3_256",
 521         [77] = "KIMD_SHA3_384",
 522         [78] = "KIMD_SHA3_512",
 523         [79] = "KIMD_SHAKE_128",
 524         [80] = "KIMD_SHAKE_256",
 525         [81] = "KIMD_GHASH",
 526         [82] = "KLMD_SHA_1",
 527         [83] = "KLMD_SHA_256",
 528         [84] = "KLMD_SHA_512",
 529         [85] = "KLMD_SHA3_224",
 530         [86] = "KLMD_SHA3_256",
 531         [87] = "KLMD_SHA3_384",
 532         [88] = "KLMD_SHA3_512",
 533         [89] = "KLMD_SHAKE_128",
 534         [90] = "KLMD_SHAKE_256",
 535         [91] = "KMAC_DEA",
 536         [92] = "KMAC_TDEA_128",
 537         [93] = "KMAC_TDEA_192",
 538         [94] = "KMAC_ENCRYPTED_DEA",
 539         [95] = "KMAC_ENCRYPTED_TDEA_128",
 540         [96] = "KMAC_ENCRYPTED_TDEA_192",
 541         [97] = "KMAC_AES_128",
 542         [98] = "KMAC_AES_192",
 543         [99] = "KMAC_AES_256",
 544         [100] = "KMAC_ENCRYPTED_AES_128",
 545         [101] = "KMAC_ENCRYPTED_AES_192",
 546         [102] = "KMAC_ENCRYPTED_AES_256",
 547         [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA",
 548         [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128",
 549         [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192",
 550         [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA",
 551         [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128",
 552         [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192",
 553         [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128",
 554         [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192",
 555         [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256",
 556         [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128",
 557         [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192",
 558         [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256A",
 559         [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128",
 560         [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256",
 561         [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128",
 562         [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256",
 563         [119] = "PCC_SCALAR_MULTIPLY_P256",
 564         [120] = "PCC_SCALAR_MULTIPLY_P384",
 565         [121] = "PCC_SCALAR_MULTIPLY_P521",
 566         [122] = "PCC_SCALAR_MULTIPLY_ED25519",
 567         [123] = "PCC_SCALAR_MULTIPLY_ED448",
 568         [124] = "PCC_SCALAR_MULTIPLY_X25519",
 569         [125] = "PCC_SCALAR_MULTIPLY_X448",
 570         [126] = "PRNO_SHA_512_DRNG",
 571         [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO",
 572         [128] = "PRNO_TRNG",
 573         [129] = "KDSA_ECDSA_VERIFY_P256",
 574         [130] = "KDSA_ECDSA_VERIFY_P384",
 575         [131] = "KDSA_ECDSA_VERIFY_P521",
 576         [132] = "KDSA_ECDSA_SIGN_P256",
 577         [133] = "KDSA_ECDSA_SIGN_P384",
 578         [134] = "KDSA_ECDSA_SIGN_P521",
 579         [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256",
 580         [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384",
 581         [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521",
 582         [138] = "KDSA_EDDSA_VERIFY_ED25519",
 583         [139] = "KDSA_EDDSA_VERIFY_ED448",
 584         [140] = "KDSA_EDDSA_SIGN_ED25519",
 585         [141] = "KDSA_EDDSA_SIGN_ED448",
 586         [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519",
 587         [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448",
 588         [144] = "PCKMO_ENCRYPT_DEA_KEY",
 589         [145] = "PCKMO_ENCRYPT_TDEA_128_KEY",
 590         [146] = "PCKMO_ENCRYPT_TDEA_192_KEY",
 591         [147] = "PCKMO_ENCRYPT_AES_128_KEY",
 592         [148] = "PCKMO_ENCRYPT_AES_192_KEY",
 593         [149] = "PCKMO_ENCRYPT_AES_256_KEY",
 594         [150] = "PCKMO_ENCRYPT_ECC_P256_KEY",
 595         [151] = "PCKMO_ENCRYPT_ECC_P384_KEY",
 596         [152] = "PCKMO_ENCRYPT_ECC_P521_KEY",
 597         [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY",
 598         [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY",
 599         [155] = "IBM_RESERVED_155",
 600         [156] = "IBM_RESERVED_156",
 601 };
 602
 603 static void __init attr_event_free(struct attribute **attrs, int num)
 604 {
 605         struct perf_pmu_events_attr *pa;
 606         int i;
 607
 608         for (i = 0; i < num; i++) {
 609                 struct device_attribute *dap;
 610
 611                 dap = container_of(attrs[i], struct device_attribute, attr);
 612                 pa = container_of(dap, struct perf_pmu_events_attr, attr);
 613                 kfree(pa);
 614         }
 615         kfree(attrs);
 616 }
 617
 618 static int __init attr_event_init_one(struct attribute **attrs, int num)
 619 {
 620         struct perf_pmu_events_attr *pa;
 621
 622         pa = kzalloc(sizeof(*pa), GFP_KERNEL);
 623         if (!pa)
 624                 return -ENOMEM;
 625
 626         sysfs_attr_init(&pa->attr.attr);
 627         pa->id = PAI_CRYPTO_BASE + num;
 628         pa->attr.attr.name = paicrypt_ctrnames[num];
 629         pa->attr.attr.mode = 0444;
 630         pa->attr.show = cpumf_events_sysfs_show;
 631         pa->attr.store = NULL;
 632         attrs[num] = &pa->attr.attr;
 633         return 0;
 634 }
 635
 636 /* Create PMU sysfs event attributes on the fly. */
 637 static int __init attr_event_init(void)
 638 {
 639         struct attribute **attrs;
 640         int ret, i;
 641
 642         attrs = kmalloc_array(ARRAY_SIZE(paicrypt_ctrnames) + 1, sizeof(*attrs),
 643                               GFP_KERNEL);
 644         if (!attrs)
 645                 return -ENOMEM;
 646         for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) {
 647                 ret = attr_event_init_one(attrs, i);
 648                 if (ret) {
 649                         attr_event_free(attrs, i - 1);
 650                         return ret;
 651                 }
 652         }
 653         attrs[i] = NULL;
 654         paicrypt_events_group.attrs = attrs;
 655         return 0;
 656 }
 657
 658 static int __init paicrypt_init(void)
 659 {
 660         struct qpaci_info_block ib;
 661         int rc;
 662
 663         if (!test_facility(196))
 664                 return 0;
 665
 666         qpaci(&ib);
 667         paicrypt_cnt = ib.num_cc;
 668         if (paicrypt_cnt == 0)
 669                 return 0;
 670         if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR)
 671                 paicrypt_cnt = PAI_CRYPTO_MAXCTR - 1;
 672
 673         rc = attr_event_init();         /* Export known PAI crypto events */
 674         if (rc) {
 675                 pr_err("Creation of PMU pai_crypto /sysfs failed\n");
 676                 return rc;
 677         }
 678
 679         /* Setup s390dbf facility */
 680         cfm_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128);
 681         if (!cfm_dbg) {
 682                 pr_err("Registration of s390dbf pai_crypto failed\n");
 683                 return -ENOMEM;
 684         }
 685         debug_register_view(cfm_dbg, &debug_sprintf_view);
 686
 687         rc = perf_pmu_register(&paicrypt, "pai_crypto", -1);
 688         if (rc) {
 689                 pr_err("Registering the pai_crypto PMU failed with rc=%i\n",
 690                        rc);
 691                 debug_unregister_view(cfm_dbg, &debug_sprintf_view);
 692                 debug_unregister(cfm_dbg);
 693                 return rc;
 694         }
 695         return 0;
 696 }
 697
 698 device_initcall(paicrypt_init);