drivers/crypto/nx/nx-842-pseries.c

   1 /*
   2  * Driver for IBM Power 842 compression accelerator
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  17  *
  18  * Copyright (C) IBM Corporation, 2012
  19  *
  20  * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
  21  *          Seth Jennings <sjenning@linux.vnet.ibm.com>
  22  */
  23
  24 #include <asm/vio.h>
  25
  26 #include "nx-842.h"
  27 #include "nx_csbcpb.h" /* struct nx_csbcpb */
  28
  29 MODULE_LICENSE("GPL");
  30 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
  31 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
  32
  33 static struct nx842_constraints nx842_pseries_constraints = {
  34         .alignment =    DDE_BUFFER_ALIGN,
  35         .multiple =     DDE_BUFFER_LAST_MULT,
  36         .minimum =      DDE_BUFFER_LAST_MULT,
  37         .maximum =      PAGE_SIZE, /* dynamic, max_sync_size */
  38 };
  39
  40 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
  41 {
  42         if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
  43                 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
  44                          in ? "input" : "output", buf,
  45                          nx842_pseries_constraints.alignment);
  46                 return -EINVAL;
  47         }
  48         if (*len % nx842_pseries_constraints.multiple) {
  49                 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
  50                          in ? "input" : "output", *len,
  51                          nx842_pseries_constraints.multiple);
  52                 if (in)
  53                         return -EINVAL;
  54                 *len = round_down(*len, nx842_pseries_constraints.multiple);
  55         }
  56         if (*len < nx842_pseries_constraints.minimum) {
  57                 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
  58                          in ? "input" : "output", *len,
  59                          nx842_pseries_constraints.minimum);
  60                 return -EINVAL;
  61         }
  62         if (*len > nx842_pseries_constraints.maximum) {
  63                 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
  64                          in ? "input" : "output", *len,
  65                          nx842_pseries_constraints.maximum);
  66                 if (in)
  67                         return -EINVAL;
  68                 *len = nx842_pseries_constraints.maximum;
  69         }
  70         return 0;
  71 }
  72
  73 /* I assume we need to align the CSB? */
  74 #define WORKMEM_ALIGN   (256)
  75
  76 struct nx842_workmem {
  77         /* scatterlist */
  78         char slin[4096];
  79         char slout[4096];
  80         /* coprocessor status/parameter block */
  81         struct nx_csbcpb csbcpb;
  82
  83         char padding[WORKMEM_ALIGN];
  84 } __aligned(WORKMEM_ALIGN);
  85
  86 /* Macros for fields within nx_csbcpb */
  87 /* Check the valid bit within the csbcpb valid field */
  88 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
  89
  90 /* CE macros operate on the completion_extension field bits in the csbcpb.
  91  * CE0 0=full completion, 1=partial completion
  92  * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
  93  * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
  94 #define NX842_CSBCPB_CE0(x)     (x & BIT_MASK(7))
  95 #define NX842_CSBCPB_CE1(x)     (x & BIT_MASK(6))
  96 #define NX842_CSBCPB_CE2(x)     (x & BIT_MASK(5))
  97
  98 /* The NX unit accepts data only on 4K page boundaries */
  99 #define NX842_HW_PAGE_SIZE      (4096)
 100 #define NX842_HW_PAGE_MASK      (~(NX842_HW_PAGE_SIZE-1))
 101
 102 enum nx842_status {
 103         UNAVAILABLE,
 104         AVAILABLE
 105 };
 106
 107 struct ibm_nx842_counters {
 108         atomic64_t comp_complete;
 109         atomic64_t comp_failed;
 110         atomic64_t decomp_complete;
 111         atomic64_t decomp_failed;
 112         atomic64_t swdecomp;
 113         atomic64_t comp_times[32];
 114         atomic64_t decomp_times[32];
 115 };
 116
 117 static struct nx842_devdata {
 118         struct vio_dev *vdev;
 119         struct device *dev;
 120         struct ibm_nx842_counters *counters;
 121         unsigned int max_sg_len;
 122         unsigned int max_sync_size;
 123         unsigned int max_sync_sg;
 124         enum nx842_status status;
 125 } __rcu *devdata;
 126 static DEFINE_SPINLOCK(devdata_mutex);
 127
 128 #define NX842_COUNTER_INC(_x) \
 129 static inline void nx842_inc_##_x( \
 130         const struct nx842_devdata *dev) { \
 131         if (dev) \
 132                 atomic64_inc(&dev->counters->_x); \
 133 }
 134 NX842_COUNTER_INC(comp_complete);
 135 NX842_COUNTER_INC(comp_failed);
 136 NX842_COUNTER_INC(decomp_complete);
 137 NX842_COUNTER_INC(decomp_failed);
 138 NX842_COUNTER_INC(swdecomp);
 139
 140 #define NX842_HIST_SLOTS 16
 141
 142 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
 143 {
 144         int bucket = fls(time);
 145
 146         if (bucket)
 147                 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
 148
 149         atomic64_inc(&times[bucket]);
 150 }
 151
 152 /* NX unit operation flags */
 153 #define NX842_OP_COMPRESS       0x0
 154 #define NX842_OP_CRC            0x1
 155 #define NX842_OP_DECOMPRESS     0x2
 156 #define NX842_OP_COMPRESS_CRC   (NX842_OP_COMPRESS | NX842_OP_CRC)
 157 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
 158 #define NX842_OP_ASYNC          (1<<23)
 159 #define NX842_OP_NOTIFY         (1<<22)
 160 #define NX842_OP_NOTIFY_INT(x)  ((x & 0xff)<<8)
 161
 162 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
 163 {
 164         /* No use of DMA mappings within the driver. */
 165         return 0;
 166 }
 167
 168 struct nx842_slentry {
 169         __be64 ptr; /* Real address (use __pa()) */
 170         __be64 len;
 171 };
 172
 173 /* pHyp scatterlist entry */
 174 struct nx842_scatterlist {
 175         int entry_nr; /* number of slentries */
 176         struct nx842_slentry *entries; /* ptr to array of slentries */
 177 };
 178
 179 /* Does not include sizeof(entry_nr) in the size */
 180 static inline unsigned long nx842_get_scatterlist_size(
 181                                 struct nx842_scatterlist *sl)
 182 {
 183         return sl->entry_nr * sizeof(struct nx842_slentry);
 184 }
 185
 186 static int nx842_build_scatterlist(unsigned long buf, int len,
 187                         struct nx842_scatterlist *sl)
 188 {
 189         unsigned long entrylen;
 190         struct nx842_slentry *entry;
 191
 192         sl->entry_nr = 0;
 193
 194         entry = sl->entries;
 195         while (len) {
 196                 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
 197                 entrylen = min_t(int, len,
 198                                  LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
 199                 entry->len = cpu_to_be64(entrylen);
 200
 201                 len -= entrylen;
 202                 buf += entrylen;
 203
 204                 sl->entry_nr++;
 205                 entry++;
 206         }
 207
 208         return 0;
 209 }
 210
 211 static int nx842_validate_result(struct device *dev,
 212         struct cop_status_block *csb)
 213 {
 214         /* The csb must be valid after returning from vio_h_cop_sync */
 215         if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
 216                 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
 217                                 __func__);
 218                 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
 219                                 csb->valid,
 220                                 csb->crb_seq_number,
 221                                 csb->completion_code,
 222                                 csb->completion_extension);
 223                 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
 224                                 be32_to_cpu(csb->processed_byte_count),
 225                                 (unsigned long)be64_to_cpu(csb->address));
 226                 return -EIO;
 227         }
 228
 229         /* Check return values from the hardware in the CSB */
 230         switch (csb->completion_code) {
 231         case 0: /* Completed without error */
 232                 break;
 233         case 64: /* Target bytes > Source bytes during compression */
 234         case 13: /* Output buffer too small */
 235                 dev_dbg(dev, "%s: Compression output larger than input\n",
 236                                         __func__);
 237                 return -ENOSPC;
 238         case 66: /* Input data contains an illegal template field */
 239         case 67: /* Template indicates data past the end of the input stream */
 240                 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
 241                                         __func__, csb->completion_code);
 242                 return -EINVAL;
 243         default:
 244                 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
 245                                         __func__, csb->completion_code);
 246                 return -EIO;
 247         }
 248
 249         /* Hardware sanity check */
 250         if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
 251                 dev_err(dev, "%s: No error returned by hardware, but "
 252                                 "data returned is unusable, contact support.\n"
 253                                 "(Additional info: csbcbp->processed bytes "
 254                                 "does not specify processed bytes for the "
 255                                 "target buffer.)\n", __func__);
 256                 return -EIO;
 257         }
 258
 259         return 0;
 260 }
 261
 262 /**
 263  * nx842_pseries_compress - Compress data using the 842 algorithm
 264  *
 265  * Compression provide by the NX842 coprocessor on IBM Power systems.
 266  * The input buffer is compressed and the result is stored in the
 267  * provided output buffer.
 268  *
 269  * Upon return from this function @outlen contains the length of the
 270  * compressed data.  If there is an error then @outlen will be 0 and an
 271  * error will be specified by the return code from this function.
 272  *
 273  * @in: Pointer to input buffer
 274  * @inlen: Length of input buffer
 275  * @out: Pointer to output buffer
 276  * @outlen: Length of output buffer
 277  * @wrkmem: ptr to buffer for working memory, size determined by
 278  *          nx842_pseries_driver.workmem_size
 279  *
 280  * Returns:
 281  *   0          Success, output of length @outlen stored in the buffer at @out
 282  *   -ENOMEM    Unable to allocate internal buffers
 283  *   -ENOSPC    Output buffer is to small
 284  *   -EIO       Internal error
 285  *   -ENODEV    Hardware unavailable
 286  */
 287 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
 288                                   unsigned char *out, unsigned int *outlen,
 289                                   void *wmem)
 290 {
 291         struct nx842_devdata *local_devdata;
 292         struct device *dev = NULL;
 293         struct nx842_workmem *workmem;
 294         struct nx842_scatterlist slin, slout;
 295         struct nx_csbcpb *csbcpb;
 296         int ret = 0, max_sync_size;
 297         unsigned long inbuf, outbuf;
 298         struct vio_pfo_op op = {
 299                 .done = NULL,
 300                 .handle = 0,
 301                 .timeout = 0,
 302         };
 303         unsigned long start = get_tb();
 304
 305         inbuf = (unsigned long)in;
 306         if (check_constraints(inbuf, &inlen, true))
 307                 return -EINVAL;
 308
 309         outbuf = (unsigned long)out;
 310         if (check_constraints(outbuf, outlen, false))
 311                 return -EINVAL;
 312
 313         rcu_read_lock();
 314         local_devdata = rcu_dereference(devdata);
 315         if (!local_devdata || !local_devdata->dev) {
 316                 rcu_read_unlock();
 317                 return -ENODEV;
 318         }
 319         max_sync_size = local_devdata->max_sync_size;
 320         dev = local_devdata->dev;
 321
 322         /* Init scatterlist */
 323         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 324         slin.entries = (struct nx842_slentry *)workmem->slin;
 325         slout.entries = (struct nx842_slentry *)workmem->slout;
 326
 327         /* Init operation */
 328         op.flags = NX842_OP_COMPRESS;
 329         csbcpb = &workmem->csbcpb;
 330         memset(csbcpb, 0, sizeof(*csbcpb));
 331         op.csbcpb = nx842_get_pa(csbcpb);
 332
 333         if ((inbuf & NX842_HW_PAGE_MASK) ==
 334             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 335                 /* Create direct DDE */
 336                 op.in = nx842_get_pa((void *)inbuf);
 337                 op.inlen = inlen;
 338         } else {
 339                 /* Create indirect DDE (scatterlist) */
 340                 nx842_build_scatterlist(inbuf, inlen, &slin);
 341                 op.in = nx842_get_pa(slin.entries);
 342                 op.inlen = -nx842_get_scatterlist_size(&slin);
 343         }
 344
 345         if ((outbuf & NX842_HW_PAGE_MASK) ==
 346             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 347                 /* Create direct DDE */
 348                 op.out = nx842_get_pa((void *)outbuf);
 349                 op.outlen = *outlen;
 350         } else {
 351                 /* Create indirect DDE (scatterlist) */
 352                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 353                 op.out = nx842_get_pa(slout.entries);
 354                 op.outlen = -nx842_get_scatterlist_size(&slout);
 355         }
 356
 357         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 358                 __func__, (unsigned long)op.in, (long)op.inlen,
 359                 (unsigned long)op.out, (long)op.outlen);
 360
 361         /* Send request to pHyp */
 362         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 363
 364         /* Check for pHyp error */
 365         if (ret) {
 366                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 367                         __func__, ret, op.hcall_err);
 368                 ret = -EIO;
 369                 goto unlock;
 370         }
 371
 372         /* Check for hardware error */
 373         ret = nx842_validate_result(dev, &csbcpb->csb);
 374         if (ret)
 375                 goto unlock;
 376
 377         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 378         dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
 379
 380 unlock:
 381         if (ret)
 382                 nx842_inc_comp_failed(local_devdata);
 383         else {
 384                 nx842_inc_comp_complete(local_devdata);
 385                 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
 386                         (get_tb() - start) / tb_ticks_per_usec);
 387         }
 388         rcu_read_unlock();
 389         return ret;
 390 }
 391
 392 /**
 393  * nx842_pseries_decompress - Decompress data using the 842 algorithm
 394  *
 395  * Decompression provide by the NX842 coprocessor on IBM Power systems.
 396  * The input buffer is decompressed and the result is stored in the
 397  * provided output buffer.  The size allocated to the output buffer is
 398  * provided by the caller of this function in @outlen.  Upon return from
 399  * this function @outlen contains the length of the decompressed data.
 400  * If there is an error then @outlen will be 0 and an error will be
 401  * specified by the return code from this function.
 402  *
 403  * @in: Pointer to input buffer
 404  * @inlen: Length of input buffer
 405  * @out: Pointer to output buffer
 406  * @outlen: Length of output buffer
 407  * @wrkmem: ptr to buffer for working memory, size determined by
 408  *          nx842_pseries_driver.workmem_size
 409  *
 410  * Returns:
 411  *   0          Success, output of length @outlen stored in the buffer at @out
 412  *   -ENODEV    Hardware decompression device is unavailable
 413  *   -ENOMEM    Unable to allocate internal buffers
 414  *   -ENOSPC    Output buffer is to small
 415  *   -EINVAL    Bad input data encountered when attempting decompress
 416  *   -EIO       Internal error
 417  */
 418 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
 419                                     unsigned char *out, unsigned int *outlen,
 420                                     void *wmem)
 421 {
 422         struct nx842_devdata *local_devdata;
 423         struct device *dev = NULL;
 424         struct nx842_workmem *workmem;
 425         struct nx842_scatterlist slin, slout;
 426         struct nx_csbcpb *csbcpb;
 427         int ret = 0, max_sync_size;
 428         unsigned long inbuf, outbuf;
 429         struct vio_pfo_op op = {
 430                 .done = NULL,
 431                 .handle = 0,
 432                 .timeout = 0,
 433         };
 434         unsigned long start = get_tb();
 435
 436         /* Ensure page alignment and size */
 437         inbuf = (unsigned long)in;
 438         if (check_constraints(inbuf, &inlen, true))
 439                 return -EINVAL;
 440
 441         outbuf = (unsigned long)out;
 442         if (check_constraints(outbuf, outlen, false))
 443                 return -EINVAL;
 444
 445         rcu_read_lock();
 446         local_devdata = rcu_dereference(devdata);
 447         if (!local_devdata || !local_devdata->dev) {
 448                 rcu_read_unlock();
 449                 return -ENODEV;
 450         }
 451         max_sync_size = local_devdata->max_sync_size;
 452         dev = local_devdata->dev;
 453
 454         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 455
 456         /* Init scatterlist */
 457         slin.entries = (struct nx842_slentry *)workmem->slin;
 458         slout.entries = (struct nx842_slentry *)workmem->slout;
 459
 460         /* Init operation */
 461         op.flags = NX842_OP_DECOMPRESS;
 462         csbcpb = &workmem->csbcpb;
 463         memset(csbcpb, 0, sizeof(*csbcpb));
 464         op.csbcpb = nx842_get_pa(csbcpb);
 465
 466         if ((inbuf & NX842_HW_PAGE_MASK) ==
 467             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 468                 /* Create direct DDE */
 469                 op.in = nx842_get_pa((void *)inbuf);
 470                 op.inlen = inlen;
 471         } else {
 472                 /* Create indirect DDE (scatterlist) */
 473                 nx842_build_scatterlist(inbuf, inlen, &slin);
 474                 op.in = nx842_get_pa(slin.entries);
 475                 op.inlen = -nx842_get_scatterlist_size(&slin);
 476         }
 477
 478         if ((outbuf & NX842_HW_PAGE_MASK) ==
 479             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 480                 /* Create direct DDE */
 481                 op.out = nx842_get_pa((void *)outbuf);
 482                 op.outlen = *outlen;
 483         } else {
 484                 /* Create indirect DDE (scatterlist) */
 485                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 486                 op.out = nx842_get_pa(slout.entries);
 487                 op.outlen = -nx842_get_scatterlist_size(&slout);
 488         }
 489
 490         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 491                 __func__, (unsigned long)op.in, (long)op.inlen,
 492                 (unsigned long)op.out, (long)op.outlen);
 493
 494         /* Send request to pHyp */
 495         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 496
 497         /* Check for pHyp error */
 498         if (ret) {
 499                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 500                         __func__, ret, op.hcall_err);
 501                 goto unlock;
 502         }
 503
 504         /* Check for hardware error */
 505         ret = nx842_validate_result(dev, &csbcpb->csb);
 506         if (ret)
 507                 goto unlock;
 508
 509         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 510
 511 unlock:
 512         if (ret)
 513                 /* decompress fail */
 514                 nx842_inc_decomp_failed(local_devdata);
 515         else {
 516                 nx842_inc_decomp_complete(local_devdata);
 517                 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
 518                         (get_tb() - start) / tb_ticks_per_usec);
 519         }
 520
 521         rcu_read_unlock();
 522         return ret;
 523 }
 524
 525 /**
 526  * nx842_OF_set_defaults -- Set default (disabled) values for devdata
 527  *
 528  * @devdata - struct nx842_devdata to update
 529  *
 530  * Returns:
 531  *  0 on success
 532  *  -ENOENT if @devdata ptr is NULL
 533  */
 534 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
 535 {
 536         if (devdata) {
 537                 devdata->max_sync_size = 0;
 538                 devdata->max_sync_sg = 0;
 539                 devdata->max_sg_len = 0;
 540                 devdata->status = UNAVAILABLE;
 541                 return 0;
 542         } else
 543                 return -ENOENT;
 544 }
 545
 546 /**
 547  * nx842_OF_upd_status -- Update the device info from OF status prop
 548  *
 549  * The status property indicates if the accelerator is enabled.  If the
 550  * device is in the OF tree it indicates that the hardware is present.
 551  * The status field indicates if the device is enabled when the status
 552  * is 'okay'.  Otherwise the device driver will be disabled.
 553  *
 554  * @devdata - struct nx842_devdata to update
 555  * @prop - struct property point containing the maxsyncop for the update
 556  *
 557  * Returns:
 558  *  0 - Device is available
 559  *  -ENODEV - Device is not available
 560  */
 561 static int nx842_OF_upd_status(struct nx842_devdata *devdata,
 562                                         struct property *prop) {
 563         int ret = 0;
 564         const char *status = (const char *)prop->value;
 565
 566         if (!strncmp(status, "okay", (size_t)prop->length)) {
 567                 devdata->status = AVAILABLE;
 568         } else {
 569                 /*
 570                  * Caller will log that the device is disabled, so only
 571                  * output if there is an unexpected status.
 572                  */
 573                 if (strncmp(status, "disabled", (size_t)prop->length)) {
 574                         dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
 575                                 __func__, status);
 576                 }
 577                 devdata->status = UNAVAILABLE;
 578                 ret = -ENODEV;
 579         }
 580
 581         return ret;
 582 }
 583
 584 /**
 585  * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
 586  *
 587  * Definition of the 'ibm,max-sg-len' OF property:
 588  *  This field indicates the maximum byte length of a scatter list
 589  *  for the platform facility. It is a single cell encoded as with encode-int.
 590  *
 591  * Example:
 592  *  # od -x ibm,max-sg-len
 593  *  0000000 0000 0ff0
 594  *
 595  *  In this example, the maximum byte length of a scatter list is
 596  *  0x0ff0 (4,080).
 597  *
 598  * @devdata - struct nx842_devdata to update
 599  * @prop - struct property point containing the maxsyncop for the update
 600  *
 601  * Returns:
 602  *  0 on success
 603  *  -EINVAL on failure
 604  */
 605 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
 606                                         struct property *prop) {
 607         int ret = 0;
 608         const unsigned int maxsglen = of_read_number(prop->value, 1);
 609
 610         if (prop->length != sizeof(maxsglen)) {
 611                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
 612                 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
 613                                 prop->length, sizeof(maxsglen));
 614                 ret = -EINVAL;
 615         } else {
 616                 devdata->max_sg_len = min_t(unsigned int,
 617                                             maxsglen, NX842_HW_PAGE_SIZE);
 618         }
 619
 620         return ret;
 621 }
 622
 623 /**
 624  * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
 625  *
 626  * Definition of the 'ibm,max-sync-cop' OF property:
 627  *  Two series of cells.  The first series of cells represents the maximums
 628  *  that can be synchronously compressed. The second series of cells
 629  *  represents the maximums that can be synchronously decompressed.
 630  *  1. The first cell in each series contains the count of the number of
 631  *     data length, scatter list elements pairs that follow – each being
 632  *     of the form
 633  *    a. One cell data byte length
 634  *    b. One cell total number of scatter list elements
 635  *
 636  * Example:
 637  *  # od -x ibm,max-sync-cop
 638  *  0000000 0000 0001 0000 1000 0000 01fe 0000 0001
 639  *  0000020 0000 1000 0000 01fe
 640  *
 641  *  In this example, compression supports 0x1000 (4,096) data byte length
 642  *  and 0x1fe (510) total scatter list elements.  Decompression supports
 643  *  0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
 644  *  elements.
 645  *
 646  * @devdata - struct nx842_devdata to update
 647  * @prop - struct property point containing the maxsyncop for the update
 648  *
 649  * Returns:
 650  *  0 on success
 651  *  -EINVAL on failure
 652  */
 653 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
 654                                         struct property *prop) {
 655         int ret = 0;
 656         unsigned int comp_data_limit, decomp_data_limit;
 657         unsigned int comp_sg_limit, decomp_sg_limit;
 658         const struct maxsynccop_t {
 659                 __be32 comp_elements;
 660                 __be32 comp_data_limit;
 661                 __be32 comp_sg_limit;
 662                 __be32 decomp_elements;
 663                 __be32 decomp_data_limit;
 664                 __be32 decomp_sg_limit;
 665         } *maxsynccop;
 666
 667         if (prop->length != sizeof(*maxsynccop)) {
 668                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
 669                 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
 670                                 sizeof(*maxsynccop));
 671                 ret = -EINVAL;
 672                 goto out;
 673         }
 674
 675         maxsynccop = (const struct maxsynccop_t *)prop->value;
 676         comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
 677         comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
 678         decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
 679         decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
 680
 681         /* Use one limit rather than separate limits for compression and
 682          * decompression. Set a maximum for this so as not to exceed the
 683          * size that the header can support and round the value down to
 684          * the hardware page size (4K) */
 685         devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
 686
 687         devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
 688                                         65536);
 689
 690         if (devdata->max_sync_size < 4096) {
 691                 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
 692                                 "less than the driver minimum, unable to use "
 693                                 "the hardware device\n",
 694                                 __func__, devdata->max_sync_size);
 695                 ret = -EINVAL;
 696                 goto out;
 697         }
 698
 699         nx842_pseries_constraints.maximum = devdata->max_sync_size;
 700
 701         devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
 702         if (devdata->max_sync_sg < 1) {
 703                 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
 704                                 "less than the driver minimum, unable to use "
 705                                 "the hardware device\n",
 706                                 __func__, devdata->max_sync_sg);
 707                 ret = -EINVAL;
 708                 goto out;
 709         }
 710
 711 out:
 712         return ret;
 713 }
 714
 715 /**
 716  *
 717  * nx842_OF_upd -- Handle OF properties updates for the device.
 718  *
 719  * Set all properties from the OF tree.  Optionally, a new property
 720  * can be provided by the @new_prop pointer to overwrite an existing value.
 721  * The device will remain disabled until all values are valid, this function
 722  * will return an error for updates unless all values are valid.
 723  *
 724  * @new_prop: If not NULL, this property is being updated.  If NULL, update
 725  *  all properties from the current values in the OF tree.
 726  *
 727  * Returns:
 728  *  0 - Success
 729  *  -ENOMEM - Could not allocate memory for new devdata structure
 730  *  -EINVAL - property value not found, new_prop is not a recognized
 731  *      property for the device or property value is not valid.
 732  *  -ENODEV - Device is not available
 733  */
 734 static int nx842_OF_upd(struct property *new_prop)
 735 {
 736         struct nx842_devdata *old_devdata = NULL;
 737         struct nx842_devdata *new_devdata = NULL;
 738         struct device_node *of_node = NULL;
 739         struct property *status = NULL;
 740         struct property *maxsglen = NULL;
 741         struct property *maxsyncop = NULL;
 742         int ret = 0;
 743         unsigned long flags;
 744
 745         spin_lock_irqsave(&devdata_mutex, flags);
 746         old_devdata = rcu_dereference_check(devdata,
 747                         lockdep_is_held(&devdata_mutex));
 748         if (old_devdata)
 749                 of_node = old_devdata->dev->of_node;
 750
 751         if (!old_devdata || !of_node) {
 752                 pr_err("%s: device is not available\n", __func__);
 753                 spin_unlock_irqrestore(&devdata_mutex, flags);
 754                 return -ENODEV;
 755         }
 756
 757         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 758         if (!new_devdata) {
 759                 dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__);
 760                 ret = -ENOMEM;
 761                 goto error_out;
 762         }
 763
 764         memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
 765         new_devdata->counters = old_devdata->counters;
 766
 767         /* Set ptrs for existing properties */
 768         status = of_find_property(of_node, "status", NULL);
 769         maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
 770         maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
 771         if (!status || !maxsglen || !maxsyncop) {
 772                 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
 773                 ret = -EINVAL;
 774                 goto error_out;
 775         }
 776
 777         /*
 778          * If this is a property update, there are only certain properties that
 779          * we care about. Bail if it isn't in the below list
 780          */
 781         if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
 782                          strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
 783                          strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
 784                 goto out;
 785
 786         /* Perform property updates */
 787         ret = nx842_OF_upd_status(new_devdata, status);
 788         if (ret)
 789                 goto error_out;
 790
 791         ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
 792         if (ret)
 793                 goto error_out;
 794
 795         ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
 796         if (ret)
 797                 goto error_out;
 798
 799 out:
 800         dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
 801                         __func__, new_devdata->max_sync_size,
 802                         old_devdata->max_sync_size);
 803         dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
 804                         __func__, new_devdata->max_sync_sg,
 805                         old_devdata->max_sync_sg);
 806         dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
 807                         __func__, new_devdata->max_sg_len,
 808                         old_devdata->max_sg_len);
 809
 810         rcu_assign_pointer(devdata, new_devdata);
 811         spin_unlock_irqrestore(&devdata_mutex, flags);
 812         synchronize_rcu();
 813         dev_set_drvdata(new_devdata->dev, new_devdata);
 814         kfree(old_devdata);
 815         return 0;
 816
 817 error_out:
 818         if (new_devdata) {
 819                 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
 820                 nx842_OF_set_defaults(new_devdata);
 821                 rcu_assign_pointer(devdata, new_devdata);
 822                 spin_unlock_irqrestore(&devdata_mutex, flags);
 823                 synchronize_rcu();
 824                 dev_set_drvdata(new_devdata->dev, new_devdata);
 825                 kfree(old_devdata);
 826         } else {
 827                 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
 828                 spin_unlock_irqrestore(&devdata_mutex, flags);
 829         }
 830
 831         if (!ret)
 832                 ret = -EINVAL;
 833         return ret;
 834 }
 835
 836 /**
 837  * nx842_OF_notifier - Process updates to OF properties for the device
 838  *
 839  * @np: notifier block
 840  * @action: notifier action
 841  * @update: struct pSeries_reconfig_prop_update pointer if action is
 842  *      PSERIES_UPDATE_PROPERTY
 843  *
 844  * Returns:
 845  *      NOTIFY_OK on success
 846  *      NOTIFY_BAD encoded with error number on failure, use
 847  *              notifier_to_errno() to decode this value
 848  */
 849 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
 850                              void *data)
 851 {
 852         struct of_reconfig_data *upd = data;
 853         struct nx842_devdata *local_devdata;
 854         struct device_node *node = NULL;
 855
 856         rcu_read_lock();
 857         local_devdata = rcu_dereference(devdata);
 858         if (local_devdata)
 859                 node = local_devdata->dev->of_node;
 860
 861         if (local_devdata &&
 862                         action == OF_RECONFIG_UPDATE_PROPERTY &&
 863                         !strcmp(upd->dn->name, node->name)) {
 864                 rcu_read_unlock();
 865                 nx842_OF_upd(upd->prop);
 866         } else
 867                 rcu_read_unlock();
 868
 869         return NOTIFY_OK;
 870 }
 871
 872 static struct notifier_block nx842_of_nb = {
 873         .notifier_call = nx842_OF_notifier,
 874 };
 875
 876 #define nx842_counter_read(_name)                                       \
 877 static ssize_t nx842_##_name##_show(struct device *dev,         \
 878                 struct device_attribute *attr,                          \
 879                 char *buf) {                                            \
 880         struct nx842_devdata *local_devdata;                    \
 881         int p = 0;                                                      \
 882         rcu_read_lock();                                                \
 883         local_devdata = rcu_dereference(devdata);                       \
 884         if (local_devdata)                                              \
 885                 p = snprintf(buf, PAGE_SIZE, "%ld\n",                   \
 886                        atomic64_read(&local_devdata->counters->_name)); \
 887         rcu_read_unlock();                                              \
 888         return p;                                                       \
 889 }
 890
 891 #define NX842DEV_COUNTER_ATTR_RO(_name)                                 \
 892         nx842_counter_read(_name);                                      \
 893         static struct device_attribute dev_attr_##_name = __ATTR(_name, \
 894                                                 0444,                   \
 895                                                 nx842_##_name##_show,\
 896                                                 NULL);
 897
 898 NX842DEV_COUNTER_ATTR_RO(comp_complete);
 899 NX842DEV_COUNTER_ATTR_RO(comp_failed);
 900 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
 901 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
 902 NX842DEV_COUNTER_ATTR_RO(swdecomp);
 903
 904 static ssize_t nx842_timehist_show(struct device *,
 905                 struct device_attribute *, char *);
 906
 907 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
 908                 nx842_timehist_show, NULL);
 909 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
 910                 0444, nx842_timehist_show, NULL);
 911
 912 static ssize_t nx842_timehist_show(struct device *dev,
 913                 struct device_attribute *attr, char *buf) {
 914         char *p = buf;
 915         struct nx842_devdata *local_devdata;
 916         atomic64_t *times;
 917         int bytes_remain = PAGE_SIZE;
 918         int bytes;
 919         int i;
 920
 921         rcu_read_lock();
 922         local_devdata = rcu_dereference(devdata);
 923         if (!local_devdata) {
 924                 rcu_read_unlock();
 925                 return 0;
 926         }
 927
 928         if (attr == &dev_attr_comp_times)
 929                 times = local_devdata->counters->comp_times;
 930         else if (attr == &dev_attr_decomp_times)
 931                 times = local_devdata->counters->decomp_times;
 932         else {
 933                 rcu_read_unlock();
 934                 return 0;
 935         }
 936
 937         for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
 938                 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
 939                                i ? (2<<(i-1)) : 0, (2<<i)-1,
 940                                atomic64_read(&times[i]));
 941                 bytes_remain -= bytes;
 942                 p += bytes;
 943         }
 944         /* The last bucket holds everything over
 945          * 2<<(NX842_HIST_SLOTS - 2) us */
 946         bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
 947                         2<<(NX842_HIST_SLOTS - 2),
 948                         atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
 949         p += bytes;
 950
 951         rcu_read_unlock();
 952         return p - buf;
 953 }
 954
 955 static struct attribute *nx842_sysfs_entries[] = {
 956         &dev_attr_comp_complete.attr,
 957         &dev_attr_comp_failed.attr,
 958         &dev_attr_decomp_complete.attr,
 959         &dev_attr_decomp_failed.attr,
 960         &dev_attr_swdecomp.attr,
 961         &dev_attr_comp_times.attr,
 962         &dev_attr_decomp_times.attr,
 963         NULL,
 964 };
 965
 966 static struct attribute_group nx842_attribute_group = {
 967         .name = NULL,           /* put in device directory */
 968         .attrs = nx842_sysfs_entries,
 969 };
 970
 971 static struct nx842_driver nx842_pseries_driver = {
 972         .name =         KBUILD_MODNAME,
 973         .owner =        THIS_MODULE,
 974         .workmem_size = sizeof(struct nx842_workmem),
 975         .constraints =  &nx842_pseries_constraints,
 976         .compress =     nx842_pseries_compress,
 977         .decompress =   nx842_pseries_decompress,
 978 };
 979
 980 static int __init nx842_probe(struct vio_dev *viodev,
 981                                   const struct vio_device_id *id)
 982 {
 983         struct nx842_devdata *old_devdata, *new_devdata = NULL;
 984         unsigned long flags;
 985         int ret = 0;
 986
 987         spin_lock_irqsave(&devdata_mutex, flags);
 988         old_devdata = rcu_dereference_check(devdata,
 989                         lockdep_is_held(&devdata_mutex));
 990
 991         if (old_devdata && old_devdata->vdev != NULL) {
 992                 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
 993                 ret = -1;
 994                 goto error_unlock;
 995         }
 996
 997         dev_set_drvdata(&viodev->dev, NULL);
 998
 999         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
1000         if (!new_devdata) {
1001                 dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__);
1002                 ret = -ENOMEM;
1003                 goto error_unlock;
1004         }
1005
1006         new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
1007                         GFP_NOFS);
1008         if (!new_devdata->counters) {
1009                 dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__);
1010                 ret = -ENOMEM;
1011                 goto error_unlock;
1012         }
1013
1014         new_devdata->vdev = viodev;
1015         new_devdata->dev = &viodev->dev;
1016         nx842_OF_set_defaults(new_devdata);
1017
1018         rcu_assign_pointer(devdata, new_devdata);
1019         spin_unlock_irqrestore(&devdata_mutex, flags);
1020         synchronize_rcu();
1021         kfree(old_devdata);
1022
1023         of_reconfig_notifier_register(&nx842_of_nb);
1024
1025         ret = nx842_OF_upd(NULL);
1026         if (ret && ret != -ENODEV) {
1027                 dev_err(&viodev->dev, "could not parse device tree. %d\n", ret);
1028                 ret = -1;
1029                 goto error;
1030         }
1031
1032         rcu_read_lock();
1033         dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1034         rcu_read_unlock();
1035
1036         if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1037                 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1038                 ret = -1;
1039                 goto error;
1040         }
1041
1042         return 0;
1043
1044 error_unlock:
1045         spin_unlock_irqrestore(&devdata_mutex, flags);
1046         if (new_devdata)
1047                 kfree(new_devdata->counters);
1048         kfree(new_devdata);
1049 error:
1050         return ret;
1051 }
1052
1053 static int __exit nx842_remove(struct vio_dev *viodev)
1054 {
1055         struct nx842_devdata *old_devdata;
1056         unsigned long flags;
1057
1058         pr_info("Removing IBM Power 842 compression device\n");
1059         sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1060
1061         spin_lock_irqsave(&devdata_mutex, flags);
1062         old_devdata = rcu_dereference_check(devdata,
1063                         lockdep_is_held(&devdata_mutex));
1064         of_reconfig_notifier_unregister(&nx842_of_nb);
1065         RCU_INIT_POINTER(devdata, NULL);
1066         spin_unlock_irqrestore(&devdata_mutex, flags);
1067         synchronize_rcu();
1068         dev_set_drvdata(&viodev->dev, NULL);
1069         if (old_devdata)
1070                 kfree(old_devdata->counters);
1071         kfree(old_devdata);
1072
1073         return 0;
1074 }
1075
1076 static struct vio_device_id nx842_vio_driver_ids[] = {
1077         {"ibm,compression-v1", "ibm,compression"},
1078         {"", ""},
1079 };
1080
1081 static struct vio_driver nx842_vio_driver = {
1082         .name = KBUILD_MODNAME,
1083         .probe = nx842_probe,
1084         .remove = __exit_p(nx842_remove),
1085         .get_desired_dma = nx842_get_desired_dma,
1086         .id_table = nx842_vio_driver_ids,
1087 };
1088
1089 static int __init nx842_pseries_init(void)
1090 {
1091         struct nx842_devdata *new_devdata;
1092         int ret;
1093
1094         pr_info("Registering IBM Power 842 compression driver\n");
1095
1096         if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
1097                 return -ENODEV;
1098
1099         RCU_INIT_POINTER(devdata, NULL);
1100         new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1101         if (!new_devdata) {
1102                 pr_err("Could not allocate memory for device data\n");
1103                 return -ENOMEM;
1104         }
1105         new_devdata->status = UNAVAILABLE;
1106         RCU_INIT_POINTER(devdata, new_devdata);
1107
1108         ret = vio_register_driver(&nx842_vio_driver);
1109         if (ret) {
1110                 pr_err("Could not register VIO driver %d\n", ret);
1111
1112                 kfree(new_devdata);
1113                 return ret;
1114         }
1115
1116         if (!nx842_platform_driver_set(&nx842_pseries_driver)) {
1117                 vio_unregister_driver(&nx842_vio_driver);
1118                 kfree(new_devdata);
1119                 return -EEXIST;
1120         }
1121
1122         return 0;
1123 }
1124
1125 module_init(nx842_pseries_init);
1126
1127 static void __exit nx842_pseries_exit(void)
1128 {
1129         struct nx842_devdata *old_devdata;
1130         unsigned long flags;
1131
1132         pr_info("Exiting IBM Power 842 compression driver\n");
1133         nx842_platform_driver_unset(&nx842_pseries_driver);
1134         spin_lock_irqsave(&devdata_mutex, flags);
1135         old_devdata = rcu_dereference_check(devdata,
1136                         lockdep_is_held(&devdata_mutex));
1137         RCU_INIT_POINTER(devdata, NULL);
1138         spin_unlock_irqrestore(&devdata_mutex, flags);
1139         synchronize_rcu();
1140         if (old_devdata && old_devdata->dev)
1141                 dev_set_drvdata(old_devdata->dev, NULL);
1142         kfree(old_devdata);
1143         vio_unregister_driver(&nx842_vio_driver);
1144 }
1145
1146 module_exit(nx842_pseries_exit);
1147