drivers/block/nvme-scsi.c

   1 /*
   2  * NVM Express device driver
   3  * Copyright (c) 2011-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 /*
  16  * Refer to the SCSI-NVMe Translation spec for details on how
  17  * each command is translated.
  18  */
  19
  20 #include <linux/nvme.h>
  21 #include <linux/bio.h>
  22 #include <linux/bitops.h>
  23 #include <linux/blkdev.h>
  24 #include <linux/compat.h>
  25 #include <linux/delay.h>
  26 #include <linux/errno.h>
  27 #include <linux/fs.h>
  28 #include <linux/genhd.h>
  29 #include <linux/idr.h>
  30 #include <linux/init.h>
  31 #include <linux/interrupt.h>
  32 #include <linux/io.h>
  33 #include <linux/kdev_t.h>
  34 #include <linux/kthread.h>
  35 #include <linux/kernel.h>
  36 #include <linux/mm.h>
  37 #include <linux/module.h>
  38 #include <linux/moduleparam.h>
  39 #include <linux/pci.h>
  40 #include <linux/poison.h>
  41 #include <linux/sched.h>
  42 #include <linux/slab.h>
  43 #include <linux/types.h>
  44 #include <scsi/sg.h>
  45 #include <scsi/scsi.h>
  46
  47
  48 static int sg_version_num = 30534;      /* 2 digits for each component */
  49
  50 #define SNTI_TRANSLATION_SUCCESS                        0
  51 #define SNTI_INTERNAL_ERROR                             1
  52
  53 /* VPD Page Codes */
  54 #define VPD_SUPPORTED_PAGES                             0x00
  55 #define VPD_SERIAL_NUMBER                               0x80
  56 #define VPD_DEVICE_IDENTIFIERS                          0x83
  57 #define VPD_EXTENDED_INQUIRY                            0x86
  58 #define VPD_BLOCK_LIMITS                                0xB0
  59 #define VPD_BLOCK_DEV_CHARACTERISTICS                   0xB1
  60
  61 /* CDB offsets */
  62 #define REPORT_LUNS_CDB_ALLOC_LENGTH_OFFSET             6
  63 #define REPORT_LUNS_SR_OFFSET                           2
  64 #define READ_CAP_16_CDB_ALLOC_LENGTH_OFFSET             10
  65 #define REQUEST_SENSE_CDB_ALLOC_LENGTH_OFFSET           4
  66 #define REQUEST_SENSE_DESC_OFFSET                       1
  67 #define REQUEST_SENSE_DESC_MASK                         0x01
  68 #define DESCRIPTOR_FORMAT_SENSE_DATA_TYPE               1
  69 #define INQUIRY_EVPD_BYTE_OFFSET                        1
  70 #define INQUIRY_PAGE_CODE_BYTE_OFFSET                   2
  71 #define INQUIRY_EVPD_BIT_MASK                           1
  72 #define INQUIRY_CDB_ALLOCATION_LENGTH_OFFSET            3
  73 #define START_STOP_UNIT_CDB_IMMED_OFFSET                1
  74 #define START_STOP_UNIT_CDB_IMMED_MASK                  0x1
  75 #define START_STOP_UNIT_CDB_POWER_COND_MOD_OFFSET       3
  76 #define START_STOP_UNIT_CDB_POWER_COND_MOD_MASK         0xF
  77 #define START_STOP_UNIT_CDB_POWER_COND_OFFSET           4
  78 #define START_STOP_UNIT_CDB_POWER_COND_MASK             0xF0
  79 #define START_STOP_UNIT_CDB_NO_FLUSH_OFFSET             4
  80 #define START_STOP_UNIT_CDB_NO_FLUSH_MASK               0x4
  81 #define START_STOP_UNIT_CDB_START_OFFSET                4
  82 #define START_STOP_UNIT_CDB_START_MASK                  0x1
  83 #define WRITE_BUFFER_CDB_MODE_OFFSET                    1
  84 #define WRITE_BUFFER_CDB_MODE_MASK                      0x1F
  85 #define WRITE_BUFFER_CDB_BUFFER_ID_OFFSET               2
  86 #define WRITE_BUFFER_CDB_BUFFER_OFFSET_OFFSET           3
  87 #define WRITE_BUFFER_CDB_PARM_LIST_LENGTH_OFFSET        6
  88 #define FORMAT_UNIT_CDB_FORMAT_PROT_INFO_OFFSET         1
  89 #define FORMAT_UNIT_CDB_FORMAT_PROT_INFO_MASK           0xC0
  90 #define FORMAT_UNIT_CDB_FORMAT_PROT_INFO_SHIFT          6
  91 #define FORMAT_UNIT_CDB_LONG_LIST_OFFSET                1
  92 #define FORMAT_UNIT_CDB_LONG_LIST_MASK                  0x20
  93 #define FORMAT_UNIT_CDB_FORMAT_DATA_OFFSET              1
  94 #define FORMAT_UNIT_CDB_FORMAT_DATA_MASK                0x10
  95 #define FORMAT_UNIT_SHORT_PARM_LIST_LEN                 4
  96 #define FORMAT_UNIT_LONG_PARM_LIST_LEN                  8
  97 #define FORMAT_UNIT_PROT_INT_OFFSET                     3
  98 #define FORMAT_UNIT_PROT_FIELD_USAGE_OFFSET             0
  99 #define FORMAT_UNIT_PROT_FIELD_USAGE_MASK               0x07
 100 #define UNMAP_CDB_PARAM_LIST_LENGTH_OFFSET              7
 101
 102 /* Misc. defines */
 103 #define NIBBLE_SHIFT                                    4
 104 #define FIXED_SENSE_DATA                                0x70
 105 #define DESC_FORMAT_SENSE_DATA                          0x72
 106 #define FIXED_SENSE_DATA_ADD_LENGTH                     10
 107 #define LUN_ENTRY_SIZE                                  8
 108 #define LUN_DATA_HEADER_SIZE                            8
 109 #define ALL_LUNS_RETURNED                               0x02
 110 #define ALL_WELL_KNOWN_LUNS_RETURNED                    0x01
 111 #define RESTRICTED_LUNS_RETURNED                        0x00
 112 #define NVME_POWER_STATE_START_VALID                    0x00
 113 #define NVME_POWER_STATE_ACTIVE                         0x01
 114 #define NVME_POWER_STATE_IDLE                           0x02
 115 #define NVME_POWER_STATE_STANDBY                        0x03
 116 #define NVME_POWER_STATE_LU_CONTROL                     0x07
 117 #define POWER_STATE_0                                   0
 118 #define POWER_STATE_1                                   1
 119 #define POWER_STATE_2                                   2
 120 #define POWER_STATE_3                                   3
 121 #define DOWNLOAD_SAVE_ACTIVATE                          0x05
 122 #define DOWNLOAD_SAVE_DEFER_ACTIVATE                    0x0E
 123 #define ACTIVATE_DEFERRED_MICROCODE                     0x0F
 124 #define FORMAT_UNIT_IMMED_MASK                          0x2
 125 #define FORMAT_UNIT_IMMED_OFFSET                        1
 126 #define KELVIN_TEMP_FACTOR                              273
 127 #define FIXED_FMT_SENSE_DATA_SIZE                       18
 128 #define DESC_FMT_SENSE_DATA_SIZE                        8
 129
 130 /* SCSI/NVMe defines and bit masks */
 131 #define INQ_STANDARD_INQUIRY_PAGE                       0x00
 132 #define INQ_SUPPORTED_VPD_PAGES_PAGE                    0x00
 133 #define INQ_UNIT_SERIAL_NUMBER_PAGE                     0x80
 134 #define INQ_DEVICE_IDENTIFICATION_PAGE                  0x83
 135 #define INQ_EXTENDED_INQUIRY_DATA_PAGE                  0x86
 136 #define INQ_BDEV_LIMITS_PAGE                            0xB0
 137 #define INQ_BDEV_CHARACTERISTICS_PAGE                   0xB1
 138 #define INQ_SERIAL_NUMBER_LENGTH                        0x14
 139 #define INQ_NUM_SUPPORTED_VPD_PAGES                     6
 140 #define VERSION_SPC_4                                   0x06
 141 #define ACA_UNSUPPORTED                                 0
 142 #define STANDARD_INQUIRY_LENGTH                         36
 143 #define ADDITIONAL_STD_INQ_LENGTH                       31
 144 #define EXTENDED_INQUIRY_DATA_PAGE_LENGTH               0x3C
 145 #define RESERVED_FIELD                                  0
 146
 147 /* SCSI READ/WRITE Defines */
 148 #define IO_CDB_WP_MASK                                  0xE0
 149 #define IO_CDB_WP_SHIFT                                 5
 150 #define IO_CDB_FUA_MASK                                 0x8
 151 #define IO_6_CDB_LBA_OFFSET                             0
 152 #define IO_6_CDB_LBA_MASK                               0x001FFFFF
 153 #define IO_6_CDB_TX_LEN_OFFSET                          4
 154 #define IO_6_DEFAULT_TX_LEN                             256
 155 #define IO_10_CDB_LBA_OFFSET                            2
 156 #define IO_10_CDB_TX_LEN_OFFSET                         7
 157 #define IO_10_CDB_WP_OFFSET                             1
 158 #define IO_10_CDB_FUA_OFFSET                            1
 159 #define IO_12_CDB_LBA_OFFSET                            2
 160 #define IO_12_CDB_TX_LEN_OFFSET                         6
 161 #define IO_12_CDB_WP_OFFSET                             1
 162 #define IO_12_CDB_FUA_OFFSET                            1
 163 #define IO_16_CDB_FUA_OFFSET                            1
 164 #define IO_16_CDB_WP_OFFSET                             1
 165 #define IO_16_CDB_LBA_OFFSET                            2
 166 #define IO_16_CDB_TX_LEN_OFFSET                         10
 167
 168 /* Mode Sense/Select defines */
 169 #define MODE_PAGE_INFO_EXCEP                            0x1C
 170 #define MODE_PAGE_CACHING                               0x08
 171 #define MODE_PAGE_CONTROL                               0x0A
 172 #define MODE_PAGE_POWER_CONDITION                       0x1A
 173 #define MODE_PAGE_RETURN_ALL                            0x3F
 174 #define MODE_PAGE_BLK_DES_LEN                           0x08
 175 #define MODE_PAGE_LLBAA_BLK_DES_LEN                     0x10
 176 #define MODE_PAGE_CACHING_LEN                           0x14
 177 #define MODE_PAGE_CONTROL_LEN                           0x0C
 178 #define MODE_PAGE_POW_CND_LEN                           0x28
 179 #define MODE_PAGE_INF_EXC_LEN                           0x0C
 180 #define MODE_PAGE_ALL_LEN                               0x54
 181 #define MODE_SENSE6_MPH_SIZE                            4
 182 #define MODE_SENSE6_ALLOC_LEN_OFFSET                    4
 183 #define MODE_SENSE_PAGE_CONTROL_OFFSET                  2
 184 #define MODE_SENSE_PAGE_CONTROL_MASK                    0xC0
 185 #define MODE_SENSE_PAGE_CODE_OFFSET                     2
 186 #define MODE_SENSE_PAGE_CODE_MASK                       0x3F
 187 #define MODE_SENSE_LLBAA_OFFSET                         1
 188 #define MODE_SENSE_LLBAA_MASK                           0x10
 189 #define MODE_SENSE_LLBAA_SHIFT                          4
 190 #define MODE_SENSE_DBD_OFFSET                           1
 191 #define MODE_SENSE_DBD_MASK                             8
 192 #define MODE_SENSE_DBD_SHIFT                            3
 193 #define MODE_SENSE10_MPH_SIZE                           8
 194 #define MODE_SENSE10_ALLOC_LEN_OFFSET                   7
 195 #define MODE_SELECT_CDB_PAGE_FORMAT_OFFSET              1
 196 #define MODE_SELECT_CDB_SAVE_PAGES_OFFSET               1
 197 #define MODE_SELECT_6_CDB_PARAM_LIST_LENGTH_OFFSET      4
 198 #define MODE_SELECT_10_CDB_PARAM_LIST_LENGTH_OFFSET     7
 199 #define MODE_SELECT_CDB_PAGE_FORMAT_MASK                0x10
 200 #define MODE_SELECT_CDB_SAVE_PAGES_MASK                 0x1
 201 #define MODE_SELECT_6_BD_OFFSET                         3
 202 #define MODE_SELECT_10_BD_OFFSET                        6
 203 #define MODE_SELECT_10_LLBAA_OFFSET                     4
 204 #define MODE_SELECT_10_LLBAA_MASK                       1
 205 #define MODE_SELECT_6_MPH_SIZE                          4
 206 #define MODE_SELECT_10_MPH_SIZE                         8
 207 #define CACHING_MODE_PAGE_WCE_MASK                      0x04
 208 #define MODE_SENSE_BLK_DESC_ENABLED                     0
 209 #define MODE_SENSE_BLK_DESC_COUNT                       1
 210 #define MODE_SELECT_PAGE_CODE_MASK                      0x3F
 211 #define SHORT_DESC_BLOCK                                8
 212 #define LONG_DESC_BLOCK                                 16
 213 #define MODE_PAGE_POW_CND_LEN_FIELD                     0x26
 214 #define MODE_PAGE_INF_EXC_LEN_FIELD                     0x0A
 215 #define MODE_PAGE_CACHING_LEN_FIELD                     0x12
 216 #define MODE_PAGE_CONTROL_LEN_FIELD                     0x0A
 217 #define MODE_SENSE_PC_CURRENT_VALUES                    0
 218
 219 /* Log Sense defines */
 220 #define LOG_PAGE_SUPPORTED_LOG_PAGES_PAGE               0x00
 221 #define LOG_PAGE_SUPPORTED_LOG_PAGES_LENGTH             0x07
 222 #define LOG_PAGE_INFORMATIONAL_EXCEPTIONS_PAGE          0x2F
 223 #define LOG_PAGE_TEMPERATURE_PAGE                       0x0D
 224 #define LOG_SENSE_CDB_SP_OFFSET                         1
 225 #define LOG_SENSE_CDB_SP_NOT_ENABLED                    0
 226 #define LOG_SENSE_CDB_PC_OFFSET                         2
 227 #define LOG_SENSE_CDB_PC_MASK                           0xC0
 228 #define LOG_SENSE_CDB_PC_SHIFT                          6
 229 #define LOG_SENSE_CDB_PC_CUMULATIVE_VALUES              1
 230 #define LOG_SENSE_CDB_PAGE_CODE_MASK                    0x3F
 231 #define LOG_SENSE_CDB_ALLOC_LENGTH_OFFSET               7
 232 #define REMAINING_INFO_EXCP_PAGE_LENGTH                 0x8
 233 #define LOG_INFO_EXCP_PAGE_LENGTH                       0xC
 234 #define REMAINING_TEMP_PAGE_LENGTH                      0xC
 235 #define LOG_TEMP_PAGE_LENGTH                            0x10
 236 #define LOG_TEMP_UNKNOWN                                0xFF
 237 #define SUPPORTED_LOG_PAGES_PAGE_LENGTH                 0x3
 238
 239 /* Read Capacity defines */
 240 #define READ_CAP_10_RESP_SIZE                           8
 241 #define READ_CAP_16_RESP_SIZE                           32
 242
 243 /* NVMe Namespace and Command Defines */
 244 #define BYTES_TO_DWORDS                                 4
 245 #define NVME_MAX_FIRMWARE_SLOT                          7
 246
 247 /* Report LUNs defines */
 248 #define REPORT_LUNS_FIRST_LUN_OFFSET                    8
 249
 250 /* SCSI ADDITIONAL SENSE Codes */
 251
 252 #define SCSI_ASC_NO_SENSE                               0x00
 253 #define SCSI_ASC_PERIPHERAL_DEV_WRITE_FAULT             0x03
 254 #define SCSI_ASC_LUN_NOT_READY                          0x04
 255 #define SCSI_ASC_WARNING                                0x0B
 256 #define SCSI_ASC_LOG_BLOCK_GUARD_CHECK_FAILED           0x10
 257 #define SCSI_ASC_LOG_BLOCK_APPTAG_CHECK_FAILED          0x10
 258 #define SCSI_ASC_LOG_BLOCK_REFTAG_CHECK_FAILED          0x10
 259 #define SCSI_ASC_UNRECOVERED_READ_ERROR                 0x11
 260 #define SCSI_ASC_MISCOMPARE_DURING_VERIFY               0x1D
 261 #define SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID           0x20
 262 #define SCSI_ASC_ILLEGAL_COMMAND                        0x20
 263 #define SCSI_ASC_ILLEGAL_BLOCK                          0x21
 264 #define SCSI_ASC_INVALID_CDB                            0x24
 265 #define SCSI_ASC_INVALID_LUN                            0x25
 266 #define SCSI_ASC_INVALID_PARAMETER                      0x26
 267 #define SCSI_ASC_FORMAT_COMMAND_FAILED                  0x31
 268 #define SCSI_ASC_INTERNAL_TARGET_FAILURE                0x44
 269
 270 /* SCSI ADDITIONAL SENSE Code Qualifiers */
 271
 272 #define SCSI_ASCQ_CAUSE_NOT_REPORTABLE                  0x00
 273 #define SCSI_ASCQ_FORMAT_COMMAND_FAILED                 0x01
 274 #define SCSI_ASCQ_LOG_BLOCK_GUARD_CHECK_FAILED          0x01
 275 #define SCSI_ASCQ_LOG_BLOCK_APPTAG_CHECK_FAILED         0x02
 276 #define SCSI_ASCQ_LOG_BLOCK_REFTAG_CHECK_FAILED         0x03
 277 #define SCSI_ASCQ_FORMAT_IN_PROGRESS                    0x04
 278 #define SCSI_ASCQ_POWER_LOSS_EXPECTED                   0x08
 279 #define SCSI_ASCQ_INVALID_LUN_ID                        0x09
 280
 281 /**
 282  * DEVICE_SPECIFIC_PARAMETER in mode parameter header (see sbc2r16) to
 283  * enable DPOFUA support type 0x10 value.
 284  */
 285 #define DEVICE_SPECIFIC_PARAMETER                       0
 286 #define VPD_ID_DESCRIPTOR_LENGTH sizeof(VPD_IDENTIFICATION_DESCRIPTOR)
 287
 288 /* MACROs to extract information from CDBs */
 289
 290 #define GET_OPCODE(cdb)         cdb[0]
 291
 292 #define GET_U8_FROM_CDB(cdb, index) (cdb[index] << 0)
 293
 294 #define GET_U16_FROM_CDB(cdb, index) ((cdb[index] << 8) | (cdb[index + 1] << 0))
 295
 296 #define GET_U24_FROM_CDB(cdb, index) ((cdb[index] << 16) | \
 297 (cdb[index + 1] <<  8) | \
 298 (cdb[index + 2] <<  0))
 299
 300 #define GET_U32_FROM_CDB(cdb, index) ((cdb[index] << 24) | \
 301 (cdb[index + 1] << 16) | \
 302 (cdb[index + 2] <<  8) | \
 303 (cdb[index + 3] <<  0))
 304
 305 #define GET_U64_FROM_CDB(cdb, index) ((((u64)cdb[index]) << 56) | \
 306 (((u64)cdb[index + 1]) << 48) | \
 307 (((u64)cdb[index + 2]) << 40) | \
 308 (((u64)cdb[index + 3]) << 32) | \
 309 (((u64)cdb[index + 4]) << 24) | \
 310 (((u64)cdb[index + 5]) << 16) | \
 311 (((u64)cdb[index + 6]) <<  8) | \
 312 (((u64)cdb[index + 7]) <<  0))
 313
 314 /* Inquiry Helper Macros */
 315 #define GET_INQ_EVPD_BIT(cdb) \
 316 ((GET_U8_FROM_CDB(cdb, INQUIRY_EVPD_BYTE_OFFSET) &              \
 317 INQUIRY_EVPD_BIT_MASK) ? 1 : 0)
 318
 319 #define GET_INQ_PAGE_CODE(cdb)                                  \
 320 (GET_U8_FROM_CDB(cdb, INQUIRY_PAGE_CODE_BYTE_OFFSET))
 321
 322 #define GET_INQ_ALLOC_LENGTH(cdb)                               \
 323 (GET_U16_FROM_CDB(cdb, INQUIRY_CDB_ALLOCATION_LENGTH_OFFSET))
 324
 325 /* Report LUNs Helper Macros */
 326 #define GET_REPORT_LUNS_ALLOC_LENGTH(cdb)                       \
 327 (GET_U32_FROM_CDB(cdb, REPORT_LUNS_CDB_ALLOC_LENGTH_OFFSET))
 328
 329 /* Read Capacity Helper Macros */
 330 #define GET_READ_CAP_16_ALLOC_LENGTH(cdb)                       \
 331 (GET_U32_FROM_CDB(cdb, READ_CAP_16_CDB_ALLOC_LENGTH_OFFSET))
 332
 333 #define IS_READ_CAP_16(cdb)                                     \
 334 ((cdb[0] == SERVICE_ACTION_IN_16 && cdb[1] == SAI_READ_CAPACITY_16) ? 1 : 0)
 335
 336 /* Request Sense Helper Macros */
 337 #define GET_REQUEST_SENSE_ALLOC_LENGTH(cdb)                     \
 338 (GET_U8_FROM_CDB(cdb, REQUEST_SENSE_CDB_ALLOC_LENGTH_OFFSET))
 339
 340 /* Mode Sense Helper Macros */
 341 #define GET_MODE_SENSE_DBD(cdb)                                 \
 342 ((GET_U8_FROM_CDB(cdb, MODE_SENSE_DBD_OFFSET) & MODE_SENSE_DBD_MASK) >> \
 343 MODE_SENSE_DBD_SHIFT)
 344
 345 #define GET_MODE_SENSE_LLBAA(cdb)                               \
 346 ((GET_U8_FROM_CDB(cdb, MODE_SENSE_LLBAA_OFFSET) &               \
 347 MODE_SENSE_LLBAA_MASK) >> MODE_SENSE_LLBAA_SHIFT)
 348
 349 #define GET_MODE_SENSE_MPH_SIZE(cdb10)                          \
 350 (cdb10 ? MODE_SENSE10_MPH_SIZE : MODE_SENSE6_MPH_SIZE)
 351
 352
 353 /* Struct to gather data that needs to be extracted from a SCSI CDB.
 354    Not conforming to any particular CDB variant, but compatible with all. */
 355
 356 struct nvme_trans_io_cdb {
 357         u8 fua;
 358         u8 prot_info;
 359         u64 lba;
 360         u32 xfer_len;
 361 };
 362
 363
 364 /* Internal Helper Functions */
 365
 366
 367 /* Copy data to userspace memory */
 368
 369 static int nvme_trans_copy_to_user(struct sg_io_hdr *hdr, void *from,
 370                                                                 unsigned long n)
 371 {
 372         int res = SNTI_TRANSLATION_SUCCESS;
 373         unsigned long not_copied;
 374         int i;
 375         void *index = from;
 376         size_t remaining = n;
 377         size_t xfer_len;
 378
 379         if (hdr->iovec_count > 0) {
 380                 struct sg_iovec sgl;
 381
 382                 for (i = 0; i < hdr->iovec_count; i++) {
 383                         not_copied = copy_from_user(&sgl, hdr->dxferp +
 384                                                 i * sizeof(struct sg_iovec),
 385                                                 sizeof(struct sg_iovec));
 386                         if (not_copied)
 387                                 return -EFAULT;
 388                         xfer_len = min(remaining, sgl.iov_len);
 389                         not_copied = copy_to_user(sgl.iov_base, index,
 390                                                                 xfer_len);
 391                         if (not_copied) {
 392                                 res = -EFAULT;
 393                                 break;
 394                         }
 395                         index += xfer_len;
 396                         remaining -= xfer_len;
 397                         if (remaining == 0)
 398                                 break;
 399                 }
 400                 return res;
 401         }
 402         not_copied = copy_to_user(hdr->dxferp, from, n);
 403         if (not_copied)
 404                 res = -EFAULT;
 405         return res;
 406 }
 407
 408 /* Copy data from userspace memory */
 409
 410 static int nvme_trans_copy_from_user(struct sg_io_hdr *hdr, void *to,
 411                                                                 unsigned long n)
 412 {
 413         int res = SNTI_TRANSLATION_SUCCESS;
 414         unsigned long not_copied;
 415         int i;
 416         void *index = to;
 417         size_t remaining = n;
 418         size_t xfer_len;
 419
 420         if (hdr->iovec_count > 0) {
 421                 struct sg_iovec sgl;
 422
 423                 for (i = 0; i < hdr->iovec_count; i++) {
 424                         not_copied = copy_from_user(&sgl, hdr->dxferp +
 425                                                 i * sizeof(struct sg_iovec),
 426                                                 sizeof(struct sg_iovec));
 427                         if (not_copied)
 428                                 return -EFAULT;
 429                         xfer_len = min(remaining, sgl.iov_len);
 430                         not_copied = copy_from_user(index, sgl.iov_base,
 431                                                                 xfer_len);
 432                         if (not_copied) {
 433                                 res = -EFAULT;
 434                                 break;
 435                         }
 436                         index += xfer_len;
 437                         remaining -= xfer_len;
 438                         if (remaining == 0)
 439                                 break;
 440                 }
 441                 return res;
 442         }
 443
 444         not_copied = copy_from_user(to, hdr->dxferp, n);
 445         if (not_copied)
 446                 res = -EFAULT;
 447         return res;
 448 }
 449
 450 /* Status/Sense Buffer Writeback */
 451
 452 static int nvme_trans_completion(struct sg_io_hdr *hdr, u8 status, u8 sense_key,
 453                                  u8 asc, u8 ascq)
 454 {
 455         int res = SNTI_TRANSLATION_SUCCESS;
 456         u8 xfer_len;
 457         u8 resp[DESC_FMT_SENSE_DATA_SIZE];
 458
 459         if (scsi_status_is_good(status)) {
 460                 hdr->status = SAM_STAT_GOOD;
 461                 hdr->masked_status = GOOD;
 462                 hdr->host_status = DID_OK;
 463                 hdr->driver_status = DRIVER_OK;
 464                 hdr->sb_len_wr = 0;
 465         } else {
 466                 hdr->status = status;
 467                 hdr->masked_status = status >> 1;
 468                 hdr->host_status = DID_OK;
 469                 hdr->driver_status = DRIVER_OK;
 470
 471                 memset(resp, 0, DESC_FMT_SENSE_DATA_SIZE);
 472                 resp[0] = DESC_FORMAT_SENSE_DATA;
 473                 resp[1] = sense_key;
 474                 resp[2] = asc;
 475                 resp[3] = ascq;
 476
 477                 xfer_len = min_t(u8, hdr->mx_sb_len, DESC_FMT_SENSE_DATA_SIZE);
 478                 hdr->sb_len_wr = xfer_len;
 479                 if (copy_to_user(hdr->sbp, resp, xfer_len) > 0)
 480                         res = -EFAULT;
 481         }
 482
 483         return res;
 484 }
 485
 486 static int nvme_trans_status_code(struct sg_io_hdr *hdr, int nvme_sc)
 487 {
 488         u8 status, sense_key, asc, ascq;
 489         int res = SNTI_TRANSLATION_SUCCESS;
 490
 491         /* For non-nvme (Linux) errors, simply return the error code */
 492         if (nvme_sc < 0)
 493                 return nvme_sc;
 494
 495         /* Mask DNR, More, and reserved fields */
 496         nvme_sc &= 0x7FF;
 497
 498         switch (nvme_sc) {
 499         /* Generic Command Status */
 500         case NVME_SC_SUCCESS:
 501                 status = SAM_STAT_GOOD;
 502                 sense_key = NO_SENSE;
 503                 asc = SCSI_ASC_NO_SENSE;
 504                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 505                 break;
 506         case NVME_SC_INVALID_OPCODE:
 507                 status = SAM_STAT_CHECK_CONDITION;
 508                 sense_key = ILLEGAL_REQUEST;
 509                 asc = SCSI_ASC_ILLEGAL_COMMAND;
 510                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 511                 break;
 512         case NVME_SC_INVALID_FIELD:
 513                 status = SAM_STAT_CHECK_CONDITION;
 514                 sense_key = ILLEGAL_REQUEST;
 515                 asc = SCSI_ASC_INVALID_CDB;
 516                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 517                 break;
 518         case NVME_SC_DATA_XFER_ERROR:
 519                 status = SAM_STAT_CHECK_CONDITION;
 520                 sense_key = MEDIUM_ERROR;
 521                 asc = SCSI_ASC_NO_SENSE;
 522                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 523                 break;
 524         case NVME_SC_POWER_LOSS:
 525                 status = SAM_STAT_TASK_ABORTED;
 526                 sense_key = ABORTED_COMMAND;
 527                 asc = SCSI_ASC_WARNING;
 528                 ascq = SCSI_ASCQ_POWER_LOSS_EXPECTED;
 529                 break;
 530         case NVME_SC_INTERNAL:
 531                 status = SAM_STAT_CHECK_CONDITION;
 532                 sense_key = HARDWARE_ERROR;
 533                 asc = SCSI_ASC_INTERNAL_TARGET_FAILURE;
 534                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 535                 break;
 536         case NVME_SC_ABORT_REQ:
 537                 status = SAM_STAT_TASK_ABORTED;
 538                 sense_key = ABORTED_COMMAND;
 539                 asc = SCSI_ASC_NO_SENSE;
 540                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 541                 break;
 542         case NVME_SC_ABORT_QUEUE:
 543                 status = SAM_STAT_TASK_ABORTED;
 544                 sense_key = ABORTED_COMMAND;
 545                 asc = SCSI_ASC_NO_SENSE;
 546                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 547                 break;
 548         case NVME_SC_FUSED_FAIL:
 549                 status = SAM_STAT_TASK_ABORTED;
 550                 sense_key = ABORTED_COMMAND;
 551                 asc = SCSI_ASC_NO_SENSE;
 552                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 553                 break;
 554         case NVME_SC_FUSED_MISSING:
 555                 status = SAM_STAT_TASK_ABORTED;
 556                 sense_key = ABORTED_COMMAND;
 557                 asc = SCSI_ASC_NO_SENSE;
 558                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 559                 break;
 560         case NVME_SC_INVALID_NS:
 561                 status = SAM_STAT_CHECK_CONDITION;
 562                 sense_key = ILLEGAL_REQUEST;
 563                 asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID;
 564                 ascq = SCSI_ASCQ_INVALID_LUN_ID;
 565                 break;
 566         case NVME_SC_LBA_RANGE:
 567                 status = SAM_STAT_CHECK_CONDITION;
 568                 sense_key = ILLEGAL_REQUEST;
 569                 asc = SCSI_ASC_ILLEGAL_BLOCK;
 570                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 571                 break;
 572         case NVME_SC_CAP_EXCEEDED:
 573                 status = SAM_STAT_CHECK_CONDITION;
 574                 sense_key = MEDIUM_ERROR;
 575                 asc = SCSI_ASC_NO_SENSE;
 576                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 577                 break;
 578         case NVME_SC_NS_NOT_READY:
 579                 status = SAM_STAT_CHECK_CONDITION;
 580                 sense_key = NOT_READY;
 581                 asc = SCSI_ASC_LUN_NOT_READY;
 582                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 583                 break;
 584
 585         /* Command Specific Status */
 586         case NVME_SC_INVALID_FORMAT:
 587                 status = SAM_STAT_CHECK_CONDITION;
 588                 sense_key = ILLEGAL_REQUEST;
 589                 asc = SCSI_ASC_FORMAT_COMMAND_FAILED;
 590                 ascq = SCSI_ASCQ_FORMAT_COMMAND_FAILED;
 591                 break;
 592         case NVME_SC_BAD_ATTRIBUTES:
 593                 status = SAM_STAT_CHECK_CONDITION;
 594                 sense_key = ILLEGAL_REQUEST;
 595                 asc = SCSI_ASC_INVALID_CDB;
 596                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 597                 break;
 598
 599         /* Media Errors */
 600         case NVME_SC_WRITE_FAULT:
 601                 status = SAM_STAT_CHECK_CONDITION;
 602                 sense_key = MEDIUM_ERROR;
 603                 asc = SCSI_ASC_PERIPHERAL_DEV_WRITE_FAULT;
 604                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 605                 break;
 606         case NVME_SC_READ_ERROR:
 607                 status = SAM_STAT_CHECK_CONDITION;
 608                 sense_key = MEDIUM_ERROR;
 609                 asc = SCSI_ASC_UNRECOVERED_READ_ERROR;
 610                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 611                 break;
 612         case NVME_SC_GUARD_CHECK:
 613                 status = SAM_STAT_CHECK_CONDITION;
 614                 sense_key = MEDIUM_ERROR;
 615                 asc = SCSI_ASC_LOG_BLOCK_GUARD_CHECK_FAILED;
 616                 ascq = SCSI_ASCQ_LOG_BLOCK_GUARD_CHECK_FAILED;
 617                 break;
 618         case NVME_SC_APPTAG_CHECK:
 619                 status = SAM_STAT_CHECK_CONDITION;
 620                 sense_key = MEDIUM_ERROR;
 621                 asc = SCSI_ASC_LOG_BLOCK_APPTAG_CHECK_FAILED;
 622                 ascq = SCSI_ASCQ_LOG_BLOCK_APPTAG_CHECK_FAILED;
 623                 break;
 624         case NVME_SC_REFTAG_CHECK:
 625                 status = SAM_STAT_CHECK_CONDITION;
 626                 sense_key = MEDIUM_ERROR;
 627                 asc = SCSI_ASC_LOG_BLOCK_REFTAG_CHECK_FAILED;
 628                 ascq = SCSI_ASCQ_LOG_BLOCK_REFTAG_CHECK_FAILED;
 629                 break;
 630         case NVME_SC_COMPARE_FAILED:
 631                 status = SAM_STAT_CHECK_CONDITION;
 632                 sense_key = MISCOMPARE;
 633                 asc = SCSI_ASC_MISCOMPARE_DURING_VERIFY;
 634                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 635                 break;
 636         case NVME_SC_ACCESS_DENIED:
 637                 status = SAM_STAT_CHECK_CONDITION;
 638                 sense_key = ILLEGAL_REQUEST;
 639                 asc = SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID;
 640                 ascq = SCSI_ASCQ_INVALID_LUN_ID;
 641                 break;
 642
 643         /* Unspecified/Default */
 644         case NVME_SC_CMDID_CONFLICT:
 645         case NVME_SC_CMD_SEQ_ERROR:
 646         case NVME_SC_CQ_INVALID:
 647         case NVME_SC_QID_INVALID:
 648         case NVME_SC_QUEUE_SIZE:
 649         case NVME_SC_ABORT_LIMIT:
 650         case NVME_SC_ABORT_MISSING:
 651         case NVME_SC_ASYNC_LIMIT:
 652         case NVME_SC_FIRMWARE_SLOT:
 653         case NVME_SC_FIRMWARE_IMAGE:
 654         case NVME_SC_INVALID_VECTOR:
 655         case NVME_SC_INVALID_LOG_PAGE:
 656         default:
 657                 status = SAM_STAT_CHECK_CONDITION;
 658                 sense_key = ILLEGAL_REQUEST;
 659                 asc = SCSI_ASC_NO_SENSE;
 660                 ascq = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
 661                 break;
 662         }
 663
 664         res = nvme_trans_completion(hdr, status, sense_key, asc, ascq);
 665
 666         return res;
 667 }
 668
 669 /* INQUIRY Helper Functions */
 670
 671 static int nvme_trans_standard_inquiry_page(struct nvme_ns *ns,
 672                                         struct sg_io_hdr *hdr, u8 *inq_response,
 673                                         int alloc_len)
 674 {
 675         struct nvme_dev *dev = ns->dev;
 676         dma_addr_t dma_addr;
 677         void *mem;
 678         struct nvme_id_ns *id_ns;
 679         int res = SNTI_TRANSLATION_SUCCESS;
 680         int nvme_sc;
 681         int xfer_len;
 682         u8 resp_data_format = 0x02;
 683         u8 protect;
 684         u8 cmdque = 0x01 << 1;
 685         u8 fw_offset = sizeof(dev->firmware_rev);
 686
 687         mem = dma_alloc_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns),
 688                                 &dma_addr, GFP_KERNEL);
 689         if (mem == NULL) {
 690                 res = -ENOMEM;
 691                 goto out_dma;
 692         }
 693
 694         /* nvme ns identify - use DPS value for PROTECT field */
 695         nvme_sc = nvme_identify(dev, ns->ns_id, 0, dma_addr);
 696         res = nvme_trans_status_code(hdr, nvme_sc);
 697         /*
 698          * If nvme_sc was -ve, res will be -ve here.
 699          * If nvme_sc was +ve, the status would bace been translated, and res
 700          *  can only be 0 or -ve.
 701          *    - If 0 && nvme_sc > 0, then go into next if where res gets nvme_sc
 702          *    - If -ve, return because its a Linux error.
 703          */
 704         if (res)
 705                 goto out_free;
 706         if (nvme_sc) {
 707                 res = nvme_sc;
 708                 goto out_free;
 709         }
 710         id_ns = mem;
 711         (id_ns->dps) ? (protect = 0x01) : (protect = 0);
 712
 713         memset(inq_response, 0, STANDARD_INQUIRY_LENGTH);
 714         inq_response[2] = VERSION_SPC_4;
 715         inq_response[3] = resp_data_format;     /*normaca=0 | hisup=0 */
 716         inq_response[4] = ADDITIONAL_STD_INQ_LENGTH;
 717         inq_response[5] = protect;      /* sccs=0 | acc=0 | tpgs=0 | pc3=0 */
 718         inq_response[7] = cmdque;       /* wbus16=0 | sync=0 | vs=0 */
 719         strncpy(&inq_response[8], "NVMe    ", 8);
 720         strncpy(&inq_response[16], dev->model, 16);
 721
 722         while (dev->firmware_rev[fw_offset - 1] == ' ' && fw_offset > 4)
 723                 fw_offset--;
 724         fw_offset -= 4;
 725         strncpy(&inq_response[32], dev->firmware_rev + fw_offset, 4);
 726
 727         xfer_len = min(alloc_len, STANDARD_INQUIRY_LENGTH);
 728         res = nvme_trans_copy_to_user(hdr, inq_response, xfer_len);
 729
 730  out_free:
 731         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns), mem,
 732                           dma_addr);
 733  out_dma:
 734         return res;
 735 }
 736
 737 static int nvme_trans_supported_vpd_pages(struct nvme_ns *ns,
 738                                         struct sg_io_hdr *hdr, u8 *inq_response,
 739                                         int alloc_len)
 740 {
 741         int res = SNTI_TRANSLATION_SUCCESS;
 742         int xfer_len;
 743
 744         memset(inq_response, 0, STANDARD_INQUIRY_LENGTH);
 745         inq_response[1] = INQ_SUPPORTED_VPD_PAGES_PAGE;   /* Page Code */
 746         inq_response[3] = INQ_NUM_SUPPORTED_VPD_PAGES;    /* Page Length */
 747         inq_response[4] = INQ_SUPPORTED_VPD_PAGES_PAGE;
 748         inq_response[5] = INQ_UNIT_SERIAL_NUMBER_PAGE;
 749         inq_response[6] = INQ_DEVICE_IDENTIFICATION_PAGE;
 750         inq_response[7] = INQ_EXTENDED_INQUIRY_DATA_PAGE;
 751         inq_response[8] = INQ_BDEV_CHARACTERISTICS_PAGE;
 752         inq_response[9] = INQ_BDEV_LIMITS_PAGE;
 753
 754         xfer_len = min(alloc_len, STANDARD_INQUIRY_LENGTH);
 755         res = nvme_trans_copy_to_user(hdr, inq_response, xfer_len);
 756
 757         return res;
 758 }
 759
 760 static int nvme_trans_unit_serial_page(struct nvme_ns *ns,
 761                                         struct sg_io_hdr *hdr, u8 *inq_response,
 762                                         int alloc_len)
 763 {
 764         struct nvme_dev *dev = ns->dev;
 765         int res = SNTI_TRANSLATION_SUCCESS;
 766         int xfer_len;
 767
 768         memset(inq_response, 0, STANDARD_INQUIRY_LENGTH);
 769         inq_response[1] = INQ_UNIT_SERIAL_NUMBER_PAGE; /* Page Code */
 770         inq_response[3] = INQ_SERIAL_NUMBER_LENGTH;    /* Page Length */
 771         strncpy(&inq_response[4], dev->serial, INQ_SERIAL_NUMBER_LENGTH);
 772
 773         xfer_len = min(alloc_len, STANDARD_INQUIRY_LENGTH);
 774         res = nvme_trans_copy_to_user(hdr, inq_response, xfer_len);
 775
 776         return res;
 777 }
 778
 779 static int nvme_trans_device_id_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 780                                         u8 *inq_response, int alloc_len)
 781 {
 782         struct nvme_dev *dev = ns->dev;
 783         dma_addr_t dma_addr;
 784         void *mem;
 785         int res = SNTI_TRANSLATION_SUCCESS;
 786         int nvme_sc;
 787         int xfer_len;
 788         __be32 tmp_id = cpu_to_be32(ns->ns_id);
 789
 790         mem = dma_alloc_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns),
 791                                         &dma_addr, GFP_KERNEL);
 792         if (mem == NULL) {
 793                 res = -ENOMEM;
 794                 goto out_dma;
 795         }
 796
 797         memset(inq_response, 0, alloc_len);
 798         inq_response[1] = INQ_DEVICE_IDENTIFICATION_PAGE;    /* Page Code */
 799         if (readl(&dev->bar->vs) >= NVME_VS(1, 1)) {
 800                 struct nvme_id_ns *id_ns = mem;
 801                 void *eui = id_ns->eui64;
 802                 int len = sizeof(id_ns->eui64);
 803
 804                 nvme_sc = nvme_identify(dev, ns->ns_id, 0, dma_addr);
 805                 res = nvme_trans_status_code(hdr, nvme_sc);
 806                 if (res)
 807                         goto out_free;
 808                 if (nvme_sc) {
 809                         res = nvme_sc;
 810                         goto out_free;
 811                 }
 812
 813                 if (readl(&dev->bar->vs) >= NVME_VS(1, 2)) {
 814                         if (bitmap_empty(eui, len * 8)) {
 815                                 eui = id_ns->nguid;
 816                                 len = sizeof(id_ns->nguid);
 817                         }
 818                 }
 819                 if (bitmap_empty(eui, len * 8))
 820                         goto scsi_string;
 821
 822                 inq_response[3] = 4 + len; /* Page Length */
 823                 /* Designation Descriptor start */
 824                 inq_response[4] = 0x01;    /* Proto ID=0h | Code set=1h */
 825                 inq_response[5] = 0x02;    /* PIV=0b | Asso=00b | Designator Type=2h */
 826                 inq_response[6] = 0x00;    /* Rsvd */
 827                 inq_response[7] = len;     /* Designator Length */
 828                 memcpy(&inq_response[8], eui, len);
 829         } else {
 830  scsi_string:
 831                 if (alloc_len < 72) {
 832                         res = nvme_trans_completion(hdr,
 833                                         SAM_STAT_CHECK_CONDITION,
 834                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
 835                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
 836                         goto out_free;
 837                 }
 838                 inq_response[3] = 0x48;    /* Page Length */
 839                 /* Designation Descriptor start */
 840                 inq_response[4] = 0x03;    /* Proto ID=0h | Code set=3h */
 841                 inq_response[5] = 0x08;    /* PIV=0b | Asso=00b | Designator Type=8h */
 842                 inq_response[6] = 0x00;    /* Rsvd */
 843                 inq_response[7] = 0x44;    /* Designator Length */
 844
 845                 sprintf(&inq_response[8], "%04x", dev->pci_dev->vendor);
 846                 memcpy(&inq_response[12], dev->model, sizeof(dev->model));
 847                 sprintf(&inq_response[52], "%04x", tmp_id);
 848                 memcpy(&inq_response[56], dev->serial, sizeof(dev->serial));
 849         }
 850         xfer_len = alloc_len;
 851         res = nvme_trans_copy_to_user(hdr, inq_response, xfer_len);
 852
 853  out_free:
 854         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns), mem,
 855                           dma_addr);
 856  out_dma:
 857         return res;
 858 }
 859
 860 static int nvme_trans_ext_inq_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 861                                         int alloc_len)
 862 {
 863         u8 *inq_response;
 864         int res = SNTI_TRANSLATION_SUCCESS;
 865         int nvme_sc;
 866         struct nvme_dev *dev = ns->dev;
 867         dma_addr_t dma_addr;
 868         void *mem;
 869         struct nvme_id_ctrl *id_ctrl;
 870         struct nvme_id_ns *id_ns;
 871         int xfer_len;
 872         u8 microcode = 0x80;
 873         u8 spt;
 874         u8 spt_lut[8] = {0, 0, 2, 1, 4, 6, 5, 7};
 875         u8 grd_chk, app_chk, ref_chk, protect;
 876         u8 uask_sup = 0x20;
 877         u8 v_sup;
 878         u8 luiclr = 0x01;
 879
 880         inq_response = kmalloc(EXTENDED_INQUIRY_DATA_PAGE_LENGTH, GFP_KERNEL);
 881         if (inq_response == NULL) {
 882                 res = -ENOMEM;
 883                 goto out_mem;
 884         }
 885
 886         mem = dma_alloc_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns),
 887                                                         &dma_addr, GFP_KERNEL);
 888         if (mem == NULL) {
 889                 res = -ENOMEM;
 890                 goto out_dma;
 891         }
 892
 893         /* nvme ns identify */
 894         nvme_sc = nvme_identify(dev, ns->ns_id, 0, dma_addr);
 895         res = nvme_trans_status_code(hdr, nvme_sc);
 896         if (res)
 897                 goto out_free;
 898         if (nvme_sc) {
 899                 res = nvme_sc;
 900                 goto out_free;
 901         }
 902         id_ns = mem;
 903         spt = spt_lut[(id_ns->dpc) & 0x07] << 3;
 904         (id_ns->dps) ? (protect = 0x01) : (protect = 0);
 905         grd_chk = protect << 2;
 906         app_chk = protect << 1;
 907         ref_chk = protect;
 908
 909         /* nvme controller identify */
 910         nvme_sc = nvme_identify(dev, 0, 1, dma_addr);
 911         res = nvme_trans_status_code(hdr, nvme_sc);
 912         if (res)
 913                 goto out_free;
 914         if (nvme_sc) {
 915                 res = nvme_sc;
 916                 goto out_free;
 917         }
 918         id_ctrl = mem;
 919         v_sup = id_ctrl->vwc;
 920
 921         memset(inq_response, 0, EXTENDED_INQUIRY_DATA_PAGE_LENGTH);
 922         inq_response[1] = INQ_EXTENDED_INQUIRY_DATA_PAGE;    /* Page Code */
 923         inq_response[2] = 0x00;    /* Page Length MSB */
 924         inq_response[3] = 0x3C;    /* Page Length LSB */
 925         inq_response[4] = microcode | spt | grd_chk | app_chk | ref_chk;
 926         inq_response[5] = uask_sup;
 927         inq_response[6] = v_sup;
 928         inq_response[7] = luiclr;
 929         inq_response[8] = 0;
 930         inq_response[9] = 0;
 931
 932         xfer_len = min(alloc_len, EXTENDED_INQUIRY_DATA_PAGE_LENGTH);
 933         res = nvme_trans_copy_to_user(hdr, inq_response, xfer_len);
 934
 935  out_free:
 936         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns), mem,
 937                           dma_addr);
 938  out_dma:
 939         kfree(inq_response);
 940  out_mem:
 941         return res;
 942 }
 943
 944 static int nvme_trans_bdev_limits_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 945                                         u8 *inq_response, int alloc_len)
 946 {
 947         __be32 max_sectors = cpu_to_be32(queue_max_hw_sectors(ns->queue));
 948         __be32 max_discard = cpu_to_be32(ns->queue->limits.max_discard_sectors);
 949         __be32 discard_desc_count = cpu_to_be32(0x100);
 950
 951         memset(inq_response, 0, STANDARD_INQUIRY_LENGTH);
 952         inq_response[1] = VPD_BLOCK_LIMITS;
 953         inq_response[3] = 0x3c; /* Page Length */
 954         memcpy(&inq_response[8], &max_sectors, sizeof(u32));
 955         memcpy(&inq_response[20], &max_discard, sizeof(u32));
 956
 957         if (max_discard)
 958                 memcpy(&inq_response[24], &discard_desc_count, sizeof(u32));
 959
 960         return nvme_trans_copy_to_user(hdr, inq_response, 0x3c);
 961 }
 962
 963 static int nvme_trans_bdev_char_page(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 964                                         int alloc_len)
 965 {
 966         u8 *inq_response;
 967         int res = SNTI_TRANSLATION_SUCCESS;
 968         int xfer_len;
 969
 970         inq_response = kzalloc(EXTENDED_INQUIRY_DATA_PAGE_LENGTH, GFP_KERNEL);
 971         if (inq_response == NULL) {
 972                 res = -ENOMEM;
 973                 goto out_mem;
 974         }
 975
 976         inq_response[1] = INQ_BDEV_CHARACTERISTICS_PAGE;    /* Page Code */
 977         inq_response[2] = 0x00;    /* Page Length MSB */
 978         inq_response[3] = 0x3C;    /* Page Length LSB */
 979         inq_response[4] = 0x00;    /* Medium Rotation Rate MSB */
 980         inq_response[5] = 0x01;    /* Medium Rotation Rate LSB */
 981         inq_response[6] = 0x00;    /* Form Factor */
 982
 983         xfer_len = min(alloc_len, EXTENDED_INQUIRY_DATA_PAGE_LENGTH);
 984         res = nvme_trans_copy_to_user(hdr, inq_response, xfer_len);
 985
 986         kfree(inq_response);
 987  out_mem:
 988         return res;
 989 }
 990
 991 /* LOG SENSE Helper Functions */
 992
 993 static int nvme_trans_log_supp_pages(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 994                                         int alloc_len)
 995 {
 996         int res = SNTI_TRANSLATION_SUCCESS;
 997         int xfer_len;
 998         u8 *log_response;
 999
1000         log_response = kzalloc(LOG_PAGE_SUPPORTED_LOG_PAGES_LENGTH, GFP_KERNEL);
1001         if (log_response == NULL) {
1002                 res = -ENOMEM;
1003                 goto out_mem;
1004         }
1005
1006         log_response[0] = LOG_PAGE_SUPPORTED_LOG_PAGES_PAGE;
1007         /* Subpage=0x00, Page Length MSB=0 */
1008         log_response[3] = SUPPORTED_LOG_PAGES_PAGE_LENGTH;
1009         log_response[4] = LOG_PAGE_SUPPORTED_LOG_PAGES_PAGE;
1010         log_response[5] = LOG_PAGE_INFORMATIONAL_EXCEPTIONS_PAGE;
1011         log_response[6] = LOG_PAGE_TEMPERATURE_PAGE;
1012
1013         xfer_len = min(alloc_len, LOG_PAGE_SUPPORTED_LOG_PAGES_LENGTH);
1014         res = nvme_trans_copy_to_user(hdr, log_response, xfer_len);
1015
1016         kfree(log_response);
1017  out_mem:
1018         return res;
1019 }
1020
1021 static int nvme_trans_log_info_exceptions(struct nvme_ns *ns,
1022                                         struct sg_io_hdr *hdr, int alloc_len)
1023 {
1024         int res = SNTI_TRANSLATION_SUCCESS;
1025         int xfer_len;
1026         u8 *log_response;
1027         struct nvme_command c;
1028         struct nvme_dev *dev = ns->dev;
1029         struct nvme_smart_log *smart_log;
1030         dma_addr_t dma_addr;
1031         void *mem;
1032         u8 temp_c;
1033         u16 temp_k;
1034
1035         log_response = kzalloc(LOG_INFO_EXCP_PAGE_LENGTH, GFP_KERNEL);
1036         if (log_response == NULL) {
1037                 res = -ENOMEM;
1038                 goto out_mem;
1039         }
1040
1041         mem = dma_alloc_coherent(&dev->pci_dev->dev,
1042                                         sizeof(struct nvme_smart_log),
1043                                         &dma_addr, GFP_KERNEL);
1044         if (mem == NULL) {
1045                 res = -ENOMEM;
1046                 goto out_dma;
1047         }
1048
1049         /* Get SMART Log Page */
1050         memset(&c, 0, sizeof(c));
1051         c.common.opcode = nvme_admin_get_log_page;
1052         c.common.nsid = cpu_to_le32(0xFFFFFFFF);
1053         c.common.prp1 = cpu_to_le64(dma_addr);
1054         c.common.cdw10[0] = cpu_to_le32((((sizeof(struct nvme_smart_log) /
1055                         BYTES_TO_DWORDS) - 1) << 16) | NVME_LOG_SMART);
1056         res = nvme_submit_admin_cmd(dev, &c, NULL);
1057         if (res != NVME_SC_SUCCESS) {
1058                 temp_c = LOG_TEMP_UNKNOWN;
1059         } else {
1060                 smart_log = mem;
1061                 temp_k = (smart_log->temperature[1] << 8) +
1062                                 (smart_log->temperature[0]);
1063                 temp_c = temp_k - KELVIN_TEMP_FACTOR;
1064         }
1065
1066         log_response[0] = LOG_PAGE_INFORMATIONAL_EXCEPTIONS_PAGE;
1067         /* Subpage=0x00, Page Length MSB=0 */
1068         log_response[3] = REMAINING_INFO_EXCP_PAGE_LENGTH;
1069         /* Informational Exceptions Log Parameter 1 Start */
1070         /* Parameter Code=0x0000 bytes 4,5 */
1071         log_response[6] = 0x23; /* DU=0, TSD=1, ETC=0, TMC=0, FMT_AND_LNK=11b */
1072         log_response[7] = 0x04; /* PARAMETER LENGTH */
1073         /* Add sense Code and qualifier = 0x00 each */
1074         /* Use Temperature from NVMe Get Log Page, convert to C from K */
1075         log_response[10] = temp_c;
1076
1077         xfer_len = min(alloc_len, LOG_INFO_EXCP_PAGE_LENGTH);
1078         res = nvme_trans_copy_to_user(hdr, log_response, xfer_len);
1079
1080         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_smart_log),
1081                           mem, dma_addr);
1082  out_dma:
1083         kfree(log_response);
1084  out_mem:
1085         return res;
1086 }
1087
1088 static int nvme_trans_log_temperature(struct nvme_ns *ns, struct sg_io_hdr *hdr,
1089                                         int alloc_len)
1090 {
1091         int res = SNTI_TRANSLATION_SUCCESS;
1092         int xfer_len;
1093         u8 *log_response;
1094         struct nvme_command c;
1095         struct nvme_dev *dev = ns->dev;
1096         struct nvme_smart_log *smart_log;
1097         dma_addr_t dma_addr;
1098         void *mem;
1099         u32 feature_resp;
1100         u8 temp_c_cur, temp_c_thresh;
1101         u16 temp_k;
1102
1103         log_response = kzalloc(LOG_TEMP_PAGE_LENGTH, GFP_KERNEL);
1104         if (log_response == NULL) {
1105                 res = -ENOMEM;
1106                 goto out_mem;
1107         }
1108
1109         mem = dma_alloc_coherent(&dev->pci_dev->dev,
1110                                         sizeof(struct nvme_smart_log),
1111                                         &dma_addr, GFP_KERNEL);
1112         if (mem == NULL) {
1113                 res = -ENOMEM;
1114                 goto out_dma;
1115         }
1116
1117         /* Get SMART Log Page */
1118         memset(&c, 0, sizeof(c));
1119         c.common.opcode = nvme_admin_get_log_page;
1120         c.common.nsid = cpu_to_le32(0xFFFFFFFF);
1121         c.common.prp1 = cpu_to_le64(dma_addr);
1122         c.common.cdw10[0] = cpu_to_le32((((sizeof(struct nvme_smart_log) /
1123                         BYTES_TO_DWORDS) - 1) << 16) | NVME_LOG_SMART);
1124         res = nvme_submit_admin_cmd(dev, &c, NULL);
1125         if (res != NVME_SC_SUCCESS) {
1126                 temp_c_cur = LOG_TEMP_UNKNOWN;
1127         } else {
1128                 smart_log = mem;
1129                 temp_k = (smart_log->temperature[1] << 8) +
1130                                 (smart_log->temperature[0]);
1131                 temp_c_cur = temp_k - KELVIN_TEMP_FACTOR;
1132         }
1133
1134         /* Get Features for Temp Threshold */
1135         res = nvme_get_features(dev, NVME_FEAT_TEMP_THRESH, 0, 0,
1136                                                                 &feature_resp);
1137         if (res != NVME_SC_SUCCESS)
1138                 temp_c_thresh = LOG_TEMP_UNKNOWN;
1139         else
1140                 temp_c_thresh = (feature_resp & 0xFFFF) - KELVIN_TEMP_FACTOR;
1141
1142         log_response[0] = LOG_PAGE_TEMPERATURE_PAGE;
1143         /* Subpage=0x00, Page Length MSB=0 */
1144         log_response[3] = REMAINING_TEMP_PAGE_LENGTH;
1145         /* Temperature Log Parameter 1 (Temperature) Start */
1146         /* Parameter Code = 0x0000 */
1147         log_response[6] = 0x01;         /* Format and Linking = 01b */
1148         log_response[7] = 0x02;         /* Parameter Length */
1149         /* Use Temperature from NVMe Get Log Page, convert to C from K */
1150         log_response[9] = temp_c_cur;
1151         /* Temperature Log Parameter 2 (Reference Temperature) Start */
1152         log_response[11] = 0x01;        /* Parameter Code = 0x0001 */
1153         log_response[12] = 0x01;        /* Format and Linking = 01b */
1154         log_response[13] = 0x02;        /* Parameter Length */
1155         /* Use Temperature Thresh from NVMe Get Log Page, convert to C from K */
1156         log_response[15] = temp_c_thresh;
1157
1158         xfer_len = min(alloc_len, LOG_TEMP_PAGE_LENGTH);
1159         res = nvme_trans_copy_to_user(hdr, log_response, xfer_len);
1160
1161         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_smart_log),
1162                           mem, dma_addr);
1163  out_dma:
1164         kfree(log_response);
1165  out_mem:
1166         return res;
1167 }
1168
1169 /* MODE SENSE Helper Functions */
1170
1171 static int nvme_trans_fill_mode_parm_hdr(u8 *resp, int len, u8 cdb10, u8 llbaa,
1172                                         u16 mode_data_length, u16 blk_desc_len)
1173 {
1174         /* Quick check to make sure I don't stomp on my own memory... */
1175         if ((cdb10 && len < 8) || (!cdb10 && len < 4))
1176                 return SNTI_INTERNAL_ERROR;
1177
1178         if (cdb10) {
1179                 resp[0] = (mode_data_length & 0xFF00) >> 8;
1180                 resp[1] = (mode_data_length & 0x00FF);
1181                 /* resp[2] and [3] are zero */
1182                 resp[4] = llbaa;
1183                 resp[5] = RESERVED_FIELD;
1184                 resp[6] = (blk_desc_len & 0xFF00) >> 8;
1185                 resp[7] = (blk_desc_len & 0x00FF);
1186         } else {
1187                 resp[0] = (mode_data_length & 0x00FF);
1188                 /* resp[1] and [2] are zero */
1189                 resp[3] = (blk_desc_len & 0x00FF);
1190         }
1191
1192         return SNTI_TRANSLATION_SUCCESS;
1193 }
1194
1195 static int nvme_trans_fill_blk_desc(struct nvme_ns *ns, struct sg_io_hdr *hdr,
1196                                     u8 *resp, int len, u8 llbaa)
1197 {
1198         int res = SNTI_TRANSLATION_SUCCESS;
1199         int nvme_sc;
1200         struct nvme_dev *dev = ns->dev;
1201         dma_addr_t dma_addr;
1202         void *mem;
1203         struct nvme_id_ns *id_ns;
1204         u8 flbas;
1205         u32 lba_length;
1206
1207         if (llbaa == 0 && len < MODE_PAGE_BLK_DES_LEN)
1208                 return SNTI_INTERNAL_ERROR;
1209         else if (llbaa > 0 && len < MODE_PAGE_LLBAA_BLK_DES_LEN)
1210                 return SNTI_INTERNAL_ERROR;
1211
1212         mem = dma_alloc_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns),
1213                                                         &dma_addr, GFP_KERNEL);
1214         if (mem == NULL) {
1215                 res = -ENOMEM;
1216                 goto out;
1217         }
1218
1219         /* nvme ns identify */
1220         nvme_sc = nvme_identify(dev, ns->ns_id, 0, dma_addr);
1221         res = nvme_trans_status_code(hdr, nvme_sc);
1222         if (res)
1223                 goto out_dma;
1224         if (nvme_sc) {
1225                 res = nvme_sc;
1226                 goto out_dma;
1227         }
1228         id_ns = mem;
1229         flbas = (id_ns->flbas) & 0x0F;
1230         lba_length = (1 << (id_ns->lbaf[flbas].ds));
1231
1232         if (llbaa == 0) {
1233                 __be32 tmp_cap = cpu_to_be32(le64_to_cpu(id_ns->ncap));
1234                 /* Byte 4 is reserved */
1235                 __be32 tmp_len = cpu_to_be32(lba_length & 0x00FFFFFF);
1236
1237                 memcpy(resp, &tmp_cap, sizeof(u32));
1238                 memcpy(&resp[4], &tmp_len, sizeof(u32));
1239         } else {
1240                 __be64 tmp_cap = cpu_to_be64(le64_to_cpu(id_ns->ncap));
1241                 __be32 tmp_len = cpu_to_be32(lba_length);
1242
1243                 memcpy(resp, &tmp_cap, sizeof(u64));
1244                 /* Bytes 8, 9, 10, 11 are reserved */
1245                 memcpy(&resp[12], &tmp_len, sizeof(u32));
1246         }
1247
1248  out_dma:
1249         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns), mem,
1250                           dma_addr);
1251  out:
1252         return res;
1253 }
1254
1255 static int nvme_trans_fill_control_page(struct nvme_ns *ns,
1256                                         struct sg_io_hdr *hdr, u8 *resp,
1257                                         int len)
1258 {
1259         if (len < MODE_PAGE_CONTROL_LEN)
1260                 return SNTI_INTERNAL_ERROR;
1261
1262         resp[0] = MODE_PAGE_CONTROL;
1263         resp[1] = MODE_PAGE_CONTROL_LEN_FIELD;
1264         resp[2] = 0x0E;         /* TST=000b, TMF_ONLY=0, DPICZ=1,
1265                                  * D_SENSE=1, GLTSD=1, RLEC=0 */
1266         resp[3] = 0x12;         /* Q_ALGO_MODIFIER=1h, NUAR=0, QERR=01b */
1267         /* Byte 4:  VS=0, RAC=0, UA_INT=0, SWP=0 */
1268         resp[5] = 0x40;         /* ATO=0, TAS=1, ATMPE=0, RWWP=0, AUTOLOAD=0 */
1269         /* resp[6] and [7] are obsolete, thus zero */
1270         resp[8] = 0xFF;         /* Busy timeout period = 0xffff */
1271         resp[9] = 0xFF;
1272         /* Bytes 10,11: Extended selftest completion time = 0x0000 */
1273
1274         return SNTI_TRANSLATION_SUCCESS;
1275 }
1276
1277 static int nvme_trans_fill_caching_page(struct nvme_ns *ns,
1278                                         struct sg_io_hdr *hdr,
1279                                         u8 *resp, int len)
1280 {
1281         int res = SNTI_TRANSLATION_SUCCESS;
1282         int nvme_sc;
1283         struct nvme_dev *dev = ns->dev;
1284         u32 feature_resp;
1285         u8 vwc;
1286
1287         if (len < MODE_PAGE_CACHING_LEN)
1288                 return SNTI_INTERNAL_ERROR;
1289
1290         nvme_sc = nvme_get_features(dev, NVME_FEAT_VOLATILE_WC, 0, 0,
1291                                                                 &feature_resp);
1292         res = nvme_trans_status_code(hdr, nvme_sc);
1293         if (res)
1294                 goto out;
1295         if (nvme_sc) {
1296                 res = nvme_sc;
1297                 goto out;
1298         }
1299         vwc = feature_resp & 0x00000001;
1300
1301         resp[0] = MODE_PAGE_CACHING;
1302         resp[1] = MODE_PAGE_CACHING_LEN_FIELD;
1303         resp[2] = vwc << 2;
1304
1305  out:
1306         return res;
1307 }
1308
1309 static int nvme_trans_fill_pow_cnd_page(struct nvme_ns *ns,
1310                                         struct sg_io_hdr *hdr, u8 *resp,
1311                                         int len)
1312 {
1313         int res = SNTI_TRANSLATION_SUCCESS;
1314
1315         if (len < MODE_PAGE_POW_CND_LEN)
1316                 return SNTI_INTERNAL_ERROR;
1317
1318         resp[0] = MODE_PAGE_POWER_CONDITION;
1319         resp[1] = MODE_PAGE_POW_CND_LEN_FIELD;
1320         /* All other bytes are zero */
1321
1322         return res;
1323 }
1324
1325 static int nvme_trans_fill_inf_exc_page(struct nvme_ns *ns,
1326                                         struct sg_io_hdr *hdr, u8 *resp,
1327                                         int len)
1328 {
1329         int res = SNTI_TRANSLATION_SUCCESS;
1330
1331         if (len < MODE_PAGE_INF_EXC_LEN)
1332                 return SNTI_INTERNAL_ERROR;
1333
1334         resp[0] = MODE_PAGE_INFO_EXCEP;
1335         resp[1] = MODE_PAGE_INF_EXC_LEN_FIELD;
1336         resp[2] = 0x88;
1337         /* All other bytes are zero */
1338
1339         return res;
1340 }
1341
1342 static int nvme_trans_fill_all_pages(struct nvme_ns *ns, struct sg_io_hdr *hdr,
1343                                      u8 *resp, int len)
1344 {
1345         int res = SNTI_TRANSLATION_SUCCESS;
1346         u16 mode_pages_offset_1 = 0;
1347         u16 mode_pages_offset_2, mode_pages_offset_3, mode_pages_offset_4;
1348
1349         mode_pages_offset_2 = mode_pages_offset_1 + MODE_PAGE_CACHING_LEN;
1350         mode_pages_offset_3 = mode_pages_offset_2 + MODE_PAGE_CONTROL_LEN;
1351         mode_pages_offset_4 = mode_pages_offset_3 + MODE_PAGE_POW_CND_LEN;
1352
1353         res = nvme_trans_fill_caching_page(ns, hdr, &resp[mode_pages_offset_1],
1354                                         MODE_PAGE_CACHING_LEN);
1355         if (res != SNTI_TRANSLATION_SUCCESS)
1356                 goto out;
1357         res = nvme_trans_fill_control_page(ns, hdr, &resp[mode_pages_offset_2],
1358                                         MODE_PAGE_CONTROL_LEN);
1359         if (res != SNTI_TRANSLATION_SUCCESS)
1360                 goto out;
1361         res = nvme_trans_fill_pow_cnd_page(ns, hdr, &resp[mode_pages_offset_3],
1362                                         MODE_PAGE_POW_CND_LEN);
1363         if (res != SNTI_TRANSLATION_SUCCESS)
1364                 goto out;
1365         res = nvme_trans_fill_inf_exc_page(ns, hdr, &resp[mode_pages_offset_4],
1366                                         MODE_PAGE_INF_EXC_LEN);
1367         if (res != SNTI_TRANSLATION_SUCCESS)
1368                 goto out;
1369
1370  out:
1371         return res;
1372 }
1373
1374 static inline int nvme_trans_get_blk_desc_len(u8 dbd, u8 llbaa)
1375 {
1376         if (dbd == MODE_SENSE_BLK_DESC_ENABLED) {
1377                 /* SPC-4: len = 8 x Num_of_descriptors if llbaa = 0, 16x if 1 */
1378                 return 8 * (llbaa + 1) * MODE_SENSE_BLK_DESC_COUNT;
1379         } else {
1380                 return 0;
1381         }
1382 }
1383
1384 static int nvme_trans_mode_page_create(struct nvme_ns *ns,
1385                                         struct sg_io_hdr *hdr, u8 *cmd,
1386                                         u16 alloc_len, u8 cdb10,
1387                                         int (*mode_page_fill_func)
1388                                         (struct nvme_ns *,
1389                                         struct sg_io_hdr *hdr, u8 *, int),
1390                                         u16 mode_pages_tot_len)
1391 {
1392         int res = SNTI_TRANSLATION_SUCCESS;
1393         int xfer_len;
1394         u8 *response;
1395         u8 dbd, llbaa;
1396         u16 resp_size;
1397         int mph_size;
1398         u16 mode_pages_offset_1;
1399         u16 blk_desc_len, blk_desc_offset, mode_data_length;
1400
1401         dbd = GET_MODE_SENSE_DBD(cmd);
1402         llbaa = GET_MODE_SENSE_LLBAA(cmd);
1403         mph_size = GET_MODE_SENSE_MPH_SIZE(cdb10);
1404         blk_desc_len = nvme_trans_get_blk_desc_len(dbd, llbaa);
1405
1406         resp_size = mph_size + blk_desc_len + mode_pages_tot_len;
1407         /* Refer spc4r34 Table 440 for calculation of Mode data Length field */
1408         mode_data_length = 3 + (3 * cdb10) + blk_desc_len + mode_pages_tot_len;
1409
1410         blk_desc_offset = mph_size;
1411         mode_pages_offset_1 = blk_desc_offset + blk_desc_len;
1412
1413         response = kzalloc(resp_size, GFP_KERNEL);
1414         if (response == NULL) {
1415                 res = -ENOMEM;
1416                 goto out_mem;
1417         }
1418
1419         res = nvme_trans_fill_mode_parm_hdr(&response[0], mph_size, cdb10,
1420                                         llbaa, mode_data_length, blk_desc_len);
1421         if (res != SNTI_TRANSLATION_SUCCESS)
1422                 goto out_free;
1423         if (blk_desc_len > 0) {
1424                 res = nvme_trans_fill_blk_desc(ns, hdr,
1425                                                &response[blk_desc_offset],
1426                                                blk_desc_len, llbaa);
1427                 if (res != SNTI_TRANSLATION_SUCCESS)
1428                         goto out_free;
1429         }
1430         res = mode_page_fill_func(ns, hdr, &response[mode_pages_offset_1],
1431                                         mode_pages_tot_len);
1432         if (res != SNTI_TRANSLATION_SUCCESS)
1433                 goto out_free;
1434
1435         xfer_len = min(alloc_len, resp_size);
1436         res = nvme_trans_copy_to_user(hdr, response, xfer_len);
1437
1438  out_free:
1439         kfree(response);
1440  out_mem:
1441         return res;
1442 }
1443
1444 /* Read Capacity Helper Functions */
1445
1446 static void nvme_trans_fill_read_cap(u8 *response, struct nvme_id_ns *id_ns,
1447                                                                 u8 cdb16)
1448 {
1449         u8 flbas;
1450         u32 lba_length;
1451         u64 rlba;
1452         u8 prot_en;
1453         u8 p_type_lut[4] = {0, 0, 1, 2};
1454         __be64 tmp_rlba;
1455         __be32 tmp_rlba_32;
1456         __be32 tmp_len;
1457
1458         flbas = (id_ns->flbas) & 0x0F;
1459         lba_length = (1 << (id_ns->lbaf[flbas].ds));
1460         rlba = le64_to_cpup(&id_ns->nsze) - 1;
1461         (id_ns->dps) ? (prot_en = 0x01) : (prot_en = 0);
1462
1463         if (!cdb16) {
1464                 if (rlba > 0xFFFFFFFF)
1465                         rlba = 0xFFFFFFFF;
1466                 tmp_rlba_32 = cpu_to_be32(rlba);
1467                 tmp_len = cpu_to_be32(lba_length);
1468                 memcpy(response, &tmp_rlba_32, sizeof(u32));
1469                 memcpy(&response[4], &tmp_len, sizeof(u32));
1470         } else {
1471                 tmp_rlba = cpu_to_be64(rlba);
1472                 tmp_len = cpu_to_be32(lba_length);
1473                 memcpy(response, &tmp_rlba, sizeof(u64));
1474                 memcpy(&response[8], &tmp_len, sizeof(u32));
1475                 response[12] = (p_type_lut[id_ns->dps & 0x3] << 1) | prot_en;
1476                 /* P_I_Exponent = 0x0 | LBPPBE = 0x0 */
1477                 /* LBPME = 0 | LBPRZ = 0 | LALBA = 0x00 */
1478                 /* Bytes 16-31 - Reserved */
1479         }
1480 }
1481
1482 /* Start Stop Unit Helper Functions */
1483
1484 static int nvme_trans_power_state(struct nvme_ns *ns, struct sg_io_hdr *hdr,
1485                                                 u8 pc, u8 pcmod, u8 start)
1486 {
1487         int res = SNTI_TRANSLATION_SUCCESS;
1488         int nvme_sc;
1489         struct nvme_dev *dev = ns->dev;
1490         dma_addr_t dma_addr;
1491         void *mem;
1492         struct nvme_id_ctrl *id_ctrl;
1493         int lowest_pow_st;      /* max npss = lowest power consumption */
1494         unsigned ps_desired = 0;
1495
1496         /* NVMe Controller Identify */
1497         mem = dma_alloc_coherent(&dev->pci_dev->dev,
1498                                 sizeof(struct nvme_id_ctrl),
1499                                 &dma_addr, GFP_KERNEL);
1500         if (mem == NULL) {
1501                 res = -ENOMEM;
1502                 goto out;
1503         }
1504         nvme_sc = nvme_identify(dev, 0, 1, dma_addr);
1505         res = nvme_trans_status_code(hdr, nvme_sc);
1506         if (res)
1507                 goto out_dma;
1508         if (nvme_sc) {
1509                 res = nvme_sc;
1510                 goto out_dma;
1511         }
1512         id_ctrl = mem;
1513         lowest_pow_st = max(POWER_STATE_0, (int)(id_ctrl->npss - 1));
1514
1515         switch (pc) {
1516         case NVME_POWER_STATE_START_VALID:
1517                 /* Action unspecified if POWER CONDITION MODIFIER != 0 */
1518                 if (pcmod == 0 && start == 0x1)
1519                         ps_desired = POWER_STATE_0;
1520                 if (pcmod == 0 && start == 0x0)
1521                         ps_desired = lowest_pow_st;
1522                 break;
1523         case NVME_POWER_STATE_ACTIVE:
1524                 /* Action unspecified if POWER CONDITION MODIFIER != 0 */
1525                 if (pcmod == 0)
1526                         ps_desired = POWER_STATE_0;
1527                 break;
1528         case NVME_POWER_STATE_IDLE:
1529                 /* Action unspecified if POWER CONDITION MODIFIER != [0,1,2] */
1530                 if (pcmod == 0x0)
1531                         ps_desired = POWER_STATE_1;
1532                 else if (pcmod == 0x1)
1533                         ps_desired = POWER_STATE_2;
1534                 else if (pcmod == 0x2)
1535                         ps_desired = POWER_STATE_3;
1536                 break;
1537         case NVME_POWER_STATE_STANDBY:
1538                 /* Action unspecified if POWER CONDITION MODIFIER != [0,1] */
1539                 if (pcmod == 0x0)
1540                         ps_desired = max(POWER_STATE_0, (lowest_pow_st - 2));
1541                 else if (pcmod == 0x1)
1542                         ps_desired = max(POWER_STATE_0, (lowest_pow_st - 1));
1543                 break;
1544         case NVME_POWER_STATE_LU_CONTROL:
1545         default:
1546                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
1547                                 ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
1548                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1549                 break;
1550         }
1551         nvme_sc = nvme_set_features(dev, NVME_FEAT_POWER_MGMT, ps_desired, 0,
1552                                     NULL);
1553         res = nvme_trans_status_code(hdr, nvme_sc);
1554         if (res)
1555                 goto out_dma;
1556         if (nvme_sc)
1557                 res = nvme_sc;
1558  out_dma:
1559         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ctrl), mem,
1560                           dma_addr);
1561  out:
1562         return res;
1563 }
1564
1565 /* Write Buffer Helper Functions */
1566 /* Also using this for Format Unit with hdr passed as NULL, and buffer_id, 0 */
1567
1568 static int nvme_trans_send_fw_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr,
1569                                         u8 opcode, u32 tot_len, u32 offset,
1570                                         u8 buffer_id)
1571 {
1572         int res = SNTI_TRANSLATION_SUCCESS;
1573         int nvme_sc;
1574         struct nvme_dev *dev = ns->dev;
1575         struct nvme_command c;
1576         struct nvme_iod *iod = NULL;
1577         unsigned length;
1578
1579         memset(&c, 0, sizeof(c));
1580         c.common.opcode = opcode;
1581         if (opcode == nvme_admin_download_fw) {
1582                 if (hdr->iovec_count > 0) {
1583                         /* Assuming SGL is not allowed for this command */
1584                         res = nvme_trans_completion(hdr,
1585                                                 SAM_STAT_CHECK_CONDITION,
1586                                                 ILLEGAL_REQUEST,
1587                                                 SCSI_ASC_INVALID_CDB,
1588                                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1589                         goto out;
1590                 }
1591                 iod = nvme_map_user_pages(dev, DMA_TO_DEVICE,
1592                                 (unsigned long)hdr->dxferp, tot_len);
1593                 if (IS_ERR(iod)) {
1594                         res = PTR_ERR(iod);
1595                         goto out;
1596                 }
1597                 length = nvme_setup_prps(dev, iod, tot_len, GFP_KERNEL);
1598                 if (length != tot_len) {
1599                         res = -ENOMEM;
1600                         goto out_unmap;
1601                 }
1602
1603                 c.dlfw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
1604                 c.dlfw.prp2 = cpu_to_le64(iod->first_dma);
1605                 c.dlfw.numd = cpu_to_le32((tot_len/BYTES_TO_DWORDS) - 1);
1606                 c.dlfw.offset = cpu_to_le32(offset/BYTES_TO_DWORDS);
1607         } else if (opcode == nvme_admin_activate_fw) {
1608                 u32 cdw10 = buffer_id | NVME_FWACT_REPL_ACTV;
1609                 c.common.cdw10[0] = cpu_to_le32(cdw10);
1610         }
1611
1612         nvme_sc = nvme_submit_admin_cmd(dev, &c, NULL);
1613         res = nvme_trans_status_code(hdr, nvme_sc);
1614         if (res)
1615                 goto out_unmap;
1616         if (nvme_sc)
1617                 res = nvme_sc;
1618
1619  out_unmap:
1620         if (opcode == nvme_admin_download_fw) {
1621                 nvme_unmap_user_pages(dev, DMA_TO_DEVICE, iod);
1622                 nvme_free_iod(dev, iod);
1623         }
1624  out:
1625         return res;
1626 }
1627
1628 /* Mode Select Helper Functions */
1629
1630 static inline void nvme_trans_modesel_get_bd_len(u8 *parm_list, u8 cdb10,
1631                                                 u16 *bd_len, u8 *llbaa)
1632 {
1633         if (cdb10) {
1634                 /* 10 Byte CDB */
1635                 *bd_len = (parm_list[MODE_SELECT_10_BD_OFFSET] << 8) +
1636                         parm_list[MODE_SELECT_10_BD_OFFSET + 1];
1637                 *llbaa = parm_list[MODE_SELECT_10_LLBAA_OFFSET] &
1638                                 MODE_SELECT_10_LLBAA_MASK;
1639         } else {
1640                 /* 6 Byte CDB */
1641                 *bd_len = parm_list[MODE_SELECT_6_BD_OFFSET];
1642         }
1643 }
1644
1645 static void nvme_trans_modesel_save_bd(struct nvme_ns *ns, u8 *parm_list,
1646                                         u16 idx, u16 bd_len, u8 llbaa)
1647 {
1648         u16 bd_num;
1649
1650         bd_num = bd_len / ((llbaa == 0) ?
1651                         SHORT_DESC_BLOCK : LONG_DESC_BLOCK);
1652         /* Store block descriptor info if a FORMAT UNIT comes later */
1653         /* TODO Saving 1st BD info; what to do if multiple BD received? */
1654         if (llbaa == 0) {
1655                 /* Standard Block Descriptor - spc4r34 7.5.5.1 */
1656                 ns->mode_select_num_blocks =
1657                                 (parm_list[idx + 1] << 16) +
1658                                 (parm_list[idx + 2] << 8) +
1659                                 (parm_list[idx + 3]);
1660
1661                 ns->mode_select_block_len =
1662                                 (parm_list[idx + 5] << 16) +
1663                                 (parm_list[idx + 6] << 8) +
1664                                 (parm_list[idx + 7]);
1665         } else {
1666                 /* Long LBA Block Descriptor - sbc3r27 6.4.2.3 */
1667                 ns->mode_select_num_blocks =
1668                                 (((u64)parm_list[idx + 0]) << 56) +
1669                                 (((u64)parm_list[idx + 1]) << 48) +
1670                                 (((u64)parm_list[idx + 2]) << 40) +
1671                                 (((u64)parm_list[idx + 3]) << 32) +
1672                                 (((u64)parm_list[idx + 4]) << 24) +
1673                                 (((u64)parm_list[idx + 5]) << 16) +
1674                                 (((u64)parm_list[idx + 6]) << 8) +
1675                                 ((u64)parm_list[idx + 7]);
1676
1677                 ns->mode_select_block_len =
1678                                 (parm_list[idx + 12] << 24) +
1679                                 (parm_list[idx + 13] << 16) +
1680                                 (parm_list[idx + 14] << 8) +
1681                                 (parm_list[idx + 15]);
1682         }
1683 }
1684
1685 static int nvme_trans_modesel_get_mp(struct nvme_ns *ns, struct sg_io_hdr *hdr,
1686                                         u8 *mode_page, u8 page_code)
1687 {
1688         int res = SNTI_TRANSLATION_SUCCESS;
1689         int nvme_sc;
1690         struct nvme_dev *dev = ns->dev;
1691         unsigned dword11;
1692
1693         switch (page_code) {
1694         case MODE_PAGE_CACHING:
1695                 dword11 = ((mode_page[2] & CACHING_MODE_PAGE_WCE_MASK) ? 1 : 0);
1696                 nvme_sc = nvme_set_features(dev, NVME_FEAT_VOLATILE_WC, dword11,
1697                                             0, NULL);
1698                 res = nvme_trans_status_code(hdr, nvme_sc);
1699                 if (res)
1700                         break;
1701                 if (nvme_sc) {
1702                         res = nvme_sc;
1703                         break;
1704                 }
1705                 break;
1706         case MODE_PAGE_CONTROL:
1707                 break;
1708         case MODE_PAGE_POWER_CONDITION:
1709                 /* Verify the OS is not trying to set timers */
1710                 if ((mode_page[2] & 0x01) != 0 || (mode_page[3] & 0x0F) != 0) {
1711                         res = nvme_trans_completion(hdr,
1712                                                 SAM_STAT_CHECK_CONDITION,
1713                                                 ILLEGAL_REQUEST,
1714                                                 SCSI_ASC_INVALID_PARAMETER,
1715                                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1716                         if (!res)
1717                                 res = SNTI_INTERNAL_ERROR;
1718                         break;
1719                 }
1720                 break;
1721         default:
1722                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
1723                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
1724                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1725                 if (!res)
1726                         res = SNTI_INTERNAL_ERROR;
1727                 break;
1728         }
1729
1730         return res;
1731 }
1732
1733 static int nvme_trans_modesel_data(struct nvme_ns *ns, struct sg_io_hdr *hdr,
1734                                         u8 *cmd, u16 parm_list_len, u8 pf,
1735                                         u8 sp, u8 cdb10)
1736 {
1737         int res = SNTI_TRANSLATION_SUCCESS;
1738         u8 *parm_list;
1739         u16 bd_len;
1740         u8 llbaa = 0;
1741         u16 index, saved_index;
1742         u8 page_code;
1743         u16 mp_size;
1744
1745         /* Get parm list from data-in/out buffer */
1746         parm_list = kmalloc(parm_list_len, GFP_KERNEL);
1747         if (parm_list == NULL) {
1748                 res = -ENOMEM;
1749                 goto out;
1750         }
1751
1752         res = nvme_trans_copy_from_user(hdr, parm_list, parm_list_len);
1753         if (res != SNTI_TRANSLATION_SUCCESS)
1754                 goto out_mem;
1755
1756         nvme_trans_modesel_get_bd_len(parm_list, cdb10, &bd_len, &llbaa);
1757         index = (cdb10) ? (MODE_SELECT_10_MPH_SIZE) : (MODE_SELECT_6_MPH_SIZE);
1758
1759         if (bd_len != 0) {
1760                 /* Block Descriptors present, parse */
1761                 nvme_trans_modesel_save_bd(ns, parm_list, index, bd_len, llbaa);
1762                 index += bd_len;
1763         }
1764         saved_index = index;
1765
1766         /* Multiple mode pages may be present; iterate through all */
1767         /* In 1st Iteration, don't do NVME Command, only check for CDB errors */
1768         do {
1769                 page_code = parm_list[index] & MODE_SELECT_PAGE_CODE_MASK;
1770                 mp_size = parm_list[index + 1] + 2;
1771                 if ((page_code != MODE_PAGE_CACHING) &&
1772                     (page_code != MODE_PAGE_CONTROL) &&
1773                     (page_code != MODE_PAGE_POWER_CONDITION)) {
1774                         res = nvme_trans_completion(hdr,
1775                                                 SAM_STAT_CHECK_CONDITION,
1776                                                 ILLEGAL_REQUEST,
1777                                                 SCSI_ASC_INVALID_CDB,
1778                                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1779                         goto out_mem;
1780                 }
1781                 index += mp_size;
1782         } while (index < parm_list_len);
1783
1784         /* In 2nd Iteration, do the NVME Commands */
1785         index = saved_index;
1786         do {
1787                 page_code = parm_list[index] & MODE_SELECT_PAGE_CODE_MASK;
1788                 mp_size = parm_list[index + 1] + 2;
1789                 res = nvme_trans_modesel_get_mp(ns, hdr, &parm_list[index],
1790                                                                 page_code);
1791                 if (res != SNTI_TRANSLATION_SUCCESS)
1792                         break;
1793                 index += mp_size;
1794         } while (index < parm_list_len);
1795
1796  out_mem:
1797         kfree(parm_list);
1798  out:
1799         return res;
1800 }
1801
1802 /* Format Unit Helper Functions */
1803
1804 static int nvme_trans_fmt_set_blk_size_count(struct nvme_ns *ns,
1805                                              struct sg_io_hdr *hdr)
1806 {
1807         int res = SNTI_TRANSLATION_SUCCESS;
1808         int nvme_sc;
1809         struct nvme_dev *dev = ns->dev;
1810         dma_addr_t dma_addr;
1811         void *mem;
1812         struct nvme_id_ns *id_ns;
1813         u8 flbas;
1814
1815         /*
1816          * SCSI Expects a MODE SELECT would have been issued prior to
1817          * a FORMAT UNIT, and the block size and number would be used
1818          * from the block descriptor in it. If a MODE SELECT had not
1819          * been issued, FORMAT shall use the current values for both.
1820          */
1821
1822         if (ns->mode_select_num_blocks == 0 || ns->mode_select_block_len == 0) {
1823                 mem = dma_alloc_coherent(&dev->pci_dev->dev,
1824                         sizeof(struct nvme_id_ns), &dma_addr, GFP_KERNEL);
1825                 if (mem == NULL) {
1826                         res = -ENOMEM;
1827                         goto out;
1828                 }
1829                 /* nvme ns identify */
1830                 nvme_sc = nvme_identify(dev, ns->ns_id, 0, dma_addr);
1831                 res = nvme_trans_status_code(hdr, nvme_sc);
1832                 if (res)
1833                         goto out_dma;
1834                 if (nvme_sc) {
1835                         res = nvme_sc;
1836                         goto out_dma;
1837                 }
1838                 id_ns = mem;
1839
1840                 if (ns->mode_select_num_blocks == 0)
1841                         ns->mode_select_num_blocks = le64_to_cpu(id_ns->ncap);
1842                 if (ns->mode_select_block_len == 0) {
1843                         flbas = (id_ns->flbas) & 0x0F;
1844                         ns->mode_select_block_len =
1845                                                 (1 << (id_ns->lbaf[flbas].ds));
1846                 }
1847  out_dma:
1848                 dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns),
1849                                   mem, dma_addr);
1850         }
1851  out:
1852         return res;
1853 }
1854
1855 static int nvme_trans_fmt_get_parm_header(struct sg_io_hdr *hdr, u8 len,
1856                                         u8 format_prot_info, u8 *nvme_pf_code)
1857 {
1858         int res = SNTI_TRANSLATION_SUCCESS;
1859         u8 *parm_list;
1860         u8 pf_usage, pf_code;
1861
1862         parm_list = kmalloc(len, GFP_KERNEL);
1863         if (parm_list == NULL) {
1864                 res = -ENOMEM;
1865                 goto out;
1866         }
1867         res = nvme_trans_copy_from_user(hdr, parm_list, len);
1868         if (res != SNTI_TRANSLATION_SUCCESS)
1869                 goto out_mem;
1870
1871         if ((parm_list[FORMAT_UNIT_IMMED_OFFSET] &
1872                                 FORMAT_UNIT_IMMED_MASK) != 0) {
1873                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
1874                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
1875                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1876                 goto out_mem;
1877         }
1878
1879         if (len == FORMAT_UNIT_LONG_PARM_LIST_LEN &&
1880             (parm_list[FORMAT_UNIT_PROT_INT_OFFSET] & 0x0F) != 0) {
1881                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
1882                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
1883                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1884                 goto out_mem;
1885         }
1886         pf_usage = parm_list[FORMAT_UNIT_PROT_FIELD_USAGE_OFFSET] &
1887                         FORMAT_UNIT_PROT_FIELD_USAGE_MASK;
1888         pf_code = (pf_usage << 2) | format_prot_info;
1889         switch (pf_code) {
1890         case 0:
1891                 *nvme_pf_code = 0;
1892                 break;
1893         case 2:
1894                 *nvme_pf_code = 1;
1895                 break;
1896         case 3:
1897                 *nvme_pf_code = 2;
1898                 break;
1899         case 7:
1900                 *nvme_pf_code = 3;
1901                 break;
1902         default:
1903                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
1904                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
1905                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1906                 break;
1907         }
1908
1909  out_mem:
1910         kfree(parm_list);
1911  out:
1912         return res;
1913 }
1914
1915 static int nvme_trans_fmt_send_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr,
1916                                    u8 prot_info)
1917 {
1918         int res = SNTI_TRANSLATION_SUCCESS;
1919         int nvme_sc;
1920         struct nvme_dev *dev = ns->dev;
1921         dma_addr_t dma_addr;
1922         void *mem;
1923         struct nvme_id_ns *id_ns;
1924         u8 i;
1925         u8 flbas, nlbaf;
1926         u8 selected_lbaf = 0xFF;
1927         u32 cdw10 = 0;
1928         struct nvme_command c;
1929
1930         /* Loop thru LBAF's in id_ns to match reqd lbaf, put in cdw10 */
1931         mem = dma_alloc_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns),
1932                                                         &dma_addr, GFP_KERNEL);
1933         if (mem == NULL) {
1934                 res = -ENOMEM;
1935                 goto out;
1936         }
1937         /* nvme ns identify */
1938         nvme_sc = nvme_identify(dev, ns->ns_id, 0, dma_addr);
1939         res = nvme_trans_status_code(hdr, nvme_sc);
1940         if (res)
1941                 goto out_dma;
1942         if (nvme_sc) {
1943                 res = nvme_sc;
1944                 goto out_dma;
1945         }
1946         id_ns = mem;
1947         flbas = (id_ns->flbas) & 0x0F;
1948         nlbaf = id_ns->nlbaf;
1949
1950         for (i = 0; i < nlbaf; i++) {
1951                 if (ns->mode_select_block_len == (1 << (id_ns->lbaf[i].ds))) {
1952                         selected_lbaf = i;
1953                         break;
1954                 }
1955         }
1956         if (selected_lbaf > 0x0F) {
1957                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
1958                                 ILLEGAL_REQUEST, SCSI_ASC_INVALID_PARAMETER,
1959                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1960         }
1961         if (ns->mode_select_num_blocks != le64_to_cpu(id_ns->ncap)) {
1962                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
1963                                 ILLEGAL_REQUEST, SCSI_ASC_INVALID_PARAMETER,
1964                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
1965         }
1966
1967         cdw10 |= prot_info << 5;
1968         cdw10 |= selected_lbaf & 0x0F;
1969         memset(&c, 0, sizeof(c));
1970         c.format.opcode = nvme_admin_format_nvm;
1971         c.format.nsid = cpu_to_le32(ns->ns_id);
1972         c.format.cdw10 = cpu_to_le32(cdw10);
1973
1974         nvme_sc = nvme_submit_admin_cmd(dev, &c, NULL);
1975         res = nvme_trans_status_code(hdr, nvme_sc);
1976         if (res)
1977                 goto out_dma;
1978         if (nvme_sc)
1979                 res = nvme_sc;
1980
1981  out_dma:
1982         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns), mem,
1983                           dma_addr);
1984  out:
1985         return res;
1986 }
1987
1988 /* Read/Write Helper Functions */
1989
1990 static inline void nvme_trans_get_io_cdb6(u8 *cmd,
1991                                         struct nvme_trans_io_cdb *cdb_info)
1992 {
1993         cdb_info->fua = 0;
1994         cdb_info->prot_info = 0;
1995         cdb_info->lba = GET_U32_FROM_CDB(cmd, IO_6_CDB_LBA_OFFSET) &
1996                                         IO_6_CDB_LBA_MASK;
1997         cdb_info->xfer_len = GET_U8_FROM_CDB(cmd, IO_6_CDB_TX_LEN_OFFSET);
1998
1999         /* sbc3r27 sec 5.32 - TRANSFER LEN of 0 implies a 256 Block transfer */
2000         if (cdb_info->xfer_len == 0)
2001                 cdb_info->xfer_len = IO_6_DEFAULT_TX_LEN;
2002 }
2003
2004 static inline void nvme_trans_get_io_cdb10(u8 *cmd,
2005                                         struct nvme_trans_io_cdb *cdb_info)
2006 {
2007         cdb_info->fua = GET_U8_FROM_CDB(cmd, IO_10_CDB_FUA_OFFSET) &
2008                                         IO_CDB_FUA_MASK;
2009         cdb_info->prot_info = GET_U8_FROM_CDB(cmd, IO_10_CDB_WP_OFFSET) &
2010                                         IO_CDB_WP_MASK >> IO_CDB_WP_SHIFT;
2011         cdb_info->lba = GET_U32_FROM_CDB(cmd, IO_10_CDB_LBA_OFFSET);
2012         cdb_info->xfer_len = GET_U16_FROM_CDB(cmd, IO_10_CDB_TX_LEN_OFFSET);
2013 }
2014
2015 static inline void nvme_trans_get_io_cdb12(u8 *cmd,
2016                                         struct nvme_trans_io_cdb *cdb_info)
2017 {
2018         cdb_info->fua = GET_U8_FROM_CDB(cmd, IO_12_CDB_FUA_OFFSET) &
2019                                         IO_CDB_FUA_MASK;
2020         cdb_info->prot_info = GET_U8_FROM_CDB(cmd, IO_12_CDB_WP_OFFSET) &
2021                                         IO_CDB_WP_MASK >> IO_CDB_WP_SHIFT;
2022         cdb_info->lba = GET_U32_FROM_CDB(cmd, IO_12_CDB_LBA_OFFSET);
2023         cdb_info->xfer_len = GET_U32_FROM_CDB(cmd, IO_12_CDB_TX_LEN_OFFSET);
2024 }
2025
2026 static inline void nvme_trans_get_io_cdb16(u8 *cmd,
2027                                         struct nvme_trans_io_cdb *cdb_info)
2028 {
2029         cdb_info->fua = GET_U8_FROM_CDB(cmd, IO_16_CDB_FUA_OFFSET) &
2030                                         IO_CDB_FUA_MASK;
2031         cdb_info->prot_info = GET_U8_FROM_CDB(cmd, IO_16_CDB_WP_OFFSET) &
2032                                         IO_CDB_WP_MASK >> IO_CDB_WP_SHIFT;
2033         cdb_info->lba = GET_U64_FROM_CDB(cmd, IO_16_CDB_LBA_OFFSET);
2034         cdb_info->xfer_len = GET_U32_FROM_CDB(cmd, IO_16_CDB_TX_LEN_OFFSET);
2035 }
2036
2037 static inline u32 nvme_trans_io_get_num_cmds(struct sg_io_hdr *hdr,
2038                                         struct nvme_trans_io_cdb *cdb_info,
2039                                         u32 max_blocks)
2040 {
2041         /* If using iovecs, send one nvme command per vector */
2042         if (hdr->iovec_count > 0)
2043                 return hdr->iovec_count;
2044         else if (cdb_info->xfer_len > max_blocks)
2045                 return ((cdb_info->xfer_len - 1) / max_blocks) + 1;
2046         else
2047                 return 1;
2048 }
2049
2050 static u16 nvme_trans_io_get_control(struct nvme_ns *ns,
2051                                         struct nvme_trans_io_cdb *cdb_info)
2052 {
2053         u16 control = 0;
2054
2055         /* When Protection information support is added, implement here */
2056
2057         if (cdb_info->fua > 0)
2058                 control |= NVME_RW_FUA;
2059
2060         return control;
2061 }
2062
2063 static int nvme_trans_do_nvme_io(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2064                                 struct nvme_trans_io_cdb *cdb_info, u8 is_write)
2065 {
2066         int res = SNTI_TRANSLATION_SUCCESS;
2067         int nvme_sc;
2068         struct nvme_dev *dev = ns->dev;
2069         u32 num_cmds;
2070         struct nvme_iod *iod;
2071         u64 unit_len;
2072         u64 unit_num_blocks;    /* Number of blocks to xfer in each nvme cmd */
2073         u32 retcode;
2074         u32 i = 0;
2075         u64 nvme_offset = 0;
2076         void __user *next_mapping_addr;
2077         struct nvme_command c;
2078         u8 opcode = (is_write ? nvme_cmd_write : nvme_cmd_read);
2079         u16 control;
2080         u32 max_blocks = queue_max_hw_sectors(ns->queue);
2081
2082         num_cmds = nvme_trans_io_get_num_cmds(hdr, cdb_info, max_blocks);
2083
2084         /*
2085          * This loop handles two cases.
2086          * First, when an SGL is used in the form of an iovec list:
2087          *   - Use iov_base as the next mapping address for the nvme command_id
2088          *   - Use iov_len as the data transfer length for the command.
2089          * Second, when we have a single buffer
2090          *   - If larger than max_blocks, split into chunks, offset
2091          *        each nvme command accordingly.
2092          */
2093         for (i = 0; i < num_cmds; i++) {
2094                 memset(&c, 0, sizeof(c));
2095                 if (hdr->iovec_count > 0) {
2096                         struct sg_iovec sgl;
2097
2098                         retcode = copy_from_user(&sgl, hdr->dxferp +
2099                                         i * sizeof(struct sg_iovec),
2100                                         sizeof(struct sg_iovec));
2101                         if (retcode)
2102                                 return -EFAULT;
2103                         unit_len = sgl.iov_len;
2104                         unit_num_blocks = unit_len >> ns->lba_shift;
2105                         next_mapping_addr = sgl.iov_base;
2106                 } else {
2107                         unit_num_blocks = min((u64)max_blocks,
2108                                         (cdb_info->xfer_len - nvme_offset));
2109                         unit_len = unit_num_blocks << ns->lba_shift;
2110                         next_mapping_addr = hdr->dxferp +
2111                                         ((1 << ns->lba_shift) * nvme_offset);
2112                 }
2113
2114                 c.rw.opcode = opcode;
2115                 c.rw.nsid = cpu_to_le32(ns->ns_id);
2116                 c.rw.slba = cpu_to_le64(cdb_info->lba + nvme_offset);
2117                 c.rw.length = cpu_to_le16(unit_num_blocks - 1);
2118                 control = nvme_trans_io_get_control(ns, cdb_info);
2119                 c.rw.control = cpu_to_le16(control);
2120
2121                 iod = nvme_map_user_pages(dev,
2122                         (is_write) ? DMA_TO_DEVICE : DMA_FROM_DEVICE,
2123                         (unsigned long)next_mapping_addr, unit_len);
2124                 if (IS_ERR(iod)) {
2125                         res = PTR_ERR(iod);
2126                         goto out;
2127                 }
2128                 retcode = nvme_setup_prps(dev, iod, unit_len, GFP_KERNEL);
2129                 if (retcode != unit_len) {
2130                         nvme_unmap_user_pages(dev,
2131                                 (is_write) ? DMA_TO_DEVICE : DMA_FROM_DEVICE,
2132                                 iod);
2133                         nvme_free_iod(dev, iod);
2134                         res = -ENOMEM;
2135                         goto out;
2136                 }
2137                 c.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
2138                 c.rw.prp2 = cpu_to_le64(iod->first_dma);
2139
2140                 nvme_offset += unit_num_blocks;
2141
2142                 nvme_sc = nvme_submit_io_cmd(dev, ns, &c, NULL);
2143                 if (nvme_sc != NVME_SC_SUCCESS) {
2144                         nvme_unmap_user_pages(dev,
2145                                 (is_write) ? DMA_TO_DEVICE : DMA_FROM_DEVICE,
2146                                 iod);
2147                         nvme_free_iod(dev, iod);
2148                         res = nvme_trans_status_code(hdr, nvme_sc);
2149                         goto out;
2150                 }
2151                 nvme_unmap_user_pages(dev,
2152                                 (is_write) ? DMA_TO_DEVICE : DMA_FROM_DEVICE,
2153                                 iod);
2154                 nvme_free_iod(dev, iod);
2155         }
2156         res = nvme_trans_status_code(hdr, NVME_SC_SUCCESS);
2157
2158  out:
2159         return res;
2160 }
2161
2162
2163 /* SCSI Command Translation Functions */
2164
2165 static int nvme_trans_io(struct nvme_ns *ns, struct sg_io_hdr *hdr, u8 is_write,
2166                                                         u8 *cmd)
2167 {
2168         int res = SNTI_TRANSLATION_SUCCESS;
2169         struct nvme_trans_io_cdb cdb_info;
2170         u8 opcode = cmd[0];
2171         u64 xfer_bytes;
2172         u64 sum_iov_len = 0;
2173         struct sg_iovec sgl;
2174         int i;
2175         size_t not_copied;
2176
2177         /* Extract Fields from CDB */
2178         switch (opcode) {
2179         case WRITE_6:
2180         case READ_6:
2181                 nvme_trans_get_io_cdb6(cmd, &cdb_info);
2182                 break;
2183         case WRITE_10:
2184         case READ_10:
2185                 nvme_trans_get_io_cdb10(cmd, &cdb_info);
2186                 break;
2187         case WRITE_12:
2188         case READ_12:
2189                 nvme_trans_get_io_cdb12(cmd, &cdb_info);
2190                 break;
2191         case WRITE_16:
2192         case READ_16:
2193                 nvme_trans_get_io_cdb16(cmd, &cdb_info);
2194                 break;
2195         default:
2196                 /* Will never really reach here */
2197                 res = SNTI_INTERNAL_ERROR;
2198                 goto out;
2199         }
2200
2201         /* Calculate total length of transfer (in bytes) */
2202         if (hdr->iovec_count > 0) {
2203                 for (i = 0; i < hdr->iovec_count; i++) {
2204                         not_copied = copy_from_user(&sgl, hdr->dxferp +
2205                                                 i * sizeof(struct sg_iovec),
2206                                                 sizeof(struct sg_iovec));
2207                         if (not_copied)
2208                                 return -EFAULT;
2209                         sum_iov_len += sgl.iov_len;
2210                         /* IO vector sizes should be multiples of block size */
2211                         if (sgl.iov_len % (1 << ns->lba_shift) != 0) {
2212                                 res = nvme_trans_completion(hdr,
2213                                                 SAM_STAT_CHECK_CONDITION,
2214                                                 ILLEGAL_REQUEST,
2215                                                 SCSI_ASC_INVALID_PARAMETER,
2216                                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2217                                 goto out;
2218                         }
2219                 }
2220         } else {
2221                 sum_iov_len = hdr->dxfer_len;
2222         }
2223
2224         /* As Per sg ioctl howto, if the lengths differ, use the lower one */
2225         xfer_bytes = min(((u64)hdr->dxfer_len), sum_iov_len);
2226
2227         /* If block count and actual data buffer size dont match, error out */
2228         if (xfer_bytes != (cdb_info.xfer_len << ns->lba_shift)) {
2229                 res = -EINVAL;
2230                 goto out;
2231         }
2232
2233         /* Check for 0 length transfer - it is not illegal */
2234         if (cdb_info.xfer_len == 0)
2235                 goto out;
2236
2237         /* Send NVMe IO Command(s) */
2238         res = nvme_trans_do_nvme_io(ns, hdr, &cdb_info, is_write);
2239         if (res != SNTI_TRANSLATION_SUCCESS)
2240                 goto out;
2241
2242  out:
2243         return res;
2244 }
2245
2246 static int nvme_trans_inquiry(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2247                                                         u8 *cmd)
2248 {
2249         int res = SNTI_TRANSLATION_SUCCESS;
2250         u8 evpd;
2251         u8 page_code;
2252         int alloc_len;
2253         u8 *inq_response;
2254
2255         evpd = GET_INQ_EVPD_BIT(cmd);
2256         page_code = GET_INQ_PAGE_CODE(cmd);
2257         alloc_len = GET_INQ_ALLOC_LENGTH(cmd);
2258
2259         inq_response = kmalloc(alloc_len, GFP_KERNEL);
2260         if (inq_response == NULL) {
2261                 res = -ENOMEM;
2262                 goto out_mem;
2263         }
2264
2265         if (evpd == 0) {
2266                 if (page_code == INQ_STANDARD_INQUIRY_PAGE) {
2267                         res = nvme_trans_standard_inquiry_page(ns, hdr,
2268                                                 inq_response, alloc_len);
2269                 } else {
2270                         res = nvme_trans_completion(hdr,
2271                                                 SAM_STAT_CHECK_CONDITION,
2272                                                 ILLEGAL_REQUEST,
2273                                                 SCSI_ASC_INVALID_CDB,
2274                                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2275                 }
2276         } else {
2277                 switch (page_code) {
2278                 case VPD_SUPPORTED_PAGES:
2279                         res = nvme_trans_supported_vpd_pages(ns, hdr,
2280                                                 inq_response, alloc_len);
2281                         break;
2282                 case VPD_SERIAL_NUMBER:
2283                         res = nvme_trans_unit_serial_page(ns, hdr, inq_response,
2284                                                                 alloc_len);
2285                         break;
2286                 case VPD_DEVICE_IDENTIFIERS:
2287                         res = nvme_trans_device_id_page(ns, hdr, inq_response,
2288                                                                 alloc_len);
2289                         break;
2290                 case VPD_EXTENDED_INQUIRY:
2291                         res = nvme_trans_ext_inq_page(ns, hdr, alloc_len);
2292                         break;
2293                 case VPD_BLOCK_LIMITS:
2294                         res = nvme_trans_bdev_limits_page(ns, hdr, inq_response,
2295                                                                 alloc_len);
2296                         break;
2297                 case VPD_BLOCK_DEV_CHARACTERISTICS:
2298                         res = nvme_trans_bdev_char_page(ns, hdr, alloc_len);
2299                         break;
2300                 default:
2301                         res = nvme_trans_completion(hdr,
2302                                                 SAM_STAT_CHECK_CONDITION,
2303                                                 ILLEGAL_REQUEST,
2304                                                 SCSI_ASC_INVALID_CDB,
2305                                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2306                         break;
2307                 }
2308         }
2309         kfree(inq_response);
2310  out_mem:
2311         return res;
2312 }
2313
2314 static int nvme_trans_log_sense(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2315                                                         u8 *cmd)
2316 {
2317         int res = SNTI_TRANSLATION_SUCCESS;
2318         u16 alloc_len;
2319         u8 sp;
2320         u8 pc;
2321         u8 page_code;
2322
2323         sp = GET_U8_FROM_CDB(cmd, LOG_SENSE_CDB_SP_OFFSET);
2324         if (sp != LOG_SENSE_CDB_SP_NOT_ENABLED) {
2325                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2326                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2327                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2328                 goto out;
2329         }
2330         pc = GET_U8_FROM_CDB(cmd, LOG_SENSE_CDB_PC_OFFSET);
2331         page_code = pc & LOG_SENSE_CDB_PAGE_CODE_MASK;
2332         pc = (pc & LOG_SENSE_CDB_PC_MASK) >> LOG_SENSE_CDB_PC_SHIFT;
2333         if (pc != LOG_SENSE_CDB_PC_CUMULATIVE_VALUES) {
2334                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2335                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2336                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2337                 goto out;
2338         }
2339         alloc_len = GET_U16_FROM_CDB(cmd, LOG_SENSE_CDB_ALLOC_LENGTH_OFFSET);
2340         switch (page_code) {
2341         case LOG_PAGE_SUPPORTED_LOG_PAGES_PAGE:
2342                 res = nvme_trans_log_supp_pages(ns, hdr, alloc_len);
2343                 break;
2344         case LOG_PAGE_INFORMATIONAL_EXCEPTIONS_PAGE:
2345                 res = nvme_trans_log_info_exceptions(ns, hdr, alloc_len);
2346                 break;
2347         case LOG_PAGE_TEMPERATURE_PAGE:
2348                 res = nvme_trans_log_temperature(ns, hdr, alloc_len);
2349                 break;
2350         default:
2351                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2352                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2353                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2354                 break;
2355         }
2356
2357  out:
2358         return res;
2359 }
2360
2361 static int nvme_trans_mode_select(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2362                                                         u8 *cmd)
2363 {
2364         int res = SNTI_TRANSLATION_SUCCESS;
2365         u8 cdb10 = 0;
2366         u16 parm_list_len;
2367         u8 page_format;
2368         u8 save_pages;
2369
2370         page_format = GET_U8_FROM_CDB(cmd, MODE_SELECT_CDB_PAGE_FORMAT_OFFSET);
2371         page_format &= MODE_SELECT_CDB_PAGE_FORMAT_MASK;
2372
2373         save_pages = GET_U8_FROM_CDB(cmd, MODE_SELECT_CDB_SAVE_PAGES_OFFSET);
2374         save_pages &= MODE_SELECT_CDB_SAVE_PAGES_MASK;
2375
2376         if (GET_OPCODE(cmd) == MODE_SELECT) {
2377                 parm_list_len = GET_U8_FROM_CDB(cmd,
2378                                 MODE_SELECT_6_CDB_PARAM_LIST_LENGTH_OFFSET);
2379         } else {
2380                 parm_list_len = GET_U16_FROM_CDB(cmd,
2381                                 MODE_SELECT_10_CDB_PARAM_LIST_LENGTH_OFFSET);
2382                 cdb10 = 1;
2383         }
2384
2385         if (parm_list_len != 0) {
2386                 /*
2387                  * According to SPC-4 r24, a paramter list length field of 0
2388                  * shall not be considered an error
2389                  */
2390                 res = nvme_trans_modesel_data(ns, hdr, cmd, parm_list_len,
2391                                                 page_format, save_pages, cdb10);
2392         }
2393
2394         return res;
2395 }
2396
2397 static int nvme_trans_mode_sense(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2398                                                         u8 *cmd)
2399 {
2400         int res = SNTI_TRANSLATION_SUCCESS;
2401         u16 alloc_len;
2402         u8 cdb10 = 0;
2403         u8 page_code;
2404         u8 pc;
2405
2406         if (GET_OPCODE(cmd) == MODE_SENSE) {
2407                 alloc_len = GET_U8_FROM_CDB(cmd, MODE_SENSE6_ALLOC_LEN_OFFSET);
2408         } else {
2409                 alloc_len = GET_U16_FROM_CDB(cmd,
2410                                                 MODE_SENSE10_ALLOC_LEN_OFFSET);
2411                 cdb10 = 1;
2412         }
2413
2414         pc = GET_U8_FROM_CDB(cmd, MODE_SENSE_PAGE_CONTROL_OFFSET) &
2415                                                 MODE_SENSE_PAGE_CONTROL_MASK;
2416         if (pc != MODE_SENSE_PC_CURRENT_VALUES) {
2417                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2418                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2419                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2420                 goto out;
2421         }
2422
2423         page_code = GET_U8_FROM_CDB(cmd, MODE_SENSE_PAGE_CODE_OFFSET) &
2424                                         MODE_SENSE_PAGE_CODE_MASK;
2425         switch (page_code) {
2426         case MODE_PAGE_CACHING:
2427                 res = nvme_trans_mode_page_create(ns, hdr, cmd, alloc_len,
2428                                                 cdb10,
2429                                                 &nvme_trans_fill_caching_page,
2430                                                 MODE_PAGE_CACHING_LEN);
2431                 break;
2432         case MODE_PAGE_CONTROL:
2433                 res = nvme_trans_mode_page_create(ns, hdr, cmd, alloc_len,
2434                                                 cdb10,
2435                                                 &nvme_trans_fill_control_page,
2436                                                 MODE_PAGE_CONTROL_LEN);
2437                 break;
2438         case MODE_PAGE_POWER_CONDITION:
2439                 res = nvme_trans_mode_page_create(ns, hdr, cmd, alloc_len,
2440                                                 cdb10,
2441                                                 &nvme_trans_fill_pow_cnd_page,
2442                                                 MODE_PAGE_POW_CND_LEN);
2443                 break;
2444         case MODE_PAGE_INFO_EXCEP:
2445                 res = nvme_trans_mode_page_create(ns, hdr, cmd, alloc_len,
2446                                                 cdb10,
2447                                                 &nvme_trans_fill_inf_exc_page,
2448                                                 MODE_PAGE_INF_EXC_LEN);
2449                 break;
2450         case MODE_PAGE_RETURN_ALL:
2451                 res = nvme_trans_mode_page_create(ns, hdr, cmd, alloc_len,
2452                                                 cdb10,
2453                                                 &nvme_trans_fill_all_pages,
2454                                                 MODE_PAGE_ALL_LEN);
2455                 break;
2456         default:
2457                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2458                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2459                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2460                 break;
2461         }
2462
2463  out:
2464         return res;
2465 }
2466
2467 static int nvme_trans_read_capacity(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2468                                                         u8 *cmd)
2469 {
2470         int res = SNTI_TRANSLATION_SUCCESS;
2471         int nvme_sc;
2472         u32 alloc_len = READ_CAP_10_RESP_SIZE;
2473         u32 resp_size = READ_CAP_10_RESP_SIZE;
2474         u32 xfer_len;
2475         u8 cdb16;
2476         struct nvme_dev *dev = ns->dev;
2477         dma_addr_t dma_addr;
2478         void *mem;
2479         struct nvme_id_ns *id_ns;
2480         u8 *response;
2481
2482         cdb16 = IS_READ_CAP_16(cmd);
2483         if (cdb16) {
2484                 alloc_len = GET_READ_CAP_16_ALLOC_LENGTH(cmd);
2485                 resp_size = READ_CAP_16_RESP_SIZE;
2486         }
2487
2488         mem = dma_alloc_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns),
2489                                                         &dma_addr, GFP_KERNEL);
2490         if (mem == NULL) {
2491                 res = -ENOMEM;
2492                 goto out;
2493         }
2494         /* nvme ns identify */
2495         nvme_sc = nvme_identify(dev, ns->ns_id, 0, dma_addr);
2496         res = nvme_trans_status_code(hdr, nvme_sc);
2497         if (res)
2498                 goto out_dma;
2499         if (nvme_sc) {
2500                 res = nvme_sc;
2501                 goto out_dma;
2502         }
2503         id_ns = mem;
2504
2505         response = kzalloc(resp_size, GFP_KERNEL);
2506         if (response == NULL) {
2507                 res = -ENOMEM;
2508                 goto out_dma;
2509         }
2510         nvme_trans_fill_read_cap(response, id_ns, cdb16);
2511
2512         xfer_len = min(alloc_len, resp_size);
2513         res = nvme_trans_copy_to_user(hdr, response, xfer_len);
2514
2515         kfree(response);
2516  out_dma:
2517         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ns), mem,
2518                           dma_addr);
2519  out:
2520         return res;
2521 }
2522
2523 static int nvme_trans_report_luns(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2524                                                         u8 *cmd)
2525 {
2526         int res = SNTI_TRANSLATION_SUCCESS;
2527         int nvme_sc;
2528         u32 alloc_len, xfer_len, resp_size;
2529         u8 select_report;
2530         u8 *response;
2531         struct nvme_dev *dev = ns->dev;
2532         dma_addr_t dma_addr;
2533         void *mem;
2534         struct nvme_id_ctrl *id_ctrl;
2535         u32 ll_length, lun_id;
2536         u8 lun_id_offset = REPORT_LUNS_FIRST_LUN_OFFSET;
2537         __be32 tmp_len;
2538
2539         alloc_len = GET_REPORT_LUNS_ALLOC_LENGTH(cmd);
2540         select_report = GET_U8_FROM_CDB(cmd, REPORT_LUNS_SR_OFFSET);
2541
2542         if ((select_report != ALL_LUNS_RETURNED) &&
2543             (select_report != ALL_WELL_KNOWN_LUNS_RETURNED) &&
2544             (select_report != RESTRICTED_LUNS_RETURNED)) {
2545                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2546                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2547                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2548                 goto out;
2549         } else {
2550                 /* NVMe Controller Identify */
2551                 mem = dma_alloc_coherent(&dev->pci_dev->dev,
2552                                         sizeof(struct nvme_id_ctrl),
2553                                         &dma_addr, GFP_KERNEL);
2554                 if (mem == NULL) {
2555                         res = -ENOMEM;
2556                         goto out;
2557                 }
2558                 nvme_sc = nvme_identify(dev, 0, 1, dma_addr);
2559                 res = nvme_trans_status_code(hdr, nvme_sc);
2560                 if (res)
2561                         goto out_dma;
2562                 if (nvme_sc) {
2563                         res = nvme_sc;
2564                         goto out_dma;
2565                 }
2566                 id_ctrl = mem;
2567                 ll_length = le32_to_cpu(id_ctrl->nn) * LUN_ENTRY_SIZE;
2568                 resp_size = ll_length + LUN_DATA_HEADER_SIZE;
2569
2570                 if (alloc_len < resp_size) {
2571                         res = nvme_trans_completion(hdr,
2572                                         SAM_STAT_CHECK_CONDITION,
2573                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2574                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2575                         goto out_dma;
2576                 }
2577
2578                 response = kzalloc(resp_size, GFP_KERNEL);
2579                 if (response == NULL) {
2580                         res = -ENOMEM;
2581                         goto out_dma;
2582                 }
2583
2584                 /* The first LUN ID will always be 0 per the SAM spec */
2585                 for (lun_id = 0; lun_id < le32_to_cpu(id_ctrl->nn); lun_id++) {
2586                         /*
2587                          * Set the LUN Id and then increment to the next LUN
2588                          * location in the parameter data.
2589                          */
2590                         __be64 tmp_id = cpu_to_be64(lun_id);
2591                         memcpy(&response[lun_id_offset], &tmp_id, sizeof(u64));
2592                         lun_id_offset += LUN_ENTRY_SIZE;
2593                 }
2594                 tmp_len = cpu_to_be32(ll_length);
2595                 memcpy(response, &tmp_len, sizeof(u32));
2596         }
2597
2598         xfer_len = min(alloc_len, resp_size);
2599         res = nvme_trans_copy_to_user(hdr, response, xfer_len);
2600
2601         kfree(response);
2602  out_dma:
2603         dma_free_coherent(&dev->pci_dev->dev, sizeof(struct nvme_id_ctrl), mem,
2604                           dma_addr);
2605  out:
2606         return res;
2607 }
2608
2609 static int nvme_trans_request_sense(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2610                                                         u8 *cmd)
2611 {
2612         int res = SNTI_TRANSLATION_SUCCESS;
2613         u8 alloc_len, xfer_len, resp_size;
2614         u8 desc_format;
2615         u8 *response;
2616
2617         alloc_len = GET_REQUEST_SENSE_ALLOC_LENGTH(cmd);
2618         desc_format = GET_U8_FROM_CDB(cmd, REQUEST_SENSE_DESC_OFFSET);
2619         desc_format &= REQUEST_SENSE_DESC_MASK;
2620
2621         resp_size = ((desc_format) ? (DESC_FMT_SENSE_DATA_SIZE) :
2622                                         (FIXED_FMT_SENSE_DATA_SIZE));
2623         response = kzalloc(resp_size, GFP_KERNEL);
2624         if (response == NULL) {
2625                 res = -ENOMEM;
2626                 goto out;
2627         }
2628
2629         if (desc_format == DESCRIPTOR_FORMAT_SENSE_DATA_TYPE) {
2630                 /* Descriptor Format Sense Data */
2631                 response[0] = DESC_FORMAT_SENSE_DATA;
2632                 response[1] = NO_SENSE;
2633                 /* TODO How is LOW POWER CONDITION ON handled? (byte 2) */
2634                 response[2] = SCSI_ASC_NO_SENSE;
2635                 response[3] = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
2636                 /* SDAT_OVFL = 0 | Additional Sense Length = 0 */
2637         } else {
2638                 /* Fixed Format Sense Data */
2639                 response[0] = FIXED_SENSE_DATA;
2640                 /* Byte 1 = Obsolete */
2641                 response[2] = NO_SENSE; /* FM, EOM, ILI, SDAT_OVFL = 0 */
2642                 /* Bytes 3-6 - Information - set to zero */
2643                 response[7] = FIXED_SENSE_DATA_ADD_LENGTH;
2644                 /* Bytes 8-11 - Cmd Specific Information - set to zero */
2645                 response[12] = SCSI_ASC_NO_SENSE;
2646                 response[13] = SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
2647                 /* Byte 14 = Field Replaceable Unit Code = 0 */
2648                 /* Bytes 15-17 - SKSV=0; Sense Key Specific = 0 */
2649         }
2650
2651         xfer_len = min(alloc_len, resp_size);
2652         res = nvme_trans_copy_to_user(hdr, response, xfer_len);
2653
2654         kfree(response);
2655  out:
2656         return res;
2657 }
2658
2659 static int nvme_trans_security_protocol(struct nvme_ns *ns,
2660                                         struct sg_io_hdr *hdr,
2661                                         u8 *cmd)
2662 {
2663         return nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2664                                 ILLEGAL_REQUEST, SCSI_ASC_ILLEGAL_COMMAND,
2665                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2666 }
2667
2668 static int nvme_trans_start_stop(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2669                                                         u8 *cmd)
2670 {
2671         int res = SNTI_TRANSLATION_SUCCESS;
2672         int nvme_sc;
2673         struct nvme_command c;
2674         u8 immed, pcmod, pc, no_flush, start;
2675
2676         immed = GET_U8_FROM_CDB(cmd, START_STOP_UNIT_CDB_IMMED_OFFSET);
2677         pcmod = GET_U8_FROM_CDB(cmd, START_STOP_UNIT_CDB_POWER_COND_MOD_OFFSET);
2678         pc = GET_U8_FROM_CDB(cmd, START_STOP_UNIT_CDB_POWER_COND_OFFSET);
2679         no_flush = GET_U8_FROM_CDB(cmd, START_STOP_UNIT_CDB_NO_FLUSH_OFFSET);
2680         start = GET_U8_FROM_CDB(cmd, START_STOP_UNIT_CDB_START_OFFSET);
2681
2682         immed &= START_STOP_UNIT_CDB_IMMED_MASK;
2683         pcmod &= START_STOP_UNIT_CDB_POWER_COND_MOD_MASK;
2684         pc = (pc & START_STOP_UNIT_CDB_POWER_COND_MASK) >> NIBBLE_SHIFT;
2685         no_flush &= START_STOP_UNIT_CDB_NO_FLUSH_MASK;
2686         start &= START_STOP_UNIT_CDB_START_MASK;
2687
2688         if (immed != 0) {
2689                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2690                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2691                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2692         } else {
2693                 if (no_flush == 0) {
2694                         /* Issue NVME FLUSH command prior to START STOP UNIT */
2695                         memset(&c, 0, sizeof(c));
2696                         c.common.opcode = nvme_cmd_flush;
2697                         c.common.nsid = cpu_to_le32(ns->ns_id);
2698
2699                         nvme_sc = nvme_submit_io_cmd(ns->dev, ns, &c, NULL);
2700                         res = nvme_trans_status_code(hdr, nvme_sc);
2701                         if (res)
2702                                 goto out;
2703                         if (nvme_sc) {
2704                                 res = nvme_sc;
2705                                 goto out;
2706                         }
2707                 }
2708                 /* Setup the expected power state transition */
2709                 res = nvme_trans_power_state(ns, hdr, pc, pcmod, start);
2710         }
2711
2712  out:
2713         return res;
2714 }
2715
2716 static int nvme_trans_synchronize_cache(struct nvme_ns *ns,
2717                                         struct sg_io_hdr *hdr, u8 *cmd)
2718 {
2719         int res = SNTI_TRANSLATION_SUCCESS;
2720         int nvme_sc;
2721         struct nvme_command c;
2722
2723         memset(&c, 0, sizeof(c));
2724         c.common.opcode = nvme_cmd_flush;
2725         c.common.nsid = cpu_to_le32(ns->ns_id);
2726
2727         nvme_sc = nvme_submit_io_cmd(ns->dev, ns, &c, NULL);
2728
2729         res = nvme_trans_status_code(hdr, nvme_sc);
2730         if (res)
2731                 goto out;
2732         if (nvme_sc)
2733                 res = nvme_sc;
2734
2735  out:
2736         return res;
2737 }
2738
2739 static int nvme_trans_format_unit(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2740                                                         u8 *cmd)
2741 {
2742         int res = SNTI_TRANSLATION_SUCCESS;
2743         u8 parm_hdr_len = 0;
2744         u8 nvme_pf_code = 0;
2745         u8 format_prot_info, long_list, format_data;
2746
2747         format_prot_info = GET_U8_FROM_CDB(cmd,
2748                                 FORMAT_UNIT_CDB_FORMAT_PROT_INFO_OFFSET);
2749         long_list = GET_U8_FROM_CDB(cmd, FORMAT_UNIT_CDB_LONG_LIST_OFFSET);
2750         format_data = GET_U8_FROM_CDB(cmd, FORMAT_UNIT_CDB_FORMAT_DATA_OFFSET);
2751
2752         format_prot_info = (format_prot_info &
2753                                 FORMAT_UNIT_CDB_FORMAT_PROT_INFO_MASK) >>
2754                                 FORMAT_UNIT_CDB_FORMAT_PROT_INFO_SHIFT;
2755         long_list &= FORMAT_UNIT_CDB_LONG_LIST_MASK;
2756         format_data &= FORMAT_UNIT_CDB_FORMAT_DATA_MASK;
2757
2758         if (format_data != 0) {
2759                 if (format_prot_info != 0) {
2760                         if (long_list == 0)
2761                                 parm_hdr_len = FORMAT_UNIT_SHORT_PARM_LIST_LEN;
2762                         else
2763                                 parm_hdr_len = FORMAT_UNIT_LONG_PARM_LIST_LEN;
2764                 }
2765         } else if (format_data == 0 && format_prot_info != 0) {
2766                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2767                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2768                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2769                 goto out;
2770         }
2771
2772         /* Get parm header from data-in/out buffer */
2773         /*
2774          * According to the translation spec, the only fields in the parameter
2775          * list we are concerned with are in the header. So allocate only that.
2776          */
2777         if (parm_hdr_len > 0) {
2778                 res = nvme_trans_fmt_get_parm_header(hdr, parm_hdr_len,
2779                                         format_prot_info, &nvme_pf_code);
2780                 if (res != SNTI_TRANSLATION_SUCCESS)
2781                         goto out;
2782         }
2783
2784         /* Attempt to activate any previously downloaded firmware image */
2785         res = nvme_trans_send_fw_cmd(ns, hdr, nvme_admin_activate_fw, 0, 0, 0);
2786
2787         /* Determine Block size and count and send format command */
2788         res = nvme_trans_fmt_set_blk_size_count(ns, hdr);
2789         if (res != SNTI_TRANSLATION_SUCCESS)
2790                 goto out;
2791
2792         res = nvme_trans_fmt_send_cmd(ns, hdr, nvme_pf_code);
2793
2794  out:
2795         return res;
2796 }
2797
2798 static int nvme_trans_test_unit_ready(struct nvme_ns *ns,
2799                                         struct sg_io_hdr *hdr,
2800                                         u8 *cmd)
2801 {
2802         int res = SNTI_TRANSLATION_SUCCESS;
2803         struct nvme_dev *dev = ns->dev;
2804
2805         if (!(readl(&dev->bar->csts) & NVME_CSTS_RDY))
2806                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2807                                             NOT_READY, SCSI_ASC_LUN_NOT_READY,
2808                                             SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2809         else
2810                 res = nvme_trans_completion(hdr, SAM_STAT_GOOD, NO_SENSE, 0, 0);
2811
2812         return res;
2813 }
2814
2815 static int nvme_trans_write_buffer(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2816                                                         u8 *cmd)
2817 {
2818         int res = SNTI_TRANSLATION_SUCCESS;
2819         u32 buffer_offset, parm_list_length;
2820         u8 buffer_id, mode;
2821
2822         parm_list_length =
2823                 GET_U24_FROM_CDB(cmd, WRITE_BUFFER_CDB_PARM_LIST_LENGTH_OFFSET);
2824         if (parm_list_length % BYTES_TO_DWORDS != 0) {
2825                 /* NVMe expects Firmware file to be a whole number of DWORDS */
2826                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2827                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2828                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2829                 goto out;
2830         }
2831         buffer_id = GET_U8_FROM_CDB(cmd, WRITE_BUFFER_CDB_BUFFER_ID_OFFSET);
2832         if (buffer_id > NVME_MAX_FIRMWARE_SLOT) {
2833                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2834                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2835                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2836                 goto out;
2837         }
2838         mode = GET_U8_FROM_CDB(cmd, WRITE_BUFFER_CDB_MODE_OFFSET) &
2839                                                 WRITE_BUFFER_CDB_MODE_MASK;
2840         buffer_offset =
2841                 GET_U24_FROM_CDB(cmd, WRITE_BUFFER_CDB_BUFFER_OFFSET_OFFSET);
2842
2843         switch (mode) {
2844         case DOWNLOAD_SAVE_ACTIVATE:
2845                 res = nvme_trans_send_fw_cmd(ns, hdr, nvme_admin_download_fw,
2846                                                 parm_list_length, buffer_offset,
2847                                                 buffer_id);
2848                 if (res != SNTI_TRANSLATION_SUCCESS)
2849                         goto out;
2850                 res = nvme_trans_send_fw_cmd(ns, hdr, nvme_admin_activate_fw,
2851                                                 parm_list_length, buffer_offset,
2852                                                 buffer_id);
2853                 break;
2854         case DOWNLOAD_SAVE_DEFER_ACTIVATE:
2855                 res = nvme_trans_send_fw_cmd(ns, hdr, nvme_admin_download_fw,
2856                                                 parm_list_length, buffer_offset,
2857                                                 buffer_id);
2858                 break;
2859         case ACTIVATE_DEFERRED_MICROCODE:
2860                 res = nvme_trans_send_fw_cmd(ns, hdr, nvme_admin_activate_fw,
2861                                                 parm_list_length, buffer_offset,
2862                                                 buffer_id);
2863                 break;
2864         default:
2865                 res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
2866                                         ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB,
2867                                         SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
2868                 break;
2869         }
2870
2871  out:
2872         return res;
2873 }
2874
2875 struct scsi_unmap_blk_desc {
2876         __be64  slba;
2877         __be32  nlb;
2878         u32     resv;
2879 };
2880
2881 struct scsi_unmap_parm_list {
2882         __be16  unmap_data_len;
2883         __be16  unmap_blk_desc_data_len;
2884         u32     resv;
2885         struct scsi_unmap_blk_desc desc[0];
2886 };
2887
2888 static int nvme_trans_unmap(struct nvme_ns *ns, struct sg_io_hdr *hdr,
2889                                                         u8 *cmd)
2890 {
2891         struct nvme_dev *dev = ns->dev;
2892         struct scsi_unmap_parm_list *plist;
2893         struct nvme_dsm_range *range;
2894         struct nvme_command c;
2895         int i, nvme_sc, res = -ENOMEM;
2896         u16 ndesc, list_len;
2897         dma_addr_t dma_addr;
2898
2899         list_len = GET_U16_FROM_CDB(cmd, UNMAP_CDB_PARAM_LIST_LENGTH_OFFSET);
2900         if (!list_len)
2901                 return -EINVAL;
2902
2903         plist = kmalloc(list_len, GFP_KERNEL);
2904         if (!plist)
2905                 return -ENOMEM;
2906
2907         res = nvme_trans_copy_from_user(hdr, plist, list_len);
2908         if (res != SNTI_TRANSLATION_SUCCESS)
2909                 goto out;
2910
2911         ndesc = be16_to_cpu(plist->unmap_blk_desc_data_len) >> 4;
2912         if (!ndesc || ndesc > 256) {
2913                 res = -EINVAL;
2914                 goto out;
2915         }
2916
2917         range = dma_alloc_coherent(&dev->pci_dev->dev, ndesc * sizeof(*range),
2918                                                         &dma_addr, GFP_KERNEL);
2919         if (!range)
2920                 goto out;
2921
2922         for (i = 0; i < ndesc; i++) {
2923                 range[i].nlb = cpu_to_le32(be32_to_cpu(plist->desc[i].nlb));
2924                 range[i].slba = cpu_to_le64(be64_to_cpu(plist->desc[i].slba));
2925                 range[i].cattr = 0;
2926         }
2927
2928         memset(&c, 0, sizeof(c));
2929         c.dsm.opcode = nvme_cmd_dsm;
2930         c.dsm.nsid = cpu_to_le32(ns->ns_id);
2931         c.dsm.prp1 = cpu_to_le64(dma_addr);
2932         c.dsm.nr = cpu_to_le32(ndesc - 1);
2933         c.dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
2934
2935         nvme_sc = nvme_submit_io_cmd(dev, ns, &c, NULL);
2936         res = nvme_trans_status_code(hdr, nvme_sc);
2937
2938         dma_free_coherent(&dev->pci_dev->dev, ndesc * sizeof(*range),
2939                                                         range, dma_addr);
2940  out:
2941         kfree(plist);
2942         return res;
2943 }
2944
2945 static int nvme_scsi_translate(struct nvme_ns *ns, struct sg_io_hdr *hdr)
2946 {
2947         u8 cmd[BLK_MAX_CDB];
2948         int retcode;
2949         unsigned int opcode;
2950
2951         if (hdr->cmdp == NULL)
2952                 return -EMSGSIZE;
2953         if (copy_from_user(cmd, hdr->cmdp, hdr->cmd_len))
2954                 return -EFAULT;
2955
2956         /*
2957          * Prime the hdr with good status for scsi commands that don't require
2958          * an nvme command for translation.
2959          */
2960         retcode = nvme_trans_status_code(hdr, NVME_SC_SUCCESS);
2961         if (retcode)
2962                 return retcode;
2963
2964         opcode = cmd[0];
2965
2966         switch (opcode) {
2967         case READ_6:
2968         case READ_10:
2969         case READ_12:
2970         case READ_16:
2971                 retcode = nvme_trans_io(ns, hdr, 0, cmd);
2972                 break;
2973         case WRITE_6:
2974         case WRITE_10:
2975         case WRITE_12:
2976         case WRITE_16:
2977                 retcode = nvme_trans_io(ns, hdr, 1, cmd);
2978                 break;
2979         case INQUIRY:
2980                 retcode = nvme_trans_inquiry(ns, hdr, cmd);
2981                 break;
2982         case LOG_SENSE:
2983                 retcode = nvme_trans_log_sense(ns, hdr, cmd);
2984                 break;
2985         case MODE_SELECT:
2986         case MODE_SELECT_10:
2987                 retcode = nvme_trans_mode_select(ns, hdr, cmd);
2988                 break;
2989         case MODE_SENSE:
2990         case MODE_SENSE_10:
2991                 retcode = nvme_trans_mode_sense(ns, hdr, cmd);
2992                 break;
2993         case READ_CAPACITY:
2994                 retcode = nvme_trans_read_capacity(ns, hdr, cmd);
2995                 break;
2996         case SERVICE_ACTION_IN_16:
2997                 if (IS_READ_CAP_16(cmd))
2998                         retcode = nvme_trans_read_capacity(ns, hdr, cmd);
2999                 else
3000                         goto out;
3001                 break;
3002         case REPORT_LUNS:
3003                 retcode = nvme_trans_report_luns(ns, hdr, cmd);
3004                 break;
3005         case REQUEST_SENSE:
3006                 retcode = nvme_trans_request_sense(ns, hdr, cmd);
3007                 break;
3008         case SECURITY_PROTOCOL_IN:
3009         case SECURITY_PROTOCOL_OUT:
3010                 retcode = nvme_trans_security_protocol(ns, hdr, cmd);
3011                 break;
3012         case START_STOP:
3013                 retcode = nvme_trans_start_stop(ns, hdr, cmd);
3014                 break;
3015         case SYNCHRONIZE_CACHE:
3016                 retcode = nvme_trans_synchronize_cache(ns, hdr, cmd);
3017                 break;
3018         case FORMAT_UNIT:
3019                 retcode = nvme_trans_format_unit(ns, hdr, cmd);
3020                 break;
3021         case TEST_UNIT_READY:
3022                 retcode = nvme_trans_test_unit_ready(ns, hdr, cmd);
3023                 break;
3024         case WRITE_BUFFER:
3025                 retcode = nvme_trans_write_buffer(ns, hdr, cmd);
3026                 break;
3027         case UNMAP:
3028                 retcode = nvme_trans_unmap(ns, hdr, cmd);
3029                 break;
3030         default:
3031  out:
3032                 retcode = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
3033                                 ILLEGAL_REQUEST, SCSI_ASC_ILLEGAL_COMMAND,
3034                                 SCSI_ASCQ_CAUSE_NOT_REPORTABLE);
3035                 break;
3036         }
3037         return retcode;
3038 }
3039
3040 int nvme_sg_io(struct nvme_ns *ns, struct sg_io_hdr __user *u_hdr)
3041 {
3042         struct sg_io_hdr hdr;
3043         int retcode;
3044
3045         if (!capable(CAP_SYS_ADMIN))
3046                 return -EACCES;
3047         if (copy_from_user(&hdr, u_hdr, sizeof(hdr)))
3048                 return -EFAULT;
3049         if (hdr.interface_id != 'S')
3050                 return -EINVAL;
3051         if (hdr.cmd_len > BLK_MAX_CDB)
3052                 return -EINVAL;
3053
3054         retcode = nvme_scsi_translate(ns, &hdr);
3055         if (retcode < 0)
3056                 return retcode;
3057         if (retcode > 0)
3058                 retcode = SNTI_TRANSLATION_SUCCESS;
3059         if (copy_to_user(u_hdr, &hdr, sizeof(sg_io_hdr_t)) > 0)
3060                 return -EFAULT;
3061
3062         return retcode;
3063 }
3064
3065 int nvme_sg_get_version_num(int __user *ip)
3066 {
3067         return put_user(sg_version_num, ip);
3068 }