engines/nvme.h

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * nvme structure declarations and helper functions for the
   4  * io_uring_cmd engine.
   5  */
   6
   7 #ifndef FIO_NVME_H
   8 #define FIO_NVME_H
   9
  10 #include <linux/nvme_ioctl.h>
  11 #include "../fio.h"
  12
  13 /*
  14  * If the uapi headers installed on the system lacks nvme uring command
  15  * support, use the local version to prevent compilation issues.
  16  */
  17 #ifndef CONFIG_NVME_URING_CMD
  18 struct nvme_uring_cmd {
  19         __u8    opcode;
  20         __u8    flags;
  21         __u16   rsvd1;
  22         __u32   nsid;
  23         __u32   cdw2;
  24         __u32   cdw3;
  25         __u64   metadata;
  26         __u64   addr;
  27         __u32   metadata_len;
  28         __u32   data_len;
  29         __u32   cdw10;
  30         __u32   cdw11;
  31         __u32   cdw12;
  32         __u32   cdw13;
  33         __u32   cdw14;
  34         __u32   cdw15;
  35         __u32   timeout_ms;
  36         __u32   rsvd2;
  37 };
  38
  39 #define NVME_URING_CMD_IO       _IOWR('N', 0x80, struct nvme_uring_cmd)
  40 #define NVME_URING_CMD_IO_VEC   _IOWR('N', 0x81, struct nvme_uring_cmd)
  41 #endif /* CONFIG_NVME_URING_CMD */
  42
  43 #define NVME_DEFAULT_IOCTL_TIMEOUT 0
  44 #define NVME_IDENTIFY_DATA_SIZE 4096
  45 #define NVME_IDENTIFY_CSI_SHIFT 24
  46 #define NVME_NQN_LENGTH 256
  47
  48 #define NVME_PI_APP_DISABLE 0xFFFF
  49 #define NVME_PI_REF_DISABLE 0xFFFFFFFF
  50
  51 #define NVME_ZNS_ZRA_REPORT_ZONES 0
  52 #define NVME_ZNS_ZRAS_FEAT_ERZ (1 << 16)
  53 #define NVME_ZNS_ZSA_RESET 0x4
  54 #define NVME_ZONE_TYPE_SEQWRITE_REQ 0x2
  55
  56 #define NVME_ATTRIBUTE_DEALLOCATE (1 << 2)
  57
  58 enum nvme_identify_cns {
  59         NVME_IDENTIFY_CNS_NS            = 0x00,
  60         NVME_IDENTIFY_CNS_CTRL          = 0x01,
  61         NVME_IDENTIFY_CNS_CSI_NS        = 0x05,
  62         NVME_IDENTIFY_CNS_CSI_CTRL      = 0x06,
  63 };
  64
  65 enum nvme_csi {
  66         NVME_CSI_NVM                    = 0,
  67         NVME_CSI_KV                     = 1,
  68         NVME_CSI_ZNS                    = 2,
  69 };
  70
  71 enum nvme_admin_opcode {
  72         nvme_admin_identify             = 0x06,
  73 };
  74
  75 enum nvme_io_opcode {
  76         nvme_cmd_write                  = 0x01,
  77         nvme_cmd_read                   = 0x02,
  78         nvme_cmd_dsm                    = 0x09,
  79         nvme_cmd_io_mgmt_recv           = 0x12,
  80         nvme_zns_cmd_mgmt_send          = 0x79,
  81         nvme_zns_cmd_mgmt_recv          = 0x7a,
  82 };
  83
  84 enum nvme_zns_zs {
  85         NVME_ZNS_ZS_EMPTY               = 0x1,
  86         NVME_ZNS_ZS_IMPL_OPEN           = 0x2,
  87         NVME_ZNS_ZS_EXPL_OPEN           = 0x3,
  88         NVME_ZNS_ZS_CLOSED              = 0x4,
  89         NVME_ZNS_ZS_READ_ONLY           = 0xd,
  90         NVME_ZNS_ZS_FULL                = 0xe,
  91         NVME_ZNS_ZS_OFFLINE             = 0xf,
  92 };
  93
  94 enum nvme_id_ctrl_ctratt {
  95         NVME_CTRL_CTRATT_ELBAS          = 1 << 15,
  96 };
  97
  98 enum {
  99         NVME_ID_NS_NVM_STS_MASK         = 0x7f,
 100         NVME_ID_NS_NVM_GUARD_SHIFT      = 7,
 101         NVME_ID_NS_NVM_GUARD_MASK       = 0x3,
 102 };
 103
 104 enum {
 105         NVME_NVM_NS_16B_GUARD           = 0,
 106         NVME_NVM_NS_32B_GUARD           = 1,
 107         NVME_NVM_NS_64B_GUARD           = 2,
 108 };
 109
 110 struct nvme_data {
 111         __u32 nsid;
 112         __u32 lba_shift;
 113         __u32 lba_size;
 114         __u32 lba_ext;
 115         __u16 ms;
 116         __u16 pi_size;
 117         __u8 pi_type;
 118         __u8 guard_type;
 119         __u8 pi_loc;
 120 };
 121
 122 enum nvme_id_ns_dps {
 123         NVME_NS_DPS_PI_NONE             = 0,
 124         NVME_NS_DPS_PI_TYPE1            = 1,
 125         NVME_NS_DPS_PI_TYPE2            = 2,
 126         NVME_NS_DPS_PI_TYPE3            = 3,
 127         NVME_NS_DPS_PI_MASK             = 7 << 0,
 128         NVME_NS_DPS_PI_FIRST            = 1 << 3,
 129 };
 130
 131 enum nvme_io_control_flags {
 132         NVME_IO_PRINFO_PRCHK_REF        = 1U << 26,
 133         NVME_IO_PRINFO_PRCHK_APP        = 1U << 27,
 134         NVME_IO_PRINFO_PRCHK_GUARD      = 1U << 28,
 135         NVME_IO_PRINFO_PRACT            = 1U << 29,
 136 };
 137
 138 struct nvme_pi_data {
 139         __u32 interval;
 140         __u32 io_flags;
 141         __u16 apptag;
 142         __u16 apptag_mask;
 143 };
 144
 145 struct nvme_lbaf {
 146         __le16                  ms;
 147         __u8                    ds;
 148         __u8                    rp;
 149 };
 150
 151 /* 16 bit guard protection Information format */
 152 struct nvme_16b_guard_pif {
 153         __be16 guard;
 154         __be16 apptag;
 155         __be32 srtag;
 156 };
 157
 158 /* 64 bit guard protection Information format */
 159 struct nvme_64b_guard_pif {
 160         __be64 guard;
 161         __be16 apptag;
 162         __u8 srtag[6];
 163 };
 164
 165 struct nvme_id_ns {
 166         __le64                  nsze;
 167         __le64                  ncap;
 168         __le64                  nuse;
 169         __u8                    nsfeat;
 170         __u8                    nlbaf;
 171         __u8                    flbas;
 172         __u8                    mc;
 173         __u8                    dpc;
 174         __u8                    dps;
 175         __u8                    nmic;
 176         __u8                    rescap;
 177         __u8                    fpi;
 178         __u8                    dlfeat;
 179         __le16                  nawun;
 180         __le16                  nawupf;
 181         __le16                  nacwu;
 182         __le16                  nabsn;
 183         __le16                  nabo;
 184         __le16                  nabspf;
 185         __le16                  noiob;
 186         __u8                    nvmcap[16];
 187         __le16                  npwg;
 188         __le16                  npwa;
 189         __le16                  npdg;
 190         __le16                  npda;
 191         __le16                  nows;
 192         __le16                  mssrl;
 193         __le32                  mcl;
 194         __u8                    msrc;
 195         __u8                    rsvd81[11];
 196         __le32                  anagrpid;
 197         __u8                    rsvd96[3];
 198         __u8                    nsattr;
 199         __le16                  nvmsetid;
 200         __le16                  endgid;
 201         __u8                    nguid[16];
 202         __u8                    eui64[8];
 203         struct nvme_lbaf        lbaf[64];
 204         __u8                    vs[3712];
 205 };
 206
 207 struct nvme_id_psd {
 208         __le16                  mp;
 209         __u8                    rsvd2;
 210         __u8                    flags;
 211         __le32                  enlat;
 212         __le32                  exlat;
 213         __u8                    rrt;
 214         __u8                    rrl;
 215         __u8                    rwt;
 216         __u8                    rwl;
 217         __le16                  idlp;
 218         __u8                    ips;
 219         __u8                    rsvd19;
 220         __le16                  actp;
 221         __u8                    apws;
 222         __u8                    rsvd23[9];
 223 };
 224
 225 struct nvme_id_ctrl {
 226         __le16                  vid;
 227         __le16                  ssvid;
 228         char                    sn[20];
 229         char                    mn[40];
 230         char                    fr[8];
 231         __u8                    rab;
 232         __u8                    ieee[3];
 233         __u8                    cmic;
 234         __u8                    mdts;
 235         __le16                  cntlid;
 236         __le32                  ver;
 237         __le32                  rtd3r;
 238         __le32                  rtd3e;
 239         __le32                  oaes;
 240         __le32                  ctratt;
 241         __le16                  rrls;
 242         __u8                    rsvd102[9];
 243         __u8                    cntrltype;
 244         __u8                    fguid[16];
 245         __le16                  crdt1;
 246         __le16                  crdt2;
 247         __le16                  crdt3;
 248         __u8                    rsvd134[119];
 249         __u8                    nvmsr;
 250         __u8                    vwci;
 251         __u8                    mec;
 252         __le16                  oacs;
 253         __u8                    acl;
 254         __u8                    aerl;
 255         __u8                    frmw;
 256         __u8                    lpa;
 257         __u8                    elpe;
 258         __u8                    npss;
 259         __u8                    avscc;
 260         __u8                    apsta;
 261         __le16                  wctemp;
 262         __le16                  cctemp;
 263         __le16                  mtfa;
 264         __le32                  hmpre;
 265         __le32                  hmmin;
 266         __u8                    tnvmcap[16];
 267         __u8                    unvmcap[16];
 268         __le32                  rpmbs;
 269         __le16                  edstt;
 270         __u8                    dsto;
 271         __u8                    fwug;
 272         __le16                  kas;
 273         __le16                  hctma;
 274         __le16                  mntmt;
 275         __le16                  mxtmt;
 276         __le32                  sanicap;
 277         __le32                  hmminds;
 278         __le16                  hmmaxd;
 279         __le16                  nsetidmax;
 280         __le16                  endgidmax;
 281         __u8                    anatt;
 282         __u8                    anacap;
 283         __le32                  anagrpmax;
 284         __le32                  nanagrpid;
 285         __le32                  pels;
 286         __le16                  domainid;
 287         __u8                    rsvd358[10];
 288         __u8                    megcap[16];
 289         __u8                    rsvd384[128];
 290         __u8                    sqes;
 291         __u8                    cqes;
 292         __le16                  maxcmd;
 293         __le32                  nn;
 294         __le16                  oncs;
 295         __le16                  fuses;
 296         __u8                    fna;
 297         __u8                    vwc;
 298         __le16                  awun;
 299         __le16                  awupf;
 300         __u8                    icsvscc;
 301         __u8                    nwpc;
 302         __le16                  acwu;
 303         __le16                  ocfs;
 304         __le32                  sgls;
 305         __le32                  mnan;
 306         __u8                    maxdna[16];
 307         __le32                  maxcna;
 308         __u8                    rsvd564[204];
 309         char                    subnqn[NVME_NQN_LENGTH];
 310         __u8                    rsvd1024[768];
 311
 312         /* Fabrics Only */
 313         __le32                  ioccsz;
 314         __le32                  iorcsz;
 315         __le16                  icdoff;
 316         __u8                    fcatt;
 317         __u8                    msdbd;
 318         __le16                  ofcs;
 319         __u8                    dctype;
 320         __u8                    rsvd1807[241];
 321
 322         struct nvme_id_psd      psd[32];
 323         __u8                    vs[1024];
 324 };
 325
 326 struct nvme_nvm_id_ns {
 327         __le64                  lbstm;
 328         __u8                    pic;
 329         __u8                    rsvd9[3];
 330         __le32                  elbaf[64];
 331         __u8                    rsvd268[3828];
 332 };
 333
 334 static inline int ilog2(uint32_t i)
 335 {
 336         int log = -1;
 337
 338         while (i) {
 339                 i >>= 1;
 340                 log++;
 341         }
 342         return log;
 343 }
 344
 345 struct nvme_zns_lbafe {
 346         __le64  zsze;
 347         __u8    zdes;
 348         __u8    rsvd9[7];
 349 };
 350
 351 struct nvme_zns_id_ns {
 352         __le16                  zoc;
 353         __le16                  ozcs;
 354         __le32                  mar;
 355         __le32                  mor;
 356         __le32                  rrl;
 357         __le32                  frl;
 358         __le32                  rrl1;
 359         __le32                  rrl2;
 360         __le32                  rrl3;
 361         __le32                  frl1;
 362         __le32                  frl2;
 363         __le32                  frl3;
 364         __le32                  numzrwa;
 365         __le16                  zrwafg;
 366         __le16                  zrwasz;
 367         __u8                    zrwacap;
 368         __u8                    rsvd53[2763];
 369         struct nvme_zns_lbafe   lbafe[64];
 370         __u8                    vs[256];
 371 };
 372
 373 struct nvme_zns_desc {
 374         __u8    zt;
 375         __u8    zs;
 376         __u8    za;
 377         __u8    zai;
 378         __u8    rsvd4[4];
 379         __le64  zcap;
 380         __le64  zslba;
 381         __le64  wp;
 382         __u8    rsvd32[32];
 383 };
 384
 385 struct nvme_zone_report {
 386         __le64                  nr_zones;
 387         __u8                    rsvd8[56];
 388         struct nvme_zns_desc    entries[];
 389 };
 390
 391 struct nvme_fdp_ruh_status_desc {
 392         __u16 pid;
 393         __u16 ruhid;
 394         __u32 earutr;
 395         __u64 ruamw;
 396         __u8  rsvd16[16];
 397 };
 398
 399 struct nvme_fdp_ruh_status {
 400         __u8  rsvd0[14];
 401         __le16 nruhsd;
 402         struct nvme_fdp_ruh_status_desc ruhss[];
 403 };
 404
 405 struct nvme_dsm_range {
 406         __le32  cattr;
 407         __le32  nlb;
 408         __le64  slba;
 409 };
 410
 411 struct nvme_dsm {
 412         __u32 nr_ranges;
 413         struct nvme_dsm_range range[];
 414 };
 415
 416 struct nvme_cmd_ext_io_opts {
 417         __u32 io_flags;
 418         __u16 apptag;
 419         __u16 apptag_mask;
 420 };
 421
 422 int fio_nvme_iomgmt_ruhs(struct thread_data *td, struct fio_file *f,
 423                          struct nvme_fdp_ruh_status *ruhs, __u32 bytes);
 424
 425 int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
 426                       struct nvme_data *data);
 427
 428 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 429                             struct iovec *iov, struct nvme_dsm *dsm);
 430
 431 void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 432                       struct nvme_cmd_ext_io_opts *opts);
 433
 434 int fio_nvme_pi_verify(struct nvme_data *data, struct io_u *io_u);
 435
 436 int fio_nvme_get_zoned_model(struct thread_data *td, struct fio_file *f,
 437                              enum zbd_zoned_model *model);
 438
 439 int fio_nvme_report_zones(struct thread_data *td, struct fio_file *f,
 440                           uint64_t offset, struct zbd_zone *zbdz,
 441                           unsigned int nr_zones);
 442
 443 int fio_nvme_reset_wp(struct thread_data *td, struct fio_file *f,
 444                       uint64_t offset, uint64_t length);
 445
 446 int fio_nvme_get_max_open_zones(struct thread_data *td, struct fio_file *f,
 447                                 unsigned int *max_open_zones);
 448
 449 static inline void put_unaligned_be48(__u64 val, __u8 *p)
 450 {
 451         *p++ = val >> 40;
 452         *p++ = val >> 32;
 453         *p++ = val >> 24;
 454         *p++ = val >> 16;
 455         *p++ = val >> 8;
 456         *p++ = val;
 457 }
 458
 459 static inline __u64 get_unaligned_be48(__u8 *p)
 460 {
 461         return (__u64)p[0] << 40 | (__u64)p[1] << 32 | (__u64)p[2] << 24 |
 462                 p[3] << 16 | p[4] << 8 | p[5];
 463 }
 464
 465 static inline bool fio_nvme_pi_ref_escape(__u8 *reftag)
 466 {
 467         __u8 ref_esc[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 468
 469         return memcmp(reftag, ref_esc, sizeof(ref_esc)) == 0;
 470 }
 471
 472 #endif