rnbd/rtrs-clt: Query and use max_segments from rtrs-clt.

With fast memory registration on write request, rnbd-clt
can do bigger IO without split. rnbd-clt now can query
rtrs-clt to get the max_segments, instead of using
BMAX_SEGMENTS.

BMAX_SEGMENTS is not longer needed, so remove it.

Link: https://lore.kernel.org/r/20210621055340.11789-6-jinpu.wang@ionos.com
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
Reviewed-by: Md Haris Iqbal <haris.iqbal@cloud.ionos.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c
index c604a402cd5c817b735242474acbbe8b1bd174db..d6f12e6c91f78f9a9c87992f1260a8d94193d933 100644 (file)
--- a/drivers/block/rnbd/rnbd-clt.c
+++ b/drivers/block/rnbd/rnbd-clt.c
@@ -92,7 +92,7 @@ static int rnbd_clt_set_dev_attr(struct rnbd_clt_dev *dev,
        dev->fua                    = !!(rsp->cache_policy & RNBD_FUA);
 
        dev->max_hw_sectors = sess->max_io_size / SECTOR_SIZE;
-       dev->max_segments = BMAX_SEGMENTS;
+       dev->max_segments = sess->max_segments;
 
        return 0;
 }
@@ -1292,7 +1292,7 @@ find_and_get_or_create_sess(const char *sessname,
        sess->rtrs = rtrs_clt_open(&rtrs_ops, sessname,
                                   paths, path_cnt, port_nr,
                                   0, /* Do not use pdu of rtrs */
-                                  RECONNECT_DELAY, BMAX_SEGMENTS,
+                                  RECONNECT_DELAY,
                                   MAX_RECONNECTS, nr_poll_queues);
        if (IS_ERR(sess->rtrs)) {
                err = PTR_ERR(sess->rtrs);
@@ -1306,6 +1306,7 @@ find_and_get_or_create_sess(const char *sessname,
        sess->max_io_size = attrs.max_io_size;
        sess->queue_depth = attrs.queue_depth;
        sess->nr_poll_queues = nr_poll_queues;
+       sess->max_segments = attrs.max_segments;
 
        err = setup_mq_tags(sess);
        if (err)

diff --git a/drivers/block/rnbd/rnbd-clt.h b/drivers/block/rnbd/rnbd-clt.h
index b5322c5aaac0fc9ed2c6c3b2825b3f0a902174c4..9ef8c4f306f2dab8bd28b1319902275118937c55 100644 (file)
--- a/drivers/block/rnbd/rnbd-clt.h
+++ b/drivers/block/rnbd/rnbd-clt.h
@@ -20,10 +20,6 @@
 #include "rnbd-proto.h"
 #include "rnbd-log.h"
 
-/* Max. number of segments per IO request, Mellanox Connect X ~ Connect X5,
- * choose minimial 30 for all, minus 1 for internal protocol, so 29.
- */
-#define BMAX_SEGMENTS 29
 /*  time in seconds between reconnect tries, default to 30 s */
 #define RECONNECT_DELAY 30
 /*
@@ -89,6 +85,7 @@ struct rnbd_clt_session {
        atomic_t                busy;
        size_t                  queue_depth;
        u32                     max_io_size;
+       u32                     max_segments;
        struct blk_mq_tag_set   tag_set;
        u32                     nr_poll_queues;
        struct mutex            lock; /* protects state and devs_list */

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index 75b6b14afd37a4167ff12a1e3efd70436ff905af..f2c40e50f25ea4956a0914b7c0232c2d5e45c3ce 100644 (file)
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -1357,7 +1357,6 @@ static void free_sess_reqs(struct rtrs_clt_sess *sess)
 static int alloc_sess_reqs(struct rtrs_clt_sess *sess)
 {
        struct rtrs_clt_io_req *req;
-       struct rtrs_clt *clt = sess->clt;
        int i, err = -ENOMEM;
 
        sess->reqs = kcalloc(sess->queue_depth, sizeof(*sess->reqs),
@@ -1466,6 +1465,8 @@ static void query_fast_reg_mode(struct rtrs_clt_sess *sess)
        sess->max_pages_per_mr =
                min3(sess->max_pages_per_mr, (u32)max_pages_per_mr,
                     ib_dev->attrs.max_fast_reg_page_list_len);
+       sess->clt->max_segments =
+               min(sess->max_pages_per_mr, sess->clt->max_segments);
 }
 
 static bool rtrs_clt_change_state_get_old(struct rtrs_clt_sess *sess,
@@ -1503,9 +1504,8 @@ static void rtrs_clt_reconnect_work(struct work_struct *work);
 static void rtrs_clt_close_work(struct work_struct *work);
 
 static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt,
-                                        const struct rtrs_addr *path,
-                                        size_t con_num, u16 max_segments,
-                                        u32 nr_poll_queues)
+                                       const struct rtrs_addr *path,
+                                       size_t con_num, u32 nr_poll_queues)
 {
        struct rtrs_clt_sess *sess;
        int err = -ENOMEM;
@@ -2668,7 +2668,6 @@ static struct rtrs_clt *alloc_clt(const char *sessname, size_t paths_num,
                                  u16 port, size_t pdu_sz, void *priv,
                                  void  (*link_ev)(void *priv,
                                                   enum rtrs_clt_link_ev ev),
-                                 unsigned int max_segments,
                                  unsigned int reconnect_delay_sec,
                                  unsigned int max_reconnect_attempts)
 {
@@ -2766,7 +2765,6 @@ static void free_clt(struct rtrs_clt *clt)
  * @port: port to be used by the RTRS session
  * @pdu_sz: Size of extra payload which can be accessed after permit allocation.
  * @reconnect_delay_sec: time between reconnect tries
- * @max_segments: Max. number of segments per IO request
  * @max_reconnect_attempts: Number of times to reconnect on error before giving
  *                         up, 0 for * disabled, -1 for forever
  * @nr_poll_queues: number of polling mode connection using IB_POLL_DIRECT flag
@@ -2781,7 +2779,6 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
                                 const struct rtrs_addr *paths,
                                 size_t paths_num, u16 port,
                                 size_t pdu_sz, u8 reconnect_delay_sec,
-                                u16 max_segments,
                                 s16 max_reconnect_attempts, u32 nr_poll_queues)
 {
        struct rtrs_clt_sess *sess, *tmp;
@@ -2790,7 +2787,7 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
 
        clt = alloc_clt(sessname, paths_num, port, pdu_sz, ops->priv,
                        ops->link_ev,
-                       max_segments, reconnect_delay_sec,
+                       reconnect_delay_sec,
                        max_reconnect_attempts);
        if (IS_ERR(clt)) {
                err = PTR_ERR(clt);
@@ -2800,7 +2797,7 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
                struct rtrs_clt_sess *sess;
 
                sess = alloc_sess(clt, &paths[i], nr_cpu_ids,
-                                 max_segments, nr_poll_queues);
+                                 nr_poll_queues);
                if (IS_ERR(sess)) {
                        err = PTR_ERR(sess);
                        goto close_all_sess;
@@ -3062,6 +3059,7 @@ int rtrs_clt_query(struct rtrs_clt *clt, struct rtrs_attrs *attr)
                return -ECOMM;
 
        attr->queue_depth      = clt->queue_depth;
+       attr->max_segments     = clt->max_segments;
        /* Cap max_io_size to min of remote buffer size and the fr pages */
        attr->max_io_size = min_t(int, clt->max_io_size,
                                  clt->max_segments * SZ_4K);
@@ -3076,7 +3074,7 @@ int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt,
        struct rtrs_clt_sess *sess;
        int err;
 
-       sess = alloc_sess(clt, addr, nr_cpu_ids, clt->max_segments, 0);
+       sess = alloc_sess(clt, addr, nr_cpu_ids, 0);
        if (IS_ERR(sess))
                return PTR_ERR(sess);
 

diff --git a/drivers/infiniband/ulp/rtrs/rtrs.h b/drivers/infiniband/ulp/rtrs/rtrs.h
index dc3e1af1a85ba63b9b7d861b647a366a349141f0..859c79685daf399a4edb48c753a11da6a314168a 100644 (file)
--- a/drivers/infiniband/ulp/rtrs/rtrs.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs.h
@@ -57,7 +57,6 @@ struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
                                 const struct rtrs_addr *paths,
                                 size_t path_cnt, u16 port,
                                 size_t pdu_sz, u8 reconnect_delay_sec,
-                                u16 max_segments,
                                 s16 max_reconnect_attempts, u32 nr_poll_queues);
 
 void rtrs_clt_close(struct rtrs_clt *sess);
@@ -110,6 +109,7 @@ int rtrs_clt_rdma_cq_direct(struct rtrs_clt *clt, unsigned int index);
 struct rtrs_attrs {
        u32             queue_depth;
        u32             max_io_size;
+       u32             max_segments;
 };
 
 int rtrs_clt_query(struct rtrs_clt *sess, struct rtrs_attrs *attr);