IB/core: Support for CMA multicast join flags
authorAlex Vesker <valex@mellanox.com>
Wed, 6 Jul 2016 13:36:35 +0000 (16:36 +0300)
committerDoug Ledford <dledford@redhat.com>
Thu, 4 Aug 2016 01:06:46 +0000 (21:06 -0400)
Added UCMA and CMA support for multicast join flags. Flags are
passed using UCMA CM join command previously reserved fields.
Currently supporting two join flags indicating two different
multicast JoinStates:

1. Full Member:
   The initiator creates the Multicast group(MCG) if it wasn't
   previously created, can send Multicast messages to the group
   and receive messages from the MCG.

2. Send Only Full Member:
   The initiator creates the Multicast group(MCG) if it wasn't
   previously created, can send Multicast messages to the group
   but doesn't receive any messages from the MCG.

   IB: Send Only Full Member requires a query of ClassPortInfo
       to determine if SM/SA supports this option. If SM/SA
       doesn't support Send-Only there will be no join request
       sent and an error will be returned.

   ETH: When Send Only Full Member is requested no IGMP join
will be sent.

Signed-off-by: Alex Vesker <valex@mellanox.com>
Reviewed by: Hal Rosenstock <hal@mellanox.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Doug Ledford <dledford@redhat.com>
drivers/infiniband/core/cma.c
drivers/infiniband/core/multicast.c
drivers/infiniband/core/ucma.c
include/rdma/ib_sa.h
include/rdma/rdma_cm.h
include/uapi/rdma/rdma_user_cm.h

index f0c91ba3178a12030a8cc4e38db7a651633759b8..0451307bea185578c6dc7221cd17eeddd83a9528 100644 (file)
@@ -68,6 +68,7 @@ MODULE_DESCRIPTION("Generic RDMA CM Agent");
 MODULE_LICENSE("Dual BSD/GPL");
 
 #define CMA_CM_RESPONSE_TIMEOUT 20
+#define CMA_QUERY_CLASSPORT_INFO_TIMEOUT 3000
 #define CMA_MAX_CM_RETRIES 15
 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
 #define CMA_IBOE_PACKET_LIFETIME 18
@@ -162,6 +163,14 @@ struct rdma_bind_list {
        unsigned short          port;
 };
 
+struct class_port_info_context {
+       struct ib_class_port_info       *class_port_info;
+       struct ib_device                *device;
+       struct completion               done;
+       struct ib_sa_query              *sa_query;
+       u8                              port_num;
+};
+
 static int cma_ps_alloc(struct net *net, enum rdma_port_space ps,
                        struct rdma_bind_list *bind_list, int snum)
 {
@@ -306,6 +315,7 @@ struct cma_multicast {
        struct sockaddr_storage addr;
        struct kref             mcref;
        bool                    igmp_joined;
+       u8                      join_state;
 };
 
 struct cma_work {
@@ -3754,10 +3764,63 @@ static void cma_set_mgid(struct rdma_id_private *id_priv,
        }
 }
 
+static void cma_query_sa_classport_info_cb(int status,
+                                          struct ib_class_port_info *rec,
+                                          void *context)
+{
+       struct class_port_info_context *cb_ctx = context;
+
+       WARN_ON(!context);
+
+       if (status || !rec) {
+               pr_debug("RDMA CM: %s port %u failed query ClassPortInfo status: %d\n",
+                        cb_ctx->device->name, cb_ctx->port_num, status);
+               goto out;
+       }
+
+       memcpy(cb_ctx->class_port_info, rec, sizeof(struct ib_class_port_info));
+
+out:
+       complete(&cb_ctx->done);
+}
+
+static int cma_query_sa_classport_info(struct ib_device *device, u8 port_num,
+                                      struct ib_class_port_info *class_port_info)
+{
+       struct class_port_info_context *cb_ctx;
+       int ret;
+
+       cb_ctx = kmalloc(sizeof(*cb_ctx), GFP_KERNEL);
+       if (!cb_ctx)
+               return -ENOMEM;
+
+       cb_ctx->device = device;
+       cb_ctx->class_port_info = class_port_info;
+       cb_ctx->port_num = port_num;
+       init_completion(&cb_ctx->done);
+
+       ret = ib_sa_classport_info_rec_query(&sa_client, device, port_num,
+                                            CMA_QUERY_CLASSPORT_INFO_TIMEOUT,
+                                            GFP_KERNEL, cma_query_sa_classport_info_cb,
+                                            cb_ctx, &cb_ctx->sa_query);
+       if (ret < 0) {
+               pr_err("RDMA CM: %s port %u failed to send ClassPortInfo query, ret: %d\n",
+                      device->name, port_num, ret);
+               goto out;
+       }
+
+       wait_for_completion(&cb_ctx->done);
+
+out:
+       kfree(cb_ctx);
+       return ret;
+}
+
 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
                                 struct cma_multicast *mc)
 {
        struct ib_sa_mcmember_rec rec;
+       struct ib_class_port_info class_port_info;
        struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
        ib_sa_comp_mask comp_mask;
        int ret;
@@ -3776,7 +3839,24 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
        rec.qkey = cpu_to_be32(id_priv->qkey);
        rdma_addr_get_sgid(dev_addr, &rec.port_gid);
        rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
-       rec.join_state = 1;
+       rec.join_state = mc->join_state;
+
+       if (rec.join_state == BIT(SENDONLY_FULLMEMBER_JOIN)) {
+               ret = cma_query_sa_classport_info(id_priv->id.device,
+                                                 id_priv->id.port_num,
+                                                 &class_port_info);
+
+               if (ret)
+                       return ret;
+
+               if (!(ib_get_cpi_capmask2(&class_port_info) &
+                     IB_SA_CAP_MASK2_SENDONLY_FULL_MEM_SUPPORT)) {
+                       pr_warn("RDMA CM: %s port %u Unable to multicast join\n"
+                               "RDMA CM: SM doesn't support Send Only Full Member option\n",
+                               id_priv->id.device->name, id_priv->id.port_num);
+                       return -EOPNOTSUPP;
+               }
+       }
 
        comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
                    IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
@@ -3845,6 +3925,9 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
        struct sockaddr *addr = (struct sockaddr *)&mc->addr;
        struct net_device *ndev = NULL;
        enum ib_gid_type gid_type;
+       bool send_only;
+
+       send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
 
        if (cma_zero_addr((struct sockaddr *)&mc->addr))
                return -EINVAL;
@@ -3878,12 +3961,14 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
        gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
                   rdma_start_port(id_priv->cma_dev->device)];
        if (addr->sa_family == AF_INET) {
-               if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
-                       err = cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid,
-                                           true);
-               if (!err) {
-                       mc->igmp_joined = true;
+               if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
                        mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;
+                       if (!send_only) {
+                               err = cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid,
+                                                   true);
+                               if (!err)
+                                       mc->igmp_joined = true;
+                       }
                }
        } else {
                if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
@@ -3913,7 +3998,7 @@ out1:
 }
 
 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
-                       void *context)
+                       u8 join_state, void *context)
 {
        struct rdma_id_private *id_priv;
        struct cma_multicast *mc;
@@ -3932,6 +4017,7 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
        mc->context = context;
        mc->id_priv = id_priv;
        mc->igmp_joined = false;
+       mc->join_state = join_state;
        spin_lock(&id_priv->lock);
        list_add(&mc->list, &id_priv->mc_list);
        spin_unlock(&id_priv->lock);
index a83ec28a147b4884492fa960fd721568791553f2..3a3c5d73bbfc833a89c6143f0e732441c8488ab8 100644 (file)
@@ -93,18 +93,6 @@ enum {
 
 struct mcast_member;
 
-/*
-* There are 4 types of join states:
-* FullMember, NonMember, SendOnlyNonMember, SendOnlyFullMember.
-*/
-enum {
-       FULLMEMBER_JOIN,
-       NONMEMBER_JOIN,
-       SENDONLY_NONMEBER_JOIN,
-       SENDONLY_FULLMEMBER_JOIN,
-       NUM_JOIN_MEMBERSHIP_TYPES,
-};
-
 struct mcast_group {
        struct ib_sa_mcmember_rec rec;
        struct rb_node          node;
index c0f3826abb30aa09d755650d5055610679c846c8..2825ece91d3c5630d5caa8a22f6bf338fe26a87b 100644 (file)
@@ -106,6 +106,7 @@ struct ucma_multicast {
        int                     events_reported;
 
        u64                     uid;
+       u8                      join_state;
        struct list_head        list;
        struct sockaddr_storage addr;
 };
@@ -1317,12 +1318,20 @@ static ssize_t ucma_process_join(struct ucma_file *file,
        struct ucma_multicast *mc;
        struct sockaddr *addr;
        int ret;
+       u8 join_state;
 
        if (out_len < sizeof(resp))
                return -ENOSPC;
 
        addr = (struct sockaddr *) &cmd->addr;
-       if (cmd->reserved || !cmd->addr_size || (cmd->addr_size != rdma_addr_size(addr)))
+       if (!cmd->addr_size || (cmd->addr_size != rdma_addr_size(addr)))
+               return -EINVAL;
+
+       if (cmd->join_flags == RDMA_MC_JOIN_FLAG_FULLMEMBER)
+               join_state = BIT(FULLMEMBER_JOIN);
+       else if (cmd->join_flags == RDMA_MC_JOIN_FLAG_SENDONLY_FULLMEMBER)
+               join_state = BIT(SENDONLY_FULLMEMBER_JOIN);
+       else
                return -EINVAL;
 
        ctx = ucma_get_ctx(file, cmd->id);
@@ -1335,10 +1344,11 @@ static ssize_t ucma_process_join(struct ucma_file *file,
                ret = -ENOMEM;
                goto err1;
        }
-
+       mc->join_state = join_state;
        mc->uid = cmd->uid;
        memcpy(&mc->addr, addr, cmd->addr_size);
-       ret = rdma_join_multicast(ctx->cm_id, (struct sockaddr *) &mc->addr, mc);
+       ret = rdma_join_multicast(ctx->cm_id, (struct sockaddr *)&mc->addr,
+                                 join_state, mc);
        if (ret)
                goto err2;
 
@@ -1382,7 +1392,7 @@ static ssize_t ucma_join_ip_multicast(struct ucma_file *file,
        join_cmd.uid = cmd.uid;
        join_cmd.id = cmd.id;
        join_cmd.addr_size = rdma_addr_size((struct sockaddr *) &cmd.addr);
-       join_cmd.reserved = 0;
+       join_cmd.join_flags = RDMA_MC_JOIN_FLAG_FULLMEMBER;
        memcpy(&join_cmd.addr, &cmd.addr, join_cmd.addr_size);
 
        return ucma_process_join(file, &join_cmd, out_len);
index 384041669489e196a1732b6419520b2a054ea651..5ee7aab95eb8499239ae2f74b0534dd393290a95 100644 (file)
@@ -94,6 +94,19 @@ enum ib_sa_selector {
        IB_SA_BEST = 3
 };
 
+/*
+ * There are 4 types of join states:
+ * FullMember, NonMember, SendOnlyNonMember, SendOnlyFullMember.
+ * The order corresponds to JoinState bits in MCMemberRecord.
+ */
+enum ib_sa_mc_join_states {
+       FULLMEMBER_JOIN,
+       NONMEMBER_JOIN,
+       SENDONLY_NONMEBER_JOIN,
+       SENDONLY_FULLMEMBER_JOIN,
+       NUM_JOIN_MEMBERSHIP_TYPES,
+};
+
 #define IB_SA_CAP_MASK2_SENDONLY_FULL_MEM_SUPPORT      BIT(12)
 
 /*
index afe44fde72a56599cb0dd1674078210a78438ae4..81fb1d15e8bb1b64d6fa7b9ba4f5eb095604738b 100644 (file)
@@ -333,11 +333,13 @@ int rdma_disconnect(struct rdma_cm_id *id);
  *   address.
  * @id: Communication identifier associated with the request.
  * @addr: Multicast address identifying the group to join.
+ * @join_state: Multicast JoinState bitmap requested by port.
+ *             Bitmap is based on IB_SA_MCMEMBER_REC_JOIN_STATE bits.
  * @context: User-defined context associated with the join request, returned
  * to the user through the private_data pointer in multicast events.
  */
 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
-                       void *context);
+                       u8 join_state, void *context);
 
 /**
  * rdma_leave_multicast - Leave the multicast group specified by the given
index 3066718eb12087ec22eb22e518a4f1756da33b21..01923d463673efdb527bb847f15683c17e323dd6 100644 (file)
@@ -244,12 +244,19 @@ struct rdma_ucm_join_ip_mcast {
        __u32 id;
 };
 
+/* Multicast join flags */
+enum {
+       RDMA_MC_JOIN_FLAG_FULLMEMBER,
+       RDMA_MC_JOIN_FLAG_SENDONLY_FULLMEMBER,
+       RDMA_MC_JOIN_FLAG_RESERVED,
+};
+
 struct rdma_ucm_join_mcast {
        __u64 response;         /* rdma_ucma_create_id_resp */
        __u64 uid;
        __u32 id;
        __u16 addr_size;
-       __u16 reserved;
+       __u16 join_flags;
        struct sockaddr_storage addr;
 };