[PATCH v2 107/127] smb: server: make use of smbdirect_connection_{create, destroy}_qp()
Stefan Metzmacher
metze at samba.org
Wed Oct 29 13:21:25 UTC 2025
It's good a use common code for this and it will allow us
to share more code in the next steps.
Calling ib_drain_qp() twice is ok.
Cc: Namjae Jeon <linkinjeon at kernel.org>
Cc: Steve French <smfrench at gmail.com>
Cc: Tom Talpey <tom at talpey.com>
Cc: linux-cifs at vger.kernel.org
Cc: samba-technical at lists.samba.org
Signed-off-by: Stefan Metzmacher <metze at samba.org>
Signed-off-by: Steve French <stfrench at microsoft.com>
---
fs/smb/server/transport_rdma.c | 217 +--------------------------------
1 file changed, 6 insertions(+), 211 deletions(-)
diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c
index 526ad5c19b6e..2f55764a5f2e 100644
--- a/fs/smb/server/transport_rdma.c
+++ b/fs/smb/server/transport_rdma.c
@@ -257,6 +257,7 @@ static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
smbdirect_socket_set_logging(sc, NULL,
smb_direct_logging_needed,
smb_direct_logging_vaprintf);
+ sc->ib.poll_ctx = IB_POLL_WORKQUEUE;
sc->send_io.mem.gfp_mask = KSMBD_DEFAULT_GFP;
sc->recv_io.mem.gfp_mask = KSMBD_DEFAULT_GFP;
/*
@@ -318,11 +319,8 @@ static void free_transport(struct smb_direct_transport *t)
if (sc->rdma.cm_id)
rdma_lock_handler(sc->rdma.cm_id);
- if (sc->ib.qp) {
+ if (sc->ib.qp)
ib_drain_qp(sc->ib.qp);
- sc->ib.qp = NULL;
- rdma_destroy_qp(sc->rdma.cm_id);
- }
ksmbd_debug(RDMA, "drain the reassembly queue\n");
do {
@@ -340,12 +338,7 @@ static void free_transport(struct smb_direct_transport *t)
} while (recvmsg);
sc->recv_io.reassembly.data_length = 0;
- if (sc->ib.send_cq)
- ib_free_cq(sc->ib.send_cq);
- if (sc->ib.recv_cq)
- ib_free_cq(sc->ib.recv_cq);
- if (sc->ib.pd)
- ib_dealloc_pd(sc->ib.pd);
+ smbdirect_connection_destroy_qp(sc);
if (sc->rdma.cm_id) {
rdma_unlock_handler(sc->rdma.cm_id);
rdma_destroy_id(sc->rdma.cm_id);
@@ -1679,206 +1672,6 @@ static int smb_direct_init_params(struct smbdirect_socket *sc)
return 0;
}
-static u32 smb_direct_rdma_rw_send_wrs(struct ib_device *dev, const struct ib_qp_init_attr *attr)
-{
- /*
- * This could be split out of rdma_rw_init_qp()
- * and be a helper function next to rdma_rw_mr_factor()
- *
- * We can't check unlikely(rdma_rw_force_mr) here,
- * but that is most likely 0 anyway.
- */
- u32 factor;
-
- WARN_ON_ONCE(attr->port_num == 0);
-
- /*
- * Each context needs at least one RDMA READ or WRITE WR.
- *
- * For some hardware we might need more, eventually we should ask the
- * HCA driver for a multiplier here.
- */
- factor = 1;
-
- /*
- * If the device needs MRs to perform RDMA READ or WRITE operations,
- * we'll need two additional MRs for the registrations and the
- * invalidation.
- */
- if (rdma_protocol_iwarp(dev, attr->port_num) || dev->attrs.max_sgl_rd)
- factor += 2; /* inv + reg */
-
- return factor * attr->cap.max_rdma_ctxs;
-}
-
-static int smb_direct_create_qpair(struct smbdirect_socket *sc)
-{
- struct smbdirect_socket_parameters *sp = &sc->parameters;
- int ret;
- struct ib_qp_cap qp_cap;
- struct ib_qp_init_attr qp_attr;
- u32 max_send_wr;
- u32 rdma_send_wr;
-
- /*
- * Note that {rdma,ib}_create_qp() will call
- * rdma_rw_init_qp() if cap->max_rdma_ctxs is not 0.
- * It will adjust cap->max_send_wr to the required
- * number of additional WRs for the RDMA RW operations.
- * It will cap cap->max_send_wr to the device limit.
- *
- * +1 for ib_drain_qp
- */
- qp_cap.max_send_wr = sp->send_credit_target + 1;
- qp_cap.max_recv_wr = sp->recv_credit_max + 1;
- qp_cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE;
- qp_cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE;
- qp_cap.max_inline_data = 0;
- qp_cap.max_rdma_ctxs = sc->rw_io.credits.max;
-
- /*
- * Find out the number of max_send_wr
- * after rdma_rw_init_qp() adjusted it.
- *
- * We only do it on a temporary variable,
- * as rdma_create_qp() will trigger
- * rdma_rw_init_qp() again.
- */
- memset(&qp_attr, 0, sizeof(qp_attr));
- qp_attr.cap = qp_cap;
- qp_attr.port_num = sc->rdma.cm_id->port_num;
- rdma_send_wr = smb_direct_rdma_rw_send_wrs(sc->ib.dev, &qp_attr);
- max_send_wr = qp_cap.max_send_wr + rdma_send_wr;
-
- if (qp_cap.max_send_wr > sc->ib.dev->attrs.max_cqe ||
- qp_cap.max_send_wr > sc->ib.dev->attrs.max_qp_wr) {
- pr_err("Possible CQE overrun: max_send_wr %d\n",
- qp_cap.max_send_wr);
- pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
- IB_DEVICE_NAME_MAX,
- sc->ib.dev->name,
- sc->ib.dev->attrs.max_cqe,
- sc->ib.dev->attrs.max_qp_wr);
- pr_err("consider lowering send_credit_target = %d\n",
- sp->send_credit_target);
- return -EINVAL;
- }
-
- if (qp_cap.max_rdma_ctxs &&
- (max_send_wr >= sc->ib.dev->attrs.max_cqe ||
- max_send_wr >= sc->ib.dev->attrs.max_qp_wr)) {
- pr_err("Possible CQE overrun: rdma_send_wr %d + max_send_wr %d = %d\n",
- rdma_send_wr, qp_cap.max_send_wr, max_send_wr);
- pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
- IB_DEVICE_NAME_MAX,
- sc->ib.dev->name,
- sc->ib.dev->attrs.max_cqe,
- sc->ib.dev->attrs.max_qp_wr);
- pr_err("consider lowering send_credit_target = %d, max_rdma_ctxs = %d\n",
- sp->send_credit_target, qp_cap.max_rdma_ctxs);
- return -EINVAL;
- }
-
- if (qp_cap.max_recv_wr > sc->ib.dev->attrs.max_cqe ||
- qp_cap.max_recv_wr > sc->ib.dev->attrs.max_qp_wr) {
- pr_err("Possible CQE overrun: max_recv_wr %d\n",
- qp_cap.max_recv_wr);
- pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
- IB_DEVICE_NAME_MAX,
- sc->ib.dev->name,
- sc->ib.dev->attrs.max_cqe,
- sc->ib.dev->attrs.max_qp_wr);
- pr_err("consider lowering receive_credit_max = %d\n",
- sp->recv_credit_max);
- return -EINVAL;
- }
-
- if (qp_cap.max_send_sge > sc->ib.dev->attrs.max_send_sge ||
- qp_cap.max_recv_sge > sc->ib.dev->attrs.max_recv_sge) {
- pr_err("device %.*s max_send_sge/max_recv_sge = %d/%d too small\n",
- IB_DEVICE_NAME_MAX,
- sc->ib.dev->name,
- sc->ib.dev->attrs.max_send_sge,
- sc->ib.dev->attrs.max_recv_sge);
- return -EINVAL;
- }
-
- sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0);
- if (IS_ERR(sc->ib.pd)) {
- pr_err("Can't create RDMA PD\n");
- ret = PTR_ERR(sc->ib.pd);
- sc->ib.pd = NULL;
- return ret;
- }
-
- sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc,
- max_send_wr,
- IB_POLL_WORKQUEUE);
- if (IS_ERR(sc->ib.send_cq)) {
- pr_err("Can't create RDMA send CQ\n");
- ret = PTR_ERR(sc->ib.send_cq);
- sc->ib.send_cq = NULL;
- goto err;
- }
-
- sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc,
- qp_cap.max_recv_wr,
- IB_POLL_WORKQUEUE);
- if (IS_ERR(sc->ib.recv_cq)) {
- pr_err("Can't create RDMA recv CQ\n");
- ret = PTR_ERR(sc->ib.recv_cq);
- sc->ib.recv_cq = NULL;
- goto err;
- }
-
- /*
- * We reset completely here!
- * As the above use was just temporary
- * to calc max_send_wr and rdma_send_wr.
- *
- * rdma_create_qp() will trigger rdma_rw_init_qp()
- * again if max_rdma_ctxs is not 0.
- */
- memset(&qp_attr, 0, sizeof(qp_attr));
- qp_attr.event_handler = smbdirect_connection_qp_event_handler;
- qp_attr.qp_context = sc;
- qp_attr.cap = qp_cap;
- qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
- qp_attr.qp_type = IB_QPT_RC;
- qp_attr.send_cq = sc->ib.send_cq;
- qp_attr.recv_cq = sc->ib.recv_cq;
- qp_attr.port_num = ~0;
-
- ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr);
- if (ret) {
- pr_err("Can't create RDMA QP: %d\n", ret);
- goto err;
- }
-
- sc->ib.qp = sc->rdma.cm_id->qp;
- sc->rdma.cm_id->event_handler = smb_direct_cm_handler;
-
- return 0;
-err:
- if (sc->ib.qp) {
- sc->ib.qp = NULL;
- rdma_destroy_qp(sc->rdma.cm_id);
- }
- if (sc->ib.recv_cq) {
- ib_destroy_cq(sc->ib.recv_cq);
- sc->ib.recv_cq = NULL;
- }
- if (sc->ib.send_cq) {
- ib_destroy_cq(sc->ib.send_cq);
- sc->ib.send_cq = NULL;
- }
- if (sc->ib.pd) {
- ib_dealloc_pd(sc->ib.pd);
- sc->ib.pd = NULL;
- }
- return ret;
-}
-
static int smb_direct_prepare(struct ksmbd_transport *t)
{
struct smb_direct_transport *st = SMBD_TRANS(t);
@@ -1970,6 +1763,8 @@ static int smb_direct_connect(struct smbdirect_socket *sc)
{
int ret;
+ sc->rdma.cm_id->event_handler = smb_direct_cm_handler;
+
ret = smb_direct_init_params(sc);
if (ret) {
pr_err("Can't configure RDMA parameters\n");
@@ -1982,7 +1777,7 @@ static int smb_direct_connect(struct smbdirect_socket *sc)
return ret;
}
- ret = smb_direct_create_qpair(sc);
+ ret = smbdirect_connection_create_qp(sc);
if (ret) {
pr_err("Can't accept RDMA client: %d\n", ret);
return ret;
--
2.43.0
More information about the samba-technical
mailing list