[PATCH] smb: client: allocate enough space for MR WRs and ib_drain_qp()

[Steve French]

applied to cifs-2.6.git for-next pending additional review and testing

On Thu, Oct 16, 2025 at 5:54 AM Stefan Metzmacher <metze at samba.org> wrote:
>
> The IB_WR_REG_MR and IB_WR_LOCAL_INV operations for smbdirect_mr_io
> structures should never fail because the submission or completion queues
> are too small. So we allocate more send_wr depending on the (local) max
> number of MRs.
>
> While there also add additional space for ib_drain_qp().
>
> This should make sure ib_post_send() will never fail
> because the submission queue is full.
>
> Fixes: f198186aa9bb ("CIFS: SMBD: Establish SMB Direct connection")
> Fixes: cc55f65dd352 ("smb: client: make use of common smbdirect_socket_parameters")
> Cc: Steve French <smfrench at gmail.com>
> Cc: Tom Talpey <tom at talpey.com>
> Cc: Long Li <longli at microsoft.com>
> Cc: Namjae Jeon <linkinjeon at kernel.org>
> Cc: linux-cifs at vger.kernel.org
> Cc: samba-technical at lists.samba.org
> Signed-off-by: Stefan Metzmacher <metze at samba.org>
> ---
>  fs/smb/client/smbdirect.c | 36 +++++++++++++++++++++++-------------
>  1 file changed, 23 insertions(+), 13 deletions(-)
>
> diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c
> index 49e2df3ad1f0..068e1069eca5 100644
> --- a/fs/smb/client/smbdirect.c
> +++ b/fs/smb/client/smbdirect.c
> @@ -1767,6 +1767,7 @@ static struct smbd_connection *_smbd_get_connection(
>         struct smbdirect_socket *sc;
>         struct smbdirect_socket_parameters *sp;
>         struct rdma_conn_param conn_param;
> +       struct ib_qp_cap qp_cap;
>         struct ib_qp_init_attr qp_attr;
>         struct sockaddr_in *addr_in = (struct sockaddr_in *) dstaddr;
>         struct ib_port_immutable port_immutable;
> @@ -1838,6 +1839,25 @@ static struct smbd_connection *_smbd_get_connection(
>                 goto config_failed;
>         }
>
> +       sp->responder_resources =
> +               min_t(u8, sp->responder_resources,
> +                     sc->ib.dev->attrs.max_qp_rd_atom);
> +       log_rdma_mr(INFO, "responder_resources=%d\n",
> +               sp->responder_resources);
> +
> +       /*
> +        * We use allocate sp->responder_resources * 2 MRs
> +        * and each MR needs WRs for REG and INV, so
> +        * we use '* 4'.
> +        *
> +        * +1 fot ib_drain_qp()
> +        */
> +       memset(&qp_cap, 0, sizeof(qp_cap));
> +       qp_cap.max_send_wr = sp->send_credit_target + sp->responder_resources * 4 + 1;
> +       qp_cap.max_recv_wr = sp->recv_credit_max + 1;
> +       qp_cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE;
> +       qp_cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE;
> +
>         sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0);
>         if (IS_ERR(sc->ib.pd)) {
>                 rc = PTR_ERR(sc->ib.pd);
> @@ -1848,7 +1868,7 @@ static struct smbd_connection *_smbd_get_connection(
>
>         sc->ib.send_cq =
>                 ib_alloc_cq_any(sc->ib.dev, sc,
> -                               sp->send_credit_target, IB_POLL_SOFTIRQ);
> +                               qp_cap.max_send_wr, IB_POLL_SOFTIRQ);
>         if (IS_ERR(sc->ib.send_cq)) {
>                 sc->ib.send_cq = NULL;
>                 goto alloc_cq_failed;
> @@ -1856,7 +1876,7 @@ static struct smbd_connection *_smbd_get_connection(
>
>         sc->ib.recv_cq =
>                 ib_alloc_cq_any(sc->ib.dev, sc,
> -                               sp->recv_credit_max, IB_POLL_SOFTIRQ);
> +                               qp_cap.max_recv_wr, IB_POLL_SOFTIRQ);
>         if (IS_ERR(sc->ib.recv_cq)) {
>                 sc->ib.recv_cq = NULL;
>                 goto alloc_cq_failed;
> @@ -1865,11 +1885,7 @@ static struct smbd_connection *_smbd_get_connection(
>         memset(&qp_attr, 0, sizeof(qp_attr));
>         qp_attr.event_handler = smbd_qp_async_error_upcall;
>         qp_attr.qp_context = sc;
> -       qp_attr.cap.max_send_wr = sp->send_credit_target;
> -       qp_attr.cap.max_recv_wr = sp->recv_credit_max;
> -       qp_attr.cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE;
> -       qp_attr.cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE;
> -       qp_attr.cap.max_inline_data = 0;
> +       qp_attr.cap = qp_cap;
>         qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
>         qp_attr.qp_type = IB_QPT_RC;
>         qp_attr.send_cq = sc->ib.send_cq;
> @@ -1883,12 +1899,6 @@ static struct smbd_connection *_smbd_get_connection(
>         }
>         sc->ib.qp = sc->rdma.cm_id->qp;
>
> -       sp->responder_resources =
> -               min_t(u8, sp->responder_resources,
> -                     sc->ib.dev->attrs.max_qp_rd_atom);
> -       log_rdma_mr(INFO, "responder_resources=%d\n",
> -               sp->responder_resources);
> -
>         memset(&conn_param, 0, sizeof(conn_param));
>         conn_param.initiator_depth = sp->initiator_depth;
>         conn_param.responder_resources = sp->responder_resources;
> --
> 2.43.0
>


-- 
Thanks,

Steve