[Samba] Possible memory leak in
Jeremy Allison
jra at samba.org
Tue Sep 14 19:26:35 UTC 2021
On Tue, Sep 14, 2021 at 08:16:49PM +0200, Ralph Boehme via samba wrote:
>
>the pool-usage of the large process you'd sent me privately contains:
>
>$ grep pthreadpool_tevent_job_state pool-usage\ smbd_32GiB.txt | wc -l
>6468
>
>which means there are 6468 SMB2 WRITE requests pending at the server.
>
>This is likely triggered because
>
>- the client is sending data much faster then the server is writing data
>to disk
>
>- Samba returns SMB2 Async Interim Response for every received WRITE request
>
>- These grant SMB2 credits to the client
>
>- As a result the client is not throttled by a crediting window
>
>Iirc we've seen this before some time ago and iirc a Windows server
>behaves either doesn't send interim async responses or stops sending
>them at some point.
>
>To me this looks like a design flaw in the crediting logic, but we can't
>change that so we have to adjust our write processing code to handle this.
>
>Until that fix materializes (which may take some tim) you could disable
>aio as a workaround in smb.conf by setting:
>
>aio read size = 0
>aio write size = 0
OK, here is a horrible test hack (the best kind :-) that should
limit the total number of outstanding aio requests overall to
MAX_TOTAL_OUTSTANDING_AIO, where that is currently set to:
#define MAX_TOTAL_OUTSTANDING_AIO 1000
Patch included here as an attachment, and also inline in
case the list strips attachment.
Matt, if you want to test this (with the number set
to taste) and let us know if this fixes the problem,
that would be an interesting data point !
Cheers,
Jeremy
---------------------------------------------
diff --git a/source3/smbd/aio.c b/source3/smbd/aio.c
index 038487ad4ba..90b218fcc7a 100644
--- a/source3/smbd/aio.c
+++ b/source3/smbd/aio.c
@@ -24,6 +24,9 @@
#include "../lib/util/tevent_ntstatus.h"
#include "../lib/util/tevent_unix.h"
+static size_t total_num_outstanding_aio = 0;
+#define MAX_TOTAL_OUTSTANDING_AIO 1000
+
/****************************************************************************
The buffer we keep around whilst an aio request is in process.
*****************************************************************************/
@@ -97,6 +100,7 @@ static int aio_del_req_from_fsp(struct aio_req_fsp_link *lnk)
DEBUG(1, ("req %p not found in fsp %p\n", req, fsp));
return 0;
}
+ total_num_outstanding_aio -= 1;
fsp->num_aio_requests -= 1;
fsp->aio_requests[i] = fsp->aio_requests[fsp->num_aio_requests];
@@ -141,6 +145,7 @@ bool aio_add_req_to_fsp(files_struct *fsp, struct tevent_req *req)
}
fsp->aio_requests[fsp->num_aio_requests] = req;
fsp->num_aio_requests += 1;
+ total_num_outstanding_aio += 1;
lnk->fsp = fsp;
lnk->req = req;
@@ -192,6 +197,10 @@ NTSTATUS schedule_aio_read_and_X(connection_struct *conn,
return NT_STATUS_RETRY;
}
+ if (total_num_outstanding_aio >= MAX_TOTAL_OUTSTANDING_AIO) {
+ return NT_STATUS_RETRY;
+ }
+
/* The following is safe from integer wrap as we've already checked
smb_maxcnt is 128k or less. Wct is 12 for read replies */
@@ -460,6 +469,10 @@ NTSTATUS schedule_aio_write_and_X(connection_struct *conn,
return NT_STATUS_RETRY;
}
+ if (total_num_outstanding_aio >= MAX_TOTAL_OUTSTANDING_AIO) {
+ return NT_STATUS_RETRY;
+ }
+
bufsize = smb_size + 6*2;
if (!(aio_ex = create_aio_extra(NULL, fsp, bufsize))) {
@@ -710,6 +723,10 @@ NTSTATUS schedule_smb2_aio_read(connection_struct *conn,
return NT_STATUS_RETRY;
}
+ if (total_num_outstanding_aio >= MAX_TOTAL_OUTSTANDING_AIO) {
+ return NT_STATUS_RETRY;
+ }
+
/* Create the out buffer. */
*preadbuf = data_blob_talloc(ctx, NULL, smb_maxcnt);
if (preadbuf->data == NULL) {
@@ -851,6 +868,10 @@ NTSTATUS schedule_aio_smb2_write(connection_struct *conn,
return NT_STATUS_RETRY;
}
+ if (total_num_outstanding_aio >= MAX_TOTAL_OUTSTANDING_AIO) {
+ return NT_STATUS_RETRY;
+ }
+
if (!(aio_ex = create_aio_extra(smbreq->smb2req, fsp, 0))) {
return NT_STATUS_NO_MEMORY;
}
-------------- next part --------------
diff --git a/source3/smbd/aio.c b/source3/smbd/aio.c
index 038487ad4ba..90b218fcc7a 100644
--- a/source3/smbd/aio.c
+++ b/source3/smbd/aio.c
@@ -24,6 +24,9 @@
#include "../lib/util/tevent_ntstatus.h"
#include "../lib/util/tevent_unix.h"
+static size_t total_num_outstanding_aio = 0;
+#define MAX_TOTAL_OUTSTANDING_AIO 1000
+
/****************************************************************************
The buffer we keep around whilst an aio request is in process.
*****************************************************************************/
@@ -97,6 +100,7 @@ static int aio_del_req_from_fsp(struct aio_req_fsp_link *lnk)
DEBUG(1, ("req %p not found in fsp %p\n", req, fsp));
return 0;
}
+ total_num_outstanding_aio -= 1;
fsp->num_aio_requests -= 1;
fsp->aio_requests[i] = fsp->aio_requests[fsp->num_aio_requests];
@@ -141,6 +145,7 @@ bool aio_add_req_to_fsp(files_struct *fsp, struct tevent_req *req)
}
fsp->aio_requests[fsp->num_aio_requests] = req;
fsp->num_aio_requests += 1;
+ total_num_outstanding_aio += 1;
lnk->fsp = fsp;
lnk->req = req;
@@ -192,6 +197,10 @@ NTSTATUS schedule_aio_read_and_X(connection_struct *conn,
return NT_STATUS_RETRY;
}
+ if (total_num_outstanding_aio >= MAX_TOTAL_OUTSTANDING_AIO) {
+ return NT_STATUS_RETRY;
+ }
+
/* The following is safe from integer wrap as we've already checked
smb_maxcnt is 128k or less. Wct is 12 for read replies */
@@ -460,6 +469,10 @@ NTSTATUS schedule_aio_write_and_X(connection_struct *conn,
return NT_STATUS_RETRY;
}
+ if (total_num_outstanding_aio >= MAX_TOTAL_OUTSTANDING_AIO) {
+ return NT_STATUS_RETRY;
+ }
+
bufsize = smb_size + 6*2;
if (!(aio_ex = create_aio_extra(NULL, fsp, bufsize))) {
@@ -710,6 +723,10 @@ NTSTATUS schedule_smb2_aio_read(connection_struct *conn,
return NT_STATUS_RETRY;
}
+ if (total_num_outstanding_aio >= MAX_TOTAL_OUTSTANDING_AIO) {
+ return NT_STATUS_RETRY;
+ }
+
/* Create the out buffer. */
*preadbuf = data_blob_talloc(ctx, NULL, smb_maxcnt);
if (preadbuf->data == NULL) {
@@ -851,6 +868,10 @@ NTSTATUS schedule_aio_smb2_write(connection_struct *conn,
return NT_STATUS_RETRY;
}
+ if (total_num_outstanding_aio >= MAX_TOTAL_OUTSTANDING_AIO) {
+ return NT_STATUS_RETRY;
+ }
+
if (!(aio_ex = create_aio_extra(smbreq->smb2req, fsp, 0))) {
return NT_STATUS_NO_MEMORY;
}
More information about the samba
mailing list