smbd: High CPU usage

[Jeremy Allison]

On Wed, Jul 18, 2018 at 09:20:10PM +0300, Vadim Lazovskiy via samba-technical wrote:
> I would not agree.
> sendfile is great. It makes you able to serve content directly from disk to
> network without unnessesery copying to userspace. It saves you lots
> syscalls and context switching.
> As for our workload profile (1-2Gbit per server) it really saves CPU time.
> 
> If someone of samba developers treat this as bug and eager to fix it I
> could provide extra information and do some debugging.

Can you try this (Linux-only) patch ?

If you can confirm it works I'll log a bug and create
fixes for the other sendfile() versions also.

Jeremy.
-------------- next part --------------
diff --git a/source3/lib/sendfile.c b/source3/lib/sendfile.c
index 3d457bd6f13..66a8e731c83 100644
--- a/source3/lib/sendfile.c
+++ b/source3/lib/sendfile.c
@@ -24,6 +24,7 @@
  */
 
 #include "includes.h"
+#include "system/filesys.h"
 
 #if defined(LINUX_SENDFILE_API)
 
@@ -36,8 +37,23 @@
 ssize_t sys_sendfile(int tofd, int fromfd, const DATA_BLOB *header, off_t offset, size_t count)
 {
 	size_t total=0;
-	ssize_t ret;
+	ssize_t ret = -1;
 	size_t hdr_len = 0;
+	int saved_errno = 0;
+	int old_flags = 0;
+
+	/*
+	 * Sendfile must complete before we can
+	 * send any other outgoing data on the socket.
+	 * Ensure socket is in blocking mode.
+	 * For SMB2 by default the socket is in non-blocking
+	 * mode.
+	 */
+	old_flags = fcntl(tofd, F_GETFL, 0);
+	ret = set_blocking(tofd, true);
+	if (ret == -1) {
+		goto out;
+	}
 
 	/*
 	 * Send the header first.
@@ -48,8 +64,9 @@ ssize_t sys_sendfile(int tofd, int fromfd, const DATA_BLOB *header, off_t offset
 		hdr_len = header->length;
 		while (total < hdr_len) {
 			ret = sys_send(tofd, header->data + total,hdr_len - total, MSG_MORE);
-			if (ret == -1)
-				return -1;
+			if (ret == -1) {
+				goto out;
+			}
 			total += ret;
 		}
 	}
@@ -59,7 +76,7 @@ ssize_t sys_sendfile(int tofd, int fromfd, const DATA_BLOB *header, off_t offset
 		ssize_t nwritten;
 		do {
 			nwritten = sendfile(tofd, fromfd, &offset, total);
-		} while (nwritten == -1 && (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK));
+		} while (nwritten == -1 && errno == EINTR);
 		if (nwritten == -1) {
 			if (errno == ENOSYS || errno == EINVAL) {
 				/* Ok - we're in a world of pain here. We just sent
@@ -72,17 +89,39 @@ ssize_t sys_sendfile(int tofd, int fromfd, const DATA_BLOB *header, off_t offset
 				 */
 				errno = EINTR; /* Normally we can never return this. */
 			}
-			return -1;
+			ret = -1;
+			goto out;
 		}
 		if (nwritten == 0) {
 			/*
 			 * EOF, return a short read
 			 */
-			return hdr_len + (count - total);
+			ret = hdr_len + (count - total);
+			goto out;
 		}
 		total -= nwritten;
 	}
-	return count + hdr_len;
+
+	ret = count + hdr_len;
+
+  out:
+
+	if (ret == -1) {
+		saved_errno = errno;
+	}
+
+	{
+		int err = fcntl(tofd, F_SETFL, old_flags);
+		if (err == -1) {
+			return -1;
+		}
+	}
+
+	if (ret == -1) {
+		errno = saved_errno;
+	}
+
+	return ret;
 }
 
 #elif defined(SOLARIS_SENDFILE_API)