[SCM] The rsync repository. - branch master updated

[Rsync CVS commit messages]

The branch, master has been updated
       via  ae1f0029 Reduce memory usage (#235)
      from  3911c238 Tweak SIMD & ASM option defaults.

https://git.samba.org/?p=rsync.git;a=shortlog;h=master


- Log -----------------------------------------------------------------
commit ae1f002999fc060178ef00d10cc021404a428e41
Author: Jindřich Makovička <makovick at gmail.com>
Date:   Fri Oct 1 21:04:59 2021 +0200

    Reduce memory usage (#235)
    
    In 2004, an allocation optimization has been added to the file
    list handling code, that preallocates 32k of file_struct pointers
    in a file_list. This optimization predates the incremental
    recursion feature, for which it is not appropriate anymore. When
    copying a tree containing a large number of small directories,
    using the incremental recursion, rsync allocates many short
    file_lists. Suddenly, the unused file_struct pointers can easily
    take 90-95% of the memory allocated by rsync.

-----------------------------------------------------------------------

Summary of changes:
 flist.c | 11 +++++++++--
 rsync.h |  5 +++--
 2 files changed, 12 insertions(+), 4 deletions(-)


Changeset truncated at 500 lines:

diff --git a/flist.c b/flist.c
index 3442d868..803490bd 100644
--- a/flist.c
+++ b/flist.c
@@ -295,6 +295,8 @@ static void flist_expand(struct file_list *flist, int extra)
 		flist->malloced = FLIST_START;
 	else if (flist->malloced >= FLIST_LINEAR)
 		flist->malloced += FLIST_LINEAR;
+	else if (flist->malloced < FLIST_START_LARGE/16)
+		flist->malloced *= 4;
 	else
 		flist->malloced *= 2;
 
@@ -305,7 +307,7 @@ static void flist_expand(struct file_list *flist, int extra)
 
 	new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced);
 
-	if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) {
+	if (DEBUG_GTE(FLIST, 1) && flist->files) {
 		rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n",
 		    who_am_i(),
 		    big_num(sizeof flist->files[0] * flist->malloced),
@@ -2186,8 +2188,10 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
 #endif
 
 	flist = cur_flist = flist_new(0, "send_file_list");
+	flist_expand(flist, FLIST_START_LARGE);
 	if (inc_recurse) {
 		dir_flist = flist_new(FLIST_TEMP, "send_file_list");
+		flist_expand(dir_flist, FLIST_START_LARGE);
 		flags |= FLAG_DIVERT_DIRS;
 	} else
 		dir_flist = cur_flist;
@@ -2541,10 +2545,13 @@ struct file_list *recv_file_list(int f, int dir_ndx)
 #endif
 
 	flist = flist_new(0, "recv_file_list");
+	flist_expand(flist, FLIST_START_LARGE);
 
 	if (inc_recurse) {
-		if (flist->ndx_start == 1)
+		if (flist->ndx_start == 1) {
 			dir_flist = flist_new(FLIST_TEMP, "recv_file_list");
+			flist_expand(dir_flist, FLIST_START_LARGE);
+		}
 		dstart = dir_flist->used;
 	} else {
 		dir_flist = flist;
diff --git a/rsync.h b/rsync.h
index 88319732..f8fcbffb 100644
--- a/rsync.h
+++ b/rsync.h
@@ -918,8 +918,9 @@ extern int xattrs_ndx;
  * Start the flist array at FLIST_START entries and grow it
  * by doubling until FLIST_LINEAR then grow by FLIST_LINEAR
  */
-#define FLIST_START	(32 * 1024)
-#define FLIST_LINEAR	(FLIST_START * 512)
+#define FLIST_START		(32)
+#define FLIST_START_LARGE	(32 * 1024)
+#define FLIST_LINEAR		(FLIST_START_LARGE * 512)
 
 /*
  * Extent size for allocation pools: A minimum size of 128KB


-- 
The rsync repository.