[PATCH] RFC: vfs cachedir

Jeremy Allison jra at samba.org
Wed Apr 19 16:55:41 UTC 2017


On Wed, Apr 19, 2017 at 03:49:24PM +0200, Aurélien Aptel wrote:
> 
> So I've written a small VFS (attached) that just caches a DIR handle
> when it's opened. It reads all the directory entries in a structure that
> is used in place of the DIR handle (which is closed as soon as we read
> all the entries). It makes a "snapshot" of the directory, so to
> speak. The rewind operation just resets the internal cursor and does not
> read the directory again.
> 
> This VFS seems to solve the issue in this specific scenario. But I'm
> sure it could fail in others (which ones?). Would you let customers use
> this VFS? Should it be merged?
> 
> 1: http://pubs.opengroup.org/onlinepubs/009695399/functions/readdir.html

So this module needs to avoid direct calls to
fdopendir/opendir/readdir/closedir and call into
SMB_VFS_NEXT_XXXXX() functions instead.

Without that it's not stackable.

If you're coming to SambaXP I'm planning to cover
this in my talk on writing VFS modules :-).

Cheers,

	Jeremy.


> SUSE Linux GmbH, Maxfeldstraße 5, 90409 Nürnberg, Germany
> GF: Felix Imendörffer, Jane Smithard, Graham Norton, HRB 21284 (AG Nürnberg)

> From 484d854add9499161ec6539dc570d24e40699459 Mon Sep 17 00:00:00 2001
> From: Aurelien Aptel <aaptel at suse.com>
> Date: Mon, 10 Apr 2017 18:44:30 +0200
> Subject: [PATCH] vfs: add vfs_cachedir
> 
> Signed-off-by: Aurelien Aptel <aaptel at suse.com>
> ---
>  source3/modules/vfs_cachedir.c | 273 +++++++++++++++++++++++++++++++++++++++++
>  source3/modules/wscript_build  |   7 ++
>  source3/wscript                |   2 +-
>  3 files changed, 281 insertions(+), 1 deletion(-)
>  create mode 100644 source3/modules/vfs_cachedir.c
> 
> diff --git a/source3/modules/vfs_cachedir.c b/source3/modules/vfs_cachedir.c
> new file mode 100644
> index 00000000000..c0113ce5627
> --- /dev/null
> +++ b/source3/modules/vfs_cachedir.c
> @@ -0,0 +1,273 @@
> +/*
> +   Unix SMB/CIFS implementation.
> +   Cache dir listing operations to have stable enumeration
> +   Copyright (C) 2017 Aurélien Aptel <aaptel at suse.com>
> +
> +   This program is free software; you can redistribute it and/or modify
> +   it under the terms of the GNU General Public License as published by
> +   the Free Software Foundation; either version 3 of the License, or
> +   (at your option) any later version.
> +
> +   This program is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +   GNU General Public License for more details.
> +
> +   You should have received a copy of the GNU General Public License
> +   along with this program.  If not, see <http://www.gnu.org/licenses/>.
> +*/
> +
> +#include "includes.h"
> +#include "smbd/smbd.h"
> +#include <dirent.h>
> +#include <sys/statvfs.h>
> +
> +#undef DBGC_CLASS
> +#define DBGC_CLASS DBGC_VFS
> +
> +#define INITIAL_CACHE_SIZE 32
> +
> +struct cachedir {
> +	size_t count;
> +	size_t offset;
> +	/* talloc allocated array, actual size embedded in talloc */
> +	struct dirent *entries;
> +};
> +
> +static struct cachedir* cachedir_alloc(TALLOC_CTX* ctx)
> +{
> +	struct cachedir *dc;
> +	dc = talloc_zero(ctx, struct cachedir);
> +	if (!dc)
> +		return NULL;
> +	dc->entries = talloc_array(dc, struct dirent, INITIAL_CACHE_SIZE);
> +	if (!dc->entries) {
> +		TALLOC_FREE(dc);
> +		return NULL;
> +	}
> +
> +	return dc;
> +}
> +
> +static int cachedir_add(struct cachedir *dc, const struct dirent *dentry)
> +{
> +	if (dc->count >= talloc_array_length(dc->entries)) {
> +		struct dirent *p = talloc_realloc(dc, dc->entries, struct dirent, dc->count*2);
> +		if (!p) {
> +			return -1;
> +		}
> +		dc->entries = p;
> +	}
> +
> +	memcpy(dc->entries + dc->count, dentry, sizeof(*dentry));
> +	dc->count++;
> +	return 0;
> +}
> +
> +static int cachedir_read_whole_dir(struct cachedir* dc, DIR* dir)
> +{
> +	int r;
> +	struct dirent *tmp;
> +	int old_errno;
> +
> +	old_errno = errno;
> +	if (errno) {
> +		DBG_WARNING("[cachedir] resetting non-zero errno (%d) before readdir calls\n", errno);
> +		errno = 0;
> +	}
> +
> +	while (1) {
> +		DBG_DEBUG("[cachedir] readdir called\n");
> +		tmp = readdir(dir);
> +		if (tmp == NULL) {
> +			if (errno) {
> +				DBG_ERR("[cachedir] readdir failed (errno %d)\n", errno);
> +				return -1;
> +			}
> +			break;
> +		}
> +
> +		r = cachedir_add(dc, tmp);
> +		if (r < 0) {
> +			DBG_ERR("[cachedir] cachedir_add failed\n");
> +			return r;
> +		}
> +	}
> +
> +	errno = old_errno;
> +
> +	return 0;
> +}
> +
> +static size_t cachedir_total_size(struct cachedir *dc)
> +{
> +	return sizeof(*dc->entries) * dc->count;
> +}
> +
> +/* Directory operations */
> +
> +static DIR *cachedirwrap_opendir(vfs_handle_struct *handle,
> +			    const struct smb_filename *smb_fname,
> +			    const char *mask,
> +			    uint32_t attr)
> +{
> +	struct cachedir *dc = NULL;
> +	DIR *dh = NULL;
> +	DIR *result = NULL;
> +	int r;
> +
> +	DBG_DEBUG("[cachedir] opendir called\n");
> +	dh = opendir(smb_fname->base_name);
> +	if (!dh) {
> +		DBG_ERR("[cachedir] opendir failed (errno %d)\n", errno);
> +		return NULL;
> +	}
> +
> +	dc = cachedir_alloc(NULL);
> +	if (!dc) {
> +		DBG_ERR("[cachedir] cachedir_alloc failed\n");
> +		goto out;
> +	}
> +
> +	r = cachedir_read_whole_dir(dc, dh);
> +	if (r) {
> +		DBG_ERR("[cachedir] cachedir_read_whole_dir failed\n");
> +		goto out;
> +	}
> +
> +	result = (DIR*)dc;
> +
> +out:
> +	if (!result) {
> +		TALLOC_FREE(dc);
> +	}
> +
> +	DBG_DEBUG("[cachedir] closedir called\n");
> +	r = closedir(dh);
> +	if (r) {
> +		DBG_ERR("[cachedir] closedir failed (errno %d)\n", errno);
> +	}
> +	return result;
> +}
> +
> +static DIR *cachedirwrap_fdopendir(vfs_handle_struct *handle,
> +			files_struct *fsp,
> +			const char *mask,
> +			uint32_t attr)
> +{
> +	DIR *result = NULL;
> +	DIR *dh = NULL;
> +	struct cachedir *dc = NULL;
> +	int r;
> +
> +	DBG_DEBUG("[cachedir] sys_fdopendir called\n");
> +	dh = sys_fdopendir(fsp->fh->fd);
> +	if (!dh) {
> +		DBG_ERR("[cachedir] sys_fdopendir failed (errno %d)\n", errno);
> +		return NULL;
> +	}
> +
> +	dc = cachedir_alloc(NULL);
> +	if (!dc) {
> +		DBG_ERR("[cachedir] cachedir_alloc failed\n");
> +		goto out;
> +	}
> +
> +	r = cachedir_read_whole_dir(dc, dh);
> +	if (r) {
> +		DBG_ERR("[cachedir] cachedir_read_whole_dir failed\n");
> +		goto out;
> +	}
> +
> +	result = (DIR*)dc;
> +
> +out:
> +	if (!result) {
> +		TALLOC_FREE(dc);
> +	}
> +
> +	DBG_DEBUG("[cachedir] closedir called\n");
> +	r = closedir(dh);
> +	if (r) {
> +		DBG_ERR("[cachedir] closedir failed (errno %d)\n", errno);
> +	}
> +	return result;
> +}
> +
> +
> +static struct dirent *cachedirwrap_readdir(vfs_handle_struct *handle,
> +				          DIR *dirp,
> +					  SMB_STRUCT_STAT *sbuf)
> +{
> +	struct dirent *result = NULL;
> +	struct cachedir *dc = (struct cachedir *)dirp;
> +
> +	if (dc->offset >= cachedir_total_size(dc)) {
> +		goto out;
> +	}
> +
> +
> +	result = (struct dirent *)(((uint8_t*)dc->entries) + dc->offset);
> +	dc->offset += sizeof(*dc->entries);
> +
> +out:
> +	if (sbuf) {
> +		/* Default Posix readdir() does not give us stat info.
> +		 * Set to invalid to indicate we didn't return this info. */
> +		SET_STAT_INVALID(*sbuf);
> +	}
> +
> +	return result;
> +}
> +
> +static NTSTATUS cachedirwrap_readdir_attr(struct vfs_handle_struct *handle,
> +				     const struct smb_filename *fname,
> +				     TALLOC_CTX *mem_ctx,
> +				     struct readdir_attr_data **attr_data)
> +{
> +	return NT_STATUS_NOT_SUPPORTED;
> +}
> +
> +static void cachedirwrap_seekdir(vfs_handle_struct *handle, DIR *dirp, long offset)
> +{
> +	struct cachedir *dc = (struct cachedir *)dirp;
> +	dc->offset = offset;
> +}
> +
> +static long cachedirwrap_telldir(vfs_handle_struct *handle, DIR *dirp)
> +{
> +	struct cachedir *dc = (struct cachedir *)dirp;
> +	return (long)dc->offset;
> +}
> +
> +static void cachedirwrap_rewinddir(vfs_handle_struct *handle, DIR *dirp)
> +{
> +	struct cachedir *dc = (struct cachedir *)dirp;
> +	dc->offset = 0;
> +}
> +
> +
> +static int cachedirwrap_closedir(vfs_handle_struct *handle, DIR *dirp)
> +{
> +	struct cachedir *dc = (struct cachedir *)dirp;
> +	TALLOC_FREE(dc);
> +	return 0;
> +}
> +
> +static struct vfs_fn_pointers vfs_cachedir_fns = {
> +	.opendir_fn = cachedirwrap_opendir,
> +	.fdopendir_fn = cachedirwrap_fdopendir,
> +	.readdir_fn = cachedirwrap_readdir,
> +	.readdir_attr_fn = cachedirwrap_readdir_attr,
> +	.seekdir_fn = cachedirwrap_seekdir,
> +	.telldir_fn = cachedirwrap_telldir,
> +	.rewind_dir_fn = cachedirwrap_rewinddir,
> +	.closedir_fn = cachedirwrap_closedir,
> +};
> +
> +NTSTATUS vfs_cachedir_init(void);
> +NTSTATUS vfs_cachedir_init(void)
> +{
> +	return smb_register_vfs(SMB_VFS_INTERFACE_VERSION,
> +				"cachedir", &vfs_cachedir_fns);
> +}
> diff --git a/source3/modules/wscript_build b/source3/modules/wscript_build
> index a5d84075872..2b3db888c41 100644
> --- a/source3/modules/wscript_build
> +++ b/source3/modules/wscript_build
> @@ -502,3 +502,10 @@ bld.SAMBA3_MODULE('vfs_fake_dfq',
>                   init_function='',
>                   internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_fake_dfq'),
>                   enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_fake_dfq'))
> +
> +bld.SAMBA3_MODULE('vfs_cachedir',
> +                 subsystem='vfs',
> +                 source='vfs_cachedir.c',
> +                 init_function='',
> +                 internal_module=bld.SAMBA3_IS_STATIC_MODULE('vfs_cachedir'),
> +                 enabled=bld.SAMBA3_IS_ENABLED_MODULE('vfs_cachedir'))
> diff --git a/source3/wscript b/source3/wscript
> index 78753687431..24c2aa677c8 100644
> --- a/source3/wscript
> +++ b/source3/wscript
> @@ -1683,7 +1683,7 @@ main() {
>                                        vfs_preopen vfs_catia
>                                        vfs_media_harmony vfs_unityed_media vfs_fruit vfs_shell_snap
>                                        vfs_commit vfs_worm vfs_crossrename vfs_linux_xfs_sgid
> -                                      vfs_time_audit vfs_offline
> +                                      vfs_time_audit vfs_offline vfs_cachedir
>                                    '''))
>      default_shared_modules.extend(TO_LIST('auth_script idmap_tdb2 idmap_script'))
>      # these have broken dependencies
> -- 
> 2.12.0
> 




More information about the samba-technical mailing list