Implement "hide new files timeout"
Jeremy Allison
jra at samba.org
Thu Nov 8 21:21:19 UTC 2018
On Thu, Nov 08, 2018 at 04:10:47PM +0100, Volker Lendecke via samba-technical wrote:
> Hi!
>
> A customer has a workflow that requires files to be hidden until they
> are really finished. Attached find a patch that implements the
> required functionality.
>
> Private build still running, but it has survived the new test.
>
> Review appreciated!
I was on the fence about making this a VFS module
instead.
But it's too simple and too useful generically,
plus it's worth having just for the cleanup to
the if-statement in source4/smbd/dir.c :-).
RB+ and pushed !
Jeremy.
> --
> SerNet GmbH, Bahnhofsallee 1b, 37081 Göttingen
> phone: +49-551-370000-0, fax: +49-551-370000-9
> AG Göttingen, HRB 2816, GF: Dr. Johannes Loxen
> http://www.sernet.de, mailto:kontakt at sernet.de
> From f5dc04a9fa3e5c8e39575de5d0cb4f7272d11255 Mon Sep 17 00:00:00 2001
> From: Volker Lendecke <vl at samba.org>
> Date: Thu, 23 Aug 2018 13:51:26 +0200
> Subject: [PATCH 1/2] smbd: Add "hide new files" option
>
> See the manpage for the description
>
> Signed-off-by: Volker Lendecke <vl at samba.org>
> ---
> docs-xml/smbdotconf/filename/hidenewfilestimeout.xml | 15 +++++++++++++++
> source3/smbd/dir.c | 18 +++++++++++++++++-
> 2 files changed, 32 insertions(+), 1 deletion(-)
> create mode 100644 docs-xml/smbdotconf/filename/hidenewfilestimeout.xml
>
> diff --git a/docs-xml/smbdotconf/filename/hidenewfilestimeout.xml b/docs-xml/smbdotconf/filename/hidenewfilestimeout.xml
> new file mode 100644
> index 00000000000..ca93e726c33
> --- /dev/null
> +++ b/docs-xml/smbdotconf/filename/hidenewfilestimeout.xml
> @@ -0,0 +1,15 @@
> +<samba:parameter name="hide new files timeout"
> + context="S"
> + type="integer"
> + xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
> + <description>
> + <para>Setting this parameter to something but 0 hides files
> + that have been modified less than N seconds ago.</para>
> + <para>It can be used for ingest/process queue style workloads. A
> + processing application should only see files that are definitely
> + finished. As many applications do not have proper external workflow
> + control, this can be a way to make sure processing does not
> + interfere with file ingest.</para>
> +</description>
> +<value type="default">0</value>
> +</samba:parameter>
> diff --git a/source3/smbd/dir.c b/source3/smbd/dir.c
> index c3af5233e98..f05d7a290e5 100644
> --- a/source3/smbd/dir.c
> +++ b/source3/smbd/dir.c
> @@ -1536,6 +1536,7 @@ bool is_visible_file(connection_struct *conn, const char *dir_path,
> bool hide_unreadable = lp_hide_unreadable(SNUM(conn));
> bool hide_unwriteable = lp_hide_unwriteable_files(SNUM(conn));
> bool hide_special = lp_hide_special_files(SNUM(conn));
> + int hide_new_files_timeout = lp_hide_new_files_timeout(SNUM(conn));
> char *entry = NULL;
> struct smb_filename *smb_fname_base = NULL;
> bool ret = false;
> @@ -1550,7 +1551,11 @@ bool is_visible_file(connection_struct *conn, const char *dir_path,
> return False;
> }
>
> - if (hide_unreadable || hide_unwriteable || hide_special) {
> + if (hide_unreadable ||
> + hide_unwriteable ||
> + hide_special ||
> + (hide_new_files_timeout != 0))
> + {
> entry = talloc_asprintf(talloc_tos(), "%s/%s", dir_path, name);
> if (!entry) {
> ret = false;
> @@ -1603,6 +1608,17 @@ bool is_visible_file(connection_struct *conn, const char *dir_path,
> ret = false;
> goto out;
> }
> +
> + if (hide_new_files_timeout != 0) {
> +
> + double age = timespec_elapsed(
> + &smb_fname_base->st.st_ex_mtime);
> +
> + if (age < (double)hide_new_files_timeout) {
> + ret = false;
> + goto out;
> + }
> + }
> }
>
> ret = true;
> --
> 2.11.0
>
>
> From 4854ab86e6da5e25de904a52a1329957ae6ef5e7 Mon Sep 17 00:00:00 2001
> From: Volker Lendecke <vl at samba.org>
> Date: Thu, 8 Nov 2018 13:27:58 +0100
> Subject: [PATCH 2/2] selftest: Test hide new files timeout
>
> Signed-off-by: Volker Lendecke <vl at samba.org>
> ---
> selftest/target/Samba3.pm | 4 +
> source3/selftest/tests.py | 12 +++
> source3/torture/proto.h | 1 +
> source3/torture/test_hidenewfiles.c | 173 ++++++++++++++++++++++++++++++++++++
> source3/torture/torture.c | 1 +
> source3/wscript_build | 1 +
> 6 files changed, 192 insertions(+)
> create mode 100644 source3/torture/test_hidenewfiles.c
>
> diff --git a/selftest/target/Samba3.pm b/selftest/target/Samba3.pm
> index 91665114661..569b5d69517 100755
> --- a/selftest/target/Samba3.pm
> +++ b/selftest/target/Samba3.pm
> @@ -838,6 +838,10 @@ sub setup_simpleserver
> path = $prefix_abs/share
> vfs objects =
> smb encrypt = desired
> +
> +[hidenewfiles]
> + path = $prefix_abs/share
> + hide new files timeout = 5
> ";
>
> my $vars = $self->provision($path, "WORKGROUP",
> diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py
> index 20b96762e7a..23b98946756 100755
> --- a/source3/selftest/tests.py
> +++ b/source3/selftest/tests.py
> @@ -128,6 +128,18 @@ for t in tests:
> plantestsuite("samba3.smbtorture_s3.vfs_aio_pthread(simpleserver).%s" % t, "simpleserver", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/vfs_aio_pthread', '$USERNAME', '$PASSWORD', smbtorture3, "", "-l $LOCAL_PATH"])
> plantestsuite("samba3.smbtorture_s3.vfs_aio_fork(simpleserver).%s" % t, "simpleserver", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/vfs_aio_fork', '$USERNAME', '$PASSWORD', smbtorture3, "", "-l $LOCAL_PATH"])
>
> +plantestsuite("samba3.smbtorture_s3.hidenewfiles(simpleserver)",
> + "simpleserver",
> + [os.path.join(samba3srcdir,
> + "script/tests/test_smbtorture_s3.sh"),
> + 'hide-new-files-timeout',
> + '//$SERVER_IP/hidenewfiles',
> + '$USERNAME',
> + '$PASSWORD',
> + smbtorture3,
> + "",
> + "-l $LOCAL_PATH"])
> +
> shares = [
> "vfs_aio_pthread_async_dosmode_default1",
> "vfs_aio_pthread_async_dosmode_default2",
> diff --git a/source3/torture/proto.h b/source3/torture/proto.h
> index 1634da49315..669e077051e 100644
> --- a/source3/torture/proto.h
> +++ b/source3/torture/proto.h
> @@ -137,5 +137,6 @@ bool run_g_lock5(int dummy);
> bool run_g_lock6(int dummy);
> bool run_g_lock_ping_pong(int dummy);
> bool run_local_namemap_cache1(int dummy);
> +bool run_hidenewfiles(int dummy);
>
> #endif /* __TORTURE_H__ */
> diff --git a/source3/torture/test_hidenewfiles.c b/source3/torture/test_hidenewfiles.c
> new file mode 100644
> index 00000000000..2f1638a6ae7
> --- /dev/null
> +++ b/source3/torture/test_hidenewfiles.c
> @@ -0,0 +1,173 @@
> +/*
> + * Unix SMB/CIFS implementation.
> + * Test pthreadpool_tevent
> + * Copyright (C) Volker Lendecke 2018
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 3 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program. If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include "includes.h"
> +#include "torture/proto.h"
> +#include "libsmb/libsmb.h"
> +#include "libcli/security/security.h"
> +
> +static NTSTATUS servertime(
> + struct cli_state *cli, const char *fname, struct timeval *tv)
> +{
> + struct smb_create_returns cr;
> + NTSTATUS status;
> + uint16_t fnum;
> +
> + status = cli_ntcreate(
> + cli,
> + fname,
> + 0,
> + FILE_GENERIC_WRITE|DELETE_ACCESS,
> + FILE_ATTRIBUTE_NORMAL,
> + 0,
> + FILE_CREATE,
> + FILE_DELETE_ON_CLOSE,
> + 0,
> + &fnum,
> + &cr);
> + if (!NT_STATUS_IS_OK(status)) {
> + d_printf("cli_ntcreate failed: %s\n", nt_errstr(status));
> + return status;
> + }
> +
> + status = cli_close(cli, fnum);
> + if (!NT_STATUS_IS_OK(status)) {
> + d_printf("cli_close failed: %s\n", nt_errstr(status));
> + return status;
> + }
> +
> + nttime_to_timeval(tv, cr.creation_time);
> +
> + return NT_STATUS_OK;
> +}
> +
> +struct have_file_state {
> + bool found;
> + const char *fname;
> +};
> +
> +static NTSTATUS have_file_fn(const char *mntpoint,
> + struct file_info *f,
> + const char *mask,
> + void *private_data)
> +{
> + struct have_file_state *state = private_data;
> + state->found |= strequal(f->name, state->fname);
> + return NT_STATUS_OK;
> +}
> +
> +static bool have_file(struct cli_state *cli, const char *fname)
> +{
> + struct have_file_state state = { .fname = fname };
> + NTSTATUS status;
> +
> + status = cli_list(
> + cli,
> + "*.*",
> + FILE_ATTRIBUTE_DIRECTORY|
> + FILE_ATTRIBUTE_SYSTEM|
> + FILE_ATTRIBUTE_HIDDEN,
> + have_file_fn,
> + &state);
> + if (!NT_STATUS_IS_OK(status)) {
> + d_printf("cli_list failed: %s\n", nt_errstr(status));
> + return false;
> + }
> +
> + return state.found;
> +}
> +
> +bool run_hidenewfiles(int dummy)
> +{
> + const char *tsname = "timestamp.txt";
> + const char *fname = "new_hidden.txt";
> + struct cli_state *cli;
> + struct smb_create_returns cr;
> + struct timeval create_time;
> + uint16_t fnum;
> + NTSTATUS status;
> + bool ret = false;
> + bool gotit = false;
> + bool ok;
> +
> + /* what is configure in smb.conf */
> + unsigned hideunreadable_seconds = 5;
> +
> + ok = torture_open_connection(&cli, 0);
> + if (!ok) {
> + return false;
> + }
> +
> + cli_unlink(cli, tsname, FILE_ATTRIBUTE_SYSTEM|FILE_ATTRIBUTE_HIDDEN);
> + cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM|FILE_ATTRIBUTE_HIDDEN);
> +
> + status = cli_ntcreate(
> + cli,
> + fname,
> + 0,
> + FILE_GENERIC_WRITE|DELETE_ACCESS,
> + FILE_ATTRIBUTE_NORMAL,
> + 0,
> + FILE_CREATE,
> + 0,
> + 0,
> + &fnum,
> + &cr);
> + if (!NT_STATUS_IS_OK(status)) {
> + d_printf("cli_ntcreate failed: %s\n", nt_errstr(status));
> + return false;
> + }
> + nttime_to_timeval(&create_time, cr.last_write_time);
> +
> + while (!gotit) {
> + struct timeval now;
> + double age;
> +
> + gotit = have_file(cli, fname);
> +
> + status = servertime(cli, tsname, &now);
> + if (!NT_STATUS_IS_OK(status)) {
> + d_printf("servertime failed: %s\n",
> + nt_errstr(status));
> + goto fail;
> + }
> + age = timeval_elapsed2(&create_time, &now);
> +
> + if ((age < hideunreadable_seconds) && gotit) {
> + d_printf("Found file at age of %f\n", age);
> + goto fail;
> + }
> + if ((age > (hideunreadable_seconds*10)) && !gotit) {
> + d_printf("Did not find file after %f seconds\n", age);
> + goto fail;
> + }
> + if (gotit) {
> + break;
> + }
> +
> + smb_msleep(1000);
> + }
> +
> + ret = true;
> +fail:
> + cli_nt_delete_on_close(cli, fnum, true);
> + cli_close(cli, fnum);
> +
> + return ret;
> +}
> diff --git a/source3/torture/torture.c b/source3/torture/torture.c
> index 333b1f1c26a..22810f081b8 100644
> --- a/source3/torture/torture.c
> +++ b/source3/torture/torture.c
> @@ -11878,6 +11878,7 @@ static struct {
> { "LOCAL-CANONICALIZE-PATH", run_local_canonicalize_path, 0 },
> { "LOCAL-NAMEMAP-CACHE1", run_local_namemap_cache1, 0 },
> { "qpathinfo-bufsize", run_qpathinfo_bufsize, 0 },
> + { "hide-new-files-timeout", run_hidenewfiles, 0 },
> {NULL, NULL, 0}};
>
> /****************************************************************************
> diff --git a/source3/wscript_build b/source3/wscript_build
> index 50d675cf3fc..a8ea8e581df 100644
> --- a/source3/wscript_build
> +++ b/source3/wscript_build
> @@ -1196,6 +1196,7 @@ bld.SAMBA3_BINARY('smbtorture' + bld.env.suffix3,
> torture/wbc_async.c
> torture/test_g_lock.c
> torture/test_namemap_cache.c
> + torture/test_hidenewfiles.c
> ''',
> deps='''
> talloc
> --
> 2.11.0
>
More information about the samba-technical
mailing list