Implement "hide new files timeout"
Volker Lendecke
Volker.Lendecke at SerNet.DE
Thu Nov 8 15:10:47 UTC 2018
Hi!
A customer has a workflow that requires files to be hidden until they
are really finished. Attached find a patch that implements the
required functionality.
Private build still running, but it has survived the new test.
Review appreciated!
Thanks, Volker
--
SerNet GmbH, Bahnhofsallee 1b, 37081 Göttingen
phone: +49-551-370000-0, fax: +49-551-370000-9
AG Göttingen, HRB 2816, GF: Dr. Johannes Loxen
http://www.sernet.de, mailto:kontakt at sernet.de
-------------- next part --------------
From f5dc04a9fa3e5c8e39575de5d0cb4f7272d11255 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Thu, 23 Aug 2018 13:51:26 +0200
Subject: [PATCH 1/2] smbd: Add "hide new files" option
See the manpage for the description
Signed-off-by: Volker Lendecke <vl at samba.org>
---
docs-xml/smbdotconf/filename/hidenewfilestimeout.xml | 15 +++++++++++++++
source3/smbd/dir.c | 18 +++++++++++++++++-
2 files changed, 32 insertions(+), 1 deletion(-)
create mode 100644 docs-xml/smbdotconf/filename/hidenewfilestimeout.xml
diff --git a/docs-xml/smbdotconf/filename/hidenewfilestimeout.xml b/docs-xml/smbdotconf/filename/hidenewfilestimeout.xml
new file mode 100644
index 00000000000..ca93e726c33
--- /dev/null
+++ b/docs-xml/smbdotconf/filename/hidenewfilestimeout.xml
@@ -0,0 +1,15 @@
+<samba:parameter name="hide new files timeout"
+ context="S"
+ type="integer"
+ xmlns:samba="http://www.samba.org/samba/DTD/samba-doc">
+ <description>
+ <para>Setting this parameter to something but 0 hides files
+ that have been modified less than N seconds ago.</para>
+ <para>It can be used for ingest/process queue style workloads. A
+ processing application should only see files that are definitely
+ finished. As many applications do not have proper external workflow
+ control, this can be a way to make sure processing does not
+ interfere with file ingest.</para>
+</description>
+<value type="default">0</value>
+</samba:parameter>
diff --git a/source3/smbd/dir.c b/source3/smbd/dir.c
index c3af5233e98..f05d7a290e5 100644
--- a/source3/smbd/dir.c
+++ b/source3/smbd/dir.c
@@ -1536,6 +1536,7 @@ bool is_visible_file(connection_struct *conn, const char *dir_path,
bool hide_unreadable = lp_hide_unreadable(SNUM(conn));
bool hide_unwriteable = lp_hide_unwriteable_files(SNUM(conn));
bool hide_special = lp_hide_special_files(SNUM(conn));
+ int hide_new_files_timeout = lp_hide_new_files_timeout(SNUM(conn));
char *entry = NULL;
struct smb_filename *smb_fname_base = NULL;
bool ret = false;
@@ -1550,7 +1551,11 @@ bool is_visible_file(connection_struct *conn, const char *dir_path,
return False;
}
- if (hide_unreadable || hide_unwriteable || hide_special) {
+ if (hide_unreadable ||
+ hide_unwriteable ||
+ hide_special ||
+ (hide_new_files_timeout != 0))
+ {
entry = talloc_asprintf(talloc_tos(), "%s/%s", dir_path, name);
if (!entry) {
ret = false;
@@ -1603,6 +1608,17 @@ bool is_visible_file(connection_struct *conn, const char *dir_path,
ret = false;
goto out;
}
+
+ if (hide_new_files_timeout != 0) {
+
+ double age = timespec_elapsed(
+ &smb_fname_base->st.st_ex_mtime);
+
+ if (age < (double)hide_new_files_timeout) {
+ ret = false;
+ goto out;
+ }
+ }
}
ret = true;
--
2.11.0
From 4854ab86e6da5e25de904a52a1329957ae6ef5e7 Mon Sep 17 00:00:00 2001
From: Volker Lendecke <vl at samba.org>
Date: Thu, 8 Nov 2018 13:27:58 +0100
Subject: [PATCH 2/2] selftest: Test hide new files timeout
Signed-off-by: Volker Lendecke <vl at samba.org>
---
selftest/target/Samba3.pm | 4 +
source3/selftest/tests.py | 12 +++
source3/torture/proto.h | 1 +
source3/torture/test_hidenewfiles.c | 173 ++++++++++++++++++++++++++++++++++++
source3/torture/torture.c | 1 +
source3/wscript_build | 1 +
6 files changed, 192 insertions(+)
create mode 100644 source3/torture/test_hidenewfiles.c
diff --git a/selftest/target/Samba3.pm b/selftest/target/Samba3.pm
index 91665114661..569b5d69517 100755
--- a/selftest/target/Samba3.pm
+++ b/selftest/target/Samba3.pm
@@ -838,6 +838,10 @@ sub setup_simpleserver
path = $prefix_abs/share
vfs objects =
smb encrypt = desired
+
+[hidenewfiles]
+ path = $prefix_abs/share
+ hide new files timeout = 5
";
my $vars = $self->provision($path, "WORKGROUP",
diff --git a/source3/selftest/tests.py b/source3/selftest/tests.py
index 20b96762e7a..23b98946756 100755
--- a/source3/selftest/tests.py
+++ b/source3/selftest/tests.py
@@ -128,6 +128,18 @@ for t in tests:
plantestsuite("samba3.smbtorture_s3.vfs_aio_pthread(simpleserver).%s" % t, "simpleserver", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/vfs_aio_pthread', '$USERNAME', '$PASSWORD', smbtorture3, "", "-l $LOCAL_PATH"])
plantestsuite("samba3.smbtorture_s3.vfs_aio_fork(simpleserver).%s" % t, "simpleserver", [os.path.join(samba3srcdir, "script/tests/test_smbtorture_s3.sh"), t, '//$SERVER_IP/vfs_aio_fork', '$USERNAME', '$PASSWORD', smbtorture3, "", "-l $LOCAL_PATH"])
+plantestsuite("samba3.smbtorture_s3.hidenewfiles(simpleserver)",
+ "simpleserver",
+ [os.path.join(samba3srcdir,
+ "script/tests/test_smbtorture_s3.sh"),
+ 'hide-new-files-timeout',
+ '//$SERVER_IP/hidenewfiles',
+ '$USERNAME',
+ '$PASSWORD',
+ smbtorture3,
+ "",
+ "-l $LOCAL_PATH"])
+
shares = [
"vfs_aio_pthread_async_dosmode_default1",
"vfs_aio_pthread_async_dosmode_default2",
diff --git a/source3/torture/proto.h b/source3/torture/proto.h
index 1634da49315..669e077051e 100644
--- a/source3/torture/proto.h
+++ b/source3/torture/proto.h
@@ -137,5 +137,6 @@ bool run_g_lock5(int dummy);
bool run_g_lock6(int dummy);
bool run_g_lock_ping_pong(int dummy);
bool run_local_namemap_cache1(int dummy);
+bool run_hidenewfiles(int dummy);
#endif /* __TORTURE_H__ */
diff --git a/source3/torture/test_hidenewfiles.c b/source3/torture/test_hidenewfiles.c
new file mode 100644
index 00000000000..2f1638a6ae7
--- /dev/null
+++ b/source3/torture/test_hidenewfiles.c
@@ -0,0 +1,173 @@
+/*
+ * Unix SMB/CIFS implementation.
+ * Test pthreadpool_tevent
+ * Copyright (C) Volker Lendecke 2018
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "includes.h"
+#include "torture/proto.h"
+#include "libsmb/libsmb.h"
+#include "libcli/security/security.h"
+
+static NTSTATUS servertime(
+ struct cli_state *cli, const char *fname, struct timeval *tv)
+{
+ struct smb_create_returns cr;
+ NTSTATUS status;
+ uint16_t fnum;
+
+ status = cli_ntcreate(
+ cli,
+ fname,
+ 0,
+ FILE_GENERIC_WRITE|DELETE_ACCESS,
+ FILE_ATTRIBUTE_NORMAL,
+ 0,
+ FILE_CREATE,
+ FILE_DELETE_ON_CLOSE,
+ 0,
+ &fnum,
+ &cr);
+ if (!NT_STATUS_IS_OK(status)) {
+ d_printf("cli_ntcreate failed: %s\n", nt_errstr(status));
+ return status;
+ }
+
+ status = cli_close(cli, fnum);
+ if (!NT_STATUS_IS_OK(status)) {
+ d_printf("cli_close failed: %s\n", nt_errstr(status));
+ return status;
+ }
+
+ nttime_to_timeval(tv, cr.creation_time);
+
+ return NT_STATUS_OK;
+}
+
+struct have_file_state {
+ bool found;
+ const char *fname;
+};
+
+static NTSTATUS have_file_fn(const char *mntpoint,
+ struct file_info *f,
+ const char *mask,
+ void *private_data)
+{
+ struct have_file_state *state = private_data;
+ state->found |= strequal(f->name, state->fname);
+ return NT_STATUS_OK;
+}
+
+static bool have_file(struct cli_state *cli, const char *fname)
+{
+ struct have_file_state state = { .fname = fname };
+ NTSTATUS status;
+
+ status = cli_list(
+ cli,
+ "*.*",
+ FILE_ATTRIBUTE_DIRECTORY|
+ FILE_ATTRIBUTE_SYSTEM|
+ FILE_ATTRIBUTE_HIDDEN,
+ have_file_fn,
+ &state);
+ if (!NT_STATUS_IS_OK(status)) {
+ d_printf("cli_list failed: %s\n", nt_errstr(status));
+ return false;
+ }
+
+ return state.found;
+}
+
+bool run_hidenewfiles(int dummy)
+{
+ const char *tsname = "timestamp.txt";
+ const char *fname = "new_hidden.txt";
+ struct cli_state *cli;
+ struct smb_create_returns cr;
+ struct timeval create_time;
+ uint16_t fnum;
+ NTSTATUS status;
+ bool ret = false;
+ bool gotit = false;
+ bool ok;
+
+ /* what is configure in smb.conf */
+ unsigned hideunreadable_seconds = 5;
+
+ ok = torture_open_connection(&cli, 0);
+ if (!ok) {
+ return false;
+ }
+
+ cli_unlink(cli, tsname, FILE_ATTRIBUTE_SYSTEM|FILE_ATTRIBUTE_HIDDEN);
+ cli_unlink(cli, fname, FILE_ATTRIBUTE_SYSTEM|FILE_ATTRIBUTE_HIDDEN);
+
+ status = cli_ntcreate(
+ cli,
+ fname,
+ 0,
+ FILE_GENERIC_WRITE|DELETE_ACCESS,
+ FILE_ATTRIBUTE_NORMAL,
+ 0,
+ FILE_CREATE,
+ 0,
+ 0,
+ &fnum,
+ &cr);
+ if (!NT_STATUS_IS_OK(status)) {
+ d_printf("cli_ntcreate failed: %s\n", nt_errstr(status));
+ return false;
+ }
+ nttime_to_timeval(&create_time, cr.last_write_time);
+
+ while (!gotit) {
+ struct timeval now;
+ double age;
+
+ gotit = have_file(cli, fname);
+
+ status = servertime(cli, tsname, &now);
+ if (!NT_STATUS_IS_OK(status)) {
+ d_printf("servertime failed: %s\n",
+ nt_errstr(status));
+ goto fail;
+ }
+ age = timeval_elapsed2(&create_time, &now);
+
+ if ((age < hideunreadable_seconds) && gotit) {
+ d_printf("Found file at age of %f\n", age);
+ goto fail;
+ }
+ if ((age > (hideunreadable_seconds*10)) && !gotit) {
+ d_printf("Did not find file after %f seconds\n", age);
+ goto fail;
+ }
+ if (gotit) {
+ break;
+ }
+
+ smb_msleep(1000);
+ }
+
+ ret = true;
+fail:
+ cli_nt_delete_on_close(cli, fnum, true);
+ cli_close(cli, fnum);
+
+ return ret;
+}
diff --git a/source3/torture/torture.c b/source3/torture/torture.c
index 333b1f1c26a..22810f081b8 100644
--- a/source3/torture/torture.c
+++ b/source3/torture/torture.c
@@ -11878,6 +11878,7 @@ static struct {
{ "LOCAL-CANONICALIZE-PATH", run_local_canonicalize_path, 0 },
{ "LOCAL-NAMEMAP-CACHE1", run_local_namemap_cache1, 0 },
{ "qpathinfo-bufsize", run_qpathinfo_bufsize, 0 },
+ { "hide-new-files-timeout", run_hidenewfiles, 0 },
{NULL, NULL, 0}};
/****************************************************************************
diff --git a/source3/wscript_build b/source3/wscript_build
index 50d675cf3fc..a8ea8e581df 100644
--- a/source3/wscript_build
+++ b/source3/wscript_build
@@ -1196,6 +1196,7 @@ bld.SAMBA3_BINARY('smbtorture' + bld.env.suffix3,
torture/wbc_async.c
torture/test_g_lock.c
torture/test_namemap_cache.c
+ torture/test_hidenewfiles.c
''',
deps='''
talloc
--
2.11.0
More information about the samba-technical
mailing list