From b354a3ae3a8c3d21d21b04f0ff17304664f3bbf1 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 14:23:50 +0100 Subject: [PATCH 01/10] s3:torture: call fault_setup() to get usage backtraces Signed-off-by: Stefan Metzmacher --- source3/torture/torture.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source3/torture/torture.c b/source3/torture/torture.c index b59ac30..5b72222 100644 --- a/source3/torture/torture.c +++ b/source3/torture/torture.c @@ -9316,6 +9316,7 @@ static void usage(void) setup_logging("smbtorture", DEBUG_STDOUT); load_case_tables(); + fault_setup(); if (is_default_dyn_CONFIGFILE()) { if(getenv("SMB_CONF_PATH")) { -- 1.7.9.5 From a125153ef61d852219f7c43885176468b1041530 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 14:29:51 +0100 Subject: [PATCH 02/10] s3:selftest: generate ${SELFTESTPREFIX}/subunit with the raw output Signed-off-by: Stefan Metzmacher --- source3/selftest/s3-selftest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source3/selftest/s3-selftest.sh b/source3/selftest/s3-selftest.sh index 10cf115..f02054b 100755 --- a/source3/selftest/s3-selftest.sh +++ b/source3/selftest/s3-selftest.sh @@ -45,7 +45,7 @@ else --srcdir="${SOURCEDIR}/.." \ --socket-wrapper ${TESTS} \ && touch ${SELFTESTPREFIX}/st_done ) | \ - ${FILTER_XFAIL} | ${SUBUNIT_FORMATTER} + tee ${SELFTESTPREFIX}/subunit | ${FILTER_XFAIL} | ${SUBUNIT_FORMATTER} EXIT_STATUS=$? st_test_done -- 1.7.9.5 From 26627e31e04ba74c4003be2ae9286530810b829e Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 17 Feb 2013 16:36:25 +0100 Subject: [PATCH 03/10] tevent: fix compiler warning in tevent_context_init_byname() Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tevent/tevent.c b/lib/tevent/tevent.c index aa758de..3b273d6 100644 --- a/lib/tevent/tevent.c +++ b/lib/tevent/tevent.c @@ -273,7 +273,7 @@ struct tevent_context *tevent_context_init_ops(TALLOC_CTX *mem_ctx, struct tevent_context *tevent_context_init_byname(TALLOC_CTX *mem_ctx, const char *name) { - struct tevent_ops *ops; + const struct tevent_ops *ops; ops = tevent_find_ops_byname(name); if (ops == NULL) { -- 1.7.9.5 From 2365159d0de9a43bbd0767f79ec6a4bbd3eda5d1 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 10:31:36 +0100 Subject: [PATCH 04/10] tevent: call epoll_panic() if EPOLL_CTL_DEL failed Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 8696215..fd0c6ff 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -240,9 +240,8 @@ static void epoll_del_event(struct epoll_event_context *epoll_ev, struct tevent_ event.events = epoll_map_flags(fde->flags); event.data.ptr = fde; if (epoll_ctl(epoll_ev->epoll_fd, EPOLL_CTL_DEL, fde->fd, &event) != 0) { - tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, - "epoll_del_event failed! probable early close bug (%s)\n", - strerror(errno)); + epoll_panic(epoll_ev, "EPOLL_CTL_DEL failed", false); + return; } fde->additional_flags &= ~EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT; } -- 1.7.9.5 From 0486cdb46e95b952686f3adda12cde8cb9e71950 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 11:24:59 +0100 Subject: [PATCH 05/10] tevent: remember the errno from select(), poll() and epoll_wait() Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 6 ++++-- lib/tevent/tevent_poll.c | 4 +++- lib/tevent/tevent_select.c | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index fd0c6ff..c7e9894 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -312,6 +312,7 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval #define MAXEVENTS 1 struct epoll_event events[MAXEVENTS]; int timeout = -1; + int wait_errno; if (epoll_ev->epoll_fd == -1) return -1; @@ -327,15 +328,16 @@ static int epoll_event_loop(struct epoll_event_context *epoll_ev, struct timeval tevent_trace_point_callback(epoll_ev->ev, TEVENT_TRACE_BEFORE_WAIT); ret = epoll_wait(epoll_ev->epoll_fd, events, MAXEVENTS, timeout); + wait_errno = errno; tevent_trace_point_callback(epoll_ev->ev, TEVENT_TRACE_AFTER_WAIT); - if (ret == -1 && errno == EINTR && epoll_ev->ev->signal_events) { + if (ret == -1 && wait_errno == EINTR && epoll_ev->ev->signal_events) { if (tevent_common_check_signal(epoll_ev->ev)) { return 0; } } - if (ret == -1 && errno != EINTR) { + if (ret == -1 && wait_errno != EINTR) { epoll_panic(epoll_ev, "epoll_wait() failed", true); return -1; } diff --git a/lib/tevent/tevent_poll.c b/lib/tevent/tevent_poll.c index 89b3bbc..81a7176 100644 --- a/lib/tevent/tevent_poll.c +++ b/lib/tevent/tevent_poll.c @@ -444,6 +444,7 @@ static int poll_event_loop_poll(struct tevent_context *ev, int timeout = -1; unsigned first_fd; unsigned i; + int poll_errno; if (ev->signal_events && tevent_common_check_signal(ev)) { return 0; @@ -462,9 +463,10 @@ static int poll_event_loop_poll(struct tevent_context *ev, tevent_trace_point_callback(poll_ev->ev, TEVENT_TRACE_BEFORE_WAIT); pollrtn = poll(poll_ev->fds, poll_ev->num_fds, timeout); + poll_errno = errno; tevent_trace_point_callback(poll_ev->ev, TEVENT_TRACE_AFTER_WAIT); - if (pollrtn == -1 && errno == EINTR && ev->signal_events) { + if (pollrtn == -1 && poll_errno == EINTR && ev->signal_events) { tevent_common_check_signal(ev); return 0; } diff --git a/lib/tevent/tevent_select.c b/lib/tevent/tevent_select.c index 7e0c927..ffb0d18 100644 --- a/lib/tevent/tevent_select.c +++ b/lib/tevent/tevent_select.c @@ -144,6 +144,7 @@ static int select_event_loop_select(struct select_event_context *select_ev, stru fd_set r_fds, w_fds; struct tevent_fd *fde; int selrtn; + int select_errno; /* we maybe need to recalculate the maxfd */ if (select_ev->maxfd == EVENT_INVALID_MAXFD) { @@ -175,15 +176,16 @@ static int select_event_loop_select(struct select_event_context *select_ev, stru tevent_trace_point_callback(select_ev->ev, TEVENT_TRACE_BEFORE_WAIT); selrtn = select(select_ev->maxfd+1, &r_fds, &w_fds, NULL, tvalp); + select_errno = errno; tevent_trace_point_callback(select_ev->ev, TEVENT_TRACE_AFTER_WAIT); - if (selrtn == -1 && errno == EINTR && + if (selrtn == -1 && select_errno == EINTR && select_ev->ev->signal_events) { tevent_common_check_signal(select_ev->ev); return 0; } - if (selrtn == -1 && errno == EBADF) { + if (selrtn == -1 && select_errno == EBADF) { /* the socket is dead! this should never happen as the socket should have first been made readable and that should have removed -- 1.7.9.5 From 1e110e120a09c6d1f95ea9744461a632cff2d48f Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 12:10:26 +0100 Subject: [PATCH 06/10] tevent: add epoll_{create,ctl}_panic_fallback() for testing This makes sure we only do random panics if a fallback handler is registered. Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 54 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index c7e9894..3132367 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -44,11 +44,50 @@ struct epoll_event_context { }; #ifdef TEST_PANIC_FALLBACK -static int epoll_wait_panic_fallback(int epfd, - struct epoll_event *events, - int maxevents, - int timeout) + +static int epoll_create_panic_fallback(struct epoll_event_context *epoll_ev, + int size) +{ + if (epoll_ev->panic_fallback == NULL) { + return epoll_create(size); + } + + /* 50% of the time, fail... */ + if ((random() % 2) == 0) { + errno = EINVAL; + return -1; + } + + return epoll_create(size); +} + +static int epoll_ctl_panic_fallback(struct epoll_event_context *epoll_ev, + int epfd, int op, int fd, + struct epoll_event *event) +{ + if (epoll_ev->panic_fallback == NULL) { + return epoll_ctl(epfd, op, fd, event); + } + + /* 50% of the time, fail... */ + if ((random() % 2) == 0) { + errno = EINVAL; + return -1; + } + + return epoll_ctl(epfd, op, fd, event); +} + +static int epoll_wait_panic_fallback(struct epoll_event_context *epoll_ev, + int epfd, + struct epoll_event *events, + int maxevents, + int timeout) { + if (epoll_ev->panic_fallback == NULL) { + return epoll_wait(epfd, events, maxevents, timeout); + } + /* 50% of the time, fail... */ if ((random() % 2) == 0) { errno = EINVAL; @@ -58,7 +97,12 @@ static int epoll_wait_panic_fallback(int epfd, return epoll_wait(epfd, events, maxevents, timeout); } -#define epoll_wait epoll_wait_panic_fallback +#define epoll_create(_size) \ + epoll_create_panic_fallback(epoll_ev, _size) +#define epoll_ctl(_epfd, _op, _fd, _event) \ + epoll_ctl_panic_fallback(epoll_ev,_epfd, _op, _fd, _event) +#define epoll_wait(_epfd, _events, _maxevents, _timeout) \ + epoll_wait_panic_fallback(epoll_ev, _epfd, _events, _maxevents, _timeout) #endif /* -- 1.7.9.5 From f3b065d5e219be8035cb771d9bfbbf2bfb8d2caf Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 15 Feb 2013 16:33:56 +0100 Subject: [PATCH 07/10] tevent: avoid any operation on epoll_ev after a epoll_panic() This calls TALLOC_FREE(ev->additional_data), which is epoll_ev within epoll_panic() before calling the fallback handler. In order to notice that a epoll_panic() happened, a caller can register a pointer to a bool variable under epoll_ev->panic_state. As epoll_check_reopen() can fail due to a epoll_panic(), we need to force the replay flag if we have called any event handler. Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent_epoll.c | 80 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 10 deletions(-) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 3132367..924ed21 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -40,6 +40,8 @@ struct epoll_event_context { pid_t pid; + bool panic_force_replay; + bool *panic_state; bool (*panic_fallback)(struct tevent_context *ev, bool replay); }; @@ -134,8 +136,21 @@ static void epoll_panic(struct epoll_event_context *epoll_ev, const char *reason, bool replay) { struct tevent_context *ev = epoll_ev->ev; + bool (*panic_fallback)(struct tevent_context *ev, bool replay); - if (epoll_ev->panic_fallback == NULL) { + panic_fallback = epoll_ev->panic_fallback; + + if (epoll_ev->panic_state != NULL) { + *epoll_ev->panic_state = true; + } + + if (epoll_ev->panic_force_replay) { + replay = true; + } + + TALLOC_FREE(ev->additional_data); + + if (panic_fallback == NULL) { tevent_debug(ev, TEVENT_DEBUG_FATAL, "%s (%s) replay[%u] - calling abort()\n", reason, strerror(errno), (unsigned)replay); @@ -146,7 +161,7 @@ static void epoll_panic(struct epoll_event_context *epoll_ev, "%s (%s) replay[%u] - calling panic_fallback\n", reason, strerror(errno), (unsigned)replay); - if (!epoll_ev->panic_fallback(ev, replay)) { + if (!panic_fallback(ev, replay)) { /* Fallback failed. */ tevent_debug(ev, TEVENT_DEBUG_FATAL, "%s (%s) replay[%u] - calling abort()\n", @@ -209,6 +224,8 @@ static void epoll_add_event(struct epoll_event_context *epoll_ev, struct tevent_ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) { struct tevent_fd *fde; + bool *caller_panic_state = epoll_ev->panic_state; + bool panic_triggered = false; if (epoll_ev->pid == getpid()) { return; @@ -217,8 +234,7 @@ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) close(epoll_ev->epoll_fd); epoll_ev->epoll_fd = epoll_create(64); if (epoll_ev->epoll_fd == -1) { - tevent_debug(epoll_ev->ev, TEVENT_DEBUG_FATAL, - "Failed to recreate epoll handle after fork\n"); + epoll_panic(epoll_ev, "epoll_create() failed", false); return; } @@ -228,9 +244,17 @@ static void epoll_check_reopen(struct epoll_event_context *epoll_ev) } epoll_ev->pid = getpid(); + epoll_ev->panic_state = &panic_triggered; for (fde=epoll_ev->ev->fd_events;fde;fde=fde->next) { epoll_add_event(epoll_ev, fde); + if (panic_triggered) { + if (caller_panic_state != NULL) { + *caller_panic_state = true; + } + return; + } } + epoll_ev->panic_state = NULL; } #define EPOLL_ADDITIONAL_FD_FLAG_HAS_EVENT (1<<0) @@ -461,15 +485,30 @@ static int epoll_event_context_init(struct tevent_context *ev) static int epoll_event_fd_destructor(struct tevent_fd *fde) { struct tevent_context *ev = fde->event_ctx; - struct epoll_event_context *epoll_ev = NULL; if (ev) { - epoll_ev = talloc_get_type(ev->additional_data, - struct epoll_event_context); + struct epoll_event_context *epoll_ev = + talloc_get_type_abort(ev->additional_data, + struct epoll_event_context); + bool panic_triggered = false; + + epoll_ev->panic_state = &panic_triggered; + + /* + * we must remove the event from the list + * otherwise a panic fallback handler may + * reuse invalid memory + */ + DLIST_REMOVE(ev->fd_events, fde); epoll_check_reopen(epoll_ev); - epoll_del_event(epoll_ev, fde); + if (!panic_triggered) { + epoll_del_event(epoll_ev, fde); + } + if (!panic_triggered) { + epoll_ev->panic_state = NULL; + } } return tevent_common_fd_destructor(fde); @@ -489,8 +528,7 @@ static struct tevent_fd *epoll_event_add_fd(struct tevent_context *ev, TALLOC_CT struct epoll_event_context *epoll_ev = talloc_get_type(ev->additional_data, struct epoll_event_context); struct tevent_fd *fde; - - epoll_check_reopen(epoll_ev); + bool panic_triggered = false; fde = tevent_common_add_fd(ev, mem_ctx, fd, flags, handler, private_data, @@ -499,6 +537,13 @@ static struct tevent_fd *epoll_event_add_fd(struct tevent_context *ev, TALLOC_CT talloc_set_destructor(fde, epoll_event_fd_destructor); + epoll_ev->panic_state = &panic_triggered; + epoll_check_reopen(epoll_ev); + if (panic_triggered) { + return fde; + } + epoll_ev->panic_state = NULL; + epoll_add_event(epoll_ev, fde); return fde; @@ -511,6 +556,7 @@ static void epoll_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) { struct tevent_context *ev; struct epoll_event_context *epoll_ev; + bool panic_triggered = false; if (fde->flags == flags) return; @@ -519,7 +565,12 @@ static void epoll_event_set_fd_flags(struct tevent_fd *fde, uint16_t flags) fde->flags = flags; + epoll_ev->panic_state = &panic_triggered; epoll_check_reopen(epoll_ev); + if (panic_triggered) { + return; + } + epoll_ev->panic_state = NULL; epoll_change_event(epoll_ev, fde); } @@ -532,6 +583,7 @@ static int epoll_event_loop_once(struct tevent_context *ev, const char *location struct epoll_event_context *epoll_ev = talloc_get_type(ev->additional_data, struct epoll_event_context); struct timeval tval; + bool panic_triggered = false; if (ev->signal_events && tevent_common_check_signal(ev)) { @@ -548,7 +600,15 @@ static int epoll_event_loop_once(struct tevent_context *ev, const char *location return 0; } + epoll_ev->panic_state = &panic_triggered; + epoll_ev->panic_force_replay = true; epoll_check_reopen(epoll_ev); + if (panic_triggered) { + errno = EINVAL; + return -1; + } + epoll_ev->panic_force_replay = false; + epoll_ev->panic_state = NULL; return epoll_event_loop(epoll_ev, &tval); } -- 1.7.9.5 From 3eacd492b58bee0115b693af73b18c238994c9de Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 17 Feb 2013 16:40:20 +0100 Subject: [PATCH 08/10] tevent: don't call TALLOC_FREE(ev->additional_data) in std_fallback_to_poll() The epoll backend has done the cleanup already. Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent_standard.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/tevent/tevent_standard.c b/lib/tevent/tevent_standard.c index d7a5bd7..a9c6170 100644 --- a/lib/tevent/tevent_standard.c +++ b/lib/tevent/tevent_standard.c @@ -68,7 +68,6 @@ static bool std_fallback_to_poll(struct tevent_context *ev, bool replay) /* First switch all the ops to poll. */ glue->epoll_ops = NULL; - TALLOC_FREE(ev->additional_data); /* * Set custom_ops the same as poll. -- 1.7.9.5 From b32f3a57f1debcad4b3f33b9af265034c0447858 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 17 Feb 2013 16:41:41 +0100 Subject: [PATCH 09/10] tevent: add std_event_loop_wait() We also need to fallback in tevent_loop_wait() otherwise we might miss events in the poll->fresh list. Signed-off-by: Stefan Metzmacher --- lib/tevent/tevent_standard.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/tevent/tevent_standard.c b/lib/tevent/tevent_standard.c index a9c6170..2584994 100644 --- a/lib/tevent/tevent_standard.c +++ b/lib/tevent/tevent_standard.c @@ -127,6 +127,22 @@ static int std_event_loop_once(struct tevent_context *ev, const char *location) return glue->poll_ops->loop_once(ev, location); } +static int std_event_loop_wait(struct tevent_context *ev, const char *location) +{ + void *glue_ptr = talloc_parent(ev->ops); + struct std_event_glue *glue = + talloc_get_type_abort(glue_ptr, + struct std_event_glue); + int ret; + + ret = glue->epoll_ops->loop_wait(ev, location); + if (glue->epoll_ops != NULL) { + /* No fallback */ + return ret; + } + + return glue->poll_ops->loop_wait(ev, location); +} /* Initialize the epoll backend and allow it to call a switch function if epoll fails at runtime. @@ -183,6 +199,7 @@ static int std_event_context_init(struct tevent_context *ev) *glue->glue_ops = *glue->epoll_ops; glue->glue_ops->context_init = std_event_context_init; glue->glue_ops->loop_once = std_event_loop_once; + glue->glue_ops->loop_wait = std_event_loop_wait; ret = glue->epoll_ops->context_init(ev); if (ret == -1) { -- 1.7.9.5 From 6836512d3c409cb1bac30f14577d233f7fb58539 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 17 Feb 2013 16:43:49 +0100 Subject: [PATCH 10/10] HACK epoll TEST_PANIC_FALLBACK --- lib/tevent/tevent_epoll.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tevent/tevent_epoll.c b/lib/tevent/tevent_epoll.c index 924ed21..288b350 100644 --- a/lib/tevent/tevent_epoll.c +++ b/lib/tevent/tevent_epoll.c @@ -45,6 +45,7 @@ struct epoll_event_context { bool (*panic_fallback)(struct tevent_context *ev, bool replay); }; +#define TEST_PANIC_FALLBACK #ifdef TEST_PANIC_FALLBACK static int epoll_create_panic_fallback(struct epoll_event_context *epoll_ev, -- 1.7.9.5