diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff6151d..21289e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -393,10 +393,12 @@ jobs: make distcheck # Branch protection should require this stable name rather than every matrix - # child. Any failed, cancelled or unexpectedly skipped dependency fails it. + # child. Any failed, cancelled or unexpectedly skipped dependency fails an + # active run. A superseded workflow must not publish a failing aggregate + # after concurrency cancellation; its replacement owns the current result. required: name: CI / required - if: always() + if: always() && !cancelled() needs: - dco - build diff --git a/INSTALL b/INSTALL index d2361a9..e37a9f6 100644 --- a/INSTALL +++ b/INSTALL @@ -43,6 +43,17 @@ please do the following: chown root:root /usr/local/spine/bin/spine chmod u+s /usr/local/spine/bin/spine + Spine is setuid root only so that it can send ICMP pings. Before it + reads any option or file it opens its raw ICMP sockets, or with + --enable-lcap keeps only CAP_NET_RAW, and then drops root for good. If + it cannot get ICMP that way it still drops root and keeps polling: ICMP + pings use datagram sockets where net.ipv4.ping_group_range allows them, + and otherwise fall back to UDP with a warning. Spine exits only if it + cannot confirm that root is gone. When root itself runs spine, for + example from root's crontab, it keeps root as before. Without the + setuid bit, ICMP works where net.ipv4.ping_group_range covers the poller + user, or where the binary has CAP_NET_RAW (setcap cap_net_raw+ep). + To compile and install Spine using MySQL versions previous to 5.5 please add the additional --with-reentrant option to the ./configure command above but please be aware that Cacti no longer officially supports MySQL prior to 5.5. diff --git a/Makefile.am b/Makefile.am index 416f2d8..fe5b685 100644 --- a/Makefile.am +++ b/Makefile.am @@ -52,7 +52,7 @@ DISTCLEANFILES = spine.1 check_PROGRAMS = test_spine_regressions dist_noinst_DATA = tests/fixtures/configuration.sql tests/fixtures/poller.sql tests/fixtures/snmpd.conf tests/fixtures/snmpv3-agent.conf tests/fixtures/snmpv3-users.conf TESTS = $(check_PROGRAMS) -test_spine_regressions_SOURCES = $(spine_SOURCES) tests/test_spine_regressions.c tests/test_spine_contracts.c tests/test_spine_reindex_contracts.c tests/test_spine_snmpv3_contracts.c tests/test_spine_settings_contracts.c tests/test_spine_cli_contracts.c tests/test_spine_stream_contracts.c tests/test_spine_output_contracts.c +test_spine_regressions_SOURCES = $(spine_SOURCES) tests/test_spine_regressions.c tests/test_spine_contracts.c tests/test_spine_reindex_contracts.c tests/test_spine_snmpv3_contracts.c tests/test_spine_settings_contracts.c tests/test_spine_cli_contracts.c tests/test_spine_stream_contracts.c tests/test_spine_output_contracts.c tests/test_spine_privilege_contracts.c # Link the real production translation units and globals, with a separate test # entry point. Only spine.c maps its own entry point in the test build. test_spine_regressions_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir) -DSPINE_TEST_PROGRAM_ENTRY @@ -65,9 +65,13 @@ test_spine_faults_SOURCES = $(spine_shared_sources) tests/test_spine_faults.c test_spine_faults_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir) test_spine_faults_LDADD = test_spine_regressions-spine.$(OBJEXT) test_spine_faults_LDFLAGS = $(AM_LDFLAGS) -Wl,--wrap=strftime -Wl,--wrap=mysql_query -Wl,--wrap=mysql_ping -Wl,--wrap=mysql_errno \ - -Wl,--wrap=pipe -Wl,--wrap=socketpair -Wl,--wrap=posix_spawn \ + -Wl,--wrap=mysql_real_connect -Wl,--wrap=mysql_thread_id -Wl,--wrap=mysql_store_result \ + -Wl,--wrap=mysql_num_rows -Wl,--wrap=mysql_fetch_row -Wl,--wrap=mysql_free_result -Wl,--wrap=mysql_num_fields \ + -Wl,--wrap=pipe -Wl,--wrap=socketpair -Wl,--wrap=posix_spawn -Wl,--wrap=posix_spawnattr_setpgroup \ -Wl,--wrap=malloc -Wl,--wrap=strdup \ - -Wl,--wrap=snmp_sess_open -Wl,--wrap=snmp_sess_close -Wl,--wrap=pthread_create + -Wl,--wrap=snmp_sess_open -Wl,--wrap=snmp_sess_close -Wl,--wrap=pthread_create \ + -Wl,--wrap=getuid -Wl,--wrap=geteuid -Wl,--wrap=getgid -Wl,--wrap=getegid \ + -Wl,--wrap=setuid -Wl,--wrap=setgid -Wl,--wrap=seteuid -Wl,--wrap=setegid -Wl,--wrap=socket endif @@ -196,8 +200,8 @@ tests_unit_test_php_runtime_DEPENDENCIES = tests/unit/php_test_server$(EXEEXT) TEST_EXTENSIONS = .sh SH_LOG_COMPILER = $(SHELL) -TESTS += tests/regression/test_remote_push.sh tests/integration/test_alloc_failure.sh tests/regression/test_child_process_safety.sh +TESTS += tests/regression/test_remote_push.sh tests/integration/test_alloc_failure.sh tests/regression/test_child_process_safety.sh tests/regression/test_setuid_privilege_drop.sh TESTS_ENVIRONMENT = top_builddir='$(abs_top_builddir)' -EXTRA_DIST += tests/regression/test_remote_push.sh tests/integration/test_alloc_failure.sh tests/regression/test_child_process_safety.sh +EXTRA_DIST += tests/regression/test_remote_push.sh tests/integration/test_alloc_failure.sh tests/regression/test_child_process_safety.sh tests/regression/test_setuid_privilege_drop.sh EXTRA_DIST += scripts/cppcheck/spine.cfg diff --git a/Makefile.in b/Makefile.in index c53c9fd..8c9bc9c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -213,7 +213,8 @@ am_test_spine_regressions_OBJECTS = $(am__objects_4) \ tests/test_spine_regressions-test_spine_settings_contracts.$(OBJEXT) \ tests/test_spine_regressions-test_spine_cli_contracts.$(OBJEXT) \ tests/test_spine_regressions-test_spine_stream_contracts.$(OBJEXT) \ - tests/test_spine_regressions-test_spine_output_contracts.$(OBJEXT) + tests/test_spine_regressions-test_spine_output_contracts.$(OBJEXT) \ + tests/test_spine_regressions-test_spine_privilege_contracts.$(OBJEXT) test_spine_regressions_OBJECTS = $(am_test_spine_regressions_OBJECTS) test_spine_regressions_LDADD = $(LDADD) am__tests_unit_php_test_server_SOURCES_DIST = \ @@ -369,6 +370,7 @@ am__depfiles_remade = ./$(DEPDIR)/error.Po ./$(DEPDIR)/nft_popen.Po \ tests/$(DEPDIR)/test_spine_regressions-test_spine_cli_contracts.Po \ tests/$(DEPDIR)/test_spine_regressions-test_spine_contracts.Po \ tests/$(DEPDIR)/test_spine_regressions-test_spine_output_contracts.Po \ + tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Po \ tests/$(DEPDIR)/test_spine_regressions-test_spine_regressions.Po \ tests/$(DEPDIR)/test_spine_regressions-test_spine_reindex_contracts.Po \ tests/$(DEPDIR)/test_spine_regressions-test_spine_settings_contracts.Po \ @@ -822,8 +824,9 @@ DISTCLEANFILES = spine.1 dist_noinst_DATA = tests/fixtures/configuration.sql tests/fixtures/poller.sql tests/fixtures/snmpd.conf tests/fixtures/snmpv3-agent.conf tests/fixtures/snmpv3-users.conf TESTS = $(check_PROGRAMS) tests/regression/test_remote_push.sh \ tests/integration/test_alloc_failure.sh \ - tests/regression/test_child_process_safety.sh -test_spine_regressions_SOURCES = $(spine_SOURCES) tests/test_spine_regressions.c tests/test_spine_contracts.c tests/test_spine_reindex_contracts.c tests/test_spine_snmpv3_contracts.c tests/test_spine_settings_contracts.c tests/test_spine_cli_contracts.c tests/test_spine_stream_contracts.c tests/test_spine_output_contracts.c + tests/regression/test_child_process_safety.sh \ + tests/regression/test_setuid_privilege_drop.sh +test_spine_regressions_SOURCES = $(spine_SOURCES) tests/test_spine_regressions.c tests/test_spine_contracts.c tests/test_spine_reindex_contracts.c tests/test_spine_snmpv3_contracts.c tests/test_spine_settings_contracts.c tests/test_spine_cli_contracts.c tests/test_spine_stream_contracts.c tests/test_spine_output_contracts.c tests/test_spine_privilege_contracts.c # Link the real production translation units and globals, with a separate test # entry point. Only spine.c maps its own entry point in the test build. test_spine_regressions_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir) -DSPINE_TEST_PROGRAM_ENTRY @@ -833,9 +836,13 @@ test_spine_regressions_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir) -DSPINE_TEST_PROGRA @HAVE_LINKER_WRAP_TRUE@test_spine_faults_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir) @HAVE_LINKER_WRAP_TRUE@test_spine_faults_LDADD = test_spine_regressions-spine.$(OBJEXT) @HAVE_LINKER_WRAP_TRUE@test_spine_faults_LDFLAGS = $(AM_LDFLAGS) -Wl,--wrap=strftime -Wl,--wrap=mysql_query -Wl,--wrap=mysql_ping -Wl,--wrap=mysql_errno \ -@HAVE_LINKER_WRAP_TRUE@ -Wl,--wrap=pipe -Wl,--wrap=socketpair -Wl,--wrap=posix_spawn \ +@HAVE_LINKER_WRAP_TRUE@ -Wl,--wrap=mysql_real_connect -Wl,--wrap=mysql_thread_id -Wl,--wrap=mysql_store_result \ +@HAVE_LINKER_WRAP_TRUE@ -Wl,--wrap=mysql_num_rows -Wl,--wrap=mysql_fetch_row -Wl,--wrap=mysql_free_result -Wl,--wrap=mysql_num_fields \ +@HAVE_LINKER_WRAP_TRUE@ -Wl,--wrap=pipe -Wl,--wrap=socketpair -Wl,--wrap=posix_spawn -Wl,--wrap=posix_spawnattr_setpgroup \ @HAVE_LINKER_WRAP_TRUE@ -Wl,--wrap=malloc -Wl,--wrap=strdup \ -@HAVE_LINKER_WRAP_TRUE@ -Wl,--wrap=snmp_sess_open -Wl,--wrap=snmp_sess_close -Wl,--wrap=pthread_create +@HAVE_LINKER_WRAP_TRUE@ -Wl,--wrap=snmp_sess_open -Wl,--wrap=snmp_sess_close -Wl,--wrap=pthread_create \ +@HAVE_LINKER_WRAP_TRUE@ -Wl,--wrap=getuid -Wl,--wrap=geteuid -Wl,--wrap=getgid -Wl,--wrap=getegid \ +@HAVE_LINKER_WRAP_TRUE@ -Wl,--wrap=setuid -Wl,--wrap=setgid -Wl,--wrap=seteuid -Wl,--wrap=setegid -Wl,--wrap=socket # Headers are not referenced by any _SOURCES, so automake will not distribute @@ -854,6 +861,7 @@ EXTRA_DIST = spine.1 spine.conf.dist tests/README.md \ tests/regression/test_remote_push.sh \ tests/integration/test_alloc_failure.sh \ tests/regression/test_child_process_safety.sh \ + tests/regression/test_setuid_privilege_drop.sh \ scripts/cppcheck/spine.cfg AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/src/core -I$(top_builddir) @HAVE_CMOCKA_TRUE@tests_unit_php_test_server_SOURCES = tests/unit/php_test_server.c @@ -1088,6 +1096,8 @@ tests/test_spine_regressions-test_spine_stream_contracts.$(OBJEXT): \ tests/$(am__dirstamp) tests/$(DEPDIR)/$(am__dirstamp) tests/test_spine_regressions-test_spine_output_contracts.$(OBJEXT): \ tests/$(am__dirstamp) tests/$(DEPDIR)/$(am__dirstamp) +tests/test_spine_regressions-test_spine_privilege_contracts.$(OBJEXT): \ + tests/$(am__dirstamp) tests/$(DEPDIR)/$(am__dirstamp) test_spine_regressions$(EXEEXT): $(test_spine_regressions_OBJECTS) $(test_spine_regressions_DEPENDENCIES) $(EXTRA_test_spine_regressions_DEPENDENCIES) @rm -f test_spine_regressions$(EXEEXT) @@ -1243,6 +1253,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/test_spine_regressions-test_spine_cli_contracts.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/test_spine_regressions-test_spine_contracts.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/test_spine_regressions-test_spine_output_contracts.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/test_spine_regressions-test_spine_regressions.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/test_spine_regressions-test_spine_reindex_contracts.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/test_spine_regressions-test_spine_settings_contracts.Po@am__quote@ # am--include-marker @@ -1711,6 +1722,20 @@ tests/test_spine_regressions-test_spine_output_contracts.obj: tests/test_spine_o @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_spine_regressions_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o tests/test_spine_regressions-test_spine_output_contracts.obj `if test -f 'tests/test_spine_output_contracts.c'; then $(CYGPATH_W) 'tests/test_spine_output_contracts.c'; else $(CYGPATH_W) '$(srcdir)/tests/test_spine_output_contracts.c'; fi` +tests/test_spine_regressions-test_spine_privilege_contracts.o: tests/test_spine_privilege_contracts.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_spine_regressions_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT tests/test_spine_regressions-test_spine_privilege_contracts.o -MD -MP -MF tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Tpo -c -o tests/test_spine_regressions-test_spine_privilege_contracts.o `test -f 'tests/test_spine_privilege_contracts.c' || echo '$(srcdir)/'`tests/test_spine_privilege_contracts.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Tpo tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='tests/test_spine_privilege_contracts.c' object='tests/test_spine_regressions-test_spine_privilege_contracts.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_spine_regressions_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o tests/test_spine_regressions-test_spine_privilege_contracts.o `test -f 'tests/test_spine_privilege_contracts.c' || echo '$(srcdir)/'`tests/test_spine_privilege_contracts.c + +tests/test_spine_regressions-test_spine_privilege_contracts.obj: tests/test_spine_privilege_contracts.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_spine_regressions_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT tests/test_spine_regressions-test_spine_privilege_contracts.obj -MD -MP -MF tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Tpo -c -o tests/test_spine_regressions-test_spine_privilege_contracts.obj `if test -f 'tests/test_spine_privilege_contracts.c'; then $(CYGPATH_W) 'tests/test_spine_privilege_contracts.c'; else $(CYGPATH_W) '$(srcdir)/tests/test_spine_privilege_contracts.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Tpo tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='tests/test_spine_privilege_contracts.c' object='tests/test_spine_regressions-test_spine_privilege_contracts.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(test_spine_regressions_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o tests/test_spine_regressions-test_spine_privilege_contracts.obj `if test -f 'tests/test_spine_privilege_contracts.c'; then $(CYGPATH_W) 'tests/test_spine_privilege_contracts.c'; else $(CYGPATH_W) '$(srcdir)/tests/test_spine_privilege_contracts.c'; fi` + tests/unit/test_linked-test_linked.o: tests/unit/test_linked.c @am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(tests_unit_test_linked_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT tests/unit/test_linked-test_linked.o -MD -MP -MF tests/unit/$(DEPDIR)/test_linked-test_linked.Tpo -c -o tests/unit/test_linked-test_linked.o `test -f 'tests/unit/test_linked.c' || echo '$(srcdir)/'`tests/unit/test_linked.c @am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) tests/unit/$(DEPDIR)/test_linked-test_linked.Tpo tests/unit/$(DEPDIR)/test_linked-test_linked.Po @@ -2691,6 +2716,7 @@ distclean: distclean-am -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_cli_contracts.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_contracts.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_output_contracts.Po + -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_regressions.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_reindex_contracts.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_settings_contracts.Po @@ -2812,6 +2838,7 @@ maintainer-clean: maintainer-clean-am -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_cli_contracts.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_contracts.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_output_contracts.Po + -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_privilege_contracts.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_regressions.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_reindex_contracts.Po -rm -f tests/$(DEPDIR)/test_spine_regressions-test_spine_settings_contracts.Po diff --git a/configure.ac b/configure.ac index 9cb25ec..7dc5e57 100644 --- a/configure.ac +++ b/configure.ac @@ -211,21 +211,27 @@ AC_ARG_ENABLE(lcap, [ ENABLED_LCAP=no ] ) +HAVE_LCAP=no if test x$ENABLED_LCAP != xno; then + AC_MSG_RESULT([yes]) if test x$FOUND_SYS_CAPABILITY_H != xno; then - AC_MSG_RESULT([yes]) AC_CHECK_LIB(cap, cap_init, [ LIBS="-lcap $LIBS" AC_DEFINE(HAVE_LCAP, 1, Linux Capabilities) - HAVE_LCAP=yes ], - [ AC_MSG_RESULT(Cannot find Linux Capabilities library(cap)...) - HAVE_LCAP=no ] - ) - else - AC_MSG_RESULT([no]) + HAVE_LCAP=yes ]) + fi + # An explicit request must not quietly produce a build without it. + if test x$HAVE_LCAP != xyes; then + AC_MSG_ERROR([--enable-lcap was given but libcap (sys/capability.h and -lcap) was not found]) fi else AC_MSG_RESULT([no]) + if test x$FOUND_SYS_CAPABILITY_H != xno; then + AC_MSG_NOTICE([libcap headers were found but are not used; pass --enable-lcap to use them]) + else + AC_MSG_NOTICE([libcap was not found; building without Linux capabilities support]) + fi + AC_MSG_NOTICE([without it a setuid root spine opens its raw ICMP sockets at startup, then drops root for good]) fi # ****************** MySQL-compatible client checks *********************** diff --git a/error.c b/error.c index 3f6a587..80ece8d 100644 --- a/error.c +++ b/error.c @@ -98,12 +98,15 @@ static void spine_signal_handler(int spine_signal) { const char *message = NULL; int saved_errno = errno; - /* A real fault (SIGSEGV/SIGBUS/...) resets to SIG_DFL and returns so the - * faulting instruction re-executes and dies with a core. SIGPIPE is a - * routine, recurring condition instead, and signal dispositions are - * process-wide: dropping it to SIG_DFL here, even briefly, would let a - * broken-pipe write on any other thread terminate the process during - * that window. Leave this handler permanently installed for SIGPIPE. */ + /* Every signal handled here except SIGSEGV, which exits with status 1 + * below, resets to SIG_DFL and is raised again before returning, so the + * process ends with that signal's default action and core. Returning alone only ended real faults, which re-execute; after + * SIGINT, SIGQUIT or a sent SIGBUS the poller logged FATAL and kept going. + * SIGPIPE is a routine, recurring condition instead, and signal + * dispositions are process-wide: dropping it to SIG_DFL here, even + * briefly, would let a broken-pipe write on any other thread terminate + * the process during that window. Leave this handler permanently + * installed for SIGPIPE. */ if (spine_signal != SIGPIPE) { signal(spine_signal, SIG_DFL); } @@ -149,6 +152,12 @@ static void spine_signal_handler(int spine_signal) { _exit(1); } + /* The signal stays blocked until this handler returns, then the default + * action ends the process. raise() is async-signal-safe. */ + if (spine_signal != SIGPIPE) { + (void) raise(spine_signal); + } + errno = saved_errno; } diff --git a/nft_popen.c b/nft_popen.c index 67898a4..42537cc 100644 --- a/nft_popen.c +++ b/nft_popen.c @@ -364,6 +364,9 @@ int nft_popen(const char * command, const char * type) { /* nft_popen() reports why process creation failed through errno, as * popen() does; cleanup below must not overwrite it. */ int failure_errno = 0; + posix_spawnattr_t attr; + int attr_valid = FALSE; + int attr_err; if (command == NULL || type == NULL) { errno = EINVAL; @@ -445,6 +448,21 @@ int nft_popen(const char * command, const char * type) { return -1; } + /* Each script leads its own process group so a timeout can kill the + * group. Killing only the shell left anything it had started running, + * still holding the pipe and counting against the process limit. */ + attr_err = posix_spawnattr_init(&attr); + if (attr_err == 0) { + attr_valid = TRUE; + attr_err = posix_spawnattr_setpgroup(&attr, 0); + } + if (attr_err == 0) attr_err = posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETPGROUP); + if (attr_err != 0) { + failure_errno = attr_err; + SPINE_LOG(("ERROR: SCRIPT: posix_spawnattr setup failed: %s", strerror(attr_err))); + goto spawn_failed; + } + /* The pipe ends are close-on-exec, which is the point: another thread * spawning in this window must not inherit them. The child needs its own * end, and dup2 clears the flag on its target, so the usual paths are @@ -521,7 +539,7 @@ int nft_popen(const char * command, const char * type) { int spawn_err; retry: - spawn_err = posix_spawn(&pid, spawn_shell, &fa, NULL, argv, environ); + spawn_err = posix_spawn(&pid, spawn_shell, &fa, &attr, argv, environ); if (spawn_err != 0) { if ((spawn_err == EAGAIN || spawn_err == ENOMEM) && retry_count < 3) { @@ -540,6 +558,7 @@ spawn_failed: * nft_pclose() in every poller thread and the daemon stops collecting * script data until it is restarted. */ posix_spawn_file_actions_destroy(&fa); + if (attr_valid) posix_spawnattr_destroy(&attr); if (inherit_fd != -1) { (void)close(inherit_fd); @@ -559,6 +578,7 @@ spawn_failed: } posix_spawn_file_actions_destroy(&fa); + posix_spawnattr_destroy(&attr); /* The child holds its own duplicate. Keeping this one would hold the pipe's * write end open, so the reader never sees EOF and exec_poll() blocks to @@ -688,7 +708,8 @@ nft_pclose(int fd) pid = cur->pid; break; case 1: - (void)kill(cur->pid, SIGKILL); + /* The negative pid reaches the script's whole process group. */ + (void)kill(-cur->pid, SIGKILL); nft_abandon_child(cur->pid, "did not exit before pipe close"); errno = ETIMEDOUT; pid = -1; @@ -699,7 +720,7 @@ nft_pclose(int fd) * outside the sweep's reach. Preserve waitpid()'s errno. */ { int saved_errno = errno; - (void)kill(cur->pid, SIGKILL); + (void)kill(-cur->pid, SIGKILL); nft_abandon_child(cur->pid, "waitpid failed"); errno = saved_errno; } diff --git a/php.c b/php.c index fe54198..bb27775 100644 --- a/php.c +++ b/php.c @@ -84,6 +84,17 @@ static char *php_undefined_result(void) { return result; } +/* Script and script server arguments can carry SNMP communities and v3 + * passphrases, so logs name only the first token, the script, never the + * arguments. */ +void php_command_script(const char *command, char *script, size_t capacity) { + size_t length = strcspn(command, " \t\r\n"); + + if (length >= capacity) length = capacity - 1; + memcpy(script, command, length); + script[length] = '\0'; +} + static void php_fail_read(int php_process, int allow_restart) { php_processes[php_process].php_state = PHP_BUSY; if (allow_restart) { @@ -106,7 +117,7 @@ void php_processes_initialize(php_t *processes, int count) { } } -static char *php_read_result(int php_process, char *command, int allow_restart); +static char *php_read_result(int php_process, const char *command, int allow_restart); /* Block SIGPIPE in the calling thread around Spine's two pipe writes. The * daemon normally catches SIGPIPE with a no-op handler process-wide, but this local guard @@ -201,7 +212,11 @@ char *php_cmd(const char *php_command, int php_process) { /* if write status is <= 0 then the script server may be hung */ if (bytes <= 0) { - SPINE_LOG(("ERROR: SS[%i] PHP Script Server communications lost sending Command[%s]. Restarting PHP Script Server", php_process, command)); + char script[SMALL_BUFSIZE]; + + php_command_script(command, script, sizeof(script)); + SPINE_LOG(("ERROR: SS[%i] PHP Script Server communications lost sending a command. Restarting PHP Script Server", php_process)); + SPINE_LOG_DEBUG(("DEBUG: SS[%i] Unsent command for script '%s'", php_process, script)); php_close(php_process); retries++; @@ -323,7 +338,7 @@ char *php_read_result_for_test(int php_process, char *command, int allow_restart } #endif -/*! \fn char *php_readpipe(int php_process, char *command) +/*! \fn char *php_readpipe(int php_process, const char *command) * \brief read a line from a PHP Script Server process * \param php_process the PHP Script Server process to obtain output from * @@ -334,7 +349,7 @@ char *php_read_result_for_test(int php_process, char *command, int allow_restart * * \return a string pointer to the PHP Script Server response */ -/*! \fn static char *php_read_result(int php_process, char *command, int allow_restart) +/*! \fn static char *php_read_result(int php_process, const char *command, int allow_restart) * \brief reads one script server response. * * allow_restart is FALSE for the startup handshake. php_init() calls this to @@ -344,7 +359,7 @@ char *php_read_result_for_test(int php_process, char *command, int allow_restart * recursion, spawning a fresh server at every level. Refusing the restart on * the handshake bounds the depth at one by construction. */ -static char *php_read_result(int php_process, char *command, int allow_restart) { +static char *php_read_result(int php_process, const char *command, int allow_restart) { fd_set fds; struct timeval timeout; double begin_time = 0; @@ -353,6 +368,7 @@ static char *php_read_result(int php_process, char *command, int allow_restart) char *result_string; int response_timeout; double read_deadline; + char script[SMALL_BUFSIZE]; ssize_t i; char *cp; @@ -366,7 +382,7 @@ static char *php_read_result(int php_process, char *command, int allow_restart) /* Public readers may bypass php_cmd()'s process-slot validation. Keep * both command and startup readers inside the allocated process table. */ if (php_processes == NULL || php_process < 0 || - php_process >= set.php_servers || php_process >= MAX_PHP_SERVERS) { + php_process >= set.php.php_servers || php_process >= MAX_PHP_SERVERS) { SET_UNDEFINED(result_string); return result_string; } @@ -458,7 +474,9 @@ static char *php_read_result(int php_process, char *command, int allow_restart) case 0: /* record end time */ end_time = get_time_as_double(); - SPINE_LOG(("WARNING: SS[%i] The PHP Script Server did not respond in time for Timeout[%0.2f], Command[%s] and will therefore be restarted", php_process, end_time - begin_time, command)); + SPINE_LOG(("WARNING: SS[%i] The PHP Script Server did not respond in time for Timeout[%0.2f] and will therefore be restarted", php_process, end_time - begin_time)); + php_command_script(command, script, sizeof(script)); + SPINE_LOG_DEBUG(("DEBUG: SS[%i] Unanswered command for script '%s'", php_process, script)); SET_UNDEFINED(result_string); php_fail_read(php_process, allow_restart); break; @@ -486,7 +504,9 @@ static char *php_read_result(int php_process, char *command, int allow_restart) int ready = spine_wait_readable(php_processes[php_process].php_read_fd, read_deadline); if (ready <= 0) { - SPINE_LOG(("WARNING: SS[%i] The PHP Script Server sent a partial response and %s, Command[%s]", php_process, ready == 0 ? "timed out" : "failed", command)); + SPINE_LOG(("WARNING: SS[%i] The PHP Script Server sent a partial response and %s", php_process, ready == 0 ? "timed out" : "failed")); + php_command_script(command, script, sizeof(script)); + SPINE_LOG_DEBUG(("DEBUG: SS[%i] Unanswered command for script '%s'", php_process, script)); SET_UNDEFINED(result_string); read_ok = FALSE; break; @@ -537,13 +557,13 @@ static char *php_read_result(int php_process, char *command, int allow_restart) return result_string; } -/*! \fn char *php_readpipe(int php_process, char *command) +/*! \fn char *php_readpipe(int php_process, const char *command) * \brief reads a script server response, restarting a server that stops * answering. * * \return a string pointer to the PHP Script Server response */ -char *php_readpipe(int php_process, char *command) { +char *php_readpipe(int php_process, const char *command) { return php_read_result(php_process, command, TRUE); } diff --git a/php.h b/php.h index 5aa3fa9..c2782a4 100644 --- a/php.h +++ b/php.h @@ -34,7 +34,8 @@ #ifndef SPINE_PHP_H #define SPINE_PHP_H extern char *php_cmd(const char *php_command, int php_process); -extern char *php_readpipe(int php_process, char *command); +extern char *php_readpipe(int php_process, const char *command); +extern void php_command_script(const char *command, char *script, size_t capacity); extern int php_init(int php_process); extern void php_close(int php_process); extern int php_get_process(void); diff --git a/ping.c b/ping.c index 613839b..b856827 100644 --- a/ping.c +++ b/ping.c @@ -274,6 +274,360 @@ spine_icmp_reply_t spine_icmp_classify_dgram_reply(const unsigned char *reply, s return SPINE_ICMP_REPLY_OK; } +#ifdef SPINE_HAVE_ICMPV6 +static const char icmp6_payload[] = "cacti-monitoring-system"; + +/*! \fn static int icmp6_reply_matches(const unsigned char *reply, ssize_t len, uint16_t id, uint16_t seq, int check_id) + * \brief decides whether an ICMPv6 message answers our echo request + * + * ICMPv6 sockets deliver the message without the IPv6 header. id and seq + * are in network order, as written into the request. A datagram socket + * rewrites the echo id, so its caller passes check_id FALSE. + */ +static int icmp6_reply_matches(const unsigned char *reply, ssize_t len, uint16_t id, uint16_t seq, int check_id) { + struct icmp6_hdr hdr; + size_t payload_len = sizeof(icmp6_payload) - 1; + + if (len < 0 || (size_t) len < sizeof(hdr) + payload_len) { + return FALSE; + } + + memcpy(&hdr, reply, sizeof(hdr)); + + if (hdr.icmp6_type != ICMP6_ECHO_REPLY || hdr.icmp6_seq != seq) { + return FALSE; + } + + if (check_id && hdr.icmp6_id != id) { + return FALSE; + } + + return memcmp(reply + sizeof(hdr), icmp6_payload, payload_len) == 0; +} +#endif + +/* Raw sockets opened by drop_privileges() before a setuid root spine gave up + * root for good. Without CAP_NET_RAW nothing can open another, so every + * poller thread shares them. A raw socket queues each ICMP message once, so a + * thread that reads a reply meant for another thread must hand it over rather + * than drop it: one thread at a time reads, matches each reply against the + * registered requests, and wakes only the thread it answers. LOCK_ICMP guards + * all of it. */ +typedef struct { + int fd; + int reading; +} icmp_shared_t; + +typedef struct icmp_waiter { + struct icmp_waiter *next; + icmp_shared_t *shared; + pthread_cond_t wake; + int family; + uint16_t id; + uint16_t seq; + struct in_addr peer; + #ifdef SPINE_HAVE_ICMPV6 + struct in6_addr peer6; + #endif + int answered; + int sleeping; +} icmp_waiter_t; + +/* A reader empties the queue in batches read without LOCK_ICMP held, so a + * flood delays neither registrations nor the waiters being woken. The bound + * on one pass keeps the reader's own deadline check running. Matching needs + * only the IP header, the ICMP header and the payload marker, so each slot + * keeps the first ICMP_SHARED_SNAPLEN bytes. The queue itself is sized for + * every poller thread rather than one. */ +#define ICMP_SHARED_DRAIN_MAX 1024 +#define ICMP_SHARED_BATCH 64 +#define ICMP_SHARED_SNAPLEN 128 +#define ICMP_SHARED_RCVBUF (4 * 1024 * 1024) + +#if defined(__linux__) +#ifndef SOL_RAW +#define SOL_RAW 255 +#endif +#ifndef ICMP_FILTER +#define ICMP_FILTER 1 +#endif +#endif + +static icmp_shared_t icmp_shared = {-1, FALSE}; +#ifdef SPINE_HAVE_ICMPV6 +static icmp_shared_t icmp6_shared = {-1, FALSE}; +#endif +static icmp_waiter_t *icmp_waiters = NULL; + +/*! \fn static int icmp_open_socket(int family, int type, int protocol) + * \brief opens an ICMP socket that PHP and script children cannot inherit + * + * Poller threads posix_spawn() scripts while others ping, so the flag is set + * atomically where SOCK_CLOEXEC exists; elsewhere fcntl() narrows the window. + */ +static int icmp_open_socket(int family, int type, int protocol) { + int fd; + + #ifdef SOCK_CLOEXEC + fd = socket(family, type | SOCK_CLOEXEC, protocol); + + if (fd != -1 || errno != EINVAL) { + return fd; + } + #endif + + fd = socket(family, type, protocol); + + if (fd != -1 && fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) { + close(fd); + return -1; + } + + return fd; +} + +static int icmp_open_shared(int family, int protocol) { + int fd = icmp_open_socket(family, SOCK_RAW, protocol); + + if (fd == -1) { + return -1; + } + + /* A descriptor select() cannot watch is no use to the readers below. */ + if (fd >= FD_SETSIZE) { + close(fd); + return -1; + } + + #if defined(__linux__) + /* The kernel copies every inbound ICMP message to a raw socket. Echo + * requests and errors aimed at this host would otherwise compete with + * the replies for the single shared queue. Best effort, as is + * ICMP6_FILTER on the IPv6 socket. */ + if (family == AF_INET) { + uint32_t blocked = ~(1U << ICMP_ECHOREPLY); + + (void) setsockopt(fd, SOL_RAW, ICMP_FILTER, &blocked, sizeof(blocked)); + } + #endif + + /* Best effort as well. SO_RCVBUFFORCE passes net.core.rmem_max, which + * is why this happens here, while spine is still root. */ + { + int rcvbuf = ICMP_SHARED_RCVBUF; + + #ifdef SO_RCVBUFFORCE + if (setsockopt(fd, SOL_SOCKET, SO_RCVBUFFORCE, &rcvbuf, sizeof(rcvbuf)) != 0) + #endif + (void) setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)); + } + + return fd; +} + +/*! \fn int ping_icmp_open_shared(void) + * \brief opens the shared raw ICMP sockets; the caller must still be root + * + * \return TRUE if the IPv4 socket is open. IPv6 is best effort. + */ +int ping_icmp_open_shared(void) { + if (icmp_shared.fd == -1) { + icmp_shared.fd = icmp_open_shared(AF_INET, IPPROTO_ICMP); + } + + #ifdef SPINE_HAVE_ICMPV6 + if (icmp6_shared.fd == -1) { + icmp6_shared.fd = icmp_open_shared(AF_INET6, IPPROTO_ICMPV6); + } + #endif + + return icmp_shared.fd != -1; +} + +int ping_icmp_shared_available(void) { + return icmp_shared.fd != -1; +} + +/* Called with LOCK_ICMP held whenever nobody is reading a socket: wake one + * sleeping waiter so it takes over. A waiter that is not asleep finds the + * socket free on its own before it next sleeps. */ +static void icmp_shared_handoff(const icmp_shared_t *shared) { + icmp_waiter_t *waiter; + + if (shared->reading) { + return; + } + + for (waiter = icmp_waiters; waiter != NULL; waiter = waiter->next) { + if (waiter->shared == shared && waiter->sleeping && !waiter->answered) { + pthread_cond_signal(&waiter->wake); + return; + } + } +} + +static void icmp_shared_register(icmp_waiter_t *waiter) { + pthread_cond_init(&waiter->wake, NULL); + + thread_mutex_lock(LOCK_ICMP); + waiter->answered = FALSE; + waiter->sleeping = FALSE; + waiter->next = icmp_waiters; + icmp_waiters = waiter; + thread_mutex_unlock(LOCK_ICMP); +} + +static void icmp_shared_unregister(icmp_waiter_t *waiter) { + icmp_waiter_t **link; + + thread_mutex_lock(LOCK_ICMP); + for (link = &icmp_waiters; *link != NULL; link = &(*link)->next) { + if (*link == waiter) { + *link = waiter->next; + break; + } + } + icmp_shared_handoff(waiter->shared); + thread_mutex_unlock(LOCK_ICMP); + + pthread_cond_destroy(&waiter->wake); +} + +/* Called with LOCK_ICMP held. A reply nobody waits for is dropped, exactly as + * a per-thread socket would drop a reply that is not its own. */ +static void icmp_shared_dispatch(int family, const unsigned char *reply, ssize_t len, + const struct sockaddr_storage *from) { + icmp_waiter_t *waiter; + + for (waiter = icmp_waiters; waiter != NULL; waiter = waiter->next) { + if (waiter->family != family || waiter->answered) { + continue; + } + + if (family == AF_INET) { + const struct sockaddr_in *source = (const struct sockaddr_in *) from; + + if (source->sin_addr.s_addr != waiter->peer.s_addr || + spine_icmp_classify_reply(reply, len, waiter->id, waiter->seq, NULL) != SPINE_ICMP_REPLY_OK) { + continue; + } + } + #ifdef SPINE_HAVE_ICMPV6 + else { + const struct sockaddr_in6 *source = (const struct sockaddr_in6 *) from; + + if (memcmp(&source->sin6_addr, &waiter->peer6, sizeof(waiter->peer6)) != 0 || + !icmp6_reply_matches(reply, len, waiter->id, waiter->seq, TRUE)) { + continue; + } + } + #endif + + waiter->answered = TRUE; + pthread_cond_signal(&waiter->wake); + return; + } +} + +/*! \fn static int icmp_shared_await(icmp_waiter_t *waiter, double deadline) + * \brief waits until the reply to a registered request arrives, or deadline + * + * Whichever waiting thread finds the socket idle reads it, so a reply is + * never stuck behind a thread whose own deadline is further away: a reader + * returns from select() as soon as anything arrives, drains the queue, and + * wakes the threads whose replies it found. + * + * \return TRUE if the reply arrived in time. + */ +static int icmp_shared_await(icmp_waiter_t *waiter, double deadline) { + icmp_shared_t *shared = waiter->shared; + unsigned char replies[ICMP_SHARED_BATCH][ICMP_SHARED_SNAPLEN]; + struct sockaddr_storage sources[ICMP_SHARED_BATCH]; + ssize_t lengths[ICMP_SHARED_BATCH]; + socklen_t fromlen; + ssize_t received; + struct timeval wait; + struct timespec until; + fd_set fds; + double now; + double remaining; + int readable; + int drained; + int count; + int slot; + int answered; + + until.tv_sec = (time_t) deadline; + until.tv_nsec = (long) ((deadline - (double) until.tv_sec) * 1000000000.0); + + thread_mutex_lock(LOCK_ICMP); + + while (!waiter->answered) { + now = get_time_as_double(); + + if (now >= deadline) { + break; + } + + if (shared->reading) { + waiter->sleeping = TRUE; + pthread_cond_timedwait(&waiter->wake, get_lock(LOCK_ICMP), &until); + waiter->sleeping = FALSE; + continue; + } + + shared->reading = TRUE; + thread_mutex_unlock(LOCK_ICMP); + + remaining = deadline - now; + wait.tv_sec = (time_t) remaining; + wait.tv_usec = (suseconds_t) ((remaining - (double) wait.tv_sec) * 1000000.0); + + FD_ZERO(&fds); + FD_SET(shared->fd, &fds); + readable = select(shared->fd + 1, &fds, NULL, NULL, &wait) > 0; + + /* Empty the queue: under a flood, one reply per pass falls behind + * and the kernel drops the replies that matter. */ + for (drained = 0; readable && drained < ICMP_SHARED_DRAIN_MAX; drained += count) { + for (count = 0; count < ICMP_SHARED_BATCH; ) { + fromlen = sizeof(sources[count]); + received = recvfrom(shared->fd, replies[count], ICMP_SHARED_SNAPLEN, MSG_DONTWAIT, + (struct sockaddr *) &sources[count], &fromlen); + + if (received < 0) { + if (errno == EINTR) { + continue; + } + readable = FALSE; + break; + } + + lengths[count++] = received; + } + + thread_mutex_lock(LOCK_ICMP); + for (slot = 0; slot < count; slot++) { + if (lengths[slot] > 0) { + icmp_shared_dispatch(waiter->family, replies[slot], lengths[slot], &sources[slot]); + } + } + thread_mutex_unlock(LOCK_ICMP); + } + + thread_mutex_lock(LOCK_ICMP); + shared->reading = FALSE; + } + + answered = waiter->answered; + + /* Leaving with the socket idle: let a sleeping waiter take over. */ + icmp_shared_handoff(shared); + thread_mutex_unlock(LOCK_ICMP); + + return answered; +} + /*! \fn int ping_icmp(host_t *host, ping_t *ping) * \brief ping a host using an ICMP packet * \param host a pointer to the current host structure @@ -291,9 +645,9 @@ int ping_icmp(const host_t *host, ping_t *ping) { int icmp_dgram; int rc = HOST_DOWN; int socket_errno = 0; - #if !(defined(__CYGWIN__) && !defined(SOLAR_PRIV)) - const int needs_seteuid = (set.availability.icmp_uses_caps != TRUE); - #endif + int icmp_use_shared = FALSE; + icmp_waiter_t waiter; + int waiting = FALSE; double begin_time, end_time, total_time; double host_timeout; @@ -326,43 +680,23 @@ int ping_icmp(const host_t *host, ping_t *ping) { * unchanged. A datagram reply arrives with the IPv4 header already * stripped, which the receive path has to account for. */ icmp_dgram = FALSE; - icmp_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP); + icmp_socket = icmp_open_socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP); if (icmp_socket != -1) { icmp_dgram = TRUE; + } else if (icmp_shared.fd != -1) { + icmp_socket = icmp_shared.fd; + icmp_use_shared = TRUE; } - /* Get a raw ICMP socket when the datagram path is unavailable. - * - * Each attempt is one complete lock/elevate/open/drop/unlock block. The - * retry used to sleep and loop back while still holding LOCK_SETEUID, so - * the second attempt relocked a non-recursive process-global mutex from - * the thread that already owned it. That deadlocks this thread at euid 0 - * in a SUID root binary and takes every other thread that needs the lock - * down with it, on nothing worse than a transient socket() failure. */ + /* Otherwise open a raw socket of our own, which needs CAP_NET_RAW or a + * real root user. Spine never regains root to do it: a setuid install + * without capabilities uses the shared socket above instead. */ retry_count = 0; while (icmp_socket == -1) { - #if !(defined(__CYGWIN__) && !defined(SOLAR_PRIV)) - if (needs_seteuid) { - thread_mutex_lock(LOCK_SETEUID); - if (seteuid(0) == -1) { - SPINE_LOG_DEBUG(("WARNING: Spine unable to obtain root privileges.")); - } - } - #endif - - icmp_socket = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP); + icmp_socket = icmp_open_socket(AF_INET, SOCK_RAW, IPPROTO_ICMP); socket_errno = errno; - #if !(defined(__CYGWIN__) && !defined(SOLAR_PRIV)) - if (needs_seteuid) { - if (seteuid(getuid()) == -1) { - SPINE_LOG_DEBUG(("WARNING: Spine unable to drop from root to local user.")); - } - thread_mutex_unlock(LOCK_SETEUID); - } - #endif - if (icmp_socket != -1) { break; } @@ -425,6 +759,17 @@ int ping_icmp(const host_t *host, ping_t *ping) { total_time = 0; begin_time = get_time_as_double(); + if (icmp_use_shared) { + memset(&waiter, 0, sizeof(waiter)); + waiter.shared = &icmp_shared; + waiter.family = AF_INET; + waiter.id = (uint16_t) (getpid() & 0xFFFF); + waiter.seq = icmp->icmp_seq; + waiter.peer = fromname.sin_addr; + icmp_shared_register(&waiter); + waiting = TRUE; + } + while (1) { if (retry_count > host->availability.retries) { snprintf(ping->ping_response, SMALL_BUFSIZE, "ICMP: Ping timed out"); @@ -439,6 +784,29 @@ int ping_icmp(const host_t *host, ping_t *ping) { SPINE_LOG_DEBUG(("Device[%i] DEBUG: Attempting to ping %s, seq %d (Retry %d of %d)", host->id, host->hostname, icmp->icmp_seq, retry_count, host->availability.retries)); } + if (waiting) { + /* Socket options would change every other thread's socket too. */ + double attempt_begin = get_time_as_double(); + + if (sendto(icmp_socket, packet, packet_len, 0, (struct sockaddr *) &fromname, sizeof(fromname)) >= 0 && + icmp_shared_await(&waiter, attempt_begin + host_timeout / one_thousand)) { + total_time = (get_time_as_double() - attempt_begin) * one_thousand; + + if (is_debug_device(host->id)) { + SPINE_LOG(("Device[%i] INFO: ICMP Device Alive, Try Count:%i, Time:%.4f ms", host->id, retry_count+1, (total_time))); + } else { + SPINE_LOG_MEDIUM(("Device[%i] INFO: ICMP Device Alive, Try Count:%i, Time:%.4f ms", host->id, retry_count+1, (total_time))); + } + snprintf(ping->ping_response, SMALL_BUFSIZE, "ICMP: Device is Alive"); + snprintf(ping->ping_status, 50, "%.5f", total_time); + rc = HOST_UP; + goto cleanup; + } + + retry_count++; + continue; + } + /* decrement the timeout value by the total time */ timeout.tv_sec = rint((host_timeout - total_time) / 1000); timeout.tv_usec = ((int) (host_timeout - total_time) % 1000) * 1000; @@ -585,14 +953,15 @@ cleanup: * socket and returned without freeing the packet, once per affected device * per cycle. See #593. * - * close() needs no privileges. Three of those exits re-entered root and - * took LOCK_SETEUID to call it, which widened the elevated window and - * serialised every poller thread on a global mutex for nothing. The raw - * socket needs root to open, which is done above, and the elevation is - * dropped there. */ + * The shared socket outlives this call; only a socket opened here is + * closed. */ SPINE_FREE(packet); - if (icmp_socket != -1) { + if (waiting) { + icmp_shared_unregister(&waiter); + } + + if (icmp_socket != -1 && !icmp_use_shared) { close(icmp_socket); } @@ -719,7 +1088,9 @@ static void apply_ipv6_scope_id(struct sockaddr_in6 *name) { static int ping_icmp_ipv6(const host_t *host, ping_t *ping) { int icmp_socket; int socket_errno = 0; - const int needs_seteuid = (set.availability.icmp_uses_caps != TRUE); + int icmp_use_shared = FALSE; + icmp_waiter_t waiter; + int waiting = FALSE; double begin_time, end_time, total_time; double host_timeout; @@ -730,7 +1101,7 @@ static int ping_icmp_ipv6(const host_t *host, ping_t *ping) { struct sockaddr_in6 fromname; char socket_reply[BUFSIZE]; int retry_count; - const char *cacti_msg = "cacti-monitoring-system"; + const char *cacti_msg = icmp6_payload; size_t msg_len; int packet_len; socklen_t fromlen; @@ -741,7 +1112,6 @@ static int ping_icmp_ipv6(const host_t *host, ping_t *ping) { static unsigned int seq = 0; struct icmp6_hdr *icmp6; - struct icmp6_hdr reply; unsigned char *packet; uint16_t our_id; uint16_t our_seq; @@ -764,31 +1134,20 @@ static int ping_icmp_ipv6(const host_t *host, ping_t *ping) { * rewrites the echo id on such a socket, which the reply match below * accounts for. */ icmp_dgram = FALSE; - icmp_socket = socket(AF_INET6, SOCK_DGRAM, IPPROTO_ICMPV6); + icmp_socket = icmp_open_socket(AF_INET6, SOCK_DGRAM, IPPROTO_ICMPV6); if (icmp_socket != -1) { icmp_dgram = TRUE; + } else if (icmp6_shared.fd != -1) { + icmp_socket = icmp6_shared.fd; + icmp_use_shared = TRUE; } - /* Each raw-socket attempt owns one complete privilege transition. */ + /* As in ping_icmp(), a socket of our own never involves regaining root. */ while (icmp_socket == -1) { - if (needs_seteuid) { - thread_mutex_lock(LOCK_SETEUID); - if (seteuid(0) == -1) { - SPINE_LOG_DEBUG(("WARNING: Spine unable to obtain root privileges.")); - } - } - - icmp_socket = socket(AF_INET6, SOCK_RAW, IPPROTO_ICMPV6); + icmp_socket = icmp_open_socket(AF_INET6, SOCK_RAW, IPPROTO_ICMPV6); socket_errno = errno; - if (needs_seteuid) { - if (seteuid(getuid()) == -1) { - SPINE_LOG_DEBUG(("WARNING: Spine unable to drop from root to local user.")); - } - thread_mutex_unlock(LOCK_SETEUID); - } - if (icmp_socket != -1) { break; } @@ -890,6 +1249,17 @@ static int ping_icmp_ipv6(const host_t *host, ping_t *ping) { total_time = 0; begin_time = get_time_as_double(); + if (icmp_use_shared) { + memset(&waiter, 0, sizeof(waiter)); + waiter.shared = &icmp6_shared; + waiter.family = AF_INET6; + waiter.id = htons(our_id); + waiter.seq = htons(our_seq); + waiter.peer6 = fromname.sin6_addr; + icmp_shared_register(&waiter); + waiting = TRUE; + } + while (1) { if (retry_count > host->availability.retries) { snprintf(ping->ping_response, SMALL_BUFSIZE, "ICMPv6: Ping timed out"); @@ -903,6 +1273,31 @@ static int ping_icmp_ipv6(const host_t *host, ping_t *ping) { SPINE_LOG_DEBUG(("DEBUG: Device[%i] Attempting to ping %s, seq %d (Retry %d of %d)", host->id, host->hostname, our_seq, retry_count, host->availability.retries)); } + if (waiting) { + /* Socket options would change every other thread's socket too. */ + double attempt_begin = get_time_as_double(); + + if (sendto(icmp_socket, packet, packet_len, 0, (struct sockaddr *) &fromname, sizeof(fromname)) >= 0 && + icmp_shared_await(&waiter, attempt_begin + host_timeout / one_thousand)) { + total_time = (get_time_as_double() - attempt_begin) * one_thousand; + + if (is_debug_device(host->id)) { + SPINE_LOG(("Device[%i] INFO: ICMPv6 Device Alive, Try Count:%i, Time:%.4f ms", host->id, retry_count+1, (total_time))); + } else { + SPINE_LOG_MEDIUM(("Device[%i] INFO: ICMPv6 Device Alive, Try Count:%i, Time:%.4f ms", host->id, retry_count+1, (total_time))); + } + + snprintf(ping->ping_response, SMALL_BUFSIZE, "ICMPv6: Device is Alive"); + snprintf(ping->ping_status, 50, "%.5f", total_time); + + result = HOST_UP; + goto cleanup; + } + + retry_count++; + continue; + } + /* decrement the timeout value by the total time */ timeout.tv_sec = rint((host_timeout - total_time) / 1000); timeout.tv_usec = ((int) (host_timeout - total_time) % 1000) * 1000; @@ -956,34 +1351,17 @@ static int ping_icmp_ipv6(const host_t *host, ping_t *ping) { goto keep_listening_ipv6; } } else { - /* a raw ICMPv6 socket delivers the ICMPv6 header without the - * IPv6 header, so anything shorter than our own probe cannot - * be the reply to it */ - if ((size_t) return_code < sizeof(struct icmp6_hdr) + msg_len) { - goto keep_listening_ipv6; - } - /* the kernel does not match the source address for us */ if (memcmp(fromname.sin6_addr.s6_addr, recvname.sin6_addr.s6_addr, sizeof(fromname.sin6_addr.s6_addr)) != 0) { /* another host responded */ goto keep_listening_ipv6; } - memcpy(&reply, socket_reply, sizeof(reply)); - - if ((reply.icmp6_type != ICMP6_ECHO_REPLY) || - (reply.icmp6_seq != htons(our_seq))) { - goto keep_listening_ipv6; - } - /* on a datagram socket the kernel assigns the echo id, so it * will not match what we wrote; sequence, source address and * payload still identify the reply */ - if (!icmp_dgram && reply.icmp6_id != htons(our_id)) { - goto keep_listening_ipv6; - } - - if (memcmp(socket_reply + sizeof(struct icmp6_hdr), cacti_msg, msg_len) != 0) { + if (!icmp6_reply_matches((const unsigned char *) socket_reply, return_code, + htons(our_id), htons(our_seq), !icmp_dgram)) { goto keep_listening_ipv6; } @@ -1017,7 +1395,11 @@ static int ping_icmp_ipv6(const host_t *host, ping_t *ping) { cleanup: SPINE_FREE(packet); - if (icmp_socket != -1) { + if (waiting) { + icmp_shared_unregister(&waiter); + } + + if (icmp_socket != -1 && !icmp_use_shared) { close(icmp_socket); } diff --git a/ping.h b/ping.h index 1e3a71c..144d2bb 100644 --- a/ping.h +++ b/ping.h @@ -164,6 +164,8 @@ extern spine_icmp_reply_t spine_icmp_classify_dgram_reply(const unsigned char *r extern int ping_host(host_t *host, ping_t *ping); extern int ping_snmp(host_t *host, ping_t *ping); extern int ping_icmp(const host_t *host, ping_t *ping); +extern int ping_icmp_open_shared(void); +extern int ping_icmp_shared_available(void); extern int ping_udp(const host_t *host, ping_t *ping); extern int ping_tcp(const host_t *host, ping_t *ping); extern name_t *get_namebyhost(const char *hostname, name_t *name); diff --git a/poller.c b/poller.c index 23d994e..2e0d8e5 100644 --- a/poller.c +++ b/poller.c @@ -213,8 +213,11 @@ static void record_result_error(const poll_error_context_t *context, const host_ context->host_id, context->thread_id, item->local_data_id, host->snmp.profile.version, host->hostname, item->rrd_name, item->arg1, result)); } else { + char script[SMALL_BUFSIZE]; + + php_command_script(item->arg1, script, sizeof(script)); SPINE_LOG(("WARNING: Invalid Response, Device[%i] HT[%i] DS[%i] SCRIPT: %s, output: %s", - context->host_id, context->thread_id, item->local_data_id, item->arg1, result)); + context->host_id, context->thread_id, item->local_data_id, script, result)); } } @@ -568,12 +571,13 @@ static void log_reindex_assertion(const reindex_evaluation_t *evaluation, const SPINE_LOG(("Device[%i] HT[%i] DQ[%i] RECACHE ASSERT FAILED: '%s%s%s'", evaluation->host->id, evaluation->work->host_thread, reindex->data_query_id, reindex->assert_value, failed ? reindex->op : "=", value != NULL ? value : "(null)")); } -static void queue_reindex(const reindex_evaluation_t *evaluation, const reindex_t *reindex) { - if (evaluation->work->host_thread != 1) return; +/* FALSE only when the queue write failed; other partitions never queue. */ +static bool queue_reindex(const reindex_evaluation_t *evaluation, const reindex_t *reindex) { + if (evaluation->work->host_thread != 1) return TRUE; char query[LRG_BUFSIZE]; snprintf(query, sizeof(query), "REPLACE INTO poller_command (poller_id, time, action, command) VALUES (%i, NOW(), %i, '%i:%i')", set.poller.poller_id, POLLER_COMMAND_REINDEX, evaluation->host->id, reindex->data_query_id); - if (set.poller.poller_id > 1 && set.poller.mode == REMOTE_ONLINE) db_insert(evaluation->remote, REMOTE, query); - else db_insert(evaluation->local, LOCAL, query); + if (set.poller.poller_id > 1 && set.poller.mode == REMOTE_ONLINE) return db_insert(evaluation->remote, REMOTE, query); + return db_insert(evaluation->local, LOCAL, query); } static void update_reindex_value(const reindex_evaluation_t *evaluation, const reindex_t *reindex, const char *value) { @@ -602,8 +606,10 @@ static bool evaluate_reindex_assertion(const reindex_evaluation_t *evaluation, c bool failed = reindex_assertion_failed(reindex, value); bool unavailable = value == NULL || IS_UNDEFINED(value) || STRIMATCH(value, "No Such Instance"); if (failed || unavailable) log_reindex_assertion(evaluation, reindex, value, failed); - if (failed) queue_reindex(evaluation, reindex); - if (failed || STRMATCH(reindex->op, ">") || STRMATCH(reindex->op, "<")) update_reindex_value(evaluation, reindex, value); + /* Advancing the stored value without a queued reindex would lose the + * reindex for good: the next poll compares against the new value. */ + bool queued = !failed || queue_reindex(evaluation, reindex); + if (queued && (failed || STRMATCH(reindex->op, ">") || STRMATCH(reindex->op, "<"))) update_reindex_value(evaluation, reindex, value); /* A failed uptime assertion means the counters reset, so the sample is a spike. */ if (failed && (STRMATCH(reindex->op, "<") || STRMATCH(reindex->arg1, ".1.3.6.1.2.1.1.3.0") || @@ -685,6 +691,7 @@ static char *poll_reindex_snmp(host_t *host, reindex_t *reindex, int host_thread static char *poll_reindex_action(host_t *host, reindex_t *reindex, int host_thread, char *sysUptime, bool *unavailable) { char *poll_result = NULL; int php_process; + char script[SMALL_BUFSIZE]; switch(reindex->action) { case POLLER_ACTION_SNMP: poll_result = poll_reindex_snmp(host, reindex, host_thread, sysUptime, unavailable); @@ -699,7 +706,8 @@ static char *poll_reindex_action(host_t *host, reindex_t *reindex, int host_thre poll_result = trim(exec_poll(host, reindex->arg1, reindex->data_query_id, "DQ")); - SPINE_LOG_DEVICE(host->id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DQ[%i] RECACHE CMD: %s, output: %s", host->id, host_thread, reindex->data_query_id, reindex->arg1, poll_result)); + php_command_script(reindex->arg1, script, sizeof(script)); + SPINE_LOG_DEVICE(host->id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DQ[%i] RECACHE CMD: %s, output: %s", host->id, host_thread, reindex->data_query_id, script, poll_result)); break; case POLLER_ACTION_PHP_SCRIPT_SERVER: /* script (php script server) */ @@ -707,7 +715,8 @@ static char *poll_reindex_action(host_t *host, reindex_t *reindex, int host_thre poll_result = trim(php_cmd(reindex->arg1, php_process)); - SPINE_LOG_DEVICE(host->id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DQ[%i] RECACHE SERVER: %s, output: %s", host->id, host_thread, reindex->data_query_id, reindex->arg1, poll_result)); + php_command_script(reindex->arg1, script, sizeof(script)); + SPINE_LOG_DEVICE(host->id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DQ[%i] RECACHE SERVER: %s, output: %s", host->id, host_thread, reindex->data_query_id, script, poll_result)); break; case POLLER_ACTION_SNMP_COUNT: { /* snmp; count items */ @@ -737,7 +746,8 @@ static char *poll_reindex_action(host_t *host, reindex_t *reindex, int host_thre snprintf(poll_result, BUFSIZE, "%d", char_count(count_result, '\n')); SPINE_FREE(count_result); - SPINE_LOG_DEVICE(host->id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DQ[%i] RECACHE CMD COUNT: %s, output: %s", host->id, host_thread, reindex->data_query_id, reindex->arg1, poll_result)); + php_command_script(reindex->arg1, script, sizeof(script)); + SPINE_LOG_DEVICE(host->id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DQ[%i] RECACHE CMD COUNT: %s, output: %s", host->id, host_thread, reindex->data_query_id, script, poll_result)); break; } @@ -753,7 +763,8 @@ static char *poll_reindex_action(host_t *host, reindex_t *reindex, int host_thre spine_snprintf(poll_result, BUFSIZE, "%d", char_count(count_result, '\n')); SPINE_FREE(count_result); - SPINE_LOG_DEVICE(host->id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DQ[%i] RECACHE SERVER COUNT: %s, output: %s", host->id, host_thread, reindex->data_query_id, reindex->arg1, poll_result)); + php_command_script(reindex->arg1, script, sizeof(script)); + SPINE_LOG_DEVICE(host->id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DQ[%i] RECACHE SERVER COUNT: %s, output: %s", host->id, host_thread, reindex->data_query_id, script, poll_result)); break; } @@ -781,9 +792,9 @@ static void load_reindex_item(reindex_t *reindex, MYSQL_ROW row) { if (row[4] != NULL) snprintf(reindex->arg1, sizeof(reindex->arg1), "%s", row[4]); } -static void poll_host_reindex(host_t *host, reindex_t *reindex, const char *query, +static bool poll_host_reindex(host_t *host, reindex_t *reindex, const char *query, const reindex_evaluation_t *evaluation) { - if (host->ignore_host || evaluation->work->host_id == 0) return; + if (host->ignore_host || evaluation->work->host_id == 0) return TRUE; int thread = evaluation->work->host_thread; MYSQL_RES *result = db_query(evaluation->local, LOCAL, query); if (result == NULL) { @@ -819,6 +830,7 @@ static void poll_host_reindex(host_t *host, reindex_t *reindex, const char *quer snmp_host_cleanup(host->snmp.session); host->snmp.session = NULL; } + return result != NULL; } static void load_poll_item(target_t *item, MYSQL_ROW row) { @@ -911,10 +923,12 @@ static void poll_script_item(host_t *host, target_t *item, } SPINE_FREE(poll_result); double thread_end = get_time_as_double(); + char script[SMALL_BUFSIZE]; + php_command_script(item->arg1, script, sizeof(script)); if (script_server) { - SPINE_LOG_DEVICE(errors->host_id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DS[%i] TT[%.2f] SS[%i] SERVER: %s, output: %s", errors->host_id, errors->thread_id, item->local_data_id, (float) ((thread_end - thread_start) * 1000), php_process, item->arg1, item->result)); + SPINE_LOG_DEVICE(errors->host_id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DS[%i] TT[%.2f] SS[%i] SERVER: %s, output: %s", errors->host_id, errors->thread_id, item->local_data_id, (float) ((thread_end - thread_start) * 1000), php_process, script, item->result)); } else { - SPINE_LOG_DEVICE(errors->host_id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DS[%i] TT[%.2f] SCRIPT: %s, output: %s", errors->host_id, errors->thread_id, item->local_data_id, (float) ((thread_end - thread_start) * 1000), item->arg1, item->result)); + SPINE_LOG_DEVICE(errors->host_id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] DS[%i] TT[%.2f] SCRIPT: %s, output: %s", errors->host_id, errors->thread_id, item->local_data_id, (float) ((thread_end - thread_start) * 1000), script, item->result)); } /* insert a NaN in place of the actual value if the snmp agent restarts */ if (!IS_UNDEFINED(item->result) && spike_kill && !strstr(item->result, ":")) SET_UNDEFINED(item->result); @@ -1197,6 +1211,7 @@ static int collect_poll_items(host_t *host, snmp_poll_batch_t *batch, int num_ro int i = 0; int rows_processed = 0; double thread_start = 0; + char script[SMALL_BUFSIZE]; while ((i < num_rows) && (!host->ignore_host)) { thread_start = get_time_as_double(); @@ -1211,7 +1226,8 @@ static int collect_poll_items(host_t *host, snmp_poll_batch_t *batch, int num_ro poll_script_item(host, &batch->items[i], batch->errors, thread_start, spike_kill, TRUE); break; default: /* unknown action, generate error */ - SPINE_LOG(("Device[%i] HT[%i] DS[%i] ERROR: Unknown Poller Action: %s", batch->errors->host_id, batch->errors->thread_id, batch->items[i].local_data_id, batch->items[i].arg1)); + php_command_script(batch->items[i].arg1, script, sizeof(script)); + SPINE_LOG(("Device[%i] HT[%i] DS[%i] ERROR: Unknown Poller Action: %s", batch->errors->host_id, batch->errors->thread_id, batch->items[i].local_data_id, script)); break; } @@ -1246,9 +1262,16 @@ static void release_poll_connections(const poller_thread_t *work, } } -/* FALSE requests the existing missing-device early return. Other database - * failures leave the host ignored and continue through the normal caller. */ -static bool load_poll_host(MYSQL *mysql, const poller_queries_t *queries, +typedef enum { + POLL_HOST_LOADED, + POLL_HOST_MISSING, + POLL_HOST_FAILED +} poll_host_load_t; + +/* A device deleted since selection returns quietly. A failed device query + * fails the device: treating it as an ignored host would still advance its + * schedule and drop the samples it never collected. */ +static poll_host_load_t load_poll_host(MYSQL *mysql, const poller_queries_t *queries, host_t *host, ping_t *ping, const poller_thread_t *work) { /* host_id=0 denotes a data source without a device. */ if (!work->host_id) { @@ -1256,22 +1279,22 @@ static bool load_poll_host(MYSQL *mysql, const poller_queries_t *queries, host->snmp.max_oids = 1; host->snmp.session = NULL; host->ignore_host = FALSE; - return TRUE; + return POLL_HOST_LOADED; } MYSQL_RES *result = db_query(mysql, LOCAL, queries->host); if (result == NULL) { - host->ignore_host = TRUE; - return TRUE; + SPINE_LOG(("Device[%i] HT[%i] ERROR: Unable to load the device", work->host_id, work->host_thread)); + return POLL_HOST_FAILED; } if (spine_count_to_int(mysql_num_rows(result)) != 1) { db_free_result(result); - return FALSE; + return POLL_HOST_MISSING; } MYSQL_ROW row = mysql_fetch_row(result); if (row == NULL) { SPINE_LOG(("Device[%i] HT[%i] ERROR: MySQL Returned a Null Device Result", host->id, work->host_thread)); host->ignore_host = TRUE; - return TRUE; + return POLL_HOST_LOADED; } load_host_metadata(mysql, row, host, work); db_free_result(result); @@ -1280,48 +1303,30 @@ static bool load_poll_host(MYSQL *mysql, const poller_queries_t *queries, if (work->host_thread == 1) { persist_host_status(mysql, host, include_system_information && host->ignore_host != TRUE); } - return TRUE; + return POLL_HOST_LOADED; } +/* Publish each partition only after its bookkeeping writes finish. The last + * partition records polling time and advances the schedule after every earlier + * write succeeded. Database waits never hold LOCK_THDET. */ static void complete_poll_host(MYSQL *mysql, poller_queries_t *queries, const poller_thread_t *work, const poll_error_context_t *error_context, - bool output_failed, double poll_start) { + bool output_failed, bool poll_failed, double poll_start) { extern poller_thread_t **details; int host_id = work->host_id; int host_thread = work->host_thread; double host_time_double = work->host_time_double; int errors = *error_context->errors; const char *error_string = error_context->buffer; - double poll_time; - /* record the polling time for the device */ - poll_time = get_time_as_double() - poll_start; + poller_thread_t *device = details[work->device_counter]; + double poll_time = get_time_as_double() - poll_start; + bool last_partition; + bool failed; + bool write_failed = FALSE; + SPINE_LOG_DEVICE(host_id, POLLER_VERBOSITY_MEDIUM, ("Device[%i] HT[%i] Total Time: %0.2g Seconds", host_id, host_thread, poll_time)); - /* record the total time for the host */ - thread_mutex_lock(LOCK_THDET); - poller_thread_t *device = details[work->device_counter]; - if (output_failed) { - device->output_failed = TRUE; - set.exit.exit_code = EXIT_FAILURE; - SPINE_LOG(("ERROR: Device[%i] HT[%i] output write failed; partial writes remain and due items stay eligible for recollection", host_id, host_thread)); - } - device->threads_complete++; - if (device->threads_complete == device->host_threads) { - /* Keep the due-item set stable until every device partition has finished. */ - if (set.poller.active_profiles != 1 && !device->output_failed) { - SPINE_LOG_MEDIUM(("Device[%i] HT[%i] Updating Poller Items for Next Poll", host_id, host_thread)); - db_query(mysql, LOCAL, queries->schedule); - } - device->complete = !device->output_failed; - - poll_time = get_time_as_double(); - queries->items[0] = '\0'; - snprintf(queries->items, BUFSIZE, "UPDATE host SET polling_time = %.3f - %.3f WHERE id = %i", poll_time, host_time_double, host_id); - db_query(mysql, LOCAL, queries->items); - - } - - if (errors > 0) { + if (!poll_failed && errors > 0) { int error_query_len = spine_count_to_int(strlen(error_string) + BUFSIZE); char *error_query = (char *)malloc(error_query_len); if (error_query == NULL) die("ERROR: Fatal malloc error: poller.c error_query!"); @@ -1333,12 +1338,52 @@ static void complete_poll_host(MYSQL *mysql, poller_queries_t *queries, " local_data_ids = CONCAT(local_data_ids, ', ', VALUES(local_data_ids))", host_id, set.poller.poller_id, errors, error_string); - db_query(mysql, LOCAL, error_query); + if (!db_insert(mysql, LOCAL, error_query)) write_failed = TRUE; free(error_query); } + + thread_mutex_lock(LOCK_THDET); + if (output_failed || write_failed) device->output_failed = TRUE; + if (poll_failed) device->poll_failed = TRUE; + failed = device->output_failed || device->poll_failed; + if (failed) { + device->complete = FALSE; + set.exit.exit_code = EXIT_FAILURE; + } + device->threads_complete++; + last_partition = device->threads_complete == device->host_threads; thread_mutex_unlock(LOCK_THDET); + + if (output_failed || poll_failed || write_failed) { + SPINE_LOG(("ERROR: Device[%i] HT[%i] polling or completion write failed; due items stay eligible for recollection", host_id, host_thread)); + } + if (!last_partition) return; + + if (!poll_failed) { + poll_time = get_time_as_double(); + snprintf(queries->items, BUFSIZE, "UPDATE host SET polling_time = %.3f - %.3f WHERE id = %i", poll_time, host_time_double, host_id); + if (!db_insert(mysql, LOCAL, queries->items)) write_failed = TRUE; + + /* Advancing the due-item set is the final write, after every partition + * and the device polling-time write have reported their outcome. */ + if (!failed && !write_failed && set.poller.active_profiles != 1) { + SPINE_LOG_MEDIUM(("Device[%i] HT[%i] Updating Poller Items for Next Poll", host_id, host_thread)); + if (!db_insert(mysql, LOCAL, queries->schedule)) write_failed = TRUE; + } + } + + thread_mutex_lock(LOCK_THDET); + if (write_failed) { + device->output_failed = TRUE; + set.exit.exit_code = EXIT_FAILURE; + } + device->complete = !device->output_failed && !device->poll_failed; + thread_mutex_unlock(LOCK_THDET); + if (write_failed) { + SPINE_LOG(("ERROR: Device[%i] HT[%i] device completion write failed; the device is not counted as polled", host_id, host_thread)); + } } void poll_host(const poller_thread_t *work, int *host_errors) { @@ -1361,6 +1406,8 @@ void poll_host(const poller_thread_t *work, int *host_errors) { int spike_kill = FALSE; int rows_processed = 0; bool output_failed = FALSE; + bool poll_failed = FALSE; + poll_host_load_t loaded; double poll_time = get_time_as_double(); @@ -1401,15 +1448,27 @@ void poll_host(const poller_thread_t *work, int *host_errors) { MYSQL_RES *result; local_cnn = db_get_connection(LOCAL); - if (local_cnn == NULL) die("ERROR: No local database connection available for polling"); - mysql = &local_cnn->mysql; - - if (set.poller.poller_id > 1 && set.poller.mode == REMOTE_ONLINE) { + if (local_cnn != NULL && set.poller.poller_id > 1 && set.poller.mode == REMOTE_ONLINE) { remote_cnn = db_get_connection(REMOTE); - if (remote_cnn == NULL) die("ERROR: No remote database connection available for polling"); - mysqlr = &remote_cnn->mysql; + if (remote_cnn == NULL) { + db_release_connection(LOCAL, local_cnn->id); + local_cnn = NULL; + } } + if (local_cnn == NULL) { + SPINE_LOG(("ERROR: Device[%i] HT[%i] No database connection available for polling", host_id, host_thread)); + complete_poll_host(NULL, NULL, work, &error_context, FALSE, TRUE, poll_time); + SPINE_FREE(error_string); + SPINE_FREE(buf_size); + SPINE_FREE(buf_errors); + mysql_thread_end(); + return; + } + + mysql = &local_cnn->mysql; + if (remote_cnn != NULL) mysqlr = &remote_cnn->mysql; + /* allocate host and ping structures with appropriate values */ if (!(host = (host_t *) malloc(sizeof(host_t)))) { die("ERROR: Fatal malloc error: poller.c host struct!"); @@ -1438,7 +1497,11 @@ void poll_host(const poller_thread_t *work, int *host_errors) { snprintf(ping->snmp_status, 50, "down"); snprintf(ping->snmp_response, SMALL_BUFSIZE, "SNMP not performed due to setting or ping result"); - if (!load_poll_host(mysql, &queries, host, ping, work)) { + loaded = load_poll_host(mysql, &queries, host, ping, work); + if (loaded == POLL_HOST_FAILED) { + complete_poll_host(mysql, &queries, work, &error_context, FALSE, TRUE, poll_time); + } + if (loaded != POLL_HOST_LOADED) { release_poll_connections(work, local_cnn, remote_cnn); SPINE_FREE(host); SPINE_FREE(reindex); @@ -1471,10 +1534,15 @@ void poll_host(const poller_thread_t *work, int *host_errors) { } const reindex_evaluation_t evaluation = {mysql, mysqlr, host, work, &errors, &spike_kill}; - poll_host_reindex(host, reindex, queries.reindex, &evaluation); + poll_failed = !poll_host_reindex(host, reindex, queries.reindex, &evaluation); /* calculate the number of poller items to poll this cycle */ - result = select_poll_items(mysql, &queries, host, work, &num_rows); + num_rows = 0; + result = NULL; + if (!poll_failed) { + result = select_poll_items(mysql, &queries, host, work, &num_rows); + poll_failed = result == NULL; + } if (num_rows > 0) { poll_item_storage_t storage = load_poll_items(result, host, num_rows); @@ -1514,7 +1582,7 @@ void poll_host(const poller_thread_t *work, int *host_errors) { SPINE_FREE(reindex); SPINE_FREE(ping); - complete_poll_host(mysql, &queries, work, &error_context, output_failed, poll_time); + complete_poll_host(mysql, &queries, work, &error_context, output_failed, poll_failed, poll_time); release_poll_connections(work, local_cnn, remote_cnn); @@ -1708,7 +1776,8 @@ static bool read_script_result(const script_result_context_t *context, int fd, d SPINE_LOG_MEDIUM(("Device[%i] ERROR: The NIFTY POPEN timed out", context->host->id)); int pid = nft_pchild(fd); if (pid > 1) { - kill(pid, SIGKILL); + /* nft_popen() made the script a group leader; take its descendants too. */ + kill(-pid, SIGKILL); } else { SPINE_LOG(("Device[%i] ERROR: Unable to find the timed-out POPEN child", context->host->id)); } @@ -1720,10 +1789,13 @@ static bool read_script_result(const script_result_context_t *context, int fd, d if (bytes > 0 && bytes < RESULTS_BUFFER) { result[bytes] = '\0'; } else { + char script[SMALL_BUFSIZE]; + + php_command_script(context->command, script, sizeof(script)); if (STRIMATCH(context->type, "DS")) { - SPINE_LOG(("Device[%i] DS[%i] ERROR: Empty result [%s]: '%s'", context->host->id, context->id, context->host->hostname, context->command)); + SPINE_LOG(("Device[%i] DS[%i] ERROR: Empty result [%s]: '%s'", context->host->id, context->id, context->host->hostname, script)); } else { - SPINE_LOG(("Device[%i] DQ[%i] ERROR: Empty result [%s]: '%s'", context->host->id, context->id, context->host->hostname, context->command)); + SPINE_LOG(("Device[%i] DQ[%i] ERROR: Empty result [%s]: '%s'", context->host->id, context->id, context->host->hostname, script)); } SET_UNDEFINED(result); } @@ -1803,7 +1875,8 @@ char *exec_poll(host_t *current_host, char *command, int id, const char *type) { strtok_r(executable, " ", &saveptr); } - SPINE_LOG_DEBUG(("The executable is '%s' in \'%s\'", executable, proc_command)); + /* executable is the first token; the arguments can carry credentials. */ + SPINE_LOG_DEBUG(("The executable is '%s'", executable)); if (access(executable, X_OK | F_OK) != -1) { cmd_fd = nft_popen(proc_command, "r"); @@ -1816,11 +1889,11 @@ char *exec_poll(host_t *current_host, char *command, int id, const char *type) { /* close pipe */ nft_pclose(cmd_fd); } else { - SPINE_LOG(("Device[%i] ERROR: Problem executing POPEN [%s]: '%s'", current_host->id, current_host->hostname, command)); + SPINE_LOG(("Device[%i] ERROR: Problem executing POPEN [%s]: '%s'", current_host->id, current_host->hostname, executable)); SET_UNDEFINED(result_string); } } else { - SPINE_LOG(("Device[%i] ERROR: Problem executing POPEN. File '%s' does not exist or is not executable.", current_host->id, command)); + SPINE_LOG(("Device[%i] ERROR: Problem executing POPEN. File '%s' does not exist or is not executable.", current_host->id, executable)); SET_UNDEFINED(result_string); } diff --git a/spine.c b/spine.c index 3d88adf..729ae1a 100644 --- a/spine.c +++ b/spine.c @@ -124,54 +124,6 @@ poller_thread_t** details = NULL; static char *getarg(char *opt, char ***pargv); static void display_help(int only_version); -#ifdef HAVE_LCAP -/* This patch is adapted (copied) patch for ntpd from Jarno Huuskonen and - * Pekka Savola that was adapted (copied) from a patch by Chris Wings to drop - * root for xntpd. - */ -void drop_root(uid_t server_uid, gid_t server_gid) { - cap_t caps; - if (prctl(PR_SET_KEEPCAPS, 1)) { - SPINE_LOG_HIGH(("prctl(PR_SET_KEEPCAPS, 1) failed")); - exit(1); - } - - if (setgroups(0, NULL) == -1) { - SPINE_LOG_HIGH(("setgroups failed.")); - exit(1); - } - - if (setegid(server_gid) == -1 || seteuid(server_uid) == -1) { - SPINE_LOG_HIGH(("setegid/seteuid to uid=%d/gid=%d failed.", server_uid, server_gid)); - exit(1); - } - - caps = cap_from_text("cap_net_raw=eip"); - if (caps == NULL) { - SPINE_LOG_HIGH(("cap_from_text failed.")); - exit(1); - } - - if (cap_set_proc(caps) == -1) { - SPINE_LOG_HIGH(("cap_set_proc failed.")); - exit(1); - } - - /* Try to free the memory from cap_from_text */ - cap_free( caps ); - - if ( setregid(server_gid, server_gid) == -1 || - setreuid(server_uid, server_uid) == -1 ) { - SPINE_LOG_HIGH(("setregid/setreuid to uid=%d/gid=%d failed.", - server_uid, server_gid)); - exit(1); - } - - SPINE_LOG_LOW(("running as uid(%d)/gid(%d) euid(%d)/egid(%d) with cap_net_raw=eip.", - getuid(), getgid(), geteuid(), getegid())); -} -#endif /* HAVE_LCAP */ - /*! \fn main(int argc, char *argv[]) * \brief The Spine program entry point * \param argc The number of arguments passed to the function plus one (+1) @@ -699,6 +651,9 @@ static void report_worker_completion(int num_rows) { if (det->output_failed) { SPINE_LOG(("ERROR: Device[%i] output persistence failed; due items remain scheduled for retry", det->host_id)); } + if (det->poll_failed) { + SPINE_LOG(("ERROR: Device[%i] polling failed on a database error; due items remain scheduled for retry", det->host_id)); + } SPINE_LOG_HIGH(("INFO: Device[%i] Thread %scomplete and %d to %d sources", det->host_id, det->complete ? "":"in", @@ -760,6 +715,15 @@ static void launch_poll_workers(MYSQL *mysql, MYSQL_RES *result, int num_rows, while (canexit == FALSE && device_counter < num_rows) { if (change_host) { mysql_row = mysql_fetch_row(result); + /* num_rows comes from this result, so a short read is not expected; + * upstream guarded it and the dispatch refactor dropped the check. */ + if (mysql_row == NULL || mysql_row[0] == NULL || mysql_row[1] == NULL) { + SPINE_LOG(("ERROR: Device list ended after %i of %i devices", device_counter, num_rows)); + thread_mutex_lock(LOCK_THDET); + set.exit.exit_code = EXIT_FAILURE; + thread_mutex_unlock(LOCK_THDET); + break; + } host_id = atoi(mysql_row[0]); partition.threads = atoi(mysql_row[1]); current_thread = 1; @@ -793,6 +757,7 @@ static void launch_poll_workers(MYSQL *mysql, MYSQL_RES *result, int num_rows, poller_details->complete = FALSE; poller_details->threads_complete = 0; poller_details->output_failed = FALSE; + poller_details->poll_failed = FALSE; thread_mutex_lock(LOCK_THDET); details[device_counter] = poller_details; @@ -914,11 +879,12 @@ static double initialize_process_defaults(void) { static int initialize_main_database(MYSQL *mysql, MYSQL *mysqlr) { MYSQL_RES *result; int mode; + int has_output_regex; /* initialize mysql objects for threads */ mysql_library_init(0, NULL, NULL); /* connect for main loop */ - db_connect(LOCAL, mysql); + if (!db_connect(LOCAL, mysql)) die("FATAL: Unable to connect to the local database"); /* setup local connection pool for hosts */ db_pool_local = (pool_t *) calloc(set.poller.threads, sizeof(pool_t)); @@ -926,7 +892,7 @@ static int initialize_main_database(MYSQL *mysql, MYSQL *mysqlr) { db_create_connection_pool(LOCAL); if (set.poller.poller_id > 1 && set.poller.mode == REMOTE_ONLINE) { - db_connect(REMOTE, mysqlr); + if (!db_connect(REMOTE, mysqlr)) die("FATAL: Unable to connect to the remote database"); mode = REMOTE; /* setup remote connection pool for hosts */ @@ -940,20 +906,23 @@ static int initialize_main_database(MYSQL *mysql, MYSQL *mysqlr) { /* check for device 0 items */ result = db_query(mysql, LOCAL, "SELECT * FROM (SELECT COUNT(*) AS items FROM poller_item WHERE host_id = 0 AND poller_id = 1) AS rs WHERE rs.items > 0"); + if (result == NULL) die("FATAL: Unable to check for Device 0 poller items"); if (mysql_num_rows(result)) { set.hosts.has_device_0 = TRUE; } db_free_result(result); /* check if poller_item has the output_regex column (added in Cacti 1.3.1) */ - if (db_column_exists(mysql, LOCAL, "poller_item", "output_regex")) { + has_output_regex = db_column_exists(mysql, LOCAL, "poller_item", "output_regex"); + if (has_output_regex < 0) die("FATAL: Unable to inspect the poller_item table"); + if (has_output_regex) { set.hosts.has_output_regex = TRUE; SPINE_LOG_DEBUG(("DEBUG: poller_item.output_regex column detected")); } - /* Since MySQL 5.7 the sql_mode defaults are too strict for cacti */ - db_insert(mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_ZERO_DATE', ''))"); - db_insert(mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY', ''))"); + /* Since MySQL 5.7 the sql_mode defaults are too strict for cacti. The + * same policy as the pool, which a reconnect of this handle reapplies. */ + if (!db_set_session_mode(mysql)) die("FATAL: Unable to configure the database session"); return mode; @@ -1071,11 +1040,8 @@ int main(int argc, char *argv[]) { start_time = get_time_as_double(); total_time = 0; - #ifdef HAVE_LCAP - if (geteuid() == 0) { - drop_root(getuid(), getgid()); - } - #endif /* HAVE_LCAP */ + /* Before any option, file or setting is read; see drop_privileges(). */ + drop_privileges(); pthread_t* threads = NULL; pthread_attr_t attr; @@ -1199,6 +1165,7 @@ int main(int argc, char *argv[]) { initialize_main_php(); result = select_poll_hosts(&mysql); + if (result == NULL) die("FATAL: Unable to select the devices to poll"); prepare_worker_storage(result, &num_rows, &threads, &ids, &host_time); diff --git a/spine_types.h b/spine_types.h index 3eadc35..48ff99e 100644 --- a/spine_types.h +++ b/spine_types.h @@ -174,7 +174,7 @@ /* threads constants */ #define LOCK_SNMP 0 -#define LOCK_SETEUID 2 +#define LOCK_ICMP 2 #define LOCK_GHBN 3 #define LOCK_POOL 4 #define LOCK_PHP 6 @@ -197,7 +197,7 @@ #define LOCK_HOST_TIME 41 #define LOCK_SNMP_O 0 -#define LOCK_SETEUID_O 2 +#define LOCK_ICMP_O 2 #define LOCK_GHBN_O 3 #define LOCK_POOL_O 4 #define LOCK_PHP_O 6 @@ -547,6 +547,7 @@ typedef struct poller_thread { int threads_complete; int complete; int output_failed; + int poll_failed; char host_time[40]; double host_time_double; spine_permits_t *thread_init_sem; diff --git a/sql.c b/sql.c index a1037ce..0eb9032 100644 --- a/sql.c +++ b/sql.c @@ -46,13 +46,22 @@ * \return TRUE if successful, or FALSE if not. * */ -static void retry_disconnected_query(MYSQL *mysql, int type, int error, const char *function, int *error_count) { - if (errno == EINTR) { - spine_sleep_usec(50000); - return; +/* Returns FALSE when the statement should not be retried. A failed reconnect + * ends the attempt at once: db_connect() has already spent its own tries, and + * repeating it here only multiplies the wait while a worker holds its device. */ +static bool retry_disconnected_query(MYSQL *mysql, int type, int error, const char *function, int *error_count) { + /* Every attempt counts. A signal that interrupted the call gets a few + * in-place retries, but the client library does not reset errno, so a + * stale EINTR used to skip both the reconnect and the count forever. */ + if (++*error_count > 30) { + SPINE_LOG(("ERROR: Too many Reconnect Attempts in Function %s", function)); + return FALSE; } - db_reconnect(mysql, type, error, function); - if (++*error_count > 30) die("FATAL: Too many Reconnect Attempts!"); + if (errno == EINTR && *error_count <= 3) { + spine_sleep_usec(50000); + return TRUE; + } + return db_reconnect(mysql, type, error, function) >= 0; } int db_insert(MYSQL *mysql, int type, const char *query) { @@ -64,8 +73,9 @@ int db_insert(MYSQL *mysql, int type, const char *query) { while (mysql_query(mysql, query) != 0) { int error = mysql_errno(mysql); if (error == 2013 || error == 2006) { - retry_disconnected_query(mysql, type, error, "db_insert", &error_count); - continue; + if (retry_disconnected_query(mysql, type, error, "db_insert", &error_count)) continue; + SPINE_LOG(("ERROR: SQL Failed! Connection lost, SQL Fragment:'%s'", query_frag)); + return FALSE; } if (error == 1213 || error == 1205) { spine_sleep_usec(50000); @@ -81,6 +91,38 @@ int db_insert(MYSQL *mysql, int type, const char *query) { return TRUE; } +/*! \fn int db_set_session_mode(MYSQL *mysql) + * \brief relaxes the session sql_mode to what Cacti's schema needs. + * + * Uses mysql_query() directly so a failure here never recurses into the + * reconnect path. Every new session needs it, including one opened by a + * reconnect, since the server starts each session at its default mode. + * SQL_readonly does not skip it: it changes no data. + * + * \return TRUE when every statement succeeded + */ +int db_set_session_mode(MYSQL *mysql) { + static const char *const modes[] = { + "NO_ZERO_DATE", "NO_ZERO_IN_DATE", "ONLY_FULL_GROUP_BY", "NO_AUTO_VALUE_ON_ZERO", + "TRADITIONAL", "STRICT_ALL_TABLES", "STRICT_TRANS_TABLES" + }; + char query[BUFSIZE]; + size_t i; + + for (i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) { + snprintf(query, sizeof(query), "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'%s', ''))", modes[i]); + if (mysql_query(mysql, query) != 0) { + SPINE_LOG(("ERROR: Unable to set the session sql_mode: %s", mysql_error(mysql))); + return FALSE; + } + } + + return TRUE; +} + +/* Returns TRUE after a reconnect, FALSE when the session was still alive, and + * -1 when the server could not be reached. After -1 the handle is initialized + * but unconnected, so the next attempt closes and reconnects it again. */ int db_reconnect(MYSQL *mysql, int type, int error, const char *function) { unsigned long mysql_thread = 0; char query[100]; @@ -93,12 +135,8 @@ int db_reconnect(MYSQL *mysql, int type, int error, const char *function) { SPINE_LOG(("WARNING: Connection Broken in Function %s with Error %i. Reconnect via mysql_ping() successful.", function, error)); snprintf(query, 100, "KILL %lu;", mysql_thread); mysql_query(mysql, query); - mysql_query(mysql, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_ZERO_DATE', ''))"); - mysql_query(mysql, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_ZERO_IN_DATE', ''))"); - mysql_query(mysql, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY', ''))"); - mysql_query(mysql, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_AUTO_VALUE_ON_ZERO', ''))"); - mysql_query(mysql, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'TRADITIONAL', ''))"); - mysql_query(mysql, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'STRICT_ALL_TABLES', ''))"); + + if (!db_set_session_mode(mysql)) return -1; sleep(1); @@ -114,15 +152,14 @@ int db_reconnect(MYSQL *mysql, int type, int error, const char *function) { SPINE_LOG(("WARNING: Connection Broken in Function %s with Error %i. Attempting explicit reconnect.", function, error)); mysql_close(mysql); - db_connect(type, mysql); - if (mysql_thread_id(mysql) > 0) { + if (db_connect(type, mysql) && mysql_thread_id(mysql) > 0 && db_set_session_mode(mysql)) { SPINE_LOG(("WARNING: Explicit reconnect successful in Function %s.", function)); return TRUE; } SPINE_LOG(("WARNING: Connection Broken with Error %i. Reconnect failed.", error)); - return FALSE; + return -1; } /*! \fn MYSQL_RES *db_query(MYSQL *mysql, int type, const char *query) @@ -132,7 +169,9 @@ int db_reconnect(MYSQL *mysql, int type, int error, const char *function) { * * This function will execute the SQL statement specified in the query variable. * - * \return MYSQL_RES a MySQL result structure + * \return MYSQL_RES a MySQL result structure, or NULL when the statement + * failed or produced no result set; the caller decides whether a + * failure ends a device, a startup step or the process * */ MYSQL_RES *db_query(MYSQL *mysql, int type, const char *query) { @@ -143,20 +182,21 @@ MYSQL_RES *db_query(MYSQL *mysql, int type, const char *query) { while (mysql_query(mysql, query) != 0) { int error = mysql_errno(mysql); if (error == 2013 || error == 2006) { - retry_disconnected_query(mysql, type, error, "db_query", &error_count); - continue; + if (retry_disconnected_query(mysql, type, error, "db_query", &error_count)) continue; + SPINE_LOG(("ERROR: Database connection lost, SQL Fragment:'%s'", query_frag)); + return NULL; } if (error == 1213 || error == 1205) { spine_sleep_usec(50000); if (++error_count > 30) { - SPINE_LOG(("FATAL: Too many Lock/Deadlock errors occurred!, SQL Fragment:'%s'", query_frag)); - exit(1); + SPINE_LOG(("ERROR: Too many Lock/Deadlock errors occurred!, SQL Fragment:'%s'", query_frag)); + return NULL; } continue; } - SPINE_LOG(("FATAL: Database Error:'%i', Message:'%s'", error, mysql_error(mysql))); + SPINE_LOG(("ERROR: Database Error:'%i', Message:'%s'", error, mysql_error(mysql))); SPINE_LOG(("ERROR: The Query Was:'%s'", query)); - exit(1); + return NULL; } return mysql_store_result(mysql); } @@ -221,7 +261,10 @@ static void db_set_ssl_options(MYSQL *mysql, int type) { } #endif -void db_connect(int type, MYSQL *mysql) { +/* Reports failure instead of exiting: worker threads reconnect through here, + * and a lost database must fail a device rather than the poller. Startup + * callers turn FALSE into a process exit. */ +int db_connect(int type, MYSQL *mysql) { int tries; int attempts; int timeout; @@ -246,8 +289,8 @@ void db_connect(int type, MYSQL *mysql) { if (mysql_init(mysql) == NULL) { db_address_release(&address); - printf("FATAL: Database unable to allocate memory and therefore can not connect\n"); - exit(1); + printf("ERROR: Database unable to allocate memory and therefore can not connect\n"); + return FALSE; } db_set_option(mysql, MYSQL_OPT_READ_TIMEOUT, &rtimeout, "read timeout"); @@ -278,11 +321,9 @@ void db_connect(int type, MYSQL *mysql) { if (!connect_error) { error = mysql_errno(mysql); - if ((error == 2002 || error == 2003 || error == 2006 || error == 2013) && errno == EINTR) { - spine_sleep_usec(5000); - tries++; - success = FALSE; - } else if (error == 2002) { + /* A stale EINTR in errno used to refund the attempt here and made + * the loop unbounded; every failed connect now spends a try. */ + if (error == 2002) { printf("Database: Connection Failed: Attempt:'%d', Error:'%u', Message:'%s'\n", attempts, mysql_errno(mysql), mysql_error(mysql)); sleep(1); success = FALSE; @@ -307,13 +348,15 @@ void db_connect(int type, MYSQL *mysql) { if (!success){ - printf("FATAL: Connection Failed, Error:'%i', Message:'%s'\n", error, mysql_error(mysql)); - exit(1); + printf("ERROR: Connection Failed, Error:'%i', Message:'%s'\n", error, mysql_error(mysql)); + return FALSE; } SPINE_LOG_DEBUG(("DEBUG: Total Connections made %i", connections)); connections++; + + return TRUE; } /*! \fn void db_disconnect(MYSQL *mysql) @@ -342,15 +385,9 @@ void db_create_connection_pool(int type) { for(id = 0; id < set.poller.threads; id++) { SPINE_LOG_DEBUG(("DEBUG: Creating Local Connection %i.", id)); - db_connect(type, &db_pool_local[id].mysql); + if (!db_connect(type, &db_pool_local[id].mysql)) die("FATAL: Unable to create the local connection pool"); - db_insert(&db_pool_local[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_ZERO_DATE', ''))"); - db_insert(&db_pool_local[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_ZERO_IN_DATE', ''))"); - db_insert(&db_pool_local[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY', ''))"); - db_insert(&db_pool_local[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_AUTO_VALUE_ON_ZERO', ''))"); - db_insert(&db_pool_local[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'TRADITIONAL', ''))"); - db_insert(&db_pool_local[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'STRICT_ALL_TABLES', ''))"); - db_insert(&db_pool_local[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'STRICT_TRANS_TABLES', ''))"); + if (!db_set_session_mode(&db_pool_local[id].mysql)) die("FATAL: Unable to configure the local connection pool"); db_pool_local[id].free = TRUE; db_pool_local[id].id = id; @@ -361,15 +398,9 @@ void db_create_connection_pool(int type) { for(id = 0; id < set.poller.threads; id++) { SPINE_LOG_DEBUG(("DEBUG: Creating Remote Connection %i.", id)); - db_connect(type, &db_pool_remote[id].mysql); + if (!db_connect(type, &db_pool_remote[id].mysql)) die("FATAL: Unable to create the remote connection pool"); - db_insert(&db_pool_remote[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_ZERO_DATE', ''))"); - db_insert(&db_pool_remote[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_ZERO_IN_DATE', ''))"); - db_insert(&db_pool_remote[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY', ''))"); - db_insert(&db_pool_remote[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_AUTO_VALUE_ON_ZERO', ''))"); - db_insert(&db_pool_remote[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'TRADITIONAL', ''))"); - db_insert(&db_pool_remote[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'STRICT_ALL_TABLES', ''))"); - db_insert(&db_pool_remote[id].mysql, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'STRICT_TRANS_TABLES', ''))"); + if (!db_set_session_mode(&db_pool_remote[id].mysql)) die("FATAL: Unable to configure the remote connection pool"); db_pool_remote[id].free = TRUE; db_pool_remote[id].id = id; @@ -530,6 +561,7 @@ void db_free_result(MYSQL_RES *result) { mysql_free_result(result); } +/* TRUE or FALSE, or -1 when the probe itself failed. */ int db_column_exists(MYSQL *mysql, int type, const char *table, const char *column) { char query_frag[BUFSIZE]; MYSQL_RES *result; @@ -543,6 +575,7 @@ int db_column_exists(MYSQL *mysql, int type, const char *table, const char *colu SPINE_LOG_DEVDBG(("DEVDBG: db_column_exists('%s','%s'): %s", table, column, query_frag)); result = db_query(mysql, type, query_frag); + if (result == NULL) return -1; if (mysql_num_rows(result)) { exists = TRUE; } else { diff --git a/sql.h b/sql.h index 787a219..06d8fda 100644 --- a/sql.h +++ b/sql.h @@ -43,7 +43,7 @@ extern void db_address_release(db_address_t *address); extern int db_insert(MYSQL *mysql, int type, const char *query); extern MYSQL_RES *db_query(MYSQL *mysql, int type, const char *query); -extern void db_connect(int type, MYSQL *mysql); +extern int db_connect(int type, MYSQL *mysql); extern void db_disconnect(MYSQL *mysql); extern void db_escape(MYSQL *mysql, char *output, int max_size, const char *input); extern void db_free_result(MYSQL_RES *result); @@ -51,6 +51,7 @@ extern void db_create_connection_pool(int type); extern void db_close_connection_pool(int type); extern pool_t *db_get_connection(int type); extern void db_release_connection(int type, int id); +extern int db_set_session_mode(MYSQL *mysql); extern int db_reconnect(MYSQL *mysql, int type, int error, const char *location); extern int db_column_exists(MYSQL *mysql, int type, const char *table, const char *column); diff --git a/src/core/locks.c b/src/core/locks.c index da83317..d305f3d 100644 --- a/src/core/locks.c +++ b/src/core/locks.c @@ -50,7 +50,7 @@ } DEFINE_SPINE_LOCK(snmp) -DEFINE_SPINE_LOCK(seteuid) +DEFINE_SPINE_LOCK(icmp) DEFINE_SPINE_LOCK(ghbn) DEFINE_SPINE_LOCK(pool) DEFINE_SPINE_LOCK(php) @@ -74,7 +74,7 @@ DEFINE_SPINE_LOCK(host_time) void init_mutexes() { pthread_once(get_attr(LOCK_SNMP_O), init_snmp_lock); - pthread_once(get_attr(LOCK_SETEUID_O), init_seteuid_lock); + pthread_once(get_attr(LOCK_ICMP_O), init_icmp_lock); pthread_once(get_attr(LOCK_GHBN_O), init_ghbn_lock); pthread_once(get_attr(LOCK_POOL_O), init_pool_lock); pthread_once(get_attr(LOCK_PHP_O), init_php_lock); @@ -100,7 +100,7 @@ void init_mutexes() { const char *get_name(int lock) { switch (lock) { case LOCK_SNMP: return "snmp"; - case LOCK_SETEUID: return "seteuid"; + case LOCK_ICMP: return "icmp"; case LOCK_GHBN: return "ghbn"; case LOCK_POOL: return "pool"; case LOCK_PHP: return "php"; @@ -131,7 +131,7 @@ pthread_cond_t* get_cond(int lock) { switch (lock) { case LOCK_SNMP: ret_val = &snmp_cond; break; - case LOCK_SETEUID: ret_val = &seteuid_cond; break; + case LOCK_ICMP: ret_val = &icmp_cond; break; case LOCK_GHBN: ret_val = &ghbn_cond; break; case LOCK_POOL: ret_val = &pool_cond; break; case LOCK_PHP: ret_val = &php_cond; break; @@ -165,7 +165,7 @@ pthread_mutex_t* get_lock(int lock) { switch (lock) { case LOCK_SNMP: ret_val = &snmp_lock; break; - case LOCK_SETEUID: ret_val = &seteuid_lock; break; + case LOCK_ICMP: ret_val = &icmp_lock; break; case LOCK_GHBN: ret_val = &ghbn_lock; break; case LOCK_POOL: ret_val = &pool_lock; break; case LOCK_PHP: ret_val = &php_lock; break; @@ -199,7 +199,7 @@ pthread_once_t* get_attr(int locko) { switch (locko) { case LOCK_SNMP_O: ret_val = &snmp_lock_o; break; - case LOCK_SETEUID_O: ret_val = &seteuid_lock_o; break; + case LOCK_ICMP_O: ret_val = &icmp_lock_o; break; case LOCK_GHBN_O: ret_val = &ghbn_lock_o; break; case LOCK_POOL_O: ret_val = &pool_lock_o; break; case LOCK_PHP_O: ret_val = &php_lock_o; break; diff --git a/tests/regression/test_child_process_safety.sh b/tests/regression/test_child_process_safety.sh index 8a5fba6..7691d01 100755 --- a/tests/regression/test_child_process_safety.sh +++ b/tests/regression/test_child_process_safety.sh @@ -37,7 +37,8 @@ grep -q 'spine_open_pipe_cloexec(php2cacti_pdes)' php.c || grep -q 'waitpid(pid, pstat, WNOHANG)' nft_popen.c || fail "nft_popen.c must reap child processes with WNOHANG" -grep -q 'kill(cur->pid, SIGKILL)' nft_popen.c || +# The negative pid kills the script's process group, descendants included. +grep -q 'kill(-cur->pid, SIGKILL)' nft_popen.c || fail "nft_popen.c must escalate a timed-out reap to SIGKILL" if grep -q 'waitpid(cur->pid, &pstat, 0)' nft_popen.c; then diff --git a/tests/regression/test_setuid_privilege_drop.sh b/tests/regression/test_setuid_privilege_drop.sh new file mode 100755 index 0000000..299a727 --- /dev/null +++ b/tests/regression/test_setuid_privilege_drop.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# A setuid root spine must give up root before it reads any option or file. +# +# Spine used to keep euid 0 through option parsing, the config file and the +# database connection, so "-O path_cactilog:" or a Cacti_Log line in a +# "-C" file made it create a root-owned file wherever the caller asked, and +# the saved uid stayed 0 for the whole run. +# +# This installs a copy of the built binary setuid root, starts it as nobody +# with a config FIFO so it blocks while reading the config, reads its ids from +# /proc, then lets it run and checks that nothing appeared in a directory only +# root can write. -O path_cactilog takes effect once the settings are read, so +# that half needs SPINE_TEST_DB_HOST: a database named SPINE_TEST_DB_NAME +# (default spine_regressions) loaded with tests/fixtures, which the default +# cactiuser/cactiuser account can read. Without it spine stops at the +# connection, which still covers the ids and the config file's Cacti_Log. +# +# Needs root, Linux and setpriv; it skips anywhere else, which includes an +# ordinary `make check`. +set -eu + +top_builddir="${top_builddir:-.}" + +skip() { + echo "SKIP: $*" + exit 77 +} + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +failed=0 +check_failed() { + echo "FAIL: $*" >&2 + failed=1 +} + +[ "$(uname -s)" = "Linux" ] || skip "Linux only" +[ "$(id -u)" -eq 0 ] || skip "needs root to install a setuid copy" +command -v setpriv >/dev/null 2>&1 || skip "setpriv not installed" +[ -x "$top_builddir/spine" ] || fail "spine has not been built" + +work=$(mktemp -d) +pid="" +cleanup() { + if [ -n "$pid" ]; then + kill "$pid" 2>/dev/null || true + fi + rm -rf "$work" +} +trap cleanup EXIT INT TERM + +chmod 755 "$work" +cp "$top_builddir/spine" "$work/spine" +chown root:root "$work/spine" +chmod 4755 "$work/spine" +mkdir "$work/rootonly" +chmod 700 "$work/rootonly" +mkfifo "$work/spine.conf" +chmod 666 "$work/spine.conf" + +setpriv --reuid=65534 --regid=65534 --clear-groups \ + "$work/spine" -R -C "$work/spine.conf" -O "path_cactilog:$work/rootonly/option.log" \ + >"$work/out" 2>&1 & +pid=$! + +# Opening the FIFO blocks until a writer appears, so a sleeping process is +# past startup and inside read_spine_config(). +state="" +tries=0 +while [ "$tries" -lt 50 ]; do + state=$(awk '/^State:/ {print $2}' "/proc/$pid/status" 2>/dev/null || true) + [ "$state" = "S" ] && break + tries=$((tries + 1)) + sleep 0.1 +done +[ "$state" = "S" ] || fail "spine did not block on its config FIFO" +sleep 0.2 + +uids=$(awk '/^Uid:/ {print $2, $3, $4, $5}' "/proc/$pid/status") +gids=$(awk '/^Gid:/ {print $2, $3, $4, $5}' "/proc/$pid/status") +echo "while reading the config: Uid $uids, Gid $gids" +[ "$uids" = "65534 65534 65534 65534" ] || check_failed "real, effective, saved or fs uid is not the caller's: $uids" +[ "$gids" = "65534 65534 65534 65534" ] || check_failed "real, effective, saved or fs gid is not the caller's: $gids" + +if [ -n "${SPINE_TEST_DB_HOST:-}" ]; then + printf 'DB_Host %s\nDB_Database %s\nCacti_Log %s\n' "$SPINE_TEST_DB_HOST" \ + "${SPINE_TEST_DB_NAME:-spine_regressions}" "$work/rootonly/config.log" >"$work/spine.conf" +else + printf 'DB_Host 127.0.0.1\nDB_Port 1\nCacti_Log %s\n' "$work/rootonly/config.log" >"$work/spine.conf" +fi +wait "$pid" || true +pid="" +cat "$work/out" +if [ -n "$(ls -A "$work/rootonly")" ]; then + ls -l "$work/rootonly" >&2 + check_failed "a setuid spine created a file in a root-only directory" +fi + +[ "$failed" -eq 0 ] || exit 1 +echo "PASS: setuid spine drops root before reading options and config" diff --git a/tests/test_spine_contracts.c b/tests/test_spine_contracts.c index 32ecd65..cc13176 100644 --- a/tests/test_spine_contracts.c +++ b/tests/test_spine_contracts.c @@ -14,6 +14,7 @@ #include "common.h" #include "spine.h" #include +#include static void test_keyword_roundtrips(void) { typedef struct { const char *word; int value; } keyword_case_t; @@ -68,12 +69,12 @@ static void *try_busy_lock(void *argument) { } static void test_lock_contracts(void) { - static const int locks[] = {LOCK_SNMP, LOCK_SETEUID, LOCK_GHBN, LOCK_POOL, LOCK_PHP, + static const int locks[] = {LOCK_SNMP, LOCK_ICMP, LOCK_GHBN, LOCK_POOL, LOCK_PHP, LOCK_PHP_PROC_0, LOCK_PHP_PROC_1, LOCK_PHP_PROC_2, LOCK_PHP_PROC_3, LOCK_PHP_PROC_4, LOCK_PHP_PROC_5, LOCK_PHP_PROC_6, LOCK_PHP_PROC_7, LOCK_PHP_PROC_8, LOCK_PHP_PROC_9, LOCK_PHP_PROC_10, LOCK_PHP_PROC_11, LOCK_PHP_PROC_12, LOCK_PHP_PROC_13, LOCK_PHP_PROC_14, LOCK_THDET, LOCK_HOST_TIME}; - static const char *const names[] = {"snmp", "seteuid", "ghbn", "pool", "php", + static const char *const names[] = {"snmp", "icmp", "ghbn", "pool", "php", "php_proc_0", "php_proc_1", "php_proc_2", "php_proc_3", "php_proc_4", "php_proc_5", "php_proc_6", "php_proc_7", "php_proc_8", "php_proc_9", "php_proc_10", "php_proc_11", "php_proc_12", "php_proc_13", "php_proc_14", "thdet", "host_time"}; @@ -200,14 +201,13 @@ static void test_fatal_signal_contracts(void) { close(diagnostic[0]); assert(dup2(diagnostic[1], STDERR_FILENO) == STDERR_FILENO); close(diagnostic[1]); + /* The default action of several cases dumps core. */ + struct rlimit no_core = {0, 0}; + assert(setrlimit(RLIMIT_CORE, &no_core) == 0); assert(signal(cases[i].signal, SIG_DFL) != SIG_ERR); install_spine_signal_handler(); assert(raise(cases[i].signal) == 0); - assert(set.exit.exit_code == cases[i].signal); - struct sigaction restored; - assert(sigaction(cases[i].signal, NULL, &restored) == 0); - assert(restored.sa_handler == SIG_DFL); - uninstall_spine_signal_handler(); + /* A fatal signal must end the process once it is logged. */ _exit(0); } close(diagnostic[1]); @@ -223,8 +223,12 @@ static void test_fatal_signal_contracts(void) { close(diagnostic[0]); int status; assert(waitpid(child, &status, 0) == child); - assert(WIFEXITED(status)); - assert(WEXITSTATUS(status) == (cases[i].signal == SIGSEGV ? 1 : 0)); + fprintf(stderr, "fatal signal %d: raw_status=%d\n", cases[i].signal, status); + if (cases[i].signal == SIGSEGV) { + assert(WIFEXITED(status) && WEXITSTATUS(status) == 1); + } else { + assert(WIFSIGNALED(status) && WTERMSIG(status) == cases[i].signal); + } assert(strstr(message, cases[i].message) != NULL); } /* A broken pipe is an ordinary write failure: it must neither terminate @@ -258,6 +262,96 @@ void test_additional_contracts(void) { puts("production additional contracts passed"); } +/* The client library never clears errno, so an EINTR left over from an + * unrelated call must not stand in for an interrupted query. Run in a child + * so an unbounded retry shows up as the alarm instead of a hung suite. */ +static void assert_stale_interrupt_is_retried(void) { + fflush(NULL); + pid_t child = fork(); + assert(child >= 0); + if (child == 0) { + alarm(10); + MYSQL administrator; + MYSQL victim; + db_connect(LOCAL, &administrator); + db_connect(LOCAL, &victim); + char query[100]; + snprintf(query, sizeof(query), "KILL CONNECTION %lu", mysql_thread_id(&victim)); + if (mysql_query(&administrator, query) != 0) _exit(2); + errno = EINTR; + _exit(db_insert(&victim, LOCAL, "SET @spine_regression_interrupt=1") == TRUE ? 0 : 3); + } + int status; + assert(waitpid(child, &status, 0) == child); + fprintf(stderr, "stale EINTR retry: raw_status=%d\n", status); + assert(WIFEXITED(status) && WEXITSTATUS(status) == 0); +} + +/* Query wrappers report failure to the caller; only startup code may turn a + * database error into a process exit. Port 1 refuses the reconnect quickly + * and deterministically, which stands in for a server that is down. */ +static void assert_database_failures_return(void) { + fflush(NULL); + pid_t child = fork(); + assert(child >= 0); + if (child == 0) { + alarm(30); + MYSQL administrator; + MYSQL victim; + if (!db_connect(LOCAL, &administrator) || !db_connect(LOCAL, &victim)) _exit(2); + if (db_query(&administrator, LOCAL, "SELECT spine_regression_missing FROM spine_regression_missing") != NULL) _exit(3); + if (db_insert(&administrator, LOCAL, "UPDATE spine_regression_missing SET value=1") != FALSE) _exit(4); + char query[100]; + snprintf(query, sizeof(query), "KILL CONNECTION %lu", mysql_thread_id(&victim)); + if (mysql_query(&administrator, query) != 0) _exit(5); + unsigned int port = set.database.port; + set.database.port = 1; + double begin = spine_monotonic_time(); + if (db_query(&victim, LOCAL, "SELECT 1") != NULL) _exit(6); + if (db_insert(&victim, LOCAL, "SET @spine_regression_down=1") != FALSE) _exit(7); + if (db_reconnect(&victim, LOCAL, 2006, "regression_down_server") != -1) _exit(8); + if (spine_monotonic_time() - begin > 20) _exit(9); + /* The same handle recovers once the server is reachable again. */ + set.database.port = port; + MYSQL_RES *result = db_query(&victim, LOCAL, "SELECT 789"); + MYSQL_ROW row = result != NULL ? mysql_fetch_row(result) : NULL; + if (row == NULL || row[0] == NULL || strcmp(row[0], "789") != 0) _exit(10); + db_free_result(result); + _exit(0); + } + int status; + assert(waitpid(child, &status, 0) == child); + fprintf(stderr, "database failure propagation: raw_status=%d\n", status); + assert(WIFEXITED(status) && WEXITSTATUS(status) == 0); +} + +static bool session_mode_contains(MYSQL *mysql, const char *mode) { + MYSQL_RES *result = db_query(mysql, LOCAL, "SELECT @@SESSION.sql_mode"); + assert(result != NULL); + MYSQL_ROW row = mysql_fetch_row(result); + assert(row != NULL && row[0] != NULL); + bool found = strstr(row[0], mode) != NULL; + db_free_result(result); + return found; +} + +/* Pooled sessions run with Cacti's relaxed sql_mode. A reconnect opens a new + * session with the server default, so it must apply the same policy again. */ +static void assert_reconnect_restores_session_mode(MYSQL *mysql) { + MYSQL victim; + assert(db_connect(LOCAL, &victim)); + /* The fixture server's default mode is strict; otherwise this proves nothing. */ + assert(session_mode_contains(&victim, "STRICT_TRANS_TABLES")); + char query[100]; + snprintf(query, sizeof(query), "KILL CONNECTION %lu", mysql_thread_id(&victim)); + assert(mysql_query(mysql, query) == 0); + bool strict = session_mode_contains(&victim, "STRICT_TRANS_TABLES"); + bool zero_date = session_mode_contains(&victim, "NO_ZERO_DATE"); + fprintf(stderr, "reconnected session mode: strict=%d no_zero_date=%d\n", strict, zero_date); + assert(!strict && !zero_date); + db_disconnect(&victim); +} + void test_additional_database_contracts(MYSQL *mysql) { assert(db_column_exists(mysql, LOCAL, "host", "id") == TRUE); assert(db_column_exists(mysql, LOCAL, "host", "regression_missing_column") == FALSE); @@ -302,4 +396,7 @@ void test_additional_database_contracts(MYSQL *mysql) { } } db_disconnect(&victim); + assert_stale_interrupt_is_retried(); + assert_database_failures_return(); + assert_reconnect_restores_session_mode(mysql); } diff --git a/tests/test_spine_faults.c b/tests/test_spine_faults.c index 012e60c..38347cb 100644 --- a/tests/test_spine_faults.c +++ b/tests/test_spine_faults.c @@ -22,6 +22,15 @@ extern size_t __real_strftime(char *, size_t, const char *, const struct tm *); extern int __real_mysql_query(MYSQL *, const char *); extern int __real_mysql_ping(MYSQL *); extern unsigned int __real_mysql_errno(MYSQL *); +extern MYSQL *__real_mysql_real_connect(MYSQL *, const char *, const char *, const char *, + const char *, unsigned int, const char *, unsigned long); +extern unsigned long __real_mysql_thread_id(MYSQL *); +extern MYSQL_RES *__real_mysql_store_result(MYSQL *); +extern my_ulonglong __real_mysql_num_rows(MYSQL_RES *); +extern MYSQL_ROW __real_mysql_fetch_row(MYSQL_RES *); +extern void __real_mysql_free_result(MYSQL_RES *); +extern unsigned int __real_mysql_num_fields(MYSQL_RES *); +extern int __real_posix_spawnattr_setpgroup(posix_spawnattr_t *, pid_t); extern int __real_pipe(int [2]); extern int __real_socketpair(int, int, int, int [2]); extern int __real_posix_spawn(pid_t *, const char *, const posix_spawn_file_actions_t *, @@ -43,6 +52,34 @@ static int observed_query_error; static int query_fault_calls; static int query_ping_calls; static MYSQL *interrupted_connection; +/* A scripted server for database failure paths that make check reaches + * without a database. While fake_database is set every client call below + * answers from this script and nothing touches the network. */ +static bool fake_database; +static int fake_errors[64]; +static int fake_error_count; +static int fake_error_next; +static int fake_errno_value; +static int fake_connect_failures; +static int fake_fail_connect_number; +static int fake_connects; +static bool fake_session_mode_fails; +static bool fake_ping_reconnects; +static int setpgroup_failures; +static unsigned long fake_thread; +static my_ulonglong fake_rows; +/* Statements starting with fake_rows_prefix report fake_rows_matched rows. */ +static const char *fake_rows_prefix; +static my_ulonglong fake_rows_matched; +static my_ulonglong fake_rows_current; +/* A statement starting with fake_fail_prefix fails; fake_item_rows rows of + * fake_item_row are handed out to the next fetches. */ +static const char *fake_fail_prefix; +static int fake_item_rows; +static char *fake_item_row[21]; +static int fake_queries; +static int fake_result_storage; +#define FAKE_RESULT ((MYSQL_RES *)(void *)&fake_result_storage) static int interrupted_queries; static int interrupted_query_calls; static int pipe_failures; @@ -67,6 +104,8 @@ static int launch_fault_calls; static int launch_checkpoints; static int launch_device_counter; static spine_permits_t *launch_startup; +static int completion_writes; +static int completion_writes_locked; int __wrap_pthread_create(pthread_t *thread, const pthread_attr_t *attributes, void *(*start)(void *), void *arg) { @@ -106,7 +145,9 @@ static void inspect_worker_launch_completion(void) { static void verify_worker_launch_exit(void) { fflush(NULL); - fprintf(stderr, "worker launch checkpoint: checkpoints=%i calls=%i faults=%i pending=%i exit=%i\n", launch_checkpoints, launch_calls, launch_fault_calls, launch_failures, set.exit.exit_code); + fprintf(stderr, "worker launch checkpoint: checkpoints=%i calls=%i faults=%i pending=%i exit=%i completion_writes=%i locked=%i\n", launch_checkpoints, launch_calls, launch_fault_calls, launch_failures, set.exit.exit_code, completion_writes, completion_writes_locked); + assert(completion_writes_locked == 0); + assert(launch_error == EPERM || completion_writes > 0); assert(launch_checkpoints == 1 && launch_failures == 0); assert(launch_fault_calls == launch_expected_faults); assert(launch_calls == (launch_error == EAGAIN ? 2 : 1)); @@ -144,7 +185,22 @@ size_t __wrap_strftime(char *output, size_t capacity, const char *format, const } int __wrap_mysql_query(MYSQL *mysql, const char *query) { + if (fake_database) { + fake_queries++; + fake_rows_current = fake_rows_prefix != NULL && strncmp(query, fake_rows_prefix, strlen(fake_rows_prefix)) == 0 ? + fake_rows_matched : fake_rows; + if (strncmp(query, "SET SESSION sql_mode", strlen("SET SESSION sql_mode")) == 0) { + fake_errno_value = fake_session_mode_fails ? 1146 : 0; + } else if (fake_fail_prefix != NULL && strncmp(query, fake_fail_prefix, strlen(fake_fail_prefix)) == 0) { + fake_errno_value = 1146; + } else { + fake_errno_value = fake_error_next < fake_error_count ? fake_errors[fake_error_next++] : 0; + } + return fake_errno_value != 0; + } static const char completion[] = "UPDATE poller_time SET end_time=NOW() WHERE poller_id="; + static const char polling_time[] = "UPDATE host SET polling_time"; + static const char host_errors[] = "INSERT INTO host_errors"; /* The interrupted handle is never connected; it must not reach the client library. */ if (mysql == interrupted_connection) { interrupted_query_calls++; @@ -158,6 +214,14 @@ int __wrap_mysql_query(MYSQL *mysql, const char *query) { if (launch_case_active && strncmp(query, completion, sizeof(completion) - 1) == 0) { inspect_worker_launch_completion(); } + /* Device bookkeeping writes can wait out a read timeout and a reconnect; + * holding LOCK_THDET across them would stall every other worker. */ + if (launch_case_active && (strncmp(query, polling_time, sizeof(polling_time) - 1) == 0 || + strncmp(query, host_errors, sizeof(host_errors) - 1) == 0)) { + completion_writes++; + if (thread_mutex_trylock(LOCK_THDET) == 0) thread_mutex_unlock(LOCK_THDET); + else completion_writes_locked++; + } if (query_fault_armed && mysql == query_fault_connection && strcmp(query, query_fault_statement) == 0) { query_fault_armed = 0; bool reconnect = FALSE; @@ -176,14 +240,77 @@ int __wrap_mysql_query(MYSQL *mysql, const char *query) { int __wrap_mysql_ping(MYSQL *mysql) { if (mysql == query_fault_connection) query_ping_calls++; + if (fake_database) { + /* A client-side auto-reconnect shows up as a new thread id. */ + if (!fake_ping_reconnects) return 1; + fake_thread++; + return 0; + } return __real_mysql_ping(mysql); } unsigned int __wrap_mysql_errno(MYSQL *mysql) { if (mysql == interrupted_connection) return 2013; + if (fake_database) return (unsigned int)fake_errno_value; return __real_mysql_errno(mysql); } +MYSQL *__wrap_mysql_real_connect(MYSQL *mysql, const char *host, const char *user, const char *password, + const char *database, unsigned int port, const char *socket, unsigned long flags) { + if (!fake_database) return __real_mysql_real_connect(mysql, host, user, password, database, port, socket, flags); + fake_connects++; + if (fake_connect_failures > 0 || fake_connects == fake_fail_connect_number) { + if (fake_connect_failures > 0) fake_connect_failures--; + fake_thread = 0; + /* 2005 is the unknown-host error db_connect() does not retry. */ + fake_errno_value = 2005; + return NULL; + } + fake_thread++; + fake_errno_value = 0; + return mysql; +} + +unsigned long __wrap_mysql_thread_id(MYSQL *mysql) { + if (fake_database) return fake_thread; + return __real_mysql_thread_id(mysql); +} + +MYSQL_RES *__wrap_mysql_store_result(MYSQL *mysql) { + if (fake_database) return fake_errno_value == 0 ? FAKE_RESULT : NULL; + return __real_mysql_store_result(mysql); +} + +my_ulonglong __wrap_mysql_num_rows(MYSQL_RES *result) { + if (result == FAKE_RESULT) return fake_rows_current; + return __real_mysql_num_rows(result); +} + +MYSQL_ROW __wrap_mysql_fetch_row(MYSQL_RES *result) { + if (result == FAKE_RESULT) { + if (fake_item_rows <= 0) return NULL; + fake_item_rows--; + return fake_item_row; + } + return __real_mysql_fetch_row(result); +} + +/* No scripted result matches a transfer plan's column count. */ +unsigned int __wrap_mysql_num_fields(MYSQL_RES *result) { + if (result == FAKE_RESULT) return 0; + return __real_mysql_num_fields(result); +} + +void __wrap_mysql_free_result(MYSQL_RES *result) { + if (result == FAKE_RESULT) return; + __real_mysql_free_result(result); +} + +int __wrap_posix_spawnattr_setpgroup(posix_spawnattr_t *attributes, pid_t group) { + if (setpgroup_failures > 0) { setpgroup_failures--; return EINVAL; } + return __real_posix_spawnattr_setpgroup(attributes, group); +} + int __wrap_pipe(int descriptors[2]) { if (pipe_failures > 0) { pipe_failures--; errno = EMFILE; return -1; } return __real_pipe(descriptors); @@ -213,6 +340,255 @@ char *__wrap_strdup(const char *text) { return __real_strdup(text); } +/* Identity and ICMP socket faults. A setuid root start cannot be reproduced + * by an unprivileged `make check`, so the ids the kernel would report, and the + * rules it applies to changing them, are modelled here for drop_privileges() + * and checkAsRoot(). Every case runs in a child so the process-wide state + * they set (the shared sockets, the setuid flag) cannot leak between cases. */ +extern uid_t __real_getuid(void); +extern uid_t __real_geteuid(void); +extern gid_t __real_getgid(void); +extern gid_t __real_getegid(void); +extern int __real_setuid(uid_t); +extern int __real_setgid(gid_t); +extern int __real_seteuid(uid_t); +extern int __real_setegid(gid_t); +extern int __real_socket(int, int, int); + +static bool ids_faked; +static uid_t fake_ruid, fake_euid, fake_suid; +static gid_t fake_rgid, fake_egid, fake_sgid; +static bool setuid_refused; +static bool setuid_keeps_saved; +static bool setgid_keeps_saved; +static bool icmp_faked; +static bool raw_icmp_allowed; +static bool dgram_icmp_allowed; + +uid_t __wrap_getuid(void) { return ids_faked ? fake_ruid : __real_getuid(); } +uid_t __wrap_geteuid(void) { return ids_faked ? fake_euid : __real_geteuid(); } +gid_t __wrap_getgid(void) { return ids_faked ? fake_rgid : __real_getgid(); } +gid_t __wrap_getegid(void) { return ids_faked ? fake_egid : __real_getegid(); } + +/* POSIX: with euid 0 setuid() sets all three ids; otherwise only the + * effective id may change, and only to the real or saved id. */ +int __wrap_setuid(uid_t uid) { + if (!ids_faked) return __real_setuid(uid); + if (setuid_refused) { errno = EAGAIN; return -1; } + if (fake_euid == 0) { + fake_ruid = fake_euid = uid; + if (!setuid_keeps_saved) fake_suid = uid; + return 0; + } + if (uid == fake_ruid || uid == fake_suid) { fake_euid = uid; return 0; } + errno = EPERM; + return -1; +} + +int __wrap_setgid(gid_t gid) { + if (!ids_faked) return __real_setgid(gid); + if (fake_euid == 0) { + fake_rgid = fake_egid = gid; + if (!setgid_keeps_saved) fake_sgid = gid; + return 0; + } + if (gid == fake_rgid || gid == fake_sgid) { fake_egid = gid; return 0; } + errno = EPERM; + return -1; +} + +int __wrap_seteuid(uid_t uid) { + if (!ids_faked) return __real_seteuid(uid); + if (fake_euid == 0 || uid == fake_ruid || uid == fake_suid) { fake_euid = uid; return 0; } + errno = EPERM; + return -1; +} + +int __wrap_setegid(gid_t gid) { + if (!ids_faked) return __real_setegid(gid); + if (fake_euid == 0 || gid == fake_rgid || gid == fake_sgid) { fake_egid = gid; return 0; } + errno = EPERM; + return -1; +} + +/* An allowed ICMP socket is stood in for by a local datagram socket. */ +int __wrap_socket(int domain, int type, int protocol) { + int base = type; + #ifdef SOCK_CLOEXEC + base &= ~SOCK_CLOEXEC; + #endif + if (!icmp_faked || (protocol != IPPROTO_ICMP && protocol != IPPROTO_ICMPV6)) { + return __real_socket(domain, type, protocol); + } + if ((base == SOCK_RAW && raw_icmp_allowed) || (base == SOCK_DGRAM && dgram_icmp_allowed)) { + return __real_socket(AF_UNIX, SOCK_DGRAM | (type & ~base), 0); + } + errno = EPERM; + return -1; +} + + +typedef void (*identity_case_t)(void); + +/* Runs one case in a child; returns its exit status with its output. */ +static int run_identity_case(identity_case_t body, char *output, size_t capacity) { + int descriptors[2]; + assert(__real_pipe(descriptors) == 0); + fflush(NULL); + pid_t child = fork(); + assert(child >= 0); + if (child == 0) { + assert(close(descriptors[0]) == 0); + assert(dup2(descriptors[1], STDOUT_FILENO) == STDOUT_FILENO); + assert(dup2(descriptors[1], STDERR_FILENO) == STDERR_FILENO); + assert(close(descriptors[1]) == 0); + set.exit.exit_code = EXIT_SUCCESS; + set.php.php_initialized = FALSE; + set.logging.log_level = POLLER_VERBOSITY_DEBUG; + set.logging.log_destination = LOGDEST_STDOUT; + body(); + fflush(NULL); + /* exit(), not _exit(), so the child's coverage counters are kept */ + exit(EXIT_SUCCESS); + } + assert(close(descriptors[1]) == 0); + size_t used = 0; + for (;;) { + ssize_t received = read(descriptors[0], output + used, capacity - used - 1); + if (received < 0 && errno == EINTR) continue; + assert(received >= 0); + if (received == 0) break; + used += (size_t) received; + assert(used < capacity - 1); + } + output[used] = '\0'; + assert(close(descriptors[0]) == 0); + int status; + while (waitpid(child, &status, 0) < 0) assert(errno == EINTR); + assert(WIFEXITED(status)); + return WEXITSTATUS(status); +} + +#ifndef HAVE_LCAP +static void fake_setuid_root_start(uid_t uid, gid_t gid) { + ids_faked = TRUE; + fake_ruid = uid; + fake_euid = fake_suid = 0; + fake_rgid = fake_egid = fake_sgid = gid; + icmp_faked = TRUE; +} + +static void setuid_start_with_raw_icmp(void) { + fake_setuid_root_start(1000, 1000); + raw_icmp_allowed = TRUE; + drop_privileges(); + assert(fake_ruid == 1000 && fake_euid == 1000 && fake_suid == 1000); + assert(fake_rgid == 1000 && fake_egid == 1000 && fake_sgid == 1000); + assert(ping_icmp_shared_available()); + /* nothing can open another raw socket now, and none is needed */ + raw_icmp_allowed = FALSE; + checkAsRoot(); + assert(set.availability.icmp_avail); +} + +static void setuid_start_without_icmp(void) { + fake_setuid_root_start(1000, 1000); + drop_privileges(); + assert(fake_euid == 1000 && fake_suid == 1000); + assert(!ping_icmp_shared_available()); + checkAsRoot(); + assert(!set.availability.icmp_avail); +} + +static void setuid_start_with_datagram_icmp(void) { + fake_setuid_root_start(1000, 1000); + drop_privileges(); + dgram_icmp_allowed = TRUE; + checkAsRoot(); + assert(set.availability.icmp_avail); +} + +static void setuid_start_setuid_refused(void) { + fake_setuid_root_start(1000, 1000); + raw_icmp_allowed = TRUE; + setuid_refused = TRUE; + drop_privileges(); +} + +static void setuid_start_saved_uid_kept(void) { + fake_setuid_root_start(1000, 1000); + setuid_keeps_saved = TRUE; + drop_privileges(); +} + +/* A setuid and setgid root install starts with an effective gid of 0. */ +static void setuid_start_saved_gid_kept(void) { + fake_setuid_root_start(1000, 1000); + fake_egid = fake_sgid = 0; + setgid_keeps_saved = TRUE; + drop_privileges(); +} + +static void real_root_start(void) { + ids_faked = TRUE; + icmp_faked = TRUE; + drop_privileges(); + assert(fake_ruid == 0 && fake_euid == 0 && fake_suid == 0); + checkAsRoot(); + assert(set.availability.icmp_avail); +} + +#endif + +static void unprivileged_start(void) { + ids_faked = TRUE; + fake_ruid = fake_euid = fake_suid = 1000; + fake_rgid = fake_egid = fake_sgid = 1000; + icmp_faked = TRUE; + drop_privileges(); + assert(!ping_icmp_shared_available()); + /* a file capability */ + raw_icmp_allowed = TRUE; + checkAsRoot(); + assert(set.availability.icmp_avail); + /* net.ipv4.ping_group_range */ + raw_icmp_allowed = FALSE; + dgram_icmp_allowed = TRUE; + checkAsRoot(); + assert(set.availability.icmp_avail); + /* neither */ + dgram_icmp_allowed = FALSE; + checkAsRoot(); + assert(!set.availability.icmp_avail); +} + +static void test_privilege_drop_faults(void) { + char output[8192]; + #ifndef HAVE_LCAP + /* With libcap the drop goes through prctl(), setgroups() and the cap_* + * calls, which these fakes do not model; regressions.yml runs it for real. */ + assert(run_identity_case(setuid_start_with_raw_icmp, output, sizeof(output)) == EXIT_SUCCESS); + assert(strstr(output, "raw ICMP sockets it opened before dropping root") != NULL); + assert(run_identity_case(setuid_start_without_icmp, output, sizeof(output)) == EXIT_SUCCESS); + assert(strstr(output, "WARNING: Spine is setuid root but could not get raw ICMP access") != NULL); + assert(run_identity_case(setuid_start_with_datagram_icmp, output, sizeof(output)) == EXIT_SUCCESS); + assert(strstr(output, "may use unprivileged ICMP sockets") != NULL); + assert(run_identity_case(setuid_start_setuid_refused, output, sizeof(output)) == EXIT_FAILURE); + assert(strstr(output, "could not drop to uid 1000/gid 1000; refusing to run") != NULL); + assert(run_identity_case(setuid_start_saved_uid_kept, output, sizeof(output)) == EXIT_FAILURE); + assert(strstr(output, "could not drop root permanently; refusing to run") != NULL); + assert(run_identity_case(setuid_start_saved_gid_kept, output, sizeof(output)) == EXIT_FAILURE); + assert(strstr(output, "could not drop root permanently; refusing to run") != NULL); + assert(run_identity_case(real_root_start, output, sizeof(output)) == EXIT_SUCCESS); + assert(strstr(output, "Spine is running as root") != NULL); + #endif + assert(run_identity_case(unprivileged_start, output, sizeof(output)) == EXIT_SUCCESS); + assert(strstr(output, "may open raw ICMP sockets") != NULL); + assert(strstr(output, "Spine has no ICMP access") != NULL); + assert(strstr(output, "WARNING: Spine is setuid root") == NULL); + puts("production privilege drop faults passed"); +} + static void test_logger_format_failure(void) { config_t previous = set; char filename[] = "spine-log-fault-XXXXXX"; @@ -308,6 +684,15 @@ static void test_child_setup_failures(unsigned int expected_descriptors) { } } +static void test_spawn_group_failure(void) { + unsigned int descriptors = open_descriptor_count(); + setpgroup_failures = 1; + assert(nft_popen("printf unused", "r") == -1 && errno == EINVAL); + assert(setpgroup_failures == 0 && open_descriptor_count() == descriptors); + assert_cancel_state(PTHREAD_CANCEL_ENABLE); + puts("production script process-group setup failure regressions passed"); +} + static void test_process_creation_failures(void) { unsigned int descriptors = open_descriptor_count(); int unrelated[2]; @@ -432,6 +817,332 @@ static void test_snmpv3_privacy_copy_failure(void) { puts("production SNMPv3 privacy copy failure regressions passed"); } +static void fake_script(const int *errors, int count) { + assert(count >= 0 && count <= (int)(sizeof(fake_errors) / sizeof(fake_errors[0]))); + for (int i = 0; i < count; i++) fake_errors[i] = errors[i]; + fake_error_count = count; + fake_error_next = 0; + fake_connect_failures = 0; + fake_fail_connect_number = 0; + fake_connects = 0; + fake_queries = 0; + fake_session_mode_fails = FALSE; + fake_ping_reconnects = FALSE; + fake_rows = 0; + fake_rows_prefix = NULL; + fake_rows_matched = 0; + fake_fail_prefix = NULL; + fake_item_rows = 0; + errno = 0; +} + +static void fake_repeat(int error, int count) { + int errors[64]; + assert(count <= (int)(sizeof(errors) / sizeof(errors[0]))); + for (int i = 0; i < count; i++) errors[i] = error; + fake_script(errors, count); +} + +/* Every wrapper outcome a worker can see: a statement error, a lost + * connection whose reconnect fails, succeeds, or cannot set the session + * mode, and both retry budgets running out. None of them may exit. */ +static void test_fake_database_wrappers(void) { + MYSQL handle; + fake_database = TRUE; + fake_thread = 0; + set.poller.SQL_readonly = FALSE; + fake_script(NULL, 0); + assert(db_connect(LOCAL, &handle) && fake_connects == 1); + + static const int statement_error[] = {1146}; + fake_script(statement_error, 1); + assert(db_query(&handle, LOCAL, "SELECT missing FROM missing") == NULL); + fake_script(statement_error, 1); + assert(db_column_exists(&handle, LOCAL, "missing", "missing") == -1); + + static const int lost[] = {2006}; + fake_script(lost, 1); + fake_connect_failures = 1; + assert(db_insert(&handle, LOCAL, "UPDATE host SET status=1") == FALSE); + assert(fake_connects == 1 && fake_connect_failures == 0); + fake_script(lost, 1); + fake_connect_failures = 1; + assert(db_query(&handle, LOCAL, "SELECT 1") == NULL); + assert(db_reconnect(&handle, LOCAL, 2006, "fake_down_server") == TRUE); + + /* The reconnect reapplies the session mode before the retry. */ + static const int recovered[] = {2013, 0}; + fake_script(recovered, 2); + assert(db_insert(&handle, LOCAL, "UPDATE host SET status=1") == TRUE); + assert(fake_connects == 1 && fake_queries == 9); + fake_script(recovered, 2); + fake_session_mode_fails = TRUE; + assert(db_insert(&handle, LOCAL, "UPDATE host SET status=1") == FALSE); + assert(fake_queries == 2); + fake_session_mode_fails = FALSE; + assert(db_set_session_mode(&handle) == TRUE); + fake_script(NULL, 0); + fake_ping_reconnects = TRUE; + fake_session_mode_fails = TRUE; + assert(db_reconnect(&handle, LOCAL, 2006, "fake_ping_reconnect") == -1); + assert(fake_connects == 0); + + fake_repeat(2013, 40); + assert(db_insert(&handle, LOCAL, "UPDATE host SET status=1") == FALSE); + assert(fake_error_next == 31 && fake_connects == 30); + fake_repeat(1213, 40); + assert(db_query(&handle, LOCAL, "SELECT 1") == NULL); + assert(fake_error_next == 31); + + fake_script(NULL, 0); + fake_rows = 1; + assert(db_column_exists(&handle, LOCAL, "host", "id") == TRUE); + db_disconnect(&handle); + fake_database = FALSE; + puts("production scripted database wrapper regressions passed"); +} + +/* Pool creation and the collector push run in the main thread; the pool must + * get the session mode and the push must skip cleanly when either side is down. */ +static void test_fake_database_main_paths(void) { + config_t previous = set; + pool_t *previous_local = db_pool_local; + pool_t *previous_remote = db_pool_remote; + fake_database = TRUE; + set.logging.log_destination = 0; + set.poller.threads = 2; + db_pool_local = calloc(2, sizeof(*db_pool_local)); + db_pool_remote = calloc(2, sizeof(*db_pool_remote)); + assert(db_pool_local != NULL && db_pool_remote != NULL); + fake_script(NULL, 0); + db_create_connection_pool(LOCAL); + db_create_connection_pool(REMOTE); + assert(fake_connects == 4 && fake_queries == 28); + assert(db_pool_local[1].free && db_pool_remote[1].free && db_pool_remote[1].id == 1); + db_close_connection_pool(LOCAL); + db_close_connection_pool(REMOTE); + /* The push connects locally first, then to the main server. */ + for (int failing = 1; failing <= 2; failing++) { + fake_script(NULL, 0); + fake_fail_connect_number = failing; + set.exit.exit_code = EXIT_SUCCESS; + poller_push_data_to_main(); + assert(fake_connects == failing && fake_queries == 0); + assert(set.exit.exit_code == EXIT_FAILURE); + } + fake_database = FALSE; + db_pool_local = previous_local; + db_pool_remote = previous_remote; + set = previous; + puts("production scripted database main-thread regressions passed"); +} + +static void *fake_poll_worker(void *argument) { + assert(mysql_thread_init() == 0); + int errors = 0; + poll_host(argument, &errors); + return NULL; +} + +static void run_fake_poll(poller_thread_t *device, int host_id) { + memset(device, 0, sizeof(*device)); + device->host_id = host_id; + device->host_thread = 1; + device->host_threads = 1; + device->host_time_double = get_time_as_double(); + STRNCOPY(device->host_time, "1791244800"); + set.exit.exit_code = EXIT_SUCCESS; + pthread_t worker; + assert(pthread_create(&worker, NULL, fake_poll_worker, device) == 0); + assert(pthread_join(worker, NULL) == 0); + assert(device->threads_complete == 1); +} + +static void assert_fake_poll_failed(const poller_thread_t *device) { + assert(device->poll_failed && !device->complete && set.exit.exit_code == EXIT_FAILURE); + assert(db_pool_local[0].free); +} + +/* A worker that cannot use the database fails its device and returns. */ +static void test_fake_database_poll_host(void) { + config_t previous = set; + pool_t *previous_local = db_pool_local; + pool_t *previous_remote = db_pool_remote; + poller_thread_t **previous_details = details; + poller_thread_t device; + poller_thread_t *device_list[1] = {&device}; + details = device_list; + fake_database = TRUE; + set.logging.log_destination = 0; + set.poller.threads = 1; + set.poller.poller_id = 1; + set.poller.active_profiles = 2; + set.poller.poller_interval = 5; + set.availability.ping_only = FALSE; + db_pool_local = calloc(1, sizeof(*db_pool_local)); + assert(db_pool_local != NULL); + fake_script(NULL, 0); + assert(db_connect(LOCAL, &db_pool_local[0].mysql)); + db_pool_local[0].free = FALSE; + run_fake_poll(&device, 0); + assert(device.poll_failed && !device.complete && set.exit.exit_code == EXIT_FAILURE); + db_pool_local[0].free = TRUE; + + set.poller.poller_id = 2; + set.poller.mode = REMOTE_ONLINE; + /* The production allocator scans one slot per configured worker. */ + pool_t busy_remote[2] = {0}; + db_pool_remote = busy_remote; + run_fake_poll(&device, 0); + assert_fake_poll_failed(&device); + set.poller.poller_id = 1; + db_pool_remote = previous_remote; + + static const int statement_error[] = {1146}; + fake_script(statement_error, 1); + run_fake_poll(&device, 902); + assert_fake_poll_failed(&device); + assert(fake_queries == 1); + fake_script(statement_error, 1); + run_fake_poll(&device, 0); + assert_fake_poll_failed(&device); + assert(fake_queries == 1); + + /* A device deleted since selection returns quietly, uncounted. */ + fake_script(NULL, 0); + memset(&device, 0, sizeof(device)); + device.host_id = 902; + device.host_thread = 1; + device.host_threads = 1; + STRNCOPY(device.host_time, "1791244800"); + pthread_t worker; + assert(pthread_create(&worker, NULL, fake_poll_worker, &device) == 0); + assert(pthread_join(worker, NULL) == 0); + assert(device.threads_complete == 0 && !device.poll_failed && db_pool_local[0].free); + /* A row count of one with no row ignores the device but completes it. */ + fake_script(NULL, 0); + fake_rows = 1; + run_fake_poll(&device, 902); + assert(!device.poll_failed && db_pool_local[0].free); + + /* An empty, successful poll advances the schedule and records the time. */ + fake_script(NULL, 0); + run_fake_poll(&device, 0); + assert(!device.poll_failed && device.complete && set.exit.exit_code == EXIT_SUCCESS); + assert(fake_queries == 3 && db_pool_local[0].free); + + /* A completion write that fails must leave the device incomplete and + * fail the run. One invalid script result makes a host_errors write. */ + static const char *const completion_writes[] = { + "UPDATE poller_item", "UPDATE host SET polling_time", "INSERT INTO host_errors" + }; + static char action[] = "1"; + static char command[] = "/usr/bin/printf invalid"; + static char local_data_id[] = "1"; + fake_item_row[0] = action; + fake_item_row[8] = command; + fake_item_row[11] = local_data_id; + assert(spine_permits_init(&available_scripts, 1) == 0); + set.php.script_timeout = 5; + /* Reach the per-item result lines, which must name only the script. */ + set.logging.spine_log_level = 2; + set.logging.log_level = POLLER_VERBOSITY_MEDIUM; + for (size_t failing = 0; failing < sizeof(completion_writes) / sizeof(completion_writes[0]); failing++) { + fake_script(NULL, 0); + fake_rows = 1; + fake_item_rows = 1; + fake_fail_prefix = completion_writes[failing]; + run_fake_poll(&device, 0); + fprintf(stderr, "completion write '%s' failing: complete=%d exit=%d queries=%d\n", + completion_writes[failing], device.complete, set.exit.exit_code, fake_queries); + assert(!device.complete && set.exit.exit_code == EXIT_FAILURE && db_pool_local[0].free); + } + assert(spine_permits_destroy(&available_scripts) == 0); + + db_disconnect(&db_pool_local[0].mysql); + free(db_pool_local); + fake_database = FALSE; + db_pool_local = previous_local; + details = previous_details; + set = previous; + puts("production scripted database worker regressions passed"); +} + +/* Startup reads still end the process on a database error. exit(), not + * _exit(), so the child's coverage is written like the parent's. */ +static void test_fake_database_startup_reads(void) { + for (int scenario = 0; scenario < 4; scenario++) { + fflush(NULL); + pid_t child = fork(); + assert(child >= 0); + if (child == 0) { + alarm(20); + fake_database = TRUE; + set.logging.log_destination = 0; + static const int statement_error[] = {1146}; + if (scenario == 0) { + fake_script(statement_error, 1); + } else { + fake_script(NULL, 0); + set.poller.poller_id = scenario == 2 ? 2 : 1; + set.poller.mode = REMOTE_ONLINE; + set.hosts.host_id_list[0] = '\0'; + if (scenario == 3) STRNCOPY(set.hosts.host_id_list, "902"); + } + read_config_options(); + exit(set.php.php_required ? 3 : 0); + } + int status; + assert(waitpid(child, &status, 0) == child); + fprintf(stderr, "fake startup read scenario=%d status=%d\n", scenario, status); + assert(WIFEXITED(status) && WEXITSTATUS(status) == (scenario == 0 ? EXIT_FAILURE : 0)); + } + puts("production scripted database startup regressions passed"); +} + +/* The whole main path against the scripted server: startup, both pools, a + * device list that ends early, completion and the collector push. */ +static int execute_fake_main_case(char *config) { + fake_database = TRUE; + fake_script(NULL, 0); + fake_rows_prefix = "SELECT SQL_NO_CACHE id, device_threads"; + fake_rows_matched = 1; + alarm(15); + char interval[] = "poller_interval:5"; + char *arguments[] = {"spine", "-C", "/nonexistent/spine-fault.conf", "--conf", config, + "-p", "2", "-t", "1", "-O", interval, "-S", "-V", "1", NULL}; + return spine_program_main((int)(sizeof(arguments) / sizeof(arguments[0]) - 1), arguments); +} + +static void test_fake_database_main(void) { + char directory[] = "/tmp/spine-fake-main-XXXXXX"; + assert(mkdtemp(directory) != NULL); + char config[SMALL_BUFSIZE]; + char log_path[SMALL_BUFSIZE]; + spine_snprintf(config, sizeof(config), "%s/spine.conf", directory); + spine_snprintf(log_path, sizeof(log_path), "%s/spine.log", directory); + FILE *file = fopen(config, "w"); + assert(file != NULL); + assert(fprintf(file, "DB_Host 127.0.0.1\nDB_Database spine_fake\nCacti_Log %s\n", log_path) > 0); + assert(fclose(file) == 0); + fflush(NULL); + pid_t process = fork(); + assert(process >= 0); + if (process == 0) { + execl("./test_spine_faults", "test_spine_faults", "--fake-main-case", config, NULL); + _exit(127); + } + int status; + assert(waitpid(process, &status, 0) == process); + fprintf(stderr, "fake main case status=%d\n", status); + /* One device was promised and none could be read: polling is incomplete. */ + assert(WIFEXITED(status) && WEXITSTATUS(status) == EXIT_FAILURE); + assert(unlink(config) == 0); + if (unlink(log_path) != 0) assert(errno == ENOENT); + assert(rmdir(directory) == 0); + puts("production scripted database main regressions passed"); +} + static void test_real_database_retry(void) { const char *hostname = getenv("SPINE_TEST_DB_HOST"); assert(hostname != NULL && hostname[0] != '\0'); @@ -693,13 +1404,23 @@ int main(int argc, char **argv) { if (argc == 4 && strcmp(argv[1], "--worker-launch-case") == 0) { return execute_worker_launch_case(argv[2], argv[3]); } + if (argc == 3 && strcmp(argv[1], "--fake-main-case") == 0) { + return execute_fake_main_case(argv[2]); + } config_defaults(); init_mutexes(); alarm(20); test_logger_format_failure(); test_process_creation_failures(); + test_spawn_group_failure(); test_interrupted_insert_retry(); test_snmpv3_privacy_copy_failure(); + test_privilege_drop_faults(); + test_fake_database_wrappers(); + test_fake_database_main_paths(); + test_fake_database_poll_host(); + test_fake_database_startup_reads(); + test_fake_database_main(); if (argc == 2 && strcmp(argv[1], "--database") == 0) { test_real_database_retry(); test_ping_only_session_lifetime(); diff --git a/tests/test_spine_privilege_contracts.c b/tests/test_spine_privilege_contracts.c new file mode 100644 index 0000000..f24c2c7 --- /dev/null +++ b/tests/test_spine_privilege_contracts.c @@ -0,0 +1,343 @@ +/* + * Copyright (C) 2004-2026 The Cacti Group + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + */ + +/* setresuid() and setresgid() let a root test reproduce the ids a setuid root + * exec hands spine: real uid of the caller, effective and saved uid of 0. */ +#define _GNU_SOURCE +#include "common.h" +#include "spine.h" +#include +#include + +#define UNPRIVILEGED_ID 65534 + +/* Linux keeps all four ids on one line, and the saved id is the one that a + * seteuid()-only drop leaves at 0. */ +static bool status_ids_are(const char *field, unsigned long id) { + #ifdef __linux__ + char line[256]; + char expected[128]; + bool found = FALSE; + FILE *status = fopen("/proc/self/status", "r"); + assert(status != NULL); + snprintf(expected, sizeof(expected), "%s\t%lu\t%lu\t%lu\t%lu\n", field, id, id, id, id); + while (fgets(line, sizeof(line), status) != NULL) { + if (strncmp(line, field, strlen(field)) == 0) { + found = strcmp(line, expected) == 0; + break; + } + } + assert(fclose(status) == 0); + return found; + #else + (void) field; + (void) id; + return TRUE; + #endif +} + +static int run_child(void (*body)(void), char *output, size_t capacity) { + int descriptors[2]; + assert(pipe(descriptors) == 0); + fflush(NULL); + pid_t child = fork(); + assert(child >= 0); + if (child == 0) { + alarm(15); + assert(close(descriptors[0]) == 0); + assert(dup2(descriptors[1], STDERR_FILENO) == STDERR_FILENO); + assert(close(descriptors[1]) == 0); + body(); + _exit(EXIT_SUCCESS); + } + assert(close(descriptors[1]) == 0); + size_t used = 0; + for (;;) { + ssize_t received = read(descriptors[0], output + used, capacity - used - 1); + if (received < 0 && errno == EINTR) continue; + assert(received >= 0); + if (received == 0) break; + used += (size_t)received; + assert(used < capacity - 1); + } + output[used] = '\0'; + assert(close(descriptors[0]) == 0); + int status; + while (waitpid(child, &status, 0) < 0) assert(errno == EINTR); + assert(WIFEXITED(status)); + return WEXITSTATUS(status); +} + +#ifdef __linux__ +/* The state execve() leaves behind for a setuid root binary run by a user. */ +static void become_setuid_root_caller(void) { + assert(setgroups(0, NULL) == 0); + assert(setresgid(UNPRIVILEGED_ID, UNPRIVILEGED_ID, UNPRIVILEGED_ID) == 0); + assert(setresuid(UNPRIVILEGED_ID, 0, 0) == 0); + assert(getuid() == UNPRIVILEGED_ID && geteuid() == 0); + assert(!privileges_dropped(UNPRIVILEGED_ID, UNPRIVILEGED_ID)); +} + +static void setuid_drop_body(void) { + become_setuid_root_caller(); + drop_privileges(); + assert(privileges_dropped(UNPRIVILEGED_ID, UNPRIVILEGED_ID)); + assert(status_ids_are("Uid:", UNPRIVILEGED_ID)); + assert(status_ids_are("Gid:", UNPRIVILEGED_ID)); + assert(seteuid(0) == -1 && geteuid() == UNPRIVILEGED_ID); + assert(setuid(0) == -1 && getuid() == UNPRIVILEGED_ID); + #ifdef HAVE_LCAP + assert(hasCaps() == TRUE); + #else + /* Opened as root, and never inherited by PHP or script children. */ + assert(ping_icmp_shared_available()); + int probe = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP); + assert(probe == -1 && errno == EPERM); + #endif +} + +/* When ICMP cannot be had, spine still drops root for good and keeps + * running; only ICMP availability is lost. */ +static void setuid_without_icmp_body(void) { + #ifdef HAVE_LCAP + cap_value_t net_raw = CAP_NET_RAW; + cap_t caps; + #else + struct rlimit none = {0, 0}; + #endif + set.exit.exit_code = EXIT_SUCCESS; + set.php.php_initialized = FALSE; + become_setuid_root_caller(); + #ifdef HAVE_LCAP + /* cap_set_proc() cannot keep a capability that is no longer permitted. */ + caps = cap_get_proc(); + assert(caps != NULL); + assert(cap_set_flag(caps, CAP_PERMITTED, 1, &net_raw, CAP_CLEAR) == 0); + assert(cap_set_flag(caps, CAP_EFFECTIVE, 1, &net_raw, CAP_CLEAR) == 0); + assert(cap_set_proc(caps) == 0); + assert(cap_free(caps) == 0); + #else + /* With no descriptor to spare, the raw sockets cannot open. */ + assert(setrlimit(RLIMIT_NOFILE, &none) == 0); + #endif + drop_privileges(); + assert(privileges_dropped(UNPRIVILEGED_ID, UNPRIVILEGED_ID)); + assert(!ping_icmp_shared_available()); + assert(hasCaps() == FALSE); + #ifdef HAVE_LCAP + /* PR_SET_KEEPCAPS must not have carried root's other capabilities over. */ + caps = cap_get_proc(); + assert(caps != NULL); + for (cap_value_t value = 0; value < 64; value++) { + cap_flag_value_t flag; + assert(cap_get_flag(caps, value, CAP_PERMITTED, &flag) != 0 || flag == CAP_CLEAR); + } + assert(cap_free(caps) == 0); + #endif + fprintf(stderr, "drop_privileges returned\n"); +} + +#ifdef HAVE_LCAP +/* A real root run of an --enable-lcap build is confined to CAP_NET_RAW. When + * that cannot be done it must stop, not carry on with every capability. */ +static void real_root_unconfinable_body(void) { + cap_value_t net_raw = CAP_NET_RAW; + cap_t caps = cap_get_proc(); + set.exit.exit_code = EXIT_SUCCESS; + set.php.php_initialized = FALSE; + assert(caps != NULL); + assert(cap_set_flag(caps, CAP_PERMITTED, 1, &net_raw, CAP_CLEAR) == 0); + assert(cap_set_flag(caps, CAP_EFFECTIVE, 1, &net_raw, CAP_CLEAR) == 0); + assert(cap_set_proc(caps) == 0); + assert(cap_free(caps) == 0); + drop_privileges(); + fprintf(stderr, "drop_privileges returned\n"); +} + +/* All ids non-zero is not enough: a capability kept in the permitted set can + * be raised again without any uid change. */ +static void retained_capabilities_body(void) { + assert(prctl(PR_SET_KEEPCAPS, 1) == 0); + assert(setgroups(0, NULL) == 0); + assert(setresgid(UNPRIVILEGED_ID, UNPRIVILEGED_ID, UNPRIVILEGED_ID) == 0); + assert(setresuid(UNPRIVILEGED_ID, UNPRIVILEGED_ID, UNPRIVILEGED_ID) == 0); + assert(seteuid(0) == -1); + assert(!privileges_dropped(UNPRIVILEGED_ID, UNPRIVILEGED_ID)); +} +#endif +#endif + +void test_privilege_contracts(void) { + char output[2048]; + uid_t uid = getuid(); + gid_t gid = getgid(); + + assert(!privileges_dropped(0, gid)); + if (geteuid() != 0) { + /* An unprivileged run has nothing to drop and keeps its ids. */ + drop_privileges(); + assert(getuid() == uid && geteuid() == uid && getgid() == gid && getegid() == gid); + assert(privileges_dropped(uid, gid)); + assert(!privileges_dropped(uid + 1, gid)); + assert(!privileges_dropped(uid, gid + 1)); + assert(!ping_icmp_shared_available()); + puts("production unprivileged startup contracts passed"); + return; + } + + #ifdef __linux__ + if (uid == 0) { + assert(run_child(setuid_drop_body, output, sizeof(output)) == EXIT_SUCCESS); + assert(run_child(setuid_without_icmp_body, output, sizeof(output)) == EXIT_SUCCESS); + assert(strstr(output, "refusing to run") == NULL); + assert(strstr(output, "drop_privileges returned") != NULL); + #ifdef HAVE_LCAP + assert(run_child(retained_capabilities_body, output, sizeof(output)) == EXIT_SUCCESS); + assert(run_child(real_root_unconfinable_body, output, sizeof(output)) == EXIT_FAILURE); + assert(strstr(output, "could not confine itself to CAP_NET_RAW") != NULL); + assert(strstr(output, "drop_privileges returned") == NULL); + #endif + puts("production setuid privilege drop contracts passed"); + } + #else + (void) output; + #endif +} + +static void *ping_loopback(void *argument) { + host_t host = {0}; + ping_t ping = {0}; + STRNCOPY(host.hostname, "127.0.0.1"); + host.id = (int)(intptr_t)argument; + host.availability.timeout = 1000; + host.availability.retries = 1; + return (void *)(intptr_t)ping_icmp(&host, &ping); +} + +/* Run from a copy of this binary installed setuid root and started by an + * unprivileged user, with net.ipv4.ping_group_range closed so the shared or + * capability path is the one that answers. */ +void test_setuid_icmp(void) { + uid_t uid = getuid(); + gid_t gid = getgid(); + pthread_t threads[8]; + + assert(uid != 0 && geteuid() == 0); + drop_privileges(); + assert(privileges_dropped(uid, gid)); + assert(status_ids_are("Uid:", uid)); + init_mutexes(); + checkAsRoot(); + assert(set.availability.icmp_avail); + for (size_t i = 0; i < sizeof(threads) / sizeof(threads[0]); i++) { + assert(pthread_create(&threads[i], NULL, ping_loopback, (void *)(intptr_t)(i + 1)) == 0); + } + for (size_t i = 0; i < sizeof(threads) / sizeof(threads[0]); i++) { + void *result; + assert(pthread_join(threads[i], &result) == 0); + assert((intptr_t)result == HOST_UP); + } + #ifdef SPINE_HAVE_ICMPV6 + /* The IPv6 socket shares the same hand-over. get_address_type() asks + * for AI_ADDRCONFIG, so ::1 only resolves on a host with a configured + * IPv6 address; skip it elsewhere. */ + host_t host6 = {0}; + ping_t ping6 = {0}; + STRNCOPY(host6.hostname, "::1"); + host6.availability.method = AVAIL_PING; + host6.availability.ping_method = PING_ICMP; + host6.availability.timeout = 1000; + host6.availability.retries = 1; + if (get_address_type(&host6) == SPINE_IPV6) { + assert(ping_host(&host6, &ping6) == HOST_UP); + assert(strcmp(ping6.ping_response, "ICMPv6: Device is Alive") == 0); + puts("setuid ICMPv6 loopback answered"); + } + #endif + assert(privileges_dropped(uid, gid)); + printf("setuid ICMP: uid=%d euid=%d shared=%d caps=%d\n", (int) getuid(), (int) geteuid(), + ping_icmp_shared_available(), hasCaps()); +} + +/* One device per thread: two in three are loopback addresses that answer, the + * rest are TEST-NET-1 addresses that never do. Timeouts and retries vary so + * readers come and go while others still wait. */ +static void *stress_device(void *argument) { + int index = (int)(intptr_t)argument; + int up = (index % 3) != 0; + host_t host = {0}; + ping_t ping = {0}; + if (up) { + snprintf(host.hostname, sizeof(host.hostname), "127.0.%d.%d", (index / 200) + 1, (index % 200) + 1); + } else { + snprintf(host.hostname, sizeof(host.hostname), "192.0.2.%d", (index % 250) + 1); + } + host.id = index; + host.availability.timeout = up ? 300 + (index % 7) * 100 : 200 + (index % 5) * 150; + host.availability.retries = index % 3; + int result = ping_icmp(&host, &ping); + if (up && result != HOST_UP) return (void *)(intptr_t)1; + if (!up && result == HOST_UP) return (void *)(intptr_t)2; + return NULL; +} + +/* Linux only; run as root. Reproduces a setuid root start (or, with + * SPINE_STRESS_REAL_ROOT set, a real root one that opens a socket per ping) + * and pings SPINE_STRESS_THREADS devices at once, `rounds` times. Run it + * with net.ipv4.ping_group_range closed so raw sockets answer, and with an + * ICMP flood aimed at the host to see how many responsive devices it hides. */ +void test_setuid_stress(int rounds) { + #ifdef __linux__ + const char *requested = getenv("SPINE_STRESS_THREADS"); + int count = requested != NULL ? atoi(requested) : 96; + int falsely_down = 0; + int falsely_up = 0; + int expected_up = 0; + double begin; + pthread_t *threads; + + assert(getuid() == 0 && count > 0 && count <= 1000 && rounds > 0); + if (getenv("SPINE_STRESS_REAL_ROOT") == NULL) { + become_setuid_root_caller(); + drop_privileges(); + assert(privileges_dropped(UNPRIVILEGED_ID, UNPRIVILEGED_ID)); + } + init_mutexes(); + checkAsRoot(); + assert(set.availability.icmp_avail); + threads = calloc((size_t) count, sizeof(*threads)); + assert(threads != NULL); + begin = get_time_as_double(); + for (int round = 0; round < rounds; round++) { + for (int i = 0; i < count; i++) { + int index = i + round * count; + if (index % 3 != 0) expected_up++; + assert(pthread_create(&threads[i], NULL, stress_device, (void *)(intptr_t) index) == 0); + } + for (int i = 0; i < count; i++) { + void *outcome; + assert(pthread_join(threads[i], &outcome) == 0); + if ((intptr_t) outcome == 1) falsely_down++; + if ((intptr_t) outcome == 2) falsely_up++; + } + } + free(threads); + printf("stress %s threads=%d rounds=%d falsely-down=%d of %d up, falsely-up=%d, elapsed=%.2fs\n", + ping_icmp_shared_available() ? "shared" : "per-ping", count, rounds, + falsely_down, expected_up, falsely_up, get_time_as_double() - begin); + #else + (void) rounds; + puts("stress mode needs Linux"); + #endif +} diff --git a/tests/test_spine_regressions.c b/tests/test_spine_regressions.c index 2f59b71..656cf7f 100644 --- a/tests/test_spine_regressions.c +++ b/tests/test_spine_regressions.c @@ -21,6 +21,9 @@ extern int spine_program_main(int argc, char **argv); extern void test_output_write_contracts(MYSQL *mysql); extern void test_additional_contracts(void); extern void test_cli_alias_contracts(void); +extern void test_privilege_contracts(void); +extern void test_setuid_icmp(void); +extern void test_setuid_stress(int rounds); extern int run_script_stream_fixture(const char *scenario); extern void test_script_stream_contracts(void); extern void test_additional_database_contracts(MYSQL *mysql); @@ -492,6 +495,52 @@ static void test_config_directives(void) { assert(read_spine_config(path) == -1); } +/* -C accepts any readable path, so a warning about a line that is not a + * directive must not reveal what the line holds. */ +static void test_config_unknown_lines_not_echoed(void) { + char path[] = "spine-config-echo-XXXXXX"; + int fd = mkstemp(path); + assert(fd >= 0); + FILE *file = fdopen(fd, "w"); + assert(file != NULL); + assert(fputs("DB_Host localhost\nroot:$6$saltsecret:19000:0:99999:7:::\n", file) != EOF); + assert(fclose(file) == 0); + int errors[2]; + assert(pipe(errors) == 0); + fflush(NULL); + pid_t child = fork(); + assert(child >= 0); + if (child == 0) { + assert(close(errors[0]) == 0); + assert(dup2(errors[1], STDERR_FILENO) == STDERR_FILENO); + assert(close(errors[1]) == 0); + memset(&set, 0, sizeof set); + set.console.stdout_notty = TRUE; + set.console.stderr_notty = FALSE; + config_defaults(); + _exit(read_spine_config(path) == 0 ? EXIT_SUCCESS : EXIT_FAILURE); + } + assert(close(errors[1]) == 0); + char message[1024]; + size_t used = 0; + for (;;) { + ssize_t received = read(errors[0], message + used, sizeof(message) - used - 1); + if (received < 0 && errno == EINTR) continue; + assert(received >= 0); + if (received == 0) break; + used += (size_t)received; + assert(used < sizeof(message) - 1); + } + message[used] = '\0'; + assert(close(errors[0]) == 0); + int status; + assert(waitpid(child, &status, 0) == child); + assert(WIFEXITED(status) && WEXITSTATUS(status) == EXIT_SUCCESS); + assert(unlink(path) == 0); + assert(strstr(message, "line 2") != NULL); + assert(strstr(message, "root") == NULL && strstr(message, "salt") == NULL); +} + static void test_config_structure_bindings(void) { config_t previous = set; memset(&set, 0, sizeof set); @@ -916,6 +965,109 @@ static void test_php_partial_response_timeout(void) { close(pipes[1]); } +/* Script server arguments can carry SNMP communities and v3 passphrases, so + * no failure line may repeat the command arguments, at any verbosity. + * Drives the timeout, partial-line and lost-server paths of the real code. */ +static void run_php_failure_logs(int level) { + static const char secret[] = "regression-community-7d1f"; + config_t previous_config = set; + php_t *previous_processes = php_processes; + char path[] = "spine-php-redaction-XXXXXX"; + int log_fd = mkstemp(path); + assert(log_fd >= 0 && close(log_fd) == 0); + set.logging.log_destination = LOGDEST_FILE; + set.logging.log_level = level; + set.logging.logfile_processed = TRUE; + set.console.stdout_notty = TRUE; + set.console.stderr_notty = TRUE; + strncopy(set.logging.path_logfile, path, sizeof(set.logging.path_logfile)); + set.php.php_servers = 1; + set.php.script_timeout = 1; + STRNCOPY(set.php.path_php, "/nonexistent-spine-regression-executable"); + char command[BUFSIZE]; + spine_snprintf(command, sizeof(command), "ss_regression.php ss_regression 127.0.0.1 %s\r\n", secret); + php_t *process = malloc(sizeof(*process)); + assert(process != NULL); + php_processes = process; + for (int partial = 0; partial <= 1; partial++) { + int pipes[2]; + assert(pipe(pipes) == 0); + if (partial) assert(write(pipes[1], "7", 1) == 1); + php_processes_initialize(process, 1); + process->php_read_fd = pipes[0]; + char *result = php_readpipe(0, command); + assert(strcmp(result, "U") == 0); + free(result); + if (process->php_read_fd >= 0) close(process->php_read_fd); + assert(close(pipes[1]) == 0); + } + int requests[2]; + assert(pipe(requests) == 0 && close(requests[0]) == 0); + int responses[2]; + assert(pipe(responses) == 0); + fflush(NULL); + pid_t server = fork(); + assert(server >= 0); + if (server == 0) { + pause(); + _exit(0); + } + php_processes_initialize(process, 1); + process->php_state = PHP_READY; + process->php_pid = server; + process->php_write_fd = requests[1]; + process->php_read_fd = responses[0]; + /* The write fails with EPIPE; restarts fail on the missing executable. */ + char *result = php_cmd(command, 0); + assert(strcmp(result, "U") == 0); + free(result); + assert(close(responses[1]) == 0); + int status; + assert(waitpid(server, &status, 0) == server); + /* The script poller logs the same commands; drive its empty-result and + * missing-file lines with the secret in the arguments. */ + assert(spine_permits_init(&available_scripts, 1) == 0); + host_t host = {0}; + STRNCOPY(host.hostname, "regression-device"); + char script_command[BUFSIZE]; + spine_snprintf(script_command, sizeof(script_command), "/usr/bin/printf '' %s", secret); + result = exec_poll(&host, script_command, 1, "DS"); + assert(strcmp(result, "U") == 0); + free(result); + spine_snprintf(script_command, sizeof(script_command), "/spine-regression/missing-script %s", secret); + result = exec_poll(&host, script_command, 1, "DS"); + assert(strcmp(result, "U") == 0); + free(result); + assert(spine_permits_destroy(&available_scripts) == 0); + FILE *file = fopen(path, "r"); + assert(file != NULL); + char logged[16384]; + size_t bytes = fread(logged, 1, sizeof(logged) - 1, file); + assert(!ferror(file) && fclose(file) == 0); + logged[bytes] = '\0'; + assert(strstr(logged, "did not respond in time") != NULL); + assert(strstr(logged, "partial response") != NULL); + assert(strstr(logged, "communications lost") != NULL); + assert(strstr(logged, "Empty result [regression-device]: '/usr/bin/printf'") != NULL); + assert(strstr(logged, "File '/spine-regression/missing-script' does not exist") != NULL); + fprintf(stderr, "php failure logs at level %d contain command arguments: %s\n", level, strstr(logged, secret) != NULL ? "yes" : "no"); + assert(strstr(logged, secret) == NULL); + /* Debug output still names the script, which is enough to find it. */ + if (level == POLLER_VERBOSITY_DEBUG) { + assert(strstr(logged, "ss_regression.php") != NULL); + assert(strstr(logged, "The executable is '/usr/bin/printf'") != NULL); + } + assert(unlink(path) == 0); + php_processes = previous_processes; + free(process); + set = previous_config; +} + +static void test_php_failure_logs_omit_command(void) { + run_php_failure_logs(POLLER_VERBOSITY_LOW); + run_php_failure_logs(POLLER_VERBOSITY_DEBUG); +} + static void test_script_execution(void) { assert(spine_permits_init(&available_scripts, 1) == 0); host_t host = {0}; @@ -971,6 +1123,59 @@ static void test_script_execution(void) { set.php.script_timeout = previous_timeout; } +/* A timed-out script is killed with everything it started. The background + * sleep inherits the write end of the witness pipe, so EOF on the read end + * proves that no descendant outlived the timeout. */ +static void test_script_timeout_kills_descendants(void) { + assert(spine_permits_init(&available_scripts, 1) == 0); + host_t host = {0}; + STRNCOPY(host.hostname, "regression-device"); + int previous_timeout = set.php.script_timeout; + set.php.script_timeout = 1; + int witness[2]; + assert(pipe(witness) == 0); + char command[] = "/bin/sh -c '/bin/sleep 30 & /bin/sleep 30'"; + char *result = exec_poll(&host, command, 1, "DS"); + assert(strcmp(result, "U") == 0); + free(result); + assert(close(witness[1]) == 0); + assert(witness[0] < FD_SETSIZE); + fd_set readable; + FD_ZERO(&readable); + FD_SET(witness[0], &readable); + struct timeval timeout = {3, 0}; + int ready = select(witness[0] + 1, &readable, NULL, NULL, &timeout); + char byte; + fprintf(stderr, "script timeout descendants: ready=%d\n", ready); + assert(ready == 1 && read(witness[0], &byte, 1) == 0); + assert(close(witness[0]) == 0); + assert(spine_permits_available(&available_scripts) == 1); + assert(spine_permits_destroy(&available_scripts) == 0); + set.php.script_timeout = previous_timeout; +} + +/* nft_pclose() kills a script that is still running after its output was + * read, and the kill must reach the script's descendants as well. */ +static void test_pclose_kills_descendants(void) { + int witness[2]; + assert(pipe(witness) == 0); + int fd = nft_popen("printf 7; /bin/sleep 30 & /bin/sleep 30", "r"); + assert(fd >= 0); + char byte; + assert(read(fd, &byte, 1) == 1 && byte == '7'); + assert(nft_pclose(fd) == -1 && errno == ETIMEDOUT); + assert(close(witness[1]) == 0); + assert(witness[0] < FD_SETSIZE); + fd_set readable; + FD_ZERO(&readable); + FD_SET(witness[0], &readable); + struct timeval timeout = {3, 0}; + int ready = select(witness[0] + 1, &readable, NULL, NULL, &timeout); + fprintf(stderr, "pclose descendants: ready=%d\n", ready); + assert(ready == 1 && read(witness[0], &byte, 1) == 0); + assert(close(witness[0]) == 0); +} + static void test_php_command(size_t length) { char command[BUFSIZE]; assert(length <= sizeof(command) - 3); @@ -1292,7 +1497,7 @@ static void test_icmp_socket_failure(void) { assert(geteuid() == getuid()); return; } - /* Failed socket retries must release the privilege mutex each time. */ + /* Failed socket retries must give up without regaining root. */ alarm(6); assert(ping_icmp(&host, &ping) == HOST_DOWN); alarm(0); @@ -1556,6 +1761,8 @@ static void *test_poll_worker(void *argument) { return NULL; } +/* A worker without a connection fails its device and returns; the process + * keeps polling other devices and still exits with a failure status. */ static void test_poll_missing_connection(const poller_thread_t *work) { for (int remote = 0; remote <= 1; remote++) { pid_t child = fork(); @@ -1567,13 +1774,23 @@ static void test_poll_missing_connection(const poller_thread_t *work) { set.poller.mode = REMOTE_ONLINE; db_pool_remote = &unavailable; } else db_pool_local[0].free = FALSE; + extern poller_thread_t **details; + poller_thread_t *device = details[work->device_counter]; + device->complete = FALSE; + device->threads_complete = 0; int errors = 0; + set.exit.exit_code = EXIT_SUCCESS; poll_host(work, &errors); - _exit(0); + if (!device->poll_failed || device->complete || device->threads_complete != 1) _exit(3); + /* The local slot borrowed for a remote poll goes back to the pool. */ + if (remote && !db_pool_local[0].free) _exit(4); + /* The failure still reaches the process exit status. */ + _exit(set.exit.exit_code == EXIT_FAILURE ? 0 : 5); } int status; assert(waitpid(child, &status, 0) == child); - assert(WIFEXITED(status) && WEXITSTATUS(status) == EXIT_FAILURE); + fprintf(stderr, "missing connection: remote=%d raw_status=%d\n", remote, status); + assert(WIFEXITED(status) && WEXITSTATUS(status) == 0); } } @@ -1637,6 +1854,28 @@ static void test_reindex_pipeline(MYSQL *mysql, test_poll_work_t *work) { assert(db_insert(mysql, LOCAL, "DELETE FROM poller_command")); } +/* A reindex that could not be queued must leave the stored assertion alone; + * advancing it would hide the change from every later poll. Hiding the + * poller_command table makes the server itself reject the queue write. */ +static void test_reindex_queue_failure(MYSQL *mysql, test_poll_work_t *work) { + assert(db_insert(mysql, LOCAL, "DELETE FROM poller_reindex")); + assert(db_insert(mysql, LOCAL, "DELETE FROM poller_command")); + assert(db_insert(mysql, LOCAL, "INSERT INTO poller_reindex(host_id,data_query_id,action,op,assert_value,arg1) VALUES (43,7,1,'=','122','/usr/bin/printf 123'),(43,8,1,'>','122','/usr/bin/printf 123')")); + assert(db_insert(mysql, LOCAL, "RENAME TABLE poller_command TO poller_command_regression_hidden")); + work->thread.complete = FALSE; + work->thread.threads_complete = 0; + work->errors = 0; + pthread_t worker; + assert(pthread_create(&worker, NULL, test_poll_worker, work) == 0); + assert(pthread_join(worker, NULL) == 0); + assert(db_insert(mysql, LOCAL, "RENAME TABLE poller_command_regression_hidden TO poller_command")); + long long kept = (long long)database_count(mysql, "SELECT COUNT(*) FROM poller_reindex WHERE (data_query_id=7 AND assert_value='122') OR (data_query_id=8 AND assert_value='122')"); + fprintf(stderr, "reindex queue failure: assertions kept=%lld\n", kept); + assert(kept == 2); + assert(work->thread.threads_complete == 1); + assert(db_insert(mysql, LOCAL, "DELETE FROM poller_reindex")); +} + static void test_reindex_query_shortcut(MYSQL *mysql, test_poll_work_t *work) { assert(db_insert(mysql, LOCAL, "DELETE FROM poller_reindex")); assert(db_insert(mysql, LOCAL, "DELETE FROM poller_command")); @@ -1695,6 +1934,51 @@ static void test_profile_schedule_completion(MYSQL *mysql, test_poll_work_t *agg set = previous; } +/* A real, non-retryable SQL error inside a worker fails that device and the + * worker returns: the schedule is not advanced, nothing is written, and the + * connection goes back to the pool for the next device. The item query names + * a column the fixture schema lacks, so MariaDB itself rejects it. */ +static void test_poll_database_failure(MYSQL *mysql, test_poll_work_t *aggregate) { + config_t previous = set; + test_poll_work_t original = *aggregate; + assert(database_count(mysql, "SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='poller_item' AND COLUMN_NAME='output_regex'") == 0); + set.poller.active_profiles = 2; + set.poller.poller_interval = 5; + assert(db_insert(mysql, LOCAL, "DELETE FROM poller_item WHERE host_id=43 AND local_data_id NOT IN (601,602)")); + for (int failing = 1; failing >= 0; failing--) { + set.hosts.has_output_regex = failing; + assert(db_insert(mysql, LOCAL, "UPDATE poller_item SET rrd_step=300,rrd_next_step=0 WHERE host_id=43")); + assert(db_insert(mysql, LOCAL, "DELETE FROM poller_output")); + assert(db_insert(mysql, LOCAL, "DELETE FROM poller_output_boost")); + aggregate->thread.host_thread = 1; + aggregate->thread.host_threads = 1; + aggregate->thread.host_data_ids = 0; + aggregate->thread.complete = FALSE; + aggregate->thread.threads_complete = 0; + aggregate->thread.output_failed = FALSE; + aggregate->thread.poll_failed = FALSE; + set.exit.exit_code = EXIT_SUCCESS; + pthread_t worker; + assert(pthread_create(&worker, NULL, test_poll_worker, aggregate) == 0); + assert(pthread_join(worker, NULL) == 0); + assert(aggregate->thread.threads_complete == 1); + assert(db_pool_local[0].free && spine_permits_available(&available_scripts) == 2); + if (failing) { + assert(aggregate->thread.poll_failed && !aggregate->thread.complete); + assert(set.exit.exit_code == EXIT_FAILURE); + assert(database_count(mysql, "SELECT COUNT(*) FROM poller_output") == 0); + assert(database_count(mysql, "SELECT COUNT(*) FROM poller_item WHERE host_id=43 AND rrd_next_step=0") == 2); + } else { + assert(!aggregate->thread.poll_failed && aggregate->thread.complete); + assert(set.exit.exit_code == EXIT_SUCCESS); + assert(database_count(mysql, "SELECT COUNT(*) FROM poller_output WHERE (local_data_id=601 AND output='123') OR (local_data_id=602 AND output='U')") == 2); + assert(database_count(mysql, "SELECT COUNT(*) FROM poller_item WHERE host_id=43 AND rrd_next_step=295") == 2); + } + } + *aggregate = original; + set = previous; +} + static void prepare_nullable_snmp_profile(MYSQL *mysql) { /* These fields are nullable in the shipped schema; missing values * must survive the real row-loader/session handoff. */ @@ -1843,7 +2127,9 @@ static void test_poll_pipeline(MYSQL *mysql) { if (host_id == 43) { test_reindex_pipeline(mysql, work); test_reindex_query_shortcut(mysql, work); + test_reindex_queue_failure(mysql, work); test_profile_schedule_completion(mysql, work); + test_poll_database_failure(mysql, work); } /* End this fixture binding before the next host or caller runs. */ details = previous_details; @@ -2445,6 +2731,19 @@ int main(int argc, char **argv) { puts("production ICMP loopback regression passed"); return 0; } + if (argc == 2 && strcmp(argv[1], "--privilege-drop") == 0) { + test_privilege_contracts(); + return 0; + } + if (argc == 3 && strcmp(argv[1], "--setuid-stress") == 0) { + test_setuid_stress(atoi(argv[2])); + return 0; + } + if (argc == 2 && strcmp(argv[1], "--setuid-icmp") == 0) { + test_setuid_icmp(); + puts("production setuid ICMP regression passed"); + return 0; + } if (argc == 2 && strcmp(argv[1], "--icmp-no-capability") == 0) { init_mutexes(); test_icmp_socket_failure(); @@ -2484,6 +2783,7 @@ int main(int argc, char **argv) { test_log_sanitization(); test_log_append_and_failures(); test_config_directives(); + test_config_unknown_lines_not_echoed(); test_config_structure_bindings(); test_date_formats(); test_device_logging(); @@ -2510,12 +2810,16 @@ int main(int argc, char **argv) { test_php_command(4); test_php_command(BUFSIZE - 3); test_invalid_php_commands(); + test_php_failure_logs_omit_command(); test_php_startup(argv[0]); test_php_owned_shutdown(); test_host_status_transitions(); test_script_execution(); + test_script_timeout_kills_descendants(); + test_pclose_kills_descendants(); test_script_stream_contracts(); test_cli_alias_contracts(); + test_privilege_contracts(); test_error_id_buffer_boundaries(); test_additional_contracts(); puts("production regression tests passed"); diff --git a/tests/unit/test_php_runtime.c b/tests/unit/test_php_runtime.c index d1165ec..8e0c7fd 100644 --- a/tests/unit/test_php_runtime.c +++ b/tests/unit/test_php_runtime.c @@ -909,12 +909,12 @@ static void test_readpipe_rejects_invalid_slots(void **state) { for (index = 0; index < sizeof(invalid_slots) / sizeof(invalid_slots[0]); index++) { assert_invalid_reader_result(invalid_slots[index]); } - set.php_servers = 0; + set.php.php_servers = 0; assert_invalid_reader_result(0); - set.php_servers = -1; + set.php.php_servers = -1; assert_invalid_reader_result(0); /* An invalid configured count cannot authorize an out-of-array slot. */ - set.php_servers = MAX_PHP_SERVERS + 1; + set.php.php_servers = MAX_PHP_SERVERS + 1; assert_invalid_reader_result(MAX_PHP_SERVERS); assert_int_equal(php_spawn_calls, 0); assert_int_equal(write_calls, 0); diff --git a/tests/unit/test_ping_icmp.c b/tests/unit/test_ping_icmp.c index 9071ce0..5d5edaa 100644 --- a/tests/unit/test_ping_icmp.c +++ b/tests/unit/test_ping_icmp.c @@ -1,9 +1,10 @@ -/* ping_icmp() resource ownership. +/* ping_icmp() resource ownership and the shared raw socket. * - * This is the least-tested and highest-consequence change in the branch: the - * function runs in a SUID-root binary, and its five exits were collapsed onto - * one cleanup label while the seteuid(0)/LOCK_SETEUID wrapper around close() - * was removed. Nothing covered it. + * The function runs in a SUID-root binary, and its five exits were collapsed + * onto one cleanup label while the seteuid(0) wrapper around close() was + * removed. Root is now dropped at startup and a setuid install without + * capabilities pings through raw sockets opened before the drop, which every + * poller thread shares. * * The live ICMP cases skip without raw-socket privilege. The ownership cases * use controlled socket and allocation sinks, so they run everywhere. @@ -36,6 +37,7 @@ static int use_controlled_socket; static int controlled_socket_fd; static int controlled_socket_type; static int controlled_socket_closed; +static int controlled_socket_cloexec; static int socket_failures_remaining; static int socket_calls; static int track_packet; @@ -49,13 +51,22 @@ static int freeaddrinfo_calls; static int controlled_reply; static uint16_t sent_icmp_id; static uint16_t sent_icmp_seq; -#ifdef SPINE_HAVE_ICMPV6 +static int reply_seq_override; +static uint16_t reply_queue[8]; +static int reply_queue_len; +static int reply_queue_pos; +static uint16_t reply_seq; +static int recvfrom_calls; +static int recvfrom_eintr_once; static int controlled_reply_v6; +static int socket_cloexec_einval; +static int fcntl_fails; static size_t reply_v6_count; -#endif +static int check_wrong_peer; static int test_socket(int domain, int type, int protocol); static int test_close(int fd); +static int test_fcntl(int fd, int command, int argument); static void *intercepted_malloc(size_t size); static void intercepted_free(void *ptr); static int test_getaddrinfo(const char *node, const char *service, @@ -72,6 +83,7 @@ static ssize_t test_recvfrom(int fd, void *buffer, size_t length, int flags, * resource sinks can be observed without root or Linux-only linker wrapping. */ #define socket test_socket #define close test_close +#define fcntl test_fcntl #define malloc intercepted_malloc #define free intercepted_free #define getaddrinfo test_getaddrinfo @@ -82,6 +94,7 @@ static ssize_t test_recvfrom(int fd, void *buffer, size_t length, int flags, #include "../../ping.c" #undef socket #undef close +#undef fcntl #undef malloc #undef free #undef getaddrinfo @@ -150,6 +163,12 @@ static ssize_t test_recvfrom(int fd, void *buffer, size_t length, int flags, if (!controlled_reply) { return recvfrom(fd, buffer, length, flags, address, address_len); } + recvfrom_calls++; + if (recvfrom_eintr_once) { + recvfrom_eintr_once = 0; + errno = EINTR; + return -1; + } #ifdef SPINE_HAVE_ICMPV6 if (controlled_reply_v6) { const char payload[] = "cacti-monitoring-system"; @@ -161,17 +180,17 @@ static ssize_t test_recvfrom(int fd, void *buffer, size_t length, int flags, assert_non_null(address); assert_non_null(address_len); assert_true(*address_len >= sizeof(*source6)); - assert_true(reply_v6_count <= sizeof(source6->sin6_addr.s6_addr)); + if (check_wrong_peer) assert_true(reply_v6_count <= sizeof(source6->sin6_addr.s6_addr)); memset(buffer, 0, response_length); reply6->icmp6_type = ICMP6_ECHO_REPLY; - reply6->icmp6_id = controlled_socket_type == SOCK_DGRAM ? sent_icmp_id ^ htons(0x100) : sent_icmp_id; + reply6->icmp6_id = check_wrong_peer && controlled_socket_type == SOCK_DGRAM ? sent_icmp_id ^ htons(0x100) : sent_icmp_id; reply6->icmp6_seq = sent_icmp_seq; memcpy((unsigned char *) buffer + sizeof(*reply6), payload, sizeof(payload) - 1); memset(source6, 0, sizeof(*source6)); source6->sin6_family = AF_INET6; source6->sin6_addr = in6addr_loopback; /* Queue one wrong source for every address octet, then the real peer. */ - if (reply_v6_count < sizeof(source6->sin6_addr.s6_addr)) { + if (check_wrong_peer && reply_v6_count < sizeof(source6->sin6_addr.s6_addr)) { source6->sin6_addr.s6_addr[reply_v6_count] ^= 0x80; } reply_v6_count++; @@ -179,16 +198,27 @@ static ssize_t test_recvfrom(int fd, void *buffer, size_t length, int flags, return (ssize_t) response_length; } #endif + /* A finite queue ends the way a drained non-blocking socket does. */ + if (reply_queue_len > 0 && reply_queue_pos >= reply_queue_len) { + errno = EAGAIN; + return -1; + } assert_true(length >= reply_length); memset(buffer, 0, reply_length); if (ip_length != 0) { ip_reply = buffer; + ip_reply->ip_v = 4; ip_reply->ip_hl = sizeof(struct ip) >> 2; + ip_reply->ip_p = IPPROTO_ICMP; } icmp_reply = (struct icmp *)((unsigned char *) buffer + ip_length); icmp_reply->icmp_type = ICMP_ECHOREPLY; icmp_reply->icmp_id = sent_icmp_id; - icmp_reply->icmp_seq = sent_icmp_seq; + if (reply_queue_len > 0) { + icmp_reply->icmp_seq = reply_queue[reply_queue_pos++]; + } else { + icmp_reply->icmp_seq = reply_seq_override ? reply_seq : sent_icmp_seq; + } if (address != NULL && address_len != NULL && *address_len >= sizeof(*source)) { source = (struct sockaddr_in *) address; memset(source, 0, sizeof(*source)); @@ -210,6 +240,17 @@ static int test_socket(int domain, int type, int protocol) { if (use_controlled_socket) { (void) domain; (void) protocol; + #ifdef SOCK_CLOEXEC + /* A kernel too old for SOCK_CLOEXEC rejects the flag. */ + if ((type & SOCK_CLOEXEC) && socket_cloexec_einval) { + errno = EINVAL; + return -1; + } + if (type & SOCK_CLOEXEC) { + controlled_socket_cloexec = 1; + type &= ~SOCK_CLOEXEC; + } + #endif controlled_socket_type = type; return controlled_socket_fd; } @@ -226,6 +267,21 @@ static int test_close(int fd) { return close(fd); } +static int test_fcntl(int fd, int command, int argument) { + if (use_controlled_socket && fd == controlled_socket_fd) { + if (fcntl_fails) { + errno = EBADF; + return -1; + } + if (command == F_SETFD && (argument & FD_CLOEXEC)) { + controlled_socket_cloexec = 1; + } + return 0; + } + + return fcntl(fd, command, argument); +} + static void *intercepted_malloc(size_t size) { void *ptr = malloc(size); @@ -269,6 +325,7 @@ static int ping_reset(void **state) { controlled_socket_fd = -1; controlled_socket_type = 0; controlled_socket_closed = 0; + controlled_socket_cloexec = 0; socket_failures_remaining = 0; socket_calls = 0; track_packet = 0; @@ -284,10 +341,22 @@ static int ping_reset(void **state) { controlled_reply = 0; sent_icmp_id = 0; sent_icmp_seq = 0; -#ifdef SPINE_HAVE_ICMPV6 + reply_seq_override = 0; + reply_queue_len = 0; + reply_queue_pos = 0; + reply_seq = 0; + recvfrom_calls = 0; + icmp_shared.fd = -1; + icmp_shared.reading = FALSE; + icmp6_shared.fd = -1; + icmp6_shared.reading = FALSE; + recvfrom_eintr_once = 0; controlled_reply_v6 = 0; + socket_cloexec_einval = 0; + fcntl_fails = 0; + icmp_waiters = NULL; reply_v6_count = 0; -#endif + check_wrong_peer = 0; return 0; } @@ -413,8 +482,8 @@ static void test_matching_reply_releases_resources(void **state) { assert_non_null(strstr(ping.ping_response, "Alive")); assert_int_equal(packet_released, 1); assert_int_equal(controlled_socket_closed, 1); - assert_int_equal(thread_mutex_trylock(LOCK_SETEUID), 0); - thread_mutex_unlock(LOCK_SETEUID); + assert_int_equal(thread_mutex_trylock(LOCK_ICMP), 0); + thread_mutex_unlock(LOCK_ICMP); } static void test_cached_capability_path_releases_resources(void **state) { @@ -430,8 +499,8 @@ static void test_cached_capability_path_releases_resources(void **state) { assert_int_equal(ping_icmp(&host, &ping), HOST_DOWN); assert_int_equal(geteuid(), getuid()); - assert_int_equal(thread_mutex_trylock(LOCK_SETEUID), 0); - thread_mutex_unlock(LOCK_SETEUID); + assert_int_equal(thread_mutex_trylock(LOCK_ICMP), 0); + thread_mutex_unlock(LOCK_ICMP); } static void test_socket_retry_can_succeed_after_one_failure(void **state) { @@ -449,13 +518,14 @@ static void test_socket_retry_can_succeed_after_one_failure(void **state) { assert_int_equal(socket_calls, 2); assert_int_equal(geteuid(), getuid()); assert_int_equal(controlled_socket_closed, 1); - assert_int_equal(thread_mutex_trylock(LOCK_SETEUID), 0); - thread_mutex_unlock(LOCK_SETEUID); + assert_int_equal(thread_mutex_trylock(LOCK_ICMP), 0); + thread_mutex_unlock(LOCK_ICMP); } -/* The socket() retry used to sleep and loop back with LOCK_SETEUID still held, - so attempt two relocked a non-recursive process-global mutex from its own - owner. That wedges the thread at euid 0 and every other thread behind it. +/* The socket() retry used to sleep and loop back with the seteuid lock still + held, so attempt two relocked a non-recursive process-global mutex from its + own owner. That wedged the thread at euid 0 and every other thread behind it. + The elevation is gone; the retry must still give up rather than hang. This runs exactly where the tests above skip: with no privilege, socket() fails with EPERM and the retry loop is what executes. An alarm turns the @@ -484,7 +554,7 @@ static void test_socket_retry_does_not_deadlock_on_seteuid(void **state) { if (sigsetjmp(ping_deadlock_env, 1) != 0) { alarm(0); sigaction(SIGALRM, &prev, NULL); - fail_msg("ping_icmp() blocked in the socket() retry; LOCK_SETEUID was held across the sleep"); + fail_msg("ping_icmp() blocked in the socket() retry"); } make_host(&host, "127.0.0.1"); @@ -502,11 +572,453 @@ static void test_socket_retry_does_not_deadlock_on_seteuid(void **state) { assert_non_null(strstr(ping.ping_response, "ICMP Socket")); /* Keep the alarm armed for the probe, and never block on a leaked lock. */ - assert_int_equal(thread_mutex_trylock(LOCK_SETEUID), 0); - thread_mutex_unlock(LOCK_SETEUID); + assert_int_equal(thread_mutex_trylock(LOCK_ICMP), 0); + thread_mutex_unlock(LOCK_ICMP); } +/* A setuid install without capabilities pings through the socket opened before + root was dropped. It must not try for a raw socket of its own, which would + fail, and must leave the shared socket open for the next caller. */ +static void test_shared_socket_answers_without_a_socket_of_its_own(void **state) { + host_t host; + ping_t ping; + + (void) state; + use_owned_controlled_socket(); + icmp_shared.fd = controlled_pair[0]; + use_controlled_socket = 0; + socket_failures_remaining = 1; + controlled_reply = 1; + track_packet = 1; + make_host(&host, "127.0.0.1"); + memset(&ping, 0, sizeof(ping)); + + assert_true(ping_icmp_shared_available()); + assert_int_equal(ping_icmp(&host, &ping), HOST_UP); + assert_non_null(strstr(ping.ping_response, "Alive")); + assert_int_equal(socket_calls, 1); + assert_int_equal(controlled_socket_closed, 0); + assert_int_equal(packet_released, 1); + assert_null(icmp_waiters); + assert_int_equal(geteuid(), getuid()); + assert_int_equal(thread_mutex_trylock(LOCK_ICMP), 0); + thread_mutex_unlock(LOCK_ICMP); +} + +static void test_shared_socket_timeout_unregisters(void **state) { + host_t host; + ping_t ping; + + (void) state; + use_owned_controlled_socket(); + icmp_shared.fd = controlled_pair[0]; + use_controlled_socket = 0; + socket_failures_remaining = 1; + make_host(&host, "127.0.0.1"); + host.availability.timeout = 50; + host.availability.retries = 1; + memset(&ping, 0, sizeof(ping)); + + assert_int_equal(ping_icmp(&host, &ping), HOST_DOWN); + assert_non_null(strstr(ping.ping_response, "timed out")); + assert_int_equal(controlled_socket_closed, 0); + assert_null(icmp_waiters); +} + +static void make_waiter(icmp_waiter_t *waiter, uint16_t seq) { + memset(waiter, 0, sizeof(*waiter)); + waiter->shared = &icmp_shared; + waiter->family = AF_INET; + waiter->id = (uint16_t) (getpid() & 0xFFFF); + waiter->seq = seq; + waiter->peer.s_addr = htonl(INADDR_LOOPBACK); +} + +/* One raw socket queues each reply once. Whoever reads a reply that answers + another thread's request must hand it over, or that device goes down. */ +static void test_shared_reply_for_another_waiter_is_handed_over(void **state) { + icmp_waiter_t mine; + icmp_waiter_t theirs; + + (void) state; + use_owned_controlled_socket(); + icmp_shared.fd = controlled_pair[0]; + controlled_reply = 1; + reply_seq_override = 1; + reply_seq = 77; + sent_icmp_id = (uint16_t) (getpid() & 0xFFFF); + make_waiter(&mine, 5); + make_waiter(&theirs, 77); + icmp_shared_register(&theirs); + icmp_shared_register(&mine); + + assert_false(icmp_shared_await(&mine, get_time_as_double() + 0.05)); + assert_true(theirs.answered); + assert_false(mine.answered); + assert_true(recvfrom_calls > 0); + + icmp_shared_unregister(&mine); + icmp_shared_unregister(&theirs); + assert_null(icmp_waiters); + assert_false(icmp_shared.reading); +} + +static void *answer_waiter(void *argument) { + icmp_waiter_t *waiter = argument; + + usleep(20000); + thread_mutex_lock(LOCK_ICMP); + waiter->answered = TRUE; + pthread_cond_signal(&waiter->wake); + thread_mutex_unlock(LOCK_ICMP); + return NULL; +} + +/* While another thread reads, a waiter sleeps on the condition variable and + never touches the socket; the reader's hand-over wakes it. */ +static void test_shared_waiter_does_not_read_while_another_thread_does(void **state) { + icmp_waiter_t mine; + pthread_t reader; + + (void) state; + use_owned_controlled_socket(); + icmp_shared.fd = controlled_pair[0]; + controlled_reply = 1; + make_waiter(&mine, 9); + icmp_shared_register(&mine); + icmp_shared.reading = TRUE; + + assert_int_equal(pthread_create(&reader, NULL, answer_waiter, &mine), 0); + assert_true(icmp_shared_await(&mine, get_time_as_double() + 2.0)); + assert_int_equal(pthread_join(reader, NULL), 0); + assert_int_equal(recvfrom_calls, 0); + + icmp_shared.reading = FALSE; + icmp_shared_unregister(&mine); +} + +/* Under a flood one reply per lock cycle falls behind the queue. A reader must + empty everything that is waiting and answer every waiter it finds. */ +static void test_shared_reader_drains_the_queue(void **state) { + icmp_waiter_t mine; + icmp_waiter_t others[3]; + + (void) state; + use_owned_controlled_socket(); + icmp_shared.fd = controlled_pair[0]; + controlled_reply = 1; + sent_icmp_id = (uint16_t) (getpid() & 0xFFFF); + for (int i = 0; i < 3; i++) { + make_waiter(&others[i], (uint16_t) (100 + i)); + icmp_shared_register(&others[i]); + reply_queue[i] = (uint16_t) (100 + i); + } + reply_queue[3] = 9; + reply_queue_len = 4; + make_waiter(&mine, 9); + icmp_shared_register(&mine); + + assert_true(icmp_shared_await(&mine, get_time_as_double() + 1.0)); + for (int i = 0; i < 3; i++) { + assert_true(others[i].answered); + icmp_shared_unregister(&others[i]); + } + /* one pass: four replies, then the empty queue */ + assert_int_equal(recvfrom_calls, 5); + icmp_shared_unregister(&mine); + assert_null(icmp_waiters); +} + +static void *release_reader(void *argument) { + icmp_shared_t *shared = argument; + + usleep(20000); + thread_mutex_lock(LOCK_ICMP); + shared->reading = FALSE; + icmp_shared_handoff(shared); + thread_mutex_unlock(LOCK_ICMP); + return NULL; +} + +/* With per-waiter wakeups, a reader that leaves must hand the socket to a + sleeping waiter, or that waiter sleeps through its own reply. */ +static void test_shared_reader_hands_over_on_leaving(void **state) { + icmp_waiter_t mine; + pthread_t reader; + double begin; + + (void) state; + use_owned_controlled_socket(); + icmp_shared.fd = controlled_pair[0]; + controlled_reply = 1; + sent_icmp_id = (uint16_t) (getpid() & 0xFFFF); + reply_queue[0] = 11; + reply_queue_len = 1; + make_waiter(&mine, 11); + icmp_shared_register(&mine); + icmp_shared.reading = TRUE; + + begin = get_time_as_double(); + assert_int_equal(pthread_create(&reader, NULL, release_reader, &icmp_shared), 0); + assert_true(icmp_shared_await(&mine, begin + 5.0)); + assert_int_equal(pthread_join(reader, NULL), 0); + assert_true(get_time_as_double() - begin < 2.0); + assert_false(icmp_shared.reading); + icmp_shared_unregister(&mine); +} + +/* Poller threads spawn scripts while others ping, so a per-ping socket must + be close-on-exec from the start or a script can inherit raw ICMP access. */ +static void test_per_ping_sockets_are_close_on_exec(void **state) { + host_t host; + ping_t ping; + + (void) state; + use_owned_controlled_socket(); + make_host(&host, "127.0.0.1"); + host.availability.timeout = 1; + host.availability.retries = 0; + + /* datagram socket */ + memset(&ping, 0, sizeof(ping)); + assert_int_equal(ping_icmp(&host, &ping), HOST_DOWN); + assert_int_equal(controlled_socket_type, SOCK_DGRAM); + assert_int_equal(controlled_socket_cloexec, 1); + + /* raw socket after the datagram one is refused */ + controlled_socket_cloexec = 0; + socket_failures_remaining = 1; + memset(&ping, 0, sizeof(ping)); + assert_int_equal(ping_icmp(&host, &ping), HOST_DOWN); + assert_int_equal(controlled_socket_type, SOCK_RAW); + assert_int_equal(controlled_socket_cloexec, 1); +} + +static void test_icmp6_reply_matching(void **state) { + unsigned char reply[sizeof(struct icmp6_hdr) + sizeof(icmp6_payload)]; + struct icmp6_hdr *header = (struct icmp6_hdr *) reply; + ssize_t length = (ssize_t) (sizeof(*header) + sizeof(icmp6_payload) - 1); + + (void) state; + memset(reply, 0, sizeof(reply)); + header->icmp6_type = ICMP6_ECHO_REPLY; + header->icmp6_id = htons(7); + header->icmp6_seq = htons(9); + memcpy(reply + sizeof(*header), icmp6_payload, sizeof(icmp6_payload) - 1); + + assert_true(icmp6_reply_matches(reply, length, htons(7), htons(9), TRUE)); + assert_false(icmp6_reply_matches(reply, -1, htons(7), htons(9), TRUE)); + assert_false(icmp6_reply_matches(reply, length - 1, htons(7), htons(9), TRUE)); + assert_false(icmp6_reply_matches(reply, length, htons(7), htons(8), TRUE)); + /* a datagram socket rewrites the id, so only the raw path checks it */ + assert_false(icmp6_reply_matches(reply, length, htons(6), htons(9), TRUE)); + assert_true(icmp6_reply_matches(reply, length, htons(6), htons(9), FALSE)); + header->icmp6_type = ICMP6_ECHO_REQUEST; + assert_false(icmp6_reply_matches(reply, length, htons(7), htons(9), TRUE)); + header->icmp6_type = ICMP6_ECHO_REPLY; + reply[sizeof(*header)] ^= 1; + assert_false(icmp6_reply_matches(reply, length, htons(7), htons(9), TRUE)); +} + +/* Where SOCK_CLOEXEC is missing or refused, fcntl() sets the flag, and a + socket whose flag cannot be set is closed rather than handed out. */ +static void test_open_socket_fcntl_fallback(void **state) { + (void) state; + use_owned_controlled_socket(); + socket_cloexec_einval = 1; + + assert_int_equal(icmp_open_socket(AF_INET, SOCK_RAW, IPPROTO_ICMP), controlled_socket_fd); + assert_int_equal(controlled_socket_cloexec, 1); + assert_int_equal(controlled_socket_closed, 0); + + fcntl_fails = 1; + assert_int_equal(icmp_open_socket(AF_INET, SOCK_RAW, IPPROTO_ICMP), -1); + assert_int_equal(controlled_socket_closed, 1); +} + +/* drop_privileges() calls this once, as root, before anything else runs. */ +static void test_open_shared_sockets(void **state) { + (void) state; + use_owned_controlled_socket(); + assert_false(ping_icmp_shared_available()); + assert_true(ping_icmp_open_shared()); + assert_true(ping_icmp_shared_available()); + assert_int_equal(icmp_shared.fd, controlled_socket_fd); + assert_int_equal(icmp6_shared.fd, controlled_socket_fd); + assert_int_equal(controlled_socket_type, SOCK_RAW); + assert_int_equal(controlled_socket_cloexec, 1); + /* opening twice keeps the sockets already open */ + socket_calls = 0; + assert_true(ping_icmp_open_shared()); + assert_int_equal(socket_calls, 0); + assert_int_equal(controlled_socket_closed, 0); +} + +static void test_open_shared_sockets_failures(void **state) { + (void) state; + socket_failures_remaining = 2; + assert_false(ping_icmp_open_shared()); + assert_int_equal(icmp6_shared.fd, -1); + + /* select() cannot watch a descriptor this high, so it is not kept */ + use_controlled_socket = 1; + controlled_socket_fd = FD_SETSIZE; + assert_false(ping_icmp_open_shared()); + assert_int_equal(controlled_socket_closed, 2); + assert_int_equal(icmp_shared.fd, -1); +} + +static void test_shared_dispatch_ipv6(void **state) { + icmp_waiter_t waiter; + unsigned char reply[sizeof(struct icmp6_hdr) + sizeof(icmp6_payload)]; + struct icmp6_hdr *header = (struct icmp6_hdr *) reply; + struct sockaddr_storage from; + struct sockaddr_in6 *source = (struct sockaddr_in6 *) &from; + ssize_t length = (ssize_t) (sizeof(*header) + sizeof(icmp6_payload) - 1); + + (void) state; + memset(&waiter, 0, sizeof(waiter)); + waiter.shared = &icmp6_shared; + waiter.family = AF_INET6; + waiter.id = htons(3); + waiter.seq = htons(4); + waiter.peer6 = in6addr_loopback; + icmp_shared_register(&waiter); + + memset(reply, 0, sizeof(reply)); + header->icmp6_type = ICMP6_ECHO_REPLY; + header->icmp6_id = htons(3); + header->icmp6_seq = htons(4); + memcpy(reply + sizeof(*header), icmp6_payload, sizeof(icmp6_payload) - 1); + memset(&from, 0, sizeof(from)); + source->sin6_family = AF_INET6; + + /* right message, wrong source */ + thread_mutex_lock(LOCK_ICMP); + icmp_shared_dispatch(AF_INET6, reply, length, &from); + assert_false(waiter.answered); + /* an IPv4 reply never answers an IPv6 request */ + icmp_shared_dispatch(AF_INET, reply, length, &from); + assert_false(waiter.answered); + source->sin6_addr = in6addr_loopback; + icmp_shared_dispatch(AF_INET6, reply, length, &from); + assert_true(waiter.answered); + thread_mutex_unlock(LOCK_ICMP); + + icmp_shared_unregister(&waiter); +} + +/* A signal during the drain must not end it or lose the reply behind it. */ +static void test_shared_drain_retries_after_eintr(void **state) { + icmp_waiter_t mine; + + (void) state; + use_owned_controlled_socket(); + icmp_shared.fd = controlled_pair[0]; + controlled_reply = 1; + recvfrom_eintr_once = 1; + sent_icmp_id = (uint16_t) (getpid() & 0xFFFF); + reply_queue[0] = 21; + reply_queue_len = 1; + make_waiter(&mine, 21); + icmp_shared_register(&mine); + + assert_true(icmp_shared_await(&mine, get_time_as_double() + 1.0)); + /* interrupted, the reply, then the empty queue */ + assert_int_equal(recvfrom_calls, 3); + icmp_shared_unregister(&mine); +} + +static void test_shared_socket_debug_device(void **state) { + host_t host; + ping_t ping; + + (void) state; + use_owned_controlled_socket(); + icmp_shared.fd = controlled_pair[0]; + use_controlled_socket = 0; + socket_failures_remaining = 1; + controlled_reply = 1; + make_host(&host, "127.0.0.1"); + pi_debug_table[0] = host.id; + memset(&ping, 0, sizeof(ping)); + + assert_int_equal(ping_icmp(&host, &ping), HOST_UP); + assert_null(icmp_waiters); +} + +static void make_host6(host_t *host) { + make_host(host, "::1"); +} + +/* A setuid install pings IPv6 devices through the shared ICMPv6 socket. */ +static void test_ipv6_shared_socket_answers(void **state) { + host_t host; + ping_t ping; + + (void) state; + use_owned_controlled_socket(); + icmp6_shared.fd = controlled_pair[0]; + use_controlled_socket = 0; + socket_failures_remaining = 1; + controlled_reply = 1; + controlled_reply_v6 = 1; + track_packet = 1; + packet_size = sizeof(struct icmp6_hdr) + sizeof(icmp6_payload) - 1; + make_host6(&host); + memset(&ping, 0, sizeof(ping)); + + assert_int_equal(ping_icmp_ipv6(&host, &ping), HOST_UP); + assert_string_equal(ping.ping_response, "ICMPv6: Device is Alive"); + assert_int_equal(socket_calls, 1); + assert_int_equal(controlled_socket_closed, 0); + assert_int_equal(packet_released, 1); + assert_null(icmp_waiters); + + /* and as a debug device */ + pi_debug_table[0] = host.id; + socket_failures_remaining = 1; + memset(&ping, 0, sizeof(ping)); + assert_int_equal(ping_icmp_ipv6(&host, &ping), HOST_UP); +} + +static void test_ipv6_shared_socket_timeout(void **state) { + host_t host; + ping_t ping; + + (void) state; + use_owned_controlled_socket(); + icmp6_shared.fd = controlled_pair[0]; + use_controlled_socket = 0; + socket_failures_remaining = 1; + make_host6(&host); + host.availability.timeout = 50; + host.availability.retries = 1; + memset(&ping, 0, sizeof(ping)); + + assert_int_equal(ping_icmp_ipv6(&host, &ping), HOST_DOWN); + assert_string_equal(ping.ping_response, "ICMPv6: Ping timed out"); + assert_null(icmp_waiters); +} + +/* The datagram path matches on sequence and payload, not the rewritten id. */ +static void test_ipv6_datagram_socket_answers(void **state) { + host_t host; + ping_t ping; + + (void) state; + use_owned_controlled_socket(); + controlled_reply = 1; + controlled_reply_v6 = 1; + make_host6(&host); + memset(&ping, 0, sizeof(ping)); + + assert_int_equal(ping_icmp_ipv6(&host, &ping), HOST_UP); + assert_int_equal(controlled_socket_type, SOCK_DGRAM); + assert_int_equal(controlled_socket_cloexec, 1); + assert_int_equal(controlled_socket_closed, 1); +} #ifdef SPINE_HAVE_ICMPV6 static void check_ipv6_peer_matching(int socket_type) { host_t host; @@ -515,8 +1027,9 @@ static void check_ipv6_peer_matching(int socket_type) { use_owned_controlled_socket(); controlled_reply = 1; controlled_reply_v6 = 1; + check_wrong_peer = 1; socket_failures_remaining = socket_type == SOCK_RAW ? 1 : 0; - set.icmp_uses_caps = TRUE; + set.availability.icmp_uses_caps = TRUE; track_packet = 1; packet_size = sizeof(struct icmp6_hdr) + strlen("cacti-monitoring-system"); make_host(&host, "::1"); @@ -552,6 +1065,23 @@ int main(void) { cmocka_unit_test_setup_teardown(test_cached_capability_path_releases_resources, ping_reset, ping_teardown), cmocka_unit_test_setup_teardown(test_socket_retry_can_succeed_after_one_failure, ping_reset, ping_teardown), cmocka_unit_test_setup_teardown(test_socket_retry_does_not_deadlock_on_seteuid, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_shared_socket_answers_without_a_socket_of_its_own, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_shared_socket_timeout_unregisters, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_shared_reply_for_another_waiter_is_handed_over, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_shared_waiter_does_not_read_while_another_thread_does, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_shared_reader_drains_the_queue, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_shared_reader_hands_over_on_leaving, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_per_ping_sockets_are_close_on_exec, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_icmp6_reply_matching, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_open_socket_fcntl_fallback, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_open_shared_sockets, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_open_shared_sockets_failures, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_shared_dispatch_ipv6, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_shared_drain_retries_after_eintr, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_shared_socket_debug_device, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_ipv6_shared_socket_answers, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_ipv6_shared_socket_timeout, ping_reset, ping_teardown), + cmocka_unit_test_setup_teardown(test_ipv6_datagram_socket_answers, ping_reset, ping_teardown), #ifdef SPINE_HAVE_ICMPV6 cmocka_unit_test_setup_teardown(test_ipv6_datagram_rejects_every_wrong_source_octet, ping_reset, ping_teardown), cmocka_unit_test_setup_teardown(test_ipv6_raw_rejects_every_wrong_source_octet, ping_reset, ping_teardown), diff --git a/tests/unit/test_util_paths.c b/tests/unit/test_util_paths.c index 1d8dcf2..5dee831 100644 --- a/tests/unit/test_util_paths.c +++ b/tests/unit/test_util_paths.c @@ -108,7 +108,7 @@ MYSQL_ROW test_mysql_fetch_row(MYSQL_RES *res) { return cells; } -void db_connect(int type, MYSQL *mysql) {} +int db_connect(int type, MYSQL *mysql) { return TRUE; } void db_disconnect(MYSQL *mysql) {} MYSQL_RES *db_query(MYSQL *mysql, int type, const char *query) { (void) mysql; (void) type; @@ -167,6 +167,8 @@ int append_hostrange(char *obuf, size_t capacity, const char *colname) { return int parse_logdest(const char *res, int default_dest) { return 0; } const char *printable_logdest(int dest) { return ""; } void php_close(int php_process) {} +int ping_icmp_open_shared(void) { return 0; } +int ping_icmp_shared_available(void) { return 0; } #include "../../util.c" diff --git a/tests/unit/test_util_strings.c b/tests/unit/test_util_strings.c index 17d8ad9..aeda4df 100644 --- a/tests/unit/test_util_strings.c +++ b/tests/unit/test_util_strings.c @@ -17,7 +17,7 @@ char config_paths[CONFIG_PATHS][BUFSIZE]; int *debug_devices = NULL; /* Mock functions needed by util.c */ -void db_connect(int type, MYSQL *mysql) {} +int db_connect(int type, MYSQL *mysql) { return TRUE; } void db_disconnect(MYSQL *mysql) {} MYSQL_RES *db_query(MYSQL *mysql, int type, const char *query) { return NULL; } void db_free_result(MYSQL_RES *result) {} @@ -27,6 +27,8 @@ int append_hostrange(char *obuf, size_t capacity, const char *colname) { return int parse_logdest(const char *res, int default_dest) { return 0; } const char *printable_logdest(int dest) { return ""; } void php_close(int php_process) {} +int ping_icmp_open_shared(void) { return 0; } +int ping_icmp_shared_available(void) { return 0; } /* Include the actual source file */ #include "../../util.c" diff --git a/util.c b/util.c index fdef09c..cbf6658 100644 --- a/util.c +++ b/util.c @@ -250,6 +250,19 @@ typedef struct setting_cache_entry { static setting_cache_t *settings_cache = NULL; static int settings_cache_count = 0; +/* Settings are read in the main thread before polling starts, so a database + * error here is a configuration failure and ends the process, as it did when + * db_query() exited on its own. An empty or missing result is not an error. */ +static MYSQL_RES *config_query(MYSQL *psql, int mode, const char *query) { + MYSQL_RES *result = db_query(psql, mode, query); + + if (result == NULL && mysql_errno(psql) != 0) { + die("FATAL: Unable to read the Cacti configuration from the database"); + } + + return result; +} + static void settings_cache_free(void) { int i; @@ -276,7 +289,7 @@ static void settings_cache_load(MYSQL *psql, int mode) { settings_cache_free(); - result = db_query(psql, mode, "SELECT SQL_NO_CACHE name, value FROM settings"); + result = config_query(psql, mode, "SELECT SQL_NO_CACHE name, value FROM settings"); if (result == NULL) return; @@ -361,7 +374,7 @@ static char *getsetting(MYSQL *psql, int mode, const char *setting) { spine_snprintf(qstring, sizeof(qstring), "SELECT SQL_NO_CACHE value FROM settings WHERE name = '%s'", setting); - result = db_query(psql, mode, qstring); + result = config_query(psql, mode, qstring); if (result == NULL) return strdup(""); mysql_row = mysql_num_rows(result) > 0 ? mysql_fetch_row(result) : NULL; retval = mysql_row != NULL && mysql_row[0] != NULL ? strdup(mysql_row[0]) : strdup(""); @@ -434,7 +447,7 @@ static char *getpsetting(MYSQL *psql, int mode, const char *setting) { spine_snprintf(qstring, sizeof(qstring), "SELECT SQL_NO_CACHE %s FROM poller WHERE id = '%d'", setting, set.poller.poller_id); - result = db_query(psql, mode, qstring); + result = config_query(psql, mode, qstring); if (result == NULL) return NULL; mysql_row = mysql_num_rows(result) > 0 ? mysql_fetch_row(result) : NULL; retval = mysql_row != NULL && mysql_row[0] != NULL ? strdup(mysql_row[0]) : NULL; @@ -513,7 +526,7 @@ static char *getglobalvariable(MYSQL *psql, int mode, const char *setting) { spine_snprintf(qstring, sizeof(qstring), "SHOW GLOBAL VARIABLES LIKE '%s'", setting); - result = db_query(psql, mode, qstring); + result = config_query(psql, mode, qstring); if (result == NULL) return NULL; mysql_row = mysql_num_rows(result) > 0 ? mysql_fetch_row(result) : NULL; retval = mysql_row != NULL && mysql_row[1] != NULL ? strdup(mysql_row[1]) : NULL; @@ -860,8 +873,8 @@ static void read_php_requirement(MYSQL *mysql) { sqlp += spine_snprintf(sqlp, sizeof(sqlbuf) - (size_t)(sqlp - sqlbuf), " AND poller_id=%i", set.poller.poller_id); spine_snprintf(sqlp, sizeof(sqlbuf) - (size_t)(sqlp - sqlbuf), " LIMIT 1"); - result = db_query(mysql, LOCAL, sqlbuf); - num_rows = spine_count_to_int(mysql_num_rows(result)); + result = config_query(mysql, LOCAL, sqlbuf); + num_rows = result != NULL ? spine_count_to_int(mysql_num_rows(result)) : 0; db_free_result(result); if (num_rows > 0) set.php.php_required = TRUE; @@ -878,8 +891,8 @@ static void read_php_requirement(MYSQL *mysql) { sqlp += spine_snprintf(sqlp, sizeof(sqlbuf) - (size_t)(sqlp - sqlbuf), " AND poller_id=%i", set.poller.poller_id); spine_snprintf(sqlp, sizeof(sqlbuf) - (size_t)(sqlp - sqlbuf), " LIMIT 1"); - result = db_query(mysql, LOCAL, sqlbuf); - num_rows = spine_count_to_int(mysql_num_rows(result)); + result = config_query(mysql, LOCAL, sqlbuf); + num_rows = result != NULL ? spine_count_to_int(mysql_num_rows(result)) : 0; db_free_result(result); if (num_rows > 0) set.php.php_required = TRUE; @@ -996,13 +1009,13 @@ void read_config_options() { int mode; char *res; - db_connect(LOCAL, &mysql); + if (!db_connect(LOCAL, &mysql)) die("FATAL: Unable to connect to the local database"); /* one round trip instead of one per setting */ settings_cache_load(&mysql, LOCAL); if (set.poller.poller_id > 1 && set.poller.mode == REMOTE_ONLINE) { - db_connect(REMOTE, &mysqlr); + if (!db_connect(REMOTE, &mysqlr)) die("FATAL: Unable to connect to the remote database"); mode = REMOTE; } else { mode = LOCAL; @@ -1224,8 +1237,20 @@ bool poller_transfer_status(MYSQL *source, MYSQL *destination) { void poller_push_data_to_main(void) { MYSQL source; MYSQL destination; - db_connect(LOCAL, &source); - db_connect(REMOTE, &destination); + /* A failed connect leaves an initialized handle that still has to be closed. */ + if (!db_connect(LOCAL, &source)) { + db_disconnect(&source); + SPINE_LOG(("ERROR: Collector synchronization skipped; the local database is unavailable. Local rows are retained for retry.")); + set.exit.exit_code = EXIT_FAILURE; + return; + } + if (!db_connect(REMOTE, &destination)) { + db_disconnect(&destination); + db_disconnect(&source); + SPINE_LOG(("ERROR: Collector synchronization skipped; the main server is unavailable. Local rows are retained for retry.")); + set.exit.exit_code = EXIT_FAILURE; + return; + } /* Preserve the supported zero-date and GROUP BY session policies. */ bool configured = db_insert(&source, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'NO_ZERO_DATE', ''))") && db_insert(&source, LOCAL, "SET SESSION sql_mode = (SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY', ''))") && @@ -1298,6 +1323,8 @@ int read_spine_config(const char *file) { char name[BUFSIZE]; char value[BUFSIZE]; char display_file[BUFSIZE]; + int line = 0; + bool line_start = TRUE; strncopy(display_file, file, sizeof(display_file)); spine_sanitize_log_message(display_file); @@ -1311,11 +1338,14 @@ int read_spine_config(const char *file) { fprintf(stdout, "SPINE: Using spine config file [%s]\n", display_file); } while (fgets(buff, sizeof(buff), fp) != NULL) { + /* A line longer than buff arrives in several pieces; count it once. */ + if (line_start) line++; + line_start = strchr(buff, '\n') != NULL; if (buff[0] == '#' || buff[0] == ' ' || buff[0] == '\n') continue; if (sscanf(buff, "%15s %255s", name, value) != 2) continue; + /* -C accepts any path, so report where the line is, never what it holds. */ if (!apply_config_directive(name, value) && !set.console.stderr_notty) { - spine_sanitize_log_message(name); - fprintf(stderr, "WARNING: Unrecognized directive: %s in %s\n", name, display_file); + fprintf(stderr, "WARNING: Unrecognized directive on line %d of %s\n", line, display_file); } } int failed = ferror(fp); @@ -1949,6 +1979,165 @@ int hex2dec(const char *str, unsigned long long *result) { return TRUE; } +#ifdef HAVE_LCAP +/* This patch is adapted (copied) patch for ntpd from Jarno Huuskonen and + * Pekka Savola that was adapted (copied) from a patch by Chris Wings to drop + * root for xntpd. + */ +/* Returns TRUE if CAP_NET_RAW was kept. Exits if the ids cannot be changed. */ +static int drop_root(uid_t server_uid, gid_t server_gid) { + cap_t caps; + int kept; + + /* Runs before logging is configured, so failures go to stderr via die(). */ + if (prctl(PR_SET_KEEPCAPS, 1)) { + die("ERROR: prctl(PR_SET_KEEPCAPS, 1) failed; refusing to run"); + } + + if (setgroups(0, NULL) == -1) { + die("ERROR: setgroups failed; refusing to run"); + } + + if (setegid(server_gid) == -1 || seteuid(server_uid) == -1) { + die("ERROR: setegid/seteuid to uid=%d/gid=%d failed; refusing to run", server_uid, server_gid); + } + + caps = cap_from_text("cap_net_raw=eip"); + kept = caps != NULL && cap_set_proc(caps) == 0; + + if (caps != NULL) { + cap_free(caps); + } + + /* Without the narrowed set, PR_SET_KEEPCAPS would carry every one of + * root's permitted capabilities across the uid change below. Clearing + * it lets the kernel empty the set, and spine continues without ICMP + * unless datagram ICMP is allowed. */ + if (!kept && prctl(PR_SET_KEEPCAPS, 0)) { + die("ERROR: prctl(PR_SET_KEEPCAPS, 0) failed; refusing to run"); + } + + if ( setregid(server_gid, server_gid) == -1 || + setreuid(server_uid, server_uid) == -1 ) { + die("ERROR: setregid/setreuid to uid=%d/gid=%d failed; refusing to run", + server_uid, server_gid); + } + + SPINE_LOG_LOW(("running as uid(%d)/gid(%d) euid(%d)/egid(%d)%s.", + getuid(), getgid(), geteuid(), getegid(), kept ? " with cap_net_raw=eip" : "")); + + return kept; +} +#endif /* HAVE_LCAP */ + +/* Set when a setuid root start could not get raw ICMP before dropping root, + * so checkAsRoot() can say so once logging is configured. */ +static int setuid_icmp_lost = FALSE; + +/*! \fn int privileges_dropped(uid_t uid, gid_t gid) + * \brief confirms root is gone for good rather than set aside + * + * seteuid(0) succeeds while the real or saved uid is still 0, and setegid(0) + * while a group id is, so a failed attempt is the proof. + * + * \return TRUE if every id is the invoking user's and root cannot come back + */ +int privileges_dropped(uid_t uid, gid_t gid) { + #ifdef HAVE_LCAP + cap_t caps; + cap_flag_value_t flag; + cap_value_t value; + int only_net_raw = TRUE; + #endif + + if (uid == 0 || getuid() != uid || geteuid() != uid || getgid() != gid || getegid() != gid) { + return FALSE; + } + + #ifdef HAVE_LCAP + /* A permitted capability could be raised again without any uid change, + * so anything beyond CAP_NET_RAW means root was not given up. */ + caps = cap_get_proc(); + if (caps == NULL) { + return FALSE; + } + + for (value = 0; value < 64; value++) { + if (cap_get_flag(caps, value, CAP_PERMITTED, &flag) == 0 && flag == CAP_SET && value != CAP_NET_RAW) { + only_net_raw = FALSE; + } + } + + cap_free(caps); + + if (!only_net_raw) { + return FALSE; + } + #endif + + if (seteuid(0) == 0) { + return FALSE; + } + + if (gid != 0 && setegid(0) == 0) { + return FALSE; + } + + return TRUE; +} + +/*! \fn void drop_privileges(void) + * \brief gives up setuid root before spine reads any input + * + * Spine is installed setuid root only to open raw ICMP sockets. Options and + * the config file can name files that spine then creates or writes, so all + * input is handled as the invoking user. This runs first in main(), opens + * what ICMP needs while still root, then drops root permanently. A real root user + * (uid 0, for example from cron) keeps running as root, as before; with + * libcap it is narrowed to CAP_NET_RAW, also as before. + * + * Never returns while spine is setuid root: it drops or exits. + */ +void drop_privileges(void) { + uid_t uid = getuid(); + gid_t gid = getgid(); + int icmp_ready; + + if (geteuid() != 0 || uid == 0) { + #ifdef HAVE_LCAP + /* --enable-lcap confines a root run to CAP_NET_RAW. Running on with + * all of root's capabilities would silently undo that, so refuse. */ + if (geteuid() == 0 && !drop_root(uid, gid)) { + die("ERROR: Spine runs as root but could not confine itself to CAP_NET_RAW; refusing to run"); + } + #endif + return; + } + + #ifdef HAVE_LCAP + icmp_ready = drop_root(uid, gid); + #else + icmp_ready = ping_icmp_open_shared(); + + /* With an effective uid of 0, POSIX setgid() and setuid() replace the + * real, effective and saved ids; seteuid() would keep root in reserve. + * Unlike drop_root(), this keeps the supplementary groups: a setuid exec + * does not change them, so they are the caller's own. */ + if (setgid(gid) != 0 || setuid(uid) != 0) { + die("ERROR: Spine is setuid root and could not drop to uid %d/gid %d; refusing to run", (int) uid, (int) gid); + } + #endif + + /* The one case that stops spine: running on with root still reachable. */ + if (!privileges_dropped(uid, gid)) { + die("ERROR: Spine is setuid root and could not drop root permanently; refusing to run"); + } + + /* Missing ICMP is not a reason to stop polling: checkAsRoot() tries + * datagram ICMP and otherwise warns, and ICMP pings fall back to UDP. */ + setuid_icmp_lost = !icmp_ready; +} + int hasCaps(void) { #ifdef HAVE_LCAP cap_t caps; @@ -1963,13 +2152,13 @@ int hasCaps(void) { /* check if cap_net_raw is in effective set */ if (cap_get_flag(caps, CAP_NET_RAW, CAP_EFFECTIVE, &capflag)) { - SPINE_LOG(("ERROR: cap_get_flag for CAP_NET_RAW failed. Attempting the setuid-root ICMP fallback when available.")); + SPINE_LOG(("ERROR: cap_get_flag for CAP_NET_RAW failed. Falling back to unprivileged ICMP where available.")); cap_free(caps); return FALSE; } if (capflag != CAP_SET) { - SPINE_LOG_MEDIUM(("WARNING: Capability CAP_NET_RAW is not set. Attempting the setuid-root ICMP fallback when available.")); + SPINE_LOG_MEDIUM(("WARNING: Capability CAP_NET_RAW is not set. Falling back to unprivileged ICMP where available.")); cap_free(caps); return FALSE; } @@ -2021,42 +2210,46 @@ void checkAsRoot(void) { boolean_t pe = priv_ineffect(PRIV_NET_ICMPACCESS); SPINE_LOG_DEBUG(("DEBUG: Privilege PRIV_NET_ICMPACCESS is: '%s'.", pe != 0 ? "Enabled" : "Disabled")); - set.availability.icmp_avail = pe; + set.availability.icmp_avail = pe || ping_icmp_shared_available(); /* Free the privset */ priv_freeset(privset); free(p); #else + /* Never regains root here: a setuid install gave it up in drop_privileges(). */ if (hasCaps() != TRUE) { SPINE_LOG_DEBUG(("DEBUG: Spine running as %d UID, %d EUID", getuid(), geteuid())); - int ret = seteuid(0); - if (ret != 0) { - SPINE_LOG_DEBUG(("WARNING: Spine NOT able to set effective UID to 0")); - } - if (geteuid() != 0) { + if (ping_icmp_shared_available()) { + SPINE_LOG_DEBUG(("DEBUG: Spine uses the raw ICMP sockets it opened before dropping root.")); + set.availability.icmp_avail = TRUE; + } else if (geteuid() != 0) { int probe; - /* Root is not the only way in. net.ipv4.ping_group_range lists - * the groups allowed to open datagram ICMP sockets, so ask the - * kernel for one instead of assuming ICMP is unavailable */ - probe = socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP); + /* A file capability (setcap cap_net_raw+ep) works without libcap + * support too, so ask the kernel rather than hasCaps(). Root is + * not the only way in either: net.ipv4.ping_group_range lists the + * groups allowed to open datagram ICMP sockets. */ + probe = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP); if (probe != -1) { + close(probe); + SPINE_LOG_DEBUG(("DEBUG: Spine may open raw ICMP sockets.")); + set.availability.icmp_avail = TRUE; + } else if ((probe = socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP)) != -1) { close(probe); SPINE_LOG_DEBUG(("DEBUG: Spine may use unprivileged ICMP sockets.")); set.availability.icmp_avail = TRUE; + } else if (setuid_icmp_lost) { + SPINE_LOG(("WARNING: Spine is setuid root but could not get raw ICMP access before dropping root, and net.ipv4.ping_group_range does not cover this user. ICMP pings fall back to UDP; SNMP polling continues.")); + set.availability.icmp_avail = FALSE; } else { - SPINE_LOG_DEBUG(("WARNING: Spine NOT running as root. This is required if using ICMP unless net.ipv4.ping_group_range covers this user. Please run \"chown root:root spine;chmod u+s spine\" to resolve.")); + SPINE_LOG_DEBUG(("WARNING: Spine has no ICMP access. Install it setuid root (chown root:root spine; chmod u+s spine), grant it CAP_NET_RAW (setcap cap_net_raw+ep spine), or widen net.ipv4.ping_group_range to cover this user.")); set.availability.icmp_avail = FALSE; } } else { SPINE_LOG_DEBUG(("DEBUG: Spine is running as root.")); set.availability.icmp_avail = TRUE; - - if (seteuid(getuid()) == -1) { - SPINE_LOG_DEBUG(("WARNING: Spine unable to drop from root to local user.")); - } } } else { SPINE_LOG_DEBUG(("DEBUG: Spine has cap_net_raw capability.")); @@ -2096,7 +2289,7 @@ static int parse_cacti_version(const char *value) { int get_cacti_version(MYSQL *psql, int mode) { assert(psql != NULL); - MYSQL_RES *result = db_query(psql, mode, "SELECT cacti FROM version LIMIT 1"); + MYSQL_RES *result = config_query(psql, mode, "SELECT cacti FROM version LIMIT 1"); if (result == NULL) return 0; MYSQL_ROW row = mysql_fetch_row(result); int version = row != NULL && row[0] != NULL ? parse_cacti_version(row[0]) : 0; diff --git a/util.h b/util.h index 95112c1..9a23d26 100644 --- a/util.h +++ b/util.h @@ -105,6 +105,8 @@ extern double get_time_as_double(void); /* function to check to see if program has capability to use raw socket with out uid = 0 */ +extern void drop_privileges(void); +extern int privileges_dropped(uid_t uid, gid_t gid); extern int hasCaps(void); /* see if we can do things as root */