#!/usr/bin/php 0) { print 'The Thold Daemon is still running' . PHP_EOL; exit(1); } else { unregister_process('thold', 'parent', 0); register_process_start('thold', 'parent', 0, $timeout); } } } // do not run the thold daemon on the remote server in central storage mode if (read_config_option('remote_storage_method') != 1 && $config['poller_id'] > 1) { print 'In Central Storage Mode, the thold_daemon only runs on the main data collector.' . PHP_EOL; exit(1); } print 'Starting Thold Daemon ... '; if (!$foreground) { if (function_exists('pcntl_fork')) { // Close the database connection db_close(); // Fork the current process to daemonize $pid = pcntl_fork(); if ($pid == -1) { // oha ... something went wrong :( print '[FAILED]' . PHP_EOL; return false; } if ($pid == 0) { // We are the child reconnect db_connect_real($database_hostname, $database_username, $database_password, $database_default, $database_type, $database_port, $database_retries, $database_ssl, $database_ssl_key, $database_ssl_cert, $database_ssl_ca); } else { cacti_log('NOTE: Thold Daemon Started on ' . gethostname(), false, 'THOLD'); // We are the parent, output and exit print '[OK]' . PHP_EOL; exit; } } else { // Windows.... awesome! But no worries print '[WARNING] This system does not support forking.' . PHP_EOL; } } else { print '[NOTE] The Thold Daemon is running in foreground mode.' . PHP_EOL; } sleep(2); $processes = read_config_option('thold_max_concurrent_processes'); thold_truncate_daemon_data(); thold_prime_distribution($processes); thold_daemon_debug('Forking Thold Daemon Child Processes'); for ($i = 1; $i <= $processes; $i++) { thold_launch_worker($i); } $prev_running = false; $start_daemon_items = 0; $counter = 0; while (true) { if (db_check_reconnect()) { $counter++; // force the check for the daemon debug $daemon_debug = read_config_option('thold_daemon_debug', true); if ($counter == 1) { thold_daemon_debug('Thold Thread Watchdog Start.'); set_config_option('thold_daemon_heartbeat', microtime(true)); } $running = thold_poller_running(); if ($running && !$prev_running) { $start = microtime(true); thold_daemon_debug('Detected Cacti Poller Start at ' . date('Y-m-d H:i:s')); $start_items = db_fetch_cell('SELECT COUNT(*) FROM plugin_thold_daemon_data'); $prev_running = true; } elseif (!$running && $prev_running) { $end_items = db_fetch_cell('SELECT COUNT(*) FROM plugin_thold_daemon_data'); $prev_running = false; $tholds = db_fetch_cell('SELECT COUNT(*) FROM thold_data WHERE thold_enabled = "on"'); thold_daemon_debug(sprintf('Detected Cacti Poller End at %s.', date('Y-m-d H:i:s'))); thold_daemon_debug(sprintf('TotalTholds:%u StartingTholds:%u, EndingTholds:%u', $tholds, $start_items, $end_items)); $end = microtime(true); cacti_log(sprintf('THOLD DAEMON STATS: TotalTime:%3.2f TotalTholds:%u StartingItems:%u EndingItems:%u', $end - $start, $tholds, $start_items, $end_items), false, 'SYSTEM'); } else { $prev_running = $running; } sleep(1); if ($counter == 10) { $counter = 0; $new_processes = read_config_option('thold_max_concurrent_processes', true); if ($new_processes != $processes) { thold_prime_distribution($new_processes, true); } else { thold_prime_distribution($new_processes); } thold_heartbeat_processes($processes, $new_processes); $processes = $new_processes; thold_daemon_debug('Thold Thread Watchdog End. Processed heartbeat.'); heartbeat_process('thold', 'parent', 0); } } else { thold_daemon_debug('WARNING: No database connection. Sleeping for 60 seconds.'); sleep(60); } } /** * Truncates the plugin_thold_daemon_data table, clearing all queued * per-thread threshold work items. * * @return void */ function thold_truncate_daemon_data() { thold_daemon_debug('Truncating historical Threshold Daemon Data'); db_execute('TRUNCATE TABLE plugin_thold_daemon_data'); } /** * Checks whether a Cacti poller run is currently in progress (has an * unfinished poller_time entry). * * @return int The number of poller_time rows with no end_time set (0 * if none, meaning no poller run is currently active). */ function thold_poller_running() { return db_fetch_cell('SELECT COUNT(*) FROM poller_time WHERE end_time = "0000-00-00"'); } /** * sig_handler - provides a generic means to catch exceptions to the Cacti log. * * @param $signo - (int) the signal that was thrown by the interface. * * @returns - null */ function sig_handler($signo) { global $config; switch ($signo) { case SIGTERM: case SIGINT: // kill any child processes $processes = db_fetch_assoc('SELECT * FROM processes WHERE tasktype = "thold" AND taskname = "child"'); if (cacti_sizeof($processes)) { foreach ($processes as $p) { thold_daemon_debug(sprintf('Killing Child Process with the pid of %u', $p['pid'])); posix_kill($p['pid'], SIGTERM); } } thold_cacti_log('WARNING: Thold Daemon Parent Process with PID[' . getmypid() . '] terminated by user', 0); unregister_process('thold', 'parent', 0); exit; default: // ignore all other signals } } /** * Launches a single Thold worker process (thold_process.php) for the * given thread number as a detached background process. * * @param int $thread The worker thread number to launch. * * @return void * * @global array $config Cacti global configuration array; used to * build the worker script's command line. * @global bool $debug Whether debug output is enabled; passed * through to the worker via --debug. */ function thold_launch_worker($thread) { global $config, $debug; $path_php = read_config_option('path_php_binary'); $process = $config['base_path'] . '/plugins/thold/thold_process.php ' . ' --thread=' . $thread . ($debug ? ' --debug' : '') . ' > /dev/null'; thold_daemon_debug('Starting Process: ' . $path_php . ' ' . $process); exec_background($path_php, $process); } /** * Monitors the registered Thold worker child processes, detecting and * cleaning up crashed processes, and launching new worker processes * (up to the configured concurrency) to replace missing ones or reach * the desired total, so that exactly $new_processes workers stay * running. * * @param array $processes The currently known worker process * registrations (unused directly; current * state is re-fetched from the database). * @param int $new_processes The desired total number of concurrent * worker processes/threads. * * @return void * * @global array $config Cacti global configuration array; used when * launching new worker processes. */ function thold_heartbeat_processes($processes, $new_processes) { global $config; $procs = db_fetch_assoc('SELECT * FROM processes WHERE tasktype = "thold" AND taskname = "child" ORDER BY taskid DESC'); $running_processes = cacti_sizeof($procs); // Check for a crashed process $process_num = -1; foreach ($procs as $id => $p) { // Check for crashed processes first if ($process_num != -1) { if ($process_num - 1 != $p['taskid']) { thold_daemon_debug(sprintf('WARNING: Detected Crashed Thold Thread. Relaunching Crashed Thread %s', $process_num - 1)); thold_launch_worker($process_num - 1); $running_processes++; } } else { // Check for hung processes next $lastupdate = strtotime($p['last_update']); $now = time(); if ($lastupdate + 120 < $now) { thold_daemon_debug(sprintf('WARNING: Detected Hung Thold Thread. Killing/Relaunching Hung Thread %s', $p['taskid'])); posix_kill($p['pid'], SIGTERM); thold_launch_worker($p['taskid']); } } $process_num = $p['taskid']; } foreach ($procs as $id => $p) { if (function_exists('posix_getpgid')) { $running = posix_getpgid($p['pid']); } elseif (function_exists('posix_kill')) { $running = posix_kill($p['pid'], 0); } if (!$running) { thold_daemon_debug(sprintf('WARNING: Thold Daemon Child[%s] Died!', $p['pid'])); cacti_log(sprintf('WARNING: Thold Daemon Child[%s] Died!', $p['pid']), false, 'THOLD'); $running_processes--; unset($procs[$id]); } } if ($running_processes != $new_processes) { if ($running_processes > $new_processes) { thold_daemon_debug(sprintf('Thold Thread Detected Process Count Change. Reducing Process Count by %s', $running_processes - $new_processes)); foreach ($procs as $id => $p) { posix_kill($p['pid'], SIGTERM); $running_processes--; if ($running_processes == $new_processes) { break; } } } else { thold_daemon_debug(sprintf('Thold Thread Detected Process Count Change. Increasing Process Count by %s', $new_processes - $running_processes)); while ($running_processes < $new_processes) { $running_processes++; thold_launch_worker($running_processes); } } } } /** * (Re)assigns each threshold's thread_id so that thresholds are evenly * distributed across the configured number of worker threads, grouped * by host (so all thresholds for a given host are always processed by * the same thread); only assigns thread ids to thresholds that * currently have none (thread_id = 0), unless a full rebalance is * forced. * * @param int $processes The current/target number of worker * threads/processes to distribute across. * @param bool $truncate Whether to force a full rebalance by * resetting all thread assignments first, even * if the thread count appears unchanged (default * false). * * @return void */ function thold_prime_distribution($processes, $truncate = false) { thold_daemon_debug('Rebalancing Thread Allocation by Device'); $seen_processes = db_fetch_cell('SELECT COUNT(DISTINCT thread_id) FROM thold_data'); if ($truncate || $seen_processes != $processes) { db_execute('UPDATE thold_data SET thread_id = 0'); } $not_set_threads = db_fetch_cell('SELECT COUNT(id) FROM thold_data WHERE thread_id = 0'); if ($not_set_threads > 0) { // Get the current distribution of threads $threads = array_rekey( db_fetch_assoc('SELECT thread_id, host_id FROM thold_data WHERE thread_id > 0'), 'host_id', 'thread_id' ); $tholds = db_fetch_assoc('SELECT id, host_id FROM thold_data WHERE thread_id = 0'); $thread_num = 1; foreach ($tholds as $t) { if (!isset($threads[$t['host_id']])) { $threads[$t['host_id']] = $thread_num; $thread_num++; } if ($thread_num > $processes) { $thread_num = 1; } } foreach ($threads as $host_id => $thread) { db_execute_prepared('UPDATE thold_data SET thread_id = ? WHERE host_id = ?', [$thread, $host_id]); } } thold_daemon_debug('Thread Rebalancing Allocation by Device Completed'); } /** * Prints a timestamped debug message directly to stdout when global * debug mode or the thold_daemon_debug setting is enabled. * * @param string $string The message to print. * * @return void * * @global bool $debug Whether debug output is enabled. */ function thold_daemon_debug($string) { global $debug; // Get the cached value $daemon_debug = read_config_option('thold_daemon_debug'); if ($debug || $daemon_debug) { $output = date('Y-m-d H:i:s') . ' DEBUG: ' . trim($string); print $output . PHP_EOL; } } /** * Prints this script's name, version, and copyright banner to stdout. * * @return void * * @global array $config Cacti global configuration array; used to * locate setup.php when plugin_thold_version() * isn't already loaded. */ function display_version() { global $config; if (!function_exists('plugin_thold_version')) { include_once($config['base_path'] . '/plugins/thold/setup.php'); } $info = plugin_thold_version(); print 'Threshold Daemon, Version ' . $info['version'] . ', ' . COPYRIGHT_YEARS . PHP_EOL; } /** * Prints this script's version banner followed by its command-line * usage instructions to stdout. * * @return void */ function display_help() { display_version(); print PHP_EOL . 'usage: thold_daemon.php [ --foreground | -f ] [ --debug ]' . PHP_EOL . PHP_EOL; print 'The Threshold Daemon processor for the Thold Plugin.' . PHP_EOL; }