", ?) AS data_source_path, GROUP_CONCAT(DISTINCT dtr.data_source_name) AS data_source_names, GROUP_CONCAT(DISTINCT dtd.rrd_step) AS rrd_step, GROUP_CONCAT(DISTINCT dtr.rrd_heartbeat) AS rrd_heartbeat, dsp.heartbeat AS profile_heartbeat, dsp.step AS profile_step FROM data_template_data AS dtd INNER JOIN data_template_rrd AS dtr ON dtr.local_data_id = dtd.local_data_id INNER JOIN graph_templates_item AS gti ON dtr.id = gti.task_item_id INNER JOIN data_source_profiles AS dsp ON dtd.data_source_profile_id = dsp.id LEFT JOIN graph_local AS gl ON gl.id = gti.local_graph_id LEFT JOIN host as h ON h.id = gl.host_id WHERE dtd.local_data_id != 0 AND h.disabled != "on" GROUP BY dtd.local_data_id', [CACTI_PATH_RRA]); if ($max_threads == 1) { $newrows[0] = $rows; return $rows; } else { $newrows = array_chunk($rows, (int) ceil(cacti_sizeof($rows) / $max_threads)); if (isset($newrows[$thread_id - 1])) { return $newrows[$thread_id - 1]; } else { return []; } } } /** * rrdcheck_debug - this simple routine prints a standard message to the console * when running in debug mode. * * @param string $message The message to display * * @return void */ function rrdcheck_debug(string $message) : void { global $debug; if ($debug) { print 'rrdcheck: ' . $message . PHP_EOL; } } /** * do_rrdcheck - this routine is a generic routine that * check all RRDfiles (missing files, ...) and stored information (NaN) * * @param int $thread_id The rrdcheck parallel thread id * * @return void */ function do_rrdcheck(int $thread_id = 1) : void { global $type; global $total_user, $total_system, $total_real, $total_dsses; global $user_time, $system_time, $real_time, $rrd_files; $user_time = 0; $system_time = 0; $real_time = 0; $dsses = 0; $poller_interval = read_config_option('poller_interval'); rrdcheck_debug(sprintf('Processing %s for Thread', $thread_id)); $max_threads = read_config_option('rrdcheck_parallel'); if (empty($max_threads)) { $max_threads = 1; set_config_option('rrdcheck_parallel', '1'); } $profiles = array_rekey( db_fetch_assoc('SELECT step, data_source_profile_id AS id, MIN(steps) AS steps, `rows` FROM data_source_profiles AS dsp INNER JOIN data_source_profiles_rra AS dspr ON dsp.id = dspr.data_source_profile_id GROUP BY data_source_profile_id'), 'id', ['step', 'steps', 'rows'] ); $rrdfiles = get_rrdfiles($thread_id, $max_threads); $stats = []; $rrd_files += cacti_sizeof($rrdfiles); $use_proxy = (read_config_option('storage_location') ? true : false); if ($use_proxy) { $rrdtool_pipe = rrd_init(false); } else { $process_pipes = rrdcheck_rrdtool_init(); $process = $process_pipes[0]; $pipes = $process_pipes[1]; // rrdcheck_rrdtool_init() returns [false, false] when RRDtool could not be // spawned; without live pipes the execute path would fwrite() to null and // crash the poller, so bail out of this run gracefully if (!is_resource($process)) { cacti_log('ERROR: rrdcheck unable to obtain RRDtool process, skipping run', false, 'RRDCHECK'); return; } } if (cacti_sizeof($rrdfiles)) { $now = time(); foreach ($rrdfiles as $rrdval) { $local_data_id = $rrdval['local_data_id']; $data_sources = []; // Rekey the Data Sources key $sources = explode(',', $rrdval['data_source_names']); foreach ($sources as $s) { $data_sources[$s] = $s; } // The first RRA may have less than 24 hours of samples $duration = $profiles[$rrdval['data_source_profile_id']]['step'] * ($profiles[$rrdval['data_source_profile_id']]['rows'] - 1); $step = $profiles[$rrdval['data_source_profile_id']]['step']; $end = $now; $start = $end - $duration; $file = $rrdval['data_source_path']; if ($use_proxy) { $file_exists = rrdtool_execute('file_exists ' . cacti_escapeshellarg($file), true, RRDTOOL_OUTPUT_BOOLEAN, false, 'RRDCHECK'); } else { clearstatcache(); $file_exists = file_exists($file); } // don't attempt to get information if the file does not exist if ($file_exists) { if (!is_resource_writable($file)) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?,NOW(),?)', [ $local_data_id, "RRDfile is not writable - $file" ] ); } if (time() > (filemtime($file) + 3600)) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?,NOW(),?)', [ $local_data_id, "RRDfile modify time older than hour - $file" ] ); } if ($rrdval['profile_step'] != $rrdval['rrd_step']) { // Fix the RRD Data Template Data Step db_execute_prepared('UPDATE data_template_data SET rrd_step = ? WHERE local_data_id = ?', [ $rrdval['profile_step'], $local_data_id ] ); } if ($rrdval['rrd_heartbeat'] != $rrdval['profile_heartbeat']) { // Fix the RRD Data Template Data Step db_execute_prepared('UPDATE data_template_rrd SET rrd_heartbeat = ? WHERE local_data_id = ?', [$rrdval['profile_heartbeat'], $rrdval['local_data_id']]); } if ($use_proxy) { $output = rrdtool_execute('info ' . cacti_escapeshellarg($file), false, RRDTOOL_OUTPUT_STDOUT, false, 'RRDCHECK'); } else { $output = rrdcheck_rrdtool_execute(['info', $file], $pipes); } $matches = []; $rrd_info = []; $output = explode("\n", $output); $last_update = false; $rrd_step = false; // Process the RRDfile information into an array foreach ($output as $line) { $line = trim($line); if ($line == '') { continue; } if (!$last_update) { if (preg_match('/^last_update = (\S+)$/', $line, $matches)) { $last_update = $matches[1]; } } if (!$rrd_step) { if (preg_match('/^step = (\S+)$/', $line, $matches)) { $rrd_step = $matches[1]; } } if (preg_match('/^ds\[(\S+)\]\.(\S+) = (\S+)$/', $line, $matches)) { $rrd_info[$matches[1]][$matches[2]] = trim($matches[3], '"'); } } if ($rrd_step != $rrdval['profile_step']) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?, NOW(), ?)', [ $local_data_id, "The RRDfile step of {$step} does not match the Data Source Profile step {$rrdval['profile_step']}. File '$file'." ] ); } if ($last_update < (time() - 3600)) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?, NOW(), ?)', [ $local_data_id, "Last update value in RRDfile is older than 1 hour. File '$file'." ] ); } $cacti_ds = rrdtool_function_info_from_ds($local_data_id); $cacti_ds = isset($cacti_ds['ds']) ? $cacti_ds['ds'] : []; foreach (rrdcheck_ds_findings($data_sources, $rrd_info, $cacti_ds, $file) as $finding) { $suffix = $finding['spec'] !== '' ? ' ' . rrdcheck_tune_rrdfile($file, $finding['spec'], $local_data_id, $use_proxy, $pipes) . '.' : ''; db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?, NOW(), ?)', [$local_data_id, $finding['message'] . $suffix]); } $output = ''; $matches = []; foreach ($rrd_info as $info_key => $info_array) { if (!isset($data_sources[$info_key])) { continue; } if ($info_array['minimal_heartbeat'] <= $poller_interval) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?, NOW(), ?)', [ $local_data_id, "The RRDfile Minimal Heart for Data Source '{$info_key}' is lower than polling interval. This will causes gaps in Graphs. File '$file.'" ] ); } if ($info_array['minimal_heartbeat'] < $rrdval['profile_heartbeat']) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?, NOW(), ?)', [ $local_data_id, "The RRDfile minimal heartbeat for Data Source '{$info_key}' should be '{$rrdval['profile_heartbeat']}' and is currently '{$info_array['minimal_heartbeat']}'. File '$file'." ] ); } } $data_source_names = []; // test stale data $pstart = $start - $step; $pend = $end - $step; $one_hour_limit = ($duration - 3600) / $step; if ($use_proxy) { $info_array = rrdtool_execute('fetch ' . cacti_escapeshellarg($file) . " LAST -s $pstart -e $pend", false, RRDTOOL_OUTPUT_STDOUT, false, 'RRDCHECK'); } else { $info_array = rrdcheck_rrdtool_execute(['fetch', $file, 'LAST', '-s', $pstart, '-e', $pend], $pipes); } // don't do anything if RRDfile did not return data $info_array = explode("\n", $info_array); if (cacti_sizeof($info_array)) { $first = true; $lines_24 = 0; $lines_1 = 0; $nan_24 = []; $nan_1 = []; foreach ($info_array as $line) { $line = trim($line); // remove line - OK u:0.03 s:0.12 r:0.33 if (str_starts_with($line, 'OK')) { continue; } if ($line == '') { continue; } if ($first) { // get the data source names $data_source_names = preg_split('/\s+/', $line) ?: []; foreach (($data_source_names ?: []) as $index => $name) { $nan_24[$index] = 0; $nan_1[$index] = 0; } $dsses += cacti_sizeof($data_source_names); $first = false; } else { $parts = explode(':', $line); $data = explode(' ', trim($parts[1])); foreach ($data as $index=>$number) { if ($index == 0) { // only onetime for each row $lines_24++; if ($lines_24 > $one_hour_limit) { // last hour $lines_1++; } } if (cacti_strtolower($number) == 'nan' || cacti_strtolower($number) == '-nan') { $nan_24[$index]++; if ($lines_24 > $one_hour_limit) { $nan_1[$index]++; } } } } } $notified = false; // 24 hour statistics foreach ($nan_24 as $index => $count) { if ($lines_24 > 0) { $ratio = $count / $lines_24; } else { $ratio = 0; db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?,NOW(),?)', [ $local_data_id, "No data returned, maybe corrupted Data Source '{$data_source_names[$index]}'. File '$file'." ] ); } if ($ratio == 1) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?,NOW(),?)', [ $local_data_id, "Stale values for last 24 hours, Data Source '{$data_source_names[$index]}'. File '$file'." ] ); $notified = true; } elseif ($ratio > 0.5) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?,NOW(),?)', [ $local_data_id, "More than 50% ($count/$lines_24) values are NaN in last 24 hours, Data Source '{$data_source_names[$index]}'. File '$file'." ] ); } } // 1 hour statistics foreach ($nan_1 as $index=>$count) { if ($notified) { // 24hour notified, skipping continue; } if ($lines_1 > 0) { $ratio = $count / $lines_1; } else { $ratio = 0; db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?,NOW(),?)', [ $local_data_id, "No data returned, maybe corrupted Data Source '{$data_source_names[$index]}'. File '$file'." ] ); } if ($ratio == 1) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?,NOW(),?)', [ $local_data_id, "Stale values for last hour, Data Source '{$data_source_names[$index]}'. File '$file'." ] ); } elseif ($ratio > 0.5) { db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?,NOW(),?)', [ $local_data_id, "More than 50% ($count/$lines_1) values are NaN in last hour, Data Source '{$data_source_names[$index]}'. File '$file'." ] ); } } } else { cacti_log("WARNING: RRDcheck - no rrd data returned - '$file'", false, 'RRDCHECK'); } } else { // rrdfile does not exist db_execute_prepared('INSERT INTO rrdcheck (local_data_id, test_date, message) VALUES (?,NOW(),?)', [$local_data_id, "RRDfile does not exist - '$file'"]); } } } if ($use_proxy) { rrd_close($rrdtool_pipe); } else { rrdcheck_rrdtool_close($process); } if (!empty($type)) { $total_user += $user_time; $total_system += $system_time; $total_real += $real_time; $total_dsses += $dsses; set_config_option('rrdcheck_rrd_system_' . $type . '_' . $thread_id, $total_system); set_config_option('rrdcheck_rrd_user_' . $type . '_' . $thread_id, $total_user); set_config_option('rrdcheck_rrd_real_' . $type . '_' . $thread_id, $total_real); set_config_option('rrdcheck_total_rrds_' . $type . '_' . $thread_id, $rrd_files); set_config_option('rrdcheck_total_dsses_' . $type . '_' . $thread_id, $total_dsses); } } /** * rrdcheck_ds_findings - compares the data sources the Graph requires against those present in an * RRDfile and returns the findings, each with the rrdtool 'tune' specification (when applicable) * needed to reconcile the difference. A data source the Graph requires but the RRDfile lacks is * added with 'DS:'; a data source the RRDfile carries but the Graph no longer requires is removed * with 'DEL:'. * * @param array $data_sources The data source names the Graph requires, keyed by name * @param array $rrd_info The data sources present in the RRDfile, keyed by name * @param array $cacti_ds The Cacti data source definitions, keyed by name * @param string $file The RRDfile path * * @return array A list of ['message' => string, 'spec' => string] findings */ function rrdcheck_ds_findings(array $data_sources, array $rrd_info, array $cacti_ds, string $file) : array { $findings = []; $db_count = cacti_sizeof($data_sources); $rrd_count = cacti_sizeof($rrd_info); if ($db_count > $rrd_count) { $findings[] = ['message' => "The Graph requires $db_count Data Sources but the RRDfile only includes $rrd_count. File '$file'.", 'spec' => '']; } elseif ($db_count < $rrd_count) { $findings[] = ['message' => "The Graph requires $db_count Data Sources but the RRDfile includes $rrd_count. File '$file'.", 'spec' => '']; } foreach ($rrd_info as $name => $info) { if (!isset($data_sources[$name])) { $spec = 'DEL:' . $name; $findings[] = ['message' => "The RRDfile includes Data Source '$name' which the Graph no longer requires. RRDtune: tune $spec.", 'spec' => $spec]; } } foreach ($data_sources as $name) { if ($name === '') { $findings[] = ['message' => "Database Data Source name is empty. File '$file'.", 'spec' => '']; continue; } if (!array_key_exists($name, $rrd_info)) { if (isset($cacti_ds[$name])) { $ds = $cacti_ds[$name]; $spec = rrdtool_tune_ds_add_spec($name, $ds['type'], $ds['minimal_heartbeat'], $ds['min'], $ds['max']); $findings[] = ['message' => "The Graph requires Data Source '$name' which is missing from the RRDfile. RRDtune: tune $spec.", 'spec' => $spec]; } else { $findings[] = ['message' => "Data Source name '$name' exists in the Database, but not in RRDfile. File '$file'.", 'spec' => '']; } } } return $findings; } /** * rrdcheck_valid_tune_spec - validate an rrdtool 'tune' specification against RRDtool's documented * data source grammar before it is sent to RRDtool. Per rrdcreate, ds-name is 1 to 19 characters * from [a-zA-Z0-9_], DST is one of the known data source types, heartbeat is a positive integer and * min/max are either a number or 'U'. * * @param string $spec The tune specification to validate * * @return bool True when the specification matches a known-good DS:/DEL: form */ function rrdcheck_valid_tune_spec(string $spec) : bool { if (preg_match('/^DEL:[a-zA-Z0-9_]{1,19}$/', $spec)) { return true; } return (bool) preg_match('/^DS:[a-zA-Z0-9_]{1,19}:(GAUGE|COUNTER|DERIVE|DCOUNTER|DDERIVE|ABSOLUTE):[0-9]+:(U|-?[0-9]+(\.[0-9]+)?):(U|-?[0-9]+(\.[0-9]+)?)$/', $spec); } /** * rrdcheck_tune_rrdfile - applies an rrdtool 'tune' specification to an RRDfile from the RRDchecker * batch process. The specification is validated against RRDtool's grammar first; for local storage * the RRDfile must resolve inside the RRA directory and be writable, while the storage proxy * enforces remote containment and permissions. The full command is written to the Cacti log. * * @param string $file The fully qualified RRDfile path * @param string $spec The rrdtool tune specification to apply * @param int $local_data_id The local data id being checked * @param bool $use_proxy Whether rrdtool runs through the storage proxy * @param array $pipes The local rrdtool pipes when not using the proxy * * @return string A short phrase describing whether the tune ran */ function rrdcheck_tune_rrdfile(string $file, string $spec, int $local_data_id, bool $use_proxy, &$pipes) : string { $command = read_config_option('path_rrdtool') . ' tune ' . cacti_escapeshellarg($file) . ' ' . cacti_escapeshellarg($spec); if (!rrdcheck_valid_tune_spec($spec)) { cacti_log("ERROR: RRDcheck - refusing malformed tune specification - DS[$local_data_id] Spec[$spec]", false, 'RRDCHECK'); return 'the tune specification was not valid so it was not run'; } if (!$use_proxy) { if (!data_source_path_within_rra($file)) { cacti_log("ERROR: RRDcheck - RRDfile resolves outside the RRA directory, tune not run - DS[$local_data_id] File[$file]", false, 'RRDCHECK'); return 'the RRDfile is outside the RRA directory so the tune was not run'; } if (!is_resource_writable($file)) { cacti_log("WARNING: RRDcheck - RRDfile not writable, tune not run - DS[$local_data_id] Command[$command]", false, 'RRDCHECK'); return 'the RRDfile is not writable so the tune was not run automatically'; } } if ($use_proxy) { $output = rrdtool_execute('tune ' . cacti_escapeshellarg($file) . ' ' . cacti_escapeshellarg($spec), false, RRDTOOL_OUTPUT_RETURN_STDERR, false, 'RRDCHECK'); $failed = ($output === false || trim((string) $output) !== ''); } else { $output = rrdcheck_rrdtool_execute(['tune', $file, $spec], $pipes); $failed = ($output === null || $output === false || strpos((string) $output, 'ERROR') !== false || strpos((string) $output, 'OK') === false); } if ($failed) { cacti_log("ERROR: RRDcheck - tune failed - DS[$local_data_id] Command[$command] Output[" . trim((string) $output) . ']', false, 'RRDCHECK'); return 'the tune command failed, see the Cacti log for details'; } cacti_log("RRDcheck - tune ran successfully - DS[$local_data_id] Command[$command]", false, 'RRDCHECK'); return 'the tune command was run automatically to correct the issue'; } /** * rrdcheck_log_statistics - provides generic timing message to both the Cacti log and the settings * table so that the statistics can be graphed as well. * * @param string $type The type of statistics to log, either 'HOURLY', 'BOOST'. * * @return void */ function rrdcheck_log_statistics(string $type) : void { global $start; rrdcheck_debug($type); if ($type == 'BOOST') { $sub_type = 'bchild'; } else { $sub_type = ''; } // take time and log performance data $end = microtime(true); if ($sub_type != '') { $rrd_user = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_rrd_user_%' . $sub_type . '%']); $rrd_system = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_rrd_system_%' . $sub_type . '%']); $rrd_real = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_rrd_real_%' . $sub_type . '%']); $rrd_files = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_total_rrds_%' . $sub_type . '%']); $dsses = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_total_dsses_%' . $sub_type . '%']); $processes = read_config_option('rrdcheck_parallel'); $cacti_stats = sprintf('Time:%01.2f Type:%s Threads:%s RRDfiles:%s DSSes:%s RRDUser:%01.2f RRDSystem:%01.2f RRDReal:%01.2f', $end - $start, $type, $processes, $rrd_files, $dsses, $rrd_user, $rrd_system, $rrd_real); db_execute("DELETE FROM settings WHERE name LIKE 'rrdcheck_rrd_%$sub_type%' OR name LIKE 'rrdcheck_total_rrds_%$sub_type%' OR name LIKE 'rrdcheck_total_dsses_%$sub_type%'"); } else { $cacti_stats = sprintf('Time:%01.2f Type:%s', $end - $start, $type); } // take time and log performance data $start = microtime(true); // log to the database set_config_option('stats_rrdcheck_' . $type, $cacti_stats); // log to the logfile cacti_log('RRDCHECK STATS: ' . $cacti_stats , true, 'SYSTEM'); } /** * rrdcheck_log_child_stats - logs rrdcheck child process information * * @param string $type The type of child, MAJOR, DAILY, BOOST * @param int $thread_id The parallel thread id * @param float $total_time The total time to collect date * * @return void */ function rrdcheck_log_child_stats(string $type, int $thread_id, float $total_time) : void { $rrd_user = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_rrd_user_%' . $type . '_' . $thread_id . '%']); $rrd_system = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_rrd_system_%' . $type . '_' . $thread_id . '%']); $rrd_real = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_rrd_real_%' . $type . '_' . $thread_id . '%']); $rrd_files = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_total_rrds_%' . $type . '_' . $thread_id . '%']); $dsses = db_fetch_cell_prepared('SELECT SUM(value) FROM settings WHERE name LIKE ?', ['rrdcheck_total_dsses_%' . $type . '_' . $thread_id . '%']); $cacti_stats = sprintf('Time:%01.2f Type:%s ProcessNumber:%s RRDfiles:%s DSSes:%s RRDUser:%01.2f RRDSystem:%01.2f RRDReal:%01.2f', $total_time, cacti_strtoupper($type), $thread_id, $rrd_files, $dsses, $rrd_user, $rrd_system, $rrd_real); cacti_log('RRDCHECK CHILD STATS: ' . $cacti_stats, true, 'SYSTEM'); } /** * rrdcheck_error_handler - this routine logs all PHP error transactions * to make sure they are properly logged. * * @param int $errno The errornum reported by the system * @param string $errmsg The error message provides by the error * @param string $filename The filename that encountered the error * @param int $linenum The line number where the error occurred * @param array $vars The current state of PHP variables. * * @return bool Always returns true for some reason */ function rrdcheck_error_handler(int $errno, string $errmsg, string $filename, int $linenum, array $vars = []) : bool { if (read_config_option('log_verbosity') >= POLLER_VERBOSITY_DEBUG) { // define all error types $errortype = [ E_ERROR => 'Error', E_WARNING => 'Warning', E_PARSE => 'Parsing Error', E_NOTICE => 'Notice', E_CORE_ERROR => 'Core Error', E_CORE_WARNING => 'Core Warning', E_COMPILE_ERROR => 'Compile Error', E_COMPILE_WARNING => 'Compile Warning', E_USER_ERROR => 'User Error', E_USER_WARNING => 'User Warning', E_USER_NOTICE => 'User Notice', E_RECOVERABLE_ERROR => 'Catchable Fatal Error' ]; // create an error string for the log $err = "ERRNO:'" . $errno . "' TYPE:'" . $errortype[$errno] . "' MESSAGE:'" . $errmsg . "' IN FILE:'" . $filename . "' LINE NO:'" . $linenum . "'"; // let's ignore some lesser issues if (substr_count($errmsg, 'date_default_timezone')) { return false; } if (substr_count($errmsg, 'Only variables')) { return false; } // log the error to the Cacti log cacti_log('PROGERR: ' . $err, false, 'RRDCHECK'); } return true; } /** * rrdcheck_boost_bottom - this routine accommodates rrdcheck after the boost process * has completed. The use of boost will require boost version 2.5 or above. The idea * if that rrdcheck will be started on the boost cycle. * * @return void */ function rrdcheck_boost_bottom() : void { if (read_config_option('rrdcheck_enable') == 'on') { include_once(CACTI_PATH_LIBRARY . '/rrd.php'); // run the daily stats. log to database to prevent secondary runs set_config_option('rrdcheck_last_run_time', time()); // run the daily stats $expected_children = rrdcheck_launch_children('bmaster'); rrdcheck_wait_for_children('bmaster', $expected_children); rrdcheck_log_statistics('BOOST'); } } /** * rrdcheck_wait_for_children - waits for launched rrdcheck children of the given * type to finish. exec_background() is non-blocking, so a bare * `while (rrdcheck_processes_running($type))` can observe zero registered * children before any of them have started, mistaking "not started yet" for * "already done" and returning immediately -- the same startup race Boost's * boost_all_children_registered() barrier was added to close (see * poller_boost.php). This applies the same shape: wait for the running count * to reach the launched count before trusting it can fall back to zero. * * @param string $type The process type (e.g. 'bmaster') * @param int $expected_children Number of children rrdcheck_launch_children() launched * * @return void */ function rrdcheck_wait_for_children(string $type, int $expected_children) : void { if ($expected_children <= 0) { return; } $startup_deadline = time() + 30; while (rrdcheck_processes_running($type) < $expected_children && time() < $startup_deadline) { sleep(1); } if (rrdcheck_processes_running($type) < $expected_children) { cacti_log(sprintf('WARNING: rrdcheck startup barrier timed out; %d of %d %s children registered before draining.', rrdcheck_processes_running($type), $expected_children, $type), false, 'RRDCHECK'); } // Wait for all processes to continue while ($running = rrdcheck_processes_running($type)) { rrdcheck_debug(sprintf('%s Processes Running, Sleeping for 2 seconds.', $running)); sleep(2); } } /** * rrdcheck_poller_bottom - this routine launches the main rrdcheck poller. * It is forked independently * to the Cacti poller after all polling has finished. * * @return void */ function rrdcheck_poller_bottom() : void { if (read_config_option('rrdcheck_enable') == 'on') { include_once(CACTI_PATH_LIBRARY . '/poller.php'); chdir(CACTI_PATH_BASE); $command_string = read_config_option('path_php_binary'); if (read_config_option('path_rrdcheck_log') != '') { // This log path reaches the shell through exec_background()'s args, // which are unescaped, so quote it. No settings form exposes this // value today; this is defense in depth and a forward-port of the // release/1.2.31 fix (issue#7473). $safe_log = cacti_escapeshellarg(read_config_option('path_rrdcheck_log')); if (CACTI_SERVER_OS == 'unix') { $extra_args = '-q ' . CACTI_PATH_BASE . '/poller_rrdcheck.php >> ' . $safe_log . ' 2>&1'; } else { $extra_args = '-q ' . CACTI_PATH_BASE . '/poller_rrdcheck.php >> ' . $safe_log; } } else { $extra_args = '-q ' . CACTI_PATH_BASE . '/poller_rrdcheck.php'; } exec_background($command_string, $extra_args); } } /** * rrdcheck_rrdtool_init - this routine provides a bi-directional socket based connection to RRDtool. * it provides a high speed connection to rrdfile in the case where the traditional Cacti call does * not when performing fetch type calls. * * @return array Includes both the process resource and the pipes to communicate with RRDtool. */ function rrdcheck_rrdtool_init() : array { if (CACTI_SERVER_OS == 'unix') { $fds = [ 0 => ['pipe', 'r'], // stdin 1 => ['pipe', 'w'], // stdout 2 => ['file', '/dev/null', 'a'] // stderr ]; } else { $fds = [ 0 => ['pipe', 'r'], // stdin 1 => ['pipe', 'w'], // stdout 2 => ['file', 'nul', 'a'] // stderr ]; } // set the rrdtool default font if (read_config_option('path_rrdtool_default_font')) { putenv('RRD_DEFAULT_FONT=' . read_config_option('path_rrdtool_default_font')); } $command = read_config_option('path_rrdtool') . ' - '; $process = proc_open($command, $fds, $pipes); // proc_open returns false if RRDtool could not be spawned; without a process // the pipes are not populated, so bail out before touching them if (!is_resource($process)) { cacti_log('ERROR: Unable to start RRDtool process for rrdcheck', false, 'RRDCHECK'); return [false, false]; } // make stdin/stdout/stderr non-blocking stream_set_blocking($pipes[0], false); stream_set_blocking($pipes[1], false); return [$process, $pipes]; } /** * rrdcheck_rrdtool_execute - this routine passes commands to RRDtool and returns the information * back to rrdcheck. It is important to note here that RRDtool needs to provide an either 'OK' * or 'ERROR' response across the pipe as it does not provide EOF characters to key upon. * This may not be the best method and may be changed after I have a conversation with a few * developers. * * @param array|string $command The rrdtool command to execute. When passed as an array * the first element is the sub-command and the remaining * elements are escaped before being joined * @param array $pipes An array of stdin and stdout pipes to read and write data from * * @return mixed The output from RRDtool */ function rrdcheck_rrdtool_execute(array|string $command, mixed &$pipes) : mixed { static $broken = false; if (is_array($command)) { // RRDtool needs the sub-command verbatim, so only its arguments are quoted $args = $command; $command = (string) array_shift($args); if (cacti_sizeof($args)) { $command .= ' ' . implode(' ', array_map('cacti_escapeshellarg', $args)); } } $stdout = ''; if ($command == '') { return null; } $command .= "\r\n"; $return_code = fwrite($pipes[0], $command); if (is_resource($pipes[1])) { while (!feof($pipes[1])) { $stdout .= fgets($pipes[1], 4096); if (substr_count($stdout, 'OK')) { break; } if (substr_count($stdout, 'ERROR')) { break; } } } elseif (!$broken) { cacti_log('ERROR: RRDtool was unable to fork. Likely RRDtool can not be found or system out of resources. Blocking subsequent messages.', false, 'POLLER'); $broken = true; } if ($stdout != '') { return $stdout; } return null; } /** * rrdcheck_rrdtool_close - this routine closes the RRDtool process thus also * closing the pipes. * * @param resource $process The process * * @return int The status of the close */ function rrdcheck_rrdtool_close($process) : int { return proc_close($process); } /** * rrdcheck_launch_children - this function will launch collector children based upon * the maximum number of threads and the process type. * * @param string $type The process type * * @return int The number of children launched */ function rrdcheck_launch_children(string $type) : int { global $debug; $processes = read_config_option('rrdcheck_parallel'); if (empty($processes)) { $processes = 1; } $php_binary = read_config_option('path_php_binary'); rrdcheck_debug("About to launch $processes processes."); $sub_type = rrdcheck_get_subtype($type); for ($i = 1; $i <= $processes; $i++) { rrdcheck_debug(sprintf('Launching rrdcheck Process Number %s for Type %s', $i, $type)); cacti_log(sprintf('NOTE: Launching rrdcheck Process Number %s for Type %s', $i, $type), false, 'BOOST', POLLER_VERBOSITY_MEDIUM); exec_background($php_binary, CACTI_PATH_BASE . "/poller_rrdcheck.php --type=$sub_type --child=$i" . ($debug ? ' --debug' : '')); } sleep(2); return (int) $processes; } /** * rrdcheck_get_subtype - this function determine the applicable * sub-type (child name) and return if based upon a type. * * @param string $type The process type * * @return string The sub type */ function rrdcheck_get_subtype(string $type) : string { switch($type) { case 'master': case 'pmaster': return 'child'; case 'bmaster': return 'bchild'; default: return 'unknown'; } } /** * rrdcheck_kill_running_processes - this function is part of an interrupt * handler to kill children processes when the parent is killed. * * @return void */ function rrdcheck_kill_running_processes() : void { global $type; if ($type == 'bmaster') { $processes = db_fetch_assoc_prepared('SELECT * FROM processes WHERE tasktype = "rrdcheck" AND taskname = "bchild" AND pid != ?', [getmypid()]); } else { $processes = db_fetch_assoc_prepared('SELECT * FROM processes WHERE tasktype = "rrdcheck" AND taskname = "child" AND pid != ?', [getmypid()]); } if (cacti_sizeof($processes)) { foreach ($processes as $p) { if (cacti_process_still_running((int) $p['pid'])) { cacti_log(sprintf('WARNING: Killing rrdcheck %s PID %d due to another due to signal or overrun.', ucfirst($p['taskname']), $p['pid']), false, 'BOOST'); cacti_process_kill((int) $p['pid'], SIGTERM, 'BOOST'); } unregister_process($p['tasktype'], $p['taskname'], $p['taskid'], $p['pid']); } } } /** * rrdcheck_processes_running - given a type, determine the number * of sub-type or children that are currently running * * @param string $type The process type * * @return int The number of running processes */ function rrdcheck_processes_running($type) : int { $sub_type = rrdcheck_get_subtype($type); $running = db_fetch_cell_prepared('SELECT COUNT(*) FROM processes WHERE tasktype = "rrdcheck" AND taskname = ?', [$sub_type]); if ($running == 0) { return 0; } return intval($running); }