mirror of
https://github.com/Cacti/cacti.git
synced 2026-10-23 11:36:53 +09:00
TheWitness
75f92c2d86
* Detect and repair RRDfile Data Source count mismatches via RRDtune Report when a Graph requires more or fewer Data Sources than its RRDfile contains, generate the rrdtool tune DS:/DEL: command to correct it, and run it when the RRDfile is writable. Applies to data_debug (rrdtool_cacti_compare) and the RRDchecker batch process (do_rrdcheck). The decision logic lives in pure helpers (rrdtool_cacti_compare_ds, rrdcheck_ds_findings) with unit tests. Resolves #8366 * rrdcheck: only probe local writability for local storage Under the storage proxy the RRDfile is remote, so is_resource_writable() checks the wrong filesystem. Skip the local probe when using the proxy and let it enforce remote permissions and report failures via its response. * rrdcheck: validate tune args, enforce RRA containment, fix failure detection Validate every tune specification against RRDtool's documented DS grammar (ds-name 1-19 [a-zA-Z0-9_], known DST, integer heartbeat, numeric-or-U min/max) and escape it before use. For local storage require the RRDfile to resolve inside the RRA directory via data_source_path_within_rra(). Detect failures from the real return value (false/null/empty or missing OK) instead of casting to string, so a failed tune is no longer logged as success. * rrd: normalize a 0/0 data source range to 0:U when building a tune add spec rrdtool_function_info_from_ds() passes min/max through unchanged, so a source stored as 0/0 produced DS:name:type:hb:0:0 which RRDtool rejects. Mirror RRD creation and emit 0:U so the repair paths can restore those sources.
1131 lines
36 KiB
PHP
1131 lines
36 KiB
PHP
<?php
|
|
/*
|
|
+-------------------------------------------------------------------------+
|
|
| Copyright (C) 2004-2026 The Cacti Group |
|
|
| |
|
|
| This program is free software; you can redistribute it and/or |
|
|
| modify it under the terms of the GNU General Public License |
|
|
| as published by the Free Software Foundation; either version 2 |
|
|
| of the License, or (at your option) any later version. |
|
|
| |
|
|
| This program is distributed in the hope that it will be useful, |
|
|
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
|
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
|
| GNU General Public License for more details. |
|
|
+-------------------------------------------------------------------------+
|
|
| Cacti: The Complete RRDtool-based Graphing Solution |
|
|
+-------------------------------------------------------------------------+
|
|
| This code is designed, written, and maintained by the Cacti Group. See |
|
|
| about.php and/or the AUTHORS file for specific developer information. |
|
|
+-------------------------------------------------------------------------+
|
|
| http://www.cacti.net/ |
|
|
+-------------------------------------------------------------------------+
|
|
*/
|
|
|
|
function get_rrdfiles(int $thread_id = 1, int $max_threads = 1) : array {
|
|
static $newrows = [];
|
|
|
|
if (cacti_sizeof($newrows)) {
|
|
return $newrows[$thread_id - 1];
|
|
}
|
|
|
|
$rows = db_fetch_assoc_prepared('SELECT dtd.data_source_profile_id, dtd.local_data_id,
|
|
REPLACE(dtd.data_source_path, "<path_rra>", ?) AS data_source_path,
|
|
GROUP_CONCAT(DISTINCT dtr.data_source_name) AS data_source_names,
|
|
GROUP_CONCAT(DISTINCT dtd.rrd_step) AS rrd_step,
|
|
GROUP_CONCAT(DISTINCT dtr.rrd_heartbeat) AS rrd_heartbeat,
|
|
dsp.heartbeat AS profile_heartbeat, dsp.step AS profile_step
|
|
FROM data_template_data AS dtd
|
|
INNER JOIN data_template_rrd AS dtr
|
|
ON dtr.local_data_id = dtd.local_data_id
|
|
INNER JOIN graph_templates_item AS gti
|
|
ON dtr.id = gti.task_item_id
|
|
INNER JOIN data_source_profiles AS dsp
|
|
ON dtd.data_source_profile_id = dsp.id
|
|
LEFT JOIN graph_local AS gl
|
|
ON gl.id = gti.local_graph_id
|
|
LEFT JOIN host as h
|
|
ON h.id = gl.host_id
|
|
WHERE dtd.local_data_id != 0
|
|
AND h.disabled != "on"
|
|
GROUP BY dtd.local_data_id',
|
|
[CACTI_PATH_RRA]);
|
|
|
|
if ($max_threads == 1) {
|
|
$newrows[0] = $rows;
|
|
|
|
return $rows;
|
|
} else {
|
|
$newrows = array_chunk($rows, (int) ceil(cacti_sizeof($rows) / $max_threads));
|
|
|
|
if (isset($newrows[$thread_id - 1])) {
|
|
return $newrows[$thread_id - 1];
|
|
} else {
|
|
return [];
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_debug - this simple routine prints a standard message to the console
|
|
* when running in debug mode.
|
|
*
|
|
* @param string $message The message to display
|
|
*
|
|
* @return void
|
|
*/
|
|
function rrdcheck_debug(string $message) : void {
|
|
global $debug;
|
|
|
|
if ($debug) {
|
|
print 'rrdcheck: ' . $message . PHP_EOL;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* do_rrdcheck - this routine is a generic routine that
|
|
* check all RRDfiles (missing files, ...) and stored information (NaN)
|
|
*
|
|
* @param int $thread_id The rrdcheck parallel thread id
|
|
*
|
|
* @return void
|
|
*/
|
|
function do_rrdcheck(int $thread_id = 1) : void {
|
|
global $type;
|
|
global $total_user, $total_system, $total_real, $total_dsses;
|
|
global $user_time, $system_time, $real_time, $rrd_files;
|
|
|
|
$user_time = 0;
|
|
$system_time = 0;
|
|
$real_time = 0;
|
|
$dsses = 0;
|
|
|
|
$poller_interval = read_config_option('poller_interval');
|
|
|
|
rrdcheck_debug(sprintf('Processing %s for Thread', $thread_id));
|
|
|
|
$max_threads = read_config_option('rrdcheck_parallel');
|
|
|
|
if (empty($max_threads)) {
|
|
$max_threads = 1;
|
|
set_config_option('rrdcheck_parallel', '1');
|
|
}
|
|
|
|
$profiles = array_rekey(
|
|
db_fetch_assoc('SELECT step, data_source_profile_id AS id, MIN(steps) AS steps, `rows`
|
|
FROM data_source_profiles AS dsp
|
|
INNER JOIN data_source_profiles_rra AS dspr
|
|
ON dsp.id = dspr.data_source_profile_id
|
|
GROUP BY data_source_profile_id'),
|
|
'id', ['step', 'steps', 'rows']
|
|
);
|
|
|
|
$rrdfiles = get_rrdfiles($thread_id, $max_threads);
|
|
$stats = [];
|
|
$rrd_files += cacti_sizeof($rrdfiles);
|
|
|
|
$use_proxy = (read_config_option('storage_location') ? true : false);
|
|
|
|
if ($use_proxy) {
|
|
$rrdtool_pipe = rrd_init(false);
|
|
} else {
|
|
$process_pipes = rrdcheck_rrdtool_init();
|
|
$process = $process_pipes[0];
|
|
$pipes = $process_pipes[1];
|
|
|
|
// rrdcheck_rrdtool_init() returns [false, false] when RRDtool could not be
|
|
// spawned; without live pipes the execute path would fwrite() to null and
|
|
// crash the poller, so bail out of this run gracefully
|
|
if (!is_resource($process)) {
|
|
cacti_log('ERROR: rrdcheck unable to obtain RRDtool process, skipping run', false, 'RRDCHECK');
|
|
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (cacti_sizeof($rrdfiles)) {
|
|
$now = time();
|
|
|
|
foreach ($rrdfiles as $rrdval) {
|
|
$local_data_id = $rrdval['local_data_id'];
|
|
$data_sources = [];
|
|
|
|
// Rekey the Data Sources key
|
|
$sources = explode(',', $rrdval['data_source_names']);
|
|
|
|
foreach ($sources as $s) {
|
|
$data_sources[$s] = $s;
|
|
}
|
|
|
|
// The first RRA may have less than 24 hours of samples
|
|
$duration = $profiles[$rrdval['data_source_profile_id']]['step'] * ($profiles[$rrdval['data_source_profile_id']]['rows'] - 1);
|
|
$step = $profiles[$rrdval['data_source_profile_id']]['step'];
|
|
|
|
$end = $now;
|
|
$start = $end - $duration;
|
|
|
|
$file = $rrdval['data_source_path'];
|
|
|
|
if ($use_proxy) {
|
|
$file_exists = rrdtool_execute('file_exists ' . cacti_escapeshellarg($file), true, RRDTOOL_OUTPUT_BOOLEAN, false, 'RRDCHECK');
|
|
} else {
|
|
clearstatcache();
|
|
$file_exists = file_exists($file);
|
|
}
|
|
|
|
// don't attempt to get information if the file does not exist
|
|
if ($file_exists) {
|
|
if (!is_resource_writable($file)) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?,NOW(),?)',
|
|
[
|
|
$local_data_id,
|
|
"RRDfile is not writable - $file"
|
|
]
|
|
);
|
|
}
|
|
|
|
if (time() > (filemtime($file) + 3600)) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?,NOW(),?)',
|
|
[
|
|
$local_data_id,
|
|
"RRDfile modify time older than hour - $file"
|
|
]
|
|
);
|
|
}
|
|
|
|
if ($rrdval['profile_step'] != $rrdval['rrd_step']) {
|
|
// Fix the RRD Data Template Data Step
|
|
db_execute_prepared('UPDATE data_template_data
|
|
SET rrd_step = ?
|
|
WHERE local_data_id = ?',
|
|
[
|
|
$rrdval['profile_step'],
|
|
$local_data_id
|
|
]
|
|
);
|
|
}
|
|
|
|
if ($rrdval['rrd_heartbeat'] != $rrdval['profile_heartbeat']) {
|
|
// Fix the RRD Data Template Data Step
|
|
db_execute_prepared('UPDATE data_template_rrd
|
|
SET rrd_heartbeat = ?
|
|
WHERE local_data_id = ?',
|
|
[$rrdval['profile_heartbeat'], $rrdval['local_data_id']]);
|
|
}
|
|
|
|
if ($use_proxy) {
|
|
$output = rrdtool_execute('info ' . cacti_escapeshellarg($file), false, RRDTOOL_OUTPUT_STDOUT, false, 'RRDCHECK');
|
|
} else {
|
|
$output = rrdcheck_rrdtool_execute(['info', $file], $pipes);
|
|
}
|
|
|
|
$matches = [];
|
|
$rrd_info = [];
|
|
$output = explode("\n", $output);
|
|
$last_update = false;
|
|
$rrd_step = false;
|
|
|
|
// Process the RRDfile information into an array
|
|
foreach ($output as $line) {
|
|
$line = trim($line);
|
|
|
|
if ($line == '') {
|
|
continue;
|
|
}
|
|
|
|
if (!$last_update) {
|
|
if (preg_match('/^last_update = (\S+)$/', $line, $matches)) {
|
|
$last_update = $matches[1];
|
|
}
|
|
}
|
|
|
|
if (!$rrd_step) {
|
|
if (preg_match('/^step = (\S+)$/', $line, $matches)) {
|
|
$rrd_step = $matches[1];
|
|
}
|
|
}
|
|
|
|
if (preg_match('/^ds\[(\S+)\]\.(\S+) = (\S+)$/', $line, $matches)) {
|
|
$rrd_info[$matches[1]][$matches[2]] = trim($matches[3], '"');
|
|
}
|
|
}
|
|
|
|
if ($rrd_step != $rrdval['profile_step']) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?, NOW(), ?)',
|
|
[
|
|
$local_data_id,
|
|
"The RRDfile step of {$step} does not match the Data Source Profile step {$rrdval['profile_step']}. File '$file'."
|
|
]
|
|
);
|
|
}
|
|
|
|
if ($last_update < (time() - 3600)) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?, NOW(), ?)',
|
|
[
|
|
$local_data_id,
|
|
"Last update value in RRDfile is older than 1 hour. File '$file'."
|
|
]
|
|
);
|
|
}
|
|
|
|
$cacti_ds = rrdtool_function_info_from_ds($local_data_id);
|
|
$cacti_ds = isset($cacti_ds['ds']) ? $cacti_ds['ds'] : [];
|
|
|
|
foreach (rrdcheck_ds_findings($data_sources, $rrd_info, $cacti_ds, $file) as $finding) {
|
|
$suffix = $finding['spec'] !== '' ? ' ' . rrdcheck_tune_rrdfile($file, $finding['spec'], $local_data_id, $use_proxy, $pipes) . '.' : '';
|
|
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?, NOW(), ?)',
|
|
[$local_data_id, $finding['message'] . $suffix]);
|
|
}
|
|
|
|
$output = '';
|
|
$matches = [];
|
|
|
|
foreach ($rrd_info as $info_key => $info_array) {
|
|
if (!isset($data_sources[$info_key])) {
|
|
continue;
|
|
}
|
|
|
|
if ($info_array['minimal_heartbeat'] <= $poller_interval) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?, NOW(), ?)',
|
|
[
|
|
$local_data_id,
|
|
"The RRDfile Minimal Heart for Data Source '{$info_key}' is lower than polling interval. This will causes gaps in Graphs. File '$file.'"
|
|
]
|
|
);
|
|
}
|
|
|
|
if ($info_array['minimal_heartbeat'] < $rrdval['profile_heartbeat']) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?, NOW(), ?)',
|
|
[
|
|
$local_data_id,
|
|
"The RRDfile minimal heartbeat for Data Source '{$info_key}' should be '{$rrdval['profile_heartbeat']}' and is currently '{$info_array['minimal_heartbeat']}'. File '$file'."
|
|
]
|
|
);
|
|
}
|
|
}
|
|
|
|
$data_source_names = [];
|
|
|
|
// test stale data
|
|
|
|
$pstart = $start - $step;
|
|
$pend = $end - $step;
|
|
$one_hour_limit = ($duration - 3600) / $step;
|
|
|
|
if ($use_proxy) {
|
|
$info_array = rrdtool_execute('fetch ' . cacti_escapeshellarg($file) . " LAST -s $pstart -e $pend", false, RRDTOOL_OUTPUT_STDOUT, false, 'RRDCHECK');
|
|
} else {
|
|
$info_array = rrdcheck_rrdtool_execute(['fetch', $file, 'LAST', '-s', $pstart, '-e', $pend], $pipes);
|
|
}
|
|
|
|
// don't do anything if RRDfile did not return data
|
|
$info_array = explode("\n", $info_array);
|
|
|
|
if (cacti_sizeof($info_array)) {
|
|
$first = true;
|
|
$lines_24 = 0;
|
|
$lines_1 = 0;
|
|
$nan_24 = [];
|
|
$nan_1 = [];
|
|
|
|
foreach ($info_array as $line) {
|
|
$line = trim($line);
|
|
|
|
// remove line - OK u:0.03 s:0.12 r:0.33
|
|
if (str_starts_with($line, 'OK')) {
|
|
continue;
|
|
}
|
|
|
|
if ($line == '') {
|
|
continue;
|
|
}
|
|
|
|
if ($first) {
|
|
// get the data source names
|
|
$data_source_names = preg_split('/\s+/', $line) ?: [];
|
|
|
|
foreach (($data_source_names ?: []) as $index => $name) {
|
|
$nan_24[$index] = 0;
|
|
$nan_1[$index] = 0;
|
|
}
|
|
|
|
$dsses += cacti_sizeof($data_source_names);
|
|
$first = false;
|
|
} else {
|
|
$parts = explode(':', $line);
|
|
$data = explode(' ', trim($parts[1]));
|
|
|
|
foreach ($data as $index=>$number) {
|
|
if ($index == 0) {
|
|
// only onetime for each row
|
|
$lines_24++;
|
|
|
|
if ($lines_24 > $one_hour_limit) { // last hour
|
|
$lines_1++;
|
|
}
|
|
}
|
|
|
|
if (cacti_strtolower($number) == 'nan' || cacti_strtolower($number) == '-nan') {
|
|
$nan_24[$index]++;
|
|
|
|
if ($lines_24 > $one_hour_limit) {
|
|
$nan_1[$index]++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
$notified = false;
|
|
|
|
// 24 hour statistics
|
|
foreach ($nan_24 as $index => $count) {
|
|
if ($lines_24 > 0) {
|
|
$ratio = $count / $lines_24;
|
|
} else {
|
|
$ratio = 0;
|
|
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?,NOW(),?)',
|
|
[
|
|
$local_data_id,
|
|
"No data returned, maybe corrupted Data Source '{$data_source_names[$index]}'. File '$file'."
|
|
]
|
|
);
|
|
}
|
|
|
|
if ($ratio == 1) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?,NOW(),?)',
|
|
[
|
|
$local_data_id,
|
|
"Stale values for last 24 hours, Data Source '{$data_source_names[$index]}'. File '$file'."
|
|
]
|
|
);
|
|
|
|
$notified = true;
|
|
} elseif ($ratio > 0.5) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?,NOW(),?)',
|
|
[
|
|
$local_data_id,
|
|
"More than 50% ($count/$lines_24) values are NaN in last 24 hours, Data Source '{$data_source_names[$index]}'. File '$file'."
|
|
]
|
|
);
|
|
}
|
|
}
|
|
|
|
// 1 hour statistics
|
|
|
|
foreach ($nan_1 as $index=>$count) {
|
|
if ($notified) {
|
|
// 24hour notified, skipping
|
|
continue;
|
|
}
|
|
|
|
if ($lines_1 > 0) {
|
|
$ratio = $count / $lines_1;
|
|
} else {
|
|
$ratio = 0;
|
|
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?,NOW(),?)',
|
|
[
|
|
$local_data_id,
|
|
"No data returned, maybe corrupted Data Source '{$data_source_names[$index]}'. File '$file'."
|
|
]
|
|
);
|
|
}
|
|
|
|
if ($ratio == 1) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?,NOW(),?)',
|
|
[
|
|
$local_data_id,
|
|
"Stale values for last hour, Data Source '{$data_source_names[$index]}'. File '$file'."
|
|
]
|
|
);
|
|
} elseif ($ratio > 0.5) {
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?,NOW(),?)',
|
|
[
|
|
$local_data_id,
|
|
"More than 50% ($count/$lines_1) values are NaN in last hour, Data Source '{$data_source_names[$index]}'. File '$file'."
|
|
]
|
|
);
|
|
}
|
|
}
|
|
} else {
|
|
cacti_log("WARNING: RRDcheck - no rrd data returned - '$file'", false, 'RRDCHECK');
|
|
}
|
|
} else { // rrdfile does not exist
|
|
db_execute_prepared('INSERT INTO rrdcheck
|
|
(local_data_id, test_date, message)
|
|
VALUES (?,NOW(),?)',
|
|
[$local_data_id, "RRDfile does not exist - '$file'"]);
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($use_proxy) {
|
|
rrd_close($rrdtool_pipe);
|
|
} else {
|
|
rrdcheck_rrdtool_close($process);
|
|
}
|
|
|
|
if (!empty($type)) {
|
|
$total_user += $user_time;
|
|
$total_system += $system_time;
|
|
$total_real += $real_time;
|
|
$total_dsses += $dsses;
|
|
|
|
set_config_option('rrdcheck_rrd_system_' . $type . '_' . $thread_id, $total_system);
|
|
set_config_option('rrdcheck_rrd_user_' . $type . '_' . $thread_id, $total_user);
|
|
set_config_option('rrdcheck_rrd_real_' . $type . '_' . $thread_id, $total_real);
|
|
set_config_option('rrdcheck_total_rrds_' . $type . '_' . $thread_id, $rrd_files);
|
|
set_config_option('rrdcheck_total_dsses_' . $type . '_' . $thread_id, $total_dsses);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_ds_findings - compares the data sources the Graph requires against those present in an
|
|
* RRDfile and returns the findings, each with the rrdtool 'tune' specification (when applicable)
|
|
* needed to reconcile the difference. A data source the Graph requires but the RRDfile lacks is
|
|
* added with 'DS:'; a data source the RRDfile carries but the Graph no longer requires is removed
|
|
* with 'DEL:'.
|
|
*
|
|
* @param array $data_sources The data source names the Graph requires, keyed by name
|
|
* @param array $rrd_info The data sources present in the RRDfile, keyed by name
|
|
* @param array $cacti_ds The Cacti data source definitions, keyed by name
|
|
* @param string $file The RRDfile path
|
|
*
|
|
* @return array A list of ['message' => string, 'spec' => string] findings
|
|
*/
|
|
function rrdcheck_ds_findings(array $data_sources, array $rrd_info, array $cacti_ds, string $file) : array {
|
|
$findings = [];
|
|
$db_count = cacti_sizeof($data_sources);
|
|
$rrd_count = cacti_sizeof($rrd_info);
|
|
|
|
if ($db_count > $rrd_count) {
|
|
$findings[] = ['message' => "The Graph requires $db_count Data Sources but the RRDfile only includes $rrd_count. File '$file'.", 'spec' => ''];
|
|
} elseif ($db_count < $rrd_count) {
|
|
$findings[] = ['message' => "The Graph requires $db_count Data Sources but the RRDfile includes $rrd_count. File '$file'.", 'spec' => ''];
|
|
}
|
|
|
|
foreach ($rrd_info as $name => $info) {
|
|
if (!isset($data_sources[$name])) {
|
|
$spec = 'DEL:' . $name;
|
|
$findings[] = ['message' => "The RRDfile includes Data Source '$name' which the Graph no longer requires. RRDtune: tune $spec.", 'spec' => $spec];
|
|
}
|
|
}
|
|
|
|
foreach ($data_sources as $name) {
|
|
if ($name === '') {
|
|
$findings[] = ['message' => "Database Data Source name is empty. File '$file'.", 'spec' => ''];
|
|
|
|
continue;
|
|
}
|
|
|
|
if (!array_key_exists($name, $rrd_info)) {
|
|
if (isset($cacti_ds[$name])) {
|
|
$ds = $cacti_ds[$name];
|
|
$spec = rrdtool_tune_ds_add_spec($name, $ds['type'], $ds['minimal_heartbeat'], $ds['min'], $ds['max']);
|
|
$findings[] = ['message' => "The Graph requires Data Source '$name' which is missing from the RRDfile. RRDtune: tune $spec.", 'spec' => $spec];
|
|
} else {
|
|
$findings[] = ['message' => "Data Source name '$name' exists in the Database, but not in RRDfile. File '$file'.", 'spec' => ''];
|
|
}
|
|
}
|
|
}
|
|
|
|
return $findings;
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_valid_tune_spec - validate an rrdtool 'tune' specification against RRDtool's documented
|
|
* data source grammar before it is sent to RRDtool. Per rrdcreate, ds-name is 1 to 19 characters
|
|
* from [a-zA-Z0-9_], DST is one of the known data source types, heartbeat is a positive integer and
|
|
* min/max are either a number or 'U'.
|
|
*
|
|
* @param string $spec The tune specification to validate
|
|
*
|
|
* @return bool True when the specification matches a known-good DS:/DEL: form
|
|
*/
|
|
function rrdcheck_valid_tune_spec(string $spec) : bool {
|
|
if (preg_match('/^DEL:[a-zA-Z0-9_]{1,19}$/', $spec)) {
|
|
return true;
|
|
}
|
|
|
|
return (bool) preg_match('/^DS:[a-zA-Z0-9_]{1,19}:(GAUGE|COUNTER|DERIVE|DCOUNTER|DDERIVE|ABSOLUTE):[0-9]+:(U|-?[0-9]+(\.[0-9]+)?):(U|-?[0-9]+(\.[0-9]+)?)$/', $spec);
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_tune_rrdfile - applies an rrdtool 'tune' specification to an RRDfile from the RRDchecker
|
|
* batch process. The specification is validated against RRDtool's grammar first; for local storage
|
|
* the RRDfile must resolve inside the RRA directory and be writable, while the storage proxy
|
|
* enforces remote containment and permissions. The full command is written to the Cacti log.
|
|
*
|
|
* @param string $file The fully qualified RRDfile path
|
|
* @param string $spec The rrdtool tune specification to apply
|
|
* @param int $local_data_id The local data id being checked
|
|
* @param bool $use_proxy Whether rrdtool runs through the storage proxy
|
|
* @param array $pipes The local rrdtool pipes when not using the proxy
|
|
*
|
|
* @return string A short phrase describing whether the tune ran
|
|
*/
|
|
function rrdcheck_tune_rrdfile(string $file, string $spec, int $local_data_id, bool $use_proxy, &$pipes) : string {
|
|
$command = read_config_option('path_rrdtool') . ' tune ' . cacti_escapeshellarg($file) . ' ' . cacti_escapeshellarg($spec);
|
|
|
|
if (!rrdcheck_valid_tune_spec($spec)) {
|
|
cacti_log("ERROR: RRDcheck - refusing malformed tune specification - DS[$local_data_id] Spec[$spec]", false, 'RRDCHECK');
|
|
|
|
return 'the tune specification was not valid so it was not run';
|
|
}
|
|
|
|
if (!$use_proxy) {
|
|
if (!data_source_path_within_rra($file)) {
|
|
cacti_log("ERROR: RRDcheck - RRDfile resolves outside the RRA directory, tune not run - DS[$local_data_id] File[$file]", false, 'RRDCHECK');
|
|
|
|
return 'the RRDfile is outside the RRA directory so the tune was not run';
|
|
}
|
|
|
|
if (!is_resource_writable($file)) {
|
|
cacti_log("WARNING: RRDcheck - RRDfile not writable, tune not run - DS[$local_data_id] Command[$command]", false, 'RRDCHECK');
|
|
|
|
return 'the RRDfile is not writable so the tune was not run automatically';
|
|
}
|
|
}
|
|
|
|
if ($use_proxy) {
|
|
$output = rrdtool_execute('tune ' . cacti_escapeshellarg($file) . ' ' . cacti_escapeshellarg($spec), false, RRDTOOL_OUTPUT_RETURN_STDERR, false, 'RRDCHECK');
|
|
$failed = ($output === false || trim((string) $output) !== '');
|
|
} else {
|
|
$output = rrdcheck_rrdtool_execute(['tune', $file, $spec], $pipes);
|
|
$failed = ($output === null || $output === false || strpos((string) $output, 'ERROR') !== false || strpos((string) $output, 'OK') === false);
|
|
}
|
|
|
|
if ($failed) {
|
|
cacti_log("ERROR: RRDcheck - tune failed - DS[$local_data_id] Command[$command] Output[" . trim((string) $output) . ']', false, 'RRDCHECK');
|
|
|
|
return 'the tune command failed, see the Cacti log for details';
|
|
}
|
|
|
|
cacti_log("RRDcheck - tune ran successfully - DS[$local_data_id] Command[$command]", false, 'RRDCHECK');
|
|
|
|
return 'the tune command was run automatically to correct the issue';
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_log_statistics - provides generic timing message to both the Cacti log and the settings
|
|
* table so that the statistics can be graphed as well.
|
|
*
|
|
* @param string $type The type of statistics to log, either 'HOURLY', 'BOOST'.
|
|
*
|
|
* @return void
|
|
*/
|
|
function rrdcheck_log_statistics(string $type) : void {
|
|
global $start;
|
|
|
|
rrdcheck_debug($type);
|
|
|
|
if ($type == 'BOOST') {
|
|
$sub_type = 'bchild';
|
|
} else {
|
|
$sub_type = '';
|
|
}
|
|
|
|
// take time and log performance data
|
|
$end = microtime(true);
|
|
|
|
if ($sub_type != '') {
|
|
$rrd_user = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_rrd_user_%' . $sub_type . '%']);
|
|
|
|
$rrd_system = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_rrd_system_%' . $sub_type . '%']);
|
|
|
|
$rrd_real = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_rrd_real_%' . $sub_type . '%']);
|
|
|
|
$rrd_files = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_total_rrds_%' . $sub_type . '%']);
|
|
|
|
$dsses = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_total_dsses_%' . $sub_type . '%']);
|
|
|
|
$processes = read_config_option('rrdcheck_parallel');
|
|
|
|
$cacti_stats = sprintf('Time:%01.2f Type:%s Threads:%s RRDfiles:%s DSSes:%s RRDUser:%01.2f RRDSystem:%01.2f RRDReal:%01.2f', $end - $start, $type, $processes, $rrd_files, $dsses, $rrd_user, $rrd_system, $rrd_real);
|
|
|
|
db_execute("DELETE FROM settings
|
|
WHERE name LIKE 'rrdcheck_rrd_%$sub_type%'
|
|
OR name LIKE 'rrdcheck_total_rrds_%$sub_type%'
|
|
OR name LIKE 'rrdcheck_total_dsses_%$sub_type%'");
|
|
} else {
|
|
$cacti_stats = sprintf('Time:%01.2f Type:%s', $end - $start, $type);
|
|
}
|
|
|
|
// take time and log performance data
|
|
$start = microtime(true);
|
|
|
|
// log to the database
|
|
set_config_option('stats_rrdcheck_' . $type, $cacti_stats);
|
|
|
|
// log to the logfile
|
|
cacti_log('RRDCHECK STATS: ' . $cacti_stats , true, 'SYSTEM');
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_log_child_stats - logs rrdcheck child process information
|
|
*
|
|
* @param string $type The type of child, MAJOR, DAILY, BOOST
|
|
* @param int $thread_id The parallel thread id
|
|
* @param float $total_time The total time to collect date
|
|
*
|
|
* @return void
|
|
*/
|
|
function rrdcheck_log_child_stats(string $type, int $thread_id, float $total_time) : void {
|
|
$rrd_user = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_rrd_user_%' . $type . '_' . $thread_id . '%']);
|
|
|
|
$rrd_system = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_rrd_system_%' . $type . '_' . $thread_id . '%']);
|
|
|
|
$rrd_real = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_rrd_real_%' . $type . '_' . $thread_id . '%']);
|
|
|
|
$rrd_files = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_total_rrds_%' . $type . '_' . $thread_id . '%']);
|
|
|
|
$dsses = db_fetch_cell_prepared('SELECT SUM(value)
|
|
FROM settings
|
|
WHERE name LIKE ?',
|
|
['rrdcheck_total_dsses_%' . $type . '_' . $thread_id . '%']);
|
|
|
|
$cacti_stats = sprintf('Time:%01.2f Type:%s ProcessNumber:%s RRDfiles:%s DSSes:%s RRDUser:%01.2f RRDSystem:%01.2f RRDReal:%01.2f', $total_time, cacti_strtoupper($type), $thread_id, $rrd_files, $dsses, $rrd_user, $rrd_system, $rrd_real);
|
|
|
|
cacti_log('RRDCHECK CHILD STATS: ' . $cacti_stats, true, 'SYSTEM');
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_error_handler - this routine logs all PHP error transactions
|
|
* to make sure they are properly logged.
|
|
*
|
|
* @param int $errno The errornum reported by the system
|
|
* @param string $errmsg The error message provides by the error
|
|
* @param string $filename The filename that encountered the error
|
|
* @param int $linenum The line number where the error occurred
|
|
* @param array $vars The current state of PHP variables.
|
|
*
|
|
* @return bool Always returns true for some reason
|
|
*/
|
|
function rrdcheck_error_handler(int $errno, string $errmsg, string $filename, int $linenum, array $vars = []) : bool {
|
|
if (read_config_option('log_verbosity') >= POLLER_VERBOSITY_DEBUG) {
|
|
// define all error types
|
|
$errortype = [
|
|
E_ERROR => 'Error',
|
|
E_WARNING => 'Warning',
|
|
E_PARSE => 'Parsing Error',
|
|
E_NOTICE => 'Notice',
|
|
E_CORE_ERROR => 'Core Error',
|
|
E_CORE_WARNING => 'Core Warning',
|
|
E_COMPILE_ERROR => 'Compile Error',
|
|
E_COMPILE_WARNING => 'Compile Warning',
|
|
E_USER_ERROR => 'User Error',
|
|
E_USER_WARNING => 'User Warning',
|
|
E_USER_NOTICE => 'User Notice',
|
|
E_RECOVERABLE_ERROR => 'Catchable Fatal Error'
|
|
];
|
|
|
|
// create an error string for the log
|
|
$err = "ERRNO:'" . $errno . "' TYPE:'" . $errortype[$errno] .
|
|
"' MESSAGE:'" . $errmsg . "' IN FILE:'" . $filename .
|
|
"' LINE NO:'" . $linenum . "'";
|
|
|
|
// let's ignore some lesser issues
|
|
if (substr_count($errmsg, 'date_default_timezone')) {
|
|
return false;
|
|
}
|
|
|
|
if (substr_count($errmsg, 'Only variables')) {
|
|
return false;
|
|
}
|
|
|
|
// log the error to the Cacti log
|
|
cacti_log('PROGERR: ' . $err, false, 'RRDCHECK');
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_boost_bottom - this routine accommodates rrdcheck after the boost process
|
|
* has completed. The use of boost will require boost version 2.5 or above. The idea
|
|
* if that rrdcheck will be started on the boost cycle.
|
|
*
|
|
* @return void
|
|
*/
|
|
function rrdcheck_boost_bottom() : void {
|
|
if (read_config_option('rrdcheck_enable') == 'on') {
|
|
include_once(CACTI_PATH_LIBRARY . '/rrd.php');
|
|
|
|
// run the daily stats. log to database to prevent secondary runs
|
|
set_config_option('rrdcheck_last_run_time', time());
|
|
|
|
// run the daily stats
|
|
$expected_children = rrdcheck_launch_children('bmaster');
|
|
|
|
rrdcheck_wait_for_children('bmaster', $expected_children);
|
|
|
|
rrdcheck_log_statistics('BOOST');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_wait_for_children - waits for launched rrdcheck children of the given
|
|
* type to finish. exec_background() is non-blocking, so a bare
|
|
* `while (rrdcheck_processes_running($type))` can observe zero registered
|
|
* children before any of them have started, mistaking "not started yet" for
|
|
* "already done" and returning immediately -- the same startup race Boost's
|
|
* boost_all_children_registered() barrier was added to close (see
|
|
* poller_boost.php). This applies the same shape: wait for the running count
|
|
* to reach the launched count before trusting it can fall back to zero.
|
|
*
|
|
* @param string $type The process type (e.g. 'bmaster')
|
|
* @param int $expected_children Number of children rrdcheck_launch_children() launched
|
|
*
|
|
* @return void
|
|
*/
|
|
function rrdcheck_wait_for_children(string $type, int $expected_children) : void {
|
|
if ($expected_children <= 0) {
|
|
return;
|
|
}
|
|
|
|
$startup_deadline = time() + 30;
|
|
|
|
while (rrdcheck_processes_running($type) < $expected_children && time() < $startup_deadline) {
|
|
sleep(1);
|
|
}
|
|
|
|
if (rrdcheck_processes_running($type) < $expected_children) {
|
|
cacti_log(sprintf('WARNING: rrdcheck startup barrier timed out; %d of %d %s children registered before draining.', rrdcheck_processes_running($type), $expected_children, $type), false, 'RRDCHECK');
|
|
}
|
|
|
|
// Wait for all processes to continue
|
|
while ($running = rrdcheck_processes_running($type)) {
|
|
rrdcheck_debug(sprintf('%s Processes Running, Sleeping for 2 seconds.', $running));
|
|
|
|
sleep(2);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_poller_bottom - this routine launches the main rrdcheck poller.
|
|
* It is forked independently
|
|
* to the Cacti poller after all polling has finished.
|
|
*
|
|
* @return void
|
|
*/
|
|
function rrdcheck_poller_bottom() : void {
|
|
if (read_config_option('rrdcheck_enable') == 'on') {
|
|
include_once(CACTI_PATH_LIBRARY . '/poller.php');
|
|
|
|
chdir(CACTI_PATH_BASE);
|
|
|
|
$command_string = read_config_option('path_php_binary');
|
|
|
|
if (read_config_option('path_rrdcheck_log') != '') {
|
|
// This log path reaches the shell through exec_background()'s args,
|
|
// which are unescaped, so quote it. No settings form exposes this
|
|
// value today; this is defense in depth and a forward-port of the
|
|
// release/1.2.31 fix (issue#7473).
|
|
$safe_log = cacti_escapeshellarg(read_config_option('path_rrdcheck_log'));
|
|
|
|
if (CACTI_SERVER_OS == 'unix') {
|
|
$extra_args = '-q ' . CACTI_PATH_BASE . '/poller_rrdcheck.php >> ' . $safe_log . ' 2>&1';
|
|
} else {
|
|
$extra_args = '-q ' . CACTI_PATH_BASE . '/poller_rrdcheck.php >> ' . $safe_log;
|
|
}
|
|
} else {
|
|
$extra_args = '-q ' . CACTI_PATH_BASE . '/poller_rrdcheck.php';
|
|
}
|
|
|
|
exec_background($command_string, $extra_args);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_rrdtool_init - this routine provides a bi-directional socket based connection to RRDtool.
|
|
* it provides a high speed connection to rrdfile in the case where the traditional Cacti call does
|
|
* not when performing fetch type calls.
|
|
*
|
|
* @return array Includes both the process resource and the pipes to communicate with RRDtool.
|
|
*/
|
|
function rrdcheck_rrdtool_init() : array {
|
|
if (CACTI_SERVER_OS == 'unix') {
|
|
$fds = [
|
|
0 => ['pipe', 'r'], // stdin
|
|
1 => ['pipe', 'w'], // stdout
|
|
2 => ['file', '/dev/null', 'a'] // stderr
|
|
];
|
|
} else {
|
|
$fds = [
|
|
0 => ['pipe', 'r'], // stdin
|
|
1 => ['pipe', 'w'], // stdout
|
|
2 => ['file', 'nul', 'a'] // stderr
|
|
];
|
|
}
|
|
|
|
// set the rrdtool default font
|
|
if (read_config_option('path_rrdtool_default_font')) {
|
|
putenv('RRD_DEFAULT_FONT=' . read_config_option('path_rrdtool_default_font'));
|
|
}
|
|
|
|
$command = read_config_option('path_rrdtool') . ' - ';
|
|
|
|
$process = proc_open($command, $fds, $pipes);
|
|
|
|
// proc_open returns false if RRDtool could not be spawned; without a process
|
|
// the pipes are not populated, so bail out before touching them
|
|
if (!is_resource($process)) {
|
|
cacti_log('ERROR: Unable to start RRDtool process for rrdcheck', false, 'RRDCHECK');
|
|
|
|
return [false, false];
|
|
}
|
|
|
|
// make stdin/stdout/stderr non-blocking
|
|
stream_set_blocking($pipes[0], false);
|
|
stream_set_blocking($pipes[1], false);
|
|
|
|
return [$process, $pipes];
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_rrdtool_execute - this routine passes commands to RRDtool and returns the information
|
|
* back to rrdcheck. It is important to note here that RRDtool needs to provide an either 'OK'
|
|
* or 'ERROR' response across the pipe as it does not provide EOF characters to key upon.
|
|
* This may not be the best method and may be changed after I have a conversation with a few
|
|
* developers.
|
|
*
|
|
* @param array|string $command The rrdtool command to execute. When passed as an array
|
|
* the first element is the sub-command and the remaining
|
|
* elements are escaped before being joined
|
|
* @param array $pipes An array of stdin and stdout pipes to read and write data from
|
|
*
|
|
* @return mixed The output from RRDtool
|
|
*/
|
|
function rrdcheck_rrdtool_execute(array|string $command, mixed &$pipes) : mixed {
|
|
static $broken = false;
|
|
|
|
if (is_array($command)) {
|
|
// RRDtool needs the sub-command verbatim, so only its arguments are quoted
|
|
$args = $command;
|
|
$command = (string) array_shift($args);
|
|
|
|
if (cacti_sizeof($args)) {
|
|
$command .= ' ' . implode(' ', array_map('cacti_escapeshellarg', $args));
|
|
}
|
|
}
|
|
|
|
$stdout = '';
|
|
|
|
if ($command == '') {
|
|
return null;
|
|
}
|
|
|
|
$command .= "\r\n";
|
|
$return_code = fwrite($pipes[0], $command);
|
|
|
|
if (is_resource($pipes[1])) {
|
|
while (!feof($pipes[1])) {
|
|
$stdout .= fgets($pipes[1], 4096);
|
|
|
|
if (substr_count($stdout, 'OK')) {
|
|
break;
|
|
}
|
|
|
|
if (substr_count($stdout, 'ERROR')) {
|
|
break;
|
|
}
|
|
}
|
|
} elseif (!$broken) {
|
|
cacti_log('ERROR: RRDtool was unable to fork. Likely RRDtool can not be found or system out of resources. Blocking subsequent messages.', false, 'POLLER');
|
|
|
|
$broken = true;
|
|
}
|
|
|
|
if ($stdout != '') {
|
|
return $stdout;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_rrdtool_close - this routine closes the RRDtool process thus also
|
|
* closing the pipes.
|
|
*
|
|
* @param resource $process The process
|
|
*
|
|
* @return int The status of the close
|
|
*/
|
|
function rrdcheck_rrdtool_close($process) : int {
|
|
return proc_close($process);
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_launch_children - this function will launch collector children based upon
|
|
* the maximum number of threads and the process type.
|
|
*
|
|
* @param string $type The process type
|
|
*
|
|
* @return int The number of children launched
|
|
*/
|
|
function rrdcheck_launch_children(string $type) : int {
|
|
global $debug;
|
|
|
|
$processes = read_config_option('rrdcheck_parallel');
|
|
|
|
if (empty($processes)) {
|
|
$processes = 1;
|
|
}
|
|
|
|
$php_binary = read_config_option('path_php_binary');
|
|
|
|
rrdcheck_debug("About to launch $processes processes.");
|
|
|
|
$sub_type = rrdcheck_get_subtype($type);
|
|
|
|
for ($i = 1; $i <= $processes; $i++) {
|
|
rrdcheck_debug(sprintf('Launching rrdcheck Process Number %s for Type %s', $i, $type));
|
|
|
|
cacti_log(sprintf('NOTE: Launching rrdcheck Process Number %s for Type %s', $i, $type), false, 'BOOST', POLLER_VERBOSITY_MEDIUM);
|
|
|
|
exec_background($php_binary, CACTI_PATH_BASE . "/poller_rrdcheck.php --type=$sub_type --child=$i" . ($debug ? ' --debug' : ''));
|
|
}
|
|
|
|
sleep(2);
|
|
|
|
return (int) $processes;
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_get_subtype - this function determine the applicable
|
|
* sub-type (child name) and return if based upon a type.
|
|
*
|
|
* @param string $type The process type
|
|
*
|
|
* @return string The sub type
|
|
*/
|
|
function rrdcheck_get_subtype(string $type) : string {
|
|
switch($type) {
|
|
case 'master':
|
|
case 'pmaster':
|
|
return 'child';
|
|
case 'bmaster':
|
|
return 'bchild';
|
|
default:
|
|
return 'unknown';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_kill_running_processes - this function is part of an interrupt
|
|
* handler to kill children processes when the parent is killed.
|
|
*
|
|
* @return void
|
|
*/
|
|
function rrdcheck_kill_running_processes() : void {
|
|
global $type;
|
|
|
|
if ($type == 'bmaster') {
|
|
$processes = db_fetch_assoc_prepared('SELECT *
|
|
FROM processes
|
|
WHERE tasktype = "rrdcheck"
|
|
AND taskname = "bchild"
|
|
AND pid != ?',
|
|
[getmypid()]);
|
|
} else {
|
|
$processes = db_fetch_assoc_prepared('SELECT *
|
|
FROM processes
|
|
WHERE tasktype = "rrdcheck"
|
|
AND taskname = "child"
|
|
AND pid != ?',
|
|
[getmypid()]);
|
|
}
|
|
|
|
if (cacti_sizeof($processes)) {
|
|
foreach ($processes as $p) {
|
|
if (cacti_process_still_running((int) $p['pid'])) {
|
|
cacti_log(sprintf('WARNING: Killing rrdcheck %s PID %d due to another due to signal or overrun.', ucfirst($p['taskname']), $p['pid']), false, 'BOOST');
|
|
|
|
cacti_process_kill((int) $p['pid'], SIGTERM, 'BOOST');
|
|
}
|
|
|
|
unregister_process($p['tasktype'], $p['taskname'], $p['taskid'], $p['pid']);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* rrdcheck_processes_running - given a type, determine the number
|
|
* of sub-type or children that are currently running
|
|
*
|
|
* @param string $type The process type
|
|
*
|
|
* @return int The number of running processes
|
|
*/
|
|
function rrdcheck_processes_running($type) : int {
|
|
$sub_type = rrdcheck_get_subtype($type);
|
|
|
|
$running = db_fetch_cell_prepared('SELECT COUNT(*)
|
|
FROM processes
|
|
WHERE tasktype = "rrdcheck"
|
|
AND taskname = ?',
|
|
[$sub_type]);
|
|
|
|
if ($running == 0) {
|
|
return 0;
|
|
}
|
|
|
|
return intval($running);
|
|
}
|