cacti/cli/batchgapfix.php

#!/usr/bin/env php
<?php
/*
 +-------------------------------------------------------------------------+
 | Copyright (C) 2004-2026 The Cacti Group                                 |
 |                                                                         |
 | This program is free software; you can redistribute it and/or           |
 | modify it under the terms of the GNU General Public License             |
 | as published by the Free Software Foundation; either version 2          |
 | of the License, or (at your option) any later version.                  |
 |                                                                         |
 | This program is distributed in the hope that it will be useful,         |
 | but WITHOUT ANY WARRANTY; without even the implied warranty of          |
 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           |
 | GNU General Public License for more details.                            |
 +-------------------------------------------------------------------------+
 | Cacti: The Complete RRDtool-based Graphing Solution                     |
 +-------------------------------------------------------------------------+
 | This code is designed, written, and maintained by the Cacti Group. See  |
 | about.php and/or the AUTHORS file for specific developer information.   |
 +-------------------------------------------------------------------------+
 | http://www.cacti.net/                                                   |
 +-------------------------------------------------------------------------+
*/

if (function_exists('pcntl_async_signals')) {
	pcntl_async_signals(true);
} else {
	declare(ticks = 100);
}

ini_set('output_buffering', 'Off');

require(__DIR__ . '/../include/cli_check.php');
require_once(CACTI_PATH_LIBRARY . '/poller.php');

if (POLLER_ID > 1) {
	print 'FATAL: This utility is designed for the main Data Collector only' . PHP_EOL;

	exit(1);
}

ini_set('memory_limit', '-1');
ini_set('max_execution_time', '0');

// process calling arguments
$parms = $_SERVER['argv'];
array_shift($parms);

global $debug, $child, $type;

$debug      = false;
$force      = false;
$host_ids   = false;
$threads    = detect_cpu_cores();

if ($threads == 0) {
	$threads = 2;
}

$child      = 0;
$type       = 'master';
$method     = 'fill';
$avgnan     = 'last';
$start_time = false;
$end_time   = false;
$php_bin    = (string) read_config_option('path_php_binary');

// install signal handlers for UNIX types only
if (function_exists('pcntl_signal')) {
	pcntl_signal(SIGTERM, 'sig_handler');
	pcntl_signal(SIGINT, 'sig_handler');
}

// take the start time to log performance data
$start = microtime(true);

foreach ($parms as $parameter) {
	if (str_contains($parameter, '=')) {
		[$arg, $value] = explode('=', $parameter, 2);
	} else {
		$arg   = $parameter;
		$value = '';
	}

	switch ($arg) {
		case '--start':
			$start_time = strtotime($value);

			break;
		case '--end':
			$end_time = strtotime($value);

			break;
		case '--threads':
			$threads = $value;

			break;
		case '--method':
			$method = $value;

			break;
		case '--avgnan':
			$avgnan = $value;

			break;
		case '--host-ids':
			$host_ids = $value;

			break;
		case '--child':
			$child = intval($value);

			break;
		case '-f':
		case '--force':
			$force = true;

			break;
		case '-d':
		case '--debug':
			$debug = true;

			break;
		case '-v':
		case '-V':
		case '--version':
			display_version();

			exit;
		case '-h':
		case '-H':
		case '--help':
			display_help();

			exit;

		default:
			print 'ERROR: Invalid Parameter ' . $parameter . PHP_EOL . PHP_EOL;
			display_help();

			exit;
	}
}

// General time checks
if ($end_time === false || $start_time === false) {
	print 'FATAL: You must provide both --start and an --end dates using \'YYYY-MM-DD HH:MM:SS\' format!' . PHP_EOL;

	exit(1);
}

// Secondary time checks
if ($end_time < $start_time) {
	print 'FATAL: End Time is less than start time!' . PHP_EOL;

	exit(1);
}

// Tertiary and subsequent time checks
if ($end_time < 0) {
	print 'FATAL: End Time is less than 0!' . PHP_EOL;

	exit(1);
}

// Tertiary and subsequent time checks
if ($start_time < 0) {
	print 'FATAL: Start Time is less than 0!' . PHP_EOL;

	exit(1);
}

if ($start_time < strtotime('2019-01-01')) {
	print 'FATAL: Start Time is less than 2019-01-01!.' . PHP_EOL;

	exit(1);
}

if ($method != 'fill' && $method != 'float') {
	print 'FATAL: Invalid --method value.  Options are \'fill\' and \'float\'.' . PHP_EOL;

	exit(1);
}

if ($avgnan != 'last' && $method != 'avg') {
	print 'FATAL: Invalid --avgnan value.  Options are \'last\' and \'avg\'.' . PHP_EOL;

	exit(1);
}

if ($threads <= 0 || $threads > 40) {
	print 'FATAL: Invalid --threads value.  Threads can be from 1 to 40 inclusive.' . PHP_EOL;

	exit(1);
}

if ($host_ids !== false) {
	$host_ids = explode(',', $host_ids);

	foreach ($host_ids as $id) {
		if (!is_numeric($id)) {
			print 'FATAL: The list of --host-ids must be a comma delimited list of numeric Cacti host_ids!' . PHP_EOL;

			exit(1);
		}
	}

	$sql_where = 'WHERE dtd.data_source_path IS NOT NULL AND gl.host_id IN(' . implode(',', $host_ids) . ')';
} else {
	$sql_where = 'WHERE dtd.data_source_path IS NOT NULL';
}

// Command takes a date
$start_date = date('Y-m-d H:i:s', $start_time);
$end_date   = date('Y-m-d H:i:s', $end_time);

// Parent Process, prep table insert records
if ($child == 0) {
	$type = 'master';

	if ($force) {
		printf('NOTE: Looking for and killing running processes.' . PHP_EOL);

		$running = db_fetch_assoc('SELECT *
			FROM processes
			WHERE tasktype = "batchgapfix"');

		if (cacti_sizeof($running)) {
			printf('NOTE: Found %s running processes found.' . PHP_EOL, cacti_sizeof($running));

			foreach ($running as $r) {
				if (cacti_process_still_running((int) $r['pid'])) {
					printf('NOTE: Process with PID: %s being killed.' . PHP_EOL, $r['pid']);

					cacti_process_kill((int) $r['pid'], SIGTERM, 'SYSTEM');
				} else {
					printf('NOTE: Process with PID: %s is no longer the registered process.' . PHP_EOL, $r['pid']);
				}
			}

			db_execute('DELETE FROM processes WHERE tasktype = "batchgapfix"');
		} else {
			printf('NOTE: No running processes found.' . PHP_EOL);
		}
	}

	if (!register_process_start('batchgapfix', 'master', 0, 250000)) {
		print 'FATAL: Detected an already running process.  Use --force to override' . PHP_EOL;

		exit(1);
	}

	if (db_table_exists('graph_local_spikekill')) {
		$running = db_fetch_cell('SELECT COUNT(*) FROM graph_local_spikekill WHERE ended = "0000-00-00"');

		if ($running > 0 && !$force) {
			print 'FATAL: You have requested a start run, and a run appears to be already running' . PHP_EOL;
			print 'FATAL: Check that no processes are running and use the --force option to override.' . PHP_EOL;

			exit(1);
		} else {
			db_execute('TRUNCATE TABLE graph_local_spikekill');
		}
	} else {
		db_execute('CREATE TABLE graph_local_spikekill (
			id INT(10) unsigned NOT NULL auto_increment,
			local_graph_id INT(10) unsigned NOT NULL default "0",
			data_source_path varchar(255) NOT NULL default "0",
			child INT(10) unsigned NOT NULL default "0",
			exit_code INT(10) unsigned NOT NULL default "0",
			started TIMESTAMP NOT NULL default "0000-00-00",
			ended TIMESTAMP NOT NULL default "0000-00-00",
			PRIMARY KEY (id),
			KEY child (child),
			KEY ended (ended))
			ENGINE=MEMORY
			COMMENT="Holds batch spike fill database"');
	}

	// Insert the local graph ids into the spikekill database
	db_execute("INSERT INTO graph_local_spikekill
		(local_graph_id, data_source_path)
		SELECT DISTINCT gl.id, dtd.data_source_path
		FROM graph_local AS gl
		INNER JOIN graph_templates_item AS gti
		ON gl.id = gti.local_graph_id
		INNER JOIN data_template_rrd AS dtr
		ON gti.task_item_id = dtr.id
		INNER JOIN data_template_data AS dtd
		ON dtd.id = dtr.local_data_id
		$sql_where");

	$rrdfiles = db_affected_rows();

	db_execute_prepared('UPDATE graph_local_spikekill
		SET data_source_path = REPLACE(data_source_path, "<path_rra>", ?)',
		[CACTI_PATH_RRA]);

	print "NOTE: There are $rrdfiles RRDfiles that will be checked for gaps and fixed" . PHP_EOL;

	$rrds_per_thread = ceil($rrdfiles / $threads);

	// Distributing RRDfiles into tasks
	for ($i = 1; $i <= $threads; $i++) {
		db_execute_prepared("UPDATE graph_local_spikekill
			SET child = ?
			WHERE child = 0
			LIMIT $rrds_per_thread",
			[$i]);
	}

	$now = date('H:i:s');

	printf('NOTE: %s, Database primed for batch gap fill.' . PHP_EOL, $now);

	// Fork Child Binaries
	for ($i = 1; $i <= $threads; $i++) {
		$args = [
			CACTI_PATH_CLI . '/batchgapfix.php',
			'--start=' . $start_date,
			'--end=' . $end_date,
			'--method=' . $method,
			'--avgnan=' . $avgnan,
			'--child=' . $i
		];

		if ($force) {
			$args[] = '--force';
		}

		if ($debug) {
			$args[] = '--debug';
		}

		$now = date('H:i:s');

		printf('NOTE: %s, Exec in Background: %s %s' . PHP_EOL, $now, $php_bin, implode(' ', $args));

		// the array form has exec_background() escape each argument
		exec_background($php_bin, $args);
	}

	$start = microtime(true);

	while (true) {
		sleep(1);

		$not_finished = db_fetch_cell_prepared('SELECT COUNT(*)
			FROM graph_local_spikekill
			WHERE ended = "0000-00-00"');

		$end = microtime(true);

		$rate = ($rrdfiles - $not_finished) / ($end - $start);

		if ($rate > 0) {
			$estimate = round($rrdfiles / $rate, 0);
			$complete = ($end - $start) - $estimate;
		} else {
			$estimate = 'unknown';
		}

		$now = date('H:i:s');

		if ($not_finished > 0) {
			printf('NOTE: %s, Status %s of %s RRDfiles processed. Total Time is %.0f.' . PHP_EOL,
				$now,
				number_format($rrdfiles - $not_finished),
				number_format($rrdfiles),
				$end - $start);
			printf(('NOTE: %s, Processing Rate: %s RRDfiles per/second, Estimated Complete ' .
				'in: %s seconds, Sleeping 1 seconds.') . PHP_EOL,
				$now,
				round($rate,
					2),
				$estimate);
		} else {
			printf('NOTE: All RRDfiles processed.  Total Time was %.2f seconds.' . PHP_EOL, $end - $start);

			break;
		}
	}

	$succeeded = db_fetch_cell('SELECT COUNT(*) FROM graph_local_spikekill WHERE exit_code = 0');
	$failed    = db_fetch_cell('SELECT COUNT(*) FROM graph_local_spikekill WHERE exit_code != 0');

	cacti_log(sprintf('BATCHFIX STATS: Time:%s, RRDfiles:%s, Threads:%s, Rate:%s, Succeeded:%s, Failed:%s',
		round($end - $start,
			2),
		$rrdfiles,
		$threads,
		round($rate,2),
		$succeeded,
		$failed),
		false,
		'SYSTEM');

	unregister_process('batchgapfix', $type, $child);

	db_execute('TRUNCATE TABLE graph_local_spikekill');

	exit(0);
} else {
	$type = 'child';

	if (!register_process_start('batchgapfix', 'child', $child, 250000)) {
		print 'FATAL: Detected an already running process.  Use --force to override' . PHP_EOL;

		exit(1);
	}
	// Child Process, remediate spikes

	$rrdfiles = db_fetch_assoc_prepared('SELECT *
		FROM graph_local_spikekill
		WHERE child = ?',
		[$child]);

	$start = microtime(true);

	$succeeded = 0;
	$failed    = 0;

	foreach ($rrdfiles as $rrdfile) {
		$output     = [];
		$return_var = 0;

		// Format the command
		$command = sprintf('%s -q %s --rrdfile=%s --outlier-start=%s --outlier-end=%s --method=%s --avgnan=%s',
			cacti_escapeshellcmd($php_bin),
			cacti_escapeshellarg(CACTI_PATH_CLI . '/removespikes.php'),
			cacti_escapeshellarg($rrdfile['data_source_path']),
			cacti_escapeshellarg($start_date),
			cacti_escapeshellarg($end_date),
			cacti_escapeshellarg($method),
			cacti_escapeshellarg($avgnan)
		);

		db_execute_prepared('UPDATE graph_local_spikekill
			SET started = NOW()
			WHERE id = ?',
			[$rrdfile['id']]);

		// Run the command
		debug("NOTE: Running command: $command");

		exec($command, $output, $return_var);

		db_execute_prepared('UPDATE graph_local_spikekill
			SET ended = NOW(), exit_code = ?
			WHERE id = ?',
			[$return_var, $rrdfile['id']]);

		if ($return_var == 0) {
			printf('SUCCESS: Gap Fills for RRDfile:%s' . PHP_EOL, $rrdfile['data_source_path']);
			$succeeded++;
		} else {
			printf('FAILED:  Gap Fills failed for RRDfile:%s' . PHP_EOL, $rrdfile['data_source_path']);
			$failed++;
		}
	}

	$end = microtime(true);

	printf('NOTE: Batch Fill Process Ended in %s seconds.  Succeeded:%s, Failed:%s' . PHP_EOL,
		round($end - $start,
			2),
		$succeeded,
		$failed);

	cacti_log(sprintf('BATCHFIX CHILD STATS: Time:%s, Thread:%s, RRDfiles:%s, Succeeded:%s, Failed:%s',
		round($end - $start,
			2),
		$child,
		cacti_sizeof($rrdfiles),
		$succeeded,
		$failed),
		false,
		'SYSTEM');

	unregister_process('batchgapfix', $type, $child);
}

exit(0);

/**
 * sig_handler - provides a generic means to catch exceptions to the Cacti log.
 *
 * @param int $signo
 *
 * @return void
 */
function sig_handler(int $signo) : void {
	global $child, $type;

	switch ($signo) {
		case SIGTERM:
		case SIGINT:
			unregister_process('batchgapfix', $type, $child);

			if ($child == 0) {
				db_execute('TRUNCATE TABLE graph_local_spikekill');
			}

			exit(1);
		default:
			// ignore all other signals
	}
}

function debug(string $string) : void {
	global $debug;

	if ($debug) {
		print 'DEBUG: ' . trim($string) . PHP_EOL;
	}
}

/**
 * display_version - displays version information
 *
 * @return void
 */
function display_version() : void {
	$version = get_cacti_cli_version();
	print "Cacti Batch Graph Gap Fill Utility, Version $version, " . COPYRIGHT_YEARS . PHP_EOL;
}

/**
 * display_help - displays help information
 *
 * @return void
 */
function display_help() : void {
	display_version();

	print PHP_EOL . 'This utility will fill gaps in graphs based upon a time range.' . PHP_EOL;
	print 'This utility will run in parallel with the given number of threads.' . PHP_EOL;
	print 'If threads argument is not specified, value is derived from the number of processor cores.' . PHP_EOL;
	print 'In case of a detection problem, 2 threads are used.' . PHP_EOL . PHP_EOL;

	print 'selected by the user.' . PHP_EOL . PHP_EOL;

	print 'usage: batchgapfix.php --start=\'YYYY-MM-DD HH:MM:SS\' --end=\'YYYY-MM-DD HH:MM:SS\' [--threads=N]' . PHP_EOL;
	print '       [--method=fill|float] [--avgnan=last|avg] [--host-ids=N,N,N,...]' . PHP_EOL;
	print '       [-f|--force] [-d|--debug]' . PHP_EOL . PHP_EOL;

	print 'Required:' . PHP_EOL;
	print '   --start=\'YYYY-MM-DD HH:MM:SS\' - The start date to check and remove gaps.' . PHP_EOL;
	print '   --end=\'YYYY-MM-DD HH:MM:SS\'   - The end date to check and remove gaps.' . PHP_EOL . PHP_EOL;

	print 'Optional:' . PHP_EOL;
	print '   --threads=N                     - The number of parallel threads [1..40]' . PHP_EOL;
	print '   --method=fill|float             - Default is \'fill\'.  The method to fill gaps.' . PHP_EOL;
	print '   --avgnan=last|avg               - Default is \'last\'.  The number to use to fill gaps.' . PHP_EOL;
	print '   --host-ids=N,N,N,...            - A comma delimited list of Cacti Device ID\'s to process.' . PHP_EOL;
	print '   --force                         - Kill the current running batch gap fill and start over.' . PHP_EOL;
	print '   --debug                         - Higher tracing level for select utilities.' . PHP_EOL . PHP_EOL;
}