<?php
/*
 +-------------------------------------------------------------------------+
 | Copyright (C) 2004-2019 The Cacti Group                                 |
 |                                                                         |
 | This program is free software; you can redistribute it and/or           |
 | modify it under the terms of the GNU General Public License             |
 | as published by the Free Software Foundation; either version 2          |
 | of the License, or (at your option) any later version.                  |
 |                                                                         |
 | This program is distributed in the hope that it will be useful,         |
 | but WITHOUT ANY WARRANTY; without even the implied warranty of          |
 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           |
 | GNU General Public License for more details.                            |
 +-------------------------------------------------------------------------+
 | Cacti: The Complete RRDtool-based Graphing Solution                     |
 +-------------------------------------------------------------------------+
 | This code is designed, written, and maintained by the Cacti Group. See  |
 | about.php and/or the AUTHORS file for specific developer information.   |
 +-------------------------------------------------------------------------+
 | http://www.cacti.net/                                                   |
 +-------------------------------------------------------------------------+
*/

class spikekill {
	/* setup defaults */
	private $std_kills = false;
	private $var_kills = false;
	private $out_kills = false;
	private $out_set   = false;
	private $username  = '';
	private $user      = '';
	private $user_info = array();

	// Required variables
	var $rrdfile   = '';

	var $method    = '';
	var $avgnan    = '';
	var $stddev    = '';

	var $out_start = '';
	var $out_end   = '';
	var $outliers  = '';
	var $percent   = '';
	var $numspike  = '';

	// Overrideable
	var $html      = true;
	var $backup    = false;
	var $debug     = false;
	var $dryrun    = false;

	// Defaults from cacti settings
	private $dmethod   = 1;
	private $dnumspike = 10;
	private $dstddev   = 10;
	private $dpercent  = 500;
	private $doutliers = 5;
	private $davgnan   = 'last';

	// Internal globals
	private $tempdir     = '';
	private $seed        = '';
	private $strout      = '';
	private $ds_min      = '';
	private $ds_max      = '';
	private $total_kills = 0;

	private $rra_cf      = array();
	private $ds_name     = array();
	private $rra_pdp     = array();
	private $step        = 0;

	// For error handling
	private $errors = array();

	public function __construct($rrdfile = '', $method = '', $avgnan = '', $stddev = '',
		$out_start = '', $out_end = '', $outliers = '', $percent = '', $numspike = '') {

		$this->username  = 'OsUser:' . get_current_user();
		$this->user_info = array();

		if (isset($_SESSION['sess_user_id'])) {
			$this->user = $_SESSION['sess_user_id'];

			/* confirm the user id is accurate */
			$this->user_info = db_fetch_row_prepared('SELECT id, username
				FROM user_auth
				WHERE id = ?',
				array($this->user));

			if (sizeof($this->user_info)) {
				$this->username = 'CactiUser:' . $this->user_info['username'];
			}
		}

		if ($rrdfile != '') {
			$this->rrdfile = $rrdfile;
		}

		if ($method != '') {
			$this->method = $method;
		}

		if ($avgnan != '') {
			$this->avgnan = $avgnan;
		}

		if ($stddev != '') {
			$this->stddev = $stddev;
		}

		if ($out_start != '') {
			$this->out_start = $out_start;
		}

		if ($out_end != '') {
			$this->out_end = $out_end;
		}

		if ($outliers != '') {
			$this->outliers = $outliers;
		}

		if ($percent != '') {
			$this->percent = $percent;
		}

		if ($numspike != '') {
			$this->numspike = $numspike;
		}

		$this->dmethod   = read_config_option('spikekill_method', 1);
		$this->dnumspike = read_config_option('spikekill_number', 10);
		$this->dstddev   = read_config_option('spikekill_deviations', 10);
		$this->dpercent  = read_config_option('spikekill_percent', 500);
		$this->doutliers = read_config_option('spikekill_outliers', 5);
		$this->davgnan   = read_config_option('spikekill_avgnan', 'last');

		return true;
	}

	public function __destruct() {
		return true;
	}

	private function set_error($string) {
		$this->errors[] = $string;
	}

	private function is_error_set() {
		return sizeof($this->errors);
	}

	public function get_errors() {
		$output = '';

		if (cacti_sizeof($this->errors)) {
			foreach($this->errors as $error) {
				$output .= ($output != '' ? ($this->html ? '<br>':"\n"):'') . $error;
			}
		}

		return $output;
	}

	public function get_output($html = true) {
		return $this->strout;
	}

	private function initialize_spikekill() {
		/* additional error check */
		if ($this->rrdfile == '') {
			$this->set_error("FATAL: You must specify an RRDfile!");
		}

		if (!file_exists($this->rrdfile)) {
			$this->set_error("FATAL: File '$this->rrdfile' does not exist.");
		} elseif (!is_writable($this->rrdfile)) {
			$this->set_error("FATAL: File '$this->rrdfile' is not writeable by '" . get_execution_user() . "'.");
		}

		$umethod   = read_user_setting('spikekill_method', $this->dmethod);
		$unumspike = read_user_setting('spikekill_number', $this->dnumspike);
		$ustddev   = read_user_setting('spikekill_deviations', $this->dstddev);
		$upercent  = read_user_setting('spikekill_percent', $this->dpercent);
		$uoutliers = read_user_setting('spikekill_outliers', $this->doutliers);
		$uavgnan   = read_user_setting('spikekill_avgnan', $this->davgnan);

		/* set the correct value */
		if ($this->avgnan == '') {
			if (!isset($uavgnan)) {
				$this->avgnan = $davgnan;
			} else {
				$this->avgnan = $uavgnan;
			}
		}

		if ($this->method == '') {
			if (!isset($umethod)) {
				$this->method = $dmethod;
			} else {
				$this->method = $umethod;
			}
		}

		if ($this->numspike == '') {
			if (!isset($unumspike)) {
				$this->numspike = $dnumspike;
			} else {
				$this->numspike = $unumspike;
			}
		}

		if ($this->stddev == '') {
			if (!isset($ustddev)) {
				$this->stddev = $dstddev;
			} else {
				$this->stddev = $ustddev;
			}
		}

		if ($this->percent == '') {
			if (!isset($upercent)) {
				$this->percent = $dpercent/100;
			} else {
				$this->percent = $upercent/100;
			}
		}

		if ($this->outliers == '') {
			if (!isset($uoutliers)) {
				$this->outliers = $doutliers;
			} else {
				$this->outliers = $uoutliers;
			}
		}

		if (!is_numeric($this->stddev) || ($this->stddev < 1)) {
			$this->set_error("FATAL: Standard Deviation must be a positive integer.");
		}

		if ($this->method == 'float' || $this->method == 'fill') {
			if (!is_numeric($this->out_start)) {
				$this->out_start == strtotime($this->out_start);
			}

			if (!is_numeric($this->out_end)) {
				$this->out_end == strtotime($this->out_end);
			}

			if ($this->out_start === false || $this->out_end === false) {
				$this->set_error("FATAL: The outlier-start and outlier-end arguments must be in the format of YYYY-MM-DD HH:MM.");
			}

			if (!is_numeric($this->outliers) || ($this->outliers < 1)) {
				$this->set_error("FATAL: The number of outliers to exlude must be a positive integer.");
			}

			$this->out_set = true;
		}

		if ($this->percent != '') {
			if (is_numeric($this->percent) && $this->percent > 0) {
				$this->percent = $this->percent/100;
			} else {
				$this->set_error("FATAL: Percent deviation must be a positive floating point number.");
			}
		}

		if (!$this->numspike != '') {
			if (!is_numeric($this->numspike) || ($this->numspike < 1)) {
				$this->set_error("FATAL: Number of spikes to remove must be a positive integer");
			}
		}

		if ((!empty($this->out_start) && empty($this->out_end)) || (!empty($this->out_end) && empty($this->out_start))) {
			$this->set_error("FATAL: Outlier time range requires outlier-start and outlier-end to be specified.");
		}

		if (!empty($this->out_start)) {
			if ($this->out_start >= $this->out_end) {
				$this->set_error("FATAL: Outlier time range requires outlier-start to be less than outlier-end.");
			}
		}

		switch($this->method) {
			case 'variance':
				$this->method = 2;
				break;
			case 'stddev':
				$this->method = 1;
				break;
			case 'float':
				$this->method = 3;
				break;
			case 'fill':
				$this->method = 4;
				break;
			default:
				$this->set_error("FATAL: You must specify either 'stddev', 'variance', 'float', or 'fill' as methods.");
		}

		if ($this->method == 3 && empty($this->out_start)) {
			$this->set_error("FATAL: The 'float' removal method requires the specification of a start and end date.");
		}

		switch($this->avgnan) {
			case 'avg':
			case 'last':
			case 'nan':
				break;
			default:
				$this->set_error("FATAL: You must specify either 'last', 'avg' or 'nan' as a replacement method.");
		}

		return false;
	}

	public function remove_spikes() {
		global $config;

		$this->initialize_spikekill();

		if ($this->is_error_set()) {
			return false;
		}

		/* determine the temporary file name */
		$this->seed = mt_rand();

		if ($config['cacti_server_os'] == 'win32') {
			$this->tempdir  = read_config_option('spikekill_backupdir');
			$xmlfile = $this->tempdir . '/' . str_replace('.rrd', '', basename($this->rrdfile)) . '.dump.' . $this->seed;
			$bakfile = $this->tempdir . '/' . str_replace('.rrd', '', basename($this->rrdfile)) . '.backup.' . $this->seed . '.rrd';
		} else {
			$this->tempdir = read_config_option('spikekill_backupdir');
			$xmlfile = $this->tempdir . '/' . str_replace('.rrd', '', basename($this->rrdfile)) . '.dump.' . $this->seed;
			$bakfile = $this->tempdir . '/' . str_replace('.rrd', '', basename($this->rrdfile)) . '.backup.' . $this->seed . '.rrd';
		}

		$this->strout = '';

		if (!empty($this->out_start) && !$this->dryrun) {
			$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') . "NOTE: Removing Outliers in Range and Replacing with Last" . ($this->html ? "</p>\n":"\n");
		}

		/* execute the dump command */
		$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') . "NOTE: Creating XML file '$xmlfile' from '$this->rrdfile'" . ($this->html ? "</p>\n":"\n");

		if (!$this->dryrun) {
			switch ($this->method) {
			case 1:
				$mm  = 'StdDev';
				$mes = "$this->username, File:" . basename($this->rrdfile) . ", Method:$mm, StdDevs:$this->stddev, AvgNan:$this->avgnan, Kills:$this->numspike, Outliers:$this->outliers";
				break;
			case 2:
				$mm  = 'Variance';
				$mes = "$this->username, File:" . basename($this->rrdfile) . ", Method:$mm, AvgNan:$this->avgnan, Kills:$this->numspike, Outliers:$this->outliers, Percent:$this->percent";
				break;
			case 3:
				$mm  = 'RangeFill';
				$mes = "$this->username, File:" . basename($this->rrdfile) . ", Method:$mm, OutStart:$this->out_start, OutEnd:$this->out_end, AvgNan:$this->avgnan";
				break;
			case 4:
				$mm  = 'GapFill';
				$mes = "$this->username, File:" . basename($this->rrdfile) . ", Method:$mm, OutStart:$this->out_start, OutEnd:$this->out_end, AvgNan:$this->avgnan";
				break;
			default:
				$mm  = 'Undefined';
				$mes = "$this->username, File:" . basename($this->rrdfile) . ", Method:$mm";
			}

			cacti_log($mes, false, 'SPIKEKILL');
		}

		shell_exec(cacti_escapeshellcmd(read_config_option('path_rrdtool')) . ' dump ' . cacti_escapeshellarg($this->rrdfile) . ' > ' . cacti_escapeshellarg($xmlfile));

		/* read the xml file into an array*/
		if (file_exists($xmlfile)) {
			$output = file($xmlfile);

			/* remove the temp file */
			unlink($xmlfile);
		} else {
			$this->set_error("FATAL: RRDtool Command Failed.  Please verify that the RRDtool path is valid in Settings->Paths!");
			return false;
		}

		/* backup the rrdfile if requested */
		if ($this->backup && !$this->dryrun) {
			if (copy($this->rrdfile, $bakfile)) {
				$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') . "NOTE: RRDfile '$this->rrdfile' backed up to '$bakfile'" . ($this->html ? "</p>\n":"\n");
			} else {
				$this->set_error("FATAL: RRDfile Backup of '$this->rrdfile' to '$bakfile' FAILED!");
				return false;
			}
		}

		if ($this->is_error_set()) {
			return false;
		}

		/* process the xml file and remove all comments */
		$output = $this->removeComments($output);

		/* Read all the rra's ds values and obtain the following pieces of information from each
		   rra archive.

		   * numsamples - The number of 'valid' non-nan samples
		   * sumofsamples - The sum of all 'valid' samples.
		   * average - The average of all samples
		   * standard_deviation - The standard deviation of all samples
		   * max_value - The maximum value of all samples
		   * min_value - The minimum value of all samples
		   * max_cutoff - Any value above this value will be set to the average.
		   * min_cutoff - Any value lower than this value will be set to the average.

		   This will end up being a n-dimensional array as follows:
		   rra[x][ds#]['totalsamples'];
		   rra[x][ds#]['numsamples'];
		   rra[x][ds#]['sumofsamples'];
		   rra[x][ds#]['average'];
		   rra[x][ds#]['stddev'];
		   rra[x][ds#]['max_value'];
		   rra[x][ds#]['min_value'];
		   rra[x][ds#]['max_cutoff'];
		   rra[x][ds#]['min_cutoff'];

		   There will also be a secondary array created with the actual samples.  This
		   array will be used to calculate the standard deviation of the sample set.
		   samples[rra_num][ds_num][timestamp];

		   Also track the min and max value for each ds and store it into the two
		   arrays: ds_min[ds#], ds_max[ds#].

		   The we don't need to know the type of rra, only it's number for this analysis
		   the same applies for the ds' as well.
		*/
		$rra     = array();
		$this->rra_cf  = array();
		$this->rra_pdp = array();

		$rra_num = 0;
		$ds_num  = 0;

		$this->total_kills = 0;

		$in_rra  = false;
		$in_db   = false;

		$this->ds_min  = array();
		$this->ds_max  = array();

		$this->ds_name = array();

		/* perform a first pass on the array and do the following:
		   1) Get the number of good samples per ds
		   2) Get the sum of the samples per ds
		   3) Get the max and min values for all samples
		   4) Build both the rra and sample arrays
		   5) Get each ds' min and max values
		*/
		if (cacti_sizeof($output)) {
			foreach($output as $line) {
				if (substr_count($line, '<v>')) {
					$linearray = explode('<v>', $line);

					/* get the timestamp */
					$timestamp_part = $linearray[0];
					if (strpos($timestamp_part, '<timestamp>') !== false) {
						$timestamp_part = str_replace('<row><timestamp>', '', $timestamp_part);
						$timestamp_part = str_replace('</timestamp>', '', $timestamp_part);
						$timestamp = trim($timestamp_part);
					} else {
						$timestamp = 0;
					}

					/* discard the row */
					array_shift($linearray);
					$ds_num = 0;
					foreach($linearray as $dsvalue) {
						/* peel off garbage */
						$dsvalue = trim(str_replace('</row>', '', str_replace('</v>', '', $dsvalue)));

						/* check for outlier territory */
						if ($timestamp > 0) {
							if (!empty($this->out_start) && $timestamp < $this->out_start) {
								$process = true;
							} elseif (!empty($this->out_end) && $timestamp > $this->out_end) {
								$process = true;
							} elseif (empty($this->out_start)) {
								$process = true;
							} else {
								$process = false;
							}
						} else {
							$process = true;
						}

						if (strtolower($dsvalue) != 'nan' && $process) {
							if (!isset($rra[$rra_num][$ds_num]['numsamples'])) {
								$rra[$rra_num][$ds_num]['numsamples'] = 1;
							} else {
								$rra[$rra_num][$ds_num]['numsamples']++;
							}

							if (!isset($rra[$rra_num][$ds_num]['sumofsamples'])) {
								$rra[$rra_num][$ds_num]['sumofsamples'] = $dsvalue;
							} elseif (is_numeric($dsvalue)) {
								$rra[$rra_num][$ds_num]['sumofsamples'] += $dsvalue;
							}

							if (!isset($rra[$rra_num][$ds_num]['max_value'])) {
								$rra[$rra_num][$ds_num]['max_value'] = $dsvalue;
							} elseif ($dsvalue > $rra[$rra_num][$ds_num]['max_value']) {
								$rra[$rra_num][$ds_num]['max_value'] = $dsvalue;
							}

							if (!isset($rra[$rra_num][$ds_num]['min_value'])) {
								$rra[$rra_num][$ds_num]['min_value'] = $dsvalue;
							} elseif ($dsvalue < $rra[$rra_num][$ds_num]['min_value']) {
								$rra[$rra_num][$ds_num]['min_value'] = $dsvalue;
							}
						}

						/* store the sample for standard deviation calculation */
						if ($timestamp == 0) {
							$samples[$rra_num][$ds_num][] = $dsvalue;
						} else {
							$samples[$rra_num][$ds_num][$timestamp] = $dsvalue;
						}

						if (!isset($rra[$rra_num][$ds_num]['totalsamples'])) {
							$rra[$rra_num][$ds_num]['totalsamples'] = 1;
						} else {
							$rra[$rra_num][$ds_num]['totalsamples']++;
						}

						$ds_num++;
					}
				} elseif (substr_count($line, '<rra>')) {
					$in_rra = true;
				} elseif (substr_count($line, '<min>')) {
					$this->ds_min[] = trim(str_replace('<min>', '', str_replace('</min>', '', trim($line))));
				} elseif (substr_count($line, '<max>')) {
					$this->ds_max[] = trim(str_replace('<max>', '', str_replace('</max>', '', trim($line))));
				} elseif (substr_count($line, '<name>')) {
					$this->ds_name[] = trim(str_replace('<name>', '', str_replace('</name>', '', trim($line))));
				} elseif (substr_count($line, '<cf>')) {
					$this->rra_cf[] = trim(str_replace('<cf>', '', str_replace('</cf>', '', trim($line))));
				} elseif (substr_count($line, '<pdp_per_row>')) {
					$this->rra_pdp[] = trim(str_replace('<pdp_per_row>', '', str_replace('</pdp_per_row>', '', trim($line))));
				} elseif (substr_count($line, '</rra>')) {
					$in_rra = false;
					$rra_num++;
				} elseif (substr_count($line, '<step>')) {
					$this->step = trim(str_replace('<step>', '', str_replace('</step>', '', trim($line))));
				}
			}
		}

		/* For all the samples determine the average with the outliers removed */
		$this->calculateVarianceAverages($rra, $samples);

		/* Now scan the rra array and the samples array and calculate the following
		   1) The standard deviation of all samples
		   2) The average of all samples per ds
		   3) The max and min cutoffs of all samples
		   4) The number of kills in each ds based upon the thresholds
		*/

		if (empty($this->out_start)) {
			$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
				"NOTE: Searching for Spikes in XML file '$xmlfile'" . ($this->html ? "</p>\n":"\n");
		}

		$this->calculateOverallStatistics($rra, $samples);

		/* debugging and/or status report */
		if ($this->debug || $this->dryrun) {
			if ($this->html) {
				$this->strout .= "<table style='width:100%' class='spikekillData' id='spikekillData'>";
			}

			$this->outputStatistics($rra);

			if ($this->html) {
				$this->strout .= '</table>';
			}
		}

		/* create an output array */
		$output = false;
		if ($this->method == 1) {
			/* standard deviation subroutine */
			if ($this->std_kills || $this->out_kills) {
				$this->debug('Either std_kills or out_kills found');

				if (!$this->dryrun) {
					$new_output = $this->updateXML($output, $rra);
					$output     = true;
				}
			} elseif (!empty($this->out_start)) {
				$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
					"NOTE: NO Window Spikes found in '$this->rrdfile'" . ($this->html ? "</p>\n":"\n");
			} else {
				$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
					"NOTE: NO Standard Deviation found in '$this->rrdfile'" . ($this->html ? "</p>\n":"\n");
			}
		} else {
			/* variance subroutine */
			if ($this->var_kills || $this->out_kills) {
				$this->debug('Either variance or out_kills found');

				if (!$this->dryrun) {
					$new_output = $this->updateXML($output, $rra);
					$output     = true;
				}
			} elseif (!empty($this->out_start)) {
				$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
					"NOTE: NO Window Fills found in '$this->rrdfile'" . ($this->html ? "</p>\n":"\n");
			} else {
				$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
					"NOTE: NO Variance Spikes found in '$this->rrdfile'" . ($this->html ? "</p>\n":"\n");
			}
		}

		/* finally update the file XML file and Reprocess the RRDfile */
		if (!$this->dryrun) {
			if ($output) {
				if ($this->writeXMLFile($new_output, $xmlfile)) {
					if ($this->backupRRDFile($this->rrdfile)) {
						$this->createRRDFileFromXML($xmlfile, $this->rrdfile);
						$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
							"NOTE: Spikes Found and Remediated.  Total Spikes ($this->total_kills)" . ($this->html ? "</p>\n":"\n");
					} else {
						$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
							"FATAL: Unable to backup '$this->rrdfile'" . ($this->html ? "</p>\n":"\n");
					}
				} else {
					$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
						"FATAL: Unable to write XML file '$xmlfile'" . ($this->html ? "</p>\n":"\n");
				}
			} else {
				$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
					"NOTE: No Spikes Found.  No remediation performed." . ($this->html ? "</p>\n":"\n");
			}
		} else {
			$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
				"NOTE: Dryrun requested.  No updates performed" . ($this->html ? "</p>\n":"\n");
		}

		$this->strout .= ($this->html ? "</table>":'');

		if ($this->total_kills > 0) {
			cacti_log("WARNING: Removed '$this->total_kills' Spikes from '$this->rrdfile', Method:'$this->method'", false, 'WEBUI');
		} elseif($this->debug) {
			cacti_log("NOTE: Removed '$this->total_kills' Spikes from '$this->rrdfile', Method:'$this->method'", false, 'WEBUI');
		}

		if (file_exists($xmlfile)) {
			unlink($xmlfile);
		}

		if (file_exists($xmlfile)) {
			unlink($bakfile);
		}

		return true;
	}

	/* All Functions */
	private function createRRDFileFromXML($xmlfile, $rrdfile) {
		/* execute the dump command */
		$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
			"NOTE: Re-Importing '$xmlfile' to '$rrdfile'" . ($this->html ? "</p>\n":"\n");

		$response = shell_exec(cacti_escapeshellcmd(read_config_option('path_rrdtool')) . ' restore -f -r ' . cacti_escapeshellarg($xmlfile) . ' ' . cacti_escapeshellarg($rrdfile));

		if ($response != '') {
			$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') . $response . ($this->html ? "</p>\n":"\n");
		}
	}

	private function writeXMLFile($output, $xmlfile) {
		return file_put_contents($xmlfile, $output);
	}

	private function backupRRDFile($rrdfile) {
		$backupdir = read_config_option('spikekill_backupdir');

		if ($backupdir == '') {
			$backupdir = $this->tempdir;
		}

		if (file_exists($backupdir . '/' . basename($rrdfile))) {
			$newfile = basename($rrdfile) . '.' . $this->seed;
		} else {
			$newfile = basename($rrdfile);
		}

		$this->strout .= ($this->html ? "<p class='spikekillNote'>":'') .
			"NOTE: Backing Up '$rrdfile' to '" . $backupdir . '/' .  $newfile . "'" . ($this->html ? "</p>\n":"\n");

		return copy($rrdfile, $backupdir . "/" . $newfile);
	}

	private function calculateVarianceAverages(&$rra, &$samples) {
		if (cacti_sizeof($samples)) {
			foreach($samples as $rra_num => $dses) {
				if (cacti_sizeof($dses)) {
					foreach($dses as $ds_num => $ds) {
						if (empty($this->out_start)) {
							if (cacti_sizeof($ds) < $this->outliers * 3) {
								$rra[$rra_num][$ds_num]['variance_avg'] = 'NAN';
							} else {
								$myds = $ds;
								$myds = array_filter($myds, array($this, 'removeNanFromSamples'));

								/* remove high outliers */
								rsort($myds, SORT_NUMERIC);
								$myds = array_slice($myds, $this->outliers);

								/* remove low outliers */
								sort($ds, SORT_NUMERIC);
								$myds = array_slice($myds, $this->outliers);

								if (cacti_sizeof($myds)) {
									$rra[$rra_num][$ds_num]['variance_avg'] = array_sum($myds) / cacti_sizeof($myds);
								} else {
									$rra[$rra_num][$ds_num]['variance_avg'] = 'NAN';
								}
							}
						} else {
							if (isset($rra[$rra_num][$ds_num]['sumofsamples']) && isset($rra[$rra_num][$ds_num]['numsamples'])) {
								if ($rra[$rra_num][$ds_num]['numsamples'] > 0) {
									$rra[$rra_num][$ds_num]['variance_avg'] = $rra[$rra_num][$ds_num]['sumofsamples'] / $rra[$rra_num][$ds_num]['numsamples'];
								} else {
									$rra[$rra_num][$ds_num]['variance_avg'] = 0;
								}
							} else {
								$rra[$rra_num][$ds_num]['variance_avg'] = 0;
							}
						}
					}
				}
			}
		}
	}

	private function removeNanFromSamples(&$string) {
		return stripos($string, 'nan') === false;
	}

	private function calculateOverallStatistics(&$rra, &$samples) {
		$rra_num = 0;
		if (cacti_sizeof($rra)) {
			foreach($rra as $dses) {
				$ds_num = 0;

				if (cacti_sizeof($dses)) {
					foreach($dses as $ds) {
						if (isset($samples[$rra_num][$ds_num])) {
							$rra[$rra_num][$ds_num]['standard_deviation'] = $this->processStandardDeviationCalculation($samples[$rra_num][$ds_num]);
							if ($rra[$rra_num][$ds_num]['standard_deviation'] == 'NAN') {
								$rra[$rra_num][$ds_num]['standard_deviation'] = 0;
							}

							if (isset($rra[$rra_num][$ds_num]['sumofsamples']) && isset($rra[$rra_num][$ds_num]['numsamples'])) {
								if ($rra[$rra_num][$ds_num]['numsamples'] > 0) {
									$rra[$rra_num][$ds_num]['average'] = $rra[$rra_num][$ds_num]['sumofsamples'] / $rra[$rra_num][$ds_num]['numsamples'];
								} else {
									$rra[$rra_num][$ds_num]['average'] = 0;
								}
							} else {
								$rra[$rra_num][$ds_num]['average'] = 0;
							}

							$rra[$rra_num][$ds_num]['min_cutoff'] = $rra[$rra_num][$ds_num]['average'] - ($this->stddev * $rra[$rra_num][$ds_num]['standard_deviation']);
							if ($rra[$rra_num][$ds_num]['min_cutoff'] < $this->ds_min[$ds_num]) {
								$rra[$rra_num][$ds_num]['min_cutoff'] = $this->ds_min[$ds_num];
							}

							$rra[$rra_num][$ds_num]['max_cutoff'] = $rra[$rra_num][$ds_num]['average'] + ($this->stddev * $rra[$rra_num][$ds_num]['standard_deviation']);
							if ($rra[$rra_num][$ds_num]['max_cutoff'] > $this->ds_max[$ds_num]) {
								$rra[$rra_num][$ds_num]['max_cutoff'] = $this->ds_max[$ds_num];
							}

							$rra[$rra_num][$ds_num]['numnksamples'] = 0;
							$rra[$rra_num][$ds_num]['sumnksamples'] = 0;
							$rra[$rra_num][$ds_num]['avgnksamples'] = 0;

							/* go through values and find cutoffs */
							$rra[$rra_num][$ds_num]['stddev_killed']   = 0;
							$rra[$rra_num][$ds_num]['variance_killed'] = 0;
							$rra[$rra_num][$ds_num]['outwind_killed']  = 0;

							/* kill what is required to be killed */
							if (cacti_sizeof($samples[$rra_num][$ds_num])) {
								foreach($samples[$rra_num][$ds_num] as $timestamp => $sample) {
									if (!empty($this->out_start) && $timestamp >= $this->out_start && $timestamp <= $this->out_end) {
										if ($this->method == 3) {
											$this->debug(sprintf("Window Kill: Value '%.4e', Time '%s'", $sample, date('Y-m-d H:i', $timestamp)));

											$rra[$rra_num][$ds_num]['outwind_killed']++;
											$this->out_kills = true;
										} elseif ($this->method == 4) {
											if ($sample > (1+$this->percent)*$rra[$rra_num][$ds_num]['variance_avg'] || strtolower($sample) == 'nan') {
												$this->debug(sprintf("Window Kill: Value '%.4e', Time '%s'", $sample, date('Y-m-d H:i', $timestamp)));

												$rra[$rra_num][$ds_num]['outwind_killed']++;
												$this->out_kills = true;
											}
										}
									} elseif (($sample > $rra[$rra_num][$ds_num]['max_cutoff']) ||
										($sample < $rra[$rra_num][$ds_num]['min_cutoff'])) {
										if ($this->method == 2) {
											$this->debug(sprintf("Std Kill: Value '%.4e', StandardDev '%.4e', StdDevLimit '%.4e'", $sample, $rra[$rra_num][$ds_num]['standard_deviation'], ($rra[$rra_num][$ds_num]['max_cutoff'] * (1+$this->percent))));

											$rra[$rra_num][$ds_num]['stddev_killed']++;
											$this->std_kills = true;
										}
									} elseif (is_numeric($sample)) {
										$rra[$rra_num][$ds_num]['numnksamples']++;
										$rra[$rra_num][$ds_num]['sumnksamples'] += $sample;
									}

									if (!empty($this->out_start) && $timestamp >= $this->out_start && $timestamp <= $this->out_end) {
										// Already calculated
									} elseif ($rra[$rra_num][$ds_num]['variance_avg'] == 'NAN') {
										/* not enought samples to calculate */
									} elseif ($sample > ($rra[$rra_num][$ds_num]['variance_avg'] * (1+$this->percent))) {
										if ($this->method == 1) {
											/* kill based upon variance */
											$this->debug(sprintf("Var Kill: Value '%.4e', VarianceDev '%.4e', VarianceLimit '%.4e'", $sample, $rra[$rra_num][$ds_num]['variance_avg'], ($rra[$rra_num][$ds_num]['variance_avg'] * (1+$this->percent))));

											$rra[$rra_num][$ds_num]['variance_killed']++;
											$this->var_kills = true;
										}
									}
								}
							}

							if ($rra[$rra_num][$ds_num]['numnksamples'] > 0) {
								$rra[$rra_num][$ds_num]['avgnksamples'] = $rra[$rra_num][$ds_num]['sumnksamples'] / $rra[$rra_num][$ds_num]['numnksamples'];
							}
						} else {
							$rra[$rra_num][$ds_num]['standard_deviation'] = 'N/A';
							$rra[$rra_num][$ds_num]['average']            = 'N/A';
							$rra[$rra_num][$ds_num]['min_cutoff']         = 'N/A';
							$rra[$rra_num][$ds_num]['max_cutoff']         = 'N/A';
							$rra[$rra_num][$ds_num]['numnksamples']       = 'N/A';
							$rra[$rra_num][$ds_num]['sumnksamples']       = 'N/A';
							$rra[$rra_num][$ds_num]['avgnksamples']       = 'N/A';
							$rra[$rra_num][$ds_num]['stddev_killed']      = 'N/A';
							$rra[$rra_num][$ds_num]['variance_killed']    = 'N/A';
							$rra[$rra_num][$ds_num]['stddev_killed']      = 'N/A';
							$rra[$rra_num][$ds_num]['outwind_killed']     = 'N/A';
							$rra[$rra_num][$ds_num]['numnksamples']       = 'N/A';
							$rra[$rra_num][$ds_num]['sumnksamples']       = 'N/A';
							$rra[$rra_num][$ds_num]['variance_killed']    = 'N/A';
							$rra[$rra_num][$ds_num]['avgnksamples']       = 'N/A';
						}

						$ds_num++;
					}
				}

				$rra_num++;
			}
		}
	}

	private function outputStatistics($rra) {
		if (cacti_sizeof($rra)) {
			if (!$this->html) {
				$this->strout .= "\n";
				$this->strout .= sprintf("%10s %16s %10s %7s %7s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s\n",
					'Size', 'DataSource', 'CF', 'Samples', 'NonNan', 'Avg', 'StdDev',
					'MaxValue', 'MinValue', 'MaxStdDev', 'MinStdDev', 'StdKilled', 'VarKilled', 'WindFilled', 'StdDevAvg', 'VarAvg');
				$this->strout .= sprintf("%10s %16s %10s %7s %7s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s\n",
					'----------', '---------------', '----------', '-------', '-------', '----------', '----------',
					'----------', '----------', '----------', '----------', '----------', '----------', '----------',
					'----------', '----------', '----------');

				foreach($rra as $rra_key => $dses) {
					if (cacti_sizeof($dses)) {
						foreach($dses as $dskey => $ds) {
							$this->strout .= sprintf('%10s %16s %10s %7s %7s ' .
								($ds['average'] < 1E6 ? '%10s ':'%10.4e ') .
								($ds['standard_deviation'] < 1E6 ? '%10s ':'%10.4e ') .
								(isset($ds['max_value'])  ? ($ds['max_value']  < 1E6 ? '%10s ':'%10.4e ') : '%10s ') .
								(isset($ds['min_value'])  ? ($ds['min_value']  < 1E6 ? '%10s ':'%10.4e ') : '%10s ') .
								(isset($ds['max_cutoff']) ? ($ds['max_cutoff'] < 1E6 ? '%10s ':'%10.4e ') : '%10s ') .
								(isset($ds['min_cutoff']) ? ($ds['min_cutoff'] < 1E6 ? '%10s ':'%10.4e ') : '%10s ') .
								'%10s %10s %10s ' .
								(isset($ds['avgnksamples']) ? ($ds['avgnksamples'] < 1E6 ? '%10s ':'%10.4e ') : '%10.4E ') .
								(isset($ds['variance_avg']) ? ($ds['variance_avg'] < 1E6 ? '%10s ':'%10.4e ') : '%10.4E ') . "\n",
								$this->displayTime($this->rra_pdp[$rra_key]),
								$this->ds_name[$dskey],
								$this->rra_cf[$rra_key],
								$ds['totalsamples'],
								(isset($ds['numsamples']) ? $ds['numsamples'] : '0'),
								($ds['average'] != 'N/A' ? round($ds['average'],2) : $ds['average']),
								($ds['standard_deviation'] != 'N/A' ? round($ds['standard_deviation'],2) : $ds['standard_deviation']),
								(isset($ds['max_value']) ? round($ds['max_value'],2) : 'N/A'),
								(isset($ds['min_value']) ? round($ds['min_value'],2) : 'N/A'),
								($ds['max_cutoff'] != 'N/A' ? round($ds['max_cutoff'],2) : $ds['max_cutoff']),
								($ds['min_cutoff'] != 'N/A' ? round($ds['min_cutoff'],2) : $ds['min_cutoff']),
								$ds['stddev_killed'],
								$ds['variance_killed'],
								$ds['outwind_killed'],
								($ds['avgnksamples'] != 'N/A' ? round($ds['avgnksamples'],2) : $ds['avgnksamples']),
								(isset($ds['variance_avg']) ? round($ds['variance_avg'],2) : 'N/A'));
						}
					}
				}

				$this->strout .= "\n";
			} else {
				$this->strout .= sprintf("<tr class='tableHeader'><th style='width:10%%;'>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th></tr>\n",
					'Size', 'DataSource', 'CF', 'Samples', 'NonNan', 'Avg', 'StdDev',
					'MaxValue', 'MinValue', 'MaxStdDev', 'MinStdDev', 'StdKilled', 'VarKilled', 'WindFilled', 'StdDevAvg', 'VarAvg');
				foreach($rra as $rra_key => $dses) {
					if (cacti_sizeof($dses)) {
						foreach($dses as $dskey => $ds) {
							$this->strout .= sprintf('<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>' .
								($ds['average'] < 1E6 ? '%s</td><td>':'%.4e</td><td>') .
								($ds['standard_deviation'] < 1E6 ? '%s</td><td>':'%.4e</td><td>') .
								(isset($ds['max_value']) ? ($ds['max_value'] < 1E6 ? '%s</td><td>':'%.4e</td><td>') : '%s</td><td>') .
								(isset($ds['min_value']) ? ($ds['min_value'] < 1E6 ? '%s</td><td>':'%.4e</td><td>') : '%s</td><td>') .
								(isset($ds['max_cutoff']) ? ($ds['max_cutoff'] < 1E6 ? '%s</td><td>':'%.4e</td><td>') : '%s</td><td>') .
								(isset($ds['min_cutoff']) ? ($ds['min_cutoff'] < 1E6 ? '%s</td><td>':'%.4e</td><td>') : '%s</td><td>') .
								'%s</td><td>%s</td><td>%s</td><td>' .
								(isset($ds['avgnksampled']) ? ($ds['avgnksamples'] < 1E6 ? '%s</td><td>':'%.4e</td><td>') : '%s</td><td>') .
								(isset($ds['variance_avg']) ? ($ds['variance_avg'] < 1E6 ? "%s</td></tr>\n":"%.4e</td></tr>\n") : "%s</td></tr>\n") . "\n",
								$this->displayTime($this->rra_pdp[$rra_key]),
								$this->ds_name[$dskey],
								$this->rra_cf[$rra_key],
								$ds['totalsamples'],
								(isset($ds['numsamples']) ? $ds['numsamples'] : '0'),
								($ds['average'] != 'N/A' ? round($ds['average'],2) : $ds['average']),
									($ds['standard_deviation'] != 'N/A' ? round($ds['standard_deviation'],2) : $ds['standard_deviation']),
								(isset($ds['max_value']) ? round($ds['max_value'],2) : 'N/A'),
								(isset($ds['min_value']) ? round($ds['min_value'],2) : 'N/A'),
								($ds['max_cutoff'] != 'N/A' ? round($ds['max_cutoff'],2) : $ds['max_cutoff']),
								($ds['min_cutoff'] != 'N/A' ? round($ds['min_cutoff'],2) : $ds['min_cutoff']),
								$ds['stddev_killed'],
								$ds['variance_killed'],
								$ds['outwind_killed'],
								($ds['avgnksamples'] != 'N/A' ? round($ds['avgnksamples'],2) : $ds['avgnksamples']),
								(isset($ds['variance_avg']) ? round($ds['variance_avg'],2) : 'N/A'));
						}
					}
				}
			}
		}
	}

	private function updateXML(&$output, &$rra) {
		/* variance subroutine */
		$rra_num   = 0;
		$ds_num    = 0;
		$kills     = 0;
		$last_num  = array();

		if (cacti_sizeof($output)) {
			foreach($output as $line) {
				if (substr_count($line, '<v>')) {
					$linearray = explode('<v>', $line);

					/* get the timestamp */
					$timestamp_part = $linearray[0];
					if (strpos($timestamp_part, '<timestamp>') !== false) {
						$timestamp_part = str_replace('<row><timestamp>', '', $timestamp_part);
						$timestamp_part = str_replace('</timestamp>', '', $timestamp_part);
						$timestamp = trim($timestamp_part);
					} else {
						$timestamp = 0;
					}

					/* discard the row */
					array_shift($linearray);

					/* initialize variables */
					$ds_num    = 0;
					$out_row   = '<row>';

					foreach($linearray as $dsvalue) {
						/* peel off garbage */
						$dsvalue = trim(str_replace('</row>', '', str_replace('</v>', '', $dsvalue)));

						if (strtolower($dsvalue) == 'nan' && !isset($last_num[$ds_num])) {
							/* do nothing, it's a NaN, and the first one */
						} elseif (!empty($this->out_start) && $timestamp > $this->out_start && $timestamp < $this->out_end) {
							if ($this->method == 3) {
								if ($this->avgnan == 'avg') {
									$dsvalue = sprintf('%1.10e', $rra[$rra_num][$ds_num]['variance_avg']);
									$kills++;
									$this->total_kills++;
								} elseif ($this->avgnan == 'last' && isset($last_num[$ds_num])) {
									$dsvalue = $last_num[$ds_num];
									$kills++;
									$this->total_kills++;
								}
							} elseif ($this->method == 4) {
								if ($dsvalue > (1+$this->percent)*$rra[$rra_num][$ds_num]['variance_avg'] || strtolower($dsvalue) == 'nan') {
									if ($this->avgnan == 'avg') {
										$dsvalue = sprintf('%1.10e', $rra[$rra_num][$ds_num]['variance_avg']);
										$kills++;
										$this->total_kills++;
									} elseif ($this->avgnan == 'last' && isset($last_num[$ds_num])) {
										$dsvalue = $last_num[$ds_num];
										$kills++;
										$this->total_kills++;
									}
								}
							}
						} elseif(strtolower($dsvalue) == 'nan' && isset($last_num[$ds_num])) {
							if ($this->method == 2) {
								if ($kills < $this->numspike) {
									if ($this->avgnan == 'avg') {
										$dsvalue = sprintf('%1.10e', $rra[$rra_num][$ds_num]['variance_avg']);
										$this->total_kills++;
										$kills++;
									} elseif ($this->avgnan == 'last' && isset($last_num[$ds_num])) {
										$dsvalue = $last_num[$ds_num];
										$this->total_kills++;
										$kills++;
									} else {
										$dsvalue = 'NaN';
									}
								}
							} elseif ($last_num[$ds_num] != 0) {
								if ($kills < $this->numspike) {
									if ($this->avgnan == 'avg') {
										$dsvalue = sprintf('%1.10e', $rra[$rra_num][$ds_num]['average']);
										$this->total_kills++;
										$kills++;
									} elseif ($this->avgnan == 'last' && isset($last_num[$ds_num])) {
										$dsvalue = $last_num[$ds_num];
										$this->total_kills++;
										$kills++;
									} else {
										$dsvalue = 'NaN';
									}
								}
							} else {
								$dsvalue = 'NaN';
							}
						} else {
							if ($this->method == 2) {
								if ($dsvalue > (1+$this->percent)*$rra[$rra_num][$ds_num]['variance_avg']) {
									if ($kills < $this->numspike) {
										if ($this->avgnan == 'avg') {
											$dsvalue = sprintf('%1.10e', $rra[$rra_num][$ds_num]['variance_avg']);
											$kills++;
											$this->total_kills++;
										} elseif ($this->avgnan == 'last' && isset($last_num[$ds_num])) {
											$dsvalue = $last_num[$ds_num];
											$kills++;
											$this->total_kills++;
										} else {
											$dsvalue = 'NaN';
										}
									}
								} else {
									$last_num[$ds_num] = $dsvalue;
								}
							} else {
								if (($dsvalue > $rra[$rra_num][$ds_num]['max_cutoff']) ||
									($dsvalue < $rra[$rra_num][$ds_num]['min_cutoff'])) {
									if ($kills < $this->numspike) {
										if ($this->avgnan == 'avg') {
											$dsvalue = sprintf('%1.10e', $rra[$rra_num][$ds_num]['average']);
											$kills++;
											$this->total_kills++;
										} elseif ($this->avgnan == 'last' && isset($last_num[$ds_num])) {
											$dsvalue = $last_num[$ds_num];
											$kills++;
											$this->total_kills++;
										} else {
											$dsvalue = 'NaN';
										}
									}
								} else {
									$last_num[$ds_num] = $dsvalue;
								}
							}
						}

						$out_row .= '<v> ' . $dsvalue . '</v>';
						$ds_num++;
					}

					$out_row .= "</row>\n";

					$new_array[] = $out_row;
				} else {
					if (substr_count($line, "</rra>\n")) {
						$ds_minmax = array();
						$rra_num++;
						$kills = 0;
						$last_num = array();
					} elseif (substr_count($line, "</database>\n")) {
						$ds_num++;
						$kills = 0;
						$last_num = array();
					}

					$new_array[] = $line;
				}
			}
		}

		return $new_array;
	}

	private function removeComments(&$output) {
		if (cacti_sizeof($output)) {
			foreach($output as $line) {
				$line = trim($line);
				if ($line == '') {
					continue;
				} else {
					/* is there a comment, remove it */
					$oline = $line;

					$comment_start = strpos($line, '<!--');
					if ($comment_start === false) {
						/* do nothing no line */
					} else {
						$comment_end = strpos($line, '-->');
						if ($comment_start == 0) {
							$line = trim(substr($line, $comment_end+3));
						} else {
							$line = trim(substr($line,0,$comment_start-1) . substr($line,$comment_end+3));
						}

						if (strpos($line, '<row>') !== false) {
							/* capture the timestamp */
							$stamp     = trim(substr($oline, $comment_start+4, $comment_end-4));
							$stamp     = explode('/', $stamp);
							$timestamp = trim($stamp[1]);
							$line = str_replace('<row><v>', "<row><timestamp> $timestamp </timestamp><v>", $line);
						}
					}

					if ($line != '') {
						$new_array[] = $line;
					}
				}
			}

			/* transfer the new array back to the original array */
			return $new_array;
		}
	}

	private function displayTime($pdp) {
		$total_time = $pdp * $this->step; // seconds

		if ($total_time < 60) {
			return $total_time . ' secs';
		} else {
			$total_time = $total_time / 60;

			if ($total_time < 60) {
				return $total_time . ' mins';
			} else {
				$total_time = $total_time / 60;

				if ($total_time < 24) {
					return $total_time . ' hours';
				} else {
					$total_time = $total_time / 24;

					return $total_time . ' days';
				}
			}
		}
	}

	private function debug($string) {
		if ($this->debug) {
			print 'DEBUG: ' . $string . "\n";
		}
	}

	private function processStandardDeviationCalculation($samples) {
		$my_samples = $samples;

		if (!empty($this->out_start)) {
			foreach($samples as $timestamp => $value) {
				if ($timestamp < $this->out_start || $timestamp > $this->out_end) {
					$my_samples[] = $value;
				}
			}
		}

		return $this->calculateStandardDeviation($my_samples);
	}

	private function calculateStandardDeviation($items) {
		if (!function_exists('stats_standard_deviation')) {
			function stats_standard_deviation($items, $sample = false) {
				$total_items = cacti_count($items);

				if ($total_items === 0) {
					return false;
				}

				if ($sample && $total_items === 1) {
					return false;
				}

				$mean  = array_sum($items) / $total_items;
				$carry = 0.0;

				foreach ($items as $val) {
					$d = ((double) $val) - $mean;
					$carry += $d * $d;
				}

				if ($sample) {
					--$total_items;
				}

				return sqrt($carry / $total_items);
			}
		}

		return stats_standard_deviation($items, false);
	}
}