":'') . "NOTE: Removing Outliers in Range and Replacing with Last" . ($html ? "
\n":"\n"); } /* execute the dump command */ $strout .= ($html ? "":'') . "NOTE: Creating XML file '$xmlfile' from '$rrdfile'" . ($html ? "
\n":"\n"); if ($using_cacti) { if (!$dryrun) { switch ($method) { case 1: $mm = 'StdDev'; $mes = "$username, File:" . basename($rrdfile) . ", Method:$mm, StdDevs:$stddev, AvgNan:$avgnan, Kills:$numspike, Outliers:$outliers"; break; case 2: $mm = 'Variance'; $mes = "$username, File:" . basename($rrdfile) . ", Method:$mm, AvgNan:$avgnan, Kills:$numspike, Outliers:$outliers, Percent:$percent"; break; case 3: $mm = 'RangeFill'; $mes = "$username, File:" . basename($rrdfile) . ", Method:$mm, OutStart:$out_start, OutEnd:$out_end, AvgNan:$avgnan"; break; case 4: $mm = 'GapFill'; $mes = "$username, File:" . basename($rrdfile) . ", Method:$mm, OutStart:$out_start, OutEnd:$out_end, AvgNan:$avgnan"; break; default: $mm = 'Undefined'; $mes = "$username, File:" . basename($rrdfile) . ", Method:$mm"; } cacti_log($mes, false, 'SPIKEKILL'); } shell_exec(read_config_option('path_rrdtool') . " dump $rrdfile > $xmlfile"); } else { shell_exec("rrdtool dump $rrdfile > $xmlfile"); } /* read the xml file into an array*/ if (file_exists($xmlfile)) { $output = file($xmlfile); /* remove the temp file */ unlink($xmlfile); } else { if ($using_cacti) { $strout .= ($html ? "":'') . "NOTE: RRDfile '$rrdfile' backed up to '$bakfile'" . ($html ? "
\n":"\n"); } else { $strout .= ($html ? "":'') . "FATAL: RRDfile Backup of '$rrdfile' to '$bakfile' FAILED!" . ($html ? "
\n":"\n"); print $strout; exit(-13); } } /* process the xml file and remove all comments */ $output = removeComments($output); /* Read all the rra's ds values and obtain the following pieces of information from each rra archive. * numsamples - The number of 'valid' non-nan samples * sumofsamples - The sum of all 'valid' samples. * average - The average of all samples * standard_deviation - The standard deviation of all samples * max_value - The maximum value of all samples * min_value - The minimum value of all samples * max_cutoff - Any value above this value will be set to the average. * min_cutoff - Any value lower than this value will be set to the average. This will end up being a n-dimensional array as follows: rra[x][ds#]['totalsamples']; rra[x][ds#]['numsamples']; rra[x][ds#]['sumofsamples']; rra[x][ds#]['average']; rra[x][ds#]['stddev']; rra[x][ds#]['max_value']; rra[x][ds#]['min_value']; rra[x][ds#]['max_cutoff']; rra[x][ds#]['min_cutoff']; There will also be a secondary array created with the actual samples. This array will be used to calculate the standard deviation of the sample set. samples[rra_num][ds_num][timestamp]; Also track the min and max value for each ds and store it into the two arrays: ds_min[ds#], ds_max[ds#]. The we don't need to know the type of rra, only it's number for this analysis the same applies for the ds' as well. */ $rra = array(); $rra_cf = array(); $rra_pdp = array(); $rra_num = 0; $ds_num = 0; $total_kills = 0; $in_rra = false; $in_db = false; $ds_min = array(); $ds_max = array(); $ds_name = array(); /* perform a first pass on the array and do the following: 1) Get the number of good samples per ds 2) Get the sum of the samples per ds 3) Get the max and min values for all samples 4) Build both the rra and sample arrays 5) Get each ds' min and max values */ if (sizeof($output)) { foreach($output as $line) { if (substr_count($line, '":'') . "NOTE: Searching for Spikes in XML file '$xmlfile'" . ($html ? "
\n":"\n"); } calculateOverallStatistics($rra, $samples); /* debugging and/or status report */ if ($debug || $dryrun) { if ($html) { $strout .= "":'') . "NOTE: NO Window Spikes found in '$rrdfile'" . ($html ? "
\n":"\n"); } else { $strout .= ($html ? "":'') . "NOTE: NO Standard Deviation found in '$rrdfile'" . ($html ? "
\n":"\n"); } } else { /* variance subroutine */ if ($var_kills || $out_kills) { debug('Either variance or out_kills found'); if (!$dryrun) { $new_output = updateXML($output, $rra); } } elseif (!empty($out_start)) { $strout .= ($html ? "":'') . "NOTE: NO Window Fills found in '$rrdfile'" . ($html ? "
\n":"\n"); } else { $strout .= ($html ? "":'') . "NOTE: NO Variance Spikes found in '$rrdfile'" . ($html ? "
\n":"\n"); } } /* finally update the file XML file and Reprocess the RRDfile */ if (!$dryrun) { if ($total_kills) { if (writeXMLFile($new_output, $xmlfile)) { if (backupRRDFile($rrdfile)) { createRRDFileFromXML($xmlfile, $rrdfile); $strout .= ($html ? "":'') . "NOTE: Spikes Found and Remediated. Total Spikes ($total_kills)" . ($html ? "
\n":"\n"); } else { $strout .= ($html ? "":'') . "FATAL: Unable to backup '$rrdfile'" . ($html ? "
\n":"\n"); } } else { $strout .= ($html ? "":'') . "FATAL: Unable to write XML file '$xmlfile'" . ($html ? "
\n":"\n"); } } else { $strout .= ($html ? "":'') . "NOTE: No Spikes Found. No remediation performed." . ($html ? "
\n":"\n"); } } else { $strout .= ($html ? "":'') . "NOTE: Dryrun requested. No updates performed" . ($html ? "
\n":"\n"); } $strout .= ($html ? "":''); if ($using_cacti) { if ($total_kills > 0) { cacti_log("WARNING: Removed '$total_kills' Spikes from '$rrdfile', Method:'$method'", false, 'WEBUI'); } elseif($debug) { cacti_log("NOTE: Removed '$total_kills' Spikes from '$rrdfile', Method:'$method'", false, 'WEBUI'); } } if ($using_cacti) { if (file_exists($xmlfile)) { unlink($xmlfile); } if (file_exists($xmlfile)) { unlink($bakfile); } } print $strout; /* All Functions */ function createRRDFileFromXML($xmlfile, $rrdfile) { global $using_cacti, $html, $strout; /* execute the dump command */ $strout .= ($html ? "":'') . "NOTE: Re-Importing '$xmlfile' to '$rrdfile'" . ($html ? "
\n":"\n"); if ($using_cacti) { $response = shell_exec(read_config_option("path_rrdtool") . " restore -f -r $xmlfile $rrdfile"); } else { $response = shell_exec("rrdtool restore -f -r $xmlfile $rrdfile"); } if ($response != '') { $strout .= ($html ? "":'') . $response . ($html ? "
\n":"\n"); } } function writeXMLFile($output, $xmlfile) { return file_put_contents($xmlfile, $output); } function backupRRDFile($rrdfile) { global $using_cacti, $tempdir, $seed, $html, $strout; if ($using_cacti) { $backupdir = read_config_option('spikekill_backupdir'); if ($backupdir == '') { $backupdir = $tempdir; } } else { $backupdir = $tempdir; } if (file_exists($backupdir . '/' . basename($rrdfile))) { $newfile = basename($rrdfile) . '.' . $seed; } else { $newfile = basename($rrdfile); } $strout .= ($html ? "":'') . "NOTE: Backing Up '$rrdfile' to '" . $backupdir . '/' . $newfile . "'" . ($html ? "
\n":"\n"); return copy($rrdfile, $backupdir . "/" . $newfile); } function calculateVarianceAverages(&$rra, &$samples) { global $outliers, $out_start, $out_end; if (sizeof($samples)) { foreach($samples as $rra_num => $dses) { if (sizeof($dses)) { foreach($dses as $ds_num => $ds) { if (empty($out_start)) { if (sizeof($ds) < $outliers * 3) { $rra[$rra_num][$ds_num]['variance_avg'] = 'NAN'; } else { $myds = $ds; $myds = array_filter($myds, 'removeNanFromSamples'); /* remove high outliers */ rsort($myds, SORT_NUMERIC); $myds = array_slice($myds, $outliers); /* remove low outliers */ sort($ds, SORT_NUMERIC); $myds = array_slice($myds, $outliers); if (sizeof($myds)) { $rra[$rra_num][$ds_num]['variance_avg'] = array_sum($myds) / sizeof($myds); } else { $rra[$rra_num][$ds_num]['variance_avg'] = 'NAN'; } } } else { if (isset($rra[$rra_num][$ds_num]['sumofsamples']) && isset($rra[$rra_num][$ds_num]['numsamples'])) { if ($rra[$rra_num][$ds_num]['numsamples'] > 0) { $rra[$rra_num][$ds_num]['variance_avg'] = $rra[$rra_num][$ds_num]['sumofsamples'] / $rra[$rra_num][$ds_num]['numsamples']; } else { $rra[$rra_num][$ds_num]['variance_avg'] = 0; } } else { $rra[$rra_num][$ds_num]['variance_avg'] = 0; } } } } } } } function removeNanFromSamples(&$string) { return stripos($string, 'nan') === false; } function calculateOverallStatistics(&$rra, &$samples) { global $percent, $stddev, $method, $ds_min, $ds_max, $var_kills, $std_kills, $out_kills, $out_start, $out_end; $rra_num = 0; if (sizeof($rra)) { foreach($rra as $dses) { $ds_num = 0; if (sizeof($dses)) { foreach($dses as $ds) { if (isset($samples[$rra_num][$ds_num])) { $rra[$rra_num][$ds_num]['standard_deviation'] = processStandardDeviationCalculation($samples[$rra_num][$ds_num]); if ($rra[$rra_num][$ds_num]['standard_deviation'] == 'NAN') { $rra[$rra_num][$ds_num]['standard_deviation'] = 0; } if (isset($rra[$rra_num][$ds_num]['sumofsamples']) && isset($rra[$rra_num][$ds_num]['numsamples'])) { if ($rra[$rra_num][$ds_num]['numsamples'] > 0) { $rra[$rra_num][$ds_num]['average'] = $rra[$rra_num][$ds_num]['sumofsamples'] / $rra[$rra_num][$ds_num]['numsamples']; } else { $rra[$rra_num][$ds_num]['average'] = 0; } } else { $rra[$rra_num][$ds_num]['average'] = 0; } $rra[$rra_num][$ds_num]['min_cutoff'] = $rra[$rra_num][$ds_num]['average'] - ($stddev * $rra[$rra_num][$ds_num]['standard_deviation']); if ($rra[$rra_num][$ds_num]['min_cutoff'] < $ds_min[$ds_num]) { $rra[$rra_num][$ds_num]['min_cutoff'] = $ds_min[$ds_num]; } $rra[$rra_num][$ds_num]['max_cutoff'] = $rra[$rra_num][$ds_num]['average'] + ($stddev * $rra[$rra_num][$ds_num]['standard_deviation']); if ($rra[$rra_num][$ds_num]['max_cutoff'] > $ds_max[$ds_num]) { $rra[$rra_num][$ds_num]['max_cutoff'] = $ds_max[$ds_num]; } $rra[$rra_num][$ds_num]['numnksamples'] = 0; $rra[$rra_num][$ds_num]['sumnksamples'] = 0; $rra[$rra_num][$ds_num]['avgnksamples'] = 0; /* go through values and find cutoffs */ $rra[$rra_num][$ds_num]['stddev_killed'] = 0; $rra[$rra_num][$ds_num]['variance_killed'] = 0; $rra[$rra_num][$ds_num]['outwind_killed'] = 0; /* kill what is required to be killed */ if (sizeof($samples[$rra_num][$ds_num])) { foreach($samples[$rra_num][$ds_num] as $timestamp => $sample) { if (!empty($out_start) && $timestamp >= $out_start && $timestamp <= $out_end) { if ($method == 3) { debug(sprintf("Window Kill: Value '%.4e', Time '%s'", $sample, date('Y-m-d H:i', $timestamp))); $rra[$rra_num][$ds_num]['outwind_killed']++; $out_kills = true; }else if ($method == 4) { if ($sample > (1+$percent)*$rra[$rra_num][$ds_num]['variance_avg'] || strtolower($sample) == 'nan') { debug(sprintf("Window Kill: Value '%.4e', Time '%s'", $sample, date('Y-m-d H:i', $timestamp))); $rra[$rra_num][$ds_num]['outwind_killed']++; $out_kills = true; } } }else if (($sample > $rra[$rra_num][$ds_num]['max_cutoff']) || ($sample < $rra[$rra_num][$ds_num]['min_cutoff'])) { debug(sprintf("Std Kill: Value '%.4e', StandardDev '%.4e', StdDevLimit '%.4e'", $sample, $rra[$rra_num][$ds_num]['standard_deviation'], ($rra[$rra_num][$ds_num]['max_cutoff'] * (1+$percent)))); $rra[$rra_num][$ds_num]['stddev_killed']++; $std_kills = true; } elseif (is_numeric($sample)) { $rra[$rra_num][$ds_num]['numnksamples']++; $rra[$rra_num][$ds_num]['sumnksamples'] += $sample; } if (!empty($out_start) && $timestamp >= $out_start && $timestamp <= $out_end) { // Already calculated }else if ($rra[$rra_num][$ds_num]['variance_avg'] == 'NAN') { /* not enought samples to calculate */ }else if ($sample > ($rra[$rra_num][$ds_num]['variance_avg'] * (1+$percent))) { /* kill based upon variance */ debug(sprintf("Var Kill: Value '%.4e', VarianceDev '%.4e', VarianceLimit '%.4e'", $sample, $rra[$rra_num][$ds_num]['variance_avg'], ($rra[$rra_num][$ds_num]['variance_avg'] * (1+$percent)))); $rra[$rra_num][$ds_num]['variance_killed']++; $var_kills = true; } } } if ($rra[$rra_num][$ds_num]['numnksamples'] > 0) { $rra[$rra_num][$ds_num]['avgnksamples'] = $rra[$rra_num][$ds_num]['sumnksamples'] / $rra[$rra_num][$ds_num]['numnksamples']; } } else { $rra[$rra_num][$ds_num]['standard_deviation'] = 'N/A'; $rra[$rra_num][$ds_num]['average'] = 'N/A'; $rra[$rra_num][$ds_num]['min_cutoff'] = 'N/A'; $rra[$rra_num][$ds_num]['max_cutoff'] = 'N/A'; $rra[$rra_num][$ds_num]['numnksamples'] = 'N/A'; $rra[$rra_num][$ds_num]['sumnksamples'] = 'N/A'; $rra[$rra_num][$ds_num]['avgnksamples'] = 'N/A'; $rra[$rra_num][$ds_num]['stddev_killed'] = 'N/A'; $rra[$rra_num][$ds_num]['variance_killed'] = 'N/A'; $rra[$rra_num][$ds_num]['stddev_killed'] = 'N/A'; $rra[$rra_num][$ds_num]['outwind_killed'] = 'N/A'; $rra[$rra_num][$ds_num]['numnksamples'] = 'N/A'; $rra[$rra_num][$ds_num]['sumnksamples'] = 'N/A'; $rra[$rra_num][$ds_num]['variance_killed'] = 'N/A'; $rra[$rra_num][$ds_num]['avgnksamples'] = 'N/A'; } $ds_num++; } } $rra_num++; } } } function outputStatistics($rra) { global $strout, $rra_cf, $rra_name, $ds_name, $rra_pdp, $html; if (sizeof($rra)) { if (!$html) { $strout .= "\n"; $strout .= sprintf("%10s %16s %10s %7s %7s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s\n", 'Size', 'DataSource', 'CF', 'Samples', 'NonNan', 'Avg', 'StdDev', 'MaxValue', 'MinValue', 'MaxStdDev', 'MinStdDev', 'StdKilled', 'VarKilled', 'WindFilled', 'StdDevAvg', 'VarAvg'); $strout .= sprintf("%10s %16s %10s %7s %7s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s\n", '----------', '---------------', '----------', '-------', '-------', '----------', '----------', '----------', '----------', '----------', '----------', '----------', '----------', '----------', '----------', '----------', '----------'); foreach($rra as $rra_key => $dses) { if (sizeof($dses)) { foreach($dses as $dskey => $ds) { $strout .= sprintf('%10s %16s %10s %7s %7s ' . ($ds['average'] < 1E6 ? '%10s ':'%10.4e ') . ($ds['standard_deviation'] < 1E6 ? '%10s ':'%10.4e ') . (isset($ds['max_value']) ? ($ds['max_value'] < 1E6 ? '%10s ':'%10.4e ') : '%10s ') . (isset($ds['min_value']) ? ($ds['min_value'] < 1E6 ? '%10s ':'%10.4e ') : '%10s ') . (isset($ds['max_cutoff']) ? ($ds['max_cutoff'] < 1E6 ? '%10s ':'%10.4e ') : '%10s ') . (isset($ds['min_cutoff']) ? ($ds['min_cutoff'] < 1E6 ? '%10s ':'%10.4e ') : '%10s ') . '%10s %10s %10s ' . (isset($ds['avgnksamples']) ? ($ds['avgnksamples'] < 1E6 ? '%10s ':'%10.4e ') : '%10.4E ') . (isset($ds['variance_avg']) ? ($ds['variance_avg'] < 1E6 ? '%10s ':'%10.4e ') : '%10.4E ') . "\n", displayTime($rra_pdp[$rra_key]), $ds_name[$dskey], $rra_cf[$rra_key], $ds['totalsamples'], (isset($ds['numsamples']) ? $ds['numsamples'] : '0'), ($ds['average'] != 'N/A' ? round($ds['average'],2) : $ds['average']), ($ds['standard_deviation'] != 'N/A' ? round($ds['standard_deviation'],2) : $ds['standard_deviation']), (isset($ds['max_value']) ? round($ds['max_value'],2) : 'N/A'), (isset($ds['min_value']) ? round($ds['min_value'],2) : 'N/A'), ($ds['max_cutoff'] != 'N/A' ? round($ds['max_cutoff'],2) : $ds['max_cutoff']), ($ds['min_cutoff'] != 'N/A' ? round($ds['min_cutoff'],2) : $ds['min_cutoff']), $ds['stddev_killed'], $ds['variance_killed'], $ds['outwind_killed'], ($ds['avgnksamples'] != 'N/A' ? round($ds['avgnksamples'],2) : $ds['avgnksamples']), (isset($ds['variance_avg']) ? round($ds['variance_avg'],2) : 'N/A')); } } } $strout .= "\n"; } else { $strout .= sprintf("