Files
cacti/tests/integration/Issue7425ProcessPidReuseGuardIntegrationTest.php
Thomas Vincent a155419ff3 Serialize process registry admission with portable locks (#7771)
* Serialize process registry admission with Symfony Lock

Signed-off-by: Thomas Vincent <thomasvincent@gmail.com>

* Fix PDO process lock mode handling

Signed-off-by: Thomas Vincent <thomasvincent@gmail.com>

* ci: run process lock coverage on PHP 8.2

Signed-off-by: Thomas Vincent <thomasvincent@gmail.com>

* fix: confirm a pid is still ours before signalling it

#7574 added the still-running guard to dsstats but left six byte-identical
siblings signalling a pid straight out of the processes table. After a crash or
a power loss the row survives, the pid is recycled by an unrelated daemon, and
the next start SIGTERMs a stranger. That is issue #7425, still live in rrdcheck,
boost, poller_commands, float_rrdfiles, poller_reindex_hosts and
rebuild_poller_cache.

Wrap each kill in cacti_process_still_running(), matching the dsstats shape, and
leave unregister_process() outside the guard so a stale row is still cleaned up.

Cover all seven with a test that walks every kill site, so a new variant cannot
reintroduce it.

Signed-off-by: Thomas Vincent <thomasvincent@gmail.com>

* test: assert the guard plainly

toContain() takes needles rather than a failure message, so the file name was
being asserted as a second required substring and the test always failed.

Signed-off-by: Thomas Vincent <thomasvincent@gmail.com>

* docs(poller): describe required lock parameters

---------

Signed-off-by: Thomas Vincent <thomasvincent@gmail.com>
2026-09-04 19:36:42 -07:00

446 lines
15 KiB
PHP

<?php
/*
+-------------------------------------------------------------------------+
| Copyright (C) 2004-2026 The Cacti Group |
| |
| This program is free software; you can redistribute it and/or |
| modify it under the terms of the GNU General Public License |
| as published by the Free Software Foundation; either version 2 |
| of the License, or (at your option) any later version. |
| |
| This program is distributed in the hope that it will be useful, |
| but WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| GNU General Public License for more details. |
+-------------------------------------------------------------------------+
| Cacti: The Complete RRDtool-based Graphing Solution |
+-------------------------------------------------------------------------+
*/
/*
* Integration coverage for issue #7425. The unit test covers
* cacti_process_still_running() on its own; these drive the registry functions
* that call it, against a real (sqlite-backed) connection, because the bug was
* about what those functions then do: refuse to start a task whose pid has been
* recycled, or send SIGTERM to whatever inherited it.
*
* The child processes here are php, so the /proc command-name comparison sees
* the same name it sees for the test runner and the kill paths are reached on
* Linux as well as where /proc is absent.
*/
require_once dirname(__DIR__, 2) . '/include/vendor/autoload.php';
require_once dirname(__DIR__, 2) . '/include/global_constants.php';
require_once dirname(__DIR__, 2) . '/lib/database.php';
require_once dirname(__DIR__, 2) . '/lib/functions.php';
require_once dirname(__DIR__) . '/Helpers/FakeMySQLPDO.php';
/* include/global.php resolves these from config and the settings table, which
would need a live install; the registry only logs through them. */
foreach ([
'CACTI_WEB' => false,
'POLLER_ID' => 1,
'CACTI_PATH_LOG' => sys_get_temp_dir(),
] as $name => $value) {
if (!defined($name)) {
define($name, $value);
}
}
require_once dirname(__DIR__, 2) . '/lib/poller.php';
/**
* A FakeMySQLPDO that also understands the two MySQL constructs the registry
* queries use. Kept local to this file rather than added to the shared helper,
* which every other integration test depends on.
*/
class ProcessRegistryPDO extends FakeMySQLPDO {
public function __construct() {
parent::__construct();
$this->setAttribute(PDO::ATTR_STATEMENT_CLASS, [ProcessRegistryStatement::class, []]);
}
public function prepare(string $query, array $options = []) : PDOStatement|false {
return parent::prepare($this->rewrite($query), $options);
}
public function query(string $statement, ?int $fetchMode = null, mixed ...$args) : PDOStatement|false {
$rewritten = $this->rewrite($statement);
return $fetchMode === null
? parent::query($rewritten)
: parent::query($rewritten, $fetchMode, ...$args);
}
public function exec(string $statement) : int|false {
return parent::exec($this->rewrite($statement));
}
/**
* Rewrites the MySQL spellings sqlite does not share.
*
* @param string $sql The statement to rewrite.
*
* @return string The sqlite equivalent.
*/
private function rewrite(string $sql) : string {
/* Both forms are cast, and that matters: strftime() returns TEXT, and
sqlite orders every integer before every string, so the unwrapped
"started + timeout < now" comparison the registry makes is true
whatever the clock says. The shared helper's own UNIX_TIMESTAMP
rewrite has the same hazard. */
$sql = preg_replace('/UNIX_TIMESTAMP\s*\(\s*\)/i', "CAST(strftime('%s','now') AS INTEGER)", $sql);
$sql = preg_replace('/UNIX_TIMESTAMP\s*\(([^)]+)\)/i', "CAST(strftime('%s', $1) AS INTEGER)", $sql);
$sql = preg_replace('/\bNOW\s*\(\s*\)/i', "datetime('now')", $sql);
$sql = preg_replace('/FROM_UNIXTIME\s*\(([^)]+)\)/i', "datetime($1, 'unixepoch')", $sql);
/* db_table_exists() probes with SHOW TABLES LIKE 'x'; without this the
registry's own table check throws on sqlite and every function returns
its "no processes table" fast path, so nothing under test runs. */
$sql = preg_replace("/SHOW\s+TABLES\s+LIKE\s+('[^']*')/i", "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE $1", $sql);
// IF(a, b, c); the word boundary keeps IFNULL and friends intact
return preg_replace('/\bIF\s*\(/i', 'iif(', $sql);
}
}
/**
* sqlite reports rowCount() as 0 for SELECT, which db_fetch_row_return() reads
* as no rows at all, so the result set is captured up front on execute().
* Because that capture drains the cursor, fetch() is served from the buffer
* too: db_fetch_row_return() reads a single row with fetch(), and without this
* it would see an exhausted cursor and report every registered row as missing.
*/
class ProcessRegistryStatement extends PDOStatement {
/** @var array<int,array<string,mixed>>|null rows of a SELECT, null otherwise */
private ?array $rows = null;
/** @var int next row fetch() returns from the buffer */
private int $cursor = 0;
protected function __construct() {
}
public function execute(?array $params = null) : bool {
$executed = parent::execute($params);
$this->rows = $this->columnCount() > 0 ? parent::fetchAll(PDO::FETCH_ASSOC) : null;
$this->cursor = 0;
return $executed;
}
public function rowCount() : int {
return $this->rows === null ? parent::rowCount() : count($this->rows);
}
public function fetch(int $mode = PDO::FETCH_DEFAULT, int $cursorOrientation = PDO::FETCH_ORI_NEXT, int $cursorOffset = 0) : mixed {
if ($this->rows === null) {
return parent::fetch($mode, $cursorOrientation, $cursorOffset);
}
return $this->rows[$this->cursor++] ?? false;
}
public function fetchAll(int $mode = PDO::FETCH_DEFAULT, mixed ...$args) : array {
return $this->rows ?? parent::fetchAll($mode, ...$args);
}
}
/**
* Creates the processes table.
*
* @return ProcessRegistryPDO A connection holding the seeded schema.
*/
function registry_seed() : ProcessRegistryPDO {
$conn = new ProcessRegistryPDO();
$conn->exec('CREATE TABLE settings (name TEXT PRIMARY KEY, value TEXT)');
// the column list from cacti.sql, so a missing one does not surface as an
// unrelated SQL error partway through a registry call
$conn->exec("CREATE TABLE processes (
id INTEGER PRIMARY KEY AUTOINCREMENT, pid INTEGER NOT NULL DEFAULT 0,
tasktype TEXT NOT NULL DEFAULT '', taskname TEXT NOT NULL DEFAULT '',
taskid INTEGER NOT NULL DEFAULT 0, timeout INTEGER DEFAULT 300,
started TEXT NOT NULL DEFAULT '', last_update TEXT NOT NULL DEFAULT ''
)");
return $conn;
}
/**
* Registers a row by hand so the state under test is explicit.
*
* @param PDO $conn The seeded connection.
* @param int $pid The pid to record.
* @param int $age How many seconds ago the process started.
* @param int $timeout The registered timeout.
*
* @return void
*/
function registry_register(PDO $conn, int $pid, int $age, int $timeout = 300) : void {
/* A timed-out row is the one the registry signals, so letting a test point
one at the runner would have it SIGTERM itself and take the suite down
with no output at all. */
if ($age > $timeout && $pid === getmypid()) {
throw new LogicException('refusing to register the test runner as a timed-out process');
}
$started = gmdate('Y-m-d H:i:s', time() - $age);
$conn->exec("INSERT INTO processes (pid, tasktype, taskname, taskid, started, timeout)
VALUES ($pid, 'poller', 'child', 1, '$started', $timeout)");
}
/**
* Starts a long-running php child, whose command name matches the runner's.
*
* @param resource|null $handle Receives the process handle.
*
* @return int The child's pid.
*/
function registry_spawn_php(&$handle) : int {
$pipes = [];
/* The array form matters: a string command is run through sh, so the child
would be sh and its /proc comm would not match the runner's. */
$handle = proc_open([PHP_BINARY, '-r', 'sleep(30);'], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes);
$pid = (int) proc_get_status($handle)['pid'];
foreach ($pipes as $pipe) {
fclose($pipe);
}
return $pid;
}
/**
* A pid that is certain to be gone, obtained by starting and reaping a child.
*
* @return int The pid of an exited process.
*/
function registry_dead_pid() : int {
$pipes = [];
$handle = proc_open([PHP_BINARY, '-r', 'exit(0);'], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes);
$pid = (int) proc_get_status($handle)['pid'];
foreach ($pipes as $pipe) {
fclose($pipe);
}
proc_close($handle);
return $pid;
}
function registry_wire(PDO $conn) : void {
$GLOBALS['database_hostname'] = 'unittest';
$GLOBALS['database_port'] = 0;
$GLOBALS['database_default'] = 'unittest';
$GLOBALS['database_total_queries'] = 0;
$GLOBALS['config'] = $GLOBALS['config'] ?? [];
$GLOBALS['database_sessions'] = ['unittest:0:unittest' => $conn];
}
beforeEach(function () {
$this->db_globals = [];
foreach (['database_sessions', 'database_hostname', 'database_port', 'database_default'] as $name) {
$this->db_globals[$name] = [
'exists' => array_key_exists($name, $GLOBALS),
'value' => $GLOBALS[$name] ?? null,
];
}
});
/* Left in place, the fake handle answers every later read_config_option() in
the run and throws on Cacti's MySQL SQL, aborting the suite. */
afterEach(function () {
foreach ($this->db_globals as $name => $snapshot) {
if ($snapshot['exists']) {
$GLOBALS[$name] = $snapshot['value'];
} else {
unset($GLOBALS[$name]);
}
}
});
test('a registration held by a live process of ours still blocks a new start', function () {
$conn = registry_seed();
registry_wire($conn);
registry_register($conn, getmypid(), 10);
expect(register_process_start('poller', 'child', 1, 300))->toBeFalse();
});
test('a registry mutation lock prevents a concurrent registration', function () {
$conn = registry_seed();
registry_wire($conn);
$held = CactiProcessLock::fromPdo($conn, 'poller', 'child', 1);
expect($held->acquire())->toBeTrue()
->and(register_process_start('poller', 'child', 1, 300))->toBeFalse()
->and($conn->query('SELECT COUNT(*) AS c FROM processes')->fetch(PDO::FETCH_ASSOC)['c'])->toBe(0);
$held->release();
});
test('registration releases its mutation lock after updating the registry', function () {
$conn = registry_seed();
registry_wire($conn);
expect(register_process_start('poller', 'child', 1, 300))->toBeTrue();
$next = CactiProcessLock::fromPdo($conn, 'poller', 'child', 1);
expect($next->acquire())->toBeTrue();
$next->release();
});
test('registry locking fails closed without a usable database connection', function () {
$GLOBALS['database_hostname'] = 'missing';
$GLOBALS['database_port'] = 0;
$GLOBALS['database_default'] = 'missing';
$GLOBALS['database_sessions'] = [];
expect(cacti_process_registry_lock('poller', 'child', 1))->toBeFalse();
});
test('registry locking temporarily enables the required database mode', function () {
$conn = registry_seed();
$conn->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_SILENT);
registry_wire($conn);
$lock = cacti_process_registry_lock('poller', 'child', 1);
expect($lock)->toBeInstanceOf(CactiProcessLock::class)
->and($conn->getAttribute(PDO::ATTR_ERRMODE))->toBe(PDO::ERRMODE_SILENT)
->and($lock->acquire())->toBeTrue()
->and($conn->getAttribute(PDO::ATTR_ERRMODE))->toBe(PDO::ERRMODE_SILENT);
$lock->release();
expect($conn->getAttribute(PDO::ATTR_ERRMODE))->toBe(PDO::ERRMODE_SILENT);
});
test('a registration whose pid has died no longer blocks a new start', function () {
$conn = registry_seed();
registry_wire($conn);
// the recycled-pid case: the row survives its process
registry_register($conn, registry_dead_pid(), 10);
expect(register_process_start('poller', 'child', 1, 300))->toBeTrue();
$row = $conn->query('SELECT pid FROM processes WHERE tasktype = "poller"')->fetch(PDO::FETCH_ASSOC);
expect((int) $row['pid'])->toBe(getmypid());
});
test('a timed-out registration whose pid has died is not signalled', function () {
$conn = registry_seed();
registry_wire($conn);
registry_register($conn, registry_dead_pid(), 600, 300);
// nothing to kill, but the row must still be replaced
expect(register_process_start('poller', 'child', 1, 300))->toBeTrue();
$row = $conn->query('SELECT pid FROM processes WHERE tasktype = "poller"')->fetch(PDO::FETCH_ASSOC);
expect((int) $row['pid'])->toBe(getmypid());
});
test('a timed-out registration held by a live process is signalled', function () {
$conn = registry_seed();
registry_wire($conn);
$handle = null;
$pid = registry_spawn_php($handle);
registry_register($conn, $pid, 600, 300);
expect(register_process_start('poller', 'child', 1, 300))->toBeTrue();
// give the signal a moment to land before asking whether it did
for ($wait = 0; $wait < 50 && proc_get_status($handle)['running']; $wait++) {
usleep(20000);
}
expect(proc_get_status($handle)['running'])->toBeFalse();
proc_close($handle);
});
test('the timeout sweep only signals pids that are still ours', function () {
$conn = registry_seed();
registry_wire($conn);
$handle = null;
$live = registry_spawn_php($handle);
registry_register($conn, $live, 600, 300);
expect(timeout_kill_registered_processes('poller', 'child', 1))->toBeTrue();
for ($wait = 0; $wait < 50 && proc_get_status($handle)['running']; $wait++) {
usleep(20000);
}
expect(proc_get_status($handle)['running'])->toBeFalse()
->and($conn->query('SELECT COUNT(*) AS c FROM processes')->fetch(PDO::FETCH_ASSOC)['c'])->toBe(0);
proc_close($handle);
});
test('is_process_running reports a live registration running and reaps a dead one', function () {
$conn = registry_seed();
registry_wire($conn);
registry_register($conn, getmypid(), 10);
expect(is_process_running('poller', 'child', 1))->toBeTrue();
$conn->exec('DELETE FROM processes');
registry_register($conn, registry_dead_pid(), 10);
// 97 is the "exited but never unregistered" answer
expect(is_process_running('poller', 'child', 1))->toBe(97)
->and($conn->query('SELECT COUNT(*) AS c FROM processes')->fetch(PDO::FETCH_ASSOC)['c'])->toBe(0);
});
/**
* The sweep's predicate read FROM_UNIXTIME(started) where every other registry
* query reads UNIX_TIMESTAMP(started). started is a timestamp column, so that
* expression never matched and nothing was ever timed out, which also left the
* pid guard above it unreachable.
*/
test('the sweep compares started as an epoch, not as a date', function () {
$poller = file_get_contents(dirname(__DIR__, 2) . '/lib/poller.php');
expect($poller)->toContain('UNIX_TIMESTAMP() > UNIX_TIMESTAMP(started) + timeout')
->and($poller)->not->toContain('FROM_UNIXTIME(started)');
});
test('a registration inside its timeout survives the sweep', function () {
$conn = registry_seed();
registry_wire($conn);
$handle = null;
$live = registry_spawn_php($handle);
registry_register($conn, $live, 10, 300);
expect(timeout_kill_registered_processes('poller', 'child', 1))->toBeTrue()
->and(proc_get_status($handle)['running'])->toBeTrue()
->and($conn->query('SELECT COUNT(*) AS c FROM processes')->fetch(PDO::FETCH_ASSOC)['c'])->toBe(1);
proc_terminate($handle);
proc_close($handle);
});