Freeze lab-tested PostgreSQL plugin engine
Extract PostgreSQL, Podman runtime and systemd lifecycle adapters while preserving existing lifecycle and scoring behavior. Update RPM layout and add guarded operator-run lab harnesses. Record successful manual stop/start, promotion, demotion and rewind-only rejoin with pinned installed hashes. The unfenced lab does not validate automatic failover, Pacemaker-managed transitions or divergent-data rewind. Validation: 103 offline tests, original-function differential checks, checksum manifest and whitespace checks passed.
This commit is contained in:
Lyncoln
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install the plugin candidate and run non-disruptive checks on the lab standby.
|
||||
|
||||
Run with sudo from the operator's terminal. Never supplies a password, changes
|
||||
maintenance mode, or runs stop/promote/demote/rejoin. Logs are root-only.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pwd
|
||||
import shutil
|
||||
import socket
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROVIDER = Path('/usr/lib/ocf/resource.d/loxthelion')
|
||||
AGENT = PROVIDER / 'managed-container'
|
||||
MODULES = PROVIDER / 'managed-container.d'
|
||||
BASELINE = ROOT / 'docs/lab-installed-before-plugin.sha256'
|
||||
BACKUP_ROOT = Path('/var/backups')
|
||||
|
||||
|
||||
def run(args, env=None, cwd=None):
|
||||
clean_env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', HOME='/root', LC_ALL='C')
|
||||
result = subprocess.run(args, env=env if env is not None else clean_env, cwd=cwd,
|
||||
text=True, capture_output=True, timeout=90)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'{Path(args[0]).name} failed (rc={result.returncode}): {result.stderr.strip()}')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def hashes(manifest):
|
||||
return dict((name, digest) for digest, name in
|
||||
(line.split(' ', 1) for line in manifest.read_text().splitlines() if line))
|
||||
|
||||
|
||||
def cluster():
|
||||
cib = ET.fromstring(run(['cibadmin', '--query']))
|
||||
require(run(['crm_attribute', '--type', 'crm_config', '--name', 'cluster-name', '--query', '--quiet'])
|
||||
== 'postgres-ra-test', 'This is not the postgres-ra-test lab')
|
||||
require(run(['crm_attribute', '--type', 'crm_config', '--name', 'maintenance-mode', '--query', '--quiet'])
|
||||
== 'true', 'Lab maintenance-mode must already be true; nothing changed')
|
||||
require({n.get('uname') for n in cib.findall('./configuration/nodes/node')} ==
|
||||
{'DAVID-ELITEDESK', 'DAVID-DV6'}, 'Unexpected cluster membership; inspect before proceeding')
|
||||
resources = cib.findall("./configuration/resources/clone[@id='pg-ha-clone']/primitive[@id='pg-ha']")
|
||||
require(len(resources) == 1, 'Expected pg-ha under pg-ha-clone')
|
||||
resource = resources[0]
|
||||
require((resource.get('class'), resource.get('provider'), resource.get('type')) ==
|
||||
('ocf', 'loxthelion', 'managed-container'), 'Unexpected resource agent')
|
||||
attributes = {}
|
||||
for group in resource.findall('instance_attributes'):
|
||||
require(not group.get('id-ref') and group.find('rule') is None,
|
||||
'Referenced/rule-based parameters require manual review')
|
||||
for pair in group.findall('nvpair'):
|
||||
name, value = pair.get('name'), pair.get('value')
|
||||
require(name not in attributes and name and value is not None, 'Ambiguous resource parameters')
|
||||
attributes[name] = value
|
||||
expected = dict(service_user='david', service_name='postgres.service', container_name='postgres',
|
||||
pg_os_user='postgres', db_user='david', db_name='david')
|
||||
require(all(attributes.get(k, v) == v for k, v in expected.items()), 'Lab service/database parameters changed')
|
||||
return dict(expected, **{k: v for k, v in attributes.items() if k not in expected})
|
||||
|
||||
|
||||
def observe():
|
||||
account = pwd.getpwnam('david')
|
||||
query = """SELECT json_build_object('recovery',pg_is_in_recovery(),
|
||||
'started',pg_postmaster_start_time()::text,
|
||||
'system_id',(SELECT system_identifier::text FROM pg_control_system()),
|
||||
'receiver',(SELECT status FROM pg_stat_wal_receiver),
|
||||
'sender',(SELECT sender_host FROM pg_stat_wal_receiver),
|
||||
'slot',(SELECT slot_name FROM pg_stat_wal_receiver));"""
|
||||
output = run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir,
|
||||
'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid),
|
||||
'timeout', '--kill-after=1s', '5s', 'podman', 'exec', '--user', 'postgres',
|
||||
'--env', 'PGOPTIONS=-c statement_timeout=2000', 'postgres',
|
||||
'psql', '-X', '-v', 'ON_ERROR_STOP=1', '-U', 'david', '-d', 'david', '-Atqc', query],
|
||||
cwd=account.pw_dir)
|
||||
state = json.loads(output)
|
||||
require(state['recovery'] is True and state['receiver'] == 'streaming'
|
||||
and state['sender'] == '192.168.1.2' and state['slot'] == 'ha_david_elitedesk',
|
||||
'Expected a streaming ELITEDESK standby of lab DV6; refusing installation/start')
|
||||
return state
|
||||
|
||||
|
||||
def main():
|
||||
require(os.geteuid() == 0, 'Run this script with sudo in your terminal; do not send passwords in chat')
|
||||
require(socket.gethostname() == 'DAVID-ELITEDESK', 'This script is only for DAVID-ELITEDESK')
|
||||
def interrupted(signum, frame):
|
||||
raise RuntimeError(f'Interrupted by signal {signum}')
|
||||
signal.signal(signal.SIGTERM, interrupted)
|
||||
signal.signal(signal.SIGINT, interrupted)
|
||||
os.umask(0o077)
|
||||
os.environ['PATH'] = '/usr/sbin:/usr/bin:/sbin:/bin'
|
||||
attributes = cluster()
|
||||
before = observe()
|
||||
for name, digest in hashes(BASELINE).items():
|
||||
path = PROVIDER / name
|
||||
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == digest,
|
||||
f'Installed baseline changed: {name}; review before replacing')
|
||||
manifest = hashes(ROOT / 'notes/REFACTOR-SHA256SUMS')
|
||||
candidates = [ROOT / 'refactored/managed-container'] + sorted(
|
||||
p for p in (ROOT / 'refactored/managed-container.d').rglob('*') if p.suffix in ('.sh', '.py') and p.is_file())
|
||||
require(len(candidates) == 15, 'Unexpected candidate file layout')
|
||||
for path in candidates:
|
||||
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
== manifest.get(str(path.relative_to(ROOT))), f'Candidate checksum mismatch: {path.name}')
|
||||
# Copy verified code into a root-owned staging directory before executing it.
|
||||
backup = Path(tempfile.mkdtemp(prefix='managed-container-plugin-lab-', dir=BACKUP_ROOT))
|
||||
print('Backup and results:', backup, flush=True)
|
||||
shutil.copy2(AGENT, backup / 'managed-container')
|
||||
shutil.copytree(MODULES, backup / 'managed-container.d', symlinks=True)
|
||||
stage = Path(tempfile.mkdtemp(prefix='.mc-plugin-stage-', dir=PROVIDER))
|
||||
for source in candidates:
|
||||
destination = stage / source.relative_to(ROOT / 'refactored')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, destination)
|
||||
destination.chmod(0o755 if destination.name == 'managed-container' else 0o644)
|
||||
require(hashlib.sha256(destination.read_bytes()).hexdigest() ==
|
||||
manifest[str(source.relative_to(ROOT))], 'Source changed while staging')
|
||||
for directory in (stage / 'managed-container.d').rglob('*'):
|
||||
if directory.is_dir(): directory.chmod(0o755)
|
||||
(stage / 'managed-container.d').chmod(0o755)
|
||||
env = dict(PATH=os.environ['PATH'], OCF_ROOT='/usr/lib/ocf', OCF_RESOURCE_INSTANCE='pg-ha',
|
||||
OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK',
|
||||
**{'OCF_RESKEY_' + k: v for k, v in attributes.items()})
|
||||
results = dict(before=before, actions=[])
|
||||
|
||||
def action(executable, name):
|
||||
output = run([str(executable), name], env=env, cwd='/')
|
||||
results['actions'].append(dict(agent=str(executable), action=name, rc=0))
|
||||
print(f'PASS {executable.parent.name}/{name}: rc=0', flush=True)
|
||||
return output
|
||||
|
||||
staged = stage / 'managed-container'
|
||||
metadata = action(staged, 'meta-data')
|
||||
require(len(ET.fromstring(metadata).findall('./parameters/parameter')) == 16, 'Unexpected plugin metadata')
|
||||
(backup / 'candidate-metadata.xml').write_text(metadata + '\n')
|
||||
action(staged, 'validate-all')
|
||||
require(cluster() == attributes, 'Cluster configuration changed before installation')
|
||||
require(observe() == before, 'PostgreSQL state changed before installation')
|
||||
retired = stage / 'previous-modules'
|
||||
modules_moved = False
|
||||
try:
|
||||
os.replace(MODULES, retired)
|
||||
modules_moved = True
|
||||
os.replace(stage / 'managed-container.d', MODULES)
|
||||
os.replace(staged, AGENT)
|
||||
run(['restorecon', '-RF', str(AGENT), str(MODULES)])
|
||||
action(AGENT, 'validate-all')
|
||||
action(AGENT, 'monitor')
|
||||
action(AGENT, 'status')
|
||||
require(cluster() == attributes and observe() == before, 'Standby/cluster changed before idempotent start')
|
||||
action(AGENT, 'start') # Already healthy standby: exercises the no-op path.
|
||||
action(AGENT, 'monitor')
|
||||
require(cluster() == attributes, 'Cluster configuration changed during checks')
|
||||
results['after'] = observe()
|
||||
require(results['after'] == before, 'Database identity/start time or replication changed during checks')
|
||||
results['score'] = run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK',
|
||||
'--lifetime=reboot', '--query', '--quiet'])
|
||||
require(results['score'] in ('100', '-INFINITY'), 'Unexpected standby score')
|
||||
# Under write load a connected standby can legitimately fail strict catch-up.
|
||||
for source in candidates:
|
||||
installed = PROVIDER / source.relative_to(ROOT / 'refactored')
|
||||
require(hashlib.sha256(installed.read_bytes()).hexdigest() ==
|
||||
manifest[str(source.relative_to(ROOT))], 'Installed payload mismatch')
|
||||
(backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
|
||||
(backup / 'installed-SHA256SUMS').write_text(''.join(
|
||||
manifest[str(p.relative_to(ROOT))] + ' ' + str(PROVIDER / p.relative_to(ROOT / 'refactored')) + '\n'
|
||||
for p in candidates))
|
||||
except BaseException:
|
||||
if modules_moved:
|
||||
print('Check failed: restoring previous agent and modules; maintenance mode is not changed.', flush=True)
|
||||
if MODULES.exists(): os.replace(MODULES, stage / 'failed-modules')
|
||||
os.replace(retired, MODULES)
|
||||
shutil.copyfile(backup / 'managed-container', stage / 'restore-agent')
|
||||
(stage / 'restore-agent').chmod(0o755)
|
||||
os.replace(stage / 'restore-agent', AGENT)
|
||||
run(['restorecon', '-RF', str(AGENT), str(MODULES)])
|
||||
(backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
|
||||
print('Staging/evidence retained:', stage, flush=True)
|
||||
raise
|
||||
shutil.rmtree(stage)
|
||||
print('PASS: installed; PostgreSQL was not restarted; maintenance remains true.', flush=True)
|
||||
print('Current score:', results['score'], '(strict standby catch-up can fluctuate)', flush=True)
|
||||
print('No stop, promote, demote, rejoin, fencing, or peer-node actions were run.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
print('STOP:', error, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user