#!/usr/bin/env python3 """Install the plugin candidate and run non-disruptive checks on the lab standby. Run with sudo from the operator's terminal. Never supplies a password, changes maintenance mode, or runs stop/promote/demote/rejoin. Logs are root-only. """ import hashlib import json import os from pathlib import Path import pwd import shutil import socket import signal import subprocess import sys import tempfile import xml.etree.ElementTree as ET ROOT = Path(__file__).resolve().parents[1] PROVIDER = Path('/usr/lib/ocf/resource.d/loxthelion') AGENT = PROVIDER / 'managed-container' MODULES = PROVIDER / 'managed-container.d' BASELINE = ROOT / 'docs/lab-installed-before-plugin.sha256' BACKUP_ROOT = Path('/var/backups') def run(args, env=None, cwd=None): clean_env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', HOME='/root', LC_ALL='C') result = subprocess.run(args, env=env if env is not None else clean_env, cwd=cwd, text=True, capture_output=True, timeout=90) if result.returncode: raise RuntimeError(f'{Path(args[0]).name} failed (rc={result.returncode}): {result.stderr.strip()}') return result.stdout.strip() def require(condition, message): if not condition: raise RuntimeError(message) def hashes(manifest): return dict((name, digest) for digest, name in (line.split(' ', 1) for line in manifest.read_text().splitlines() if line)) def cluster(): cib = ET.fromstring(run(['cibadmin', '--query'])) require(run(['crm_attribute', '--type', 'crm_config', '--name', 'cluster-name', '--query', '--quiet']) == 'postgres-ra-test', 'This is not the postgres-ra-test lab') require(run(['crm_attribute', '--type', 'crm_config', '--name', 'maintenance-mode', '--query', '--quiet']) == 'true', 'Lab maintenance-mode must already be true; nothing changed') require({n.get('uname') for n in cib.findall('./configuration/nodes/node')} == {'DAVID-ELITEDESK', 'DAVID-DV6'}, 'Unexpected cluster membership; inspect before proceeding') resources = cib.findall("./configuration/resources/clone[@id='pg-ha-clone']/primitive[@id='pg-ha']") require(len(resources) == 1, 'Expected pg-ha under pg-ha-clone') resource = resources[0] require((resource.get('class'), resource.get('provider'), resource.get('type')) == ('ocf', 'loxthelion', 'managed-container'), 'Unexpected resource agent') attributes = {} for group in resource.findall('instance_attributes'): require(not group.get('id-ref') and group.find('rule') is None, 'Referenced/rule-based parameters require manual review') for pair in group.findall('nvpair'): name, value = pair.get('name'), pair.get('value') require(name not in attributes and name and value is not None, 'Ambiguous resource parameters') attributes[name] = value expected = dict(service_user='david', service_name='postgres.service', container_name='postgres', pg_os_user='postgres', db_user='david', db_name='david') require(all(attributes.get(k, v) == v for k, v in expected.items()), 'Lab service/database parameters changed') return dict(expected, **{k: v for k, v in attributes.items() if k not in expected}) def observe(): account = pwd.getpwnam('david') query = """SELECT json_build_object('recovery',pg_is_in_recovery(), 'started',pg_postmaster_start_time()::text, 'system_id',(SELECT system_identifier::text FROM pg_control_system()), 'receiver',(SELECT status FROM pg_stat_wal_receiver), 'sender',(SELECT sender_host FROM pg_stat_wal_receiver), 'slot',(SELECT slot_name FROM pg_stat_wal_receiver));""" output = run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir, 'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid), 'timeout', '--kill-after=1s', '5s', 'podman', 'exec', '--user', 'postgres', '--env', 'PGOPTIONS=-c statement_timeout=2000', 'postgres', 'psql', '-X', '-v', 'ON_ERROR_STOP=1', '-U', 'david', '-d', 'david', '-Atqc', query], cwd=account.pw_dir) state = json.loads(output) require(state['recovery'] is True and state['receiver'] == 'streaming' and state['sender'] == '192.168.1.2' and state['slot'] == 'ha_david_elitedesk', 'Expected a streaming ELITEDESK standby of lab DV6; refusing installation/start') return state def main(): require(os.geteuid() == 0, 'Run this script with sudo in your terminal; do not send passwords in chat') require(socket.gethostname() == 'DAVID-ELITEDESK', 'This script is only for DAVID-ELITEDESK') def interrupted(signum, frame): raise RuntimeError(f'Interrupted by signal {signum}') signal.signal(signal.SIGTERM, interrupted) signal.signal(signal.SIGINT, interrupted) os.umask(0o077) os.environ['PATH'] = '/usr/sbin:/usr/bin:/sbin:/bin' attributes = cluster() before = observe() for name, digest in hashes(BASELINE).items(): path = PROVIDER / name require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == digest, f'Installed baseline changed: {name}; review before replacing') manifest = hashes(ROOT / 'notes/REFACTOR-SHA256SUMS') candidates = [ROOT / 'refactored/managed-container'] + sorted( p for p in (ROOT / 'refactored/managed-container.d').rglob('*') if p.suffix in ('.sh', '.py') and p.is_file()) require(len(candidates) == 15, 'Unexpected candidate file layout') for path in candidates: require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == manifest.get(str(path.relative_to(ROOT))), f'Candidate checksum mismatch: {path.name}') # Copy verified code into a root-owned staging directory before executing it. backup = Path(tempfile.mkdtemp(prefix='managed-container-plugin-lab-', dir=BACKUP_ROOT)) print('Backup and results:', backup, flush=True) shutil.copy2(AGENT, backup / 'managed-container') shutil.copytree(MODULES, backup / 'managed-container.d', symlinks=True) stage = Path(tempfile.mkdtemp(prefix='.mc-plugin-stage-', dir=PROVIDER)) for source in candidates: destination = stage / source.relative_to(ROOT / 'refactored') destination.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(source, destination) destination.chmod(0o755 if destination.name == 'managed-container' else 0o644) require(hashlib.sha256(destination.read_bytes()).hexdigest() == manifest[str(source.relative_to(ROOT))], 'Source changed while staging') for directory in (stage / 'managed-container.d').rglob('*'): if directory.is_dir(): directory.chmod(0o755) (stage / 'managed-container.d').chmod(0o755) env = dict(PATH=os.environ['PATH'], OCF_ROOT='/usr/lib/ocf', OCF_RESOURCE_INSTANCE='pg-ha', OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK', **{'OCF_RESKEY_' + k: v for k, v in attributes.items()}) results = dict(before=before, actions=[]) def action(executable, name): output = run([str(executable), name], env=env, cwd='/') results['actions'].append(dict(agent=str(executable), action=name, rc=0)) print(f'PASS {executable.parent.name}/{name}: rc=0', flush=True) return output staged = stage / 'managed-container' metadata = action(staged, 'meta-data') require(len(ET.fromstring(metadata).findall('./parameters/parameter')) == 16, 'Unexpected plugin metadata') (backup / 'candidate-metadata.xml').write_text(metadata + '\n') action(staged, 'validate-all') require(cluster() == attributes, 'Cluster configuration changed before installation') require(observe() == before, 'PostgreSQL state changed before installation') retired = stage / 'previous-modules' modules_moved = False try: os.replace(MODULES, retired) modules_moved = True os.replace(stage / 'managed-container.d', MODULES) os.replace(staged, AGENT) run(['restorecon', '-RF', str(AGENT), str(MODULES)]) action(AGENT, 'validate-all') action(AGENT, 'monitor') action(AGENT, 'status') require(cluster() == attributes and observe() == before, 'Standby/cluster changed before idempotent start') action(AGENT, 'start') # Already healthy standby: exercises the no-op path. action(AGENT, 'monitor') require(cluster() == attributes, 'Cluster configuration changed during checks') results['after'] = observe() require(results['after'] == before, 'Database identity/start time or replication changed during checks') results['score'] = run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK', '--lifetime=reboot', '--query', '--quiet']) require(results['score'] in ('100', '-INFINITY'), 'Unexpected standby score') # Under write load a connected standby can legitimately fail strict catch-up. for source in candidates: installed = PROVIDER / source.relative_to(ROOT / 'refactored') require(hashlib.sha256(installed.read_bytes()).hexdigest() == manifest[str(source.relative_to(ROOT))], 'Installed payload mismatch') (backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n') (backup / 'installed-SHA256SUMS').write_text(''.join( manifest[str(p.relative_to(ROOT))] + ' ' + str(PROVIDER / p.relative_to(ROOT / 'refactored')) + '\n' for p in candidates)) except BaseException: if modules_moved: print('Check failed: restoring previous agent and modules; maintenance mode is not changed.', flush=True) if MODULES.exists(): os.replace(MODULES, stage / 'failed-modules') os.replace(retired, MODULES) shutil.copyfile(backup / 'managed-container', stage / 'restore-agent') (stage / 'restore-agent').chmod(0o755) os.replace(stage / 'restore-agent', AGENT) run(['restorecon', '-RF', str(AGENT), str(MODULES)]) (backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n') print('Staging/evidence retained:', stage, flush=True) raise shutil.rmtree(stage) print('PASS: installed; PostgreSQL was not restarted; maintenance remains true.', flush=True) print('Current score:', results['score'], '(strict standby catch-up can fluctuate)', flush=True) print('No stop, promote, demote, rejoin, fencing, or peer-node actions were run.', flush=True) if __name__ == '__main__': try: main() except Exception as error: print('STOP:', error, file=sys.stderr) sys.exit(1)