Lyncoln
ec2077f979
Extract PostgreSQL, Podman runtime and systemd lifecycle adapters while preserving existing lifecycle and scoring behavior. Update RPM layout and add guarded operator-run lab harnesses. Record successful manual stop/start, promotion, demotion and rewind-only rejoin with pinned installed hashes. The unfenced lab does not validate automatic failover, Pacemaker-managed transitions or divergent-data rewind. Validation: 103 offline tests, original-function differential checks, checksum manifest and whitespace checks passed.
204 lines
11 KiB
Python
204 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Install the plugin candidate and run non-disruptive checks on the lab standby.
|
|
|
|
Run with sudo from the operator's terminal. Never supplies a password, changes
|
|
maintenance mode, or runs stop/promote/demote/rejoin. Logs are root-only.
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import pwd
|
|
import shutil
|
|
import socket
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import xml.etree.ElementTree as ET
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PROVIDER = Path('/usr/lib/ocf/resource.d/loxthelion')
|
|
AGENT = PROVIDER / 'managed-container'
|
|
MODULES = PROVIDER / 'managed-container.d'
|
|
BASELINE = ROOT / 'docs/lab-installed-before-plugin.sha256'
|
|
BACKUP_ROOT = Path('/var/backups')
|
|
|
|
|
|
def run(args, env=None, cwd=None):
|
|
clean_env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', HOME='/root', LC_ALL='C')
|
|
result = subprocess.run(args, env=env if env is not None else clean_env, cwd=cwd,
|
|
text=True, capture_output=True, timeout=90)
|
|
if result.returncode:
|
|
raise RuntimeError(f'{Path(args[0]).name} failed (rc={result.returncode}): {result.stderr.strip()}')
|
|
return result.stdout.strip()
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise RuntimeError(message)
|
|
|
|
|
|
def hashes(manifest):
|
|
return dict((name, digest) for digest, name in
|
|
(line.split(' ', 1) for line in manifest.read_text().splitlines() if line))
|
|
|
|
|
|
def cluster():
|
|
cib = ET.fromstring(run(['cibadmin', '--query']))
|
|
require(run(['crm_attribute', '--type', 'crm_config', '--name', 'cluster-name', '--query', '--quiet'])
|
|
== 'postgres-ra-test', 'This is not the postgres-ra-test lab')
|
|
require(run(['crm_attribute', '--type', 'crm_config', '--name', 'maintenance-mode', '--query', '--quiet'])
|
|
== 'true', 'Lab maintenance-mode must already be true; nothing changed')
|
|
require({n.get('uname') for n in cib.findall('./configuration/nodes/node')} ==
|
|
{'DAVID-ELITEDESK', 'DAVID-DV6'}, 'Unexpected cluster membership; inspect before proceeding')
|
|
resources = cib.findall("./configuration/resources/clone[@id='pg-ha-clone']/primitive[@id='pg-ha']")
|
|
require(len(resources) == 1, 'Expected pg-ha under pg-ha-clone')
|
|
resource = resources[0]
|
|
require((resource.get('class'), resource.get('provider'), resource.get('type')) ==
|
|
('ocf', 'loxthelion', 'managed-container'), 'Unexpected resource agent')
|
|
attributes = {}
|
|
for group in resource.findall('instance_attributes'):
|
|
require(not group.get('id-ref') and group.find('rule') is None,
|
|
'Referenced/rule-based parameters require manual review')
|
|
for pair in group.findall('nvpair'):
|
|
name, value = pair.get('name'), pair.get('value')
|
|
require(name not in attributes and name and value is not None, 'Ambiguous resource parameters')
|
|
attributes[name] = value
|
|
expected = dict(service_user='david', service_name='postgres.service', container_name='postgres',
|
|
pg_os_user='postgres', db_user='david', db_name='david')
|
|
require(all(attributes.get(k, v) == v for k, v in expected.items()), 'Lab service/database parameters changed')
|
|
return dict(expected, **{k: v for k, v in attributes.items() if k not in expected})
|
|
|
|
|
|
def observe():
|
|
account = pwd.getpwnam('david')
|
|
query = """SELECT json_build_object('recovery',pg_is_in_recovery(),
|
|
'started',pg_postmaster_start_time()::text,
|
|
'system_id',(SELECT system_identifier::text FROM pg_control_system()),
|
|
'receiver',(SELECT status FROM pg_stat_wal_receiver),
|
|
'sender',(SELECT sender_host FROM pg_stat_wal_receiver),
|
|
'slot',(SELECT slot_name FROM pg_stat_wal_receiver));"""
|
|
output = run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir,
|
|
'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid),
|
|
'timeout', '--kill-after=1s', '5s', 'podman', 'exec', '--user', 'postgres',
|
|
'--env', 'PGOPTIONS=-c statement_timeout=2000', 'postgres',
|
|
'psql', '-X', '-v', 'ON_ERROR_STOP=1', '-U', 'david', '-d', 'david', '-Atqc', query],
|
|
cwd=account.pw_dir)
|
|
state = json.loads(output)
|
|
require(state['recovery'] is True and state['receiver'] == 'streaming'
|
|
and state['sender'] == '192.168.1.2' and state['slot'] == 'ha_david_elitedesk',
|
|
'Expected a streaming ELITEDESK standby of lab DV6; refusing installation/start')
|
|
return state
|
|
|
|
|
|
def main():
|
|
require(os.geteuid() == 0, 'Run this script with sudo in your terminal; do not send passwords in chat')
|
|
require(socket.gethostname() == 'DAVID-ELITEDESK', 'This script is only for DAVID-ELITEDESK')
|
|
def interrupted(signum, frame):
|
|
raise RuntimeError(f'Interrupted by signal {signum}')
|
|
signal.signal(signal.SIGTERM, interrupted)
|
|
signal.signal(signal.SIGINT, interrupted)
|
|
os.umask(0o077)
|
|
os.environ['PATH'] = '/usr/sbin:/usr/bin:/sbin:/bin'
|
|
attributes = cluster()
|
|
before = observe()
|
|
for name, digest in hashes(BASELINE).items():
|
|
path = PROVIDER / name
|
|
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == digest,
|
|
f'Installed baseline changed: {name}; review before replacing')
|
|
manifest = hashes(ROOT / 'notes/REFACTOR-SHA256SUMS')
|
|
candidates = [ROOT / 'refactored/managed-container'] + sorted(
|
|
p for p in (ROOT / 'refactored/managed-container.d').rglob('*') if p.suffix in ('.sh', '.py') and p.is_file())
|
|
require(len(candidates) == 15, 'Unexpected candidate file layout')
|
|
for path in candidates:
|
|
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest()
|
|
== manifest.get(str(path.relative_to(ROOT))), f'Candidate checksum mismatch: {path.name}')
|
|
# Copy verified code into a root-owned staging directory before executing it.
|
|
backup = Path(tempfile.mkdtemp(prefix='managed-container-plugin-lab-', dir=BACKUP_ROOT))
|
|
print('Backup and results:', backup, flush=True)
|
|
shutil.copy2(AGENT, backup / 'managed-container')
|
|
shutil.copytree(MODULES, backup / 'managed-container.d', symlinks=True)
|
|
stage = Path(tempfile.mkdtemp(prefix='.mc-plugin-stage-', dir=PROVIDER))
|
|
for source in candidates:
|
|
destination = stage / source.relative_to(ROOT / 'refactored')
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(source, destination)
|
|
destination.chmod(0o755 if destination.name == 'managed-container' else 0o644)
|
|
require(hashlib.sha256(destination.read_bytes()).hexdigest() ==
|
|
manifest[str(source.relative_to(ROOT))], 'Source changed while staging')
|
|
for directory in (stage / 'managed-container.d').rglob('*'):
|
|
if directory.is_dir(): directory.chmod(0o755)
|
|
(stage / 'managed-container.d').chmod(0o755)
|
|
env = dict(PATH=os.environ['PATH'], OCF_ROOT='/usr/lib/ocf', OCF_RESOURCE_INSTANCE='pg-ha',
|
|
OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK',
|
|
**{'OCF_RESKEY_' + k: v for k, v in attributes.items()})
|
|
results = dict(before=before, actions=[])
|
|
|
|
def action(executable, name):
|
|
output = run([str(executable), name], env=env, cwd='/')
|
|
results['actions'].append(dict(agent=str(executable), action=name, rc=0))
|
|
print(f'PASS {executable.parent.name}/{name}: rc=0', flush=True)
|
|
return output
|
|
|
|
staged = stage / 'managed-container'
|
|
metadata = action(staged, 'meta-data')
|
|
require(len(ET.fromstring(metadata).findall('./parameters/parameter')) == 16, 'Unexpected plugin metadata')
|
|
(backup / 'candidate-metadata.xml').write_text(metadata + '\n')
|
|
action(staged, 'validate-all')
|
|
require(cluster() == attributes, 'Cluster configuration changed before installation')
|
|
require(observe() == before, 'PostgreSQL state changed before installation')
|
|
retired = stage / 'previous-modules'
|
|
modules_moved = False
|
|
try:
|
|
os.replace(MODULES, retired)
|
|
modules_moved = True
|
|
os.replace(stage / 'managed-container.d', MODULES)
|
|
os.replace(staged, AGENT)
|
|
run(['restorecon', '-RF', str(AGENT), str(MODULES)])
|
|
action(AGENT, 'validate-all')
|
|
action(AGENT, 'monitor')
|
|
action(AGENT, 'status')
|
|
require(cluster() == attributes and observe() == before, 'Standby/cluster changed before idempotent start')
|
|
action(AGENT, 'start') # Already healthy standby: exercises the no-op path.
|
|
action(AGENT, 'monitor')
|
|
require(cluster() == attributes, 'Cluster configuration changed during checks')
|
|
results['after'] = observe()
|
|
require(results['after'] == before, 'Database identity/start time or replication changed during checks')
|
|
results['score'] = run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK',
|
|
'--lifetime=reboot', '--query', '--quiet'])
|
|
require(results['score'] in ('100', '-INFINITY'), 'Unexpected standby score')
|
|
# Under write load a connected standby can legitimately fail strict catch-up.
|
|
for source in candidates:
|
|
installed = PROVIDER / source.relative_to(ROOT / 'refactored')
|
|
require(hashlib.sha256(installed.read_bytes()).hexdigest() ==
|
|
manifest[str(source.relative_to(ROOT))], 'Installed payload mismatch')
|
|
(backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
|
|
(backup / 'installed-SHA256SUMS').write_text(''.join(
|
|
manifest[str(p.relative_to(ROOT))] + ' ' + str(PROVIDER / p.relative_to(ROOT / 'refactored')) + '\n'
|
|
for p in candidates))
|
|
except BaseException:
|
|
if modules_moved:
|
|
print('Check failed: restoring previous agent and modules; maintenance mode is not changed.', flush=True)
|
|
if MODULES.exists(): os.replace(MODULES, stage / 'failed-modules')
|
|
os.replace(retired, MODULES)
|
|
shutil.copyfile(backup / 'managed-container', stage / 'restore-agent')
|
|
(stage / 'restore-agent').chmod(0o755)
|
|
os.replace(stage / 'restore-agent', AGENT)
|
|
run(['restorecon', '-RF', str(AGENT), str(MODULES)])
|
|
(backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
|
|
print('Staging/evidence retained:', stage, flush=True)
|
|
raise
|
|
shutil.rmtree(stage)
|
|
print('PASS: installed; PostgreSQL was not restarted; maintenance remains true.', flush=True)
|
|
print('Current score:', results['score'], '(strict standby catch-up can fluctuate)', flush=True)
|
|
print('No stop, promote, demote, rejoin, fencing, or peer-node actions were run.', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Exception as error:
|
|
print('STOP:', error, file=sys.stderr)
|
|
sys.exit(1)
|