#!/usr/bin/env python3 """One guarded RA stop/start cycle on the ELITEDESK lab standby only.""" import importlib.util import json import os from pathlib import Path import pwd import socket import subprocess import sys import tempfile import time import hashlib spec = importlib.util.spec_from_file_location('lab', Path(__file__).with_name('test-elitedesk-plugin.py')) lab = importlib.util.module_from_spec(spec) spec.loader.exec_module(lab) def owner_command(*args): account = pwd.getpwnam('david') return lab.run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir, 'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid), 'timeout', '--kill-after=1s', '5s', *args], cwd=account.pw_dir) def score(): return lab.run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK', '--lifetime=reboot', '--query', '--quiet']) def verify_payload(): entries = lab.hashes(lab.ROOT / 'docs/lab-installed-plugin.sha256') lab.require(len(entries) == 15, 'Expected the 15-file installed plugin snapshot') for name, digest in entries.items(): path = Path(name) lab.require(lab.PROVIDER in path.parents and not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == digest, 'Installed code changed; inspect before testing: ' + name) def main(): lab.require(os.geteuid() == 0, 'Run with sudo in your terminal; do not send passwords in chat') lab.require(socket.gethostname() == 'DAVID-ELITEDESK', 'Only the ELITEDESK lab standby is supported') os.umask(0o077) attributes = lab.cluster() # Exact lab membership/resource; maintenance=true. before = lab.observe() # Running standby streaming from DV6 with its own slot. verify_payload() evidence = Path(tempfile.mkdtemp(prefix='managed-container-cycle-lab-', dir=lab.BACKUP_ROOT)) print('Results:', evidence, flush=True) env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', OCF_ROOT='/usr/lib/ocf', OCF_RESOURCE_INSTANCE='pg-ha', OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK', **{'OCF_RESKEY_' + k: v for k, v in attributes.items()}) results = dict(before=before, actions=[], status='running') def save(): (evidence / 'results.json').write_text(json.dumps(results, indent=2) + '\n') def unchanged_cluster(): lab.require(lab.cluster() == attributes, 'Lab configuration or maintenance state changed; halt') def action(name, expected=0): unchanged_cluster() # GNU timeout bounds the RA process group, not just its parent process. result = subprocess.run(['timeout', '--kill-after=5s', '60s', str(lab.AGENT), name], env=env, cwd='/', text=True, capture_output=True) results['actions'].append(dict(action=name, rc=result.returncode, expected=expected)) (evidence / f'{len(results["actions"]):02d}-{name}.log').write_text(result.stdout + result.stderr) save() lab.require(result.returncode == expected, f'{name}: expected rc={expected}, got {result.returncode}; see {evidence}') print(f'PASS {name}: rc={result.returncode}', flush=True) save() try: action('validate-all') action('monitor') lab.require(lab.observe() == before, 'Standby changed before stop') action('stop') active = owner_command('systemctl', '--user', 'show', 'postgres.service', '--property=ActiveState', '--value') containers = owner_command('podman', 'ps', '--all', '--filter', 'name=^postgres$', '--format', '{{.State}}') results['stopped'] = dict(service=active, container=containers or 'absent') lab.require(active == 'inactive' and containers in ('', 'exited', 'stopped'), 'Service/container did not reach the expected stopped state') action('monitor', 7) results['stopped']['score'] = score() lab.require(results['stopped']['score'] == '-INFINITY', 'Stopped standby retained an eligible score') save() print('PASS stopped state: service inactive, container stopped/absent, score=-INFINITY', flush=True) verify_payload() action('start') # Existing standby.signal guard remains in the RA. deadline = time.monotonic() + 60 last_error = 'No streaming observation yet' while time.monotonic() < deadline: unchanged_cluster() try: after = lab.observe() except RuntimeError as error: last_error = str(error) time.sleep(2) continue lab.require(after['system_id'] == before['system_id'], 'PostgreSQL system identifier changed') lab.require(after['started'] != before['started'], 'Postmaster start time did not change') results['after'] = after action('monitor') results['score'] = score() save() if results['score'] == '100': break last_error = 'Streaming resumed but strict standby score has not reached 100' time.sleep(2) else: raise RuntimeError('Post-start verification deadline expired: ' + last_error) action('status') unchanged_cluster() verify_payload() results['status'] = 'passed' save() print('PASS cycle: same database, new postmaster, streaming from DV6, score=100.', flush=True) print('Maintenance remains true. No promotion, demotion, rejoin, fencing, or peer actions ran.', flush=True) except BaseException as error: results['status'] = 'failed' results['error'] = str(error) save() print('HALTED: no automatic retry, recovery, or data replacement. Inspect results:', evidence, flush=True) raise if __name__ == '__main__': try: main() except Exception as error: print('STOP:', error, file=sys.stderr) sys.exit(1)