Freeze lab-tested PostgreSQL plugin engine

Extract PostgreSQL, Podman runtime and systemd lifecycle adapters while preserving existing lifecycle and scoring behavior. Update RPM layout and add guarded operator-run lab harnesses. Record successful manual stop/start, promotion, demotion and rewind-only rejoin with pinned installed hashes. The unfenced lab does not validate automatic failover, Pacemaker-managed transitions or divergent-data rewind. Validation: 103 offline tests, original-function differential checks, checksum manifest and whitespace checks passed.
2026-09-13 01:57:04 -04:00
parent 488a1672d7
commit ec2077f979
41 changed files with 1925 additions and 331 deletions
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""One guarded RA stop/start cycle on the ELITEDESK lab standby only."""
+import importlib.util
+import json
+import os
+from pathlib import Path
+import pwd
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import hashlib
+
+spec = importlib.util.spec_from_file_location('lab', Path(__file__).with_name('test-elitedesk-plugin.py'))
+lab = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(lab)
+
+
+def owner_command(*args):
+    account = pwd.getpwnam('david')
+    return lab.run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir,
+                    'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid),
+                    'timeout', '--kill-after=1s', '5s', *args], cwd=account.pw_dir)
+
+
+def score():
+    return lab.run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK',
+                    '--lifetime=reboot', '--query', '--quiet'])
+
+
+def verify_payload():
+    entries = lab.hashes(lab.ROOT / 'docs/lab-installed-plugin.sha256')
+    lab.require(len(entries) == 15, 'Expected the 15-file installed plugin snapshot')
+    for name, digest in entries.items():
+        path = Path(name)
+        lab.require(lab.PROVIDER in path.parents and not path.is_symlink()
+                    and hashlib.sha256(path.read_bytes()).hexdigest() == digest,
+                    'Installed code changed; inspect before testing: ' + name)
+
+
+def main():
+    lab.require(os.geteuid() == 0, 'Run with sudo in your terminal; do not send passwords in chat')
+    lab.require(socket.gethostname() == 'DAVID-ELITEDESK', 'Only the ELITEDESK lab standby is supported')
+    os.umask(0o077)
+    attributes = lab.cluster()  # Exact lab membership/resource; maintenance=true.
+    before = lab.observe()     # Running standby streaming from DV6 with its own slot.
+    verify_payload()
+    evidence = Path(tempfile.mkdtemp(prefix='managed-container-cycle-lab-', dir=lab.BACKUP_ROOT))
+    print('Results:', evidence, flush=True)
+    env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', OCF_ROOT='/usr/lib/ocf',
+               OCF_RESOURCE_INSTANCE='pg-ha', OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK',
+               **{'OCF_RESKEY_' + k: v for k, v in attributes.items()})
+    results = dict(before=before, actions=[], status='running')
+
+    def save():
+        (evidence / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
+
+    def unchanged_cluster():
+        lab.require(lab.cluster() == attributes, 'Lab configuration or maintenance state changed; halt')
+
+    def action(name, expected=0):
+        unchanged_cluster()
+        # GNU timeout bounds the RA process group, not just its parent process.
+        result = subprocess.run(['timeout', '--kill-after=5s', '60s', str(lab.AGENT), name],
+                                env=env, cwd='/', text=True, capture_output=True)
+        results['actions'].append(dict(action=name, rc=result.returncode, expected=expected))
+        (evidence / f'{len(results["actions"]):02d}-{name}.log').write_text(result.stdout + result.stderr)
+        save()
+        lab.require(result.returncode == expected,
+                    f'{name}: expected rc={expected}, got {result.returncode}; see {evidence}')
+        print(f'PASS {name}: rc={result.returncode}', flush=True)
+
+    save()
+    try:
+        action('validate-all')
+        action('monitor')
+        lab.require(lab.observe() == before, 'Standby changed before stop')
+        action('stop')
+        active = owner_command('systemctl', '--user', 'show', 'postgres.service', '--property=ActiveState', '--value')
+        containers = owner_command('podman', 'ps', '--all', '--filter', 'name=^postgres$', '--format', '{{.State}}')
+        results['stopped'] = dict(service=active, container=containers or 'absent')
+        lab.require(active == 'inactive' and containers in ('', 'exited', 'stopped'),
+                    'Service/container did not reach the expected stopped state')
+        action('monitor', 7)
+        results['stopped']['score'] = score()
+        lab.require(results['stopped']['score'] == '-INFINITY', 'Stopped standby retained an eligible score')
+        save()
+        print('PASS stopped state: service inactive, container stopped/absent, score=-INFINITY', flush=True)
+        verify_payload()
+        action('start')  # Existing standby.signal guard remains in the RA.
+        deadline = time.monotonic() + 60
+        last_error = 'No streaming observation yet'
+        while time.monotonic() < deadline:
+            unchanged_cluster()
+            try:
+                after = lab.observe()
+            except RuntimeError as error:
+                last_error = str(error)
+                time.sleep(2)
+                continue
+            lab.require(after['system_id'] == before['system_id'], 'PostgreSQL system identifier changed')
+            lab.require(after['started'] != before['started'], 'Postmaster start time did not change')
+            results['after'] = after
+            action('monitor')
+            results['score'] = score()
+            save()
+            if results['score'] == '100':
+                break
+            last_error = 'Streaming resumed but strict standby score has not reached 100'
+            time.sleep(2)
+        else:
+            raise RuntimeError('Post-start verification deadline expired: ' + last_error)
+        action('status')
+        unchanged_cluster()
+        verify_payload()
+        results['status'] = 'passed'
+        save()
+        print('PASS cycle: same database, new postmaster, streaming from DV6, score=100.', flush=True)
+        print('Maintenance remains true. No promotion, demotion, rejoin, fencing, or peer actions ran.', flush=True)
+    except BaseException as error:
+        results['status'] = 'failed'
+        results['error'] = str(error)
+        save()
+        print('HALTED: no automatic retry, recovery, or data replacement. Inspect results:', evidence, flush=True)
+        raise
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as error:
+        print('STOP:', error, file=sys.stderr)
+        sys.exit(1)