ocf-managed-container/scripts/test-elitedesk-cycle.py

#!/usr/bin/env python3
"""One guarded RA stop/start cycle on the ELITEDESK lab standby only."""
import importlib.util
import json
import os
from pathlib import Path
import pwd
import socket
import subprocess
import sys
import tempfile
import time
import hashlib

spec = importlib.util.spec_from_file_location('lab', Path(__file__).with_name('test-elitedesk-plugin.py'))
lab = importlib.util.module_from_spec(spec)
spec.loader.exec_module(lab)


def owner_command(*args):
    account = pwd.getpwnam('david')
    return lab.run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir,
                    'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid),
                    'timeout', '--kill-after=1s', '5s', *args], cwd=account.pw_dir)


def score():
    return lab.run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK',
                    '--lifetime=reboot', '--query', '--quiet'])


def verify_payload():
    entries = lab.hashes(lab.ROOT / 'docs/lab-installed-plugin.sha256')
    lab.require(len(entries) == 15, 'Expected the 15-file installed plugin snapshot')
    for name, digest in entries.items():
        path = Path(name)
        lab.require(lab.PROVIDER in path.parents and not path.is_symlink()
                    and hashlib.sha256(path.read_bytes()).hexdigest() == digest,
                    'Installed code changed; inspect before testing: ' + name)


def main():
    lab.require(os.geteuid() == 0, 'Run with sudo in your terminal; do not send passwords in chat')
    lab.require(socket.gethostname() == 'DAVID-ELITEDESK', 'Only the ELITEDESK lab standby is supported')
    os.umask(0o077)
    attributes = lab.cluster()  # Exact lab membership/resource; maintenance=true.
    before = lab.observe()     # Running standby streaming from DV6 with its own slot.
    verify_payload()
    evidence = Path(tempfile.mkdtemp(prefix='managed-container-cycle-lab-', dir=lab.BACKUP_ROOT))
    print('Results:', evidence, flush=True)
    env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', OCF_ROOT='/usr/lib/ocf',
               OCF_RESOURCE_INSTANCE='pg-ha', OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK',
               **{'OCF_RESKEY_' + k: v for k, v in attributes.items()})
    results = dict(before=before, actions=[], status='running')

    def save():
        (evidence / 'results.json').write_text(json.dumps(results, indent=2) + '\n')

    def unchanged_cluster():
        lab.require(lab.cluster() == attributes, 'Lab configuration or maintenance state changed; halt')

    def action(name, expected=0):
        unchanged_cluster()
        # GNU timeout bounds the RA process group, not just its parent process.
        result = subprocess.run(['timeout', '--kill-after=5s', '60s', str(lab.AGENT), name],
                                env=env, cwd='/', text=True, capture_output=True)
        results['actions'].append(dict(action=name, rc=result.returncode, expected=expected))
        (evidence / f'{len(results["actions"]):02d}-{name}.log').write_text(result.stdout + result.stderr)
        save()
        lab.require(result.returncode == expected,
                    f'{name}: expected rc={expected}, got {result.returncode}; see {evidence}')
        print(f'PASS {name}: rc={result.returncode}', flush=True)

    save()
    try:
        action('validate-all')
        action('monitor')
        lab.require(lab.observe() == before, 'Standby changed before stop')
        action('stop')
        active = owner_command('systemctl', '--user', 'show', 'postgres.service', '--property=ActiveState', '--value')
        containers = owner_command('podman', 'ps', '--all', '--filter', 'name=^postgres$', '--format', '{{.State}}')
        results['stopped'] = dict(service=active, container=containers or 'absent')
        lab.require(active == 'inactive' and containers in ('', 'exited', 'stopped'),
                    'Service/container did not reach the expected stopped state')
        action('monitor', 7)
        results['stopped']['score'] = score()
        lab.require(results['stopped']['score'] == '-INFINITY', 'Stopped standby retained an eligible score')
        save()
        print('PASS stopped state: service inactive, container stopped/absent, score=-INFINITY', flush=True)
        verify_payload()
        action('start')  # Existing standby.signal guard remains in the RA.
        deadline = time.monotonic() + 60
        last_error = 'No streaming observation yet'
        while time.monotonic() < deadline:
            unchanged_cluster()
            try:
                after = lab.observe()
            except RuntimeError as error:
                last_error = str(error)
                time.sleep(2)
                continue
            lab.require(after['system_id'] == before['system_id'], 'PostgreSQL system identifier changed')
            lab.require(after['started'] != before['started'], 'Postmaster start time did not change')
            results['after'] = after
            action('monitor')
            results['score'] = score()
            save()
            if results['score'] == '100':
                break
            last_error = 'Streaming resumed but strict standby score has not reached 100'
            time.sleep(2)
        else:
            raise RuntimeError('Post-start verification deadline expired: ' + last_error)
        action('status')
        unchanged_cluster()
        verify_payload()
        results['status'] = 'passed'
        save()
        print('PASS cycle: same database, new postmaster, streaming from DV6, score=100.', flush=True)
        print('Maintenance remains true. No promotion, demotion, rejoin, fencing, or peer actions ran.', flush=True)
    except BaseException as error:
        results['status'] = 'failed'
        results['error'] = str(error)
        save()
        print('HALTED: no automatic retry, recovery, or data replacement. Inspect results:', evidence, flush=True)
        raise


if __name__ == '__main__':
    try:
        main()
    except Exception as error:
        print('STOP:', error, file=sys.stderr)
        sys.exit(1)