Freeze lab-tested PostgreSQL plugin engine
Extract PostgreSQL, Podman runtime and systemd lifecycle adapters while preserving existing lifecycle and scoring behavior. Update RPM layout and add guarded operator-run lab harnesses. Record successful manual stop/start, promotion, demotion and rewind-only rejoin with pinned installed hashes. The unfenced lab does not validate automatic failover, Pacemaker-managed transitions or divergent-data rewind. Validation: 103 offline tests, original-function differential checks, checksum manifest and whitespace checks passed.
This commit is contained in:
Lyncoln
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One guarded RA stop/start cycle on the ELITEDESK lab standby only."""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pwd
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import hashlib
|
||||
|
||||
spec = importlib.util.spec_from_file_location('lab', Path(__file__).with_name('test-elitedesk-plugin.py'))
|
||||
lab = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(lab)
|
||||
|
||||
|
||||
def owner_command(*args):
|
||||
account = pwd.getpwnam('david')
|
||||
return lab.run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir,
|
||||
'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid),
|
||||
'timeout', '--kill-after=1s', '5s', *args], cwd=account.pw_dir)
|
||||
|
||||
|
||||
def score():
|
||||
return lab.run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK',
|
||||
'--lifetime=reboot', '--query', '--quiet'])
|
||||
|
||||
|
||||
def verify_payload():
|
||||
entries = lab.hashes(lab.ROOT / 'docs/lab-installed-plugin.sha256')
|
||||
lab.require(len(entries) == 15, 'Expected the 15-file installed plugin snapshot')
|
||||
for name, digest in entries.items():
|
||||
path = Path(name)
|
||||
lab.require(lab.PROVIDER in path.parents and not path.is_symlink()
|
||||
and hashlib.sha256(path.read_bytes()).hexdigest() == digest,
|
||||
'Installed code changed; inspect before testing: ' + name)
|
||||
|
||||
|
||||
def main():
|
||||
lab.require(os.geteuid() == 0, 'Run with sudo in your terminal; do not send passwords in chat')
|
||||
lab.require(socket.gethostname() == 'DAVID-ELITEDESK', 'Only the ELITEDESK lab standby is supported')
|
||||
os.umask(0o077)
|
||||
attributes = lab.cluster() # Exact lab membership/resource; maintenance=true.
|
||||
before = lab.observe() # Running standby streaming from DV6 with its own slot.
|
||||
verify_payload()
|
||||
evidence = Path(tempfile.mkdtemp(prefix='managed-container-cycle-lab-', dir=lab.BACKUP_ROOT))
|
||||
print('Results:', evidence, flush=True)
|
||||
env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', OCF_ROOT='/usr/lib/ocf',
|
||||
OCF_RESOURCE_INSTANCE='pg-ha', OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK',
|
||||
**{'OCF_RESKEY_' + k: v for k, v in attributes.items()})
|
||||
results = dict(before=before, actions=[], status='running')
|
||||
|
||||
def save():
|
||||
(evidence / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
|
||||
|
||||
def unchanged_cluster():
|
||||
lab.require(lab.cluster() == attributes, 'Lab configuration or maintenance state changed; halt')
|
||||
|
||||
def action(name, expected=0):
|
||||
unchanged_cluster()
|
||||
# GNU timeout bounds the RA process group, not just its parent process.
|
||||
result = subprocess.run(['timeout', '--kill-after=5s', '60s', str(lab.AGENT), name],
|
||||
env=env, cwd='/', text=True, capture_output=True)
|
||||
results['actions'].append(dict(action=name, rc=result.returncode, expected=expected))
|
||||
(evidence / f'{len(results["actions"]):02d}-{name}.log').write_text(result.stdout + result.stderr)
|
||||
save()
|
||||
lab.require(result.returncode == expected,
|
||||
f'{name}: expected rc={expected}, got {result.returncode}; see {evidence}')
|
||||
print(f'PASS {name}: rc={result.returncode}', flush=True)
|
||||
|
||||
save()
|
||||
try:
|
||||
action('validate-all')
|
||||
action('monitor')
|
||||
lab.require(lab.observe() == before, 'Standby changed before stop')
|
||||
action('stop')
|
||||
active = owner_command('systemctl', '--user', 'show', 'postgres.service', '--property=ActiveState', '--value')
|
||||
containers = owner_command('podman', 'ps', '--all', '--filter', 'name=^postgres$', '--format', '{{.State}}')
|
||||
results['stopped'] = dict(service=active, container=containers or 'absent')
|
||||
lab.require(active == 'inactive' and containers in ('', 'exited', 'stopped'),
|
||||
'Service/container did not reach the expected stopped state')
|
||||
action('monitor', 7)
|
||||
results['stopped']['score'] = score()
|
||||
lab.require(results['stopped']['score'] == '-INFINITY', 'Stopped standby retained an eligible score')
|
||||
save()
|
||||
print('PASS stopped state: service inactive, container stopped/absent, score=-INFINITY', flush=True)
|
||||
verify_payload()
|
||||
action('start') # Existing standby.signal guard remains in the RA.
|
||||
deadline = time.monotonic() + 60
|
||||
last_error = 'No streaming observation yet'
|
||||
while time.monotonic() < deadline:
|
||||
unchanged_cluster()
|
||||
try:
|
||||
after = lab.observe()
|
||||
except RuntimeError as error:
|
||||
last_error = str(error)
|
||||
time.sleep(2)
|
||||
continue
|
||||
lab.require(after['system_id'] == before['system_id'], 'PostgreSQL system identifier changed')
|
||||
lab.require(after['started'] != before['started'], 'Postmaster start time did not change')
|
||||
results['after'] = after
|
||||
action('monitor')
|
||||
results['score'] = score()
|
||||
save()
|
||||
if results['score'] == '100':
|
||||
break
|
||||
last_error = 'Streaming resumed but strict standby score has not reached 100'
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise RuntimeError('Post-start verification deadline expired: ' + last_error)
|
||||
action('status')
|
||||
unchanged_cluster()
|
||||
verify_payload()
|
||||
results['status'] = 'passed'
|
||||
save()
|
||||
print('PASS cycle: same database, new postmaster, streaming from DV6, score=100.', flush=True)
|
||||
print('Maintenance remains true. No promotion, demotion, rejoin, fencing, or peer actions ran.', flush=True)
|
||||
except BaseException as error:
|
||||
results['status'] = 'failed'
|
||||
results['error'] = str(error)
|
||||
save()
|
||||
print('HALTED: no automatic retry, recovery, or data replacement. Inspect results:', evidence, flush=True)
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
print('STOP:', error, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install the plugin candidate and run non-disruptive checks on the lab standby.
|
||||
|
||||
Run with sudo from the operator's terminal. Never supplies a password, changes
|
||||
maintenance mode, or runs stop/promote/demote/rejoin. Logs are root-only.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pwd
|
||||
import shutil
|
||||
import socket
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROVIDER = Path('/usr/lib/ocf/resource.d/loxthelion')
|
||||
AGENT = PROVIDER / 'managed-container'
|
||||
MODULES = PROVIDER / 'managed-container.d'
|
||||
BASELINE = ROOT / 'docs/lab-installed-before-plugin.sha256'
|
||||
BACKUP_ROOT = Path('/var/backups')
|
||||
|
||||
|
||||
def run(args, env=None, cwd=None):
|
||||
clean_env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', HOME='/root', LC_ALL='C')
|
||||
result = subprocess.run(args, env=env if env is not None else clean_env, cwd=cwd,
|
||||
text=True, capture_output=True, timeout=90)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'{Path(args[0]).name} failed (rc={result.returncode}): {result.stderr.strip()}')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def hashes(manifest):
|
||||
return dict((name, digest) for digest, name in
|
||||
(line.split(' ', 1) for line in manifest.read_text().splitlines() if line))
|
||||
|
||||
|
||||
def cluster():
|
||||
cib = ET.fromstring(run(['cibadmin', '--query']))
|
||||
require(run(['crm_attribute', '--type', 'crm_config', '--name', 'cluster-name', '--query', '--quiet'])
|
||||
== 'postgres-ra-test', 'This is not the postgres-ra-test lab')
|
||||
require(run(['crm_attribute', '--type', 'crm_config', '--name', 'maintenance-mode', '--query', '--quiet'])
|
||||
== 'true', 'Lab maintenance-mode must already be true; nothing changed')
|
||||
require({n.get('uname') for n in cib.findall('./configuration/nodes/node')} ==
|
||||
{'DAVID-ELITEDESK', 'DAVID-DV6'}, 'Unexpected cluster membership; inspect before proceeding')
|
||||
resources = cib.findall("./configuration/resources/clone[@id='pg-ha-clone']/primitive[@id='pg-ha']")
|
||||
require(len(resources) == 1, 'Expected pg-ha under pg-ha-clone')
|
||||
resource = resources[0]
|
||||
require((resource.get('class'), resource.get('provider'), resource.get('type')) ==
|
||||
('ocf', 'loxthelion', 'managed-container'), 'Unexpected resource agent')
|
||||
attributes = {}
|
||||
for group in resource.findall('instance_attributes'):
|
||||
require(not group.get('id-ref') and group.find('rule') is None,
|
||||
'Referenced/rule-based parameters require manual review')
|
||||
for pair in group.findall('nvpair'):
|
||||
name, value = pair.get('name'), pair.get('value')
|
||||
require(name not in attributes and name and value is not None, 'Ambiguous resource parameters')
|
||||
attributes[name] = value
|
||||
expected = dict(service_user='david', service_name='postgres.service', container_name='postgres',
|
||||
pg_os_user='postgres', db_user='david', db_name='david')
|
||||
require(all(attributes.get(k, v) == v for k, v in expected.items()), 'Lab service/database parameters changed')
|
||||
return dict(expected, **{k: v for k, v in attributes.items() if k not in expected})
|
||||
|
||||
|
||||
def observe():
|
||||
account = pwd.getpwnam('david')
|
||||
query = """SELECT json_build_object('recovery',pg_is_in_recovery(),
|
||||
'started',pg_postmaster_start_time()::text,
|
||||
'system_id',(SELECT system_identifier::text FROM pg_control_system()),
|
||||
'receiver',(SELECT status FROM pg_stat_wal_receiver),
|
||||
'sender',(SELECT sender_host FROM pg_stat_wal_receiver),
|
||||
'slot',(SELECT slot_name FROM pg_stat_wal_receiver));"""
|
||||
output = run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir,
|
||||
'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid),
|
||||
'timeout', '--kill-after=1s', '5s', 'podman', 'exec', '--user', 'postgres',
|
||||
'--env', 'PGOPTIONS=-c statement_timeout=2000', 'postgres',
|
||||
'psql', '-X', '-v', 'ON_ERROR_STOP=1', '-U', 'david', '-d', 'david', '-Atqc', query],
|
||||
cwd=account.pw_dir)
|
||||
state = json.loads(output)
|
||||
require(state['recovery'] is True and state['receiver'] == 'streaming'
|
||||
and state['sender'] == '192.168.1.2' and state['slot'] == 'ha_david_elitedesk',
|
||||
'Expected a streaming ELITEDESK standby of lab DV6; refusing installation/start')
|
||||
return state
|
||||
|
||||
|
||||
def main():
|
||||
require(os.geteuid() == 0, 'Run this script with sudo in your terminal; do not send passwords in chat')
|
||||
require(socket.gethostname() == 'DAVID-ELITEDESK', 'This script is only for DAVID-ELITEDESK')
|
||||
def interrupted(signum, frame):
|
||||
raise RuntimeError(f'Interrupted by signal {signum}')
|
||||
signal.signal(signal.SIGTERM, interrupted)
|
||||
signal.signal(signal.SIGINT, interrupted)
|
||||
os.umask(0o077)
|
||||
os.environ['PATH'] = '/usr/sbin:/usr/bin:/sbin:/bin'
|
||||
attributes = cluster()
|
||||
before = observe()
|
||||
for name, digest in hashes(BASELINE).items():
|
||||
path = PROVIDER / name
|
||||
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == digest,
|
||||
f'Installed baseline changed: {name}; review before replacing')
|
||||
manifest = hashes(ROOT / 'notes/REFACTOR-SHA256SUMS')
|
||||
candidates = [ROOT / 'refactored/managed-container'] + sorted(
|
||||
p for p in (ROOT / 'refactored/managed-container.d').rglob('*') if p.suffix in ('.sh', '.py') and p.is_file())
|
||||
require(len(candidates) == 15, 'Unexpected candidate file layout')
|
||||
for path in candidates:
|
||||
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
== manifest.get(str(path.relative_to(ROOT))), f'Candidate checksum mismatch: {path.name}')
|
||||
# Copy verified code into a root-owned staging directory before executing it.
|
||||
backup = Path(tempfile.mkdtemp(prefix='managed-container-plugin-lab-', dir=BACKUP_ROOT))
|
||||
print('Backup and results:', backup, flush=True)
|
||||
shutil.copy2(AGENT, backup / 'managed-container')
|
||||
shutil.copytree(MODULES, backup / 'managed-container.d', symlinks=True)
|
||||
stage = Path(tempfile.mkdtemp(prefix='.mc-plugin-stage-', dir=PROVIDER))
|
||||
for source in candidates:
|
||||
destination = stage / source.relative_to(ROOT / 'refactored')
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, destination)
|
||||
destination.chmod(0o755 if destination.name == 'managed-container' else 0o644)
|
||||
require(hashlib.sha256(destination.read_bytes()).hexdigest() ==
|
||||
manifest[str(source.relative_to(ROOT))], 'Source changed while staging')
|
||||
for directory in (stage / 'managed-container.d').rglob('*'):
|
||||
if directory.is_dir(): directory.chmod(0o755)
|
||||
(stage / 'managed-container.d').chmod(0o755)
|
||||
env = dict(PATH=os.environ['PATH'], OCF_ROOT='/usr/lib/ocf', OCF_RESOURCE_INSTANCE='pg-ha',
|
||||
OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK',
|
||||
**{'OCF_RESKEY_' + k: v for k, v in attributes.items()})
|
||||
results = dict(before=before, actions=[])
|
||||
|
||||
def action(executable, name):
|
||||
output = run([str(executable), name], env=env, cwd='/')
|
||||
results['actions'].append(dict(agent=str(executable), action=name, rc=0))
|
||||
print(f'PASS {executable.parent.name}/{name}: rc=0', flush=True)
|
||||
return output
|
||||
|
||||
staged = stage / 'managed-container'
|
||||
metadata = action(staged, 'meta-data')
|
||||
require(len(ET.fromstring(metadata).findall('./parameters/parameter')) == 16, 'Unexpected plugin metadata')
|
||||
(backup / 'candidate-metadata.xml').write_text(metadata + '\n')
|
||||
action(staged, 'validate-all')
|
||||
require(cluster() == attributes, 'Cluster configuration changed before installation')
|
||||
require(observe() == before, 'PostgreSQL state changed before installation')
|
||||
retired = stage / 'previous-modules'
|
||||
modules_moved = False
|
||||
try:
|
||||
os.replace(MODULES, retired)
|
||||
modules_moved = True
|
||||
os.replace(stage / 'managed-container.d', MODULES)
|
||||
os.replace(staged, AGENT)
|
||||
run(['restorecon', '-RF', str(AGENT), str(MODULES)])
|
||||
action(AGENT, 'validate-all')
|
||||
action(AGENT, 'monitor')
|
||||
action(AGENT, 'status')
|
||||
require(cluster() == attributes and observe() == before, 'Standby/cluster changed before idempotent start')
|
||||
action(AGENT, 'start') # Already healthy standby: exercises the no-op path.
|
||||
action(AGENT, 'monitor')
|
||||
require(cluster() == attributes, 'Cluster configuration changed during checks')
|
||||
results['after'] = observe()
|
||||
require(results['after'] == before, 'Database identity/start time or replication changed during checks')
|
||||
results['score'] = run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK',
|
||||
'--lifetime=reboot', '--query', '--quiet'])
|
||||
require(results['score'] in ('100', '-INFINITY'), 'Unexpected standby score')
|
||||
# Under write load a connected standby can legitimately fail strict catch-up.
|
||||
for source in candidates:
|
||||
installed = PROVIDER / source.relative_to(ROOT / 'refactored')
|
||||
require(hashlib.sha256(installed.read_bytes()).hexdigest() ==
|
||||
manifest[str(source.relative_to(ROOT))], 'Installed payload mismatch')
|
||||
(backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
|
||||
(backup / 'installed-SHA256SUMS').write_text(''.join(
|
||||
manifest[str(p.relative_to(ROOT))] + ' ' + str(PROVIDER / p.relative_to(ROOT / 'refactored')) + '\n'
|
||||
for p in candidates))
|
||||
except BaseException:
|
||||
if modules_moved:
|
||||
print('Check failed: restoring previous agent and modules; maintenance mode is not changed.', flush=True)
|
||||
if MODULES.exists(): os.replace(MODULES, stage / 'failed-modules')
|
||||
os.replace(retired, MODULES)
|
||||
shutil.copyfile(backup / 'managed-container', stage / 'restore-agent')
|
||||
(stage / 'restore-agent').chmod(0o755)
|
||||
os.replace(stage / 'restore-agent', AGENT)
|
||||
run(['restorecon', '-RF', str(AGENT), str(MODULES)])
|
||||
(backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
|
||||
print('Staging/evidence retained:', stage, flush=True)
|
||||
raise
|
||||
shutil.rmtree(stage)
|
||||
print('PASS: installed; PostgreSQL was not restarted; maintenance remains true.', flush=True)
|
||||
print('Current score:', results['score'], '(strict standby catch-up can fluctuate)', flush=True)
|
||||
print('No stop, promote, demote, rejoin, fencing, or peer-node actions were run.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
print('STOP:', error, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator-run, one-stage-at-a-time role tests for the two-node lab.
|
||||
|
||||
No installation, automatic recovery, fencing, or maintenance-mode changes.
|
||||
See docs/lab-role-tests.md before running a mutating stage.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pwd
|
||||
import shlex
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
PROVIDER = Path('/usr/lib/ocf/resource.d/loxthelion')
|
||||
AGENT = PROVIDER / 'managed-container'
|
||||
NODES = {'DAVID-ELITEDESK': '192.168.1.6', 'DAVID-DV6': '192.168.1.2'}
|
||||
PINNED = {'DAVID-DV6': {'managed-container': 'a4b775a0ddcabdb0558da9b34c7fe2a883c07d0ef308c1c7224526abf53f63a0',
|
||||
'managed-container.d/auth.sh': '93191771d3cb8aefadb2a16d2f89d64a8b6c584071a473439be818237bdf7f17',
|
||||
'managed-container.d/common.sh': '97e93d9164e854e5f3688cf9d3cc656b001eb30ef3c2073c907450ca43588f3d',
|
||||
'managed-container.d/container.sh': 'ac6bc363a2301f1491d162ef609103dbfbceb3b099a3556614c6165952309374',
|
||||
'managed-container.d/demote.sh': '0af3bb362a303017372c631354ba94ef9e723bf14e64d2eff91b6662e54a5be0',
|
||||
'managed-container.d/notify.sh': '6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670',
|
||||
'managed-container.d/postgres.sh': 'd2779ce6e7d1f426ffca2e41bbb8de77503005dc38d658d69f184020365796eb',
|
||||
'managed-container.d/promote.sh': '3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58',
|
||||
'managed-container.d/promotion_state.py': '5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7',
|
||||
'managed-container.d/rejoin.sh': '21beea72bcb7f24a51c17ea48666ac87df9436702ad77b166b45f4647e28e8ad',
|
||||
'managed-container.d/replication.sh': '6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547'},
|
||||
'DAVID-ELITEDESK': {'managed-container': '01eb9b161c8e3db289f09bf63953be779921e15bdf8b1a4d8ce7e60032b826e4',
|
||||
'managed-container.d/applications/postgresql/auth.sh': '10932919c4339b553c7c9dbf30b45a917c51c0a08f362e12678003d58b12fa82',
|
||||
'managed-container.d/applications/postgresql/demote.sh': 'da4e58cadf1df3953cb97a0512e3931ee1b9d3ee61c39a1ad1a6047ef537addc',
|
||||
'managed-container.d/applications/postgresql/notify.sh': '6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670',
|
||||
'managed-container.d/applications/postgresql/plugin.sh': 'd3a989de97ceb61a56828f006d5ccb87b7c34fc155a944585bedf928785021fc',
|
||||
'managed-container.d/applications/postgresql/postgres.sh': 'fa222e6767249999767eb82fac2d85e7e35404da240517a8b630571e77282053',
|
||||
'managed-container.d/applications/postgresql/promote.sh': '3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58',
|
||||
'managed-container.d/applications/postgresql/promotion_state.py': '5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7',
|
||||
'managed-container.d/applications/postgresql/rejoin.sh': '28b0e44554a88ccb87cd16a68315c045dd37d354aceb2fa25da7c7399a659b23',
|
||||
'managed-container.d/applications/postgresql/replication.sh': '6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547',
|
||||
'managed-container.d/common.sh': 'a482ef3cc1048e6a4f427e76dd2dce173ffd00760dbed1afa601a3fd05b0ad99',
|
||||
'managed-container.d/container.sh': '3a27ac9c63c61310da8b8711fa2c4400b37ea8eae2ee2a79e28bc63180b06c7c',
|
||||
'managed-container.d/engine.sh': '53978e2367394695d61c6fb38f2ee46814474c5a4e0448d2692f1a3ce583af93',
|
||||
'managed-container.d/lifecycles/systemd.sh': '6c92c095ce10ac95d843fdc1c49e31c19491ca22fdac960affd8649606d76d82',
|
||||
'managed-container.d/runtimes/podman.sh': 'dc5820067a78c4a1c531426902793aae97a03fd64912b017b3b102031dcc1e04'}}
|
||||
|
||||
QUERY = """SELECT json_build_object(
|
||||
'recovery',pg_is_in_recovery(), 'paused',pg_is_wal_replay_paused(),
|
||||
'system_id',(SELECT system_identifier::text FROM pg_control_system()),
|
||||
'started',pg_postmaster_start_time()::text,
|
||||
'receiver',(SELECT status FROM pg_stat_wal_receiver),
|
||||
'sender',(SELECT sender_host FROM pg_stat_wal_receiver),
|
||||
'slot',(SELECT slot_name FROM pg_stat_wal_receiver),
|
||||
'replay',pg_last_wal_replay_lsn()::text);"""
|
||||
# pg_is_wal_replay_paused() is only callable during recovery.
|
||||
QUERY = QUERY.replace('pg_is_wal_replay_paused()',
|
||||
'CASE WHEN pg_is_in_recovery() THEN pg_is_wal_replay_paused() ELSE false END')
|
||||
|
||||
|
||||
def require(ok, message):
|
||||
if not ok:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def run(args, **kwargs):
|
||||
result = subprocess.run(args, text=True, capture_output=True, timeout=90,
|
||||
env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', LC_ALL='C'), **kwargs)
|
||||
require(result.returncode == 0, f'{args[0]} failed: {result.stderr.strip()}')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def owner_args(*args):
|
||||
account = pwd.getpwnam('david')
|
||||
return ['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir,
|
||||
'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid), *args]
|
||||
|
||||
|
||||
def cluster():
|
||||
cib = ET.fromstring(run(['cibadmin', '--query']))
|
||||
def prop(name):
|
||||
return run(['crm_attribute', '--type', 'crm_config', '--name', name, '--query', '--quiet'])
|
||||
require(prop('cluster-name') == 'postgres-ra-test' and prop('maintenance-mode') == 'true',
|
||||
'Expected postgres-ra-test with maintenance=true; nothing changed')
|
||||
require({n.get('uname') for n in cib.findall('./configuration/nodes/node')} == set(NODES),
|
||||
'Unexpected cluster membership')
|
||||
resource = cib.find("./configuration/resources/clone[@id='pg-ha-clone']/primitive[@id='pg-ha']")
|
||||
require(resource is not None and (resource.get('class'), resource.get('provider'), resource.get('type'))
|
||||
== ('ocf', 'loxthelion', 'managed-container'), 'Unexpected resource')
|
||||
attrs = {}
|
||||
for group in resource.findall('instance_attributes'):
|
||||
require(not group.get('id-ref') and group.find('rule') is None, 'Unsupported parameter rules')
|
||||
for pair in group.findall('nvpair'):
|
||||
key, value = pair.get('name'), pair.get('value')
|
||||
require(key and value is not None and key not in attrs, 'Ambiguous parameters')
|
||||
attrs[key] = value
|
||||
defaults = dict(service_user='david', service_name='postgres.service', container_name='postgres',
|
||||
pg_os_user='postgres', db_user='david', db_name='david',
|
||||
application='postgresql', container_runtime='podman', lifecycle_manager='systemd')
|
||||
require(all(attrs.get(k, v) == v for k, v in defaults.items()), 'Unexpected service/application settings')
|
||||
return dict(defaults, **attrs)
|
||||
|
||||
|
||||
def snapshot(node):
|
||||
# Executes only read operations as david on either host. Never inspects container environment.
|
||||
code = """import hashlib,json,pathlib,socket,subprocess
|
||||
def cmd(args):
|
||||
p=subprocess.run(['timeout','--kill-after=1s','5s']+args,text=True,capture_output=True)
|
||||
if p.returncode: raise RuntimeError('Read-only observation failed: '+args[0])
|
||||
return p.stdout.strip()
|
||||
files=FILES
|
||||
base=pathlib.Path('/usr/lib/ocf/resource.d/loxthelion')
|
||||
assert all(not (base/n).is_symlink() for n in files)
|
||||
print(json.dumps(dict(node=socket.gethostname(),
|
||||
hashes={n:hashlib.sha256((base/n).read_bytes()).hexdigest() for n in files},
|
||||
service=cmd(['systemctl','--user','show','postgres.service','--property=ActiveState','--value']),
|
||||
container=cmd(['podman','inspect','--format','{{.Id}}','postgres']),
|
||||
db=json.loads(cmd(['podman','exec','--user','postgres','--env','PGOPTIONS=-c statement_timeout=2000',
|
||||
'postgres','psql','-X','-v','ON_ERROR_STOP=1','-U','david','-d','david','-Atqc',QUERY])))))
|
||||
""".replace('FILES', repr(list(PINNED[node]))).replace('QUERY', repr(QUERY))
|
||||
if node == socket.gethostname():
|
||||
args = owner_args('python3', '-c', code)
|
||||
else:
|
||||
args = owner_args('ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=5',
|
||||
'-o', 'StrictHostKeyChecking=yes', 'david@' + NODES[node],
|
||||
'python3 -c ' + shlex.quote(code))
|
||||
value = json.loads(run(args, cwd=pwd.getpwnam('david').pw_dir))
|
||||
require(value['node'] == node and value['hashes'] == PINNED[node], 'Node or installed code differs: ' + node)
|
||||
require(value['service'] == 'active', 'Maintenance service not active: ' + node)
|
||||
return value
|
||||
|
||||
|
||||
def standby(db, source):
|
||||
return (db['recovery'] is True and db['paused'] is False and db['receiver'] == 'streaming'
|
||||
and db['sender'] == NODES[source])
|
||||
|
||||
|
||||
def guard(stage, node, local, peer):
|
||||
require(local['system_id'] == peer['system_id'], 'Database system identifiers differ')
|
||||
other = next(n for n in NODES if n != node)
|
||||
if stage == 'demote':
|
||||
require(local['recovery'] is False and standby(peer, node), 'Need local primary and streaming peer standby')
|
||||
elif stage == 'promote':
|
||||
require(local['recovery'] is True and local['paused'] is False and standby(peer, node),
|
||||
'Both nodes must be in recovery and the demoted peer must stream from this node')
|
||||
elif stage == 'rejoin':
|
||||
require(node == 'DAVID-ELITEDESK' and standby(local, other) and peer['recovery'] is False,
|
||||
'Rejoin test requires ELITEDESK streaming from primary DV6')
|
||||
|
||||
|
||||
def main(stage):
|
||||
node = socket.gethostname()
|
||||
require(os.geteuid() == 0 and node in NODES, 'Run with sudo on an expected lab node')
|
||||
os.umask(0o077)
|
||||
peer = next(n for n in NODES if n != node)
|
||||
attrs = cluster()
|
||||
before, remote = snapshot(node), snapshot(peer)
|
||||
guard(stage, node, before['db'], remote['db'])
|
||||
if stage == 'inspect':
|
||||
print(json.dumps(dict(local=before, peer=remote, maintenance=True), indent=2))
|
||||
return
|
||||
evidence = Path(tempfile.mkdtemp(prefix='managed-container-roles-lab-', dir='/var/backups'))
|
||||
print('Results:', evidence, flush=True)
|
||||
result = dict(stage=stage, node=node, before=before, peer_before=remote, actions=[], status='running')
|
||||
env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', OCF_ROOT='/usr/lib/ocf', OCF_RESOURCE_INSTANCE='pg-ha',
|
||||
OCF_RESKEY_CRM_meta_on_node=node, **{'OCF_RESKEY_' + k: v for k, v in attrs.items()})
|
||||
def save():
|
||||
(evidence / 'results.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||
def action(name, expected=0, extra=None):
|
||||
require(cluster() == attrs, 'Cluster configuration or maintenance changed')
|
||||
response = subprocess.run(['timeout', '--kill-after=5s', '120s', str(AGENT), name],
|
||||
env=dict(env, **(extra or {})), cwd='/', text=True, capture_output=True)
|
||||
result['actions'].append(dict(action=name, rc=response.returncode, expected=expected))
|
||||
(evidence / f'{len(result["actions"]):02d}-{name}.log').write_text(response.stdout + response.stderr)
|
||||
save()
|
||||
require(response.returncode == expected, f'{name} returned {response.returncode}; expected {expected}')
|
||||
print(f'PASS {name}: rc={response.returncode}', flush=True)
|
||||
save()
|
||||
try:
|
||||
action('validate-all')
|
||||
action('monitor', 8 if before['db']['recovery'] is False else 0)
|
||||
# Reobserve immediately before mutation. No action retry or inferred fencing certificate.
|
||||
current, remote = snapshot(node), snapshot(peer)
|
||||
require(current['container'] == before['container'] and
|
||||
all(current['db'][k] == before['db'][k] for k in ('system_id', 'started', 'recovery')),
|
||||
'Local incarnation/role changed during preflight')
|
||||
guard(stage, node, current['db'], remote['db'])
|
||||
if stage == 'demote':
|
||||
action('notify', extra=dict(OCF_RESKEY_CRM_meta_notify_type='pre',
|
||||
OCF_RESKEY_CRM_meta_notify_operation='demote',
|
||||
OCF_RESKEY_CRM_meta_notify_demote_uname=node,
|
||||
OCF_RESKEY_CRM_meta_notify_promote_uname=peer))
|
||||
action('demote')
|
||||
elif stage == 'promote':
|
||||
action('promote')
|
||||
elif stage == 'rejoin':
|
||||
# This stage tests rewind/recovery of a healthy standby, not divergent former-primary repair.
|
||||
run(owner_args('timeout', '--kill-after=5s', '60s', 'podman', 'exec', '--user', 'postgres',
|
||||
'postgres', 'sh', '-c',
|
||||
'exec "$(pg_config --bindir)/pg_ctl" -D "$PGDATA" stop -m fast -w'))
|
||||
action('monitor', 7)
|
||||
require(run(['crm_attribute', '--promotion=pg-ha', '--node', node,
|
||||
'--lifetime=reboot', '--query', '--quiet']) == '-INFINITY',
|
||||
'Stopped PostgreSQL retained an eligible score')
|
||||
require(snapshot(peer)['db']['recovery'] is False, 'DV6 is no longer primary')
|
||||
action('rejoin', extra=dict(OCF_RESKEY_rejoin_source_host=NODES[peer],
|
||||
OCF_RESKEY_replication_slot='ha_david_elitedesk',
|
||||
OCF_RESKEY_application_name=node,
|
||||
OCF_RESKEY_rejoin_allow_basebackup='false'))
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
after, remote = snapshot(node), snapshot(peer)
|
||||
require(after['db']['system_id'] == before['db']['system_id'], 'Local database identity changed')
|
||||
require(after['container'] == before['container'], 'Maintenance container was replaced')
|
||||
require(remote['db']['system_id'] == before['db']['system_id'], 'Peer database identity changed')
|
||||
if stage == 'promote':
|
||||
require(after['db']['recovery'] is False and standby(remote['db'], node), 'Unexpected roles after promotion')
|
||||
expected_score = '1000'
|
||||
else:
|
||||
require(standby(after['db'], peer), 'Local node did not return to streaming recovery')
|
||||
require(after['db']['started'] != before['db']['started'], 'Expected a new postmaster')
|
||||
require(remote['db']['recovery'] is (stage == 'demote'), 'Unexpected peer role')
|
||||
expected_score = '100'
|
||||
action('monitor', 8 if stage == 'promote' else 0)
|
||||
value = run(['crm_attribute', '--promotion=pg-ha', '--node', node, '--lifetime=reboot', '--query', '--quiet'])
|
||||
if value == expected_score:
|
||||
break
|
||||
require(time.monotonic() < deadline, 'Expected promotion score did not converge')
|
||||
time.sleep(2)
|
||||
result.update(after=after, peer_after=remote, score=value, status='passed')
|
||||
require(cluster() == attrs, 'Cluster configuration changed')
|
||||
save()
|
||||
print(f'PASS {stage}: verified roles, streaming, same database/container, score={value}.', flush=True)
|
||||
print('Maintenance remains true. No fencing or automatic recovery ran.', flush=True)
|
||||
except BaseException as error:
|
||||
result.update(status='failed', error=str(error))
|
||||
save()
|
||||
print('HALTED. Do not continue to the next stage. Inspect:', evidence, flush=True)
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('stage', choices=['inspect', 'demote', 'promote', 'rejoin'])
|
||||
try:
|
||||
main(parser.parse_args().stage)
|
||||
except Exception as error:
|
||||
raise SystemExit('STOP: ' + str(error))
|
||||
Reference in New Issue
Block a user