200 lines
9.5 KiB
Python
200 lines
9.5 KiB
Python
#!/usr/bin/env python3
|
|
"""State and monitor integration tests: temporary files and fake boundaries only."""
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import tempfile
|
|
import unittest
|
|
|
|
from test_promotion_scores import Scores, LOAD, FIXTURE
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
HELPER = ROOT / 'refactored/managed-container.d/promotion_state.py'
|
|
spec = importlib.util.spec_from_file_location('promotion_state', HELPER)
|
|
policy = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(policy)
|
|
CONNECTED = dict(identity=['7684451577990111248', '/data/pg18', '1789200000.123456'],
|
|
recovery=True, paused=False, receiver='streaming', replay='0/30',
|
|
flush='0/20', strict=True)
|
|
DISCONNECTED = dict(CONNECTED, receiver=None, flush=None, strict=None)
|
|
|
|
|
|
class State(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory(prefix='mc-evidence-')
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.evidence = policy.Evidence(Path(self.tmp.name) / 'promotion',
|
|
['pg-ha:0', 'owner', 'postgres.service', 'postgres'],
|
|
'test-boot', 1000.0, 1789201000.0, os.getuid())
|
|
|
|
def check(self, obs=DISCONNECTED, age=300):
|
|
return self.evidence.evaluate(copy.deepcopy(obs), age)
|
|
|
|
def seed(self):
|
|
self.assertEqual(self.check(CONNECTED), '100')
|
|
return json.loads(self.evidence.path.read_text())
|
|
|
|
def test_streaming_disconnected_eligible_and_persisted_lsn(self):
|
|
record = self.seed()
|
|
self.assertEqual(record['replay'], '0/30')
|
|
self.assertEqual(record['flush'], '0/20')
|
|
self.evidence.now += 50
|
|
self.assertEqual(self.check(), '50')
|
|
self.assertEqual(json.loads(self.evidence.path.read_text()), record)
|
|
self.assertEqual(self.evidence.path.stat().st_mode & 0o777, 0o600)
|
|
self.assertEqual(self.evidence.directory.stat().st_mode & 0o777, 0o700)
|
|
|
|
def test_never_verified(self):
|
|
self.assertEqual(self.check(), '-INFINITY')
|
|
|
|
def test_stale_and_exact_boundary(self):
|
|
self.seed(); self.evidence.now += 300
|
|
self.assertEqual(self.check(), '50')
|
|
self.evidence.now += .001
|
|
self.assertEqual(self.check(), '-INFINITY')
|
|
self.assertFalse(self.evidence.path.exists())
|
|
|
|
def test_disabled_and_invalid_age(self):
|
|
self.seed()
|
|
self.assertEqual(self.check(age=0), '-INFINITY')
|
|
for age in (-1, True, 100000, '300'):
|
|
with self.assertRaises(ValueError): self.check(age=age)
|
|
|
|
def test_malformed_state(self):
|
|
for value in ('{', '[]', '{}', 'null', '{"version":true}', '{"version":1,"version":1}', 'x' * 4097):
|
|
self.seed(); self.evidence.path.write_text(value)
|
|
self.assertEqual(self.check(), '-INFINITY', value[:40])
|
|
self.assertFalse(self.evidence.path.exists())
|
|
|
|
def test_invalid_record_fields(self):
|
|
for key, value in [('replay', '0/ZZ'), ('flush', '1/0'), ('boot', 'other'),
|
|
('observed_boottime', float('nan')), ('observed_boottime', 1001),
|
|
('observed_unix', True), ('context', ['other']), ('version', True)]:
|
|
record = self.seed(); record[key] = value
|
|
self.evidence.path.write_text(json.dumps(record))
|
|
self.assertEqual(self.check(), '-INFINITY', key)
|
|
|
|
def test_system_data_path_and_postmaster_identity_changes(self):
|
|
for index, value in enumerate(['999', '/new/pgdata', '1789200100.123456']):
|
|
self.seed(); obs = copy.deepcopy(DISCONNECTED); obs['identity'][index] = value
|
|
self.assertEqual(self.check(obs), '-INFINITY')
|
|
self.assertFalse(self.evidence.path.exists())
|
|
|
|
def test_reinitialization_requires_reverification_then_replaces_state(self):
|
|
self.seed()
|
|
disconnected = copy.deepcopy(DISCONNECTED)
|
|
disconnected['identity'][2] = '1789200100.123456' # Same system ID/basebackup.
|
|
self.assertEqual(self.check(disconnected), '-INFINITY')
|
|
connected = dict(CONNECTED, identity=disconnected['identity'])
|
|
self.assertEqual(self.check(connected), '100')
|
|
self.assertEqual(self.check(disconnected), '50')
|
|
self.assertEqual(json.loads(self.evidence.path.read_text())['identity'], connected['identity'])
|
|
|
|
def test_paused_primary_and_replay_regression_invalidate(self):
|
|
for changes in ({'paused': True}, {'recovery': False, 'paused': True},
|
|
{'replay': '0/10'}, {'replay': None}):
|
|
self.seed()
|
|
self.assertEqual(self.check(dict(DISCONNECTED, **changes)), '-INFINITY')
|
|
self.assertFalse(self.evidence.path.exists())
|
|
|
|
def test_connected_failure_cannot_use_disconnected_fallback(self):
|
|
for changes in ({'flush': '0/40'}, {'flush': None}, {'paused': True}):
|
|
self.seed()
|
|
self.assertEqual(self.check(dict(CONNECTED, **changes, strict=False)), '-INFINITY')
|
|
self.assertFalse(self.evidence.path.exists())
|
|
|
|
def test_reconnect_refreshes_timestamp_and_lsns(self):
|
|
self.seed(); self.evidence.now += 100
|
|
self.assertEqual(self.check(), '50')
|
|
self.assertEqual(self.check(dict(CONNECTED, replay='0/50', flush='0/40')), '100')
|
|
record = json.loads(self.evidence.path.read_text())
|
|
self.assertEqual(record['observed_boottime'], 1100)
|
|
self.assertEqual(record['replay'], '0/50')
|
|
self.assertEqual(self.check(dict(DISCONNECTED, replay='0/50')), '50')
|
|
|
|
def test_wall_clock_steps_do_not_extend_or_expire_observation(self):
|
|
self.seed(); self.evidence.wall -= 100000; self.evidence.now += 100
|
|
self.assertEqual(self.check(), '50')
|
|
self.evidence.wall += 200000; self.evidence.now += 201
|
|
self.assertEqual(self.check(), '-INFINITY')
|
|
|
|
def test_symlink_state_rejected_without_reading_target(self):
|
|
self.seed(); self.evidence.path.unlink()
|
|
target = Path(self.tmp.name) / 'target'; target.write_text('unchanged')
|
|
self.evidence.path.symlink_to(target)
|
|
self.assertEqual(self.check(), '-INFINITY')
|
|
self.assertEqual(target.read_text(), 'unchanged')
|
|
|
|
def test_unsafe_directory_rejected(self):
|
|
self.seed(); self.evidence.directory.chmod(0o777)
|
|
with self.assertRaises(ValueError): self.check()
|
|
|
|
|
|
# Reuse fake crm_attribute and the real monitor wrapper. The real state evaluator
|
|
# runs as this test user against a private directory; never writes /run or CIB.
|
|
class Monitor(unittest.TestCase):
|
|
setUp = Scores.setUp
|
|
run_shell = Scores.run_shell
|
|
state = Scores.state
|
|
score = Scores.score
|
|
|
|
def test_invalid_observation_query_and_helper_failure_replace_score(self):
|
|
fixture = monitor_fixture(self)
|
|
self.env['OBSERVATION'] = json.dumps(CONNECTED)
|
|
for failure in ('mc_pg_standby_observation() { echo malformed; }',
|
|
'mc_pg_standby_observation() { return 1; }',
|
|
'python3() { return 124; }'):
|
|
self.run_shell('mc_promotion_monitor', fixture=fixture)
|
|
self.score('100')
|
|
self.run_shell(failure + '; mc_promotion_monitor', fixture=fixture)
|
|
self.score('-INFINITY')
|
|
|
|
|
|
def monitor_fixture(test):
|
|
driver = test.path / 'driver.py'
|
|
driver.write_text('''import importlib.util,json,os,sys,time
|
|
from pathlib import Path
|
|
spec=importlib.util.spec_from_file_location('policy',sys.argv[1])
|
|
p=importlib.util.module_from_spec(spec);spec.loader.exec_module(p)
|
|
e=p.Evidence(Path(sys.argv[2]),['pg-ha:0','owner','service','container'],'boot',float(os.environ.get('TEST_NOW','1000')),1789201000,os.getuid())
|
|
print(e.evaluate(json.load(sys.stdin),300))
|
|
''')
|
|
# Restore the production score helper after the original tests' mock.
|
|
return FIXTURE + '\n. ' + shlex.quote(str(HELPER.with_name('postgres.sh'))) + '''
|
|
service_running() { [ "$SERVICE" = 1 ]; }
|
|
container_running() { [ "$CONTAINER" = 1 ]; }
|
|
pg_running() { [ "$RUNNING" = 1 ]; }
|
|
pg_ready() { return 0; }
|
|
pg_role() { echo "$ROLE"; }
|
|
mc_pg_standby_observation() { printf '%s' "$OBSERVATION"; }
|
|
python3() { command python3 ''' + ' '.join(map(shlex.quote, [str(driver), str(HELPER), str(test.path / 'evidence')])) + '''; }
|
|
timeout() { shift 2; "$@"; }
|
|
'''
|
|
|
|
|
|
def test_monitor_disconnect_reprobe_stopped_reconnect(self):
|
|
fixture = monitor_fixture(self)
|
|
self.env['OBSERVATION'] = json.dumps(CONNECTED)
|
|
self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('100')
|
|
self.env['OBSERVATION'] = json.dumps(DISCONNECTED)
|
|
self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('50')
|
|
data = self.state(); data['scores'] = {}; self.db.write_text(json.dumps(data))
|
|
self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('50')
|
|
self.run_shell('RUNNING=0; mc_promotion_monitor', 7, fixture=fixture); self.score('-INFINITY')
|
|
self.env['OBSERVATION'] = json.dumps(CONNECTED)
|
|
self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('100')
|
|
self.env['OBSERVATION'] = json.dumps(DISCONNECTED); self.env['TEST_NOW'] = '1301'
|
|
self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('-INFINITY')
|
|
|
|
|
|
Monitor.test_monitor_disconnect_reprobe_stopped_reconnect = test_monitor_disconnect_reprobe_stopped_reconnect
|
|
|
|
if __name__ == '__main__':
|
|
# Explicit suite excludes imported Scores from duplicate discovery.
|
|
unittest.TextTestRunner(verbosity=2).run(unittest.TestSuite(
|
|
unittest.defaultTestLoader.loadTestsFromTestCase(c) for c in (State, Monitor)
|
|
)).wasSuccessful() or __import__('sys').exit(1)
|