#!/usr/bin/env python3 """State and monitor integration tests: temporary files and fake boundaries only.""" import copy import importlib.util import json import os from pathlib import Path import shlex import tempfile import unittest from test_promotion_scores import Scores, LOAD, FIXTURE ROOT = Path(__file__).resolve().parents[1] HELPER = ROOT / 'refactored/managed-container.d/applications/postgresql/promotion_state.py' spec = importlib.util.spec_from_file_location('promotion_state', HELPER) policy = importlib.util.module_from_spec(spec) spec.loader.exec_module(policy) CONNECTED = dict(identity=['7684451577990111248', '/data/pg18', '1789200000.123456'], recovery=True, paused=False, receiver='streaming', replay='0/30', flush='0/20', strict=True) DISCONNECTED = dict(CONNECTED, receiver=None, flush=None, strict=None) class State(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory(prefix='mc-evidence-') self.addCleanup(self.tmp.cleanup) self.evidence = policy.Evidence(Path(self.tmp.name) / 'promotion', ['pg-ha:0', 'owner', 'postgres.service', 'postgres'], 'test-boot', 1000.0, 1789201000.0, os.getuid()) def check(self, obs=DISCONNECTED, age=300): return self.evidence.evaluate(copy.deepcopy(obs), age) def seed(self): self.assertEqual(self.check(CONNECTED), '100') return json.loads(self.evidence.path.read_text()) def test_streaming_disconnected_eligible_and_persisted_lsn(self): record = self.seed() self.assertEqual(record['replay'], '0/30') self.assertEqual(record['flush'], '0/20') self.evidence.now += 50 self.assertEqual(self.check(), '50') self.assertEqual(json.loads(self.evidence.path.read_text()), record) self.assertEqual(self.evidence.path.stat().st_mode & 0o777, 0o600) self.assertEqual(self.evidence.directory.stat().st_mode & 0o777, 0o700) def test_never_verified(self): self.assertEqual(self.check(), '-INFINITY') def test_stale_and_exact_boundary(self): self.seed(); self.evidence.now += 300 self.assertEqual(self.check(), '50') self.evidence.now += .001 self.assertEqual(self.check(), '-INFINITY') self.assertFalse(self.evidence.path.exists()) def test_disabled_and_invalid_age(self): self.seed() self.assertEqual(self.check(age=0), '-INFINITY') for age in (-1, True, 100000, '300'): with self.assertRaises(ValueError): self.check(age=age) def test_malformed_state(self): for value in ('{', '[]', '{}', 'null', '{"version":true}', '{"version":1,"version":1}', 'x' * 4097): self.seed(); self.evidence.path.write_text(value) self.assertEqual(self.check(), '-INFINITY', value[:40]) self.assertFalse(self.evidence.path.exists()) def test_invalid_record_fields(self): for key, value in [('replay', '0/ZZ'), ('flush', '1/0'), ('boot', 'other'), ('observed_boottime', float('nan')), ('observed_boottime', 1001), ('observed_unix', True), ('context', ['other']), ('version', True)]: record = self.seed(); record[key] = value self.evidence.path.write_text(json.dumps(record)) self.assertEqual(self.check(), '-INFINITY', key) def test_system_data_path_and_postmaster_identity_changes(self): for index, value in enumerate(['999', '/new/pgdata', '1789200100.123456']): self.seed(); obs = copy.deepcopy(DISCONNECTED); obs['identity'][index] = value self.assertEqual(self.check(obs), '-INFINITY') self.assertFalse(self.evidence.path.exists()) def test_reinitialization_requires_reverification_then_replaces_state(self): self.seed() disconnected = copy.deepcopy(DISCONNECTED) disconnected['identity'][2] = '1789200100.123456' # Same system ID/basebackup. self.assertEqual(self.check(disconnected), '-INFINITY') connected = dict(CONNECTED, identity=disconnected['identity']) self.assertEqual(self.check(connected), '100') self.assertEqual(self.check(disconnected), '50') self.assertEqual(json.loads(self.evidence.path.read_text())['identity'], connected['identity']) def test_paused_primary_and_replay_regression_invalidate(self): for changes in ({'paused': True}, {'recovery': False, 'paused': True}, {'replay': '0/10'}, {'replay': None}): self.seed() self.assertEqual(self.check(dict(DISCONNECTED, **changes)), '-INFINITY') self.assertFalse(self.evidence.path.exists()) def test_connected_failure_cannot_use_disconnected_fallback(self): for changes in ({'flush': '0/40'}, {'flush': None}, {'paused': True}): self.seed() self.assertEqual(self.check(dict(CONNECTED, **changes, strict=False)), '-INFINITY') self.assertFalse(self.evidence.path.exists()) def test_reconnect_refreshes_timestamp_and_lsns(self): self.seed(); self.evidence.now += 100 self.assertEqual(self.check(), '50') self.assertEqual(self.check(dict(CONNECTED, replay='0/50', flush='0/40')), '100') record = json.loads(self.evidence.path.read_text()) self.assertEqual(record['observed_boottime'], 1100) self.assertEqual(record['replay'], '0/50') self.assertEqual(self.check(dict(DISCONNECTED, replay='0/50')), '50') def test_wall_clock_steps_do_not_extend_or_expire_observation(self): self.seed(); self.evidence.wall -= 100000; self.evidence.now += 100 self.assertEqual(self.check(), '50') self.evidence.wall += 200000; self.evidence.now += 201 self.assertEqual(self.check(), '-INFINITY') def test_symlink_state_rejected_without_reading_target(self): self.seed(); self.evidence.path.unlink() target = Path(self.tmp.name) / 'target'; target.write_text('unchanged') self.evidence.path.symlink_to(target) self.assertEqual(self.check(), '-INFINITY') self.assertEqual(target.read_text(), 'unchanged') def test_unsafe_directory_rejected(self): self.seed(); self.evidence.directory.chmod(0o777) with self.assertRaises(ValueError): self.check() # Reuse fake crm_attribute and the real monitor wrapper. The real state evaluator # runs as this test user against a private directory; never writes /run or CIB. class Monitor(unittest.TestCase): setUp = Scores.setUp run_shell = Scores.run_shell state = Scores.state score = Scores.score def test_invalid_observation_query_and_helper_failure_replace_score(self): fixture = monitor_fixture(self) self.env['OBSERVATION'] = json.dumps(CONNECTED) for failure in ('mc_pg_standby_observation() { echo malformed; }', 'mc_pg_standby_observation() { return 1; }', 'python3() { return 124; }'): self.run_shell('mc_promotion_monitor', fixture=fixture) self.score('100') self.run_shell(failure + '; mc_promotion_monitor', fixture=fixture) self.score('-INFINITY') def monitor_fixture(test): driver = test.path / 'driver.py' driver.write_text('''import importlib.util,json,os,sys,time from pathlib import Path spec=importlib.util.spec_from_file_location('policy',sys.argv[1]) p=importlib.util.module_from_spec(spec);spec.loader.exec_module(p) e=p.Evidence(Path(sys.argv[2]),['pg-ha:0','owner','service','container'],'boot',float(os.environ.get('TEST_NOW','1000')),1789201000,os.getuid()) print(e.evaluate(json.load(sys.stdin),300)) ''') # Restore the production score helper after the original tests' mock. return FIXTURE + '\n. ' + shlex.quote(str(HELPER.with_name('postgres.sh'))) + ''' service_running() { [ "$SERVICE" = 1 ]; } container_running() { [ "$CONTAINER" = 1 ]; } pg_running() { [ "$RUNNING" = 1 ]; } pg_ready() { return 0; } pg_role() { echo "$ROLE"; } mc_pg_standby_observation() { printf '%s' "$OBSERVATION"; } python3() { command python3 ''' + ' '.join(map(shlex.quote, [str(driver), str(HELPER), str(test.path / 'evidence')])) + '''; } timeout() { shift 2; "$@"; } ''' def test_monitor_disconnect_reprobe_stopped_reconnect(self): fixture = monitor_fixture(self) self.env['OBSERVATION'] = json.dumps(CONNECTED) self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('100') self.env['OBSERVATION'] = json.dumps(DISCONNECTED) self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('50') data = self.state(); data['scores'] = {}; self.db.write_text(json.dumps(data)) self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('50') self.run_shell('RUNNING=0; mc_promotion_monitor', 7, fixture=fixture); self.score('-INFINITY') self.env['OBSERVATION'] = json.dumps(CONNECTED) self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('100') self.env['OBSERVATION'] = json.dumps(DISCONNECTED); self.env['TEST_NOW'] = '1301' self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('-INFINITY') Monitor.test_monitor_disconnect_reprobe_stopped_reconnect = test_monitor_disconnect_reprobe_stopped_reconnect if __name__ == '__main__': # Explicit suite excludes imported Scores from duplicate discovery. unittest.TextTestRunner(verbosity=2).run(unittest.TestSuite( unittest.defaultTestLoader.loadTestsFromTestCase(c) for c in (State, Monitor) )).wasSuccessful() or __import__('sys').exit(1)