Freeze HA scoring and document reported production failover

2026-09-12 18:07:28 -04:00
parent c8636ed6d4
commit 488a1672d7
29 changed files with 2285 additions and 46 deletions
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""State and monitor integration tests: temporary files and fake boundaries only."""
+import copy
+import importlib.util
+import json
+import os
+from pathlib import Path
+import shlex
+import tempfile
+import unittest
+
+from test_promotion_scores import Scores, LOAD, FIXTURE
+
+ROOT = Path(__file__).resolve().parents[1]
+HELPER = ROOT / 'refactored/managed-container.d/promotion_state.py'
+spec = importlib.util.spec_from_file_location('promotion_state', HELPER)
+policy = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(policy)
+CONNECTED = dict(identity=['7684451577990111248', '/data/pg18', '1789200000.123456'],
+                 recovery=True, paused=False, receiver='streaming', replay='0/30',
+                 flush='0/20', strict=True)
+DISCONNECTED = dict(CONNECTED, receiver=None, flush=None, strict=None)
+
+
+class State(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory(prefix='mc-evidence-')
+        self.addCleanup(self.tmp.cleanup)
+        self.evidence = policy.Evidence(Path(self.tmp.name) / 'promotion',
+                                       ['pg-ha:0', 'owner', 'postgres.service', 'postgres'],
+                                       'test-boot', 1000.0, 1789201000.0, os.getuid())
+
+    def check(self, obs=DISCONNECTED, age=300):
+        return self.evidence.evaluate(copy.deepcopy(obs), age)
+
+    def seed(self):
+        self.assertEqual(self.check(CONNECTED), '100')
+        return json.loads(self.evidence.path.read_text())
+
+    def test_streaming_disconnected_eligible_and_persisted_lsn(self):
+        record = self.seed()
+        self.assertEqual(record['replay'], '0/30')
+        self.assertEqual(record['flush'], '0/20')
+        self.evidence.now += 50
+        self.assertEqual(self.check(), '50')
+        self.assertEqual(json.loads(self.evidence.path.read_text()), record)
+        self.assertEqual(self.evidence.path.stat().st_mode & 0o777, 0o600)
+        self.assertEqual(self.evidence.directory.stat().st_mode & 0o777, 0o700)
+
+    def test_never_verified(self):
+        self.assertEqual(self.check(), '-INFINITY')
+
+    def test_stale_and_exact_boundary(self):
+        self.seed(); self.evidence.now += 300
+        self.assertEqual(self.check(), '50')
+        self.evidence.now += .001
+        self.assertEqual(self.check(), '-INFINITY')
+        self.assertFalse(self.evidence.path.exists())
+
+    def test_disabled_and_invalid_age(self):
+        self.seed()
+        self.assertEqual(self.check(age=0), '-INFINITY')
+        for age in (-1, True, 100000, '300'):
+            with self.assertRaises(ValueError): self.check(age=age)
+
+    def test_malformed_state(self):
+        for value in ('{', '[]', '{}', 'null', '{"version":true}', '{"version":1,"version":1}', 'x' * 4097):
+            self.seed(); self.evidence.path.write_text(value)
+            self.assertEqual(self.check(), '-INFINITY', value[:40])
+            self.assertFalse(self.evidence.path.exists())
+
+    def test_invalid_record_fields(self):
+        for key, value in [('replay', '0/ZZ'), ('flush', '1/0'), ('boot', 'other'),
+                           ('observed_boottime', float('nan')), ('observed_boottime', 1001),
+                           ('observed_unix', True), ('context', ['other']), ('version', True)]:
+            record = self.seed(); record[key] = value
+            self.evidence.path.write_text(json.dumps(record))
+            self.assertEqual(self.check(), '-INFINITY', key)
+
+    def test_system_data_path_and_postmaster_identity_changes(self):
+        for index, value in enumerate(['999', '/new/pgdata', '1789200100.123456']):
+            self.seed(); obs = copy.deepcopy(DISCONNECTED); obs['identity'][index] = value
+            self.assertEqual(self.check(obs), '-INFINITY')
+            self.assertFalse(self.evidence.path.exists())
+
+    def test_reinitialization_requires_reverification_then_replaces_state(self):
+        self.seed()
+        disconnected = copy.deepcopy(DISCONNECTED)
+        disconnected['identity'][2] = '1789200100.123456'  # Same system ID/basebackup.
+        self.assertEqual(self.check(disconnected), '-INFINITY')
+        connected = dict(CONNECTED, identity=disconnected['identity'])
+        self.assertEqual(self.check(connected), '100')
+        self.assertEqual(self.check(disconnected), '50')
+        self.assertEqual(json.loads(self.evidence.path.read_text())['identity'], connected['identity'])
+
+    def test_paused_primary_and_replay_regression_invalidate(self):
+        for changes in ({'paused': True}, {'recovery': False, 'paused': True},
+                        {'replay': '0/10'}, {'replay': None}):
+            self.seed()
+            self.assertEqual(self.check(dict(DISCONNECTED, **changes)), '-INFINITY')
+            self.assertFalse(self.evidence.path.exists())
+
+    def test_connected_failure_cannot_use_disconnected_fallback(self):
+        for changes in ({'flush': '0/40'}, {'flush': None}, {'paused': True}):
+            self.seed()
+            self.assertEqual(self.check(dict(CONNECTED, **changes, strict=False)), '-INFINITY')
+            self.assertFalse(self.evidence.path.exists())
+
+    def test_reconnect_refreshes_timestamp_and_lsns(self):
+        self.seed(); self.evidence.now += 100
+        self.assertEqual(self.check(), '50')
+        self.assertEqual(self.check(dict(CONNECTED, replay='0/50', flush='0/40')), '100')
+        record = json.loads(self.evidence.path.read_text())
+        self.assertEqual(record['observed_boottime'], 1100)
+        self.assertEqual(record['replay'], '0/50')
+        self.assertEqual(self.check(dict(DISCONNECTED, replay='0/50')), '50')
+
+    def test_wall_clock_steps_do_not_extend_or_expire_observation(self):
+        self.seed(); self.evidence.wall -= 100000; self.evidence.now += 100
+        self.assertEqual(self.check(), '50')
+        self.evidence.wall += 200000; self.evidence.now += 201
+        self.assertEqual(self.check(), '-INFINITY')
+
+    def test_symlink_state_rejected_without_reading_target(self):
+        self.seed(); self.evidence.path.unlink()
+        target = Path(self.tmp.name) / 'target'; target.write_text('unchanged')
+        self.evidence.path.symlink_to(target)
+        self.assertEqual(self.check(), '-INFINITY')
+        self.assertEqual(target.read_text(), 'unchanged')
+
+    def test_unsafe_directory_rejected(self):
+        self.seed(); self.evidence.directory.chmod(0o777)
+        with self.assertRaises(ValueError): self.check()
+
+
+# Reuse fake crm_attribute and the real monitor wrapper. The real state evaluator
+# runs as this test user against a private directory; never writes /run or CIB.
+class Monitor(unittest.TestCase):
+    setUp = Scores.setUp
+    run_shell = Scores.run_shell
+    state = Scores.state
+    score = Scores.score
+
+    def test_invalid_observation_query_and_helper_failure_replace_score(self):
+        fixture = monitor_fixture(self)
+        self.env['OBSERVATION'] = json.dumps(CONNECTED)
+        for failure in ('mc_pg_standby_observation() { echo malformed; }',
+                        'mc_pg_standby_observation() { return 1; }',
+                        'python3() { return 124; }'):
+            self.run_shell('mc_promotion_monitor', fixture=fixture)
+            self.score('100')
+            self.run_shell(failure + '; mc_promotion_monitor', fixture=fixture)
+            self.score('-INFINITY')
+
+
+def monitor_fixture(test):
+    driver = test.path / 'driver.py'
+    driver.write_text('''import importlib.util,json,os,sys,time
+from pathlib import Path
+spec=importlib.util.spec_from_file_location('policy',sys.argv[1])
+p=importlib.util.module_from_spec(spec);spec.loader.exec_module(p)
+e=p.Evidence(Path(sys.argv[2]),['pg-ha:0','owner','service','container'],'boot',float(os.environ.get('TEST_NOW','1000')),1789201000,os.getuid())
+print(e.evaluate(json.load(sys.stdin),300))
+''')
+    # Restore the production score helper after the original tests' mock.
+    return FIXTURE + '\n. ' + shlex.quote(str(HELPER.with_name('postgres.sh'))) + '''
+service_running() { [ "$SERVICE" = 1 ]; }
+container_running() { [ "$CONTAINER" = 1 ]; }
+pg_running() { [ "$RUNNING" = 1 ]; }
+pg_ready() { return 0; }
+pg_role() { echo "$ROLE"; }
+mc_pg_standby_observation() { printf '%s' "$OBSERVATION"; }
+python3() { command python3 ''' + ' '.join(map(shlex.quote, [str(driver), str(HELPER), str(test.path / 'evidence')])) + '''; }
+timeout() { shift 2; "$@"; }
+'''
+
+
+def test_monitor_disconnect_reprobe_stopped_reconnect(self):
+    fixture = monitor_fixture(self)
+    self.env['OBSERVATION'] = json.dumps(CONNECTED)
+    self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('100')
+    self.env['OBSERVATION'] = json.dumps(DISCONNECTED)
+    self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('50')
+    data = self.state(); data['scores'] = {}; self.db.write_text(json.dumps(data))
+    self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('50')
+    self.run_shell('RUNNING=0; mc_promotion_monitor', 7, fixture=fixture); self.score('-INFINITY')
+    self.env['OBSERVATION'] = json.dumps(CONNECTED)
+    self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('100')
+    self.env['OBSERVATION'] = json.dumps(DISCONNECTED); self.env['TEST_NOW'] = '1301'
+    self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('-INFINITY')
+
+
+Monitor.test_monitor_disconnect_reprobe_stopped_reconnect = test_monitor_disconnect_reprobe_stopped_reconnect
+
+if __name__ == '__main__':
+    # Explicit suite excludes imported Scores from duplicate discovery.
+    unittest.TextTestRunner(verbosity=2).run(unittest.TestSuite(
+        unittest.defaultTestLoader.loadTestsFromTestCase(c) for c in (State, Monitor)
+    )).wasSuccessful() or __import__('sys').exit(1)