diff --git a/plugins/postgresql/manifest.json b/plugins/postgresql/manifest.json index e4f753e..3a0983b 100644 --- a/plugins/postgresql/manifest.json +++ b/plugins/postgresql/manifest.json @@ -1,11 +1,12 @@ { "name": "postgresql", - "version": 3, + "version": 4, "module": "plugin.py", "actions": [ "monitor", "notify", "notify.target", + "promote.database", "demote.prepare", "demote.database" ], diff --git a/plugins/postgresql/plugin.py b/plugins/postgresql/plugin.py index 6ac4523..310e449 100644 --- a/plugins/postgresql/plugin.py +++ b/plugins/postgresql/plugin.py @@ -531,6 +531,191 @@ def notify_target(config, runtime_state, request_parameters): } + +def promote_database(config, runtime_state, request_parameters): + """ + Read-only PostgreSQL promotion preflight. + + Validate that Pacemaker selected this local node for promotion, that the + local PostgreSQL instance is still an eligible standby, and that no + reachable configured peer is currently writable. + + This action never promotes PostgreSQL. The OCF resource agent remains + authoritative for the actual pg_ctl promote operation and for local + filesystem state such as the rejoin-required marker. + """ + del request_parameters + + node_name = str( + config.get("PACEMAKER_NODE_NAME", "") + ).strip().lower() + + if not node_name: + raise ValueError( + "PACEMAKER_NODE_NAME is required for promote.database" + ) + + notification = _postgres_runtime_state( + runtime_state + ).get("notification") + + if not isinstance(notification, dict): + return { + "ocf_rc": 1, + "ready": False, + "mutation": False, + "reason": "notification_missing", + "node": node_name, + } + + if notification.get("type") != "pre": + return { + "ocf_rc": 1, + "ready": False, + "mutation": False, + "reason": "notification_type_invalid", + "node": node_name, + } + + if notification.get("operation") != "promote": + return { + "ocf_rc": 1, + "ready": False, + "mutation": False, + "reason": "notification_operation_invalid", + "node": node_name, + } + + received_at = notification.get("received_at") + + if not isinstance(received_at, (int, float)): + return { + "ocf_rc": 1, + "ready": False, + "mutation": False, + "reason": "notification_timestamp_invalid", + "node": node_name, + } + + age = time.time() - received_at + + if age < 0 or age > NOTIFY_TARGET_MAX_AGE_SECONDS: + return { + "ocf_rc": 1, + "ready": False, + "mutation": False, + "reason": "notification_stale", + "age_seconds": age, + "node": node_name, + } + + target_node = str( + notification.get("promote_uname", "") + ).strip().lower() + + previous_primary_node = str( + notification.get("demote_uname", "") + ).strip().lower() + + if not target_node: + return { + "ocf_rc": 1, + "ready": False, + "mutation": False, + "reason": "promotion_target_missing", + "node": node_name, + } + + if target_node != node_name: + return { + "ocf_rc": 1, + "ready": False, + "mutation": False, + "reason": "local_node_not_target", + "node": node_name, + "target_node": target_node, + "previous_primary_node": previous_primary_node, + } + + local = monitor(config, runtime_state) + local_ha = local.get("ha") or {} + + checks = { + "notification": True, + "local_target": True, + "local_monitor_success": ( + local_ha.get("ocf_rc") == OCF_SUCCESS + ), + "local_standby": ( + local.get("role") == "standby" + and local.get("in_recovery") is True + ), + "promotion_eligible": ( + local_ha.get("promotion_eligible") is True + ), + } + + if not all(checks.values()): + return { + "ocf_rc": 1, + "ready": False, + "mutation": False, + "reason": "local_not_promotion_eligible", + "node": node_name, + "target_node": target_node, + "previous_primary_node": previous_primary_node, + "local": local_ha, + "checks": checks, + } + + try: + password = read_secret(config["PGPASSWORD_FILE"]) + peer_state = probe_replication_peers(config, password) + except (KeyError, TypeError, ValueError, psycopg2.Error) as exc: + return { + "ocf_rc": 1, + "ready": False, + "mutation": False, + "reason": "peer_probe_failed", + "node": node_name, + "target_node": target_node, + "previous_primary_node": previous_primary_node, + "error": str(exc).splitlines()[0], + "checks": checks, + } + + writable_primaries = [ + str(name).strip().lower() + for name in peer_state.get("writable_primaries", []) + ] + + checks["no_reachable_writable_primary"] = ( + len(writable_primaries) == 0 + ) + + ready = all(checks.values()) + + return { + "ocf_rc": OCF_SUCCESS if ready else OCF_ERR_GENERIC, + "ready": ready, + "mutation": False, + "reason": ( + "promotion_preflight_ready" + if ready + else "reachable_writable_primary" + ), + "node": node_name, + "target_node": target_node, + "previous_primary_node": previous_primary_node, + "writable_primaries": writable_primaries, + "writable_primary_count": len(writable_primaries), + "peer_state": peer_state, + "checks": checks, + "age_seconds": age, + } + + + def demote_prepare(config, runtime_state, request_parameters): """ Read-only demotion preparation. diff --git a/tests/test-promote-database.py b/tests/test-promote-database.py new file mode 100755 index 0000000..7233e49 --- /dev/null +++ b/tests/test-promote-database.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 + +import importlib.util +import sys +import time +import types +from pathlib import Path + + +fake_psycopg2 = types.ModuleType("psycopg2") +fake_psycopg2.Error = Exception +fake_psycopg2.OperationalError = OSError + +fake_extras = types.ModuleType("psycopg2.extras") +fake_extras.RealDictCursor = object +fake_psycopg2.extras = fake_extras + +sys.modules["psycopg2"] = fake_psycopg2 +sys.modules["psycopg2.extras"] = fake_extras + + +spec = importlib.util.spec_from_file_location( + "postgresql_plugin", + Path("plugins/postgresql/plugin.py"), +) + +plugin = importlib.util.module_from_spec(spec) +spec.loader.exec_module(plugin) + + +def standby_monitor(*, eligible=True): + return { + "ha": { + "ocf_rc": plugin.OCF_SUCCESS, + "health": "healthy", + "ha_role": "unpromoted", + "state": "streaming", + "promotion_eligible": eligible, + }, + "role": "standby", + "in_recovery": True, + "streaming": True, + } + + +def peer_state(*primaries): + return { + "peers": [], + "writable_primaries": list(primaries), + "writable_primary_count": len(primaries), + } + + +config = { + "PACEMAKER_NODE_NAME": "david-eve", + "PGPASSWORD_FILE": "/run/secrets/postgres_password", + "PG_REPLICATION_PEERS": ( + "david-prodesk=192.168.1.4:5432," + "david-macbookpro=192.168.1.5:5432," + "david-eve=192.168.1.3:5432" + ), + "PGUSER": "david", + "PGDATABASE": "david", +} + + +def fresh_state(): + return { + "postgresql": { + "notification": { + "type": "pre", + "operation": "promote", + "promote_uname": "david-eve", + "demote_uname": "david-prodesk", + "received_at": time.time(), + }, + }, + } + + +plugin.read_secret = lambda path: "test-password" +plugin.monitor = lambda config, runtime_state: standby_monitor() +plugin.probe_replication_peers = ( + lambda config, password: peer_state() +) + + +print("===== PROMOTE DATABASE ACTION =====") + +runtime_state = fresh_state() + +result = plugin.promote_database( + config, + runtime_state, + {}, +) + +assert result["ocf_rc"] == plugin.OCF_SUCCESS +assert result["ready"] is True +assert result["mutation"] is False +assert result["reason"] == "promotion_preflight_ready" +assert result["target_node"] == "david-eve" +assert result["previous_primary_node"] == "david-prodesk" +assert result["writable_primary_count"] == 0 +assert result["checks"]["local_target"] is True +assert result["checks"]["local_standby"] is True +assert result["checks"]["promotion_eligible"] is True +assert result["checks"]["no_reachable_writable_primary"] is True + +print("valid_promotion_preflight accepted") + + +runtime_state = fresh_state() +runtime_state["postgresql"]["notification"]["promote_uname"] = ( + "david-macbookpro" +) + +result = plugin.promote_database( + config, + runtime_state, + {}, +) + +assert result["ocf_rc"] == 1 +assert result["ready"] is False +assert result["reason"] == "local_node_not_target" + +print("wrong_target rejected") + + +runtime_state = fresh_state() +runtime_state["postgresql"]["notification"]["received_at"] = ( + time.time() - 121 +) + +result = plugin.promote_database( + config, + runtime_state, + {}, +) + +assert result["ocf_rc"] == 1 +assert result["reason"] == "notification_stale" + +print("stale_notification rejected") + + +result = plugin.promote_database( + config, + {}, + {}, +) + +assert result["ocf_rc"] == 1 +assert result["reason"] == "notification_missing" + +print("missing_notification rejected") + + +runtime_state = fresh_state() + +plugin.monitor = lambda config, runtime_state: { + "ha": { + "ocf_rc": plugin.OCF_RUNNING_PROMOTED, + "promotion_eligible": True, + }, + "role": "primary", + "in_recovery": False, +} + +result = plugin.promote_database( + config, + runtime_state, + {}, +) + +assert result["ocf_rc"] == 1 +assert result["ready"] is False +assert result["reason"] == "local_not_promotion_eligible" +assert result["checks"]["local_standby"] is False + +print("local_primary rejected") + + +plugin.monitor = lambda config, runtime_state: standby_monitor( + eligible=False +) + +result = plugin.promote_database( + config, + fresh_state(), + {}, +) + +assert result["ocf_rc"] == 1 +assert result["ready"] is False +assert result["checks"]["promotion_eligible"] is False + +print("ineligible_standby rejected") + + +plugin.monitor = lambda config, runtime_state: standby_monitor() +plugin.probe_replication_peers = ( + lambda config, password: peer_state("david-prodesk") +) + +result = plugin.promote_database( + config, + fresh_state(), + {}, +) + +assert result["ocf_rc"] == plugin.OCF_ERR_GENERIC +assert result["ready"] is False +assert result["mutation"] is False +assert result["reason"] == "reachable_writable_primary" +assert result["writable_primary_count"] == 1 +assert result["writable_primaries"] == ["david-prodesk"] +assert result["checks"]["no_reachable_writable_primary"] is False + +print("existing_primary rejected") + + +print() +print("PROMOTE_DATABASE_TEST_PASS=1") diff --git a/tests/test-runtime-state.py b/tests/test-runtime-state.py index 119e824..92f4e07 100755 --- a/tests/test-runtime-state.py +++ b/tests/test-runtime-state.py @@ -179,8 +179,8 @@ production_manifest = json.loads( ) require( - production_manifest.get("version") == 3, - "PostgreSQL manifest version is not 3", + production_manifest.get("version") == 4, + "PostgreSQL manifest version is not 4", ) require( "PG_REPLICATION_PEERS"