Handle PostgreSQL self-restart demotion preflight

plugins/postgresql/manifest.json

@@ -1,11 +1,12 @@
 {
   "name": "postgresql",
-  "version": 3,
+  "version": 4,
   "module": "plugin.py",
   "actions": [
     "monitor",
     "notify",
     "notify.target",
+    "promote.database",
     "demote.prepare",
     "demote.database"
   ],

plugins/postgresql/plugin.py

@@ -531,6 +531,191 @@ def notify_target(config, runtime_state, request_parameters):
     }
 
 
+
+def promote_database(config, runtime_state, request_parameters):
+    """
+    Read-only PostgreSQL promotion preflight.
+
+    Validate that Pacemaker selected this local node for promotion, that the
+    local PostgreSQL instance is still an eligible standby, and that no
+    reachable configured peer is currently writable.
+
+    This action never promotes PostgreSQL.  The OCF resource agent remains
+    authoritative for the actual pg_ctl promote operation and for local
+    filesystem state such as the rejoin-required marker.
+    """
+    del request_parameters
+
+    node_name = str(
+        config.get("PACEMAKER_NODE_NAME", "")
+    ).strip().lower()
+
+    if not node_name:
+        raise ValueError(
+            "PACEMAKER_NODE_NAME is required for promote.database"
+        )
+
+    notification = _postgres_runtime_state(
+        runtime_state
+    ).get("notification")
+
+    if not isinstance(notification, dict):
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "mutation": False,
+            "reason": "notification_missing",
+            "node": node_name,
+        }
+
+    if notification.get("type") != "pre":
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "mutation": False,
+            "reason": "notification_type_invalid",
+            "node": node_name,
+        }
+
+    if notification.get("operation") != "promote":
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "mutation": False,
+            "reason": "notification_operation_invalid",
+            "node": node_name,
+        }
+
+    received_at = notification.get("received_at")
+
+    if not isinstance(received_at, (int, float)):
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "mutation": False,
+            "reason": "notification_timestamp_invalid",
+            "node": node_name,
+        }
+
+    age = time.time() - received_at
+
+    if age < 0 or age > NOTIFY_TARGET_MAX_AGE_SECONDS:
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "mutation": False,
+            "reason": "notification_stale",
+            "age_seconds": age,
+            "node": node_name,
+        }
+
+    target_node = str(
+        notification.get("promote_uname", "")
+    ).strip().lower()
+
+    previous_primary_node = str(
+        notification.get("demote_uname", "")
+    ).strip().lower()
+
+    if not target_node:
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "mutation": False,
+            "reason": "promotion_target_missing",
+            "node": node_name,
+        }
+
+    if target_node != node_name:
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "mutation": False,
+            "reason": "local_node_not_target",
+            "node": node_name,
+            "target_node": target_node,
+            "previous_primary_node": previous_primary_node,
+        }
+
+    local = monitor(config, runtime_state)
+    local_ha = local.get("ha") or {}
+
+    checks = {
+        "notification": True,
+        "local_target": True,
+        "local_monitor_success": (
+            local_ha.get("ocf_rc") == OCF_SUCCESS
+        ),
+        "local_standby": (
+            local.get("role") == "standby"
+            and local.get("in_recovery") is True
+        ),
+        "promotion_eligible": (
+            local_ha.get("promotion_eligible") is True
+        ),
+    }
+
+    if not all(checks.values()):
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "mutation": False,
+            "reason": "local_not_promotion_eligible",
+            "node": node_name,
+            "target_node": target_node,
+            "previous_primary_node": previous_primary_node,
+            "local": local_ha,
+            "checks": checks,
+        }
+
+    try:
+        password = read_secret(config["PGPASSWORD_FILE"])
+        peer_state = probe_replication_peers(config, password)
+    except (KeyError, TypeError, ValueError, psycopg2.Error) as exc:
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "mutation": False,
+            "reason": "peer_probe_failed",
+            "node": node_name,
+            "target_node": target_node,
+            "previous_primary_node": previous_primary_node,
+            "error": str(exc).splitlines()[0],
+            "checks": checks,
+        }
+
+    writable_primaries = [
+        str(name).strip().lower()
+        for name in peer_state.get("writable_primaries", [])
+    ]
+
+    checks["no_reachable_writable_primary"] = (
+        len(writable_primaries) == 0
+    )
+
+    ready = all(checks.values())
+
+    return {
+        "ocf_rc": OCF_SUCCESS if ready else OCF_ERR_GENERIC,
+        "ready": ready,
+        "mutation": False,
+        "reason": (
+            "promotion_preflight_ready"
+            if ready
+            else "reachable_writable_primary"
+        ),
+        "node": node_name,
+        "target_node": target_node,
+        "previous_primary_node": previous_primary_node,
+        "writable_primaries": writable_primaries,
+        "writable_primary_count": len(writable_primaries),
+        "peer_state": peer_state,
+        "checks": checks,
+        "age_seconds": age,
+    }
+
+
+
 def demote_prepare(config, runtime_state, request_parameters):
     """
     Read-only demotion preparation.
@@ -617,6 +802,47 @@ def demote_prepare(config, runtime_state, request_parameters):
             "successor_node": successor_node,
         }
 
+    promoted_nodes = {
+        str(name).strip().lower()
+        for name in str(
+            notification.get("promoted_uname", "")
+        ).split()
+        if str(name).strip()
+    }
+
+    if successor_node == node_name:
+        if node_name not in promoted_nodes:
+            return {
+                "ocf_rc": 1,
+                "ready": False,
+                "reason": "self_restart_not_current_primary",
+                "node": node_name,
+                "target_node": target_node,
+                "successor_node": successor_node,
+            }
+
+        return {
+            "ocf_rc": 0,
+            "ready": False,
+            "identity_validated": True,
+            "reason": "local_restart_checks_pending",
+            "mutation": False,
+            "node": node_name,
+            "target_node": target_node,
+            "successor_node": successor_node,
+            "handoff_mode": "local_restart",
+            "final_handoff_required": False,
+            "age_seconds": age,
+            "checks": {
+                "notification": True,
+                "local_target": True,
+                "self_successor": True,
+                "local_currently_promoted": True,
+                "postgresql_mutated": False,
+                "database_checks": False,
+            },
+        }
+
     if not successor_node:
         return {
             "ocf_rc": 1,
@@ -700,6 +926,7 @@ def demote_database(config, runtime_state, request_parameters):
 
     if preparation.get("reason") not in (
         "database_checks_pending",
+        "local_restart_checks_pending",
     ):
         return preparation
 
@@ -713,6 +940,26 @@ def demote_database(config, runtime_state, request_parameters):
             "local": local["ha"],
         }
 
+    if preparation.get("reason") == "local_restart_checks_pending":
+        return {
+            "ocf_rc": 0,
+            "ready": True,
+            "mutation": False,
+            "final_handoff_required": False,
+            "handoff_mode": "local_restart",
+            "reason": "local_restart_demotion_ready",
+            "node": preparation["node"],
+            "target_node": preparation["target_node"],
+            "successor_node": preparation["successor_node"],
+            "checks": {
+                "notification": True,
+                "local_target": True,
+                "local_primary": True,
+                "self_successor": True,
+                "remote_handoff_required": False,
+            },
+        }
+
     successor_node = preparation["successor_node"]
     peers = parse_replication_peers(
         config.get("PG_REPLICATION_PEERS", "")

tests/test-demote-database.py

@@ -137,3 +137,56 @@ assert result["reason"] == "successor_unreachable"
 print("successor_unreachable    rejected")
 
 print("DEMOTE_DATABASE_FAILURE_TEST_PASS=1")
+
+print()
+print("===== SELF-RESTART DATABASE GATE =====")
+
+plugin.monitor = lambda config, runtime_state: {
+    "ha": {"ocf_rc": plugin.OCF_RUNNING_PROMOTED},
+    "primary": {"current_wal_lsn": "14/1C07A920"},
+}
+
+self_restart_state = {
+    "postgresql": {
+        "notification": {
+            "type": "pre",
+            "operation": "demote",
+            "demote_uname": "david-prodesk",
+            "promote_uname": "david-prodesk",
+            "promoted_uname": "david-prodesk",
+            "unpromoted_uname": "david-eve",
+            "received_at": time.time(),
+        },
+    },
+}
+
+connect_called = False
+
+def must_not_connect(**kwargs):
+    global connect_called
+    connect_called = True
+    raise AssertionError(
+        "self-restart demotion must not probe a remote successor"
+    )
+
+plugin.psycopg2.connect = must_not_connect
+
+result = plugin.demote_database(
+    config,
+    self_restart_state,
+    {},
+)
+
+assert result["ocf_rc"] == 0
+assert result["ready"] is True
+assert result["mutation"] is False
+assert result["final_handoff_required"] is False
+assert result["handoff_mode"] == "local_restart"
+assert result["reason"] == "local_restart_demotion_ready"
+assert result["successor_node"] == "david-prodesk"
+assert result["checks"]["local_primary"] is True
+assert result["checks"]["remote_handoff_required"] is False
+assert connect_called is False
+
+print("self_restart_database     accepted")
+print("remote_probe_skipped      accepted")

tests/test-demote-prepare.py

@@ -124,3 +124,63 @@ result = plugin.demote_prepare(
 assert result["ocf_rc"] == 1
 assert result["reason"] == "successor_not_configured"
 print("unconfigured_successor  rejected")
+
+print()
+print("===== SELF-RESTART PREPARE =====")
+
+self_restart_state = {
+    "postgresql": {
+        "notification": {
+            "type": "pre",
+            "operation": "demote",
+            "demote_uname": "david-eve",
+            "promote_uname": "david-eve",
+            "promoted_uname": "david-eve",
+            "unpromoted_uname": "david-prodesk david-macbookpro",
+            "received_at": time.time(),
+        },
+    },
+}
+
+result = plugin.demote_prepare(
+    {
+        "PACEMAKER_NODE_NAME": "david-eve",
+        "PG_REPLICATION_PEERS": (
+            "david-prodesk=192.168.1.4:5432,"
+            "david-eve=192.168.1.3:5432"
+        ),
+    },
+    self_restart_state,
+    {},
+)
+
+assert result["ocf_rc"] == 0
+assert result["ready"] is False
+assert result["identity_validated"] is True
+assert result["reason"] == "local_restart_checks_pending"
+assert result["handoff_mode"] == "local_restart"
+assert result["final_handoff_required"] is False
+assert result["successor_node"] == "david-eve"
+assert result["checks"]["self_successor"] is True
+assert result["checks"]["local_currently_promoted"] is True
+
+print("self_restart_prepare      accepted")
+
+self_restart_state["postgresql"]["notification"]["promoted_uname"] = ""
+
+result = plugin.demote_prepare(
+    {
+        "PACEMAKER_NODE_NAME": "david-eve",
+        "PG_REPLICATION_PEERS": (
+            "david-prodesk=192.168.1.4:5432,"
+            "david-eve=192.168.1.3:5432"
+        ),
+    },
+    self_restart_state,
+    {},
+)
+
+assert result["ocf_rc"] == 1
+assert result["reason"] == "self_restart_not_current_primary"
+
+print("false_self_restart        rejected")

tests/test-promote-database.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+
+import importlib.util
+import sys
+import time
+import types
+from pathlib import Path
+
+
+fake_psycopg2 = types.ModuleType("psycopg2")
+fake_psycopg2.Error = Exception
+fake_psycopg2.OperationalError = OSError
+
+fake_extras = types.ModuleType("psycopg2.extras")
+fake_extras.RealDictCursor = object
+fake_psycopg2.extras = fake_extras
+
+sys.modules["psycopg2"] = fake_psycopg2
+sys.modules["psycopg2.extras"] = fake_extras
+
+
+spec = importlib.util.spec_from_file_location(
+    "postgresql_plugin",
+    Path("plugins/postgresql/plugin.py"),
+)
+
+plugin = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(plugin)
+
+
+def standby_monitor(*, eligible=True):
+    return {
+        "ha": {
+            "ocf_rc": plugin.OCF_SUCCESS,
+            "health": "healthy",
+            "ha_role": "unpromoted",
+            "state": "streaming",
+            "promotion_eligible": eligible,
+        },
+        "role": "standby",
+        "in_recovery": True,
+        "streaming": True,
+    }
+
+
+def peer_state(*primaries):
+    return {
+        "peers": [],
+        "writable_primaries": list(primaries),
+        "writable_primary_count": len(primaries),
+    }
+
+
+config = {
+    "PACEMAKER_NODE_NAME": "david-eve",
+    "PGPASSWORD_FILE": "/run/secrets/postgres_password",
+    "PG_REPLICATION_PEERS": (
+        "david-prodesk=192.168.1.4:5432,"
+        "david-macbookpro=192.168.1.5:5432,"
+        "david-eve=192.168.1.3:5432"
+    ),
+    "PGUSER": "david",
+    "PGDATABASE": "david",
+}
+
+
+def fresh_state():
+    return {
+        "postgresql": {
+            "notification": {
+                "type": "pre",
+                "operation": "promote",
+                "promote_uname": "david-eve",
+                "demote_uname": "david-prodesk",
+                "received_at": time.time(),
+            },
+        },
+    }
+
+
+plugin.read_secret = lambda path: "test-password"
+plugin.monitor = lambda config, runtime_state: standby_monitor()
+plugin.probe_replication_peers = (
+    lambda config, password: peer_state()
+)
+
+
+print("===== PROMOTE DATABASE ACTION =====")
+
+runtime_state = fresh_state()
+
+result = plugin.promote_database(
+    config,
+    runtime_state,
+    {},
+)
+
+assert result["ocf_rc"] == plugin.OCF_SUCCESS
+assert result["ready"] is True
+assert result["mutation"] is False
+assert result["reason"] == "promotion_preflight_ready"
+assert result["target_node"] == "david-eve"
+assert result["previous_primary_node"] == "david-prodesk"
+assert result["writable_primary_count"] == 0
+assert result["checks"]["local_target"] is True
+assert result["checks"]["local_standby"] is True
+assert result["checks"]["promotion_eligible"] is True
+assert result["checks"]["no_reachable_writable_primary"] is True
+
+print("valid_promotion_preflight accepted")
+
+
+runtime_state = fresh_state()
+runtime_state["postgresql"]["notification"]["promote_uname"] = (
+    "david-macbookpro"
+)
+
+result = plugin.promote_database(
+    config,
+    runtime_state,
+    {},
+)
+
+assert result["ocf_rc"] == 1
+assert result["ready"] is False
+assert result["reason"] == "local_node_not_target"
+
+print("wrong_target              rejected")
+
+
+runtime_state = fresh_state()
+runtime_state["postgresql"]["notification"]["received_at"] = (
+    time.time() - 121
+)
+
+result = plugin.promote_database(
+    config,
+    runtime_state,
+    {},
+)
+
+assert result["ocf_rc"] == 1
+assert result["reason"] == "notification_stale"
+
+print("stale_notification        rejected")
+
+
+result = plugin.promote_database(
+    config,
+    {},
+    {},
+)
+
+assert result["ocf_rc"] == 1
+assert result["reason"] == "notification_missing"
+
+print("missing_notification      rejected")
+
+
+runtime_state = fresh_state()
+
+plugin.monitor = lambda config, runtime_state: {
+    "ha": {
+        "ocf_rc": plugin.OCF_RUNNING_PROMOTED,
+        "promotion_eligible": True,
+    },
+    "role": "primary",
+    "in_recovery": False,
+}
+
+result = plugin.promote_database(
+    config,
+    runtime_state,
+    {},
+)
+
+assert result["ocf_rc"] == 1
+assert result["ready"] is False
+assert result["reason"] == "local_not_promotion_eligible"
+assert result["checks"]["local_standby"] is False
+
+print("local_primary             rejected")
+
+
+plugin.monitor = lambda config, runtime_state: standby_monitor(
+    eligible=False
+)
+
+result = plugin.promote_database(
+    config,
+    fresh_state(),
+    {},
+)
+
+assert result["ocf_rc"] == 1
+assert result["ready"] is False
+assert result["checks"]["promotion_eligible"] is False
+
+print("ineligible_standby        rejected")
+
+
+plugin.monitor = lambda config, runtime_state: standby_monitor()
+plugin.probe_replication_peers = (
+    lambda config, password: peer_state("david-prodesk")
+)
+
+result = plugin.promote_database(
+    config,
+    fresh_state(),
+    {},
+)
+
+assert result["ocf_rc"] == plugin.OCF_ERR_GENERIC
+assert result["ready"] is False
+assert result["mutation"] is False
+assert result["reason"] == "reachable_writable_primary"
+assert result["writable_primary_count"] == 1
+assert result["writable_primaries"] == ["david-prodesk"]
+assert result["checks"]["no_reachable_writable_primary"] is False
+
+print("existing_primary          rejected")
+
+
+print()
+print("PROMOTE_DATABASE_TEST_PASS=1")

tests/test-runtime-state.py

@@ -179,8 +179,8 @@ production_manifest = json.loads(
 )
 
 require(
-    production_manifest.get("version") == 3,
-    "PostgreSQL manifest version is not 3",
+    production_manifest.get("version") == 4,
+    "PostgreSQL manifest version is not 4",
 )
 require(
     "PG_REPLICATION_PEERS"