Add read-only PostgreSQL demotion database probe

2026-09-17 18:15:39 -04:00
parent 10df464b00
commit 5fcd0bf931
3 changed files with 296 additions and 1 deletions
@@ -676,3 +676,158 @@ def demote_prepare(config, runtime_state, request_parameters):
            "database_checks": False,
        },
    }
+
+
+def _lsn_to_int(value):
+    if not value:
+        return None
+
+    upper, lower = value.split("/", 1)
+    return (int(upper, 16) << 32) + int(lower, 16)
+
+
+def demote_database(config, runtime_state, request_parameters):
+    """
+    Read-only PostgreSQL successor validation for demotion.
+    """
+    del request_parameters
+
+    preparation = demote_prepare(
+        config,
+        runtime_state,
+        {},
+    )
+
+    if preparation.get("reason") not in (
+        "database_checks_pending",
+    ):
+        return preparation
+
+    local = monitor(config, runtime_state)
+
+    if local["ha"]["ocf_rc"] != OCF_RUNNING_PROMOTED:
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "reason": "local_not_primary",
+            "local": local["ha"],
+        }
+
+    successor_node = preparation["successor_node"]
+    peers = parse_replication_peers(
+        config.get("PG_REPLICATION_PEERS", "")
+    )
+
+    successor = next(
+        (
+            peer for peer in peers
+            if peer["name"].strip().lower() == successor_node
+        ),
+        None,
+    )
+
+    if successor is None:
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "reason": "successor_not_configured",
+            "successor_node": successor_node,
+        }
+
+    try:
+        password = read_secret(config["PGPASSWORD_FILE"])
+
+        conn = psycopg2.connect(
+            host=successor["host"],
+            port=successor["port"],
+            user=config["PGUSER"],
+            dbname=config["PGDATABASE"],
+            password=password,
+            connect_timeout=3,
+            application_name="managed-containerd-demote-check",
+        )
+
+        try:
+            conn.autocommit = True
+
+            with conn.cursor(
+                cursor_factory=psycopg2.extras.RealDictCursor
+            ) as cur:
+                cur.execute("""
+                    SELECT
+                        pg_is_in_recovery() AS in_recovery,
+                        pg_is_wal_replay_paused() AS replay_paused,
+                        pg_last_wal_receive_lsn()::text AS receive_lsn,
+                        pg_last_wal_replay_lsn()::text AS replay_lsn,
+                        (
+                            SELECT status
+                            FROM pg_stat_wal_receiver
+                            LIMIT 1
+                        ) AS receiver_status,
+                        (
+                            SELECT slot_name
+                            FROM pg_stat_wal_receiver
+                            LIMIT 1
+                        ) AS slot_name
+                """)
+                successor_state = dict(cur.fetchone())
+        finally:
+            conn.close()
+
+    except psycopg2.Error as exc:
+        return {
+            "ocf_rc": 1,
+            "ready": False,
+            "reason": "successor_unreachable",
+            "successor_node": successor_node,
+            "successor_host": successor["host"],
+            "successor_port": successor["port"],
+            "error": str(exc).splitlines()[0],
+        }
+
+    primary_lsn = local["primary"]["current_wal_lsn"]
+    replay_lsn = successor_state["replay_lsn"]
+
+    primary_value = _lsn_to_int(primary_lsn)
+    replay_value = _lsn_to_int(replay_lsn)
+
+    wal_lag_bytes = None
+    if primary_value is not None and replay_value is not None:
+        wal_lag_bytes = max(0, primary_value - replay_value)
+
+    checks = {
+        "local_primary": True,
+        "successor_reachable": True,
+        "successor_in_recovery": (
+            successor_state["in_recovery"] is True
+        ),
+        "successor_streaming": (
+            successor_state["receiver_status"] == "streaming"
+        ),
+        "successor_replay_not_paused": (
+            successor_state["replay_paused"] is False
+        ),
+        "successor_slot_present": bool(
+            successor_state["slot_name"]
+        ),
+    }
+
+    ready = all(checks.values())
+
+    return {
+        "ocf_rc": 0 if ready else 1,
+        "ready": ready,
+        "mutation": False,
+        "final_handoff_required": True,
+        "node": preparation["node"],
+        "target_node": preparation["target_node"],
+        "successor_node": successor_node,
+        "successor_host": successor["host"],
+        "successor_port": successor["port"],
+        "primary_wal_lsn": primary_lsn,
+        "successor_receive_lsn": successor_state["receive_lsn"],
+        "successor_replay_lsn": replay_lsn,
+        "wal_lag_bytes": wal_lag_bytes,
+        "successor_slot_name": successor_state["slot_name"],
+        "checks": checks,
+    }