Freeze HA scoring and document reported production failover

2026-09-12 18:07:28 -04:00
parent c8636ed6d4
commit 488a1672d7
29 changed files with 2285 additions and 46 deletions
@@ -290,3 +290,76 @@ managed_stop()

    return "${OCF_SUCCESS}"
 }
+
+# Conservative scoring eligibility only; this does not replace role detection,
+# fencing, the promotion action, or the existing controlled-demote WAL checks.
+# This strict predicate remains unchanged; disconnected policy is separate below.
+mc_pg_promotion_eligible()
+{
+    local eligible
+
+    eligible="$(
+        as_service_user timeout --kill-after=1s 3s \
+            podman exec --user "${PG_OS_USER}" \
+            --env 'PGOPTIONS=-c statement_timeout=2000' \
+            "${CONTAINER_NAME}" \
+            psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc "
+                SELECT CASE WHEN pg_is_in_recovery() THEN
+                    NOT pg_is_wal_replay_paused()
+                    AND EXISTS (
+                        SELECT 1 FROM pg_stat_wal_receiver
+                        WHERE status = 'streaming'
+                          AND flushed_lsn IS NOT NULL
+                          AND pg_last_wal_replay_lsn() >= flushed_lsn
+                    )
+                ELSE false END;
+            " 2>/dev/null
+    )" || return 1
+    [ "${eligible}" = t ]
+}
+
+# One observation supplies both strict connected eligibility and disconnected
+# evidence. PostgreSQL restart time deliberately invalidates prior observations,
+# including a new basebackup sharing the same PostgreSQL system identifier.
+mc_pg_standby_observation()
+{
+    as_service_user timeout --kill-after=1s 3s \
+        podman exec --user "${PG_OS_USER}" \
+        --env 'PGOPTIONS=-c statement_timeout=2000' \
+        "${CONTAINER_NAME}" \
+        psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc "
+            WITH s AS MATERIALIZED (
+                SELECT pg_is_in_recovery() AS recovery,
+                       pg_last_wal_replay_lsn() AS replay,
+                       r.status AS receiver, r.flushed_lsn AS flush
+                FROM (SELECT 1) AS singleton
+                LEFT JOIN pg_stat_wal_receiver r ON true
+            )
+            SELECT json_build_object(
+                'identity', json_build_array(
+                    (SELECT system_identifier::text FROM pg_control_system()),
+                    current_setting('data_directory'),
+                    extract(epoch FROM pg_postmaster_start_time())::text),
+                'recovery', recovery,
+                'paused', CASE WHEN recovery THEN pg_is_wal_replay_paused() ELSE true END,
+                'receiver', receiver, 'replay', replay::text, 'flush', flush::text,
+                'strict', CASE WHEN recovery THEN
+                    NOT pg_is_wal_replay_paused() AND receiver = 'streaming'
+                    AND flush IS NOT NULL AND replay >= flush
+                    ELSE false END)
+            FROM s;
+        " 2>/dev/null
+}
+
+mc_pg_standby_score()
+{
+    local observation
+    observation="$(mc_pg_standby_observation)" || return 1
+    # The helper is root-owned alongside the modules; state never enters the
+    # rootless container. A failed/slow helper always withdraws eligibility.
+    printf '%s' "${observation}" | timeout --kill-after=1s 1s python3 \
+        "${MC_MODULE_DIR}/promotion_state.py" \
+        "${MC_PROMOTION_RESOURCE_INSTANCE:-}" \
+        "${SERVICE_USER}" "${SERVICE_NAME}" "${CONTAINER_NAME}" \
+        "${OCF_RESKEY_disconnected_max_age:-300}"
+}