Accept validated PostgreSQL self-restart demotion

checksums/SHA256SUMS

@@ -2,12 +2,12 @@
 5aa02fa065d1dba55486838fa4f0cd8eba982edaaf73cdc7d818223a09cbadc6  ocf/loxthelion/managed-container
 10932919c4339b553c7c9dbf30b45a917c51c0a08f362e12678003d58b12fa82  ocf/loxthelion/managed-container.d/applications/postgresql/auth.sh
 61d64b4a3a15c354f101ef34c5dc0425723f4204207d359b5377d2b9f04f6b23  ocf/loxthelion/managed-container.d/applications/postgresql/demote.sh
-e111dfe454a6e1edd80edc0c303119e045e7c88fe77b69a2745e58a022d0d317  ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh
+86bb5bacd27e4082a7f753108c1f5d5e777ff04fba8699d62ed148c17224abdf  ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh
 bcfde96b98f4727a8c97d42a39b7c6ec925b1b89d17ef67cc58eb7bd67afbfae  ocf/loxthelion/managed-container.d/applications/postgresql/notify.sh
 957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548  ocf/loxthelion/managed-container.d/applications/postgresql/peer_config.sh
-99bb3478931eced9471a8f054c14d152780aec73801449d1a6b2f09a4b656bec  ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh
+286d282565239bdb600563bac116575f3f56580e06f8f8df86c1a0fd0af96f07  ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh
 c1d878dabf22ad3928d66a5542bda3c150e5e9b039926629e7c5fd3401d29be4  ocf/loxthelion/managed-container.d/applications/postgresql/postgres.sh
-f482853a0c7992981e59b4ad78d1ee69851d5d60fdec1f695673228017dd8244  ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh
+d8534993c8e93f630bc0987301f5837f66e93c90d02b1063f46fdcdb559c0391  ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh
 5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7  ocf/loxthelion/managed-container.d/applications/postgresql/promotion_state.py
 b0ebad749ed736f08ec748a693f12e8dc2541430e0d3db75bbb6b2b7e517f84b  ocf/loxthelion/managed-container.d/applications/postgresql/rejoin.sh
 0d3017fdca92b2dc5dba406065bc269622ce5140e3fce4a959ca95ce4a78e755  ocf/loxthelion/managed-container.d/applications/postgresql/replication.sh
@@ -23,13 +23,17 @@ a42bb59a058ffeb59c1baa96e403856910bbbaea98f9104ff93daf4b5970c780  postgres-host/
 957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548  postgres-host/postgres-ha-peer-config.sh
 deb09e3ae40629f43b262ba4f9a7ebf59a1dca3da9d8c94478f670aea28bd0e1  quadlet/postgres.container
 04a125068677bbee6e71d313ca9d95d35faa32851b79de781cd7b9f466561445  quadlet/postgres.env.example
-a4bedb1c423294e4cafe4c93e9daff3d2b774a8f8e4a6b989322acbc2b842a04  tests/run-all.sh
+ad248df2911e87fbe456c095f4e30dd54b2da52c9e2efa0619d3a2bf16a0f30c  tests/run-all.sh
 a373c0560391a2d8f63c3ba31609ab16b219d9b532b4ab9f8aea6dc45de10555  tests/test-demote-database-gate.sh
 688eeb756c7f346e7e55acc555004ba12b48bc7c3d309073658d5e84a6e5e348  tests/test-demote-database-parameter.sh
+a3b3b2dad6bfdab8ffc7b3cabc6e1b9f188ff743fcadf073e5e8bdcf1827b2b9  tests/test-demote-database-response-matrix.sh
 22d3eb56e8cd3ae1a58e2616f44abd3818f79d7e5df55c1c158e36dca7c77f42  tests/test-demote-database-shadow.sh
 5e5d4fdd2b6c7e0b8b146de7aad06c1f96b221a10fec525b00d40e0d85709fb0  tests/test-entrypoint-structure.sh
 2faee72cf2fdcbfa667c3e1c2a7aabc053b96a9b8127ff7440ddb717fd2dc314  tests/test-notify-daemon-shadow.sh
 9150004af86a819d7d1b5ae2b3fa9d4ddd0480f55babd094137052109864b057  tests/test-notify-target-shadow.sh
+ea2dc89483c5addc59cee0b8f070a08894c4803a204263507e2eaffb5416718e  tests/test-promote-database-gate.sh
+fc292a8612911423fdc8b9c5d187e2a19dd240d1bf35340d0b307888543beaa0  tests/test-promote-database-parameter.sh
+7ea6ec5aaf2364f140f2987b4c2cf65ce5a132e284b6be22968de0cc4b63b3d5  tests/test-promote-database-shadow.sh
 b83f0415c09f0be34c51722370e5b07c51b9a9756d305d3b3f50827f484aa4dd  tests/test-proxy-client-actions.py
 cc4482d229805c64d7df258a8a82984f244128d27fff4ea43b36caa004b56bef  tests/test-proxy-client-parameters.py
 c3b988d7b2b1b138bb2530df9172f96bb0f5edc2f1561f38f0055695f4126134  tests/test-proxy-protocol.sh

ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh

@@ -71,12 +71,23 @@ import os
 
 response = json.loads(os.environ["MC_DAEMON_RESPONSE"])
 
-if (
-    response.get("ocf_rc") != 0
-    or response.get("ready") is not True
-    or response.get("mutation") is not False
-    or response.get("final_handoff_required") is not True
-):
+common_valid = (
+    response.get("ocf_rc") == 0
+    and response.get("ready") is True
+    and response.get("mutation") is False
+)
+
+peer_handoff = (
+    response.get("final_handoff_required") is True
+)
+
+local_restart = (
+    response.get("final_handoff_required") is False
+    and response.get("handoff_mode") == "local_restart"
+    and response.get("reason") == "local_restart_demotion_ready"
+)
+
+if not common_valid or not (peer_handoff or local_restart):
     raise SystemExit(1)
 '; then
         ocf_log info \

ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh

@@ -14,6 +14,7 @@ OCF_RESKEY_application_name_default=""
 OCF_RESKEY_rejoin_allow_basebackup_default="false"
 OCF_RESKEY_rejoin_basebackup_headroom_percent_default="50"
 OCF_RESKEY_demote_database_gate_default="false"
+OCF_RESKEY_promote_database_gate_default="false"
 
 : "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}"
 : "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}"
@@ -29,6 +30,7 @@ OCF_RESKEY_demote_database_gate_default="false"
 : "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}"
 : "${OCF_RESKEY_rejoin_basebackup_headroom_percent:=${OCF_RESKEY_rejoin_basebackup_headroom_percent_default}}"
 : "${OCF_RESKEY_demote_database_gate:=${OCF_RESKEY_demote_database_gate_default}}"
+: "${OCF_RESKEY_promote_database_gate:=${OCF_RESKEY_promote_database_gate_default}}"
 
 SERVICE_USER="${OCF_RESKEY_service_user}"
 SERVICE_NAME="${OCF_RESKEY_service_name}"
@@ -44,7 +46,9 @@ APPLICATION_NAME="${OCF_RESKEY_application_name}"
 REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}"
 REJOIN_BASEBACKUP_HEADROOM_PERCENT="${OCF_RESKEY_rejoin_basebackup_headroom_percent}"
 DEMOTE_DATABASE_GATE="${OCF_RESKEY_demote_database_gate}"
+PROMOTE_DATABASE_GATE="${OCF_RESKEY_promote_database_gate}"
 export MC_DEMOTE_DATABASE_GATE="${DEMOTE_DATABASE_GATE}"
+export MC_PROMOTE_DATABASE_GATE="${PROMOTE_DATABASE_GATE}"
 
 # Resolve PostgreSQL configuration from the managed service when Pacemaker has
 # not explicitly supplied it. The application configuration remains the source
@@ -303,6 +307,15 @@ validate_all()
             ;;
     esac
 
+    case "${PROMOTE_DATABASE_GATE}" in
+        true|false)
+            ;;
+        *)
+            ocf_exit_reason "promote_database_gate must be true or false"
+            return "${OCF_ERR_CONFIGURED}"
+            ;;
+    esac
+
     case "${REJOIN_BASEBACKUP_HEADROOM_PERCENT}" in
         ""|*[!0-9]*)
             ocf_exit_reason "rejoin_basebackup_headroom_percent must be a non-negative integer"
@@ -352,6 +365,11 @@ mc_promotion_metadata()
       <shortdesc lang="en">Gate demotion on database checks</shortdesc>\
       <content type="boolean" default="false"/>\
     </parameter>\
+    <parameter name="promote_database_gate" unique="0" required="0">\
+      <longdesc lang="en">When true, reject native PostgreSQL promotion unless the managed-containerd promote.database preflight agrees that this node is the notified promotion target, is an eligible standby, and no reachable writable peer exists. When false, the daemon check remains shadow-only and native promotion fails open.</longdesc>\
+      <shortdesc lang="en">Gate promotion on database checks</shortdesc>\
+      <content type="boolean" default="false"/>\
+    </parameter>\
     <parameter name="replication_peers" unique="0" required="0">\
       <longdesc lang="en">Opt-in stable peer map: node=host:port,node=host:port. Include the local node; exclude the client VIP. IPv4 or DNS hosts only. Requires a configured per-node replication slot, supplied explicitly or resolved from PostgreSQL/the managed container environment, plus peer authentication and slots on every upstream. Uses native primary-only selection; demotion verifies final WAL then ready recovery before successor promotion.</longdesc>\
       <shortdesc lang="en">Native replication peer map</shortdesc>\

ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh

@@ -1,5 +1,70 @@
 # Sourced by managed-container; uses the shared OCF and parameter environment.
 
+mc_pg_promote_database_shadow()
+{
+    local socket_path resource response rc
+
+    socket_path="${MC_PROXY_SOCKET:-/run/user/1000/managed-containerd/managed-containerd.sock}"
+    resource="${OCF_RESOURCE_INSTANCE:-}"
+    resource="${resource%%:*}"
+
+    # Standalone/resource-agent tests may not have a real resource identity.
+    [ -n "${resource}" ] || return 0
+
+    if response="$(
+        timeout --kill-after=1s 5s \
+            python3 \
+            "${MC_PROXY_CLIENT:-/usr/lib/ocf/resource.d/loxthelion/managed-container.d/proxy_client.py}" \
+            "${socket_path}" \
+            "${resource}" \
+            "promote.database" \
+            10 \
+            11
+    )"; then
+        rc=0
+    else
+        rc=$?
+    fi
+
+    if [ "${rc}" -ne 0 ]; then
+        ocf_log warn \
+            "managed-containerd promote.database shadow unavailable rc=${rc}"
+
+        if [ "${MC_PROMOTE_DATABASE_GATE:-false}" = true ]; then
+            return 1
+        fi
+
+        return 0
+    fi
+
+    if MC_DAEMON_RESPONSE="${response}" python3 -c '
+import json
+import os
+
+response = json.loads(os.environ["MC_DAEMON_RESPONSE"])
+
+if (
+    response.get("ocf_rc") != 0
+    or response.get("ready") is not True
+    or response.get("mutation") is not False
+):
+    raise SystemExit(1)
+'; then
+        ocf_log info \
+            "managed-containerd promote.database shadow: comparison=MATCH response=${response}"
+    else
+        ocf_log err \
+            "managed-containerd promote.database shadow MISMATCH response=${response}"
+
+        if [ "${MC_PROMOTE_DATABASE_GATE:-false}" = true ]; then
+            return 1
+        fi
+    fi
+
+    return 0
+}
+
+
 managed_promote()
 {
     local monitor_rc role i
@@ -58,6 +123,15 @@ managed_promote()
     ocf_log info \
         "Promoting PostgreSQL standby inside ${CONTAINER_NAME}"
 
+    # Independent managed-containerd promotion preflight.
+    # Shadow-only by default; when promote_database_gate=true,
+    # daemon disagreement or transport failure denies native promotion.
+    if ! mc_pg_promote_database_shadow; then
+        ocf_exit_reason \
+            "Promotion database gate rejected native PostgreSQL promotion"
+        return "${OCF_ERR_GENERIC}"
+    fi
+
     if ! pg_command '
         BINDIR="$(pg_config --bindir)" || exit 1
         exec "$BINDIR/pg_ctl" \

tests/run-all.sh

@@ -27,8 +27,12 @@ python3 ./tests/test-proxy-client-parameters.py
 ./tests/test-notify-daemon-shadow.sh
 ./tests/test-notify-target-shadow.sh
 ./tests/test-demote-database-shadow.sh
+./tests/test-demote-database-response-matrix.sh
+./tests/test-promote-database-shadow.sh
 ./tests/test-demote-database-gate.sh
+./tests/test-promote-database-gate.sh
 ./tests/test-demote-database-parameter.sh
+./tests/test-promote-database-parameter.sh
 
 echo
 echo ">>> Python syntax"

tests/test-demote-database-response-matrix.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh"
+FAKE="$(mktemp)"
+
+cleanup()
+{
+    rm -f "$FAKE"
+}
+
+trap cleanup EXIT
+
+ocf_log()
+{
+    :
+}
+
+export OCF_RESOURCE_INSTANCE=postgresql-ha
+export MC_PROXY_CLIENT="$FAKE"
+export MC_DEMOTE_DATABASE_GATE=true
+
+# shellcheck disable=SC1090
+. "$FILE"
+
+write_response()
+{
+    cat > "$FAKE"
+    chmod 0755 "$FAKE"
+}
+
+echo "===== REMOTE HANDOFF ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": True,
+    "reason": "demotion_preflight_ready",
+}))
+PY
+
+if ! mc_pg_demote_database_shadow; then
+    echo "FAIL: valid remote handoff response was rejected"
+    exit 1
+fi
+
+echo "remote_handoff            accepted"
+
+echo
+echo "===== LOCAL RESTART ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": False,
+    "handoff_mode": "local_restart",
+    "reason": "local_restart_demotion_ready",
+}))
+PY
+
+if ! mc_pg_demote_database_shadow; then
+    echo "FAIL: valid local restart response was rejected"
+    exit 1
+fi
+
+echo "local_restart             accepted"
+
+echo
+echo "===== ARBITRARY FALSE HANDOFF ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": False,
+}))
+PY
+
+if mc_pg_demote_database_shadow; then
+    echo "FAIL: arbitrary final_handoff_required=false was accepted"
+    exit 1
+fi
+
+echo "arbitrary_false           rejected"
+
+echo
+echo "===== FAKE LOCAL RESTART ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": False,
+    "handoff_mode": "local_restart",
+    "reason": "wrong_reason",
+}))
+PY
+
+if mc_pg_demote_database_shadow; then
+    echo "FAIL: fake local restart response was accepted"
+    exit 1
+fi
+
+echo "fake_local_restart        rejected"
+
+echo
+echo "DEMOTE_DATABASE_RESPONSE_MATRIX_TEST_PASS=1"

tests/test-promote-database-gate.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh"
+FAKE="$(mktemp)"
+
+cleanup()
+{
+    rm -f "$FAKE"
+}
+
+trap cleanup EXIT
+
+ocf_log()
+{
+    :
+}
+
+export OCF_RESOURCE_INSTANCE=postgresql-ha
+export MC_PROXY_CLIENT="$FAKE"
+
+# Valid disagreement.
+cat > "$FAKE" <<'EOF'
+import json
+print(json.dumps({
+    "ocf_rc": 1,
+    "ready": False,
+    "mutation": False,
+}))
+EOF
+
+chmod 0755 "$FAKE"
+
+# shellcheck disable=SC1090
+. "$FILE"
+
+unset MC_PROMOTE_DATABASE_GATE
+mc_pg_promote_database_shadow
+
+export MC_PROMOTE_DATABASE_GATE=true
+
+if mc_pg_promote_database_shadow; then
+    echo "FAIL: promotion mismatch was not gated"
+    exit 1
+fi
+
+# Transport failure.
+cat > "$FAKE" <<'EOF'
+raise SystemExit(10)
+EOF
+
+chmod 0755 "$FAKE"
+
+unset MC_PROMOTE_DATABASE_GATE
+mc_pg_promote_database_shadow
+
+export MC_PROMOTE_DATABASE_GATE=true
+
+if mc_pg_promote_database_shadow; then
+    echo "FAIL: promotion transport failure was not gated"
+    exit 1
+fi
+
+# Positive daemon agreement must still pass with the gate enabled.
+cat > "$FAKE" <<'EOF'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+}))
+EOF
+
+chmod 0755 "$FAKE"
+
+export MC_PROMOTE_DATABASE_GATE=true
+
+mc_pg_promote_database_shadow
+
+echo "PROMOTE_DATABASE_GATE_RUNTIME_TEST_PASS=1"

tests/test-promote-database-parameter.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh"
+
+grep -q \
+  'OCF_RESKEY_promote_database_gate_default="false"' \
+  "$FILE"
+
+grep -q \
+  ': "${OCF_RESKEY_promote_database_gate:=' \
+  "$FILE"
+
+grep -q \
+  'PROMOTE_DATABASE_GATE="${OCF_RESKEY_promote_database_gate}"' \
+  "$FILE"
+
+grep -q \
+  'export MC_PROMOTE_DATABASE_GATE="${PROMOTE_DATABASE_GATE}"' \
+  "$FILE"
+
+grep -q \
+  'promote_database_gate must be true or false' \
+  "$FILE"
+
+grep -q \
+  '<parameter name="promote_database_gate"' \
+  "$FILE"
+
+grep -q \
+  '<content type="boolean" default="false"/>' \
+  "$FILE"
+
+echo "PROMOTE_DATABASE_PARAMETER_TEST_PASS=1"

tests/test-promote-database-shadow.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh"
+
+grep -q '^mc_pg_promote_database_shadow()' "$FILE"
+grep -q '"promote.database"' "$FILE"
+grep -q 'response.get("ready") is not True' "$FILE"
+grep -q 'response.get("mutation") is not False' "$FILE"
+grep -q 'comparison=MATCH' "$FILE"
+grep -q 'promote.database shadow MISMATCH' "$FILE"
+
+SHADOW_LINE="$(
+    grep -n 'if ! mc_pg_promote_database_shadow; then' "$FILE" |
+    head -1 |
+    cut -d: -f1
+)"
+
+PROMOTE_LINE="$(
+    grep -n 'exec "\$BINDIR/pg_ctl"' "$FILE" |
+    head -1 |
+    cut -d: -f1
+)"
+
+[ -n "$SHADOW_LINE" ]
+[ -n "$PROMOTE_LINE" ]
+[ "$SHADOW_LINE" -lt "$PROMOTE_LINE" ]
+
+echo "PROMOTE_DATABASE_SHADOW_TEST_PASS=1"