Remove legacy monitor shadow execution

Add daemon-selected container monitor delegation
Accept validated PostgreSQL self-restart demotion
2026-09-18 15:13:13 -04:00 · 2026-09-18 14:09:46 -04:00 · 2026-09-18 09:43:41 -04:00 · 2026-09-18 02:46:40 -04:00 · 2026-09-17 22:54:26 -04:00
15 changed files with 803 additions and 27 deletions
@@ -2,12 +2,12 @@
 5aa02fa065d1dba55486838fa4f0cd8eba982edaaf73cdc7d818223a09cbadc6  ocf/loxthelion/managed-container
 10932919c4339b553c7c9dbf30b45a917c51c0a08f362e12678003d58b12fa82  ocf/loxthelion/managed-container.d/applications/postgresql/auth.sh
 61d64b4a3a15c354f101ef34c5dc0425723f4204207d359b5377d2b9f04f6b23  ocf/loxthelion/managed-container.d/applications/postgresql/demote.sh
-e111dfe454a6e1edd80edc0c303119e045e7c88fe77b69a2745e58a022d0d317  ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh
+86bb5bacd27e4082a7f753108c1f5d5e777ff04fba8699d62ed148c17224abdf  ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh
 bcfde96b98f4727a8c97d42a39b7c6ec925b1b89d17ef67cc58eb7bd67afbfae  ocf/loxthelion/managed-container.d/applications/postgresql/notify.sh
 957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548  ocf/loxthelion/managed-container.d/applications/postgresql/peer_config.sh
-99bb3478931eced9471a8f054c14d152780aec73801449d1a6b2f09a4b656bec  ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh
+286d282565239bdb600563bac116575f3f56580e06f8f8df86c1a0fd0af96f07  ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh
 c1d878dabf22ad3928d66a5542bda3c150e5e9b039926629e7c5fd3401d29be4  ocf/loxthelion/managed-container.d/applications/postgresql/postgres.sh
-f482853a0c7992981e59b4ad78d1ee69851d5d60fdec1f695673228017dd8244  ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh
+d8534993c8e93f630bc0987301f5837f66e93c90d02b1063f46fdcdb559c0391  ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh
 5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7  ocf/loxthelion/managed-container.d/applications/postgresql/promotion_state.py
 b0ebad749ed736f08ec748a693f12e8dc2541430e0d3db75bbb6b2b7e517f84b  ocf/loxthelion/managed-container.d/applications/postgresql/rejoin.sh
 0d3017fdca92b2dc5dba406065bc269622ce5140e3fce4a959ca95ce4a78e755  ocf/loxthelion/managed-container.d/applications/postgresql/replication.sh
@@ -23,13 +23,17 @@ a42bb59a058ffeb59c1baa96e403856910bbbaea98f9104ff93daf4b5970c780  postgres-host/
 957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548  postgres-host/postgres-ha-peer-config.sh
 deb09e3ae40629f43b262ba4f9a7ebf59a1dca3da9d8c94478f670aea28bd0e1  quadlet/postgres.container
 04a125068677bbee6e71d313ca9d95d35faa32851b79de781cd7b9f466561445  quadlet/postgres.env.example
-a4bedb1c423294e4cafe4c93e9daff3d2b774a8f8e4a6b989322acbc2b842a04  tests/run-all.sh
+ad248df2911e87fbe456c095f4e30dd54b2da52c9e2efa0619d3a2bf16a0f30c  tests/run-all.sh
 a373c0560391a2d8f63c3ba31609ab16b219d9b532b4ab9f8aea6dc45de10555  tests/test-demote-database-gate.sh
 688eeb756c7f346e7e55acc555004ba12b48bc7c3d309073658d5e84a6e5e348  tests/test-demote-database-parameter.sh
+a3b3b2dad6bfdab8ffc7b3cabc6e1b9f188ff743fcadf073e5e8bdcf1827b2b9  tests/test-demote-database-response-matrix.sh
 22d3eb56e8cd3ae1a58e2616f44abd3818f79d7e5df55c1c158e36dca7c77f42  tests/test-demote-database-shadow.sh
 5e5d4fdd2b6c7e0b8b146de7aad06c1f96b221a10fec525b00d40e0d85709fb0  tests/test-entrypoint-structure.sh
 2faee72cf2fdcbfa667c3e1c2a7aabc053b96a9b8127ff7440ddb717fd2dc314  tests/test-notify-daemon-shadow.sh
 9150004af86a819d7d1b5ae2b3fa9d4ddd0480f55babd094137052109864b057  tests/test-notify-target-shadow.sh
+ea2dc89483c5addc59cee0b8f070a08894c4803a204263507e2eaffb5416718e  tests/test-promote-database-gate.sh
+fc292a8612911423fdc8b9c5d187e2a19dd240d1bf35340d0b307888543beaa0  tests/test-promote-database-parameter.sh
+7ea6ec5aaf2364f140f2987b4c2cf65ce5a132e284b6be22968de0cc4b63b3d5  tests/test-promote-database-shadow.sh
 b83f0415c09f0be34c51722370e5b07c51b9a9756d305d3b3f50827f484aa4dd  tests/test-proxy-client-actions.py
 cc4482d229805c64d7df258a8a82984f244128d27fff4ea43b36caa004b56bef  tests/test-proxy-client-parameters.py
 c3b988d7b2b1b138bb2530df9172f96bb0f5edc2f1561f38f0055695f4126134  tests/test-proxy-protocol.sh
@@ -71,12 +71,23 @@ import os

 response = json.loads(os.environ["MC_DAEMON_RESPONSE"])

-if (
-    response.get("ocf_rc") != 0
-    or response.get("ready") is not True
-    or response.get("mutation") is not False
-    or response.get("final_handoff_required") is not True
-):
+common_valid = (
+    response.get("ocf_rc") == 0
+    and response.get("ready") is True
+    and response.get("mutation") is False
+)
+
+peer_handoff = (
+    response.get("final_handoff_required") is True
+)
+
+local_restart = (
+    response.get("final_handoff_required") is False
+    and response.get("handoff_mode") == "local_restart"
+    and response.get("reason") == "local_restart_demotion_ready"
+)
+
+if not common_valid or not (peer_handoff or local_restart):
    raise SystemExit(1)
 '; then
        ocf_log info \
@@ -14,6 +14,7 @@ OCF_RESKEY_application_name_default=""
 OCF_RESKEY_rejoin_allow_basebackup_default="false"
 OCF_RESKEY_rejoin_basebackup_headroom_percent_default="50"
 OCF_RESKEY_demote_database_gate_default="false"
+OCF_RESKEY_promote_database_gate_default="false"

 : "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}"
 : "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}"
@@ -29,6 +30,7 @@ OCF_RESKEY_demote_database_gate_default="false"
 : "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}"
 : "${OCF_RESKEY_rejoin_basebackup_headroom_percent:=${OCF_RESKEY_rejoin_basebackup_headroom_percent_default}}"
 : "${OCF_RESKEY_demote_database_gate:=${OCF_RESKEY_demote_database_gate_default}}"
+: "${OCF_RESKEY_promote_database_gate:=${OCF_RESKEY_promote_database_gate_default}}"

 SERVICE_USER="${OCF_RESKEY_service_user}"
 SERVICE_NAME="${OCF_RESKEY_service_name}"
@@ -44,7 +46,9 @@ APPLICATION_NAME="${OCF_RESKEY_application_name}"
 REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}"
 REJOIN_BASEBACKUP_HEADROOM_PERCENT="${OCF_RESKEY_rejoin_basebackup_headroom_percent}"
 DEMOTE_DATABASE_GATE="${OCF_RESKEY_demote_database_gate}"
+PROMOTE_DATABASE_GATE="${OCF_RESKEY_promote_database_gate}"
 export MC_DEMOTE_DATABASE_GATE="${DEMOTE_DATABASE_GATE}"
+export MC_PROMOTE_DATABASE_GATE="${PROMOTE_DATABASE_GATE}"

 # Resolve PostgreSQL configuration from the managed service when Pacemaker has
 # not explicitly supplied it. The application configuration remains the source
@@ -303,6 +307,15 @@ validate_all()
            ;;
    esac

+    case "${PROMOTE_DATABASE_GATE}" in
+        true|false)
+            ;;
+        *)
+            ocf_exit_reason "promote_database_gate must be true or false"
+            return "${OCF_ERR_CONFIGURED}"
+            ;;
+    esac
+
    case "${REJOIN_BASEBACKUP_HEADROOM_PERCENT}" in
        ""|*[!0-9]*)
            ocf_exit_reason "rejoin_basebackup_headroom_percent must be a non-negative integer"
@@ -352,6 +365,11 @@ mc_promotion_metadata()
      <shortdesc lang="en">Gate demotion on database checks</shortdesc>\
      <content type="boolean" default="false"/>\
    </parameter>\
+    <parameter name="promote_database_gate" unique="0" required="0">\
+      <longdesc lang="en">When true, reject native PostgreSQL promotion unless the managed-containerd promote.database preflight agrees that this node is the notified promotion target, is an eligible standby, and no reachable writable peer exists. When false, the daemon check remains shadow-only and native promotion fails open.</longdesc>\
+      <shortdesc lang="en">Gate promotion on database checks</shortdesc>\
+      <content type="boolean" default="false"/>\
+    </parameter>\
    <parameter name="replication_peers" unique="0" required="0">\
      <longdesc lang="en">Opt-in stable peer map: node=host:port,node=host:port. Include the local node; exclude the client VIP. IPv4 or DNS hosts only. Requires a configured per-node replication slot, supplied explicitly or resolved from PostgreSQL/the managed container environment, plus peer authentication and slots on every upstream. Uses native primary-only selection; demotion verifies final WAL then ready recovery before successor promotion.</longdesc>\
      <shortdesc lang="en">Native replication peer map</shortdesc>\
@@ -1,5 +1,70 @@
 # Sourced by managed-container; uses the shared OCF and parameter environment.

+mc_pg_promote_database_shadow()
+{
+    local socket_path resource response rc
+
+    socket_path="${MC_PROXY_SOCKET:-/run/user/1000/managed-containerd/managed-containerd.sock}"
+    resource="${OCF_RESOURCE_INSTANCE:-}"
+    resource="${resource%%:*}"
+
+    # Standalone/resource-agent tests may not have a real resource identity.
+    [ -n "${resource}" ] || return 0
+
+    if response="$(
+        timeout --kill-after=1s 5s \
+            python3 \
+            "${MC_PROXY_CLIENT:-/usr/lib/ocf/resource.d/loxthelion/managed-container.d/proxy_client.py}" \
+            "${socket_path}" \
+            "${resource}" \
+            "promote.database" \
+            10 \
+            11
+    )"; then
+        rc=0
+    else
+        rc=$?
+    fi
+
+    if [ "${rc}" -ne 0 ]; then
+        ocf_log warn \
+            "managed-containerd promote.database shadow unavailable rc=${rc}"
+
+        if [ "${MC_PROMOTE_DATABASE_GATE:-false}" = true ]; then
+            return 1
+        fi
+
+        return 0
+    fi
+
+    if MC_DAEMON_RESPONSE="${response}" python3 -c '
+import json
+import os
+
+response = json.loads(os.environ["MC_DAEMON_RESPONSE"])
+
+if (
+    response.get("ocf_rc") != 0
+    or response.get("ready") is not True
+    or response.get("mutation") is not False
+):
+    raise SystemExit(1)
+'; then
+        ocf_log info \
+            "managed-containerd promote.database shadow: comparison=MATCH response=${response}"
+    else
+        ocf_log err \
+            "managed-containerd promote.database shadow MISMATCH response=${response}"
+
+        if [ "${MC_PROMOTE_DATABASE_GATE:-false}" = true ]; then
+            return 1
+        fi
+    fi
+
+    return 0
+}
+
+
 managed_promote()
 {
    local monitor_rc role i
@@ -58,6 +123,15 @@ managed_promote()
    ocf_log info \
        "Promoting PostgreSQL standby inside ${CONTAINER_NAME}"

+    # Independent managed-containerd promotion preflight.
+    # Shadow-only by default; when promote_database_gate=true,
+    # daemon disagreement or transport failure denies native promotion.
+    if ! mc_pg_promote_database_shadow; then
+        ocf_exit_reason \
+            "Promotion database gate rejected native PostgreSQL promotion"
+        return "${OCF_ERR_GENERIC}"
+    fi
+
    if ! pg_command '
        BINDIR="$(pg_config --bindir)" || exit 1
        exec "$BINDIR/pg_ctl" \
@@ -54,12 +54,12 @@ mc_promotion_score_from_monitor()
 }

 # managed-containerd is authoritative for recurring monitor/status results.
-# The legacy application monitor remains temporarily available as a shadow
-# comparison only. Promotion scoring is still supplied by the existing
-# application scoring hook during this migration stage.
+# The packaged application monitor is used only when managed-containerd
+# explicitly delegates monitoring to the container backend. Promotion scoring
+# remains supplied by the existing application scoring hook.
 mc_promotion_monitor()
 {
-    local monitor_rc legacy_rc proxy_result proxy_rc cached_rc
+    local monitor_rc proxy_result proxy_rc cached_rc

    proxy_result="$(mc_proxy_monitor)"
    proxy_rc=$?
@@ -91,13 +91,6 @@ mc_promotion_monitor()
                    ocf_log warn \
                        "managed-containerd unavailable; using recent cached authoritative rc=${monitor_rc} for bounded control-plane grace"

-                    mc_application_monitor
-                    legacy_rc=$?
-
-                    mc_proxy_legacy_shadow_monitor \
-                        "${monitor_rc}" \
-                        "${legacy_rc}" || true
-
                    return "${monitor_rc}"
                fi

@@ -119,6 +112,70 @@ mc_promotion_monitor()
        esac
    fi

+    # A successful managed-containerd response may explicitly delegate
+    # monitoring to the RA's already-loaded packaged container/runtime path.
+    #
+    # The daemon remains authoritative for backend selection: this path is
+    # reachable only after managed-containerd successfully requested it.
+    case "${proxy_result}" in
+        delegate\|*)
+            local delegate_tag delegate_backend delegate_runtime delegate_extra
+            local configured_runtime
+
+            IFS='|' read -r                 delegate_tag                 delegate_backend                 delegate_runtime                 delegate_extra <<EOF
+${proxy_result}
+EOF
+
+            if [ "${delegate_tag}" != delegate ] ||
+               [ "${delegate_backend}" != container ] ||
+               [ -z "${delegate_runtime}" ] ||
+               [ -n "${delegate_extra}" ]; then
+
+                mc_proxy_monitor_cache_clear || true
+                mc_promotion_score_set -INFINITY || true
+                ocf_exit_reason                     "managed-containerd returned invalid monitor delegation: ${proxy_result}"
+                return "${OCF_ERR_GENERIC}"
+            fi
+
+            configured_runtime="${OCF_RESKEY_container_runtime:-podman}"
+
+            if [ "${delegate_runtime}" != "${configured_runtime}" ]; then
+                mc_proxy_monitor_cache_clear || true
+                mc_promotion_score_set -INFINITY || true
+                ocf_exit_reason                     "managed-containerd delegated runtime ${delegate_runtime} does not match loaded RA runtime ${configured_runtime}"
+                return "${OCF_ERR_GENERIC}"
+            fi
+
+            ocf_log info                 "managed-containerd delegated authoritative monitor: resource=${OCF_RESOURCE_INSTANCE:-unknown} backend=${delegate_backend} runtime=${delegate_runtime}"
+
+            mc_application_monitor
+            monitor_rc=$?
+
+            case "${monitor_rc}" in
+                "${OCF_SUCCESS}"|                "${OCF_ERR_GENERIC}"|                "${OCF_NOT_RUNNING}"|                "${OCF_RUNNING_PROMOTED}")
+                    ;;
+                *)
+                    mc_proxy_monitor_cache_clear || true
+                    mc_promotion_score_set -INFINITY || true
+                    ocf_exit_reason                         "delegated application monitor returned invalid rc ${monitor_rc}"
+                    return "${OCF_ERR_GENERIC}"
+                    ;;
+            esac
+
+            # The delegated application result is now the authoritative
+            # observation. Cache only healthy 0/8 results using the same
+            # control-plane grace mechanism as daemon-native monitoring.
+            mc_proxy_monitor_cache_write "${monitor_rc}" || {
+                ocf_log warn                     "Unable to update delegated authoritative monitor cache"
+            }
+
+            ocf_log info                 "managed-containerd delegated monitor result: resource=${OCF_RESOURCE_INSTANCE:-unknown} rc=${monitor_rc} backend=${delegate_backend} runtime=${delegate_runtime}"
+
+            mc_promotion_score_from_monitor "${monitor_rc}" || true
+            return "${monitor_rc}"
+            ;;
+    esac
+
    monitor_rc="${proxy_result%%|*}"

    case "${monitor_rc}" in
@@ -136,12 +193,9 @@ mc_promotion_monitor()

    ocf_log info         "managed-containerd authoritative monitor: resource=${OCF_RESOURCE_INSTANCE:-unknown} proxy=${proxy_result}"

-    # Legacy PostgreSQL monitoring remains observational only.
-    mc_application_monitor
-    legacy_rc=$?
-
-    mc_proxy_legacy_shadow_monitor         "${monitor_rc}"         "${legacy_rc}" || true
-
+    # The daemon-native monitor is authoritative. The packaged application
+    # monitor is invoked only when managed-containerd explicitly delegates to
+    # the container backend.
    mc_promotion_score_from_monitor "${monitor_rc}" || true
    return "${monitor_rc}"
 }
@@ -88,6 +88,16 @@ mc_proxy_monitor()
        return "${MC_PROXY_RC_HARD_FAILURE}"
    fi

+    # Delegated monitor execution is a valid control-plane response, but it is
+    # not itself an OCF observation and must never be written to the healthy
+    # role cache. The caller validates and executes the packaged runtime path.
+    case "${response}" in
+        delegate\|*)
+            printf '%s\n' "${response}"
+            return 0
+            ;;
+    esac
+
    rc="${response%%|*}"

    # Cache only healthy daemon-authoritative role observations. An
@@ -117,6 +117,22 @@ def format_monitor_result(result):
    if not isinstance(result, dict):
        raise ProtocolFailure

+    if result.get("delegated") is True:
+        backend = result.get("backend")
+        runtime = result.get("container_runtime")
+
+        if backend != "container":
+            raise ProtocolFailure
+
+        if runtime not in ("podman", "docker"):
+            raise ProtocolFailure
+
+        return "|".join([
+            "delegate",
+            backend,
+            runtime,
+        ])
+
    ha = result.get("ha")

    if not isinstance(ha, dict):
@@ -27,8 +27,12 @@ python3 ./tests/test-proxy-client-parameters.py
 ./tests/test-notify-daemon-shadow.sh
 ./tests/test-notify-target-shadow.sh
 ./tests/test-demote-database-shadow.sh
+./tests/test-demote-database-response-matrix.sh
+./tests/test-promote-database-shadow.sh
 ./tests/test-demote-database-gate.sh
+./tests/test-promote-database-gate.sh
 ./tests/test-demote-database-parameter.sh
+./tests/test-promote-database-parameter.sh

 echo
 echo ">>> Python syntax"
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh"
+FAKE="$(mktemp)"
+
+cleanup()
+{
+    rm -f "$FAKE"
+}
+
+trap cleanup EXIT
+
+ocf_log()
+{
+    :
+}
+
+export OCF_RESOURCE_INSTANCE=postgresql-ha
+export MC_PROXY_CLIENT="$FAKE"
+export MC_DEMOTE_DATABASE_GATE=true
+
+# shellcheck disable=SC1090
+. "$FILE"
+
+write_response()
+{
+    cat > "$FAKE"
+    chmod 0755 "$FAKE"
+}
+
+echo "===== REMOTE HANDOFF ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": True,
+    "reason": "demotion_preflight_ready",
+}))
+PY
+
+if ! mc_pg_demote_database_shadow; then
+    echo "FAIL: valid remote handoff response was rejected"
+    exit 1
+fi
+
+echo "remote_handoff            accepted"
+
+echo
+echo "===== LOCAL RESTART ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": False,
+    "handoff_mode": "local_restart",
+    "reason": "local_restart_demotion_ready",
+}))
+PY
+
+if ! mc_pg_demote_database_shadow; then
+    echo "FAIL: valid local restart response was rejected"
+    exit 1
+fi
+
+echo "local_restart             accepted"
+
+echo
+echo "===== ARBITRARY FALSE HANDOFF ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": False,
+}))
+PY
+
+if mc_pg_demote_database_shadow; then
+    echo "FAIL: arbitrary final_handoff_required=false was accepted"
+    exit 1
+fi
+
+echo "arbitrary_false           rejected"
+
+echo
+echo "===== FAKE LOCAL RESTART ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": False,
+    "handoff_mode": "local_restart",
+    "reason": "wrong_reason",
+}))
+PY
+
+if mc_pg_demote_database_shadow; then
+    echo "FAIL: fake local restart response was accepted"
+    exit 1
+fi
+
+echo "fake_local_restart        rejected"
+
+echo
+echo "DEMOTE_DATABASE_RESPONSE_MATRIX_TEST_PASS=1"
@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+
+set -u
+
+OCF_SUCCESS=0
+OCF_ERR_GENERIC=1
+OCF_NOT_RUNNING=7
+OCF_RUNNING_PROMOTED=8
+
+MC_PROXY_RC_TRANSPORT=10
+MC_PROXY_RC_HARD_FAILURE=11
+
+OCF_RESOURCE_INSTANCE=postgresql-ha
+OCF_RESKEY_container_runtime=podman
+
+logs=()
+exit_reason=""
+score=""
+cache_rc=""
+application_rc=0
+application_calls=0
+proxy_result_value="delegate|container|podman"
+
+
+# Load the real function under test first. Test doubles defined below then
+# override its external dependencies, including promotion-score helpers.
+source ocf/loxthelion/managed-container.d/common.sh
+
+
+ocf_log()
+{
+    logs+=("$*")
+}
+
+ocf_exit_reason()
+{
+    exit_reason="$*"
+}
+
+mc_proxy_monitor()
+{
+    printf '%s\n' "${proxy_result_value}"
+    return 0
+}
+
+mc_proxy_cached_monitor()
+{
+    return 1
+}
+
+mc_proxy_monitor_cache_clear()
+{
+    cache_rc="cleared"
+    return 0
+}
+
+mc_proxy_monitor_cache_write()
+{
+    cache_rc="$1"
+    return 0
+}
+
+mc_proxy_legacy_shadow_monitor()
+{
+    echo "ERROR: shadow monitor must not execute for delegation" >&2
+    return 99
+}
+
+mc_application_monitor()
+{
+    application_calls=$((application_calls + 1))
+    return "${application_rc}"
+}
+
+mc_promotion_score_set()
+{
+    score="$1"
+    return 0
+}
+
+mc_promotion_score_from_monitor()
+{
+    case "$1" in
+        0|8)
+            score="healthy:$1"
+            ;;
+        *)
+            score="-INFINITY"
+            ;;
+    esac
+
+    return 0
+}
+
+echo "===== PODMAN DELEGATION SUCCESS ====="
+
+application_rc=0
+proxy_result_value="delegate|container|podman"
+cache_rc=""
+score=""
+application_calls=0
+
+mc_promotion_monitor
+rc=$?
+
+[ "$rc" -eq 0 ]
+[ "$application_calls" -eq 1 ]
+[ "$cache_rc" = 0 ]
+[ "$score" = "healthy:0" ]
+
+echo "podman_success          accepted"
+
+
+echo
+echo "===== PROMOTED DELEGATION ====="
+
+application_rc=8
+proxy_result_value="delegate|container|podman"
+cache_rc=""
+score=""
+application_calls=0
+
+mc_promotion_monitor
+rc=$?
+
+[ "$rc" -eq 8 ]
+[ "$application_calls" -eq 1 ]
+[ "$cache_rc" = 8 ]
+[ "$score" = "healthy:8" ]
+
+echo "podman_promoted         accepted"
+
+
+echo
+echo "===== RUNTIME MISMATCH ====="
+
+application_rc=0
+proxy_result_value="delegate|container|docker"
+cache_rc=""
+score=""
+application_calls=0
+
+mc_promotion_monitor
+rc=$?
+
+[ "$rc" -eq 1 ]
+[ "$application_calls" -eq 0 ]
+[ "$cache_rc" = "cleared" ]
+[ "$score" = "-INFINITY" ]
+
+echo "runtime_mismatch        rejected"
+
+
+echo
+echo "===== MALFORMED DELEGATION ====="
+
+proxy_result_value="delegate|container|podman|extra"
+cache_rc=""
+score=""
+application_calls=0
+
+mc_promotion_monitor
+rc=$?
+
+[ "$rc" -eq 1 ]
+[ "$application_calls" -eq 0 ]
+[ "$cache_rc" = "cleared" ]
+[ "$score" = "-INFINITY" ]
+
+echo "malformed_delegation    rejected"
+
+
+echo
+echo "MONITOR_DELEGATION_SHELL_TEST_PASS=1"
@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh"
+FAKE="$(mktemp)"
+
+cleanup()
+{
+    rm -f "$FAKE"
+}
+
+trap cleanup EXIT
+
+ocf_log()
+{
+    :
+}
+
+export OCF_RESOURCE_INSTANCE=postgresql-ha
+export MC_PROXY_CLIENT="$FAKE"
+
+# Valid disagreement.
+cat > "$FAKE" <<'EOF'
+import json
+print(json.dumps({
+    "ocf_rc": 1,
+    "ready": False,
+    "mutation": False,
+}))
+EOF
+
+chmod 0755 "$FAKE"
+
+# shellcheck disable=SC1090
+. "$FILE"
+
+unset MC_PROMOTE_DATABASE_GATE
+mc_pg_promote_database_shadow
+
+export MC_PROMOTE_DATABASE_GATE=true
+
+if mc_pg_promote_database_shadow; then
+    echo "FAIL: promotion mismatch was not gated"
+    exit 1
+fi
+
+# Transport failure.
+cat > "$FAKE" <<'EOF'
+raise SystemExit(10)
+EOF
+
+chmod 0755 "$FAKE"
+
+unset MC_PROMOTE_DATABASE_GATE
+mc_pg_promote_database_shadow
+
+export MC_PROMOTE_DATABASE_GATE=true
+
+if mc_pg_promote_database_shadow; then
+    echo "FAIL: promotion transport failure was not gated"
+    exit 1
+fi
+
+# Positive daemon agreement must still pass with the gate enabled.
+cat > "$FAKE" <<'EOF'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+}))
+EOF
+
+chmod 0755 "$FAKE"
+
+export MC_PROMOTE_DATABASE_GATE=true
+
+mc_pg_promote_database_shadow
+
+echo "PROMOTE_DATABASE_GATE_RUNTIME_TEST_PASS=1"
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh"
+
+grep -q \
+  'OCF_RESKEY_promote_database_gate_default="false"' \
+  "$FILE"
+
+grep -q \
+  ': "${OCF_RESKEY_promote_database_gate:=' \
+  "$FILE"
+
+grep -q \
+  'PROMOTE_DATABASE_GATE="${OCF_RESKEY_promote_database_gate}"' \
+  "$FILE"
+
+grep -q \
+  'export MC_PROMOTE_DATABASE_GATE="${PROMOTE_DATABASE_GATE}"' \
+  "$FILE"
+
+grep -q \
+  'promote_database_gate must be true or false' \
+  "$FILE"
+
+grep -q \
+  '<parameter name="promote_database_gate"' \
+  "$FILE"
+
+grep -q \
+  '<content type="boolean" default="false"/>' \
+  "$FILE"
+
+echo "PROMOTE_DATABASE_PARAMETER_TEST_PASS=1"
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh"
+
+grep -q '^mc_pg_promote_database_shadow()' "$FILE"
+grep -q '"promote.database"' "$FILE"
+grep -q 'response.get("ready") is not True' "$FILE"
+grep -q 'response.get("mutation") is not False' "$FILE"
+grep -q 'comparison=MATCH' "$FILE"
+grep -q 'promote.database shadow MISMATCH' "$FILE"
+
+SHADOW_LINE="$(
+    grep -n 'if ! mc_pg_promote_database_shadow; then' "$FILE" |
+    head -1 |
+    cut -d: -f1
+)"
+
+PROMOTE_LINE="$(
+    grep -n 'exec "\$BINDIR/pg_ctl"' "$FILE" |
+    head -1 |
+    cut -d: -f1
+)"
+
+[ -n "$SHADOW_LINE" ]
+[ -n "$PROMOTE_LINE" ]
+[ "$SHADOW_LINE" -lt "$PROMOTE_LINE" ]
+
+echo "PROMOTE_DATABASE_SHADOW_TEST_PASS=1"
@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+
+import importlib.util
+from pathlib import Path
+
+
+spec = importlib.util.spec_from_file_location(
+    "proxy_client",
+    Path(
+        "ocf/loxthelion/"
+        "managed-container.d/"
+        "proxy_client.py"
+    ),
+)
+
+proxy = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(proxy)
+
+
+print("===== NORMAL API MONITOR RESULT =====")
+
+normal = {
+    "ha": {
+        "ocf_rc": 0,
+        "health": "healthy",
+        "ha_role": "unpromoted",
+        "state": "streaming",
+        "promotion_eligible": True,
+    },
+    "role": "standby",
+    "streaming": True,
+    "wal_receiver": {
+        "slot_name": "ha_pg_node1",
+        "sender_host": "192.0.2.10",
+    },
+}
+
+formatted = proxy.format_monitor_result(normal)
+
+expected = (
+    "0|healthy|unpromoted|streaming|true|"
+    "standby|true|ha_pg_node1|192.0.2.10"
+)
+
+assert formatted == expected, formatted
+
+print("api_monitor_format       accepted")
+
+
+print()
+print("===== PODMAN DELEGATION =====")
+
+delegated = {
+    "delegated": True,
+    "backend": "container",
+    "container_runtime": "podman",
+}
+
+formatted = proxy.format_monitor_result(delegated)
+
+assert formatted == "delegate|container|podman", formatted
+
+print("podman_delegation        accepted")
+
+
+print()
+print("===== DOCKER DELEGATION =====")
+
+delegated = {
+    "delegated": True,
+    "backend": "container",
+    "container_runtime": "docker",
+}
+
+formatted = proxy.format_monitor_result(delegated)
+
+assert formatted == "delegate|container|docker", formatted
+
+print("docker_delegation        accepted")
+
+
+print()
+print("===== INVALID DELEGATED BACKEND =====")
+
+bad = {
+    "delegated": True,
+    "backend": "api",
+    "container_runtime": "podman",
+}
+
+try:
+    proxy.format_monitor_result(bad)
+except proxy.ProtocolFailure:
+    print("invalid_backend          rejected")
+else:
+    raise AssertionError(
+        "delegation with non-container backend was accepted"
+    )
+
+
+print()
+print("===== INVALID DELEGATED RUNTIME =====")
+
+bad = {
+    "delegated": True,
+    "backend": "container",
+    "container_runtime": "lxc",
+}
+
+try:
+    proxy.format_monitor_result(bad)
+except proxy.ProtocolFailure:
+    print("invalid_runtime          rejected")
+else:
+    raise AssertionError(
+        "delegation with invalid runtime was accepted"
+    )
+
+
+print()
+print("===== MALFORMED DELEGATION =====")
+
+bad = {
+    "delegated": True,
+    "backend": "container",
+}
+
+try:
+    proxy.format_monitor_result(bad)
+except proxy.ProtocolFailure:
+    print("missing_runtime          rejected")
+else:
+    raise AssertionError(
+        "delegation without runtime was accepted"
+    )
+
+
+print()
+print("PROXY_MONITOR_DELEGATION_TEST_PASS=1")
@@ -150,6 +150,13 @@ with conn:
        elif mode == "bad_rc":
            response["result"]["ha"]["ocf_rc"] = 42

+        elif mode == "delegate":
+            response["result"] = {
+                "delegated": True,
+                "backend": "container",
+                "container_runtime": "podman",
+            }
+
    conn.sendall(
        json.dumps(response).encode("utf-8") + b"\n"
    )
@@ -248,6 +255,7 @@ PY_REFUSED
 echo "===== PROXY PROTOCOL CLASSIFICATION ====="

 run_case valid             valid             0
+run_case delegation        delegate          0
 run_case missing_protocol  missing_protocol  11
 run_case wrong_protocol    wrong_protocol    11
 run_case wrong_resource    wrong_resource    11
Author	SHA1	Message	Date
Lyncoln	3d1714d995	Remove legacy monitor shadow execution	2026-09-18 15:13:13 -04:00
Lyncoln	cbb865729a	Add daemon-selected container monitor delegation	2026-09-18 14:09:46 -04:00
Lyncoln	3937de82dc	Accept validated PostgreSQL self-restart demotion	2026-09-18 09:43:41 -04:00
Lyncoln	798aa40ae5	Add PostgreSQL promotion database gate	2026-09-18 02:46:40 -04:00
Lyncoln	4229008216	Shadow PostgreSQL promotion database preflight	2026-09-17 22:54:26 -04:00