Add daemon-selected container monitor delegation

checksums/SHA256SUMS

@@ -2,7 +2,7 @@
 5aa02fa065d1dba55486838fa4f0cd8eba982edaaf73cdc7d818223a09cbadc6  ocf/loxthelion/managed-container
 10932919c4339b553c7c9dbf30b45a917c51c0a08f362e12678003d58b12fa82  ocf/loxthelion/managed-container.d/applications/postgresql/auth.sh
 61d64b4a3a15c354f101ef34c5dc0425723f4204207d359b5377d2b9f04f6b23  ocf/loxthelion/managed-container.d/applications/postgresql/demote.sh
-e111dfe454a6e1edd80edc0c303119e045e7c88fe77b69a2745e58a022d0d317  ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh
+86bb5bacd27e4082a7f753108c1f5d5e777ff04fba8699d62ed148c17224abdf  ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh
 bcfde96b98f4727a8c97d42a39b7c6ec925b1b89d17ef67cc58eb7bd67afbfae  ocf/loxthelion/managed-container.d/applications/postgresql/notify.sh
 957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548  ocf/loxthelion/managed-container.d/applications/postgresql/peer_config.sh
 286d282565239bdb600563bac116575f3f56580e06f8f8df86c1a0fd0af96f07  ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh
@@ -23,9 +23,10 @@ a42bb59a058ffeb59c1baa96e403856910bbbaea98f9104ff93daf4b5970c780  postgres-host/
 957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548  postgres-host/postgres-ha-peer-config.sh
 deb09e3ae40629f43b262ba4f9a7ebf59a1dca3da9d8c94478f670aea28bd0e1  quadlet/postgres.container
 04a125068677bbee6e71d313ca9d95d35faa32851b79de781cd7b9f466561445  quadlet/postgres.env.example
-186c2350d1f4eaddd756fbc680319ce2bfb7ee3462cf1d421f549ba8ccca8d24  tests/run-all.sh
+ad248df2911e87fbe456c095f4e30dd54b2da52c9e2efa0619d3a2bf16a0f30c  tests/run-all.sh
 a373c0560391a2d8f63c3ba31609ab16b219d9b532b4ab9f8aea6dc45de10555  tests/test-demote-database-gate.sh
 688eeb756c7f346e7e55acc555004ba12b48bc7c3d309073658d5e84a6e5e348  tests/test-demote-database-parameter.sh
+a3b3b2dad6bfdab8ffc7b3cabc6e1b9f188ff743fcadf073e5e8bdcf1827b2b9  tests/test-demote-database-response-matrix.sh
 22d3eb56e8cd3ae1a58e2616f44abd3818f79d7e5df55c1c158e36dca7c77f42  tests/test-demote-database-shadow.sh
 5e5d4fdd2b6c7e0b8b146de7aad06c1f96b221a10fec525b00d40e0d85709fb0  tests/test-entrypoint-structure.sh
 2faee72cf2fdcbfa667c3e1c2a7aabc053b96a9b8127ff7440ddb717fd2dc314  tests/test-notify-daemon-shadow.sh

ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh

@@ -71,12 +71,23 @@ import os
 
 response = json.loads(os.environ["MC_DAEMON_RESPONSE"])
 
-if (
+common_valid = (
-    response.get("ocf_rc") != 0
+    response.get("ocf_rc") == 0
-    or response.get("ready") is not True
+    and response.get("ready") is True
-    or response.get("mutation") is not False
+    and response.get("mutation") is False
-    or response.get("final_handoff_required") is not True
+)
-):
+
+peer_handoff = (
+    response.get("final_handoff_required") is True
+)
+
+local_restart = (
+    response.get("final_handoff_required") is False
+    and response.get("handoff_mode") == "local_restart"
+    and response.get("reason") == "local_restart_demotion_ready"
+)
+
+if not common_valid or not (peer_handoff or local_restart):
     raise SystemExit(1)
 '; then
         ocf_log info \

ocf/loxthelion/managed-container.d/common.sh

@@ -119,6 +119,70 @@ mc_promotion_monitor()
         esac
     fi
 
+    # A successful managed-containerd response may explicitly delegate
+    # monitoring to the RA's already-loaded packaged container/runtime path.
+    #
+    # The daemon remains authoritative for backend selection: this path is
+    # reachable only after managed-containerd successfully requested it.
+    case "${proxy_result}" in
+        delegate\|*)
+            local delegate_tag delegate_backend delegate_runtime delegate_extra
+            local configured_runtime
+
+            IFS='|' read -r                 delegate_tag                 delegate_backend                 delegate_runtime                 delegate_extra <<EOF
+${proxy_result}
+EOF
+
+            if [ "${delegate_tag}" != delegate ] ||
+               [ "${delegate_backend}" != container ] ||
+               [ -z "${delegate_runtime}" ] ||
+               [ -n "${delegate_extra}" ]; then
+
+                mc_proxy_monitor_cache_clear || true
+                mc_promotion_score_set -INFINITY || true
+                ocf_exit_reason                     "managed-containerd returned invalid monitor delegation: ${proxy_result}"
+                return "${OCF_ERR_GENERIC}"
+            fi
+
+            configured_runtime="${OCF_RESKEY_container_runtime:-podman}"
+
+            if [ "${delegate_runtime}" != "${configured_runtime}" ]; then
+                mc_proxy_monitor_cache_clear || true
+                mc_promotion_score_set -INFINITY || true
+                ocf_exit_reason                     "managed-containerd delegated runtime ${delegate_runtime} does not match loaded RA runtime ${configured_runtime}"
+                return "${OCF_ERR_GENERIC}"
+            fi
+
+            ocf_log info                 "managed-containerd delegated authoritative monitor: resource=${OCF_RESOURCE_INSTANCE:-unknown} backend=${delegate_backend} runtime=${delegate_runtime}"
+
+            mc_application_monitor
+            monitor_rc=$?
+
+            case "${monitor_rc}" in
+                "${OCF_SUCCESS}"|                "${OCF_ERR_GENERIC}"|                "${OCF_NOT_RUNNING}"|                "${OCF_RUNNING_PROMOTED}")
+                    ;;
+                *)
+                    mc_proxy_monitor_cache_clear || true
+                    mc_promotion_score_set -INFINITY || true
+                    ocf_exit_reason                         "delegated application monitor returned invalid rc ${monitor_rc}"
+                    return "${OCF_ERR_GENERIC}"
+                    ;;
+            esac
+
+            # The delegated application result is now the authoritative
+            # observation. Cache only healthy 0/8 results using the same
+            # control-plane grace mechanism as daemon-native monitoring.
+            mc_proxy_monitor_cache_write "${monitor_rc}" || {
+                ocf_log warn                     "Unable to update delegated authoritative monitor cache"
+            }
+
+            ocf_log info                 "managed-containerd delegated monitor result: resource=${OCF_RESOURCE_INSTANCE:-unknown} rc=${monitor_rc} backend=${delegate_backend} runtime=${delegate_runtime}"
+
+            mc_promotion_score_from_monitor "${monitor_rc}" || true
+            return "${monitor_rc}"
+            ;;
+    esac
+
     monitor_rc="${proxy_result%%|*}"
 
     case "${monitor_rc}" in

ocf/loxthelion/managed-container.d/proxy.sh

@@ -88,6 +88,16 @@ mc_proxy_monitor()
         return "${MC_PROXY_RC_HARD_FAILURE}"
     fi
 
+    # Delegated monitor execution is a valid control-plane response, but it is
+    # not itself an OCF observation and must never be written to the healthy
+    # role cache. The caller validates and executes the packaged runtime path.
+    case "${response}" in
+        delegate\|*)
+            printf '%s\n' "${response}"
+            return 0
+            ;;
+    esac
+
     rc="${response%%|*}"
 
     # Cache only healthy daemon-authoritative role observations. An

ocf/loxthelion/managed-container.d/proxy_client.py

@@ -117,6 +117,22 @@ def format_monitor_result(result):
     if not isinstance(result, dict):
         raise ProtocolFailure
 
+    if result.get("delegated") is True:
+        backend = result.get("backend")
+        runtime = result.get("container_runtime")
+
+        if backend != "container":
+            raise ProtocolFailure
+
+        if runtime not in ("podman", "docker"):
+            raise ProtocolFailure
+
+        return "|".join([
+            "delegate",
+            backend,
+            runtime,
+        ])
+
     ha = result.get("ha")
 
     if not isinstance(ha, dict):

tests/run-all.sh

@@ -27,6 +27,7 @@ python3 ./tests/test-proxy-client-parameters.py
 ./tests/test-notify-daemon-shadow.sh
 ./tests/test-notify-target-shadow.sh
 ./tests/test-demote-database-shadow.sh
+./tests/test-demote-database-response-matrix.sh
 ./tests/test-promote-database-shadow.sh
 ./tests/test-demote-database-gate.sh
 ./tests/test-promote-database-gate.sh

tests/test-demote-database-response-matrix.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
+FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh"
+FAKE="$(mktemp)"
+
+cleanup()
+{
+    rm -f "$FAKE"
+}
+
+trap cleanup EXIT
+
+ocf_log()
+{
+    :
+}
+
+export OCF_RESOURCE_INSTANCE=postgresql-ha
+export MC_PROXY_CLIENT="$FAKE"
+export MC_DEMOTE_DATABASE_GATE=true
+
+# shellcheck disable=SC1090
+. "$FILE"
+
+write_response()
+{
+    cat > "$FAKE"
+    chmod 0755 "$FAKE"
+}
+
+echo "===== REMOTE HANDOFF ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": True,
+    "reason": "demotion_preflight_ready",
+}))
+PY
+
+if ! mc_pg_demote_database_shadow; then
+    echo "FAIL: valid remote handoff response was rejected"
+    exit 1
+fi
+
+echo "remote_handoff            accepted"
+
+echo
+echo "===== LOCAL RESTART ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": False,
+    "handoff_mode": "local_restart",
+    "reason": "local_restart_demotion_ready",
+}))
+PY
+
+if ! mc_pg_demote_database_shadow; then
+    echo "FAIL: valid local restart response was rejected"
+    exit 1
+fi
+
+echo "local_restart             accepted"
+
+echo
+echo "===== ARBITRARY FALSE HANDOFF ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": False,
+}))
+PY
+
+if mc_pg_demote_database_shadow; then
+    echo "FAIL: arbitrary final_handoff_required=false was accepted"
+    exit 1
+fi
+
+echo "arbitrary_false           rejected"
+
+echo
+echo "===== FAKE LOCAL RESTART ====="
+
+write_response <<'PY'
+import json
+print(json.dumps({
+    "ocf_rc": 0,
+    "ready": True,
+    "mutation": False,
+    "final_handoff_required": False,
+    "handoff_mode": "local_restart",
+    "reason": "wrong_reason",
+}))
+PY
+
+if mc_pg_demote_database_shadow; then
+    echo "FAIL: fake local restart response was accepted"
+    exit 1
+fi
+
+echo "fake_local_restart        rejected"
+
+echo
+echo "DEMOTE_DATABASE_RESPONSE_MATRIX_TEST_PASS=1"

tests/test-monitor-delegation.sh

@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+
+set -u
+
+OCF_SUCCESS=0
+OCF_ERR_GENERIC=1
+OCF_NOT_RUNNING=7
+OCF_RUNNING_PROMOTED=8
+
+MC_PROXY_RC_TRANSPORT=10
+MC_PROXY_RC_HARD_FAILURE=11
+
+OCF_RESOURCE_INSTANCE=postgresql-ha
+OCF_RESKEY_container_runtime=podman
+
+logs=()
+exit_reason=""
+score=""
+cache_rc=""
+application_rc=0
+application_calls=0
+proxy_result_value="delegate|container|podman"
+
+
+# Load the real function under test first. Test doubles defined below then
+# override its external dependencies, including promotion-score helpers.
+source ocf/loxthelion/managed-container.d/common.sh
+
+
+ocf_log()
+{
+    logs+=("$*")
+}
+
+ocf_exit_reason()
+{
+    exit_reason="$*"
+}
+
+mc_proxy_monitor()
+{
+    printf '%s\n' "${proxy_result_value}"
+    return 0
+}
+
+mc_proxy_cached_monitor()
+{
+    return 1
+}
+
+mc_proxy_monitor_cache_clear()
+{
+    cache_rc="cleared"
+    return 0
+}
+
+mc_proxy_monitor_cache_write()
+{
+    cache_rc="$1"
+    return 0
+}
+
+mc_proxy_legacy_shadow_monitor()
+{
+    echo "ERROR: shadow monitor must not execute for delegation" >&2
+    return 99
+}
+
+mc_application_monitor()
+{
+    application_calls=$((application_calls + 1))
+    return "${application_rc}"
+}
+
+mc_promotion_score_set()
+{
+    score="$1"
+    return 0
+}
+
+mc_promotion_score_from_monitor()
+{
+    case "$1" in
+        0|8)
+            score="healthy:$1"
+            ;;
+        *)
+            score="-INFINITY"
+            ;;
+    esac
+
+    return 0
+}
+
+echo "===== PODMAN DELEGATION SUCCESS ====="
+
+application_rc=0
+proxy_result_value="delegate|container|podman"
+cache_rc=""
+score=""
+application_calls=0
+
+mc_promotion_monitor
+rc=$?
+
+[ "$rc" -eq 0 ]
+[ "$application_calls" -eq 1 ]
+[ "$cache_rc" = 0 ]
+[ "$score" = "healthy:0" ]
+
+echo "podman_success          accepted"
+
+
+echo
+echo "===== PROMOTED DELEGATION ====="
+
+application_rc=8
+proxy_result_value="delegate|container|podman"
+cache_rc=""
+score=""
+application_calls=0
+
+mc_promotion_monitor
+rc=$?
+
+[ "$rc" -eq 8 ]
+[ "$application_calls" -eq 1 ]
+[ "$cache_rc" = 8 ]
+[ "$score" = "healthy:8" ]
+
+echo "podman_promoted         accepted"
+
+
+echo
+echo "===== RUNTIME MISMATCH ====="
+
+application_rc=0
+proxy_result_value="delegate|container|docker"
+cache_rc=""
+score=""
+application_calls=0
+
+mc_promotion_monitor
+rc=$?
+
+[ "$rc" -eq 1 ]
+[ "$application_calls" -eq 0 ]
+[ "$cache_rc" = "cleared" ]
+[ "$score" = "-INFINITY" ]
+
+echo "runtime_mismatch        rejected"
+
+
+echo
+echo "===== MALFORMED DELEGATION ====="
+
+proxy_result_value="delegate|container|podman|extra"
+cache_rc=""
+score=""
+application_calls=0
+
+mc_promotion_monitor
+rc=$?
+
+[ "$rc" -eq 1 ]
+[ "$application_calls" -eq 0 ]
+[ "$cache_rc" = "cleared" ]
+[ "$score" = "-INFINITY" ]
+
+echo "malformed_delegation    rejected"
+
+
+echo
+echo "MONITOR_DELEGATION_SHELL_TEST_PASS=1"

tests/test-proxy-monitor-delegation.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+
+import importlib.util
+from pathlib import Path
+
+
+spec = importlib.util.spec_from_file_location(
+    "proxy_client",
+    Path(
+        "ocf/loxthelion/"
+        "managed-container.d/"
+        "proxy_client.py"
+    ),
+)
+
+proxy = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(proxy)
+
+
+print("===== NORMAL API MONITOR RESULT =====")
+
+normal = {
+    "ha": {
+        "ocf_rc": 0,
+        "health": "healthy",
+        "ha_role": "unpromoted",
+        "state": "streaming",
+        "promotion_eligible": True,
+    },
+    "role": "standby",
+    "streaming": True,
+    "wal_receiver": {
+        "slot_name": "ha_pg_node1",
+        "sender_host": "192.0.2.10",
+    },
+}
+
+formatted = proxy.format_monitor_result(normal)
+
+expected = (
+    "0|healthy|unpromoted|streaming|true|"
+    "standby|true|ha_pg_node1|192.0.2.10"
+)
+
+assert formatted == expected, formatted
+
+print("api_monitor_format       accepted")
+
+
+print()
+print("===== PODMAN DELEGATION =====")
+
+delegated = {
+    "delegated": True,
+    "backend": "container",
+    "container_runtime": "podman",
+}
+
+formatted = proxy.format_monitor_result(delegated)
+
+assert formatted == "delegate|container|podman", formatted
+
+print("podman_delegation        accepted")
+
+
+print()
+print("===== DOCKER DELEGATION =====")
+
+delegated = {
+    "delegated": True,
+    "backend": "container",
+    "container_runtime": "docker",
+}
+
+formatted = proxy.format_monitor_result(delegated)
+
+assert formatted == "delegate|container|docker", formatted
+
+print("docker_delegation        accepted")
+
+
+print()
+print("===== INVALID DELEGATED BACKEND =====")
+
+bad = {
+    "delegated": True,
+    "backend": "api",
+    "container_runtime": "podman",
+}
+
+try:
+    proxy.format_monitor_result(bad)
+except proxy.ProtocolFailure:
+    print("invalid_backend          rejected")
+else:
+    raise AssertionError(
+        "delegation with non-container backend was accepted"
+    )
+
+
+print()
+print("===== INVALID DELEGATED RUNTIME =====")
+
+bad = {
+    "delegated": True,
+    "backend": "container",
+    "container_runtime": "lxc",
+}
+
+try:
+    proxy.format_monitor_result(bad)
+except proxy.ProtocolFailure:
+    print("invalid_runtime          rejected")
+else:
+    raise AssertionError(
+        "delegation with invalid runtime was accepted"
+    )
+
+
+print()
+print("===== MALFORMED DELEGATION =====")
+
+bad = {
+    "delegated": True,
+    "backend": "container",
+}
+
+try:
+    proxy.format_monitor_result(bad)
+except proxy.ProtocolFailure:
+    print("missing_runtime          rejected")
+else:
+    raise AssertionError(
+        "delegation without runtime was accepted"
+    )
+
+
+print()
+print("PROXY_MONITOR_DELEGATION_TEST_PASS=1")

tests/test-proxy-protocol.sh

@@ -150,6 +150,13 @@ with conn:
         elif mode == "bad_rc":
             response["result"]["ha"]["ocf_rc"] = 42
 
+        elif mode == "delegate":
+            response["result"] = {
+                "delegated": True,
+                "backend": "container",
+                "container_runtime": "podman",
+            }
+
     conn.sendall(
         json.dumps(response).encode("utf-8") + b"\n"
     )
@@ -248,6 +255,7 @@ PY_REFUSED
 echo "===== PROXY PROTOCOL CLASSIFICATION ====="
 
 run_case valid             valid             0
+run_case delegation        delegate          0
 run_case missing_protocol  missing_protocol  11
 run_case wrong_protocol    wrong_protocol    11
 run_case wrong_resource    wrong_resource    11