From 798aa40ae54d84b51af445489b4d60e001e02e41 Mon Sep 17 00:00:00 2001 From: Lyncoln Date: Fri, 18 Sep 2026 02:46:40 -0400 Subject: [PATCH] Add PostgreSQL promotion database gate --- checksums/SHA256SUMS | 10 ++- .../applications/postgresql/plugin.sh | 18 +++++ .../applications/postgresql/promote.sh | 21 +++-- tests/run-all.sh | 2 + tests/test-promote-database-gate.sh | 81 +++++++++++++++++++ tests/test-promote-database-parameter.sh | 35 ++++++++ tests/test-promote-database-shadow.sh | 4 +- 7 files changed, 160 insertions(+), 11 deletions(-) create mode 100755 tests/test-promote-database-gate.sh create mode 100755 tests/test-promote-database-parameter.sh diff --git a/checksums/SHA256SUMS b/checksums/SHA256SUMS index 695e3a9..4cb534c 100644 --- a/checksums/SHA256SUMS +++ b/checksums/SHA256SUMS @@ -5,9 +5,9 @@ e111dfe454a6e1edd80edc0c303119e045e7c88fe77b69a2745e58a022d0d317 ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh bcfde96b98f4727a8c97d42a39b7c6ec925b1b89d17ef67cc58eb7bd67afbfae ocf/loxthelion/managed-container.d/applications/postgresql/notify.sh 957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548 ocf/loxthelion/managed-container.d/applications/postgresql/peer_config.sh -99bb3478931eced9471a8f054c14d152780aec73801449d1a6b2f09a4b656bec ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh +286d282565239bdb600563bac116575f3f56580e06f8f8df86c1a0fd0af96f07 ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh c1d878dabf22ad3928d66a5542bda3c150e5e9b039926629e7c5fd3401d29be4 ocf/loxthelion/managed-container.d/applications/postgresql/postgres.sh -78b32fd6354bec61375ee73a31898059b21d51949a906cc6c7f23419a79670d6 ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh +d8534993c8e93f630bc0987301f5837f66e93c90d02b1063f46fdcdb559c0391 ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh 5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7 ocf/loxthelion/managed-container.d/applications/postgresql/promotion_state.py b0ebad749ed736f08ec748a693f12e8dc2541430e0d3db75bbb6b2b7e517f84b ocf/loxthelion/managed-container.d/applications/postgresql/rejoin.sh 0d3017fdca92b2dc5dba406065bc269622ce5140e3fce4a959ca95ce4a78e755 ocf/loxthelion/managed-container.d/applications/postgresql/replication.sh @@ -23,14 +23,16 @@ a42bb59a058ffeb59c1baa96e403856910bbbaea98f9104ff93daf4b5970c780 postgres-host/ 957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548 postgres-host/postgres-ha-peer-config.sh deb09e3ae40629f43b262ba4f9a7ebf59a1dca3da9d8c94478f670aea28bd0e1 quadlet/postgres.container 04a125068677bbee6e71d313ca9d95d35faa32851b79de781cd7b9f466561445 quadlet/postgres.env.example -ac72a62381a686dbc7dcebf2f81bd3f96e84e83ba63ff956440546a8026037fb tests/run-all.sh +186c2350d1f4eaddd756fbc680319ce2bfb7ee3462cf1d421f549ba8ccca8d24 tests/run-all.sh a373c0560391a2d8f63c3ba31609ab16b219d9b532b4ab9f8aea6dc45de10555 tests/test-demote-database-gate.sh 688eeb756c7f346e7e55acc555004ba12b48bc7c3d309073658d5e84a6e5e348 tests/test-demote-database-parameter.sh 22d3eb56e8cd3ae1a58e2616f44abd3818f79d7e5df55c1c158e36dca7c77f42 tests/test-demote-database-shadow.sh 5e5d4fdd2b6c7e0b8b146de7aad06c1f96b221a10fec525b00d40e0d85709fb0 tests/test-entrypoint-structure.sh 2faee72cf2fdcbfa667c3e1c2a7aabc053b96a9b8127ff7440ddb717fd2dc314 tests/test-notify-daemon-shadow.sh 9150004af86a819d7d1b5ae2b3fa9d4ddd0480f55babd094137052109864b057 tests/test-notify-target-shadow.sh -dd7ad3d1c06cac922c026b6bc30db5a5e66aa5d82948ebed983384c7b0bbba33 tests/test-promote-database-shadow.sh +ea2dc89483c5addc59cee0b8f070a08894c4803a204263507e2eaffb5416718e tests/test-promote-database-gate.sh +fc292a8612911423fdc8b9c5d187e2a19dd240d1bf35340d0b307888543beaa0 tests/test-promote-database-parameter.sh +7ea6ec5aaf2364f140f2987b4c2cf65ce5a132e284b6be22968de0cc4b63b3d5 tests/test-promote-database-shadow.sh b83f0415c09f0be34c51722370e5b07c51b9a9756d305d3b3f50827f484aa4dd tests/test-proxy-client-actions.py cc4482d229805c64d7df258a8a82984f244128d27fff4ea43b36caa004b56bef tests/test-proxy-client-parameters.py c3b988d7b2b1b138bb2530df9172f96bb0f5edc2f1561f38f0055695f4126134 tests/test-proxy-protocol.sh diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh index b6b9b06..b2ce858 100755 --- a/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh @@ -14,6 +14,7 @@ OCF_RESKEY_application_name_default="" OCF_RESKEY_rejoin_allow_basebackup_default="false" OCF_RESKEY_rejoin_basebackup_headroom_percent_default="50" OCF_RESKEY_demote_database_gate_default="false" +OCF_RESKEY_promote_database_gate_default="false" : "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}" : "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}" @@ -29,6 +30,7 @@ OCF_RESKEY_demote_database_gate_default="false" : "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}" : "${OCF_RESKEY_rejoin_basebackup_headroom_percent:=${OCF_RESKEY_rejoin_basebackup_headroom_percent_default}}" : "${OCF_RESKEY_demote_database_gate:=${OCF_RESKEY_demote_database_gate_default}}" +: "${OCF_RESKEY_promote_database_gate:=${OCF_RESKEY_promote_database_gate_default}}" SERVICE_USER="${OCF_RESKEY_service_user}" SERVICE_NAME="${OCF_RESKEY_service_name}" @@ -44,7 +46,9 @@ APPLICATION_NAME="${OCF_RESKEY_application_name}" REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}" REJOIN_BASEBACKUP_HEADROOM_PERCENT="${OCF_RESKEY_rejoin_basebackup_headroom_percent}" DEMOTE_DATABASE_GATE="${OCF_RESKEY_demote_database_gate}" +PROMOTE_DATABASE_GATE="${OCF_RESKEY_promote_database_gate}" export MC_DEMOTE_DATABASE_GATE="${DEMOTE_DATABASE_GATE}" +export MC_PROMOTE_DATABASE_GATE="${PROMOTE_DATABASE_GATE}" # Resolve PostgreSQL configuration from the managed service when Pacemaker has # not explicitly supplied it. The application configuration remains the source @@ -303,6 +307,15 @@ validate_all() ;; esac + case "${PROMOTE_DATABASE_GATE}" in + true|false) + ;; + *) + ocf_exit_reason "promote_database_gate must be true or false" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + case "${REJOIN_BASEBACKUP_HEADROOM_PERCENT}" in ""|*[!0-9]*) ocf_exit_reason "rejoin_basebackup_headroom_percent must be a non-negative integer" @@ -352,6 +365,11 @@ mc_promotion_metadata() Gate demotion on database checks\ \ \ + \ + When true, reject native PostgreSQL promotion unless the managed-containerd promote.database preflight agrees that this node is the notified promotion target, is an eligible standby, and no reachable writable peer exists. When false, the daemon check remains shadow-only and native promotion fails open.\ + Gate promotion on database checks\ + \ + \ \ Opt-in stable peer map: node=host:port,node=host:port. Include the local node; exclude the client VIP. IPv4 or DNS hosts only. Requires a configured per-node replication slot, supplied explicitly or resolved from PostgreSQL/the managed container environment, plus peer authentication and slots on every upstream. Uses native primary-only selection; demotion verifies final WAL then ready recovery before successor promotion.\ Native replication peer map\ diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh index a38b57e..952f49a 100644 --- a/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh @@ -29,6 +29,11 @@ mc_pg_promote_database_shadow() if [ "${rc}" -ne 0 ]; then ocf_log warn \ "managed-containerd promote.database shadow unavailable rc=${rc}" + + if [ "${MC_PROMOTE_DATABASE_GATE:-false}" = true ]; then + return 1 + fi + return 0 fi @@ -50,10 +55,12 @@ if ( else ocf_log err \ "managed-containerd promote.database shadow MISMATCH response=${response}" + + if [ "${MC_PROMOTE_DATABASE_GATE:-false}" = true ]; then + return 1 + fi fi - # Shadow-only milestone: daemon disagreement must not alter the - # authoritative native promotion result yet. return 0 } @@ -117,9 +124,13 @@ managed_promote() "Promoting PostgreSQL standby inside ${CONTAINER_NAME}" # Independent managed-containerd promotion preflight. - # Shadow-only for now: disagreement is logged but does not block - # the authoritative native PostgreSQL promotion. - mc_pg_promote_database_shadow + # Shadow-only by default; when promote_database_gate=true, + # daemon disagreement or transport failure denies native promotion. + if ! mc_pg_promote_database_shadow; then + ocf_exit_reason \ + "Promotion database gate rejected native PostgreSQL promotion" + return "${OCF_ERR_GENERIC}" + fi if ! pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 diff --git a/tests/run-all.sh b/tests/run-all.sh index cd8fc2d..a287169 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -29,7 +29,9 @@ python3 ./tests/test-proxy-client-parameters.py ./tests/test-demote-database-shadow.sh ./tests/test-promote-database-shadow.sh ./tests/test-demote-database-gate.sh +./tests/test-promote-database-gate.sh ./tests/test-demote-database-parameter.sh +./tests/test-promote-database-parameter.sh echo echo ">>> Python syntax" diff --git a/tests/test-promote-database-gate.sh b/tests/test-promote-database-gate.sh new file mode 100755 index 0000000..7da633e --- /dev/null +++ b/tests/test-promote-database-gate.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh" +FAKE="$(mktemp)" + +cleanup() +{ + rm -f "$FAKE" +} + +trap cleanup EXIT + +ocf_log() +{ + : +} + +export OCF_RESOURCE_INSTANCE=postgresql-ha +export MC_PROXY_CLIENT="$FAKE" + +# Valid disagreement. +cat > "$FAKE" <<'EOF' +import json +print(json.dumps({ + "ocf_rc": 1, + "ready": False, + "mutation": False, +})) +EOF + +chmod 0755 "$FAKE" + +# shellcheck disable=SC1090 +. "$FILE" + +unset MC_PROMOTE_DATABASE_GATE +mc_pg_promote_database_shadow + +export MC_PROMOTE_DATABASE_GATE=true + +if mc_pg_promote_database_shadow; then + echo "FAIL: promotion mismatch was not gated" + exit 1 +fi + +# Transport failure. +cat > "$FAKE" <<'EOF' +raise SystemExit(10) +EOF + +chmod 0755 "$FAKE" + +unset MC_PROMOTE_DATABASE_GATE +mc_pg_promote_database_shadow + +export MC_PROMOTE_DATABASE_GATE=true + +if mc_pg_promote_database_shadow; then + echo "FAIL: promotion transport failure was not gated" + exit 1 +fi + +# Positive daemon agreement must still pass with the gate enabled. +cat > "$FAKE" <<'EOF' +import json +print(json.dumps({ + "ocf_rc": 0, + "ready": True, + "mutation": False, +})) +EOF + +chmod 0755 "$FAKE" + +export MC_PROMOTE_DATABASE_GATE=true + +mc_pg_promote_database_shadow + +echo "PROMOTE_DATABASE_GATE_RUNTIME_TEST_PASS=1" diff --git a/tests/test-promote-database-parameter.sh b/tests/test-promote-database-parameter.sh new file mode 100755 index 0000000..f6f99fa --- /dev/null +++ b/tests/test-promote-database-parameter.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh" + +grep -q \ + 'OCF_RESKEY_promote_database_gate_default="false"' \ + "$FILE" + +grep -q \ + ': "${OCF_RESKEY_promote_database_gate:=' \ + "$FILE" + +grep -q \ + 'PROMOTE_DATABASE_GATE="${OCF_RESKEY_promote_database_gate}"' \ + "$FILE" + +grep -q \ + 'export MC_PROMOTE_DATABASE_GATE="${PROMOTE_DATABASE_GATE}"' \ + "$FILE" + +grep -q \ + 'promote_database_gate must be true or false' \ + "$FILE" + +grep -q \ + '' \ + "$FILE" + +echo "PROMOTE_DATABASE_PARAMETER_TEST_PASS=1" diff --git a/tests/test-promote-database-shadow.sh b/tests/test-promote-database-shadow.sh index 2c2d772..6ce6290 100755 --- a/tests/test-promote-database-shadow.sh +++ b/tests/test-promote-database-shadow.sh @@ -12,8 +12,8 @@ grep -q 'comparison=MATCH' "$FILE" grep -q 'promote.database shadow MISMATCH' "$FILE" SHADOW_LINE="$( - grep -n '^[[:space:]]*mc_pg_promote_database_shadow$' "$FILE" | - tail -1 | + grep -n 'if ! mc_pg_promote_database_shadow; then' "$FILE" | + head -1 | cut -d: -f1 )"