Add PostgreSQL promotion database gate

This commit is contained in:
Lyncoln
2026-09-18 02:46:40 -04:00
parent 4229008216
commit 798aa40ae5
7 changed files with 160 additions and 11 deletions
+6 -4
View File
@@ -5,9 +5,9 @@
e111dfe454a6e1edd80edc0c303119e045e7c88fe77b69a2745e58a022d0d317 ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh
bcfde96b98f4727a8c97d42a39b7c6ec925b1b89d17ef67cc58eb7bd67afbfae ocf/loxthelion/managed-container.d/applications/postgresql/notify.sh
957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548 ocf/loxthelion/managed-container.d/applications/postgresql/peer_config.sh
99bb3478931eced9471a8f054c14d152780aec73801449d1a6b2f09a4b656bec ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh
286d282565239bdb600563bac116575f3f56580e06f8f8df86c1a0fd0af96f07 ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh
c1d878dabf22ad3928d66a5542bda3c150e5e9b039926629e7c5fd3401d29be4 ocf/loxthelion/managed-container.d/applications/postgresql/postgres.sh
78b32fd6354bec61375ee73a31898059b21d51949a906cc6c7f23419a79670d6 ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh
d8534993c8e93f630bc0987301f5837f66e93c90d02b1063f46fdcdb559c0391 ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh
5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7 ocf/loxthelion/managed-container.d/applications/postgresql/promotion_state.py
b0ebad749ed736f08ec748a693f12e8dc2541430e0d3db75bbb6b2b7e517f84b ocf/loxthelion/managed-container.d/applications/postgresql/rejoin.sh
0d3017fdca92b2dc5dba406065bc269622ce5140e3fce4a959ca95ce4a78e755 ocf/loxthelion/managed-container.d/applications/postgresql/replication.sh
@@ -23,14 +23,16 @@ a42bb59a058ffeb59c1baa96e403856910bbbaea98f9104ff93daf4b5970c780 postgres-host/
957a5a72ed277d0216e070abb80733483be3d421b9fa488e27fdd75bb2919548 postgres-host/postgres-ha-peer-config.sh
deb09e3ae40629f43b262ba4f9a7ebf59a1dca3da9d8c94478f670aea28bd0e1 quadlet/postgres.container
04a125068677bbee6e71d313ca9d95d35faa32851b79de781cd7b9f466561445 quadlet/postgres.env.example
ac72a62381a686dbc7dcebf2f81bd3f96e84e83ba63ff956440546a8026037fb tests/run-all.sh
186c2350d1f4eaddd756fbc680319ce2bfb7ee3462cf1d421f549ba8ccca8d24 tests/run-all.sh
a373c0560391a2d8f63c3ba31609ab16b219d9b532b4ab9f8aea6dc45de10555 tests/test-demote-database-gate.sh
688eeb756c7f346e7e55acc555004ba12b48bc7c3d309073658d5e84a6e5e348 tests/test-demote-database-parameter.sh
22d3eb56e8cd3ae1a58e2616f44abd3818f79d7e5df55c1c158e36dca7c77f42 tests/test-demote-database-shadow.sh
5e5d4fdd2b6c7e0b8b146de7aad06c1f96b221a10fec525b00d40e0d85709fb0 tests/test-entrypoint-structure.sh
2faee72cf2fdcbfa667c3e1c2a7aabc053b96a9b8127ff7440ddb717fd2dc314 tests/test-notify-daemon-shadow.sh
9150004af86a819d7d1b5ae2b3fa9d4ddd0480f55babd094137052109864b057 tests/test-notify-target-shadow.sh
dd7ad3d1c06cac922c026b6bc30db5a5e66aa5d82948ebed983384c7b0bbba33 tests/test-promote-database-shadow.sh
ea2dc89483c5addc59cee0b8f070a08894c4803a204263507e2eaffb5416718e tests/test-promote-database-gate.sh
fc292a8612911423fdc8b9c5d187e2a19dd240d1bf35340d0b307888543beaa0 tests/test-promote-database-parameter.sh
7ea6ec5aaf2364f140f2987b4c2cf65ce5a132e284b6be22968de0cc4b63b3d5 tests/test-promote-database-shadow.sh
b83f0415c09f0be34c51722370e5b07c51b9a9756d305d3b3f50827f484aa4dd tests/test-proxy-client-actions.py
cc4482d229805c64d7df258a8a82984f244128d27fff4ea43b36caa004b56bef tests/test-proxy-client-parameters.py
c3b988d7b2b1b138bb2530df9172f96bb0f5edc2f1561f38f0055695f4126134 tests/test-proxy-protocol.sh
@@ -14,6 +14,7 @@ OCF_RESKEY_application_name_default=""
OCF_RESKEY_rejoin_allow_basebackup_default="false"
OCF_RESKEY_rejoin_basebackup_headroom_percent_default="50"
OCF_RESKEY_demote_database_gate_default="false"
OCF_RESKEY_promote_database_gate_default="false"
: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}"
: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}"
@@ -29,6 +30,7 @@ OCF_RESKEY_demote_database_gate_default="false"
: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}"
: "${OCF_RESKEY_rejoin_basebackup_headroom_percent:=${OCF_RESKEY_rejoin_basebackup_headroom_percent_default}}"
: "${OCF_RESKEY_demote_database_gate:=${OCF_RESKEY_demote_database_gate_default}}"
: "${OCF_RESKEY_promote_database_gate:=${OCF_RESKEY_promote_database_gate_default}}"
SERVICE_USER="${OCF_RESKEY_service_user}"
SERVICE_NAME="${OCF_RESKEY_service_name}"
@@ -44,7 +46,9 @@ APPLICATION_NAME="${OCF_RESKEY_application_name}"
REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}"
REJOIN_BASEBACKUP_HEADROOM_PERCENT="${OCF_RESKEY_rejoin_basebackup_headroom_percent}"
DEMOTE_DATABASE_GATE="${OCF_RESKEY_demote_database_gate}"
PROMOTE_DATABASE_GATE="${OCF_RESKEY_promote_database_gate}"
export MC_DEMOTE_DATABASE_GATE="${DEMOTE_DATABASE_GATE}"
export MC_PROMOTE_DATABASE_GATE="${PROMOTE_DATABASE_GATE}"
# Resolve PostgreSQL configuration from the managed service when Pacemaker has
# not explicitly supplied it. The application configuration remains the source
@@ -303,6 +307,15 @@ validate_all()
;;
esac
case "${PROMOTE_DATABASE_GATE}" in
true|false)
;;
*)
ocf_exit_reason "promote_database_gate must be true or false"
return "${OCF_ERR_CONFIGURED}"
;;
esac
case "${REJOIN_BASEBACKUP_HEADROOM_PERCENT}" in
""|*[!0-9]*)
ocf_exit_reason "rejoin_basebackup_headroom_percent must be a non-negative integer"
@@ -352,6 +365,11 @@ mc_promotion_metadata()
<shortdesc lang="en">Gate demotion on database checks</shortdesc>\
<content type="boolean" default="false"/>\
</parameter>\
<parameter name="promote_database_gate" unique="0" required="0">\
<longdesc lang="en">When true, reject native PostgreSQL promotion unless the managed-containerd promote.database preflight agrees that this node is the notified promotion target, is an eligible standby, and no reachable writable peer exists. When false, the daemon check remains shadow-only and native promotion fails open.</longdesc>\
<shortdesc lang="en">Gate promotion on database checks</shortdesc>\
<content type="boolean" default="false"/>\
</parameter>\
<parameter name="replication_peers" unique="0" required="0">\
<longdesc lang="en">Opt-in stable peer map: node=host:port,node=host:port. Include the local node; exclude the client VIP. IPv4 or DNS hosts only. Requires a configured per-node replication slot, supplied explicitly or resolved from PostgreSQL/the managed container environment, plus peer authentication and slots on every upstream. Uses native primary-only selection; demotion verifies final WAL then ready recovery before successor promotion.</longdesc>\
<shortdesc lang="en">Native replication peer map</shortdesc>\
@@ -29,6 +29,11 @@ mc_pg_promote_database_shadow()
if [ "${rc}" -ne 0 ]; then
ocf_log warn \
"managed-containerd promote.database shadow unavailable rc=${rc}"
if [ "${MC_PROMOTE_DATABASE_GATE:-false}" = true ]; then
return 1
fi
return 0
fi
@@ -50,10 +55,12 @@ if (
else
ocf_log err \
"managed-containerd promote.database shadow MISMATCH response=${response}"
if [ "${MC_PROMOTE_DATABASE_GATE:-false}" = true ]; then
return 1
fi
fi
# Shadow-only milestone: daemon disagreement must not alter the
# authoritative native promotion result yet.
return 0
}
@@ -117,9 +124,13 @@ managed_promote()
"Promoting PostgreSQL standby inside ${CONTAINER_NAME}"
# Independent managed-containerd promotion preflight.
# Shadow-only for now: disagreement is logged but does not block
# the authoritative native PostgreSQL promotion.
mc_pg_promote_database_shadow
# Shadow-only by default; when promote_database_gate=true,
# daemon disagreement or transport failure denies native promotion.
if ! mc_pg_promote_database_shadow; then
ocf_exit_reason \
"Promotion database gate rejected native PostgreSQL promotion"
return "${OCF_ERR_GENERIC}"
fi
if ! pg_command '
BINDIR="$(pg_config --bindir)" || exit 1
+2
View File
@@ -29,7 +29,9 @@ python3 ./tests/test-proxy-client-parameters.py
./tests/test-demote-database-shadow.sh
./tests/test-promote-database-shadow.sh
./tests/test-demote-database-gate.sh
./tests/test-promote-database-gate.sh
./tests/test-demote-database-parameter.sh
./tests/test-promote-database-parameter.sh
echo
echo ">>> Python syntax"
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh"
FAKE="$(mktemp)"
cleanup()
{
rm -f "$FAKE"
}
trap cleanup EXIT
ocf_log()
{
:
}
export OCF_RESOURCE_INSTANCE=postgresql-ha
export MC_PROXY_CLIENT="$FAKE"
# Valid disagreement.
cat > "$FAKE" <<'EOF'
import json
print(json.dumps({
"ocf_rc": 1,
"ready": False,
"mutation": False,
}))
EOF
chmod 0755 "$FAKE"
# shellcheck disable=SC1090
. "$FILE"
unset MC_PROMOTE_DATABASE_GATE
mc_pg_promote_database_shadow
export MC_PROMOTE_DATABASE_GATE=true
if mc_pg_promote_database_shadow; then
echo "FAIL: promotion mismatch was not gated"
exit 1
fi
# Transport failure.
cat > "$FAKE" <<'EOF'
raise SystemExit(10)
EOF
chmod 0755 "$FAKE"
unset MC_PROMOTE_DATABASE_GATE
mc_pg_promote_database_shadow
export MC_PROMOTE_DATABASE_GATE=true
if mc_pg_promote_database_shadow; then
echo "FAIL: promotion transport failure was not gated"
exit 1
fi
# Positive daemon agreement must still pass with the gate enabled.
cat > "$FAKE" <<'EOF'
import json
print(json.dumps({
"ocf_rc": 0,
"ready": True,
"mutation": False,
}))
EOF
chmod 0755 "$FAKE"
export MC_PROMOTE_DATABASE_GATE=true
mc_pg_promote_database_shadow
echo "PROMOTE_DATABASE_GATE_RUNTIME_TEST_PASS=1"
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
FILE="$ROOT/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh"
grep -q \
'OCF_RESKEY_promote_database_gate_default="false"' \
"$FILE"
grep -q \
': "${OCF_RESKEY_promote_database_gate:=' \
"$FILE"
grep -q \
'PROMOTE_DATABASE_GATE="${OCF_RESKEY_promote_database_gate}"' \
"$FILE"
grep -q \
'export MC_PROMOTE_DATABASE_GATE="${PROMOTE_DATABASE_GATE}"' \
"$FILE"
grep -q \
'promote_database_gate must be true or false' \
"$FILE"
grep -q \
'<parameter name="promote_database_gate"' \
"$FILE"
grep -q \
'<content type="boolean" default="false"/>' \
"$FILE"
echo "PROMOTE_DATABASE_PARAMETER_TEST_PASS=1"
+2 -2
View File
@@ -12,8 +12,8 @@ grep -q 'comparison=MATCH' "$FILE"
grep -q 'promote.database shadow MISMATCH' "$FILE"
SHADOW_LINE="$(
grep -n '^[[:space:]]*mc_pg_promote_database_shadow$' "$FILE" |
tail -1 |
grep -n 'if ! mc_pg_promote_database_shadow; then' "$FILE" |
head -1 |
cut -d: -f1
)"