Initial managed-container HA resource agent baseline
This commit is contained in:
Lyncoln
@@ -0,0 +1,88 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
pg_replication_exec()
|
||||
{
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
if [ -z "${POSTGRES_REPLICATION_PASSWORD:-}" ]; then
|
||||
printf "%s\n" \
|
||||
"POSTGRES_REPLICATION_PASSWORD is missing from container environment" \
|
||||
>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD}"
|
||||
|
||||
exec "$@"
|
||||
' sh "$@"
|
||||
}
|
||||
|
||||
ensure_rewind_privileges()
|
||||
{
|
||||
local repl_user
|
||||
|
||||
repl_user="$(
|
||||
pg_command '
|
||||
printf "%s" "${POSTGRES_REPLICATION_USER:-}"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to read POSTGRES_REPLICATION_USER from container"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ -z "${repl_user}" ]; then
|
||||
ocf_exit_reason \
|
||||
"POSTGRES_REPLICATION_USER is empty"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
case "${repl_user}" in
|
||||
*[!A-Za-z0-9_]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid replication role name: ${repl_user}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
ocf_log info \
|
||||
"Ensuring pg_rewind source privileges for role ${repl_user}"
|
||||
|
||||
if ! pg_command "
|
||||
psql \
|
||||
-U '${DB_USER}' \
|
||||
-d postgres \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-c \"
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean)
|
||||
TO ${repl_user};
|
||||
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_stat_file(text, boolean)
|
||||
TO ${repl_user};
|
||||
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_read_binary_file(text)
|
||||
TO ${repl_user};
|
||||
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_read_binary_file(
|
||||
text,
|
||||
bigint,
|
||||
bigint,
|
||||
boolean
|
||||
)
|
||||
TO ${repl_user};
|
||||
\"
|
||||
" >/dev/null
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed ensuring pg_rewind privileges for ${repl_user}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
node_to_addr()
|
||||
{
|
||||
local target="${1:-}"
|
||||
|
||||
[ -n "${target}" ] || return 1
|
||||
|
||||
corosync-cmapctl 2>/dev/null |
|
||||
awk -v target="${target}" '
|
||||
/nodelist\.node\.[0-9]+\.name / {
|
||||
key=$1
|
||||
sub(/\.name$/, "", key)
|
||||
names[key]=$NF
|
||||
}
|
||||
|
||||
/nodelist\.node\.[0-9]+\.ring0_addr / {
|
||||
key=$1
|
||||
sub(/\.ring0_addr$/, "", key)
|
||||
addrs[key]=$NF
|
||||
}
|
||||
|
||||
END {
|
||||
for (key in names) {
|
||||
if (names[key] == target && addrs[key] != "") {
|
||||
print addrs[key]
|
||||
exit 0
|
||||
}
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
'
|
||||
}
|
||||
|
||||
# Read the final checkpoint only after pg_ctl has confirmed clean primary shutdown.
|
||||
mc_pg_shutdown_checkpoint()
|
||||
{
|
||||
local control summary
|
||||
control="$(mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" --env LC_ALL=C \
|
||||
"${CONTAINER_NAME}" sh -c \
|
||||
'exec "$(pg_config --bindir)/pg_controldata" "$PGDATA"')" || return 1
|
||||
summary="$(printf '%s\n' "${control}" | awk -F ': *' '
|
||||
/^Database cluster state:/ { state=$2; ns++ }
|
||||
/^Database system identifier:/ { sid=$2; ni++ }
|
||||
/^Latest checkpoint location:/ { checkpoint=$2; nc++ }
|
||||
/^Latest checkpoint.s TimeLineID:/ { timeline=$2; nt++ }
|
||||
END {
|
||||
if (ns != 1 || ni != 1 || nc != 1 || nt != 1 || state != "shut down") exit 1;
|
||||
print sid "|" timeline "|" checkpoint;
|
||||
}')" || return 1
|
||||
[[ "${summary}" =~ ^[0-9]+\|[1-9][0-9]*\|[0-9A-F]+/[0-9A-F]+$ ]] || return 1
|
||||
printf '%s\n' "${summary}"
|
||||
}
|
||||
|
||||
# A checkpoint LSN is its record START. Replay must be strictly beyond it to
|
||||
# establish that the complete shutdown checkpoint has been replayed.
|
||||
mc_pg_wait_shutdown_replay()
|
||||
{
|
||||
local host="$1" port="$2" observation="$3" deadline="$4"
|
||||
local system timeline checkpoint response ready identity rest got_system got_timeline
|
||||
local remaining duration
|
||||
IFS='|' read -r system timeline checkpoint <<< "${observation}"
|
||||
[[ "${observation}" =~ ^[0-9]+\|[1-9][0-9]*\|[0-9A-F]+/[0-9A-F]+$ ]] || return 1
|
||||
while (( SECONDS < deadline )); do
|
||||
remaining=$((deadline - SECONDS))
|
||||
duration=3
|
||||
(( remaining < duration )) && duration=${remaining}
|
||||
response="$(mc_runtime_exec_bounded "${duration}s" --user "${PG_OS_USER}" \
|
||||
--env PGCONNECT_TIMEOUT=2 --env 'PGOPTIONS=-c statement_timeout=1000' \
|
||||
"${CONTAINER_NAME}" sh -c '
|
||||
export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD:?}"
|
||||
psql -X -w -v ON_ERROR_STOP=1 -h "$1" -p "$2" -U "$3" -d postgres -Atqc \
|
||||
"SELECT CASE WHEN pg_is_in_recovery() THEN
|
||||
NOT pg_is_wal_replay_paused() AND
|
||||
pg_last_wal_replay_lsn() > '\''$4'\''::pg_lsn
|
||||
ELSE false END;" || exit 1
|
||||
exec psql -X -w -v ON_ERROR_STOP=1 -h "$1" -p "$2" -U "$3" \
|
||||
-d replication=true -Atqc IDENTIFY_SYSTEM
|
||||
' sh "${host}" "${port}" "${REPLICATION_USER}" "${checkpoint}" 2>/dev/null)" || response=''
|
||||
ready="${response%%$'\n'*}"
|
||||
identity="${response#*$'\n'}"
|
||||
IFS='|' read -r got_system got_timeline rest <<< "${identity}"
|
||||
if [ "${ready}" = t ] && [ "${got_system}" = "${system}" ] &&
|
||||
[ "${got_timeline}" = "${timeline}" ]; then
|
||||
return 0
|
||||
fi
|
||||
(( SECONDS < deadline )) && sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
managed_demote()
|
||||
{
|
||||
local monitor_rc
|
||||
local target_node target_addr
|
||||
local local_node slot_name target_port
|
||||
local role receiver_status i checkpoint handoff_deadline
|
||||
local action_started=${SECONDS} action_budget=${OCF_RESKEY_CRM_meta_timeout:-60000}
|
||||
[[ "${action_budget}" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}"
|
||||
action_budget=$((10#${action_budget} / 1000))
|
||||
if (( action_budget < 15 )); then
|
||||
ocf_exit_reason "Demotion timeout is too short for a verified shutdown-WAL handoff"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
;;
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL is already unpromoted"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
"${OCF_NOT_RUNNING}")
|
||||
ocf_exit_reason \
|
||||
"Cannot complete demotion because PostgreSQL is stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
target_node="$(demote_target)" || {
|
||||
ocf_exit_reason \
|
||||
"No fresh valid Pacemaker pre-demote target is available"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
target_addr="$(node_to_addr "${target_node}")" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to resolve Pacemaker promotion target ${target_node} through Corosync"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
target_port="${REJOIN_SOURCE_PORT:-5432}"
|
||||
local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}"
|
||||
slot_name="ha_$(printf '%s' "${local_node}" |
|
||||
tr '[:upper:]-' '[:lower:]_')"
|
||||
|
||||
ocf_log info \
|
||||
"Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})"
|
||||
|
||||
if ! candidate_is_standby "${target_addr}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not a reachable PostgreSQL standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! candidate_caught_up "${target_node}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then
|
||||
ocf_exit_reason \
|
||||
"Unable to ensure physical replication slot ${slot_name} on ${target_node}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Demotion safety checks passed; stopping PostgreSQL primary"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"pg_ctl stop failed during controlled demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL remained running after pg_ctl stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! service_running || ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service/container stopped unexpectedly during demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
# Keep at most 20 seconds for the handoff, reserving 10 seconds of the
|
||||
# action budget for subsequent work. Pacemaker remains the overall timeout.
|
||||
checkpoint="$(mc_pg_shutdown_checkpoint)" || {
|
||||
ocf_exit_reason "Unable to read a valid clean-shutdown checkpoint; PostgreSQL remains stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
handoff_deadline=$((action_started + action_budget - 10))
|
||||
(( handoff_deadline > SECONDS + 20 )) && handoff_deadline=$((SECONDS + 20))
|
||||
if ! mc_pg_wait_shutdown_replay "${target_addr}" "${target_port}" "${checkpoint}" "${handoff_deadline}"; then
|
||||
ocf_exit_reason "Successor has not verified final shutdown-checkpoint replay; PostgreSQL remains stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Configuring ${local_node} as cascading standby of ${target_node}"
|
||||
|
||||
if ! mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
--env "RA_PRIMARY_HOST=${target_addr}" \
|
||||
--env "RA_PRIMARY_PORT=${target_port}" \
|
||||
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
||||
--env "RA_REPLICATION_SLOT=${slot_name}" \
|
||||
--env "RA_APPLICATION_NAME=${local_node}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
set -e
|
||||
|
||||
sed -i \
|
||||
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \
|
||||
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \
|
||||
"$PGDATA/postgresql.auto.conf"
|
||||
|
||||
printf "%s
|
||||
" \
|
||||
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \
|
||||
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \
|
||||
>> "$PGDATA/postgresql.auto.conf"
|
||||
|
||||
touch "$PGDATA/standby.signal"
|
||||
|
||||
chmod 600 \
|
||||
"$PGDATA/postgresql.auto.conf" \
|
||||
"$PGDATA/standby.signal"
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to configure PostgreSQL as standby of ${target_node}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL as cascading standby of ${target_node}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL failed to start as cascading standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null || true)"
|
||||
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
||||
|
||||
if [ "${role}" = "t" ] &&
|
||||
[ "${receiver_status}" = "streaming" ]
|
||||
then
|
||||
ocf_log info \
|
||||
"Controlled demotion complete: ${local_node} is streaming from ${target_node}"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL started after demotion but did not become a streaming standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
# Optional native peer lifecycle; defaults continue to use the legacy path.
|
||||
mc_pg_peer_validate()
|
||||
{
|
||||
[ -n "${OCF_RESKEY_replication_peers:-}" ] || return 0
|
||||
mc_peer_conninfo "${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" >/dev/null || {
|
||||
ocf_exit_reason "Invalid replication_peers map or local identity"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
}
|
||||
# The container may legitimately not exist yet during validate/start.
|
||||
# Validate an explicitly supplied slot now, but defer runtime discovery
|
||||
# until an operation actually requires the slot.
|
||||
if [ -n "${REPLICATION_SLOT}" ]; then
|
||||
case "${REPLICATION_SLOT}" in ''|*[!a-z0-9_]*)
|
||||
ocf_exit_reason "Configured replication slot is invalid; expected lowercase letters, digits, and underscore"
|
||||
return "${OCF_ERR_CONFIGURED}";; esac
|
||||
[ ${#REPLICATION_SLOT} -le 63 ] || return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
}
|
||||
|
||||
mc_pg_write_peer_recovery()
|
||||
{
|
||||
local code
|
||||
# Same canonical POSIX helper is mounted into bootstrap; no duplicated writer.
|
||||
code="$(cat "${MC_APPLICATION_DIR}/peer_config.sh")" || return 1
|
||||
mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" sh -c \
|
||||
"${code}
|
||||
mc_peer_write_recovery \"\$PGDATA\" \"\$1\" \"\$2\" \"\$3\" \"\$4\" \"\$5\"" sh \
|
||||
"${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" \
|
||||
"${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_SLOT}"
|
||||
}
|
||||
|
||||
mc_pg_native_demote()
|
||||
(
|
||||
local mc_native_started=${SECONDS}
|
||||
local mc_native_ms=${OCF_RESKEY_CRM_meta_timeout:-60000}
|
||||
[[ "$mc_native_ms" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}"
|
||||
local mc_native_deadline=$((mc_native_started + 10#$mc_native_ms / 1000 - 5))
|
||||
local mc_original_bounded
|
||||
mc_original_bounded=$(declare -f mc_runtime_exec_bounded)
|
||||
eval "${mc_original_bounded/mc_runtime_exec_bounded/mc_native_original_bounded}"
|
||||
mc_runtime_exec_bounded() {
|
||||
local requested=$1 remaining=$((mc_native_deadline - SECONDS))
|
||||
shift
|
||||
(( remaining > 0 )) || return 124
|
||||
# Existing bounded call sites use integer seconds.
|
||||
[[ "$requested" =~ ^[0-9]+s$ ]] || return 124
|
||||
(( ${requested%s} < remaining )) && remaining=${requested%s}
|
||||
mc_native_original_bounded "${remaining}s" "$@"
|
||||
}
|
||||
mc_runtime_exec() {
|
||||
local remaining=$((mc_native_deadline - SECONDS))
|
||||
(( remaining > 0 )) || return 124
|
||||
mc_native_original_bounded "${remaining}s" "$@"
|
||||
}
|
||||
local monitor_rc
|
||||
local target_node target_addr
|
||||
local local_node slot_name target_port
|
||||
local role receiver_status i checkpoint handoff_deadline
|
||||
local self_restart=false
|
||||
local action_started=${mc_native_started} action_budget=${OCF_RESKEY_CRM_meta_timeout:-60000}
|
||||
[[ "${action_budget}" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}"
|
||||
action_budget=$((10#${action_budget} / 1000))
|
||||
if (( action_budget < 15 )); then
|
||||
ocf_exit_reason "Demotion timeout is too short for a verified shutdown-WAL handoff"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
mc_pg_peer_validate || return "$?"
|
||||
mc_pg_require_replication_slot || return "$?"
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
;;
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL is already unpromoted"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
"${OCF_NOT_RUNNING}")
|
||||
ocf_exit_reason \
|
||||
"Cannot complete demotion because PostgreSQL is stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}"
|
||||
slot_name=${REPLICATION_SLOT}
|
||||
local native_deadline=$((action_started + action_budget - 5))
|
||||
|
||||
target_node="$(mc_pg_native_demote_target)" || {
|
||||
ocf_exit_reason \
|
||||
"No fresh valid Pacemaker pre-demote target is available"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "$(printf '%s' "${target_node}" | tr '[:upper:]' '[:lower:]')" = \
|
||||
"$(printf '%s' "${local_node}" | tr '[:upper:]' '[:lower:]')" ]; then
|
||||
self_restart=true
|
||||
ocf_log info \
|
||||
"Pacemaker requested a self-restart demotion of ${local_node}; performing clean local demotion without remote handoff"
|
||||
else
|
||||
local peer_pair
|
||||
peer_pair=$(mc_peer_endpoints "${OCF_RESKEY_replication_peers}" "${local_node}" "${target_node}") || {
|
||||
ocf_exit_reason "Successor is missing from the validated peer map"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
}
|
||||
target_addr=${peer_pair%|*}
|
||||
target_port=${peer_pair#*|}
|
||||
# Existing candidate/slot SQL helpers use REJOIN_SOURCE_PORT.
|
||||
local REJOIN_SOURCE_PORT=${target_port}
|
||||
|
||||
ocf_log info \
|
||||
"Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})"
|
||||
|
||||
if ! candidate_is_standby "${target_addr}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not a reachable PostgreSQL standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! candidate_caught_up "${target_node}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then
|
||||
ocf_exit_reason \
|
||||
"Unable to ensure physical replication slot ${slot_name} on ${target_node}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Demotion safety checks passed; stopping PostgreSQL primary"
|
||||
fi
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"pg_ctl stop failed during controlled demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL remained running after pg_ctl stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! service_running || ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service/container stopped unexpectedly during demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
checkpoint="$(mc_pg_shutdown_checkpoint)" || {
|
||||
ocf_exit_reason "Unable to read a valid clean-shutdown checkpoint; PostgreSQL remains stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${self_restart}" != true ]; then
|
||||
# Keep at most 20 seconds for the handoff, reserving 10 seconds of the
|
||||
# action budget for subsequent work. Pacemaker remains the overall timeout.
|
||||
handoff_deadline=$((action_started + action_budget - 10))
|
||||
(( handoff_deadline > SECONDS + 20 )) && handoff_deadline=$((SECONDS + 20))
|
||||
if ! mc_pg_wait_shutdown_replay "${target_addr}" "${target_port}" "${checkpoint}" "${handoff_deadline}"; then
|
||||
ocf_exit_reason "Successor has not verified final shutdown-checkpoint replay; PostgreSQL remains stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
else
|
||||
ocf_log info \
|
||||
"Clean shutdown checkpoint verified locally for Pacemaker self-restart demotion"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Configuring ${local_node} with stable native replication peers"
|
||||
|
||||
if ! mc_pg_write_peer_recovery; then
|
||||
ocf_exit_reason "Failed to write native peer recovery configuration"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL in recovery with native primary selection"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL failed to start as cascading standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
while (( SECONDS < native_deadline )); do
|
||||
if pg_ready && [ "$(pg_role 2>/dev/null)" = t ]; then
|
||||
local state
|
||||
state=$(mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" psql -X -w -v ON_ERROR_STOP=1 -U "${DB_USER}" -d postgres -Atqc "SELECT system_identifier::text || '|' || (NOT pg_is_wal_replay_paused())::text FROM pg_control_system()" 2>/dev/null) || state=''
|
||||
if [ "${state}" = "${checkpoint%%|*}|true" ]; then
|
||||
if [ "${self_restart}" = true ]; then
|
||||
ocf_log info "Pacemaker self-restart demotion complete: clean former primary is ready in recovery for possible re-promotion"
|
||||
else
|
||||
ocf_log info "Native peer demotion complete: final WAL verified; old primary ready in recovery awaiting successor promotion"
|
||||
fi
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
ocf_exit_reason "Old primary did not become verified ready recovery within demotion budget"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
)
|
||||
@@ -0,0 +1,120 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
mc_pg_demote_notification_target()
|
||||
{
|
||||
local allow_self="${1:-false}"
|
||||
local state_file
|
||||
local type operation promote_uname demote_uname local_node
|
||||
local now mtime age
|
||||
local promote_norm local_norm demote_norm
|
||||
|
||||
state_file="/run/loxthelion/${OCF_RESOURCE_INSTANCE:-managed-container}.notify"
|
||||
local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}"
|
||||
|
||||
[ -r "${state_file}" ] || return 1
|
||||
|
||||
now="$(date +%s)" || return 1
|
||||
mtime="$(stat -c %Y "${state_file}" 2>/dev/null)" || return 1
|
||||
age=$((now - mtime))
|
||||
|
||||
[ "${age}" -ge 0 ] || return 1
|
||||
[ "${age}" -le 120 ] || return 1
|
||||
|
||||
type="$(sed -n 's/^type=//p' "${state_file}" | head -n1)"
|
||||
operation="$(sed -n 's/^operation=//p' "${state_file}" | head -n1)"
|
||||
promote_uname="$(sed -n 's/^promote_uname=//p' "${state_file}" | head -n1)"
|
||||
demote_uname="$(sed -n 's/^demote_uname=//p' "${state_file}" | head -n1)"
|
||||
|
||||
[ "${type}" = "pre" ] || return 1
|
||||
[ "${operation}" = "demote" ] || return 1
|
||||
|
||||
local_norm="$(printf '%s' "${local_node}" | tr '[:upper:]' '[:lower:]')"
|
||||
demote_norm="$(printf '%s' "${demote_uname}" | tr '[:upper:]' '[:lower:]')"
|
||||
promote_norm="$(printf '%s' "${promote_uname}" | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
[ "${demote_norm}" = "${local_norm}" ] || return 1
|
||||
[ -n "${promote_uname}" ] || return 1
|
||||
|
||||
case "${promote_uname}" in
|
||||
*" "*) return 1 ;;
|
||||
esac
|
||||
|
||||
if [ "${allow_self}" != true ] && [ "${promote_norm}" = "${local_norm}" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "${promote_uname}"
|
||||
}
|
||||
|
||||
# Legacy callers retain the historical requirement for a distinct successor.
|
||||
demote_target()
|
||||
{
|
||||
mc_pg_demote_notification_target false
|
||||
}
|
||||
|
||||
# Native-peer demotion can distinguish a true handoff from Pacemaker's
|
||||
# demote/restart/re-promote of the same node after a resource-definition change.
|
||||
mc_pg_native_demote_target()
|
||||
{
|
||||
mc_pg_demote_notification_target true
|
||||
}
|
||||
|
||||
managed_notify()
|
||||
{
|
||||
local state_dir state_file
|
||||
|
||||
state_dir="/run/loxthelion"
|
||||
state_file="${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify"
|
||||
|
||||
ocf_log info \
|
||||
"notify: type=${OCF_RESKEY_CRM_meta_notify_type:-unset} operation=${OCF_RESKEY_CRM_meta_notify_operation:-unset}"
|
||||
|
||||
ocf_log info \
|
||||
"notify: promote_uname=${OCF_RESKEY_CRM_meta_notify_promote_uname:-none} demote_uname=${OCF_RESKEY_CRM_meta_notify_demote_uname:-none}"
|
||||
|
||||
ocf_log info \
|
||||
"notify: promoted_uname=${OCF_RESKEY_CRM_meta_notify_promoted_uname:-none} unpromoted_uname=${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-none}"
|
||||
|
||||
if ! mkdir -p -m 0755 "${state_dir}"; then
|
||||
ocf_exit_reason \
|
||||
"Unable to create notification state directory ${state_dir}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! {
|
||||
printf 'timestamp=%s\n' "$(date --iso-8601=seconds)"
|
||||
printf 'node=%s\n' "$(hostname)"
|
||||
printf 'type=%s\n' "${OCF_RESKEY_CRM_meta_notify_type:-}"
|
||||
printf 'operation=%s\n' "${OCF_RESKEY_CRM_meta_notify_operation:-}"
|
||||
printf 'promote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}"
|
||||
printf 'demote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}"
|
||||
printf 'promoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}"
|
||||
printf 'unpromoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}"
|
||||
} > "${state_file}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to write notification state ${state_file}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
chmod 0600 "${state_file}" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to secure notification state ${state_file}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
{
|
||||
printf '%s node=%s type=%s operation=%s promote=[%s] demote=[%s] promoted=[%s] unpromoted=[%s]\n' \
|
||||
"$(date --iso-8601=seconds)" \
|
||||
"$(hostname)" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_type:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_operation:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" \
|
||||
>> "${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify.log"
|
||||
} || true
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
# POSIX-compatible shared normalizer. Also mounted beside the bootstrap script.
|
||||
# Inputs are data, never evaluated shell. IPv4/DNS endpoints only in API 1.
|
||||
mc_peer_endpoints()
|
||||
{
|
||||
LC_ALL=C awk -v peers="$1" -v self="$2" -v select_node="${3:-}" '
|
||||
BEGIN {
|
||||
n=split(peers, entries, ","); self=tolower(self); select_node=tolower(select_node);
|
||||
if(n<2 || n>32 || self=="") exit 1;
|
||||
for(i=1;i<=n;i++) {
|
||||
if(split(entries[i],pair,"=")!=2 || pair[1]!~/^[A-Za-z0-9_][A-Za-z0-9_.-]*$/) exit 1;
|
||||
node=tolower(pair[1]); if(seen[node]++) exit 1;
|
||||
if(split(pair[2],addr,":")!=2 || addr[1]!~/^[A-Za-z0-9][A-Za-z0-9.-]*$/ || addr[2]!~/^[0-9]+$/ || addr[2]+0<1 || addr[2]+0>65535) exit 1;
|
||||
endpoint=tolower(addr[1]) ":" (addr[2]+0); if(addresses[endpoint]++) exit 1;
|
||||
if(node==self) { own++; continue; }
|
||||
if(select_node!="" && node!=select_node) continue;
|
||||
hosts=hosts (count?",":"") addr[1]; ports=ports (count?",":"") (addr[2]+0); count++;
|
||||
}
|
||||
if(own!=1 || count<1 || (select_node!="" && count!=1)) exit 1;
|
||||
print hosts "|" ports;
|
||||
}'
|
||||
}
|
||||
|
||||
mc_peer_conninfo()
|
||||
{
|
||||
# peers, self, replication user, application name
|
||||
case "$3" in ''|*[!a-zA-Z0-9_.-]*) return 1;; esac
|
||||
case "$4" in ''|*[!a-zA-Z0-9_.-]*) return 1;; esac
|
||||
mc_peer_pair=$(mc_peer_endpoints "$1" "$2") || return 1
|
||||
printf 'host=%s port=%s user=%s application_name=%s target_session_attrs=primary connect_timeout=2\n' \
|
||||
"${mc_peer_pair%|*}" "${mc_peer_pair#*|}" "$3" "$4"
|
||||
}
|
||||
|
||||
mc_peer_write_recovery()
|
||||
(
|
||||
# PGDATA, peers, self, user, application, slot. No credentials in output.
|
||||
set -eu
|
||||
data=$1
|
||||
conn=$(mc_peer_conninfo "$2" "$3" "$4" "$5") || exit 1
|
||||
case "$6" in ''|*[!a-z0-9_]*) exit 1;; esac
|
||||
[ ${#6} -le 63 ] || exit 1
|
||||
[ -d "$data" ] && [ -f "$data/postgresql.auto.conf" ] && [ ! -L "$data/postgresql.auto.conf" ] || exit 1
|
||||
[ ! -L "$data/standby.signal" ] || exit 1
|
||||
umask 077
|
||||
tmp=$(mktemp "$data/.peer-recovery.XXXXXXXX")
|
||||
trap 'rm -f "$tmp"' EXIT HUP INT TERM
|
||||
sed -e '/^[[:space:]]*primary_conninfo[[:space:]]*=/d' \
|
||||
-e '/^[[:space:]]*primary_slot_name[[:space:]]*=/d' "$data/postgresql.auto.conf" > "$tmp"
|
||||
printf "primary_conninfo = '%s'\nprimary_slot_name = '%s'\n" "$conn" "$6" >> "$tmp"
|
||||
chmod 600 "$tmp"
|
||||
mv -f "$tmp" "$data/postgresql.auto.conf"
|
||||
[ ! -L "$data/standby.signal" ] || exit 1
|
||||
touch "$data/standby.signal"
|
||||
chmod 600 "$data/standby.signal"
|
||||
)
|
||||
@@ -0,0 +1,391 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
OCF_RESKEY_service_user_default="david"
|
||||
OCF_RESKEY_service_name_default="postgres.service"
|
||||
OCF_RESKEY_container_name_default="postgres"
|
||||
OCF_RESKEY_pg_os_user_default="postgres"
|
||||
OCF_RESKEY_db_user_default="david"
|
||||
OCF_RESKEY_db_name_default="david"
|
||||
OCF_RESKEY_rejoin_source_host_default=""
|
||||
OCF_RESKEY_rejoin_source_port_default="5432"
|
||||
OCF_RESKEY_replication_user_default="replicator"
|
||||
OCF_RESKEY_replication_slot_default=""
|
||||
OCF_RESKEY_application_name_default=""
|
||||
OCF_RESKEY_rejoin_allow_basebackup_default="false"
|
||||
OCF_RESKEY_rejoin_basebackup_headroom_percent_default="50"
|
||||
|
||||
: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}"
|
||||
: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}"
|
||||
: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}"
|
||||
: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}"
|
||||
: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}"
|
||||
: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}"
|
||||
: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}"
|
||||
: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}"
|
||||
: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}"
|
||||
: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}"
|
||||
: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}"
|
||||
: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}"
|
||||
: "${OCF_RESKEY_rejoin_basebackup_headroom_percent:=${OCF_RESKEY_rejoin_basebackup_headroom_percent_default}}"
|
||||
|
||||
SERVICE_USER="${OCF_RESKEY_service_user}"
|
||||
SERVICE_NAME="${OCF_RESKEY_service_name}"
|
||||
CONTAINER_NAME="${OCF_RESKEY_container_name}"
|
||||
PG_OS_USER="${OCF_RESKEY_pg_os_user}"
|
||||
DB_USER="${OCF_RESKEY_db_user}"
|
||||
DB_NAME="${OCF_RESKEY_db_name}"
|
||||
REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}"
|
||||
REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}"
|
||||
REPLICATION_USER="${OCF_RESKEY_replication_user}"
|
||||
REPLICATION_SLOT="${OCF_RESKEY_replication_slot}"
|
||||
APPLICATION_NAME="${OCF_RESKEY_application_name}"
|
||||
REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}"
|
||||
REJOIN_BASEBACKUP_HEADROOM_PERCENT="${OCF_RESKEY_rejoin_basebackup_headroom_percent}"
|
||||
|
||||
# Resolve PostgreSQL configuration from the managed service when Pacemaker has
|
||||
# not explicitly supplied it. The application configuration remains the source
|
||||
# of truth; no slot naming convention is inferred.
|
||||
mc_pg_resolve_replication_slot()
|
||||
{
|
||||
local resolved
|
||||
|
||||
if [ -n "${REPLICATION_SLOT}" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Prefer PostgreSQL's effective setting when the server is already up.
|
||||
# This asks PostgreSQL itself rather than reconstructing configuration.
|
||||
if declare -F pg_running >/dev/null 2>&1 && pg_running; then
|
||||
resolved="$(
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql -X -U "${DB_USER}" -d "${DB_NAME}" -Atqc \
|
||||
'SHOW primary_slot_name;' 2>/dev/null
|
||||
)" || resolved=
|
||||
if [ -n "${resolved}" ]; then
|
||||
REPLICATION_SLOT=${resolved}
|
||||
OCF_RESKEY_replication_slot=${resolved}
|
||||
ocf_log debug "Resolved PostgreSQL replication slot from PostgreSQL effective configuration"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# When PostgreSQL is stopped but its maintenance container is running,
|
||||
# obtain the exact value injected into the managed container. Runtime
|
||||
# adapters own this lookup; the application plugin does not parse Quadlets.
|
||||
if declare -F mc_runtime_environment >/dev/null 2>&1; then
|
||||
resolved="$(
|
||||
mc_runtime_environment \
|
||||
"${CONTAINER_NAME}" \
|
||||
POSTGRES_REPLICATION_SLOT
|
||||
)" || resolved=
|
||||
if [ -n "${resolved}" ]; then
|
||||
REPLICATION_SLOT=${resolved}
|
||||
OCF_RESKEY_replication_slot=${resolved}
|
||||
ocf_log debug "Resolved PostgreSQL replication slot from managed container environment"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
mc_pg_require_replication_slot()
|
||||
{
|
||||
mc_pg_resolve_replication_slot || {
|
||||
ocf_exit_reason "PostgreSQL replication slot is required but could not be resolved from replication_slot, PostgreSQL, or the managed container environment"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
}
|
||||
|
||||
case "${REPLICATION_SLOT}" in
|
||||
''|*[!a-z0-9_]*)
|
||||
ocf_exit_reason "Resolved replication slot is invalid; expected lowercase letters, digits, and underscore"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ ${#REPLICATION_SLOT} -gt 63 ]; then
|
||||
ocf_exit_reason "Resolved replication slot exceeds PostgreSQL's 63-character identifier limit"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
if [ -z "${APPLICATION_NAME}" ]; then
|
||||
APPLICATION_NAME="$(hostname -s)"
|
||||
fi
|
||||
|
||||
meta_data()
|
||||
{
|
||||
cat <<EOF_META
|
||||
<?xml version="1.0"?>
|
||||
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
|
||||
<resource-agent name="managed-container" version="0.4">
|
||||
<version>1.0</version>
|
||||
|
||||
<longdesc lang="en">
|
||||
Manage an application inside a persistent rootless container.
|
||||
|
||||
Version 0.4 implements role-aware monitoring, promotion, and demotion.
|
||||
The systemd user Quadlet owns the container lifecycle while pg_ctl owns
|
||||
the PostgreSQL lifecycle inside the container.
|
||||
</longdesc>
|
||||
|
||||
<shortdesc lang="en">
|
||||
Manage PostgreSQL independently inside a persistent container
|
||||
</shortdesc>
|
||||
|
||||
<parameters>
|
||||
|
||||
<parameter name="service_user" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
Linux user owning the rootless systemd/Podman service.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Rootless service owner</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_service_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="service_name" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
systemd user service generated by Quadlet.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Quadlet service name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_service_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="container_name" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
Podman container containing PostgreSQL.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Container name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_container_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="pg_os_user" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
Operating-system user running PostgreSQL inside the container.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">PostgreSQL OS user</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_pg_os_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="db_user" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL database user used for health queries.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Database user</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_db_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="db_name" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL database used for health queries.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Database name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_db_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
|
||||
<parameter name="rejoin_source_host" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL primary host used as the source for rejoin operations.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Rejoin source host</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_rejoin_source_host_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="rejoin_source_port" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL port on the rejoin source.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Rejoin source port</shortdesc>
|
||||
<content type="integer" default="${OCF_RESKEY_rejoin_source_port_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="replication_user" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL role used for replication and pg_rewind.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Replication user</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_replication_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="replication_slot" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
Physical replication slot used by this standby. If unset, the agent
|
||||
resolves the effective PostgreSQL primary_slot_name when available,
|
||||
otherwise POSTGRES_REPLICATION_SLOT from the running managed container.
|
||||
No slot name is derived or guessed.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Physical replication slot</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_replication_slot_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="application_name" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL replication application_name. Defaults to the local hostname.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Replication application name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_application_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="rejoin_allow_basebackup" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
Allow rejoin to fall back to pg_basebackup when pg_rewind cannot
|
||||
repair the local PostgreSQL data directory. The old PGDATA is
|
||||
preserved before replacement. This must only be enabled when the
|
||||
operator has established that this node may safely discard its
|
||||
divergent PostgreSQL history.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Allow pg_basebackup rejoin fallback</shortdesc>
|
||||
<content type="boolean" default="${OCF_RESKEY_rejoin_allow_basebackup_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="rejoin_basebackup_headroom_percent" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
Additional free-space headroom, as a percentage of the estimated
|
||||
source database payload, required before pg_basebackup fallback.
|
||||
The old PGDATA is renamed in place and therefore is not counted as
|
||||
another free-space copy. A minimum 1 GiB headroom is always required.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">pg_basebackup free-space headroom percent</shortdesc>
|
||||
<content type="integer" default="${OCF_RESKEY_rejoin_basebackup_headroom_percent_default}"/>
|
||||
</parameter>
|
||||
|
||||
</parameters>
|
||||
|
||||
<actions>
|
||||
<action name="start" timeout="60s"/>
|
||||
<action name="stop" timeout="60s"/>
|
||||
<action name="promote" timeout="60s"/>
|
||||
<action name="demote" timeout="60s"/>
|
||||
<action name="notify" timeout="20s"/>
|
||||
<action name="monitor" timeout="20s" interval="10s"/>
|
||||
<action name="validate-all" timeout="20s"/>
|
||||
<action name="meta-data" timeout="5s"/>
|
||||
</actions>
|
||||
</resource-agent>
|
||||
EOF_META
|
||||
}
|
||||
|
||||
validate_all()
|
||||
{
|
||||
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
|
||||
ocf_exit_reason "Service user '${SERVICE_USER}' does not exist"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
for cmd in runuser systemctl podman getent; do
|
||||
if ! command -v "${cmd}" >/dev/null 2>&1; then
|
||||
ocf_exit_reason "Required command '${cmd}' is not installed"
|
||||
return "${OCF_ERR_INSTALLED}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ! -d "/run/user/$(service_uid)" ]; then
|
||||
ocf_exit_reason \
|
||||
"Runtime directory /run/user/$(service_uid) does not exist"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
case "${REJOIN_BASEBACKUP_HEADROOM_PERCENT}" in
|
||||
""|*[!0-9]*)
|
||||
ocf_exit_reason "rejoin_basebackup_headroom_percent must be a non-negative integer"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
if [ "${REJOIN_BASEBACKUP_HEADROOM_PERCENT}" -gt 500 ]; then
|
||||
ocf_exit_reason "rejoin_basebackup_headroom_percent must be between 0 and 500"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
if ! mc_lifecycle_cat "${SERVICE_NAME}" >/dev/null 2>&1
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
local podman_error
|
||||
|
||||
podman_error="$(
|
||||
mc_runtime_info 2>&1 >/dev/null
|
||||
)"
|
||||
local podman_rc=$?
|
||||
|
||||
if [ "${podman_rc}" -ne 0 ]; then
|
||||
ocf_exit_reason \
|
||||
"Rootless Podman failed for ${SERVICE_USER}: ${podman_error}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
|
||||
# Keep the original metadata function intact; advertise the added policy here.
|
||||
mc_promotion_metadata()
|
||||
{
|
||||
meta_data | sed '/<\/parameters>/i\
|
||||
<parameter name="disconnected_max_age" unique="0" required="0">\
|
||||
<longdesc lang="en">Maximum age in seconds of a verified streaming observation for disconnected promotion eligibility. Default 300; 0 disables disconnected eligibility. Requires fencing managed by Pacemaker. PostgreSQL restart requires fresh streaming verification.</longdesc>\
|
||||
<shortdesc lang="en">Disconnected standby observation lifetime</shortdesc>\
|
||||
<content type="integer" default="300"/>\
|
||||
</parameter>\
|
||||
<parameter name="replication_peers" unique="0" required="0">\
|
||||
<longdesc lang="en">Opt-in stable peer map: node=host:port,node=host:port. Include the local node; exclude the client VIP. IPv4 or DNS hosts only. Requires a configured per-node replication slot, supplied explicitly or resolved from PostgreSQL/the managed container environment, plus peer authentication and slots on every upstream. Uses native primary-only selection; demotion verifies final WAL then ready recovery before successor promotion.</longdesc>\
|
||||
<shortdesc lang="en">Native replication peer map</shortdesc>\
|
||||
<content type="string" default=""/>\
|
||||
</parameter>'
|
||||
}
|
||||
|
||||
MC_APPLICATION_API=1
|
||||
MC_APPLICATION_DIR="${MC_MODULE_DIR}/applications/postgresql"
|
||||
for MC_PG_MODULE in peer_config native_peers postgres auth replication notify promote demote rejoin; do
|
||||
if ! . "${MC_APPLICATION_DIR}/${MC_PG_MODULE}.sh"; then
|
||||
ocf_exit_reason "Unable to load resource-agent module ${MC_APPLICATION_DIR}/${MC_PG_MODULE}.sh"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
done
|
||||
unset MC_PG_MODULE
|
||||
|
||||
mc_application_monitor()
|
||||
{
|
||||
managed_monitor
|
||||
}
|
||||
|
||||
mc_application_standby_score()
|
||||
{
|
||||
mc_pg_standby_score
|
||||
}
|
||||
|
||||
mc_application_validate()
|
||||
{
|
||||
mc_pg_peer_validate || return "$?"
|
||||
validate_all
|
||||
}
|
||||
|
||||
mc_application_metadata()
|
||||
{
|
||||
mc_promotion_metadata
|
||||
}
|
||||
|
||||
mc_application_notify()
|
||||
{
|
||||
managed_notify
|
||||
}
|
||||
|
||||
mc_application_action()
|
||||
{
|
||||
mc_pg_peer_validate || return "$?"
|
||||
if [ "$1" = demote ] && [ -n "${OCF_RESKEY_replication_peers:-}" ]; then
|
||||
mc_pg_native_demote
|
||||
return "$?"
|
||||
fi
|
||||
case "$1" in
|
||||
start|stop|promote|demote|rejoin) "managed_$1" ;;
|
||||
*) return "${OCF_ERR_UNIMPLEMENTED}" ;;
|
||||
esac
|
||||
}
|
||||
@@ -0,0 +1,696 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
pg_command()
|
||||
{
|
||||
local command="$1"
|
||||
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c "${command}"
|
||||
}
|
||||
|
||||
pgdata_path()
|
||||
{
|
||||
pg_command '
|
||||
if [ -z "$PGDATA" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "%s\n" "$PGDATA"
|
||||
'
|
||||
}
|
||||
|
||||
pg_running()
|
||||
{
|
||||
pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
"$BINDIR/pg_ctl" status -D "$PGDATA" >/dev/null 2>&1
|
||||
'
|
||||
}
|
||||
|
||||
pg_ready()
|
||||
{
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
pg_isready -q \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}"
|
||||
}
|
||||
|
||||
pg_role()
|
||||
{
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}" \
|
||||
-Atqc 'SELECT pg_is_in_recovery();'
|
||||
}
|
||||
|
||||
pg_wal_receiver_status()
|
||||
{
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}" \
|
||||
-Atqc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;"
|
||||
}
|
||||
|
||||
pg_streaming()
|
||||
{
|
||||
[ "$(pg_wal_receiver_status 2>/dev/null || true)" = "streaming" ]
|
||||
}
|
||||
|
||||
pg_wal_receiver_sender()
|
||||
{
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}" \
|
||||
-Atqc "
|
||||
SELECT sender_host || '|' || sender_port
|
||||
FROM pg_stat_wal_receiver
|
||||
WHERE status = 'streaming'
|
||||
LIMIT 1;
|
||||
"
|
||||
}
|
||||
|
||||
mc_pg_rejoin_running_standby()
|
||||
{
|
||||
local primary_info primary_rc primary_node primary_host primary_port
|
||||
|
||||
[ -n "${OCF_RESKEY_replication_peers:-}" ] || return 1
|
||||
|
||||
primary_info="$(mc_pg_find_remote_primary 2>/dev/null)"
|
||||
primary_rc=$?
|
||||
|
||||
case "${primary_rc}" in
|
||||
0)
|
||||
primary_node="${primary_info%%|*}"
|
||||
primary_info="${primary_info#*|}"
|
||||
primary_host="${primary_info%%|*}"
|
||||
primary_port="${primary_info#*|}"
|
||||
|
||||
ocf_log warn \
|
||||
"Standby is not streaming while remote primary ${primary_node} (${primary_host}:${primary_port}) is available; forcing managed rejoin"
|
||||
|
||||
if pg_running; then
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to stop non-streaming standby before managed rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
mc_pg_require_replication_slot || return "$?"
|
||||
|
||||
REJOIN_SOURCE_HOST="${primary_host}"
|
||||
REJOIN_SOURCE_PORT="${primary_port}"
|
||||
OCF_RESKEY_rejoin_source_host="${primary_host}"
|
||||
OCF_RESKEY_rejoin_source_port="${primary_port}"
|
||||
|
||||
if ! managed_rejoin; then
|
||||
ocf_exit_reason \
|
||||
"Managed rejoin of non-streaming standby against ${primary_node} failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
pg_clear_rejoin_pending >/dev/null 2>&1 || true
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
1)
|
||||
# No writable peer exists. This is a normal failover-election
|
||||
# condition; leave the standby running Unpromoted and let the
|
||||
# promotion scorer decide whether it is eligible.
|
||||
return 1
|
||||
;;
|
||||
|
||||
2)
|
||||
ocf_exit_reason \
|
||||
"Multiple writable PostgreSQL peers detected while repairing non-streaming standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Unable to determine PostgreSQL peer-primary state while repairing non-streaming standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
|
||||
pg_rejoin_pending()
|
||||
{
|
||||
pg_command 'test -f "$PGDATA/.loxthelion-ha-rejoin-required"'
|
||||
}
|
||||
|
||||
pg_mark_rejoin_pending()
|
||||
{
|
||||
pg_command '
|
||||
umask 077
|
||||
: > "$PGDATA/.loxthelion-ha-rejoin-required"
|
||||
chmod 600 "$PGDATA/.loxthelion-ha-rejoin-required"
|
||||
'
|
||||
}
|
||||
|
||||
pg_clear_rejoin_pending()
|
||||
{
|
||||
pg_command 'rm -f "$PGDATA/.loxthelion-ha-rejoin-required"'
|
||||
}
|
||||
|
||||
mc_pg_reconcile_rejoin_marker()
|
||||
{
|
||||
local primary_info primary_rc primary_node primary_host primary_port
|
||||
local receiver_sender
|
||||
|
||||
[ -n "${OCF_RESKEY_replication_peers:-}" ] || return 1
|
||||
|
||||
primary_info="$(mc_pg_find_remote_primary 2>/dev/null)"
|
||||
primary_rc=$?
|
||||
|
||||
[ "${primary_rc}" -eq 0 ] || return 1
|
||||
|
||||
primary_node="${primary_info%%|*}"
|
||||
primary_info="${primary_info#*|}"
|
||||
primary_host="${primary_info%%|*}"
|
||||
primary_port="${primary_info#*|}"
|
||||
|
||||
receiver_sender="$(pg_wal_receiver_sender 2>/dev/null || true)"
|
||||
|
||||
if ! pg_streaming; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "${receiver_sender}" != "${primary_host}|${primary_port}" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! pg_clear_rejoin_pending; then
|
||||
ocf_log warn \
|
||||
"Standby is streaming from elected primary ${primary_node}, but rejoin-required marker could not be cleared"
|
||||
return 1
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Standby is verified streaming from elected primary ${primary_node} (${primary_host}:${primary_port}); cleared provisional rejoin marker"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
pg_has_standby_signal()
|
||||
{
|
||||
pg_command 'test -f "$PGDATA/standby.signal"'
|
||||
}
|
||||
|
||||
managed_monitor()
|
||||
{
|
||||
local role
|
||||
|
||||
if ! service_running; then
|
||||
ocf_log debug "User service ${SERVICE_NAME} is not active"
|
||||
return "${OCF_NOT_RUNNING}"
|
||||
fi
|
||||
|
||||
if ! container_running; then
|
||||
ocf_log warn \
|
||||
"Service ${SERVICE_NAME} is active but container ${CONTAINER_NAME} is not running"
|
||||
return "${OCF_NOT_RUNNING}"
|
||||
fi
|
||||
|
||||
if ! pg_running; then
|
||||
ocf_log debug \
|
||||
"Container ${CONTAINER_NAME} is running but PostgreSQL is stopped"
|
||||
return "${OCF_NOT_RUNNING}"
|
||||
fi
|
||||
|
||||
if ! pg_ready; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL process exists but pg_isready failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL is accepting connections but role query failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${role}" in
|
||||
f)
|
||||
ocf_log debug \
|
||||
"PostgreSQL healthy: role=primary (promoted)"
|
||||
return "${OCF_RUNNING_PROMOTED}"
|
||||
;;
|
||||
t)
|
||||
# A former primary may be started provisionally in recovery when
|
||||
# no writable peer exists yet, solely so Pacemaker can decide
|
||||
# which Unpromoted instance to promote. Once another node becomes
|
||||
# primary, force recovery of this provisional instance through a
|
||||
# stop/start cycle so start() can pg_rewind it against the winner.
|
||||
if pg_rejoin_pending && [ -n "${OCF_RESKEY_replication_peers:-}" ]; then
|
||||
local primary_info primary_rc
|
||||
primary_info="$(mc_pg_find_remote_primary 2>/dev/null)"
|
||||
primary_rc=$?
|
||||
|
||||
case "${primary_rc}" in
|
||||
0)
|
||||
ocf_exit_reason \
|
||||
"A remote primary is now available (${primary_info%%|*}); provisional former-primary standby requires rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
2)
|
||||
ocf_exit_reason \
|
||||
"Multiple writable PostgreSQL peers detected while monitoring provisional standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if pg_streaming; then
|
||||
ocf_log debug \
|
||||
"PostgreSQL healthy: role=standby (unpromoted), WAL receiver=streaming"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
# A standby can legitimately lose its receiver while the cluster
|
||||
# has no writable primary (for example during failover election).
|
||||
# Do not declare that condition unhealthy: promotion eligibility
|
||||
# is decided separately by the promotion-scoring evidence. If a
|
||||
# writable primary already exists, however, a non-streaming
|
||||
# standby is broken and must be repaired rather than reported OK.
|
||||
if [ -n "${OCF_RESKEY_replication_peers:-}" ]; then
|
||||
local primary_info primary_rc
|
||||
primary_info="$(mc_pg_find_remote_primary 2>/dev/null)"
|
||||
primary_rc=$?
|
||||
|
||||
case "${primary_rc}" in
|
||||
0)
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL standby is in recovery but WAL receiver is not streaming while remote primary ${primary_info%%|*} is available"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
1)
|
||||
ocf_log debug \
|
||||
"PostgreSQL standby is disconnected and no remote primary is currently identifiable; leaving promotion eligibility to Pacemaker scoring"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
2)
|
||||
ocf_exit_reason \
|
||||
"Multiple writable PostgreSQL peers detected while monitoring standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Unable to determine PostgreSQL peer-primary state while monitoring standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Backward-compatible behavior when native peer discovery is not
|
||||
# configured: role-aware monitoring cannot prove an upstream.
|
||||
ocf_log debug \
|
||||
"PostgreSQL healthy: role=standby (unpromoted); native peer discovery is not configured"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Unexpected pg_is_in_recovery() result: '${role}'"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
}
|
||||
|
||||
managed_start()
|
||||
{
|
||||
local i monitor_rc
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL resource already running unpromoted"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
ocf_exit_reason \
|
||||
"Refusing start: PostgreSQL is already primary; promotion must occur through promote()"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
"${OCF_NOT_RUNNING}")
|
||||
# Expected for a stopped resource. Continue.
|
||||
;;
|
||||
|
||||
*)
|
||||
# If start() is called for an already-running standby that is in
|
||||
# recovery but cannot stream from an existing primary, repair it
|
||||
# instead of simply returning the monitor failure. This catches a
|
||||
# previous partial rejoin whose standby.signal survived restart.
|
||||
if service_running && container_running && pg_running; then
|
||||
local existing_role
|
||||
existing_role="$(pg_role 2>/dev/null || true)"
|
||||
if [ "${existing_role}" = "t" ] && ! pg_streaming; then
|
||||
if mc_pg_rejoin_running_standby; then
|
||||
managed_monitor
|
||||
return $?
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
ocf_exit_reason \
|
||||
"Cannot start unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
ocf_log info \
|
||||
"Ensuring container service ${SERVICE_NAME} is running"
|
||||
|
||||
if ! service_running; then
|
||||
if ! mc_lifecycle_start "${SERVICE_NAME}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to start ${SERVICE_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
for i in $(seq 1 20); do
|
||||
container_running && break
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Container ${CONTAINER_NAME} did not become running"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! pg_running; then
|
||||
#
|
||||
# OCF promotable-resource rule:
|
||||
# start() must produce an Unpromoted instance. Missing standby.signal
|
||||
# therefore never authorizes a primary start. It means either:
|
||||
# * a former primary can be rewound against an existing winner, or
|
||||
# * no winner exists yet, in which case place this PGDATA into
|
||||
# recovery and let Pacemaker decide whether to promote it.
|
||||
#
|
||||
if pg_rejoin_pending || ! pg_has_standby_signal; then
|
||||
local primary_info primary_rc primary_node primary_host primary_port
|
||||
|
||||
if [ -z "${OCF_RESKEY_replication_peers:-}" ]; then
|
||||
ocf_exit_reason \
|
||||
"Cannot safely recover PGDATA without standby.signal: replication_peers is required for automatic primary discovery"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
# At this point the maintenance container is running, so resolve
|
||||
# the exact configured slot from PostgreSQL/container state before
|
||||
# any rewind or recovery configuration is written.
|
||||
mc_pg_require_replication_slot || return "$?"
|
||||
|
||||
primary_info="$(mc_pg_find_remote_primary 2>/dev/null)"
|
||||
primary_rc=$?
|
||||
|
||||
case "${primary_rc}" in
|
||||
0)
|
||||
primary_node="${primary_info%%|*}"
|
||||
primary_info="${primary_info#*|}"
|
||||
primary_host="${primary_info%%|*}"
|
||||
primary_port="${primary_info#*|}"
|
||||
|
||||
ocf_log info \
|
||||
"Existing remote primary ${primary_node} (${primary_host}:${primary_port}) found; rejoining local PGDATA before start"
|
||||
|
||||
REJOIN_SOURCE_HOST="${primary_host}"
|
||||
REJOIN_SOURCE_PORT="${primary_port}"
|
||||
OCF_RESKEY_rejoin_source_host="${primary_host}"
|
||||
OCF_RESKEY_rejoin_source_port="${primary_port}"
|
||||
|
||||
if ! managed_rejoin; then
|
||||
ocf_exit_reason \
|
||||
"Automatic rejoin against ${primary_node} failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
pg_clear_rejoin_pending >/dev/null 2>&1 || true
|
||||
managed_monitor
|
||||
return $?
|
||||
;;
|
||||
|
||||
1)
|
||||
ocf_log info \
|
||||
"No remote primary is currently identifiable; preparing local PGDATA as Unpromoted and leaving promotion exclusively to Pacemaker"
|
||||
|
||||
if ! mc_pg_write_peer_recovery; then
|
||||
ocf_exit_reason \
|
||||
"Unable to place PGDATA into native standby recovery while no primary exists"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! pg_mark_rejoin_pending; then
|
||||
ocf_exit_reason \
|
||||
"Unable to mark provisional standby as requiring rejoin if another node wins promotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
;;
|
||||
|
||||
2)
|
||||
ocf_exit_reason \
|
||||
"Multiple writable PostgreSQL peers detected; refusing start"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Unable to determine PostgreSQL peer-primary state"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL as an unpromoted standby inside ${CONTAINER_NAME}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "pg_ctl start failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL start completed in unpromoted state"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL became primary during start; stopping it to preserve OCF role semantics"
|
||||
|
||||
pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
' >/dev/null 2>&1 || true
|
||||
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
*)
|
||||
# standby.signal can exist while the standby is still unusable
|
||||
# (for example, its required WAL was removed after a failed
|
||||
# primary repair). If a writable peer exists, route the instance
|
||||
# through managed_rejoin(), whose guarded basebackup fallback can
|
||||
# rebuild PGDATA when rewind alone cannot establish streaming.
|
||||
if pg_running; then
|
||||
local final_role
|
||||
final_role="$(pg_role 2>/dev/null || true)"
|
||||
if [ "${final_role}" = "t" ] && ! pg_streaming; then
|
||||
if mc_pg_rejoin_running_standby; then
|
||||
managed_monitor
|
||||
return $?
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
managed_stop()
|
||||
{
|
||||
local i
|
||||
|
||||
if container_running && pg_running; then
|
||||
ocf_log info \
|
||||
"Stopping PostgreSQL cleanly inside ${CONTAINER_NAME}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "pg_ctl stop failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if service_running; then
|
||||
ocf_log info \
|
||||
"Stopping container service ${SERVICE_NAME}"
|
||||
|
||||
if ! mc_lifecycle_stop "${SERVICE_NAME}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to stop ${SERVICE_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
for i in $(seq 1 20); do
|
||||
if ! service_running && ! container_running; then
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if service_running; then
|
||||
ocf_exit_reason \
|
||||
"${SERVICE_NAME} remained active after stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if container_running; then
|
||||
ocf_exit_reason \
|
||||
"${CONTAINER_NAME} remained running after stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
# Conservative scoring eligibility only; this does not replace role detection,
|
||||
# fencing, the promotion action, or the existing controlled-demote WAL checks.
|
||||
# This strict predicate remains unchanged; disconnected policy is separate below.
|
||||
mc_pg_promotion_eligible()
|
||||
{
|
||||
local eligible
|
||||
|
||||
# A provisional former primary must never become promotion-eligible.
|
||||
# It may run in recovery so Pacemaker can observe it, but it must first
|
||||
# complete a real rejoin/rewind against the elected primary.
|
||||
if pg_rejoin_pending; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
eligible="$(
|
||||
mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" \
|
||||
--env 'PGOPTIONS=-c statement_timeout=2000' \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc "
|
||||
SELECT CASE WHEN pg_is_in_recovery() THEN
|
||||
NOT pg_is_wal_replay_paused()
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM pg_stat_wal_receiver
|
||||
WHERE status = 'streaming'
|
||||
AND flushed_lsn IS NOT NULL
|
||||
AND pg_last_wal_replay_lsn() >= flushed_lsn
|
||||
)
|
||||
ELSE false END;
|
||||
" 2>/dev/null
|
||||
)" || return 1
|
||||
[ "${eligible}" = t ]
|
||||
}
|
||||
|
||||
# One observation supplies both strict connected eligibility and disconnected
|
||||
# evidence. PostgreSQL restart time deliberately invalidates prior observations,
|
||||
# including a new basebackup sharing the same PostgreSQL system identifier.
|
||||
mc_pg_standby_observation()
|
||||
{
|
||||
mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" \
|
||||
--env 'PGOPTIONS=-c statement_timeout=2000' \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc "
|
||||
WITH s AS MATERIALIZED (
|
||||
SELECT pg_is_in_recovery() AS recovery,
|
||||
pg_last_wal_replay_lsn() AS replay,
|
||||
r.status AS receiver, r.flushed_lsn AS flush
|
||||
FROM (SELECT 1) AS singleton
|
||||
LEFT JOIN pg_stat_wal_receiver r ON true
|
||||
)
|
||||
SELECT json_build_object(
|
||||
'identity', json_build_array(
|
||||
(SELECT system_identifier::text FROM pg_control_system()),
|
||||
current_setting('data_directory'),
|
||||
extract(epoch FROM pg_postmaster_start_time())::text),
|
||||
'recovery', recovery,
|
||||
'paused', CASE WHEN recovery THEN pg_is_wal_replay_paused() ELSE true END,
|
||||
'receiver', receiver, 'replay', replay::text, 'flush', flush::text,
|
||||
'strict', CASE WHEN recovery THEN
|
||||
NOT pg_is_wal_replay_paused() AND receiver = 'streaming'
|
||||
AND flush IS NOT NULL AND replay >= flush
|
||||
ELSE false END)
|
||||
FROM s;
|
||||
" 2>/dev/null
|
||||
}
|
||||
|
||||
mc_pg_standby_score()
|
||||
{
|
||||
local observation
|
||||
|
||||
# Hard promotion veto for provisional former-primary PGDATA. Returning
|
||||
# -INFINITY here ensures Pacemaker cannot choose this instance while the
|
||||
# rejoin-required marker exists, even though monitor may report it as a
|
||||
# healthy Unpromoted standby.
|
||||
if pg_rejoin_pending; then
|
||||
if ! mc_pg_reconcile_rejoin_marker; then
|
||||
printf '%s\n' '-INFINITY'
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
observation="$(mc_pg_standby_observation)" || return 1
|
||||
# The helper is root-owned alongside the modules; state never enters the
|
||||
# rootless container. A failed/slow helper always withdraws eligibility.
|
||||
printf '%s' "${observation}" | timeout --kill-after=1s 1s python3 \
|
||||
"${MC_APPLICATION_DIR}/promotion_state.py" \
|
||||
"${MC_PROMOTION_RESOURCE_INSTANCE:-}" \
|
||||
"${SERVICE_USER}" "${SERVICE_NAME}" "${CONTAINER_NAME}" \
|
||||
"${OCF_RESKEY_disconnected_max_age:-300}"
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
managed_promote()
|
||||
{
|
||||
local monitor_rc role i
|
||||
|
||||
# Belt-and-suspenders safety: promotion scoring already returns
|
||||
# -INFINITY while this marker exists, but refuse the action itself too.
|
||||
# This protects against an administrative/manual promotion request or a
|
||||
# future scoring regression.
|
||||
if pg_rejoin_pending; then
|
||||
ocf_exit_reason \
|
||||
"Cannot promote PostgreSQL while rejoin-required is set; instance must rejoin the elected primary first"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
ocf_log info \
|
||||
"PostgreSQL is already primary; reconciling primary invariants"
|
||||
|
||||
if ! ensure_local_ha_slots; then
|
||||
ocf_exit_reason \
|
||||
"HA slot reconciliation failed on existing primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_rewind_privileges; then
|
||||
ocf_exit_reason \
|
||||
"pg_rewind privilege reconciliation failed on existing primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
pg_clear_rejoin_pending >/dev/null 2>&1 || true
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
"${OCF_SUCCESS}")
|
||||
# Healthy standby: continue with promotion.
|
||||
;;
|
||||
|
||||
"${OCF_NOT_RUNNING}")
|
||||
ocf_exit_reason \
|
||||
"Cannot promote PostgreSQL because the resource is not running"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot promote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
ocf_log info \
|
||||
"Promoting PostgreSQL standby inside ${CONTAINER_NAME}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
promote -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "pg_ctl promote failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
sleep 1
|
||||
continue
|
||||
}
|
||||
|
||||
if [ "${role}" = "f" ]; then
|
||||
ocf_log info \
|
||||
"PostgreSQL became primary; reconciling primary invariants"
|
||||
|
||||
if ! ensure_local_ha_slots; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL promoted but HA slot reconciliation failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_rewind_privileges; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL promoted but pg_rewind privilege reconciliation failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! pg_clear_rejoin_pending; then
|
||||
ocf_log warn \
|
||||
"PostgreSQL promoted successfully but provisional rejoin marker could not be cleared"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"PostgreSQL promotion completed successfully"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL did not become primary after promotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Local promotion evidence. Invoked by the root RA; never invokes cluster tools.
|
||||
|
||||
State is a bounded JSON record, never executable input. The public CLI uses a
|
||||
fixed root-owned /run directory and CLOCK_BOOTTIME, including suspend duration.
|
||||
"""
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
STATE_DIR = Path('/run/managed-container/promotion')
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError('duplicate JSON key')
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def lsn(value):
|
||||
if not isinstance(value, str) or not re.fullmatch(r'[0-9A-F]{1,8}/[0-9A-F]{1,8}', value):
|
||||
raise ValueError('invalid LSN')
|
||||
high, low = value.split('/')
|
||||
return (int(high, 16) << 32) | int(low, 16)
|
||||
|
||||
|
||||
def number(value):
|
||||
return type(value) in (int, float) and math.isfinite(value) and value >= 0
|
||||
|
||||
|
||||
def validate_observation(obs):
|
||||
if set(obs) != {'identity', 'recovery', 'paused', 'receiver', 'replay', 'flush', 'strict'}:
|
||||
raise ValueError('invalid observation fields')
|
||||
identity = obs['identity']
|
||||
if not isinstance(identity, list) or len(identity) != 3:
|
||||
raise ValueError('invalid identity')
|
||||
system, path, started = identity
|
||||
if (not isinstance(system, str) or not re.fullmatch(r'[0-9]{1,20}', system)
|
||||
or not isinstance(path, str) or not path.startswith('/') or len(path) > 2048
|
||||
or not isinstance(started, str) or not re.fullmatch(r'[0-9]{1,12}\.[0-9]{1,6}', started)):
|
||||
raise ValueError('invalid identity values')
|
||||
if type(obs['recovery']) is not bool or type(obs['paused']) is not bool:
|
||||
raise ValueError('invalid role')
|
||||
if obs['strict'] is not None and type(obs['strict']) is not bool:
|
||||
raise ValueError('invalid strict flag')
|
||||
if obs['receiver'] is not None and obs['receiver'] not in (
|
||||
'starting', 'streaming', 'waiting', 'restarting', 'stopping'):
|
||||
raise ValueError('invalid receiver status')
|
||||
replay = lsn(obs['replay']) if obs['replay'] is not None else None
|
||||
flush = lsn(obs['flush']) if obs['flush'] is not None else None
|
||||
strict = (obs['recovery'] and not obs['paused'] and obs['receiver'] == 'streaming'
|
||||
and replay is not None and flush is not None and replay >= flush)
|
||||
if (obs['strict'] is True) != strict:
|
||||
raise ValueError('inconsistent strict observation')
|
||||
return strict
|
||||
|
||||
|
||||
class Evidence:
|
||||
def __init__(self, directory, context, boot, now, wall, owner=0):
|
||||
self.directory, self.context = Path(directory), context
|
||||
self.boot, self.now, self.wall, self.owner = boot, now, wall, owner
|
||||
key = hashlib.sha256(json.dumps(context, separators=(',', ':')).encode()).hexdigest()
|
||||
self.path = self.directory / (key + '.json')
|
||||
self.lock = self.directory / (key + '.lock')
|
||||
|
||||
def secure(self, path, mode):
|
||||
info = path.lstat()
|
||||
if (info.st_uid != self.owner or stat.S_IMODE(info.st_mode) != mode
|
||||
or (not stat.S_ISREG(info.st_mode) if mode == 0o600 else not stat.S_ISDIR(info.st_mode))):
|
||||
raise ValueError('unsafe state ownership/type/mode')
|
||||
|
||||
def discard(self):
|
||||
self.path.unlink(missing_ok=True)
|
||||
|
||||
def evaluate(self, obs, max_age):
|
||||
if type(max_age) is not int or not 0 <= max_age <= 99999:
|
||||
raise ValueError('invalid freshness threshold')
|
||||
self.directory.mkdir(mode=0o700, exist_ok=True)
|
||||
self.secure(self.directory, 0o700)
|
||||
fd = os.open(self.lock, os.O_CREAT | os.O_NOFOLLOW | os.O_RDWR, 0o600)
|
||||
with os.fdopen(fd, 'r+') as lock:
|
||||
self.secure(self.lock, 0o600)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
try:
|
||||
return self._evaluate(obs, max_age)
|
||||
except (ValueError, TypeError, KeyError, OverflowError):
|
||||
self.discard()
|
||||
return '-INFINITY'
|
||||
|
||||
def _evaluate(self, obs, max_age):
|
||||
strict = validate_observation(obs)
|
||||
if strict:
|
||||
record = dict(version=1, context=self.context, boot=self.boot,
|
||||
identity=obs['identity'], observed_boottime=self.now,
|
||||
observed_unix=self.wall, replay=obs['replay'], flush=obs['flush'])
|
||||
# Same-directory atomic replace; failed persistence cannot authorize
|
||||
# subsequent disconnected promotion. No secrets are recorded.
|
||||
fd, temporary = tempfile.mkstemp(prefix='.observation-', dir=self.directory)
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as out:
|
||||
json.dump(record, out, separators=(',', ':'))
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
os.replace(temporary, self.path)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
return '100'
|
||||
if (not obs['recovery'] or obs['paused'] or obs['receiver'] == 'streaming'
|
||||
or obs['replay'] is None or max_age == 0):
|
||||
self.discard()
|
||||
return '-INFINITY'
|
||||
if not self.path.exists():
|
||||
return '-INFINITY'
|
||||
self.secure(self.path, 0o600)
|
||||
if self.path.stat().st_size > 4096:
|
||||
raise ValueError('oversized state')
|
||||
with self.path.open() as source:
|
||||
record = json.load(source, object_pairs_hook=unique_object)
|
||||
if set(record) != {'version', 'context', 'boot', 'identity', 'observed_boottime',
|
||||
'observed_unix', 'replay', 'flush'}:
|
||||
raise ValueError('invalid record fields')
|
||||
if (type(record['version']) is not int or record['version'] != 1
|
||||
or record['context'] != self.context or record['boot'] != self.boot
|
||||
or record['identity'] != obs['identity']
|
||||
or not number(record['observed_boottime']) or not number(record['observed_unix'])
|
||||
or not 0 <= self.now - record['observed_boottime'] <= max_age
|
||||
or lsn(record['replay']) < lsn(record['flush'])
|
||||
or lsn(obs['replay']) < lsn(record['replay'])):
|
||||
raise ValueError('expired or incompatible observation')
|
||||
return '50'
|
||||
|
||||
|
||||
def main():
|
||||
if os.geteuid() != 0 or len(sys.argv) != 6 or not all(sys.argv[1:5]):
|
||||
raise ValueError('root and explicit resource context required')
|
||||
age = sys.argv[5]
|
||||
if not re.fullmatch(r'(0|[1-9][0-9]{0,4})', age):
|
||||
raise ValueError('invalid freshness threshold')
|
||||
# /run is trusted root-owned; validate our parent before using a child.
|
||||
parent = STATE_DIR.parent
|
||||
parent.mkdir(mode=0o700, exist_ok=True)
|
||||
info = parent.lstat()
|
||||
if info.st_uid != 0 or not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700:
|
||||
raise ValueError('unsafe state parent')
|
||||
raw = sys.stdin.read(8193)
|
||||
if len(raw) > 8192:
|
||||
raise ValueError('oversized observation')
|
||||
evidence = Evidence(STATE_DIR, sys.argv[1:5],
|
||||
Path('/proc/sys/kernel/random/boot_id').read_text().strip(),
|
||||
time.clock_gettime(time.CLOCK_BOOTTIME), time.time())
|
||||
try:
|
||||
obs = json.loads(raw, object_pairs_hook=unique_object)
|
||||
except ValueError:
|
||||
obs = {} # Invalid query output withdraws and removes prior evidence.
|
||||
print(evidence.evaluate(obs, int(age)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (OSError, ValueError, TypeError, KeyError, OverflowError):
|
||||
print('-INFINITY')
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,391 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
mc_pg_rejoin_basebackup_enabled()
|
||||
{
|
||||
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
||||
true|1|yes)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
mc_pg_rejoin_basebackup_fallback()
|
||||
{
|
||||
local pgdata="$1"
|
||||
local reason="${2:-rejoin fallback requested}"
|
||||
local source_bytes free_bytes required_bytes headroom_bytes min_headroom_bytes
|
||||
local timestamp preserved_pgdata failed_pgdata source_role_retry
|
||||
|
||||
if ! mc_pg_rejoin_basebackup_enabled; then
|
||||
ocf_exit_reason \
|
||||
"${reason}; rejoin_allow_basebackup is not enabled"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"${reason}; evaluating pg_basebackup fallback"
|
||||
|
||||
# Revalidate the source immediately before replacing local PGDATA.
|
||||
source_role_retry="$(pg_remote_query \
|
||||
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to revalidate rejoin source before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${source_role_retry}" != "f" ]; then
|
||||
ocf_exit_reason \
|
||||
"Rejoin source is no longer a writable primary; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL is still active; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! service_running || ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service/container is unavailable; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_remote_slot \
|
||||
"${REJOIN_SOURCE_HOST}" \
|
||||
"${REPLICATION_SLOT}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
# Estimate the replacement payload. Existing PGDATA is renamed in place on
|
||||
# the same filesystem, so it consumes no additional free space. Require one
|
||||
# replacement copy plus configurable safety headroom.
|
||||
source_bytes="$(pg_remote_query \
|
||||
"SELECT COALESCE(sum(pg_database_size(datname)),0)
|
||||
FROM pg_database
|
||||
WHERE datallowconn;" 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to estimate source database size before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${source_bytes}" in
|
||||
""|*[!0-9]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid source database size returned before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
free_bytes="$(
|
||||
pg_command '
|
||||
blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")"
|
||||
case "$blocks" in
|
||||
""|*[!0-9]*) exit 1 ;;
|
||||
esac
|
||||
printf "%s\\n" "$((blocks * 1024))"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to determine free space for pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${free_bytes}" in
|
||||
""|*[!0-9]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid free-space value before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
min_headroom_bytes=$((1024 * 1024 * 1024))
|
||||
headroom_bytes=$((source_bytes * REJOIN_BASEBACKUP_HEADROOM_PERCENT / 100))
|
||||
if [ "${headroom_bytes}" -lt "${min_headroom_bytes}" ]; then
|
||||
headroom_bytes=${min_headroom_bytes}
|
||||
fi
|
||||
required_bytes=$((source_bytes + headroom_bytes))
|
||||
|
||||
ocf_log info \
|
||||
"pg_basebackup space check: source-estimate=${source_bytes} headroom=${headroom_bytes} required=${required_bytes} free=${free_bytes}"
|
||||
|
||||
if [ "${free_bytes}" -lt "${required_bytes}" ]; then
|
||||
ocf_exit_reason \
|
||||
"Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes} source-estimate=${source_bytes} headroom=${headroom_bytes}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
timestamp="$(date -u +%Y%m%d-%H%M%S)"
|
||||
preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}"
|
||||
failed_pgdata="${pgdata}.failed-basebackup-${timestamp}"
|
||||
|
||||
ocf_log warn \
|
||||
"Preserving existing PGDATA as ${preserved_pgdata}"
|
||||
|
||||
if ! pg_command "
|
||||
set -e
|
||||
[ -d '${pgdata}' ]
|
||||
[ ! -e '${preserved_pgdata}' ]
|
||||
[ ! -e '${failed_pgdata}' ]
|
||||
mv '${pgdata}' '${preserved_pgdata}'
|
||||
install -d -m 0700 '${pgdata}'
|
||||
"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to preserve existing PGDATA for pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"Running pg_basebackup from ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
if ! pg_replication_exec \
|
||||
pg_basebackup \
|
||||
--dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \
|
||||
--pgdata="${pgdata}" \
|
||||
--format=plain \
|
||||
--wal-method=stream \
|
||||
--slot="${REPLICATION_SLOT}" \
|
||||
--write-recovery-conf \
|
||||
--progress
|
||||
then
|
||||
ocf_log err \
|
||||
"pg_basebackup failed; preserving failed copy and restoring previous PGDATA"
|
||||
|
||||
if ! pg_command "
|
||||
set -e
|
||||
[ ! -e '${failed_pgdata}' ]
|
||||
if [ -e '${pgdata}' ]; then
|
||||
mv '${pgdata}' '${failed_pgdata}'
|
||||
fi
|
||||
mv '${preserved_pgdata}' '${pgdata}'
|
||||
"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_exit_reason \
|
||||
"pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}"
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
mc_pg_rejoin_configure_standby()
|
||||
{
|
||||
if [ -n "${OCF_RESKEY_replication_peers:-}" ]; then
|
||||
mc_pg_write_peer_recovery || {
|
||||
ocf_exit_reason "Failed to configure native peer recovery"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
if ! mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
--env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \
|
||||
--env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \
|
||||
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
||||
--env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \
|
||||
--env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
set -e
|
||||
sed -i \\
|
||||
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \\
|
||||
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \\
|
||||
"$PGDATA/postgresql.auto.conf"
|
||||
printf "%s\\n" \\
|
||||
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \\
|
||||
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \\
|
||||
>> "$PGDATA/postgresql.auto.conf"
|
||||
touch "$PGDATA/standby.signal"
|
||||
chmod 600 "$PGDATA/postgresql.auto.conf" "$PGDATA/standby.signal"
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "Failed to configure standby settings"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
mc_pg_rejoin_start_and_wait_streaming()
|
||||
{
|
||||
local role receiver_status i
|
||||
|
||||
ocf_log info "Starting PostgreSQL as standby"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "PostgreSQL failed to start after rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
sleep 1
|
||||
continue
|
||||
}
|
||||
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
||||
|
||||
if [ "${role}" = "t" ] && [ "${receiver_status}" = "streaming" ]; then
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
managed_rejoin()
|
||||
{
|
||||
local source_role pgdata
|
||||
local fallback_used=false
|
||||
|
||||
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
||||
true|false|1|0|yes|no) ;;
|
||||
*)
|
||||
ocf_exit_reason "rejoin_allow_basebackup must be true or false"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "${REJOIN_SOURCE_HOST}" ]; then
|
||||
ocf_exit_reason "rejoin_source_host is required for rejoin"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
if ! service_running; then
|
||||
ocf_exit_reason "Maintenance service ${SERVICE_NAME} must be running before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! container_running; then
|
||||
ocf_exit_reason "Maintenance container ${CONTAINER_NAME} must be running before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
mc_pg_require_replication_slot || return "$?"
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason "PostgreSQL must be stopped before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info "Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}"
|
||||
source_role="$(pg_remote_query 'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
||||
ocf_exit_reason "Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${source_role}" != "f" ]; then
|
||||
ocf_exit_reason "Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info "Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}"; then
|
||||
ocf_exit_reason "Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
pgdata="$(pgdata_path 2>/dev/null)" || {
|
||||
ocf_exit_reason "Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
if [ -z "${pgdata}" ]; then
|
||||
ocf_exit_reason "PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info "Running pg_rewind against ${REJOIN_SOURCE_HOST}"
|
||||
if ! pg_replication_exec \
|
||||
pg_rewind \
|
||||
--target-pgdata="${pgdata}" \
|
||||
--source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
||||
--progress
|
||||
then
|
||||
mc_pg_rejoin_basebackup_fallback "${pgdata}" "pg_rewind failed" || return "$?"
|
||||
fallback_used=true
|
||||
fi
|
||||
|
||||
ocf_log info "Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}"
|
||||
mc_pg_rejoin_configure_standby || return "$?"
|
||||
|
||||
if mc_pg_rejoin_start_and_wait_streaming; then
|
||||
pg_clear_rejoin_pending >/dev/null 2>&1 || true
|
||||
ocf_log info "PostgreSQL successfully rejoined as streaming standby"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
# A successful pg_rewind does not guarantee that the WAL required to join
|
||||
# the new timeline is still available. If the standby cannot reach
|
||||
# streaming, stop it and perform the same guarded full-resync fallback.
|
||||
if [ "${fallback_used}" = "true" ]; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL did not become a streaming standby even after pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! mc_pg_rejoin_basebackup_enabled; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL started in recovery but did not become streaming; rejoin_allow_basebackup is not enabled"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"PostgreSQL entered recovery but did not reach streaming after pg_rewind; stopping it for pg_basebackup fallback"
|
||||
|
||||
# The failed post-rewind start may already have exited. Stopping an
|
||||
# already-stopped server is success for fallback purposes.
|
||||
if pg_running; then
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" -D "$PGDATA" stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to stop non-streaming standby before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
else
|
||||
ocf_log info \
|
||||
"PostgreSQL is already stopped; proceeding directly to pg_basebackup fallback"
|
||||
fi
|
||||
|
||||
mc_pg_rejoin_basebackup_fallback \
|
||||
"${pgdata}" \
|
||||
"pg_rewind completed but standby could not establish WAL streaming" || return "$?"
|
||||
|
||||
ocf_log info "Reconfiguring PostgreSQL standby after pg_basebackup fallback"
|
||||
mc_pg_rejoin_configure_standby || return "$?"
|
||||
|
||||
if ! mc_pg_rejoin_start_and_wait_streaming; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL did not become a streaming standby after pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
pg_clear_rejoin_pending >/dev/null 2>&1 || true
|
||||
ocf_log info \
|
||||
"PostgreSQL successfully rejoined as streaming standby after pg_basebackup fallback"
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
pg_remote_query()
|
||||
{
|
||||
local sql="$1"
|
||||
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
||||
-Atqc "${sql}"
|
||||
}
|
||||
|
||||
|
||||
# Return exactly one reachable remote writable PostgreSQL primary from the
|
||||
# validated native peer map as: node|host|port.
|
||||
# Return 1 when no remote primary is currently reachable, and 2 when more
|
||||
# than one writable peer is observed (unsafe / split-brain indication).
|
||||
mc_pg_find_remote_primary()
|
||||
{
|
||||
local self peers line node host port role
|
||||
local found_count=0 found_node='' found_host='' found_port=''
|
||||
|
||||
peers="${OCF_RESKEY_replication_peers:-}"
|
||||
self="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}"
|
||||
|
||||
[ -n "${peers}" ] || return 1
|
||||
|
||||
while IFS='|' read -r node host port; do
|
||||
[ -n "${node}" ] || continue
|
||||
|
||||
role="$(
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres connect_timeout=2" \
|
||||
-X -w -Atqc 'SELECT pg_is_in_recovery();' \
|
||||
2>/dev/null
|
||||
)" || continue
|
||||
|
||||
if [ "${role}" = "f" ]; then
|
||||
found_count=$((found_count + 1))
|
||||
found_node="${node}"
|
||||
found_host="${host}"
|
||||
found_port="${port}"
|
||||
fi
|
||||
done <<EOF_PEERS
|
||||
$(LC_ALL=C awk -v peers="${peers}" -v self="${self}" '
|
||||
BEGIN {
|
||||
n=split(peers, entries, ","); self=tolower(self);
|
||||
for(i=1;i<=n;i++) {
|
||||
if(split(entries[i], pair, "=")!=2) continue;
|
||||
node=pair[1];
|
||||
if(tolower(node)==self) continue;
|
||||
if(split(pair[2], addr, ":")!=2) continue;
|
||||
print node "|" addr[1] "|" addr[2];
|
||||
}
|
||||
}')
|
||||
EOF_PEERS
|
||||
|
||||
if [ "${found_count}" -gt 1 ]; then
|
||||
ocf_log err "Multiple writable PostgreSQL peers detected while selecting a rejoin source"
|
||||
return 2
|
||||
fi
|
||||
|
||||
[ "${found_count}" -eq 1 ] || return 1
|
||||
printf '%s|%s|%s\n' "${found_node}" "${found_host}" "${found_port}"
|
||||
}
|
||||
|
||||
ensure_local_ha_slots()
|
||||
{
|
||||
local slots raw_slot slot slot_type
|
||||
|
||||
#
|
||||
# POSTGRES_HA_SLOTS lives in the container environment supplied by
|
||||
# postgres.env. It is the cluster-wide list of physical slots that
|
||||
# every writable primary must provide.
|
||||
#
|
||||
slots="$(
|
||||
pg_command '
|
||||
printf "%s" "${POSTGRES_HA_SLOTS:-}"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to read POSTGRES_HA_SLOTS from PostgreSQL container"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ -z "${slots}" ]; then
|
||||
ocf_log warn \
|
||||
"POSTGRES_HA_SLOTS is empty; no local HA slots will be reconciled"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
while IFS= read -r raw_slot; do
|
||||
slot="$(printf '%s' "${raw_slot}" | tr -d '[:space:]')"
|
||||
|
||||
[ -n "${slot}" ] || continue
|
||||
|
||||
case "${slot}" in
|
||||
*[!a-z0-9_]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid physical replication slot name: ${slot}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
slot_type="$(
|
||||
pg_command "
|
||||
psql \
|
||||
-U '${DB_USER}' \
|
||||
-d postgres \
|
||||
-At \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-c \"
|
||||
SELECT slot_type
|
||||
FROM pg_replication_slots
|
||||
WHERE slot_name = '${slot}';
|
||||
\"
|
||||
" 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Failed querying replication slot ${slot}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
slot_type="$(printf '%s' "${slot_type}" | tr -d '[:space:]')"
|
||||
|
||||
case "${slot_type}" in
|
||||
physical)
|
||||
ocf_log info \
|
||||
"Physical HA slot already exists: ${slot}"
|
||||
;;
|
||||
|
||||
"")
|
||||
ocf_log info \
|
||||
"Creating physical HA slot: ${slot}"
|
||||
|
||||
if ! pg_command "
|
||||
psql \
|
||||
-U '${DB_USER}' \
|
||||
-d postgres \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-c \"
|
||||
SELECT pg_create_physical_replication_slot('${slot}');
|
||||
\"
|
||||
" >/dev/null
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed creating physical replication slot ${slot}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Replication slot ${slot} exists but is type ${slot_type}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
done <<EOF
|
||||
$(printf '%s\n' "${slots}" | tr ',' '\n')
|
||||
EOF
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
ensure_remote_slot()
|
||||
{
|
||||
local host="${1:-}"
|
||||
local slot="${2:-}"
|
||||
local result
|
||||
|
||||
[ -n "${host}" ] || return 1
|
||||
[ -n "${slot}" ] || return 1
|
||||
|
||||
result="$(
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \
|
||||
-Atc "
|
||||
SELECT slot_type
|
||||
FROM pg_replication_slots
|
||||
WHERE slot_name = '${slot}';
|
||||
" 2>/dev/null
|
||||
)" || return 1
|
||||
|
||||
case "${result}" in
|
||||
physical)
|
||||
return 0
|
||||
;;
|
||||
|
||||
"")
|
||||
if ! pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \
|
||||
-Atc "
|
||||
SELECT slot_name
|
||||
FROM pg_create_physical_replication_slot('${slot}');
|
||||
" >/dev/null
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_log err \
|
||||
"Replication slot ${slot} exists on ${host} but is not physical"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
candidate_caught_up()
|
||||
{
|
||||
local candidate="${1:-}"
|
||||
local result
|
||||
|
||||
[ -n "${candidate}" ] || return 1
|
||||
|
||||
result="$(pg_command "
|
||||
psql -U '${DB_USER}' -d '${DB_NAME}' -Atc \"
|
||||
SELECT CASE
|
||||
WHEN state = 'streaming'
|
||||
AND replay_lsn >= pg_current_wal_flush_lsn()
|
||||
THEN 'yes'
|
||||
ELSE 'no'
|
||||
END
|
||||
FROM pg_stat_replication
|
||||
WHERE application_name = '${candidate}'
|
||||
LIMIT 1;
|
||||
\"
|
||||
" 2>/dev/null)" || return 1
|
||||
|
||||
[ "${result}" = "yes" ]
|
||||
}
|
||||
|
||||
candidate_is_standby()
|
||||
{
|
||||
local host="${1:-}"
|
||||
local result
|
||||
local port="${REJOIN_SOURCE_PORT:-5432}"
|
||||
|
||||
[ -n "${host}" ] || return 1
|
||||
|
||||
result="$(
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres" \
|
||||
-Atc "SELECT pg_is_in_recovery();" \
|
||||
2>/dev/null
|
||||
)" || return 1
|
||||
|
||||
[ "${result}" = "t" ]
|
||||
}
|
||||
Reference in New Issue
Block a user