279 lines
9.8 KiB
Bash
279 lines
9.8 KiB
Bash
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
|
|
|
node_to_addr()
|
|
{
|
|
local target="${1:-}"
|
|
|
|
[ -n "${target}" ] || return 1
|
|
|
|
corosync-cmapctl 2>/dev/null |
|
|
awk -v target="${target}" '
|
|
/nodelist\.node\.[0-9]+\.name / {
|
|
key=$1
|
|
sub(/\.name$/, "", key)
|
|
names[key]=$NF
|
|
}
|
|
|
|
/nodelist\.node\.[0-9]+\.ring0_addr / {
|
|
key=$1
|
|
sub(/\.ring0_addr$/, "", key)
|
|
addrs[key]=$NF
|
|
}
|
|
|
|
END {
|
|
for (key in names) {
|
|
if (names[key] == target && addrs[key] != "") {
|
|
print addrs[key]
|
|
exit 0
|
|
}
|
|
}
|
|
exit 1
|
|
}
|
|
'
|
|
}
|
|
|
|
# Read the final checkpoint only after pg_ctl has confirmed clean primary shutdown.
|
|
mc_pg_shutdown_checkpoint()
|
|
{
|
|
local control summary
|
|
control="$(mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" --env LC_ALL=C \
|
|
"${CONTAINER_NAME}" sh -c \
|
|
'exec "$(pg_config --bindir)/pg_controldata" "$PGDATA"')" || return 1
|
|
summary="$(printf '%s\n' "${control}" | awk -F ': *' '
|
|
/^Database cluster state:/ { state=$2; ns++ }
|
|
/^Database system identifier:/ { sid=$2; ni++ }
|
|
/^Latest checkpoint location:/ { checkpoint=$2; nc++ }
|
|
/^Latest checkpoint.s TimeLineID:/ { timeline=$2; nt++ }
|
|
END {
|
|
if (ns != 1 || ni != 1 || nc != 1 || nt != 1 || state != "shut down") exit 1;
|
|
print sid "|" timeline "|" checkpoint;
|
|
}')" || return 1
|
|
[[ "${summary}" =~ ^[0-9]+\|[1-9][0-9]*\|[0-9A-F]+/[0-9A-F]+$ ]] || return 1
|
|
printf '%s\n' "${summary}"
|
|
}
|
|
|
|
# A checkpoint LSN is its record START. Replay must be strictly beyond it to
|
|
# establish that the complete shutdown checkpoint has been replayed.
|
|
mc_pg_wait_shutdown_replay()
|
|
{
|
|
local host="$1" port="$2" observation="$3" deadline="$4"
|
|
local system timeline checkpoint response ready identity rest got_system got_timeline
|
|
local remaining duration
|
|
IFS='|' read -r system timeline checkpoint <<< "${observation}"
|
|
[[ "${observation}" =~ ^[0-9]+\|[1-9][0-9]*\|[0-9A-F]+/[0-9A-F]+$ ]] || return 1
|
|
while (( SECONDS < deadline )); do
|
|
remaining=$((deadline - SECONDS))
|
|
duration=3
|
|
(( remaining < duration )) && duration=${remaining}
|
|
response="$(mc_runtime_exec_bounded "${duration}s" --user "${PG_OS_USER}" \
|
|
--env PGCONNECT_TIMEOUT=2 --env 'PGOPTIONS=-c statement_timeout=1000' \
|
|
"${CONTAINER_NAME}" sh -c '
|
|
export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD:?}"
|
|
psql -X -w -v ON_ERROR_STOP=1 -h "$1" -p "$2" -U "$3" -d postgres -Atqc \
|
|
"SELECT CASE WHEN pg_is_in_recovery() THEN
|
|
NOT pg_is_wal_replay_paused() AND
|
|
pg_last_wal_replay_lsn() > '\''$4'\''::pg_lsn
|
|
ELSE false END;" || exit 1
|
|
exec psql -X -w -v ON_ERROR_STOP=1 -h "$1" -p "$2" -U "$3" \
|
|
-d replication=true -Atqc IDENTIFY_SYSTEM
|
|
' sh "${host}" "${port}" "${REPLICATION_USER}" "${checkpoint}" 2>/dev/null)" || response=''
|
|
ready="${response%%$'\n'*}"
|
|
identity="${response#*$'\n'}"
|
|
IFS='|' read -r got_system got_timeline rest <<< "${identity}"
|
|
if [ "${ready}" = t ] && [ "${got_system}" = "${system}" ] &&
|
|
[ "${got_timeline}" = "${timeline}" ]; then
|
|
return 0
|
|
fi
|
|
(( SECONDS < deadline )) && sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
managed_demote()
|
|
{
|
|
local monitor_rc
|
|
local target_node target_addr
|
|
local local_node slot_name target_port
|
|
local role receiver_status i checkpoint handoff_deadline
|
|
local action_started=${SECONDS} action_budget=${OCF_RESKEY_CRM_meta_timeout:-60000}
|
|
[[ "${action_budget}" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}"
|
|
action_budget=$((10#${action_budget} / 1000))
|
|
if (( action_budget < 15 )); then
|
|
ocf_exit_reason "Demotion timeout is too short for a verified shutdown-WAL handoff"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
managed_monitor
|
|
monitor_rc=$?
|
|
|
|
case "${monitor_rc}" in
|
|
"${OCF_RUNNING_PROMOTED}")
|
|
;;
|
|
"${OCF_SUCCESS}")
|
|
ocf_log info \
|
|
"PostgreSQL is already unpromoted"
|
|
return "${OCF_SUCCESS}"
|
|
;;
|
|
"${OCF_NOT_RUNNING}")
|
|
ocf_exit_reason \
|
|
"Cannot complete demotion because PostgreSQL is stopped"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
*)
|
|
ocf_exit_reason \
|
|
"Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
|
return "${monitor_rc}"
|
|
;;
|
|
esac
|
|
|
|
target_node="$(demote_target)" || {
|
|
ocf_exit_reason \
|
|
"No fresh valid Pacemaker pre-demote target is available"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
target_addr="$(node_to_addr "${target_node}")" || {
|
|
ocf_exit_reason \
|
|
"Unable to resolve Pacemaker promotion target ${target_node} through Corosync"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
target_port="${REJOIN_SOURCE_PORT:-5432}"
|
|
local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}"
|
|
slot_name="ha_$(printf '%s' "${local_node}" |
|
|
tr '[:upper:]-' '[:lower:]_')"
|
|
|
|
ocf_log info \
|
|
"Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})"
|
|
|
|
if ! candidate_is_standby "${target_addr}"; then
|
|
ocf_exit_reason \
|
|
"Promotion target ${target_node} is not a reachable PostgreSQL standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! candidate_caught_up "${target_node}"; then
|
|
ocf_exit_reason \
|
|
"Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then
|
|
ocf_exit_reason \
|
|
"Unable to ensure physical replication slot ${slot_name} on ${target_node}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Demotion safety checks passed; stopping PostgreSQL primary"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
stop -m fast -w
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"pg_ctl stop failed during controlled demotion"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if pg_running; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL remained running after pg_ctl stop"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! service_running || ! container_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance service/container stopped unexpectedly during demotion"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
# Keep at most 20 seconds for the handoff, reserving 10 seconds of the
|
|
# action budget for subsequent work. Pacemaker remains the overall timeout.
|
|
checkpoint="$(mc_pg_shutdown_checkpoint)" || {
|
|
ocf_exit_reason "Unable to read a valid clean-shutdown checkpoint; PostgreSQL remains stopped"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
handoff_deadline=$((action_started + action_budget - 10))
|
|
(( handoff_deadline > SECONDS + 20 )) && handoff_deadline=$((SECONDS + 20))
|
|
if ! mc_pg_wait_shutdown_replay "${target_addr}" "${target_port}" "${checkpoint}" "${handoff_deadline}"; then
|
|
ocf_exit_reason "Successor has not verified final shutdown-checkpoint replay; PostgreSQL remains stopped"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Configuring ${local_node} as cascading standby of ${target_node}"
|
|
|
|
if ! mc_runtime_exec \
|
|
--user "${PG_OS_USER}" \
|
|
--env "RA_PRIMARY_HOST=${target_addr}" \
|
|
--env "RA_PRIMARY_PORT=${target_port}" \
|
|
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
|
--env "RA_REPLICATION_SLOT=${slot_name}" \
|
|
--env "RA_APPLICATION_NAME=${local_node}" \
|
|
"${CONTAINER_NAME}" \
|
|
sh -c '
|
|
set -e
|
|
|
|
sed -i \
|
|
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \
|
|
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \
|
|
"$PGDATA/postgresql.auto.conf"
|
|
|
|
printf "%s
|
|
" \
|
|
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \
|
|
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \
|
|
>> "$PGDATA/postgresql.auto.conf"
|
|
|
|
touch "$PGDATA/standby.signal"
|
|
|
|
chmod 600 \
|
|
"$PGDATA/postgresql.auto.conf" \
|
|
"$PGDATA/standby.signal"
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"Failed to configure PostgreSQL as standby of ${target_node}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Starting PostgreSQL as cascading standby of ${target_node}"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
-l "$PGDATA/postgresql.log" \
|
|
start -w
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"PostgreSQL failed to start as cascading standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
for i in $(seq 1 30); do
|
|
role="$(pg_role 2>/dev/null || true)"
|
|
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
|
|
|
if [ "${role}" = "t" ] &&
|
|
[ "${receiver_status}" = "streaming" ]
|
|
then
|
|
ocf_log info \
|
|
"Controlled demotion complete: ${local_node} is streaming from ${target_node}"
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
ocf_exit_reason \
|
|
"PostgreSQL started after demotion but did not become a streaming standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|