Initial managed-container HA resource agent baseline
This commit is contained in:
Lyncoln
+228
@@ -0,0 +1,228 @@
|
||||
# Optional native peer lifecycle; defaults continue to use the legacy path.
|
||||
mc_pg_peer_validate()
|
||||
{
|
||||
[ -n "${OCF_RESKEY_replication_peers:-}" ] || return 0
|
||||
mc_peer_conninfo "${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" >/dev/null || {
|
||||
ocf_exit_reason "Invalid replication_peers map or local identity"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
}
|
||||
# The container may legitimately not exist yet during validate/start.
|
||||
# Validate an explicitly supplied slot now, but defer runtime discovery
|
||||
# until an operation actually requires the slot.
|
||||
if [ -n "${REPLICATION_SLOT}" ]; then
|
||||
case "${REPLICATION_SLOT}" in ''|*[!a-z0-9_]*)
|
||||
ocf_exit_reason "Configured replication slot is invalid; expected lowercase letters, digits, and underscore"
|
||||
return "${OCF_ERR_CONFIGURED}";; esac
|
||||
[ ${#REPLICATION_SLOT} -le 63 ] || return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
}
|
||||
|
||||
mc_pg_write_peer_recovery()
|
||||
{
|
||||
local code
|
||||
# Same canonical POSIX helper is mounted into bootstrap; no duplicated writer.
|
||||
code="$(cat "${MC_APPLICATION_DIR}/peer_config.sh")" || return 1
|
||||
mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" sh -c \
|
||||
"${code}
|
||||
mc_peer_write_recovery \"\$PGDATA\" \"\$1\" \"\$2\" \"\$3\" \"\$4\" \"\$5\"" sh \
|
||||
"${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" \
|
||||
"${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_SLOT}"
|
||||
}
|
||||
|
||||
mc_pg_native_demote()
|
||||
(
|
||||
local mc_native_started=${SECONDS}
|
||||
local mc_native_ms=${OCF_RESKEY_CRM_meta_timeout:-60000}
|
||||
[[ "$mc_native_ms" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}"
|
||||
local mc_native_deadline=$((mc_native_started + 10#$mc_native_ms / 1000 - 5))
|
||||
local mc_original_bounded
|
||||
mc_original_bounded=$(declare -f mc_runtime_exec_bounded)
|
||||
eval "${mc_original_bounded/mc_runtime_exec_bounded/mc_native_original_bounded}"
|
||||
mc_runtime_exec_bounded() {
|
||||
local requested=$1 remaining=$((mc_native_deadline - SECONDS))
|
||||
shift
|
||||
(( remaining > 0 )) || return 124
|
||||
# Existing bounded call sites use integer seconds.
|
||||
[[ "$requested" =~ ^[0-9]+s$ ]] || return 124
|
||||
(( ${requested%s} < remaining )) && remaining=${requested%s}
|
||||
mc_native_original_bounded "${remaining}s" "$@"
|
||||
}
|
||||
mc_runtime_exec() {
|
||||
local remaining=$((mc_native_deadline - SECONDS))
|
||||
(( remaining > 0 )) || return 124
|
||||
mc_native_original_bounded "${remaining}s" "$@"
|
||||
}
|
||||
local monitor_rc
|
||||
local target_node target_addr
|
||||
local local_node slot_name target_port
|
||||
local role receiver_status i checkpoint handoff_deadline
|
||||
local self_restart=false
|
||||
local action_started=${mc_native_started} action_budget=${OCF_RESKEY_CRM_meta_timeout:-60000}
|
||||
[[ "${action_budget}" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}"
|
||||
action_budget=$((10#${action_budget} / 1000))
|
||||
if (( action_budget < 15 )); then
|
||||
ocf_exit_reason "Demotion timeout is too short for a verified shutdown-WAL handoff"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
mc_pg_peer_validate || return "$?"
|
||||
mc_pg_require_replication_slot || return "$?"
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
;;
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL is already unpromoted"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
"${OCF_NOT_RUNNING}")
|
||||
ocf_exit_reason \
|
||||
"Cannot complete demotion because PostgreSQL is stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}"
|
||||
slot_name=${REPLICATION_SLOT}
|
||||
local native_deadline=$((action_started + action_budget - 5))
|
||||
|
||||
target_node="$(mc_pg_native_demote_target)" || {
|
||||
ocf_exit_reason \
|
||||
"No fresh valid Pacemaker pre-demote target is available"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "$(printf '%s' "${target_node}" | tr '[:upper:]' '[:lower:]')" = \
|
||||
"$(printf '%s' "${local_node}" | tr '[:upper:]' '[:lower:]')" ]; then
|
||||
self_restart=true
|
||||
ocf_log info \
|
||||
"Pacemaker requested a self-restart demotion of ${local_node}; performing clean local demotion without remote handoff"
|
||||
else
|
||||
local peer_pair
|
||||
peer_pair=$(mc_peer_endpoints "${OCF_RESKEY_replication_peers}" "${local_node}" "${target_node}") || {
|
||||
ocf_exit_reason "Successor is missing from the validated peer map"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
}
|
||||
target_addr=${peer_pair%|*}
|
||||
target_port=${peer_pair#*|}
|
||||
# Existing candidate/slot SQL helpers use REJOIN_SOURCE_PORT.
|
||||
local REJOIN_SOURCE_PORT=${target_port}
|
||||
|
||||
ocf_log info \
|
||||
"Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})"
|
||||
|
||||
if ! candidate_is_standby "${target_addr}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not a reachable PostgreSQL standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! candidate_caught_up "${target_node}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then
|
||||
ocf_exit_reason \
|
||||
"Unable to ensure physical replication slot ${slot_name} on ${target_node}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Demotion safety checks passed; stopping PostgreSQL primary"
|
||||
fi
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"pg_ctl stop failed during controlled demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL remained running after pg_ctl stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! service_running || ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service/container stopped unexpectedly during demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
checkpoint="$(mc_pg_shutdown_checkpoint)" || {
|
||||
ocf_exit_reason "Unable to read a valid clean-shutdown checkpoint; PostgreSQL remains stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${self_restart}" != true ]; then
|
||||
# Keep at most 20 seconds for the handoff, reserving 10 seconds of the
|
||||
# action budget for subsequent work. Pacemaker remains the overall timeout.
|
||||
handoff_deadline=$((action_started + action_budget - 10))
|
||||
(( handoff_deadline > SECONDS + 20 )) && handoff_deadline=$((SECONDS + 20))
|
||||
if ! mc_pg_wait_shutdown_replay "${target_addr}" "${target_port}" "${checkpoint}" "${handoff_deadline}"; then
|
||||
ocf_exit_reason "Successor has not verified final shutdown-checkpoint replay; PostgreSQL remains stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
else
|
||||
ocf_log info \
|
||||
"Clean shutdown checkpoint verified locally for Pacemaker self-restart demotion"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Configuring ${local_node} with stable native replication peers"
|
||||
|
||||
if ! mc_pg_write_peer_recovery; then
|
||||
ocf_exit_reason "Failed to write native peer recovery configuration"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL in recovery with native primary selection"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL failed to start as cascading standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
while (( SECONDS < native_deadline )); do
|
||||
if pg_ready && [ "$(pg_role 2>/dev/null)" = t ]; then
|
||||
local state
|
||||
state=$(mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" psql -X -w -v ON_ERROR_STOP=1 -U "${DB_USER}" -d postgres -Atqc "SELECT system_identifier::text || '|' || (NOT pg_is_wal_replay_paused())::text FROM pg_control_system()" 2>/dev/null) || state=''
|
||||
if [ "${state}" = "${checkpoint%%|*}|true" ]; then
|
||||
if [ "${self_restart}" = true ]; then
|
||||
ocf_log info "Pacemaker self-restart demotion complete: clean former primary is ready in recovery for possible re-promotion"
|
||||
else
|
||||
ocf_log info "Native peer demotion complete: final WAL verified; old primary ready in recovery awaiting successor promotion"
|
||||
fi
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
ocf_exit_reason "Old primary did not become verified ready recovery within demotion budget"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
)
|
||||
Reference in New Issue
Block a user