311 lines
11 KiB
Bash
Executable File
311 lines
11 KiB
Bash
Executable File
# Optional native peer lifecycle; defaults continue to use the legacy path.
|
|
mc_pg_peer_validate()
|
|
{
|
|
[ -n "${OCF_RESKEY_replication_peers:-}" ] || return 0
|
|
mc_peer_conninfo "${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" >/dev/null || {
|
|
ocf_exit_reason "Invalid replication_peers map or local identity"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
}
|
|
# The container may legitimately not exist yet during validate/start.
|
|
# Validate an explicitly supplied slot now, but defer runtime discovery
|
|
# until an operation actually requires the slot.
|
|
if [ -n "${REPLICATION_SLOT}" ]; then
|
|
case "${REPLICATION_SLOT}" in ''|*[!a-z0-9_]*)
|
|
ocf_exit_reason "Configured replication slot is invalid; expected lowercase letters, digits, and underscore"
|
|
return "${OCF_ERR_CONFIGURED}";; esac
|
|
[ ${#REPLICATION_SLOT} -le 63 ] || return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
}
|
|
|
|
mc_pg_write_peer_recovery()
|
|
{
|
|
local code
|
|
# Same canonical POSIX helper is mounted into bootstrap; no duplicated writer.
|
|
code="$(cat "${MC_APPLICATION_DIR}/peer_config.sh")" || return 1
|
|
mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" sh -c \
|
|
"${code}
|
|
mc_peer_write_recovery \"\$PGDATA\" \"\$1\" \"\$2\" \"\$3\" \"\$4\" \"\$5\"" sh \
|
|
"${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" \
|
|
"${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_SLOT}"
|
|
}
|
|
|
|
mc_pg_demote_database_shadow()
|
|
{
|
|
local socket_path resource response rc
|
|
|
|
socket_path="${MC_PROXY_SOCKET:-/run/user/1000/managed-containerd/managed-containerd.sock}"
|
|
resource="${OCF_RESOURCE_INSTANCE:-}"
|
|
resource="${resource%%:*}"
|
|
|
|
[ -n "${resource}" ] || return 0
|
|
|
|
if response="$(
|
|
timeout --kill-after=1s 5s \
|
|
python3 \
|
|
"${MC_PROXY_CLIENT:-/usr/lib/ocf/resource.d/loxthelion/managed-container.d/proxy_client.py}" \
|
|
"${socket_path}" \
|
|
"${resource}" \
|
|
"demote.database" \
|
|
10 \
|
|
11
|
|
)"; then
|
|
rc=0
|
|
else
|
|
rc=$?
|
|
fi
|
|
|
|
if [ "${rc}" -ne 0 ]; then
|
|
ocf_log warn \
|
|
"managed-containerd demote.database shadow unavailable rc=${rc}"
|
|
|
|
if [ "${MC_DEMOTE_DATABASE_GATE:-false}" = true ]; then
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
fi
|
|
|
|
if MC_DAEMON_RESPONSE="${response}" python3 -c '
|
|
import json
|
|
import os
|
|
|
|
response = json.loads(os.environ["MC_DAEMON_RESPONSE"])
|
|
|
|
common_valid = (
|
|
response.get("ocf_rc") == 0
|
|
and response.get("ready") is True
|
|
and response.get("mutation") is False
|
|
)
|
|
|
|
peer_handoff = (
|
|
response.get("final_handoff_required") is True
|
|
)
|
|
|
|
local_restart = (
|
|
response.get("final_handoff_required") is False
|
|
and response.get("handoff_mode") == "local_restart"
|
|
and response.get("reason") == "local_restart_demotion_ready"
|
|
)
|
|
|
|
if not common_valid or not (peer_handoff or local_restart):
|
|
raise SystemExit(1)
|
|
'; then
|
|
ocf_log info \
|
|
"managed-containerd demote.database shadow: comparison=MATCH response=${response}"
|
|
else
|
|
ocf_log err \
|
|
"managed-containerd demote.database shadow MISMATCH response=${response}"
|
|
|
|
if [ "${MC_DEMOTE_DATABASE_GATE:-false}" = true ]; then
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
|
|
mc_pg_native_demote()
|
|
(
|
|
local mc_native_started=${SECONDS}
|
|
local mc_native_ms=${OCF_RESKEY_CRM_meta_timeout:-60000}
|
|
[[ "$mc_native_ms" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}"
|
|
local mc_native_deadline=$((mc_native_started + 10#$mc_native_ms / 1000 - 5))
|
|
local mc_original_bounded
|
|
mc_original_bounded=$(declare -f mc_runtime_exec_bounded)
|
|
eval "${mc_original_bounded/mc_runtime_exec_bounded/mc_native_original_bounded}"
|
|
mc_runtime_exec_bounded() {
|
|
local requested=$1 remaining=$((mc_native_deadline - SECONDS))
|
|
shift
|
|
(( remaining > 0 )) || return 124
|
|
# Existing bounded call sites use integer seconds.
|
|
[[ "$requested" =~ ^[0-9]+s$ ]] || return 124
|
|
(( ${requested%s} < remaining )) && remaining=${requested%s}
|
|
mc_native_original_bounded "${remaining}s" "$@"
|
|
}
|
|
mc_runtime_exec() {
|
|
local remaining=$((mc_native_deadline - SECONDS))
|
|
(( remaining > 0 )) || return 124
|
|
mc_native_original_bounded "${remaining}s" "$@"
|
|
}
|
|
local monitor_rc
|
|
local target_node target_addr
|
|
local local_node slot_name target_port
|
|
local role receiver_status i checkpoint handoff_deadline
|
|
local self_restart=false
|
|
local action_started=${mc_native_started} action_budget=${OCF_RESKEY_CRM_meta_timeout:-60000}
|
|
[[ "${action_budget}" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}"
|
|
action_budget=$((10#${action_budget} / 1000))
|
|
if (( action_budget < 15 )); then
|
|
ocf_exit_reason "Demotion timeout is too short for a verified shutdown-WAL handoff"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
mc_pg_peer_validate || return "$?"
|
|
mc_pg_require_replication_slot || return "$?"
|
|
managed_monitor
|
|
monitor_rc=$?
|
|
|
|
case "${monitor_rc}" in
|
|
"${OCF_RUNNING_PROMOTED}")
|
|
;;
|
|
"${OCF_SUCCESS}")
|
|
ocf_log info \
|
|
"PostgreSQL is already unpromoted"
|
|
return "${OCF_SUCCESS}"
|
|
;;
|
|
"${OCF_NOT_RUNNING}")
|
|
ocf_exit_reason \
|
|
"Cannot complete demotion because PostgreSQL is stopped"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
*)
|
|
ocf_exit_reason \
|
|
"Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
|
return "${monitor_rc}"
|
|
;;
|
|
esac
|
|
|
|
local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}"
|
|
slot_name=${REPLICATION_SLOT}
|
|
local native_deadline=$((action_started + action_budget - 5))
|
|
|
|
target_node="$(mc_pg_native_demote_target)" || {
|
|
ocf_exit_reason \
|
|
"No fresh valid Pacemaker pre-demote target is available"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if ! mc_pg_demote_database_shadow; then
|
|
ocf_exit_reason \
|
|
"managed-containerd demote.database gate rejected demotion"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if [ "$(printf '%s' "${target_node}" | tr '[:upper:]' '[:lower:]')" = \
|
|
"$(printf '%s' "${local_node}" | tr '[:upper:]' '[:lower:]')" ]; then
|
|
self_restart=true
|
|
ocf_log info \
|
|
"Pacemaker requested a self-restart demotion of ${local_node}; performing clean local demotion without remote handoff"
|
|
else
|
|
local peer_pair
|
|
peer_pair=$(mc_peer_endpoints "${OCF_RESKEY_replication_peers}" "${local_node}" "${target_node}") || {
|
|
ocf_exit_reason "Successor is missing from the validated peer map"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
}
|
|
target_addr=${peer_pair%|*}
|
|
target_port=${peer_pair#*|}
|
|
# Existing candidate/slot SQL helpers use REJOIN_SOURCE_PORT.
|
|
local REJOIN_SOURCE_PORT=${target_port}
|
|
|
|
ocf_log info \
|
|
"Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})"
|
|
|
|
if ! candidate_is_standby "${target_addr}"; then
|
|
ocf_exit_reason \
|
|
"Promotion target ${target_node} is not a reachable PostgreSQL standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! candidate_caught_up "${target_node}"; then
|
|
ocf_exit_reason \
|
|
"Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then
|
|
ocf_exit_reason \
|
|
"Unable to ensure physical replication slot ${slot_name} on ${target_node}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Demotion safety checks passed; stopping PostgreSQL primary"
|
|
fi
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
stop -m fast -w
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"pg_ctl stop failed during controlled demotion"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if pg_running; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL remained running after pg_ctl stop"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! service_running || ! container_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance service/container stopped unexpectedly during demotion"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
checkpoint="$(mc_pg_shutdown_checkpoint)" || {
|
|
ocf_exit_reason "Unable to read a valid clean-shutdown checkpoint; PostgreSQL remains stopped"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ "${self_restart}" != true ]; then
|
|
# Keep at most 20 seconds for the handoff, reserving 10 seconds of the
|
|
# action budget for subsequent work. Pacemaker remains the overall timeout.
|
|
handoff_deadline=$((action_started + action_budget - 10))
|
|
(( handoff_deadline > SECONDS + 20 )) && handoff_deadline=$((SECONDS + 20))
|
|
if ! mc_pg_wait_shutdown_replay "${target_addr}" "${target_port}" "${checkpoint}" "${handoff_deadline}"; then
|
|
ocf_exit_reason "Successor has not verified final shutdown-checkpoint replay; PostgreSQL remains stopped"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
else
|
|
ocf_log info \
|
|
"Clean shutdown checkpoint verified locally for Pacemaker self-restart demotion"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Configuring ${local_node} with stable native replication peers"
|
|
|
|
if ! mc_pg_write_peer_recovery; then
|
|
ocf_exit_reason "Failed to write native peer recovery configuration"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Starting PostgreSQL in recovery with native primary selection"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
-l "$PGDATA/postgresql.log" \
|
|
start -w
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"PostgreSQL failed to start as cascading standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
while (( SECONDS < native_deadline )); do
|
|
if pg_ready && [ "$(pg_role 2>/dev/null)" = t ]; then
|
|
local state
|
|
state=$(mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" psql -X -w -v ON_ERROR_STOP=1 -U "${DB_USER}" -d postgres -Atqc "SELECT system_identifier::text || '|' || (NOT pg_is_wal_replay_paused())::text FROM pg_control_system()" 2>/dev/null) || state=''
|
|
if [ "${state}" = "${checkpoint%%|*}|true" ]; then
|
|
if [ "${self_restart}" = true ]; then
|
|
ocf_log info "Pacemaker self-restart demotion complete: clean former primary is ready in recovery for possible re-promotion"
|
|
else
|
|
ocf_log info "Native peer demotion complete: final WAL verified; old primary ready in recovery awaiting successor promotion"
|
|
fi
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
fi
|
|
sleep 1
|
|
done
|
|
ocf_exit_reason "Old primary did not become verified ready recovery within demotion budget"
|
|
return "${OCF_ERR_GENERIC}"
|
|
)
|