# Optional native peer lifecycle; defaults continue to use the legacy path. mc_pg_peer_validate() { [ -n "${OCF_RESKEY_replication_peers:-}" ] || return 0 mc_peer_conninfo "${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" >/dev/null || { ocf_exit_reason "Invalid replication_peers map or local identity" return "${OCF_ERR_CONFIGURED}" } # The container may legitimately not exist yet during validate/start. # Validate an explicitly supplied slot now, but defer runtime discovery # until an operation actually requires the slot. if [ -n "${REPLICATION_SLOT}" ]; then case "${REPLICATION_SLOT}" in ''|*[!a-z0-9_]*) ocf_exit_reason "Configured replication slot is invalid; expected lowercase letters, digits, and underscore" return "${OCF_ERR_CONFIGURED}";; esac [ ${#REPLICATION_SLOT} -le 63 ] || return "${OCF_ERR_CONFIGURED}" fi } mc_pg_write_peer_recovery() { local code # Same canonical POSIX helper is mounted into bootstrap; no duplicated writer. code="$(cat "${MC_APPLICATION_DIR}/peer_config.sh")" || return 1 mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" sh -c \ "${code} mc_peer_write_recovery \"\$PGDATA\" \"\$1\" \"\$2\" \"\$3\" \"\$4\" \"\$5\"" sh \ "${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" \ "${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_SLOT}" } mc_pg_demote_database_shadow() { local socket_path resource response rc socket_path="${MC_PROXY_SOCKET:-/run/user/1000/managed-containerd/managed-containerd.sock}" resource="${OCF_RESOURCE_INSTANCE:-}" resource="${resource%%:*}" [ -n "${resource}" ] || return 0 if response="$( timeout --kill-after=1s 5s \ python3 \ "${MC_PROXY_CLIENT:-/usr/lib/ocf/resource.d/loxthelion/managed-container.d/proxy_client.py}" \ "${socket_path}" \ "${resource}" \ "demote.database" \ 10 \ 11 )"; then rc=0 else rc=$? fi if [ "${rc}" -ne 0 ]; then ocf_log warn \ "managed-containerd demote.database shadow unavailable rc=${rc}" if [ "${MC_DEMOTE_DATABASE_GATE:-false}" = true ]; then return 1 fi return 0 fi if MC_DAEMON_RESPONSE="${response}" python3 -c ' import json import os response = json.loads(os.environ["MC_DAEMON_RESPONSE"]) if ( response.get("ocf_rc") != 0 or response.get("ready") is not True or response.get("mutation") is not False or response.get("final_handoff_required") is not True ): raise SystemExit(1) '; then ocf_log info \ "managed-containerd demote.database shadow: comparison=MATCH response=${response}" else ocf_log err \ "managed-containerd demote.database shadow MISMATCH response=${response}" if [ "${MC_DEMOTE_DATABASE_GATE:-false}" = true ]; then return 1 fi fi return 0 } mc_pg_native_demote() ( local mc_native_started=${SECONDS} local mc_native_ms=${OCF_RESKEY_CRM_meta_timeout:-60000} [[ "$mc_native_ms" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}" local mc_native_deadline=$((mc_native_started + 10#$mc_native_ms / 1000 - 5)) local mc_original_bounded mc_original_bounded=$(declare -f mc_runtime_exec_bounded) eval "${mc_original_bounded/mc_runtime_exec_bounded/mc_native_original_bounded}" mc_runtime_exec_bounded() { local requested=$1 remaining=$((mc_native_deadline - SECONDS)) shift (( remaining > 0 )) || return 124 # Existing bounded call sites use integer seconds. [[ "$requested" =~ ^[0-9]+s$ ]] || return 124 (( ${requested%s} < remaining )) && remaining=${requested%s} mc_native_original_bounded "${remaining}s" "$@" } mc_runtime_exec() { local remaining=$((mc_native_deadline - SECONDS)) (( remaining > 0 )) || return 124 mc_native_original_bounded "${remaining}s" "$@" } local monitor_rc local target_node target_addr local local_node slot_name target_port local role receiver_status i checkpoint handoff_deadline local self_restart=false local action_started=${mc_native_started} action_budget=${OCF_RESKEY_CRM_meta_timeout:-60000} [[ "${action_budget}" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}" action_budget=$((10#${action_budget} / 1000)) if (( action_budget < 15 )); then ocf_exit_reason "Demotion timeout is too short for a verified shutdown-WAL handoff" return "${OCF_ERR_CONFIGURED}" fi mc_pg_peer_validate || return "$?" mc_pg_require_replication_slot || return "$?" managed_monitor monitor_rc=$? case "${monitor_rc}" in "${OCF_RUNNING_PROMOTED}") ;; "${OCF_SUCCESS}") ocf_log info \ "PostgreSQL is already unpromoted" return "${OCF_SUCCESS}" ;; "${OCF_NOT_RUNNING}") ocf_exit_reason \ "Cannot complete demotion because PostgreSQL is stopped" return "${OCF_ERR_GENERIC}" ;; *) ocf_exit_reason \ "Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" return "${monitor_rc}" ;; esac local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" slot_name=${REPLICATION_SLOT} local native_deadline=$((action_started + action_budget - 5)) target_node="$(mc_pg_native_demote_target)" || { ocf_exit_reason \ "No fresh valid Pacemaker pre-demote target is available" return "${OCF_ERR_GENERIC}" } if ! mc_pg_demote_database_shadow; then ocf_exit_reason \ "managed-containerd demote.database gate rejected demotion" return "${OCF_ERR_GENERIC}" fi if [ "$(printf '%s' "${target_node}" | tr '[:upper:]' '[:lower:]')" = \ "$(printf '%s' "${local_node}" | tr '[:upper:]' '[:lower:]')" ]; then self_restart=true ocf_log info \ "Pacemaker requested a self-restart demotion of ${local_node}; performing clean local demotion without remote handoff" else local peer_pair peer_pair=$(mc_peer_endpoints "${OCF_RESKEY_replication_peers}" "${local_node}" "${target_node}") || { ocf_exit_reason "Successor is missing from the validated peer map" return "${OCF_ERR_CONFIGURED}" } target_addr=${peer_pair%|*} target_port=${peer_pair#*|} # Existing candidate/slot SQL helpers use REJOIN_SOURCE_PORT. local REJOIN_SOURCE_PORT=${target_port} ocf_log info \ "Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})" if ! candidate_is_standby "${target_addr}"; then ocf_exit_reason \ "Promotion target ${target_node} is not a reachable PostgreSQL standby" return "${OCF_ERR_GENERIC}" fi if ! candidate_caught_up "${target_node}"; then ocf_exit_reason \ "Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position" return "${OCF_ERR_GENERIC}" fi if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then ocf_exit_reason \ "Unable to ensure physical replication slot ${slot_name} on ${target_node}" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Demotion safety checks passed; stopping PostgreSQL primary" fi if ! pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 exec "$BINDIR/pg_ctl" \ -D "$PGDATA" \ stop -m fast -w ' then ocf_exit_reason \ "pg_ctl stop failed during controlled demotion" return "${OCF_ERR_GENERIC}" fi if pg_running; then ocf_exit_reason \ "PostgreSQL remained running after pg_ctl stop" return "${OCF_ERR_GENERIC}" fi if ! service_running || ! container_running; then ocf_exit_reason \ "Maintenance service/container stopped unexpectedly during demotion" return "${OCF_ERR_GENERIC}" fi checkpoint="$(mc_pg_shutdown_checkpoint)" || { ocf_exit_reason "Unable to read a valid clean-shutdown checkpoint; PostgreSQL remains stopped" return "${OCF_ERR_GENERIC}" } if [ "${self_restart}" != true ]; then # Keep at most 20 seconds for the handoff, reserving 10 seconds of the # action budget for subsequent work. Pacemaker remains the overall timeout. handoff_deadline=$((action_started + action_budget - 10)) (( handoff_deadline > SECONDS + 20 )) && handoff_deadline=$((SECONDS + 20)) if ! mc_pg_wait_shutdown_replay "${target_addr}" "${target_port}" "${checkpoint}" "${handoff_deadline}"; then ocf_exit_reason "Successor has not verified final shutdown-checkpoint replay; PostgreSQL remains stopped" return "${OCF_ERR_GENERIC}" fi else ocf_log info \ "Clean shutdown checkpoint verified locally for Pacemaker self-restart demotion" fi ocf_log info \ "Configuring ${local_node} with stable native replication peers" if ! mc_pg_write_peer_recovery; then ocf_exit_reason "Failed to write native peer recovery configuration" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Starting PostgreSQL in recovery with native primary selection" if ! pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 exec "$BINDIR/pg_ctl" \ -D "$PGDATA" \ -l "$PGDATA/postgresql.log" \ start -w ' then ocf_exit_reason \ "PostgreSQL failed to start as cascading standby" return "${OCF_ERR_GENERIC}" fi while (( SECONDS < native_deadline )); do if pg_ready && [ "$(pg_role 2>/dev/null)" = t ]; then local state state=$(mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" psql -X -w -v ON_ERROR_STOP=1 -U "${DB_USER}" -d postgres -Atqc "SELECT system_identifier::text || '|' || (NOT pg_is_wal_replay_paused())::text FROM pg_control_system()" 2>/dev/null) || state='' if [ "${state}" = "${checkpoint%%|*}|true" ]; then if [ "${self_restart}" = true ]; then ocf_log info "Pacemaker self-restart demotion complete: clean former primary is ready in recovery for possible re-promotion" else ocf_log info "Native peer demotion complete: final WAL verified; old primary ready in recovery awaiting successor promotion" fi return "${OCF_SUCCESS}" fi fi sleep 1 done ocf_exit_reason "Old primary did not become verified ready recovery within demotion budget" return "${OCF_ERR_GENERIC}" )