392 lines
13 KiB
Bash
Executable File
392 lines
13 KiB
Bash
Executable File
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
|
|
|
mc_pg_rejoin_basebackup_enabled()
|
|
{
|
|
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
|
true|1|yes)
|
|
return 0
|
|
;;
|
|
*)
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
mc_pg_rejoin_basebackup_fallback()
|
|
{
|
|
local pgdata="$1"
|
|
local reason="${2:-rejoin fallback requested}"
|
|
local source_bytes free_bytes required_bytes headroom_bytes min_headroom_bytes
|
|
local timestamp preserved_pgdata failed_pgdata source_role_retry
|
|
|
|
if ! mc_pg_rejoin_basebackup_enabled; then
|
|
ocf_exit_reason \
|
|
"${reason}; rejoin_allow_basebackup is not enabled"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log warn \
|
|
"${reason}; evaluating pg_basebackup fallback"
|
|
|
|
# Revalidate the source immediately before replacing local PGDATA.
|
|
source_role_retry="$(pg_remote_query \
|
|
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to revalidate rejoin source before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ "${source_role_retry}" != "f" ]; then
|
|
ocf_exit_reason \
|
|
"Rejoin source is no longer a writable primary; refusing pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if pg_running; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL is still active; refusing pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! service_running || ! container_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance service/container is unavailable; refusing pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! ensure_remote_slot \
|
|
"${REJOIN_SOURCE_HOST}" \
|
|
"${REPLICATION_SLOT}"
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
# Estimate the replacement payload. Existing PGDATA is renamed in place on
|
|
# the same filesystem, so it consumes no additional free space. Require one
|
|
# replacement copy plus configurable safety headroom.
|
|
source_bytes="$(pg_remote_query \
|
|
"SELECT COALESCE(sum(pg_database_size(datname)),0)
|
|
FROM pg_database
|
|
WHERE datallowconn;" 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to estimate source database size before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
case "${source_bytes}" in
|
|
""|*[!0-9]*)
|
|
ocf_exit_reason \
|
|
"Invalid source database size returned before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
esac
|
|
|
|
free_bytes="$(
|
|
pg_command '
|
|
blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")"
|
|
case "$blocks" in
|
|
""|*[!0-9]*) exit 1 ;;
|
|
esac
|
|
printf "%s\\n" "$((blocks * 1024))"
|
|
' 2>/dev/null
|
|
)" || {
|
|
ocf_exit_reason \
|
|
"Unable to determine free space for pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
case "${free_bytes}" in
|
|
""|*[!0-9]*)
|
|
ocf_exit_reason \
|
|
"Invalid free-space value before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
esac
|
|
|
|
min_headroom_bytes=$((1024 * 1024 * 1024))
|
|
headroom_bytes=$((source_bytes * REJOIN_BASEBACKUP_HEADROOM_PERCENT / 100))
|
|
if [ "${headroom_bytes}" -lt "${min_headroom_bytes}" ]; then
|
|
headroom_bytes=${min_headroom_bytes}
|
|
fi
|
|
required_bytes=$((source_bytes + headroom_bytes))
|
|
|
|
ocf_log info \
|
|
"pg_basebackup space check: source-estimate=${source_bytes} headroom=${headroom_bytes} required=${required_bytes} free=${free_bytes}"
|
|
|
|
if [ "${free_bytes}" -lt "${required_bytes}" ]; then
|
|
ocf_exit_reason \
|
|
"Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes} source-estimate=${source_bytes} headroom=${headroom_bytes}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
timestamp="$(date -u +%Y%m%d-%H%M%S)"
|
|
preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}"
|
|
failed_pgdata="${pgdata}.failed-basebackup-${timestamp}"
|
|
|
|
ocf_log warn \
|
|
"Preserving existing PGDATA as ${preserved_pgdata}"
|
|
|
|
if ! pg_command "
|
|
set -e
|
|
[ -d '${pgdata}' ]
|
|
[ ! -e '${preserved_pgdata}' ]
|
|
[ ! -e '${failed_pgdata}' ]
|
|
mv '${pgdata}' '${preserved_pgdata}'
|
|
install -d -m 0700 '${pgdata}'
|
|
"
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to preserve existing PGDATA for pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log warn \
|
|
"Running pg_basebackup from ${REJOIN_SOURCE_HOST}"
|
|
|
|
if ! pg_replication_exec \
|
|
pg_basebackup \
|
|
--dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \
|
|
--pgdata="${pgdata}" \
|
|
--format=plain \
|
|
--wal-method=stream \
|
|
--slot="${REPLICATION_SLOT}" \
|
|
--write-recovery-conf \
|
|
--progress
|
|
then
|
|
ocf_log err \
|
|
"pg_basebackup failed; preserving failed copy and restoring previous PGDATA"
|
|
|
|
if ! pg_command "
|
|
set -e
|
|
[ ! -e '${failed_pgdata}' ]
|
|
if [ -e '${pgdata}' ]; then
|
|
mv '${pgdata}' '${failed_pgdata}'
|
|
fi
|
|
mv '${preserved_pgdata}' '${pgdata}'
|
|
"
|
|
then
|
|
ocf_exit_reason \
|
|
"pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_exit_reason \
|
|
"pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}"
|
|
return "${OCF_SUCCESS}"
|
|
}
|
|
|
|
mc_pg_rejoin_configure_standby()
|
|
{
|
|
if [ -n "${OCF_RESKEY_replication_peers:-}" ]; then
|
|
mc_pg_write_peer_recovery || {
|
|
ocf_exit_reason "Failed to configure native peer recovery"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
|
|
if ! mc_runtime_exec \
|
|
--user "${PG_OS_USER}" \
|
|
--env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \
|
|
--env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \
|
|
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
|
--env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \
|
|
--env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \
|
|
"${CONTAINER_NAME}" \
|
|
sh -c '
|
|
set -e
|
|
sed -i \\
|
|
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \\
|
|
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \\
|
|
"$PGDATA/postgresql.auto.conf"
|
|
printf "%s\\n" \\
|
|
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \\
|
|
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \\
|
|
>> "$PGDATA/postgresql.auto.conf"
|
|
touch "$PGDATA/standby.signal"
|
|
chmod 600 "$PGDATA/postgresql.auto.conf" "$PGDATA/standby.signal"
|
|
'
|
|
then
|
|
ocf_exit_reason "Failed to configure standby settings"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
return "${OCF_SUCCESS}"
|
|
}
|
|
|
|
mc_pg_rejoin_start_and_wait_streaming()
|
|
{
|
|
local role receiver_status i
|
|
|
|
ocf_log info "Starting PostgreSQL as standby"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
-l "$PGDATA/postgresql.log" \
|
|
start -w
|
|
'
|
|
then
|
|
ocf_exit_reason "PostgreSQL failed to start after rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
for i in $(seq 1 30); do
|
|
role="$(pg_role 2>/dev/null)" || {
|
|
sleep 1
|
|
continue
|
|
}
|
|
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
|
|
|
if [ "${role}" = "t" ] && [ "${receiver_status}" = "streaming" ]; then
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
managed_rejoin()
|
|
{
|
|
local source_role pgdata
|
|
local fallback_used=false
|
|
|
|
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
|
true|false|1|0|yes|no) ;;
|
|
*)
|
|
ocf_exit_reason "rejoin_allow_basebackup must be true or false"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
;;
|
|
esac
|
|
|
|
if [ -z "${REJOIN_SOURCE_HOST}" ]; then
|
|
ocf_exit_reason "rejoin_source_host is required for rejoin"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
if ! service_running; then
|
|
ocf_exit_reason "Maintenance service ${SERVICE_NAME} must be running before rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! container_running; then
|
|
ocf_exit_reason "Maintenance container ${CONTAINER_NAME} must be running before rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
mc_pg_require_replication_slot || return "$?"
|
|
|
|
if pg_running; then
|
|
ocf_exit_reason "PostgreSQL must be stopped before rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info "Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}"
|
|
source_role="$(pg_remote_query 'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
|
ocf_exit_reason "Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ "${source_role}" != "f" ]; then
|
|
ocf_exit_reason "Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info "Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
|
if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}"; then
|
|
ocf_exit_reason "Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
pgdata="$(pgdata_path 2>/dev/null)" || {
|
|
ocf_exit_reason "Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
if [ -z "${pgdata}" ]; then
|
|
ocf_exit_reason "PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info "Running pg_rewind against ${REJOIN_SOURCE_HOST}"
|
|
if ! pg_replication_exec \
|
|
pg_rewind \
|
|
--target-pgdata="${pgdata}" \
|
|
--source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
|
--progress
|
|
then
|
|
mc_pg_rejoin_basebackup_fallback "${pgdata}" "pg_rewind failed" || return "$?"
|
|
fallback_used=true
|
|
fi
|
|
|
|
ocf_log info "Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}"
|
|
mc_pg_rejoin_configure_standby || return "$?"
|
|
|
|
if mc_pg_rejoin_start_and_wait_streaming; then
|
|
pg_clear_rejoin_pending >/dev/null 2>&1 || true
|
|
ocf_log info "PostgreSQL successfully rejoined as streaming standby"
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
|
|
# A successful pg_rewind does not guarantee that the WAL required to join
|
|
# the new timeline is still available. If the standby cannot reach
|
|
# streaming, stop it and perform the same guarded full-resync fallback.
|
|
if [ "${fallback_used}" = "true" ]; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL did not become a streaming standby even after pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! mc_pg_rejoin_basebackup_enabled; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL started in recovery but did not become streaming; rejoin_allow_basebackup is not enabled"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log warn \
|
|
"PostgreSQL entered recovery but did not reach streaming after pg_rewind; stopping it for pg_basebackup fallback"
|
|
|
|
# The failed post-rewind start may already have exited. Stopping an
|
|
# already-stopped server is success for fallback purposes.
|
|
if pg_running; then
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" -D "$PGDATA" stop -m fast -w
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to stop non-streaming standby before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
else
|
|
ocf_log info \
|
|
"PostgreSQL is already stopped; proceeding directly to pg_basebackup fallback"
|
|
fi
|
|
|
|
mc_pg_rejoin_basebackup_fallback \
|
|
"${pgdata}" \
|
|
"pg_rewind completed but standby could not establish WAL streaming" || return "$?"
|
|
|
|
ocf_log info "Reconfiguring PostgreSQL standby after pg_basebackup fallback"
|
|
mc_pg_rejoin_configure_standby || return "$?"
|
|
|
|
if ! mc_pg_rejoin_start_and_wait_streaming; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL did not become a streaming standby after pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
pg_clear_rejoin_pending >/dev/null 2>&1 || true
|
|
ocf_log info \
|
|
"PostgreSQL successfully rejoined as streaming standby after pg_basebackup fallback"
|
|
return "${OCF_SUCCESS}"
|
|
}
|