Initial managed-container HA resource agent baseline
This commit is contained in:
Lyncoln
@@ -0,0 +1,391 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
mc_pg_rejoin_basebackup_enabled()
|
||||
{
|
||||
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
||||
true|1|yes)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
mc_pg_rejoin_basebackup_fallback()
|
||||
{
|
||||
local pgdata="$1"
|
||||
local reason="${2:-rejoin fallback requested}"
|
||||
local source_bytes free_bytes required_bytes headroom_bytes min_headroom_bytes
|
||||
local timestamp preserved_pgdata failed_pgdata source_role_retry
|
||||
|
||||
if ! mc_pg_rejoin_basebackup_enabled; then
|
||||
ocf_exit_reason \
|
||||
"${reason}; rejoin_allow_basebackup is not enabled"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"${reason}; evaluating pg_basebackup fallback"
|
||||
|
||||
# Revalidate the source immediately before replacing local PGDATA.
|
||||
source_role_retry="$(pg_remote_query \
|
||||
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to revalidate rejoin source before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${source_role_retry}" != "f" ]; then
|
||||
ocf_exit_reason \
|
||||
"Rejoin source is no longer a writable primary; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL is still active; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! service_running || ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service/container is unavailable; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_remote_slot \
|
||||
"${REJOIN_SOURCE_HOST}" \
|
||||
"${REPLICATION_SLOT}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
# Estimate the replacement payload. Existing PGDATA is renamed in place on
|
||||
# the same filesystem, so it consumes no additional free space. Require one
|
||||
# replacement copy plus configurable safety headroom.
|
||||
source_bytes="$(pg_remote_query \
|
||||
"SELECT COALESCE(sum(pg_database_size(datname)),0)
|
||||
FROM pg_database
|
||||
WHERE datallowconn;" 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to estimate source database size before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${source_bytes}" in
|
||||
""|*[!0-9]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid source database size returned before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
free_bytes="$(
|
||||
pg_command '
|
||||
blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")"
|
||||
case "$blocks" in
|
||||
""|*[!0-9]*) exit 1 ;;
|
||||
esac
|
||||
printf "%s\\n" "$((blocks * 1024))"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to determine free space for pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${free_bytes}" in
|
||||
""|*[!0-9]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid free-space value before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
min_headroom_bytes=$((1024 * 1024 * 1024))
|
||||
headroom_bytes=$((source_bytes * REJOIN_BASEBACKUP_HEADROOM_PERCENT / 100))
|
||||
if [ "${headroom_bytes}" -lt "${min_headroom_bytes}" ]; then
|
||||
headroom_bytes=${min_headroom_bytes}
|
||||
fi
|
||||
required_bytes=$((source_bytes + headroom_bytes))
|
||||
|
||||
ocf_log info \
|
||||
"pg_basebackup space check: source-estimate=${source_bytes} headroom=${headroom_bytes} required=${required_bytes} free=${free_bytes}"
|
||||
|
||||
if [ "${free_bytes}" -lt "${required_bytes}" ]; then
|
||||
ocf_exit_reason \
|
||||
"Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes} source-estimate=${source_bytes} headroom=${headroom_bytes}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
timestamp="$(date -u +%Y%m%d-%H%M%S)"
|
||||
preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}"
|
||||
failed_pgdata="${pgdata}.failed-basebackup-${timestamp}"
|
||||
|
||||
ocf_log warn \
|
||||
"Preserving existing PGDATA as ${preserved_pgdata}"
|
||||
|
||||
if ! pg_command "
|
||||
set -e
|
||||
[ -d '${pgdata}' ]
|
||||
[ ! -e '${preserved_pgdata}' ]
|
||||
[ ! -e '${failed_pgdata}' ]
|
||||
mv '${pgdata}' '${preserved_pgdata}'
|
||||
install -d -m 0700 '${pgdata}'
|
||||
"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to preserve existing PGDATA for pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"Running pg_basebackup from ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
if ! pg_replication_exec \
|
||||
pg_basebackup \
|
||||
--dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \
|
||||
--pgdata="${pgdata}" \
|
||||
--format=plain \
|
||||
--wal-method=stream \
|
||||
--slot="${REPLICATION_SLOT}" \
|
||||
--write-recovery-conf \
|
||||
--progress
|
||||
then
|
||||
ocf_log err \
|
||||
"pg_basebackup failed; preserving failed copy and restoring previous PGDATA"
|
||||
|
||||
if ! pg_command "
|
||||
set -e
|
||||
[ ! -e '${failed_pgdata}' ]
|
||||
if [ -e '${pgdata}' ]; then
|
||||
mv '${pgdata}' '${failed_pgdata}'
|
||||
fi
|
||||
mv '${preserved_pgdata}' '${pgdata}'
|
||||
"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_exit_reason \
|
||||
"pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}"
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
mc_pg_rejoin_configure_standby()
|
||||
{
|
||||
if [ -n "${OCF_RESKEY_replication_peers:-}" ]; then
|
||||
mc_pg_write_peer_recovery || {
|
||||
ocf_exit_reason "Failed to configure native peer recovery"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
if ! mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
--env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \
|
||||
--env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \
|
||||
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
||||
--env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \
|
||||
--env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
set -e
|
||||
sed -i \\
|
||||
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \\
|
||||
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \\
|
||||
"$PGDATA/postgresql.auto.conf"
|
||||
printf "%s\\n" \\
|
||||
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \\
|
||||
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \\
|
||||
>> "$PGDATA/postgresql.auto.conf"
|
||||
touch "$PGDATA/standby.signal"
|
||||
chmod 600 "$PGDATA/postgresql.auto.conf" "$PGDATA/standby.signal"
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "Failed to configure standby settings"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
mc_pg_rejoin_start_and_wait_streaming()
|
||||
{
|
||||
local role receiver_status i
|
||||
|
||||
ocf_log info "Starting PostgreSQL as standby"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "PostgreSQL failed to start after rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
sleep 1
|
||||
continue
|
||||
}
|
||||
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
||||
|
||||
if [ "${role}" = "t" ] && [ "${receiver_status}" = "streaming" ]; then
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
managed_rejoin()
|
||||
{
|
||||
local source_role pgdata
|
||||
local fallback_used=false
|
||||
|
||||
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
||||
true|false|1|0|yes|no) ;;
|
||||
*)
|
||||
ocf_exit_reason "rejoin_allow_basebackup must be true or false"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "${REJOIN_SOURCE_HOST}" ]; then
|
||||
ocf_exit_reason "rejoin_source_host is required for rejoin"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
if ! service_running; then
|
||||
ocf_exit_reason "Maintenance service ${SERVICE_NAME} must be running before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! container_running; then
|
||||
ocf_exit_reason "Maintenance container ${CONTAINER_NAME} must be running before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
mc_pg_require_replication_slot || return "$?"
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason "PostgreSQL must be stopped before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info "Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}"
|
||||
source_role="$(pg_remote_query 'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
||||
ocf_exit_reason "Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${source_role}" != "f" ]; then
|
||||
ocf_exit_reason "Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info "Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}"; then
|
||||
ocf_exit_reason "Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
pgdata="$(pgdata_path 2>/dev/null)" || {
|
||||
ocf_exit_reason "Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
if [ -z "${pgdata}" ]; then
|
||||
ocf_exit_reason "PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info "Running pg_rewind against ${REJOIN_SOURCE_HOST}"
|
||||
if ! pg_replication_exec \
|
||||
pg_rewind \
|
||||
--target-pgdata="${pgdata}" \
|
||||
--source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
||||
--progress
|
||||
then
|
||||
mc_pg_rejoin_basebackup_fallback "${pgdata}" "pg_rewind failed" || return "$?"
|
||||
fallback_used=true
|
||||
fi
|
||||
|
||||
ocf_log info "Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}"
|
||||
mc_pg_rejoin_configure_standby || return "$?"
|
||||
|
||||
if mc_pg_rejoin_start_and_wait_streaming; then
|
||||
pg_clear_rejoin_pending >/dev/null 2>&1 || true
|
||||
ocf_log info "PostgreSQL successfully rejoined as streaming standby"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
# A successful pg_rewind does not guarantee that the WAL required to join
|
||||
# the new timeline is still available. If the standby cannot reach
|
||||
# streaming, stop it and perform the same guarded full-resync fallback.
|
||||
if [ "${fallback_used}" = "true" ]; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL did not become a streaming standby even after pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! mc_pg_rejoin_basebackup_enabled; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL started in recovery but did not become streaming; rejoin_allow_basebackup is not enabled"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"PostgreSQL entered recovery but did not reach streaming after pg_rewind; stopping it for pg_basebackup fallback"
|
||||
|
||||
# The failed post-rewind start may already have exited. Stopping an
|
||||
# already-stopped server is success for fallback purposes.
|
||||
if pg_running; then
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" -D "$PGDATA" stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to stop non-streaming standby before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
else
|
||||
ocf_log info \
|
||||
"PostgreSQL is already stopped; proceeding directly to pg_basebackup fallback"
|
||||
fi
|
||||
|
||||
mc_pg_rejoin_basebackup_fallback \
|
||||
"${pgdata}" \
|
||||
"pg_rewind completed but standby could not establish WAL streaming" || return "$?"
|
||||
|
||||
ocf_log info "Reconfiguring PostgreSQL standby after pg_basebackup fallback"
|
||||
mc_pg_rejoin_configure_standby || return "$?"
|
||||
|
||||
if ! mc_pg_rejoin_start_and_wait_streaming; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL did not become a streaming standby after pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
pg_clear_rejoin_pending >/dev/null 2>&1 || true
|
||||
ocf_log info \
|
||||
"PostgreSQL successfully rejoined as streaming standby after pg_basebackup fallback"
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
Reference in New Issue
Block a user