Lyncoln
ec2077f979
Extract PostgreSQL, Podman runtime and systemd lifecycle adapters while preserving existing lifecycle and scoring behavior. Update RPM layout and add guarded operator-run lab harnesses. Record successful manual stop/start, promotion, demotion and rewind-only rejoin with pinned installed hashes. The unfenced lab does not validate automatic failover, Pacemaker-managed transitions or divergent-data rewind. Validation: 103 offline tests, original-function differential checks, checksum manifest and whitespace checks passed.
343 lines
11 KiB
Bash
343 lines
11 KiB
Bash
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
|
|
|
managed_rejoin()
|
|
{
|
|
local source_role role receiver_status pgdata i
|
|
|
|
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
|
true|false|1|0|yes|no)
|
|
;;
|
|
*)
|
|
ocf_exit_reason "rejoin_allow_basebackup must be true or false"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
;;
|
|
esac
|
|
|
|
if [ -z "${REJOIN_SOURCE_HOST}" ]; then
|
|
ocf_exit_reason \
|
|
"rejoin_source_host is required for rejoin"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
if [ -z "${REPLICATION_SLOT}" ]; then
|
|
ocf_exit_reason \
|
|
"replication_slot is required for rejoin"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
if ! service_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance service ${SERVICE_NAME} must be running before rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! container_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance container ${CONTAINER_NAME} must be running before rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if pg_running; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL must be stopped before rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}"
|
|
|
|
source_role="$(pg_remote_query \
|
|
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ "${source_role}" != "f" ]; then
|
|
ocf_exit_reason \
|
|
"Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
|
|
|
if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}"
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Running pg_rewind against ${REJOIN_SOURCE_HOST}"
|
|
|
|
pgdata="$(pgdata_path 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ -z "${pgdata}" ]; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! pg_replication_exec \
|
|
pg_rewind \
|
|
--target-pgdata="${pgdata}" \
|
|
--source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
|
--progress
|
|
then
|
|
local fallback_enabled source_bytes free_bytes required_bytes
|
|
local timestamp preserved_pgdata failed_pgdata source_role_retry
|
|
|
|
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
|
true|1|yes)
|
|
fallback_enabled=true
|
|
;;
|
|
*)
|
|
fallback_enabled=false
|
|
;;
|
|
esac
|
|
|
|
if [ "${fallback_enabled}" != "true" ]; then
|
|
ocf_exit_reason \
|
|
"pg_rewind failed and rejoin_allow_basebackup is not enabled"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log warn \
|
|
"pg_rewind failed; evaluating pg_basebackup fallback"
|
|
|
|
# Revalidate the source immediately before replacing local PGDATA.
|
|
source_role_retry="$(pg_remote_query \
|
|
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to revalidate rejoin source before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ "${source_role_retry}" != "f" ]; then
|
|
ocf_exit_reason \
|
|
"Rejoin source is no longer a writable primary; refusing pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if pg_running; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL became active during rejoin; refusing pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! service_running || ! container_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance service/container is unavailable; refusing pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! ensure_remote_slot \
|
|
"${REJOIN_SOURCE_HOST}" \
|
|
"${REPLICATION_SLOT}"
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
# Estimate source database payload and require 2x that amount free.
|
|
# This is deliberately conservative and does not pretend to be an
|
|
# exact prediction of the resulting PGDATA size.
|
|
source_bytes="$(pg_remote_query \
|
|
"SELECT COALESCE(sum(pg_database_size(datname)),0)
|
|
FROM pg_database
|
|
WHERE datallowconn;" 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to estimate source database size before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
case "${source_bytes}" in
|
|
""|*[!0-9]*)
|
|
ocf_exit_reason \
|
|
"Invalid source database size returned before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
esac
|
|
|
|
free_bytes="$(
|
|
pg_command '
|
|
blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")"
|
|
case "$blocks" in
|
|
""|*[!0-9]*)
|
|
exit 1
|
|
;;
|
|
esac
|
|
printf "%s\n" "$((blocks * 1024))"
|
|
' 2>/dev/null
|
|
)" || {
|
|
ocf_exit_reason \
|
|
"Unable to determine free space for pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
case "${free_bytes}" in
|
|
""|*[!0-9]*)
|
|
ocf_exit_reason \
|
|
"Invalid free-space value before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
esac
|
|
|
|
required_bytes=$((source_bytes * 2))
|
|
|
|
if [ "${free_bytes}" -lt "${required_bytes}" ]; then
|
|
ocf_exit_reason \
|
|
"Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
timestamp="$(date -u +%Y%m%d-%H%M%S)"
|
|
preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}"
|
|
failed_pgdata="${pgdata}.failed-basebackup-${timestamp}"
|
|
|
|
ocf_log warn \
|
|
"Preserving existing PGDATA as ${preserved_pgdata}"
|
|
|
|
if ! pg_command "
|
|
set -e
|
|
|
|
[ -d '${pgdata}' ]
|
|
[ ! -e '${preserved_pgdata}' ]
|
|
[ ! -e '${failed_pgdata}' ]
|
|
|
|
mv '${pgdata}' '${preserved_pgdata}'
|
|
install -d -m 0700 '${pgdata}'
|
|
"
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to preserve existing PGDATA for pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log warn \
|
|
"Running pg_basebackup from ${REJOIN_SOURCE_HOST}"
|
|
|
|
if ! pg_replication_exec \
|
|
pg_basebackup \
|
|
--dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \
|
|
--pgdata="${pgdata}" \
|
|
--format=plain \
|
|
--wal-method=stream \
|
|
--slot="${REPLICATION_SLOT}" \
|
|
--write-recovery-conf \
|
|
--progress
|
|
then
|
|
ocf_log err \
|
|
"pg_basebackup failed; preserving failed copy and restoring previous PGDATA"
|
|
|
|
if ! pg_command "
|
|
set -e
|
|
|
|
if [ -e '${failed_pgdata}' ]; then
|
|
exit 1
|
|
fi
|
|
|
|
if [ -e '${pgdata}' ]; then
|
|
mv '${pgdata}' '${failed_pgdata}'
|
|
fi
|
|
|
|
mv '${preserved_pgdata}' '${pgdata}'
|
|
"
|
|
then
|
|
ocf_exit_reason \
|
|
"pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_exit_reason \
|
|
"pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}"
|
|
|
|
if ! mc_runtime_exec \
|
|
--user "${PG_OS_USER}" \
|
|
--env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \
|
|
--env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \
|
|
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
|
--env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \
|
|
--env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \
|
|
"${CONTAINER_NAME}" \
|
|
sh -c '
|
|
set -e
|
|
|
|
sed -i \
|
|
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \
|
|
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \
|
|
"$PGDATA/postgresql.auto.conf"
|
|
|
|
printf "%s\n" \
|
|
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \
|
|
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \
|
|
>> "$PGDATA/postgresql.auto.conf"
|
|
|
|
touch "$PGDATA/standby.signal"
|
|
|
|
chmod 600 \
|
|
"$PGDATA/postgresql.auto.conf" \
|
|
"$PGDATA/standby.signal"
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"Failed to configure standby settings"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Starting PostgreSQL as standby"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
-l "$PGDATA/postgresql.log" \
|
|
start -w
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"PostgreSQL failed to start after rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
for i in $(seq 1 30); do
|
|
role="$(pg_role 2>/dev/null)" || {
|
|
sleep 1
|
|
continue
|
|
}
|
|
|
|
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
|
|
|
if [ "${role}" = "t" ] &&
|
|
[ "${receiver_status}" = "streaming" ]
|
|
then
|
|
ocf_log info \
|
|
"PostgreSQL successfully rejoined as streaming standby"
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
ocf_exit_reason \
|
|
"PostgreSQL started but did not become a streaming standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|