1722 lines
49 KiB
Bash
Executable File
1722 lines
49 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# OCF Resource Agent:
|
|
# loxthelion:managed-container
|
|
#
|
|
# v0.4
|
|
# Container lifecycle: systemd user Quadlet
|
|
# Application profile: PostgreSQL
|
|
#
|
|
# Version 0.4 adds explicit PostgreSQL demotion scaffolding.
|
|
# Healthy primaries report OCF_RUNNING_PROMOTED while healthy standbys
|
|
# report OCF_SUCCESS.
|
|
|
|
: "${OCF_ROOT:=/usr/lib/ocf}"
|
|
OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat"
|
|
. "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs"
|
|
|
|
# Pacemaker uses "Promoted" terminology, while many OCF shell libraries
|
|
# still expose the historical OCF_RUNNING_MASTER constant for return code 8.
|
|
# Normalize both forms here so the agent remains portable.
|
|
: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}"
|
|
|
|
###############################################################################
|
|
# Defaults
|
|
###############################################################################
|
|
|
|
OCF_RESKEY_service_user_default="david"
|
|
OCF_RESKEY_service_name_default="postgres.service"
|
|
OCF_RESKEY_container_name_default="postgres"
|
|
OCF_RESKEY_pg_os_user_default="postgres"
|
|
OCF_RESKEY_db_user_default="david"
|
|
OCF_RESKEY_db_name_default="david"
|
|
OCF_RESKEY_rejoin_source_host_default=""
|
|
OCF_RESKEY_rejoin_source_port_default="5432"
|
|
OCF_RESKEY_replication_user_default="replicator"
|
|
OCF_RESKEY_replication_slot_default=""
|
|
OCF_RESKEY_application_name_default=""
|
|
OCF_RESKEY_rejoin_allow_basebackup_default="false"
|
|
|
|
: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}"
|
|
: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}"
|
|
: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}"
|
|
: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}"
|
|
: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}"
|
|
: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}"
|
|
: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}"
|
|
: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}"
|
|
: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}"
|
|
: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}"
|
|
: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}"
|
|
: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}"
|
|
|
|
SERVICE_USER="${OCF_RESKEY_service_user}"
|
|
SERVICE_NAME="${OCF_RESKEY_service_name}"
|
|
CONTAINER_NAME="${OCF_RESKEY_container_name}"
|
|
PG_OS_USER="${OCF_RESKEY_pg_os_user}"
|
|
DB_USER="${OCF_RESKEY_db_user}"
|
|
DB_NAME="${OCF_RESKEY_db_name}"
|
|
REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}"
|
|
REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}"
|
|
REPLICATION_USER="${OCF_RESKEY_replication_user}"
|
|
REPLICATION_SLOT="${OCF_RESKEY_replication_slot}"
|
|
APPLICATION_NAME="${OCF_RESKEY_application_name}"
|
|
REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}"
|
|
|
|
if [ -z "${APPLICATION_NAME}" ]; then
|
|
APPLICATION_NAME="$(hostname -s)"
|
|
fi
|
|
|
|
###############################################################################
|
|
# Metadata
|
|
###############################################################################
|
|
|
|
meta_data()
|
|
{
|
|
cat <<EOF_META
|
|
<?xml version="1.0"?>
|
|
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
|
|
<resource-agent name="managed-container" version="0.4">
|
|
<version>1.0</version>
|
|
|
|
<longdesc lang="en">
|
|
Manage an application inside a persistent rootless container.
|
|
|
|
Version 0.4 implements role-aware monitoring, promotion, and demotion.
|
|
The systemd user Quadlet owns the container lifecycle while pg_ctl owns
|
|
the PostgreSQL lifecycle inside the container.
|
|
</longdesc>
|
|
|
|
<shortdesc lang="en">
|
|
Manage PostgreSQL independently inside a persistent container
|
|
</shortdesc>
|
|
|
|
<parameters>
|
|
|
|
<parameter name="service_user" unique="0" required="1">
|
|
<longdesc lang="en">
|
|
Linux user owning the rootless systemd/Podman service.
|
|
</longdesc>
|
|
<shortdesc lang="en">Rootless service owner</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_service_user_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="service_name" unique="0" required="1">
|
|
<longdesc lang="en">
|
|
systemd user service generated by Quadlet.
|
|
</longdesc>
|
|
<shortdesc lang="en">Quadlet service name</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_service_name_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="container_name" unique="0" required="1">
|
|
<longdesc lang="en">
|
|
Podman container containing PostgreSQL.
|
|
</longdesc>
|
|
<shortdesc lang="en">Container name</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_container_name_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="pg_os_user" unique="0" required="1">
|
|
<longdesc lang="en">
|
|
Operating-system user running PostgreSQL inside the container.
|
|
</longdesc>
|
|
<shortdesc lang="en">PostgreSQL OS user</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_pg_os_user_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="db_user" unique="0" required="1">
|
|
<longdesc lang="en">
|
|
PostgreSQL database user used for health queries.
|
|
</longdesc>
|
|
<shortdesc lang="en">Database user</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_db_user_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="db_name" unique="0" required="1">
|
|
<longdesc lang="en">
|
|
PostgreSQL database used for health queries.
|
|
</longdesc>
|
|
<shortdesc lang="en">Database name</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_db_name_default}"/>
|
|
</parameter>
|
|
|
|
|
|
<parameter name="rejoin_source_host" unique="0" required="0">
|
|
<longdesc lang="en">
|
|
PostgreSQL primary host used as the source for rejoin operations.
|
|
</longdesc>
|
|
<shortdesc lang="en">Rejoin source host</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_rejoin_source_host_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="rejoin_source_port" unique="0" required="0">
|
|
<longdesc lang="en">
|
|
PostgreSQL port on the rejoin source.
|
|
</longdesc>
|
|
<shortdesc lang="en">Rejoin source port</shortdesc>
|
|
<content type="integer" default="${OCF_RESKEY_rejoin_source_port_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="replication_user" unique="0" required="0">
|
|
<longdesc lang="en">
|
|
PostgreSQL role used for replication and pg_rewind.
|
|
</longdesc>
|
|
<shortdesc lang="en">Replication user</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_replication_user_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="replication_slot" unique="0" required="0">
|
|
<longdesc lang="en">
|
|
Physical replication slot used by this standby.
|
|
</longdesc>
|
|
<shortdesc lang="en">Physical replication slot</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_replication_slot_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="application_name" unique="0" required="0">
|
|
<longdesc lang="en">
|
|
PostgreSQL replication application_name. Defaults to the local hostname.
|
|
</longdesc>
|
|
<shortdesc lang="en">Replication application name</shortdesc>
|
|
<content type="string" default="${OCF_RESKEY_application_name_default}"/>
|
|
</parameter>
|
|
|
|
<parameter name="rejoin_allow_basebackup" unique="0" required="0">
|
|
<longdesc lang="en">
|
|
Allow rejoin to fall back to pg_basebackup when pg_rewind cannot
|
|
repair the local PostgreSQL data directory. The old PGDATA is
|
|
preserved before replacement. This must only be enabled when the
|
|
operator has established that this node may safely discard its
|
|
divergent PostgreSQL history.
|
|
</longdesc>
|
|
<shortdesc lang="en">Allow pg_basebackup rejoin fallback</shortdesc>
|
|
<content type="boolean" default="${OCF_RESKEY_rejoin_allow_basebackup_default}"/>
|
|
</parameter>
|
|
|
|
</parameters>
|
|
|
|
<actions>
|
|
<action name="start" timeout="60s"/>
|
|
<action name="stop" timeout="60s"/>
|
|
<action name="promote" timeout="60s"/>
|
|
<action name="demote" timeout="60s"/>
|
|
<action name="notify" timeout="20s"/>
|
|
<action name="monitor" timeout="20s" interval="10s"/>
|
|
<action name="validate-all" timeout="20s"/>
|
|
<action name="meta-data" timeout="5s"/>
|
|
</actions>
|
|
</resource-agent>
|
|
EOF_META
|
|
}
|
|
|
|
###############################################################################
|
|
# Rootless-user helpers
|
|
###############################################################################
|
|
|
|
service_uid()
|
|
{
|
|
id -u "${SERVICE_USER}"
|
|
}
|
|
|
|
service_home()
|
|
{
|
|
getent passwd "${SERVICE_USER}" | cut -d: -f6
|
|
}
|
|
|
|
as_service_user()
|
|
{
|
|
local uid home
|
|
|
|
uid="$(id -u "$SERVICE_USER")" || return 1
|
|
home="$(getent passwd "$SERVICE_USER" | cut -d: -f6)" || return 1
|
|
|
|
runuser -u "$SERVICE_USER" -- \
|
|
sh -c '
|
|
home="$1"
|
|
runtime="$2"
|
|
shift 2
|
|
|
|
cd "$home" || exit 1
|
|
|
|
export HOME="$home"
|
|
export XDG_RUNTIME_DIR="$runtime"
|
|
|
|
exec "$@"
|
|
' sh \
|
|
"$home" \
|
|
"/run/user/$uid" \
|
|
"$@"
|
|
}
|
|
|
|
###############################################################################
|
|
# Container / application status
|
|
###############################################################################
|
|
|
|
service_running()
|
|
{
|
|
as_service_user \
|
|
systemctl --user is-active --quiet "${SERVICE_NAME}"
|
|
}
|
|
|
|
container_running()
|
|
{
|
|
local state
|
|
|
|
state="$(
|
|
as_service_user podman inspect \
|
|
--format '{{.State.Running}}' \
|
|
"${CONTAINER_NAME}" 2>/dev/null
|
|
)" || return 1
|
|
|
|
[ "${state}" = "true" ]
|
|
}
|
|
|
|
pg_command()
|
|
{
|
|
local command="$1"
|
|
|
|
as_service_user \
|
|
podman exec \
|
|
--user "${PG_OS_USER}" \
|
|
"${CONTAINER_NAME}" \
|
|
sh -c "${command}"
|
|
}
|
|
|
|
pg_replication_exec()
|
|
{
|
|
as_service_user \
|
|
podman exec \
|
|
--user "${PG_OS_USER}" \
|
|
"${CONTAINER_NAME}" \
|
|
sh -c '
|
|
if [ -z "${POSTGRES_REPLICATION_PASSWORD:-}" ]; then
|
|
printf "%s\n" \
|
|
"POSTGRES_REPLICATION_PASSWORD is missing from container environment" \
|
|
>&2
|
|
exit 1
|
|
fi
|
|
|
|
export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD}"
|
|
|
|
exec "$@"
|
|
' sh "$@"
|
|
}
|
|
|
|
pgdata_path()
|
|
{
|
|
pg_command '
|
|
if [ -z "$PGDATA" ]; then
|
|
exit 1
|
|
fi
|
|
|
|
printf "%s\n" "$PGDATA"
|
|
'
|
|
}
|
|
|
|
pg_running()
|
|
{
|
|
pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
"$BINDIR/pg_ctl" status -D "$PGDATA" >/dev/null 2>&1
|
|
'
|
|
}
|
|
|
|
pg_ready()
|
|
{
|
|
as_service_user \
|
|
podman exec \
|
|
--user "${PG_OS_USER}" \
|
|
"${CONTAINER_NAME}" \
|
|
pg_isready -q \
|
|
-U "${DB_USER}" \
|
|
-d "${DB_NAME}"
|
|
}
|
|
|
|
pg_role()
|
|
{
|
|
as_service_user \
|
|
podman exec \
|
|
--user "${PG_OS_USER}" \
|
|
"${CONTAINER_NAME}" \
|
|
psql \
|
|
-U "${DB_USER}" \
|
|
-d "${DB_NAME}" \
|
|
-Atqc 'SELECT pg_is_in_recovery();'
|
|
}
|
|
|
|
|
|
pg_remote_query()
|
|
{
|
|
local sql="$1"
|
|
|
|
pg_replication_exec \
|
|
psql \
|
|
"host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
|
-Atqc "${sql}"
|
|
}
|
|
|
|
pg_wal_receiver_status()
|
|
{
|
|
as_service_user \
|
|
podman exec \
|
|
--user "${PG_OS_USER}" \
|
|
"${CONTAINER_NAME}" \
|
|
psql \
|
|
-U "${DB_USER}" \
|
|
-d "${DB_NAME}" \
|
|
-Atqc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;"
|
|
}
|
|
|
|
###############################################################################
|
|
# Validation
|
|
###############################################################################
|
|
|
|
validate_all()
|
|
{
|
|
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
|
|
ocf_exit_reason "Service user '${SERVICE_USER}' does not exist"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
for cmd in runuser systemctl podman getent; do
|
|
if ! command -v "${cmd}" >/dev/null 2>&1; then
|
|
ocf_exit_reason "Required command '${cmd}' is not installed"
|
|
return "${OCF_ERR_INSTALLED}"
|
|
fi
|
|
done
|
|
|
|
if [ ! -d "/run/user/$(service_uid)" ]; then
|
|
ocf_exit_reason \
|
|
"Runtime directory /run/user/$(service_uid) does not exist"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! as_service_user \
|
|
systemctl --user cat "${SERVICE_NAME}" >/dev/null 2>&1
|
|
then
|
|
ocf_exit_reason \
|
|
"User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
local podman_error
|
|
|
|
podman_error="$(
|
|
as_service_user podman info 2>&1 >/dev/null
|
|
)"
|
|
local podman_rc=$?
|
|
|
|
if [ "${podman_rc}" -ne 0 ]; then
|
|
ocf_exit_reason \
|
|
"Rootless Podman failed for ${SERVICE_USER}: ${podman_error}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
return "${OCF_SUCCESS}"
|
|
}
|
|
|
|
###############################################################################
|
|
# Monitor
|
|
###############################################################################
|
|
|
|
managed_monitor()
|
|
{
|
|
local role
|
|
|
|
if ! service_running; then
|
|
ocf_log debug "User service ${SERVICE_NAME} is not active"
|
|
return "${OCF_NOT_RUNNING}"
|
|
fi
|
|
|
|
if ! container_running; then
|
|
ocf_log warn \
|
|
"Service ${SERVICE_NAME} is active but container ${CONTAINER_NAME} is not running"
|
|
return "${OCF_NOT_RUNNING}"
|
|
fi
|
|
|
|
if ! pg_running; then
|
|
ocf_log debug \
|
|
"Container ${CONTAINER_NAME} is running but PostgreSQL is stopped"
|
|
return "${OCF_NOT_RUNNING}"
|
|
fi
|
|
|
|
if ! pg_ready; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL process exists but pg_isready failed"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
role="$(pg_role 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"PostgreSQL is accepting connections but role query failed"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
case "${role}" in
|
|
f)
|
|
ocf_log debug \
|
|
"PostgreSQL healthy: role=primary (promoted)"
|
|
return "${OCF_RUNNING_PROMOTED}"
|
|
;;
|
|
t)
|
|
ocf_log debug \
|
|
"PostgreSQL healthy: role=standby (unpromoted)"
|
|
return "${OCF_SUCCESS}"
|
|
;;
|
|
*)
|
|
ocf_exit_reason \
|
|
"Unexpected pg_is_in_recovery() result: '${role}'"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
esac
|
|
|
|
}
|
|
|
|
###############################################################################
|
|
# Start
|
|
###############################################################################
|
|
|
|
managed_start()
|
|
{
|
|
local i monitor_rc
|
|
|
|
managed_monitor
|
|
monitor_rc=$?
|
|
|
|
case "${monitor_rc}" in
|
|
"${OCF_SUCCESS}")
|
|
ocf_log info \
|
|
"PostgreSQL resource already running unpromoted"
|
|
return "${OCF_SUCCESS}"
|
|
;;
|
|
|
|
"${OCF_RUNNING_PROMOTED}")
|
|
ocf_exit_reason \
|
|
"Refusing start: PostgreSQL is already primary; promotion must occur through promote()"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
|
|
"${OCF_NOT_RUNNING}")
|
|
# Expected for a stopped resource. Continue.
|
|
;;
|
|
|
|
*)
|
|
ocf_exit_reason \
|
|
"Cannot start unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
|
return "${monitor_rc}"
|
|
;;
|
|
esac
|
|
|
|
ocf_log info \
|
|
"Ensuring container service ${SERVICE_NAME} is running"
|
|
|
|
if ! service_running; then
|
|
if ! as_service_user \
|
|
systemctl --user start "${SERVICE_NAME}"
|
|
then
|
|
ocf_exit_reason \
|
|
"Failed to start ${SERVICE_NAME}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
fi
|
|
|
|
for i in $(seq 1 20); do
|
|
container_running && break
|
|
sleep 0.5
|
|
done
|
|
|
|
if ! container_running; then
|
|
ocf_exit_reason \
|
|
"Container ${CONTAINER_NAME} did not become running"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! pg_running; then
|
|
#
|
|
# OCF promotable-resource rule:
|
|
# start() must produce an Unpromoted instance.
|
|
#
|
|
# A PostgreSQL data directory without standby.signal would start
|
|
# writable/primary, so never start it from this action.
|
|
#
|
|
if ! pg_command '
|
|
test -f "$PGDATA/standby.signal"
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"Refusing start: PGDATA is not configured as a standby (standby.signal missing)"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Starting PostgreSQL as an unpromoted standby inside ${CONTAINER_NAME}"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
-l "$PGDATA/postgresql.log" \
|
|
start -w
|
|
'
|
|
then
|
|
ocf_exit_reason "pg_ctl start failed"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
fi
|
|
|
|
managed_monitor
|
|
monitor_rc=$?
|
|
|
|
case "${monitor_rc}" in
|
|
"${OCF_SUCCESS}")
|
|
ocf_log info \
|
|
"PostgreSQL start completed in unpromoted state"
|
|
return "${OCF_SUCCESS}"
|
|
;;
|
|
|
|
"${OCF_RUNNING_PROMOTED}")
|
|
ocf_exit_reason \
|
|
"PostgreSQL became primary during start; stopping it to preserve OCF role semantics"
|
|
|
|
pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
stop -m fast -w
|
|
' >/dev/null 2>&1 || true
|
|
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
|
|
*)
|
|
return "${monitor_rc}"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
ensure_rewind_privileges()
|
|
{
|
|
local repl_user
|
|
|
|
repl_user="$(
|
|
pg_command '
|
|
printf "%s" "${POSTGRES_REPLICATION_USER:-}"
|
|
' 2>/dev/null
|
|
)" || {
|
|
ocf_exit_reason \
|
|
"Unable to read POSTGRES_REPLICATION_USER from container"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ -z "${repl_user}" ]; then
|
|
ocf_exit_reason \
|
|
"POSTGRES_REPLICATION_USER is empty"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
case "${repl_user}" in
|
|
*[!A-Za-z0-9_]*)
|
|
ocf_exit_reason \
|
|
"Invalid replication role name: ${repl_user}"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
;;
|
|
esac
|
|
|
|
ocf_log info \
|
|
"Ensuring pg_rewind source privileges for role ${repl_user}"
|
|
|
|
if ! pg_command "
|
|
psql \
|
|
-U '${DB_USER}' \
|
|
-d postgres \
|
|
-v ON_ERROR_STOP=1 \
|
|
-c \"
|
|
GRANT EXECUTE
|
|
ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean)
|
|
TO ${repl_user};
|
|
|
|
GRANT EXECUTE
|
|
ON FUNCTION pg_catalog.pg_stat_file(text, boolean)
|
|
TO ${repl_user};
|
|
|
|
GRANT EXECUTE
|
|
ON FUNCTION pg_catalog.pg_read_binary_file(text)
|
|
TO ${repl_user};
|
|
|
|
GRANT EXECUTE
|
|
ON FUNCTION pg_catalog.pg_read_binary_file(
|
|
text,
|
|
bigint,
|
|
bigint,
|
|
boolean
|
|
)
|
|
TO ${repl_user};
|
|
\"
|
|
" >/dev/null
|
|
then
|
|
ocf_exit_reason \
|
|
"Failed ensuring pg_rewind privileges for ${repl_user}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
return "${OCF_SUCCESS}"
|
|
}
|
|
|
|
|
|
ensure_local_ha_slots()
|
|
{
|
|
local slots raw_slot slot slot_type
|
|
|
|
#
|
|
# POSTGRES_HA_SLOTS lives in the container environment supplied by
|
|
# postgres.env. It is the cluster-wide list of physical slots that
|
|
# every writable primary must provide.
|
|
#
|
|
slots="$(
|
|
pg_command '
|
|
printf "%s" "${POSTGRES_HA_SLOTS:-}"
|
|
' 2>/dev/null
|
|
)" || {
|
|
ocf_exit_reason \
|
|
"Unable to read POSTGRES_HA_SLOTS from PostgreSQL container"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ -z "${slots}" ]; then
|
|
ocf_log warn \
|
|
"POSTGRES_HA_SLOTS is empty; no local HA slots will be reconciled"
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
|
|
while IFS= read -r raw_slot; do
|
|
slot="$(printf '%s' "${raw_slot}" | tr -d '[:space:]')"
|
|
|
|
[ -n "${slot}" ] || continue
|
|
|
|
case "${slot}" in
|
|
*[!a-z0-9_]*)
|
|
ocf_exit_reason \
|
|
"Invalid physical replication slot name: ${slot}"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
;;
|
|
esac
|
|
|
|
slot_type="$(
|
|
pg_command "
|
|
psql \
|
|
-U '${DB_USER}' \
|
|
-d postgres \
|
|
-At \
|
|
-v ON_ERROR_STOP=1 \
|
|
-c \"
|
|
SELECT slot_type
|
|
FROM pg_replication_slots
|
|
WHERE slot_name = '${slot}';
|
|
\"
|
|
" 2>/dev/null
|
|
)" || {
|
|
ocf_exit_reason \
|
|
"Failed querying replication slot ${slot}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
slot_type="$(printf '%s' "${slot_type}" | tr -d '[:space:]')"
|
|
|
|
case "${slot_type}" in
|
|
physical)
|
|
ocf_log info \
|
|
"Physical HA slot already exists: ${slot}"
|
|
;;
|
|
|
|
"")
|
|
ocf_log info \
|
|
"Creating physical HA slot: ${slot}"
|
|
|
|
if ! pg_command "
|
|
psql \
|
|
-U '${DB_USER}' \
|
|
-d postgres \
|
|
-v ON_ERROR_STOP=1 \
|
|
-c \"
|
|
SELECT pg_create_physical_replication_slot('${slot}');
|
|
\"
|
|
" >/dev/null
|
|
then
|
|
ocf_exit_reason \
|
|
"Failed creating physical replication slot ${slot}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
;;
|
|
|
|
*)
|
|
ocf_exit_reason \
|
|
"Replication slot ${slot} exists but is type ${slot_type}"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
;;
|
|
esac
|
|
|
|
done <<EOF
|
|
$(printf '%s\n' "${slots}" | tr ',' '\n')
|
|
EOF
|
|
|
|
return "${OCF_SUCCESS}"
|
|
}
|
|
|
|
|
|
managed_promote()
|
|
{
|
|
local monitor_rc role i
|
|
|
|
managed_monitor
|
|
monitor_rc=$?
|
|
|
|
case "${monitor_rc}" in
|
|
"${OCF_RUNNING_PROMOTED}")
|
|
ocf_log info \
|
|
"PostgreSQL is already primary; reconciling primary invariants"
|
|
|
|
if ! ensure_local_ha_slots; then
|
|
ocf_exit_reason \
|
|
"HA slot reconciliation failed on existing primary"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! ensure_rewind_privileges; then
|
|
ocf_exit_reason \
|
|
"pg_rewind privilege reconciliation failed on existing primary"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
return "${OCF_SUCCESS}"
|
|
;;
|
|
|
|
"${OCF_SUCCESS}")
|
|
# Healthy standby: continue with promotion.
|
|
;;
|
|
|
|
"${OCF_NOT_RUNNING}")
|
|
ocf_exit_reason \
|
|
"Cannot promote PostgreSQL because the resource is not running"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
|
|
*)
|
|
ocf_exit_reason \
|
|
"Cannot promote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
|
return "${monitor_rc}"
|
|
;;
|
|
esac
|
|
|
|
ocf_log info \
|
|
"Promoting PostgreSQL standby inside ${CONTAINER_NAME}"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
promote -w
|
|
'
|
|
then
|
|
ocf_exit_reason "pg_ctl promote failed"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
for i in $(seq 1 30); do
|
|
role="$(pg_role 2>/dev/null)" || {
|
|
sleep 1
|
|
continue
|
|
}
|
|
|
|
if [ "${role}" = "f" ]; then
|
|
ocf_log info \
|
|
"PostgreSQL became primary; reconciling primary invariants"
|
|
|
|
if ! ensure_local_ha_slots; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL promoted but HA slot reconciliation failed"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! ensure_rewind_privileges; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL promoted but pg_rewind privilege reconciliation failed"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"PostgreSQL promotion completed successfully"
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
ocf_exit_reason \
|
|
"PostgreSQL did not become primary after promotion"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
###############################################################################
|
|
# Stop
|
|
###############################################################################
|
|
|
|
node_to_addr()
|
|
{
|
|
local target="${1:-}"
|
|
|
|
[ -n "${target}" ] || return 1
|
|
|
|
corosync-cmapctl 2>/dev/null |
|
|
awk -v target="${target}" '
|
|
/nodelist\.node\.[0-9]+\.name / {
|
|
key=$1
|
|
sub(/\.name$/, "", key)
|
|
names[key]=$NF
|
|
}
|
|
|
|
/nodelist\.node\.[0-9]+\.ring0_addr / {
|
|
key=$1
|
|
sub(/\.ring0_addr$/, "", key)
|
|
addrs[key]=$NF
|
|
}
|
|
|
|
END {
|
|
for (key in names) {
|
|
if (names[key] == target && addrs[key] != "") {
|
|
print addrs[key]
|
|
exit 0
|
|
}
|
|
}
|
|
exit 1
|
|
}
|
|
'
|
|
}
|
|
|
|
candidate_caught_up()
|
|
{
|
|
local candidate="${1:-}"
|
|
local result
|
|
|
|
[ -n "${candidate}" ] || return 1
|
|
|
|
result="$(pg_command "
|
|
psql -U '${DB_USER}' -d '${DB_NAME}' -Atc \"
|
|
SELECT CASE
|
|
WHEN state = 'streaming'
|
|
AND replay_lsn >= pg_current_wal_flush_lsn()
|
|
THEN 'yes'
|
|
ELSE 'no'
|
|
END
|
|
FROM pg_stat_replication
|
|
WHERE application_name = '${candidate}'
|
|
LIMIT 1;
|
|
\"
|
|
" 2>/dev/null)" || return 1
|
|
|
|
[ "${result}" = "yes" ]
|
|
}
|
|
|
|
demote_target()
|
|
{
|
|
local state_file
|
|
local type operation promote_uname demote_uname local_node
|
|
local now mtime age
|
|
|
|
state_file="/run/loxthelion/${OCF_RESOURCE_INSTANCE:-managed-container}.notify"
|
|
local_node="$(hostname)"
|
|
|
|
[ -r "${state_file}" ] || return 1
|
|
|
|
now="$(date +%s)" || return 1
|
|
mtime="$(stat -c %Y "${state_file}" 2>/dev/null)" || return 1
|
|
age=$((now - mtime))
|
|
|
|
[ "${age}" -ge 0 ] || return 1
|
|
[ "${age}" -le 120 ] || return 1
|
|
|
|
type="$(
|
|
sed -n 's/^type=//p' "${state_file}" | head -n1
|
|
)"
|
|
|
|
operation="$(
|
|
sed -n 's/^operation=//p' "${state_file}" | head -n1
|
|
)"
|
|
|
|
promote_uname="$(
|
|
sed -n 's/^promote_uname=//p' "${state_file}" | head -n1
|
|
)"
|
|
|
|
demote_uname="$(
|
|
sed -n 's/^demote_uname=//p' "${state_file}" | head -n1
|
|
)"
|
|
|
|
[ "${type}" = "pre" ] || return 1
|
|
[ "${operation}" = "demote" ] || return 1
|
|
[ "${demote_uname}" = "${local_node}" ] || return 1
|
|
[ -n "${promote_uname}" ] || return 1
|
|
[ "${promote_uname}" != "${local_node}" ] || return 1
|
|
|
|
case "${promote_uname}" in
|
|
*" "*) return 1 ;;
|
|
esac
|
|
|
|
printf '%s
|
|
' "${promote_uname}"
|
|
}
|
|
|
|
|
|
ensure_remote_slot()
|
|
{
|
|
local host="${1:-}"
|
|
local slot="${2:-}"
|
|
local result
|
|
|
|
[ -n "${host}" ] || return 1
|
|
[ -n "${slot}" ] || return 1
|
|
|
|
result="$(
|
|
pg_replication_exec \
|
|
psql \
|
|
"host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \
|
|
-Atc "
|
|
SELECT slot_type
|
|
FROM pg_replication_slots
|
|
WHERE slot_name = '${slot}';
|
|
" 2>/dev/null
|
|
)" || return 1
|
|
|
|
case "${result}" in
|
|
physical)
|
|
return 0
|
|
;;
|
|
|
|
"")
|
|
if ! pg_replication_exec \
|
|
psql \
|
|
"host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \
|
|
-Atc "
|
|
SELECT slot_name
|
|
FROM pg_create_physical_replication_slot('${slot}');
|
|
" >/dev/null
|
|
then
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
;;
|
|
|
|
*)
|
|
ocf_log err \
|
|
"Replication slot ${slot} exists on ${host} but is not physical"
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
candidate_is_standby()
|
|
{
|
|
local host="${1:-}"
|
|
local result
|
|
local port="${REJOIN_SOURCE_PORT:-5432}"
|
|
|
|
[ -n "${host}" ] || return 1
|
|
|
|
result="$(
|
|
pg_replication_exec \
|
|
psql \
|
|
"host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres" \
|
|
-Atc "SELECT pg_is_in_recovery();" \
|
|
2>/dev/null
|
|
)" || return 1
|
|
|
|
[ "${result}" = "t" ]
|
|
}
|
|
|
|
managed_demote()
|
|
{
|
|
local monitor_rc
|
|
local target_node target_addr
|
|
local local_node slot_name target_port
|
|
local role receiver_status i
|
|
|
|
managed_monitor
|
|
monitor_rc=$?
|
|
|
|
case "${monitor_rc}" in
|
|
"${OCF_RUNNING_PROMOTED}")
|
|
;;
|
|
"${OCF_SUCCESS}")
|
|
ocf_log info \
|
|
"PostgreSQL is already unpromoted"
|
|
return "${OCF_SUCCESS}"
|
|
;;
|
|
"${OCF_NOT_RUNNING}")
|
|
ocf_exit_reason \
|
|
"Cannot complete demotion because PostgreSQL is stopped"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
*)
|
|
ocf_exit_reason \
|
|
"Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
|
return "${monitor_rc}"
|
|
;;
|
|
esac
|
|
|
|
target_node="$(demote_target)" || {
|
|
ocf_exit_reason \
|
|
"No fresh valid Pacemaker pre-demote target is available"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
target_addr="$(node_to_addr "${target_node}")" || {
|
|
ocf_exit_reason \
|
|
"Unable to resolve Pacemaker promotion target ${target_node} through Corosync"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
target_port="${REJOIN_SOURCE_PORT:-5432}"
|
|
local_node="$(hostname)"
|
|
slot_name="ha_$(printf '%s' "${local_node}" |
|
|
tr '[:upper:]-' '[:lower:]_')"
|
|
|
|
ocf_log info \
|
|
"Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})"
|
|
|
|
if ! candidate_is_standby "${target_addr}"; then
|
|
ocf_exit_reason \
|
|
"Promotion target ${target_node} is not a reachable PostgreSQL standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! candidate_caught_up "${target_node}"; then
|
|
ocf_exit_reason \
|
|
"Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then
|
|
ocf_exit_reason \
|
|
"Unable to ensure physical replication slot ${slot_name} on ${target_node}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Demotion safety checks passed; stopping PostgreSQL primary"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
stop -m fast -w
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"pg_ctl stop failed during controlled demotion"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if pg_running; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL remained running after pg_ctl stop"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! service_running || ! container_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance service/container stopped unexpectedly during demotion"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Configuring ${local_node} as cascading standby of ${target_node}"
|
|
|
|
if ! as_service_user \
|
|
podman exec \
|
|
--user "${PG_OS_USER}" \
|
|
--env "RA_PRIMARY_HOST=${target_addr}" \
|
|
--env "RA_PRIMARY_PORT=${target_port}" \
|
|
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
|
--env "RA_REPLICATION_SLOT=${slot_name}" \
|
|
--env "RA_APPLICATION_NAME=${local_node}" \
|
|
"${CONTAINER_NAME}" \
|
|
sh -c '
|
|
set -e
|
|
|
|
sed -i \
|
|
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \
|
|
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \
|
|
"$PGDATA/postgresql.auto.conf"
|
|
|
|
printf "%s
|
|
" \
|
|
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \
|
|
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \
|
|
>> "$PGDATA/postgresql.auto.conf"
|
|
|
|
touch "$PGDATA/standby.signal"
|
|
|
|
chmod 600 \
|
|
"$PGDATA/postgresql.auto.conf" \
|
|
"$PGDATA/standby.signal"
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"Failed to configure PostgreSQL as standby of ${target_node}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Starting PostgreSQL as cascading standby of ${target_node}"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
-l "$PGDATA/postgresql.log" \
|
|
start -w
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"PostgreSQL failed to start as cascading standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
for i in $(seq 1 30); do
|
|
role="$(pg_role 2>/dev/null || true)"
|
|
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
|
|
|
if [ "${role}" = "t" ] &&
|
|
[ "${receiver_status}" = "streaming" ]
|
|
then
|
|
ocf_log info \
|
|
"Controlled demotion complete: ${local_node} is streaming from ${target_node}"
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
ocf_exit_reason \
|
|
"PostgreSQL started after demotion but did not become a streaming standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
|
|
managed_rejoin()
|
|
{
|
|
local source_role role receiver_status pgdata i
|
|
|
|
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
|
true|false|1|0|yes|no)
|
|
;;
|
|
*)
|
|
ocf_exit_reason "rejoin_allow_basebackup must be true or false"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
;;
|
|
esac
|
|
|
|
if [ -z "${REJOIN_SOURCE_HOST}" ]; then
|
|
ocf_exit_reason \
|
|
"rejoin_source_host is required for rejoin"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
if [ -z "${REPLICATION_SLOT}" ]; then
|
|
ocf_exit_reason \
|
|
"replication_slot is required for rejoin"
|
|
return "${OCF_ERR_CONFIGURED}"
|
|
fi
|
|
|
|
if ! service_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance service ${SERVICE_NAME} must be running before rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! container_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance container ${CONTAINER_NAME} must be running before rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if pg_running; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL must be stopped before rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}"
|
|
|
|
source_role="$(pg_remote_query \
|
|
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ "${source_role}" != "f" ]; then
|
|
ocf_exit_reason \
|
|
"Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
|
|
|
if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}"
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Running pg_rewind against ${REJOIN_SOURCE_HOST}"
|
|
|
|
pgdata="$(pgdata_path 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ -z "${pgdata}" ]; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! pg_replication_exec \
|
|
pg_rewind \
|
|
--target-pgdata="${pgdata}" \
|
|
--source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
|
--progress
|
|
then
|
|
local fallback_enabled source_bytes free_bytes required_bytes
|
|
local timestamp preserved_pgdata failed_pgdata source_role_retry
|
|
|
|
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
|
true|1|yes)
|
|
fallback_enabled=true
|
|
;;
|
|
*)
|
|
fallback_enabled=false
|
|
;;
|
|
esac
|
|
|
|
if [ "${fallback_enabled}" != "true" ]; then
|
|
ocf_exit_reason \
|
|
"pg_rewind failed and rejoin_allow_basebackup is not enabled"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log warn \
|
|
"pg_rewind failed; evaluating pg_basebackup fallback"
|
|
|
|
# Revalidate the source immediately before replacing local PGDATA.
|
|
source_role_retry="$(pg_remote_query \
|
|
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to revalidate rejoin source before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
if [ "${source_role_retry}" != "f" ]; then
|
|
ocf_exit_reason \
|
|
"Rejoin source is no longer a writable primary; refusing pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if pg_running; then
|
|
ocf_exit_reason \
|
|
"PostgreSQL became active during rejoin; refusing pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! service_running || ! container_running; then
|
|
ocf_exit_reason \
|
|
"Maintenance service/container is unavailable; refusing pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! ensure_remote_slot \
|
|
"${REJOIN_SOURCE_HOST}" \
|
|
"${REPLICATION_SLOT}"
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
# Estimate source database payload and require 2x that amount free.
|
|
# This is deliberately conservative and does not pretend to be an
|
|
# exact prediction of the resulting PGDATA size.
|
|
source_bytes="$(pg_remote_query \
|
|
"SELECT COALESCE(sum(pg_database_size(datname)),0)
|
|
FROM pg_database
|
|
WHERE datallowconn;" 2>/dev/null)" || {
|
|
ocf_exit_reason \
|
|
"Unable to estimate source database size before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
case "${source_bytes}" in
|
|
""|*[!0-9]*)
|
|
ocf_exit_reason \
|
|
"Invalid source database size returned before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
esac
|
|
|
|
free_bytes="$(
|
|
pg_command '
|
|
blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")"
|
|
case "$blocks" in
|
|
""|*[!0-9]*)
|
|
exit 1
|
|
;;
|
|
esac
|
|
printf "%s\n" "$((blocks * 1024))"
|
|
' 2>/dev/null
|
|
)" || {
|
|
ocf_exit_reason \
|
|
"Unable to determine free space for pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
case "${free_bytes}" in
|
|
""|*[!0-9]*)
|
|
ocf_exit_reason \
|
|
"Invalid free-space value before pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
;;
|
|
esac
|
|
|
|
required_bytes=$((source_bytes * 2))
|
|
|
|
if [ "${free_bytes}" -lt "${required_bytes}" ]; then
|
|
ocf_exit_reason \
|
|
"Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
timestamp="$(date -u +%Y%m%d-%H%M%S)"
|
|
preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}"
|
|
failed_pgdata="${pgdata}.failed-basebackup-${timestamp}"
|
|
|
|
ocf_log warn \
|
|
"Preserving existing PGDATA as ${preserved_pgdata}"
|
|
|
|
if ! pg_command "
|
|
set -e
|
|
|
|
[ -d '${pgdata}' ]
|
|
[ ! -e '${preserved_pgdata}' ]
|
|
[ ! -e '${failed_pgdata}' ]
|
|
|
|
mv '${pgdata}' '${preserved_pgdata}'
|
|
install -d -m 0700 '${pgdata}'
|
|
"
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to preserve existing PGDATA for pg_basebackup fallback"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log warn \
|
|
"Running pg_basebackup from ${REJOIN_SOURCE_HOST}"
|
|
|
|
if ! pg_replication_exec \
|
|
pg_basebackup \
|
|
--dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \
|
|
--pgdata="${pgdata}" \
|
|
--format=plain \
|
|
--wal-method=stream \
|
|
--slot="${REPLICATION_SLOT}" \
|
|
--write-recovery-conf \
|
|
--progress
|
|
then
|
|
ocf_log err \
|
|
"pg_basebackup failed; preserving failed copy and restoring previous PGDATA"
|
|
|
|
if ! pg_command "
|
|
set -e
|
|
|
|
if [ -e '${failed_pgdata}' ]; then
|
|
exit 1
|
|
fi
|
|
|
|
if [ -e '${pgdata}' ]; then
|
|
mv '${pgdata}' '${failed_pgdata}'
|
|
fi
|
|
|
|
mv '${preserved_pgdata}' '${pgdata}'
|
|
"
|
|
then
|
|
ocf_exit_reason \
|
|
"pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_exit_reason \
|
|
"pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}"
|
|
|
|
if ! as_service_user \
|
|
podman exec \
|
|
--user "${PG_OS_USER}" \
|
|
--env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \
|
|
--env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \
|
|
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
|
--env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \
|
|
--env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \
|
|
"${CONTAINER_NAME}" \
|
|
sh -c '
|
|
set -e
|
|
|
|
sed -i \
|
|
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \
|
|
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \
|
|
"$PGDATA/postgresql.auto.conf"
|
|
|
|
printf "%s\n" \
|
|
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \
|
|
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \
|
|
>> "$PGDATA/postgresql.auto.conf"
|
|
|
|
touch "$PGDATA/standby.signal"
|
|
|
|
chmod 600 \
|
|
"$PGDATA/postgresql.auto.conf" \
|
|
"$PGDATA/standby.signal"
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"Failed to configure standby settings"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
ocf_log info \
|
|
"Starting PostgreSQL as standby"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
-l "$PGDATA/postgresql.log" \
|
|
start -w
|
|
'
|
|
then
|
|
ocf_exit_reason \
|
|
"PostgreSQL failed to start after rejoin"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
for i in $(seq 1 30); do
|
|
role="$(pg_role 2>/dev/null)" || {
|
|
sleep 1
|
|
continue
|
|
}
|
|
|
|
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
|
|
|
if [ "${role}" = "t" ] &&
|
|
[ "${receiver_status}" = "streaming" ]
|
|
then
|
|
ocf_log info \
|
|
"PostgreSQL successfully rejoined as streaming standby"
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
ocf_exit_reason \
|
|
"PostgreSQL started but did not become a streaming standby"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
managed_stop()
|
|
{
|
|
local i
|
|
|
|
if container_running && pg_running; then
|
|
ocf_log info \
|
|
"Stopping PostgreSQL cleanly inside ${CONTAINER_NAME}"
|
|
|
|
if ! pg_command '
|
|
BINDIR="$(pg_config --bindir)" || exit 1
|
|
exec "$BINDIR/pg_ctl" \
|
|
-D "$PGDATA" \
|
|
stop -m fast -w
|
|
'
|
|
then
|
|
ocf_exit_reason "pg_ctl stop failed"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
fi
|
|
|
|
if service_running; then
|
|
ocf_log info \
|
|
"Stopping container service ${SERVICE_NAME}"
|
|
|
|
if ! as_service_user \
|
|
systemctl --user stop "${SERVICE_NAME}"
|
|
then
|
|
ocf_exit_reason \
|
|
"Failed to stop ${SERVICE_NAME}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
fi
|
|
|
|
for i in $(seq 1 20); do
|
|
if ! service_running && ! container_running; then
|
|
return "${OCF_SUCCESS}"
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
if service_running; then
|
|
ocf_exit_reason \
|
|
"${SERVICE_NAME} remained active after stop"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if container_running; then
|
|
ocf_exit_reason \
|
|
"${CONTAINER_NAME} remained running after stop"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
return "${OCF_SUCCESS}"
|
|
}
|
|
|
|
managed_notify()
|
|
{
|
|
local state_dir state_file
|
|
|
|
state_dir="/run/loxthelion"
|
|
state_file="${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify"
|
|
|
|
ocf_log info \
|
|
"notify: type=${OCF_RESKEY_CRM_meta_notify_type:-unset} operation=${OCF_RESKEY_CRM_meta_notify_operation:-unset}"
|
|
|
|
ocf_log info \
|
|
"notify: promote_uname=${OCF_RESKEY_CRM_meta_notify_promote_uname:-none} demote_uname=${OCF_RESKEY_CRM_meta_notify_demote_uname:-none}"
|
|
|
|
ocf_log info \
|
|
"notify: promoted_uname=${OCF_RESKEY_CRM_meta_notify_promoted_uname:-none} unpromoted_uname=${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-none}"
|
|
|
|
if ! mkdir -p -m 0755 "${state_dir}"; then
|
|
ocf_exit_reason \
|
|
"Unable to create notification state directory ${state_dir}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
if ! {
|
|
printf 'timestamp=%s\n' "$(date --iso-8601=seconds)"
|
|
printf 'node=%s\n' "$(hostname)"
|
|
printf 'type=%s\n' "${OCF_RESKEY_CRM_meta_notify_type:-}"
|
|
printf 'operation=%s\n' "${OCF_RESKEY_CRM_meta_notify_operation:-}"
|
|
printf 'promote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}"
|
|
printf 'demote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}"
|
|
printf 'promoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}"
|
|
printf 'unpromoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}"
|
|
} > "${state_file}"
|
|
then
|
|
ocf_exit_reason \
|
|
"Unable to write notification state ${state_file}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
fi
|
|
|
|
chmod 0600 "${state_file}" || {
|
|
ocf_exit_reason \
|
|
"Unable to secure notification state ${state_file}"
|
|
return "${OCF_ERR_GENERIC}"
|
|
}
|
|
|
|
{
|
|
printf '%s node=%s type=%s operation=%s promote=[%s] demote=[%s] promoted=[%s] unpromoted=[%s]\n' \
|
|
"$(date --iso-8601=seconds)" \
|
|
"$(hostname)" \
|
|
"${OCF_RESKEY_CRM_meta_notify_type:-}" \
|
|
"${OCF_RESKEY_CRM_meta_notify_operation:-}" \
|
|
"${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" \
|
|
"${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" \
|
|
"${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" \
|
|
"${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" \
|
|
>> "${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify.log"
|
|
} || true
|
|
|
|
return "${OCF_SUCCESS}"
|
|
}
|
|
|
|
###############################################################################
|
|
# Dispatcher
|
|
###############################################################################
|
|
|
|
case "$1" in
|
|
meta-data)
|
|
meta_data
|
|
exit "${OCF_SUCCESS}"
|
|
;;
|
|
|
|
validate-all)
|
|
validate_all
|
|
exit $?
|
|
;;
|
|
|
|
monitor|status)
|
|
managed_monitor
|
|
exit $?
|
|
;;
|
|
|
|
start)
|
|
validate_all || exit $?
|
|
managed_start
|
|
exit $?
|
|
;;
|
|
|
|
stop)
|
|
managed_stop
|
|
exit $?
|
|
;;
|
|
|
|
promote)
|
|
validate_all || exit $?
|
|
managed_promote
|
|
exit $?
|
|
;;
|
|
|
|
demote)
|
|
validate_all || exit $?
|
|
managed_demote
|
|
exit $?
|
|
;;
|
|
|
|
notify)
|
|
managed_notify
|
|
exit $?
|
|
;;
|
|
|
|
rejoin)
|
|
validate_all || exit $?
|
|
managed_rejoin
|
|
exit $?
|
|
;;
|
|
|
|
*)
|
|
echo "Usage: $0 {start|stop|promote|demote|notify|rejoin|monitor|status|validate-all|meta-data}"
|
|
exit "${OCF_ERR_UNIMPLEMENTED}"
|
|
;;
|
|
esac
|