Add PostgreSQL HA OCF agent modular baseline
This commit is contained in:
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
|
||||
# OCF Resource Agent:
|
||||
# loxthelion:managed-container
|
||||
#
|
||||
# v0.4
|
||||
# Container lifecycle: systemd user Quadlet
|
||||
# Application profile: PostgreSQL
|
||||
#
|
||||
# Version 0.4 adds explicit PostgreSQL demotion scaffolding.
|
||||
# Healthy primaries report OCF_RUNNING_PROMOTED while healthy standbys
|
||||
# report OCF_SUCCESS.
|
||||
|
||||
: "${OCF_ROOT:=/usr/lib/ocf}"
|
||||
OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat"
|
||||
. "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs"
|
||||
|
||||
# Pacemaker uses "Promoted" terminology, while many OCF shell libraries
|
||||
# still expose the historical OCF_RUNNING_MASTER constant for return code 8.
|
||||
# Normalize both forms here so the agent remains portable.
|
||||
: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}"
|
||||
|
||||
|
||||
# Resolve modules relative to this entrypoint, independent of Pacemaker's cwd.
|
||||
MC_MODULE_DIR="$(CDPATH= cd -- "$(command dirname -- "${BASH_SOURCE[0]}")" && pwd)/managed-container.d"
|
||||
for MC_MODULE in common container postgres auth replication notify promote demote rejoin; do
|
||||
if ! . "${MC_MODULE_DIR}/${MC_MODULE}.sh"; then
|
||||
ocf_exit_reason "Unable to load resource-agent module ${MC_MODULE_DIR}/${MC_MODULE}.sh"
|
||||
exit "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
done
|
||||
unset MC_MODULE
|
||||
|
||||
case "$1" in
|
||||
meta-data)
|
||||
meta_data
|
||||
exit "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
validate-all)
|
||||
validate_all
|
||||
exit $?
|
||||
;;
|
||||
|
||||
monitor|status)
|
||||
managed_monitor
|
||||
exit $?
|
||||
;;
|
||||
|
||||
start)
|
||||
validate_all || exit $?
|
||||
managed_start
|
||||
exit $?
|
||||
;;
|
||||
|
||||
stop)
|
||||
managed_stop
|
||||
exit $?
|
||||
;;
|
||||
|
||||
promote)
|
||||
validate_all || exit $?
|
||||
managed_promote
|
||||
exit $?
|
||||
;;
|
||||
|
||||
demote)
|
||||
validate_all || exit $?
|
||||
managed_demote
|
||||
exit $?
|
||||
;;
|
||||
|
||||
notify)
|
||||
managed_notify
|
||||
exit $?
|
||||
;;
|
||||
|
||||
rejoin)
|
||||
validate_all || exit $?
|
||||
managed_rejoin
|
||||
exit $?
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 {start|stop|promote|demote|notify|rejoin|monitor|status|validate-all|meta-data}"
|
||||
exit "${OCF_ERR_UNIMPLEMENTED}"
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,89 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
pg_replication_exec()
|
||||
{
|
||||
as_service_user \
|
||||
podman exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
if [ -z "${POSTGRES_REPLICATION_PASSWORD:-}" ]; then
|
||||
printf "%s\n" \
|
||||
"POSTGRES_REPLICATION_PASSWORD is missing from container environment" \
|
||||
>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD}"
|
||||
|
||||
exec "$@"
|
||||
' sh "$@"
|
||||
}
|
||||
|
||||
ensure_rewind_privileges()
|
||||
{
|
||||
local repl_user
|
||||
|
||||
repl_user="$(
|
||||
pg_command '
|
||||
printf "%s" "${POSTGRES_REPLICATION_USER:-}"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to read POSTGRES_REPLICATION_USER from container"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ -z "${repl_user}" ]; then
|
||||
ocf_exit_reason \
|
||||
"POSTGRES_REPLICATION_USER is empty"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
case "${repl_user}" in
|
||||
*[!A-Za-z0-9_]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid replication role name: ${repl_user}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
ocf_log info \
|
||||
"Ensuring pg_rewind source privileges for role ${repl_user}"
|
||||
|
||||
if ! pg_command "
|
||||
psql \
|
||||
-U '${DB_USER}' \
|
||||
-d postgres \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-c \"
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean)
|
||||
TO ${repl_user};
|
||||
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_stat_file(text, boolean)
|
||||
TO ${repl_user};
|
||||
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_read_binary_file(text)
|
||||
TO ${repl_user};
|
||||
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_read_binary_file(
|
||||
text,
|
||||
bigint,
|
||||
bigint,
|
||||
boolean
|
||||
)
|
||||
TO ${repl_user};
|
||||
\"
|
||||
" >/dev/null
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed ensuring pg_rewind privileges for ${repl_user}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
OCF_RESKEY_service_user_default="david"
|
||||
OCF_RESKEY_service_name_default="postgres.service"
|
||||
OCF_RESKEY_container_name_default="postgres"
|
||||
OCF_RESKEY_pg_os_user_default="postgres"
|
||||
OCF_RESKEY_db_user_default="david"
|
||||
OCF_RESKEY_db_name_default="david"
|
||||
OCF_RESKEY_rejoin_source_host_default=""
|
||||
OCF_RESKEY_rejoin_source_port_default="5432"
|
||||
OCF_RESKEY_replication_user_default="replicator"
|
||||
OCF_RESKEY_replication_slot_default=""
|
||||
OCF_RESKEY_application_name_default=""
|
||||
OCF_RESKEY_rejoin_allow_basebackup_default="false"
|
||||
|
||||
: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}"
|
||||
: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}"
|
||||
: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}"
|
||||
: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}"
|
||||
: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}"
|
||||
: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}"
|
||||
: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}"
|
||||
: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}"
|
||||
: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}"
|
||||
: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}"
|
||||
: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}"
|
||||
: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}"
|
||||
|
||||
SERVICE_USER="${OCF_RESKEY_service_user}"
|
||||
SERVICE_NAME="${OCF_RESKEY_service_name}"
|
||||
CONTAINER_NAME="${OCF_RESKEY_container_name}"
|
||||
PG_OS_USER="${OCF_RESKEY_pg_os_user}"
|
||||
DB_USER="${OCF_RESKEY_db_user}"
|
||||
DB_NAME="${OCF_RESKEY_db_name}"
|
||||
REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}"
|
||||
REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}"
|
||||
REPLICATION_USER="${OCF_RESKEY_replication_user}"
|
||||
REPLICATION_SLOT="${OCF_RESKEY_replication_slot}"
|
||||
APPLICATION_NAME="${OCF_RESKEY_application_name}"
|
||||
REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}"
|
||||
|
||||
if [ -z "${APPLICATION_NAME}" ]; then
|
||||
APPLICATION_NAME="$(hostname -s)"
|
||||
fi
|
||||
|
||||
meta_data()
|
||||
{
|
||||
cat <<EOF_META
|
||||
<?xml version="1.0"?>
|
||||
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
|
||||
<resource-agent name="managed-container" version="0.4">
|
||||
<version>1.0</version>
|
||||
|
||||
<longdesc lang="en">
|
||||
Manage an application inside a persistent rootless container.
|
||||
|
||||
Version 0.4 implements role-aware monitoring, promotion, and demotion.
|
||||
The systemd user Quadlet owns the container lifecycle while pg_ctl owns
|
||||
the PostgreSQL lifecycle inside the container.
|
||||
</longdesc>
|
||||
|
||||
<shortdesc lang="en">
|
||||
Manage PostgreSQL independently inside a persistent container
|
||||
</shortdesc>
|
||||
|
||||
<parameters>
|
||||
|
||||
<parameter name="service_user" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
Linux user owning the rootless systemd/Podman service.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Rootless service owner</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_service_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="service_name" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
systemd user service generated by Quadlet.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Quadlet service name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_service_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="container_name" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
Podman container containing PostgreSQL.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Container name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_container_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="pg_os_user" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
Operating-system user running PostgreSQL inside the container.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">PostgreSQL OS user</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_pg_os_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="db_user" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL database user used for health queries.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Database user</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_db_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="db_name" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL database used for health queries.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Database name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_db_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
|
||||
<parameter name="rejoin_source_host" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL primary host used as the source for rejoin operations.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Rejoin source host</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_rejoin_source_host_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="rejoin_source_port" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL port on the rejoin source.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Rejoin source port</shortdesc>
|
||||
<content type="integer" default="${OCF_RESKEY_rejoin_source_port_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="replication_user" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL role used for replication and pg_rewind.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Replication user</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_replication_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="replication_slot" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
Physical replication slot used by this standby.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Physical replication slot</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_replication_slot_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="application_name" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL replication application_name. Defaults to the local hostname.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Replication application name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_application_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="rejoin_allow_basebackup" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
Allow rejoin to fall back to pg_basebackup when pg_rewind cannot
|
||||
repair the local PostgreSQL data directory. The old PGDATA is
|
||||
preserved before replacement. This must only be enabled when the
|
||||
operator has established that this node may safely discard its
|
||||
divergent PostgreSQL history.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Allow pg_basebackup rejoin fallback</shortdesc>
|
||||
<content type="boolean" default="${OCF_RESKEY_rejoin_allow_basebackup_default}"/>
|
||||
</parameter>
|
||||
|
||||
</parameters>
|
||||
|
||||
<actions>
|
||||
<action name="start" timeout="60s"/>
|
||||
<action name="stop" timeout="60s"/>
|
||||
<action name="promote" timeout="60s"/>
|
||||
<action name="demote" timeout="60s"/>
|
||||
<action name="notify" timeout="20s"/>
|
||||
<action name="monitor" timeout="20s" interval="10s"/>
|
||||
<action name="validate-all" timeout="20s"/>
|
||||
<action name="meta-data" timeout="5s"/>
|
||||
</actions>
|
||||
</resource-agent>
|
||||
EOF_META
|
||||
}
|
||||
|
||||
validate_all()
|
||||
{
|
||||
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
|
||||
ocf_exit_reason "Service user '${SERVICE_USER}' does not exist"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
for cmd in runuser systemctl podman getent; do
|
||||
if ! command -v "${cmd}" >/dev/null 2>&1; then
|
||||
ocf_exit_reason "Required command '${cmd}' is not installed"
|
||||
return "${OCF_ERR_INSTALLED}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ! -d "/run/user/$(service_uid)" ]; then
|
||||
ocf_exit_reason \
|
||||
"Runtime directory /run/user/$(service_uid) does not exist"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! as_service_user \
|
||||
systemctl --user cat "${SERVICE_NAME}" >/dev/null 2>&1
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
local podman_error
|
||||
|
||||
podman_error="$(
|
||||
as_service_user podman info 2>&1 >/dev/null
|
||||
)"
|
||||
local podman_rc=$?
|
||||
|
||||
if [ "${podman_rc}" -ne 0 ]; then
|
||||
ocf_exit_reason \
|
||||
"Rootless Podman failed for ${SERVICE_USER}: ${podman_error}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
service_uid()
|
||||
{
|
||||
id -u "${SERVICE_USER}"
|
||||
}
|
||||
|
||||
service_home()
|
||||
{
|
||||
getent passwd "${SERVICE_USER}" | cut -d: -f6
|
||||
}
|
||||
|
||||
as_service_user()
|
||||
{
|
||||
local uid home
|
||||
|
||||
uid="$(id -u "$SERVICE_USER")" || return 1
|
||||
home="$(getent passwd "$SERVICE_USER" | cut -d: -f6)" || return 1
|
||||
|
||||
runuser -u "$SERVICE_USER" -- \
|
||||
sh -c '
|
||||
home="$1"
|
||||
runtime="$2"
|
||||
shift 2
|
||||
|
||||
cd "$home" || exit 1
|
||||
|
||||
export HOME="$home"
|
||||
export XDG_RUNTIME_DIR="$runtime"
|
||||
|
||||
exec "$@"
|
||||
' sh \
|
||||
"$home" \
|
||||
"/run/user/$uid" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
service_running()
|
||||
{
|
||||
as_service_user \
|
||||
systemctl --user is-active --quiet "${SERVICE_NAME}"
|
||||
}
|
||||
|
||||
container_running()
|
||||
{
|
||||
local state
|
||||
|
||||
state="$(
|
||||
as_service_user podman inspect \
|
||||
--format '{{.State.Running}}' \
|
||||
"${CONTAINER_NAME}" 2>/dev/null
|
||||
)" || return 1
|
||||
|
||||
[ "${state}" = "true" ]
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
node_to_addr()
|
||||
{
|
||||
local target="${1:-}"
|
||||
|
||||
[ -n "${target}" ] || return 1
|
||||
|
||||
corosync-cmapctl 2>/dev/null |
|
||||
awk -v target="${target}" '
|
||||
/nodelist\.node\.[0-9]+\.name / {
|
||||
key=$1
|
||||
sub(/\.name$/, "", key)
|
||||
names[key]=$NF
|
||||
}
|
||||
|
||||
/nodelist\.node\.[0-9]+\.ring0_addr / {
|
||||
key=$1
|
||||
sub(/\.ring0_addr$/, "", key)
|
||||
addrs[key]=$NF
|
||||
}
|
||||
|
||||
END {
|
||||
for (key in names) {
|
||||
if (names[key] == target && addrs[key] != "") {
|
||||
print addrs[key]
|
||||
exit 0
|
||||
}
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
'
|
||||
}
|
||||
|
||||
managed_demote()
|
||||
{
|
||||
local monitor_rc
|
||||
local target_node target_addr
|
||||
local local_node slot_name target_port
|
||||
local role receiver_status i
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
;;
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL is already unpromoted"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
"${OCF_NOT_RUNNING}")
|
||||
ocf_exit_reason \
|
||||
"Cannot complete demotion because PostgreSQL is stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
target_node="$(demote_target)" || {
|
||||
ocf_exit_reason \
|
||||
"No fresh valid Pacemaker pre-demote target is available"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
target_addr="$(node_to_addr "${target_node}")" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to resolve Pacemaker promotion target ${target_node} through Corosync"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
target_port="${REJOIN_SOURCE_PORT:-5432}"
|
||||
local_node="$(hostname)"
|
||||
slot_name="ha_$(printf '%s' "${local_node}" |
|
||||
tr '[:upper:]-' '[:lower:]_')"
|
||||
|
||||
ocf_log info \
|
||||
"Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})"
|
||||
|
||||
if ! candidate_is_standby "${target_addr}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not a reachable PostgreSQL standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! candidate_caught_up "${target_node}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then
|
||||
ocf_exit_reason \
|
||||
"Unable to ensure physical replication slot ${slot_name} on ${target_node}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Demotion safety checks passed; stopping PostgreSQL primary"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"pg_ctl stop failed during controlled demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL remained running after pg_ctl stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! service_running || ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service/container stopped unexpectedly during demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Configuring ${local_node} as cascading standby of ${target_node}"
|
||||
|
||||
if ! as_service_user \
|
||||
podman exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
--env "RA_PRIMARY_HOST=${target_addr}" \
|
||||
--env "RA_PRIMARY_PORT=${target_port}" \
|
||||
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
||||
--env "RA_REPLICATION_SLOT=${slot_name}" \
|
||||
--env "RA_APPLICATION_NAME=${local_node}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
set -e
|
||||
|
||||
sed -i \
|
||||
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \
|
||||
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \
|
||||
"$PGDATA/postgresql.auto.conf"
|
||||
|
||||
printf "%s
|
||||
" \
|
||||
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \
|
||||
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \
|
||||
>> "$PGDATA/postgresql.auto.conf"
|
||||
|
||||
touch "$PGDATA/standby.signal"
|
||||
|
||||
chmod 600 \
|
||||
"$PGDATA/postgresql.auto.conf" \
|
||||
"$PGDATA/standby.signal"
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to configure PostgreSQL as standby of ${target_node}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL as cascading standby of ${target_node}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL failed to start as cascading standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null || true)"
|
||||
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
||||
|
||||
if [ "${role}" = "t" ] &&
|
||||
[ "${receiver_status}" = "streaming" ]
|
||||
then
|
||||
ocf_log info \
|
||||
"Controlled demotion complete: ${local_node} is streaming from ${target_node}"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL started after demotion but did not become a streaming standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
demote_target()
|
||||
{
|
||||
local state_file
|
||||
local type operation promote_uname demote_uname local_node
|
||||
local now mtime age
|
||||
|
||||
state_file="/run/loxthelion/${OCF_RESOURCE_INSTANCE:-managed-container}.notify"
|
||||
local_node="$(hostname)"
|
||||
|
||||
[ -r "${state_file}" ] || return 1
|
||||
|
||||
now="$(date +%s)" || return 1
|
||||
mtime="$(stat -c %Y "${state_file}" 2>/dev/null)" || return 1
|
||||
age=$((now - mtime))
|
||||
|
||||
[ "${age}" -ge 0 ] || return 1
|
||||
[ "${age}" -le 120 ] || return 1
|
||||
|
||||
type="$(
|
||||
sed -n 's/^type=//p' "${state_file}" | head -n1
|
||||
)"
|
||||
|
||||
operation="$(
|
||||
sed -n 's/^operation=//p' "${state_file}" | head -n1
|
||||
)"
|
||||
|
||||
promote_uname="$(
|
||||
sed -n 's/^promote_uname=//p' "${state_file}" | head -n1
|
||||
)"
|
||||
|
||||
demote_uname="$(
|
||||
sed -n 's/^demote_uname=//p' "${state_file}" | head -n1
|
||||
)"
|
||||
|
||||
[ "${type}" = "pre" ] || return 1
|
||||
[ "${operation}" = "demote" ] || return 1
|
||||
[ "${demote_uname}" = "${local_node}" ] || return 1
|
||||
[ -n "${promote_uname}" ] || return 1
|
||||
[ "${promote_uname}" != "${local_node}" ] || return 1
|
||||
|
||||
case "${promote_uname}" in
|
||||
*" "*) return 1 ;;
|
||||
esac
|
||||
|
||||
printf '%s
|
||||
' "${promote_uname}"
|
||||
}
|
||||
|
||||
managed_notify()
|
||||
{
|
||||
local state_dir state_file
|
||||
|
||||
state_dir="/run/loxthelion"
|
||||
state_file="${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify"
|
||||
|
||||
ocf_log info \
|
||||
"notify: type=${OCF_RESKEY_CRM_meta_notify_type:-unset} operation=${OCF_RESKEY_CRM_meta_notify_operation:-unset}"
|
||||
|
||||
ocf_log info \
|
||||
"notify: promote_uname=${OCF_RESKEY_CRM_meta_notify_promote_uname:-none} demote_uname=${OCF_RESKEY_CRM_meta_notify_demote_uname:-none}"
|
||||
|
||||
ocf_log info \
|
||||
"notify: promoted_uname=${OCF_RESKEY_CRM_meta_notify_promoted_uname:-none} unpromoted_uname=${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-none}"
|
||||
|
||||
if ! mkdir -p -m 0755 "${state_dir}"; then
|
||||
ocf_exit_reason \
|
||||
"Unable to create notification state directory ${state_dir}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! {
|
||||
printf 'timestamp=%s\n' "$(date --iso-8601=seconds)"
|
||||
printf 'node=%s\n' "$(hostname)"
|
||||
printf 'type=%s\n' "${OCF_RESKEY_CRM_meta_notify_type:-}"
|
||||
printf 'operation=%s\n' "${OCF_RESKEY_CRM_meta_notify_operation:-}"
|
||||
printf 'promote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}"
|
||||
printf 'demote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}"
|
||||
printf 'promoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}"
|
||||
printf 'unpromoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}"
|
||||
} > "${state_file}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to write notification state ${state_file}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
chmod 0600 "${state_file}" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to secure notification state ${state_file}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
{
|
||||
printf '%s node=%s type=%s operation=%s promote=[%s] demote=[%s] promoted=[%s] unpromoted=[%s]\n' \
|
||||
"$(date --iso-8601=seconds)" \
|
||||
"$(hostname)" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_type:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_operation:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" \
|
||||
>> "${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify.log"
|
||||
} || true
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
pg_command()
|
||||
{
|
||||
local command="$1"
|
||||
|
||||
as_service_user \
|
||||
podman exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c "${command}"
|
||||
}
|
||||
|
||||
pgdata_path()
|
||||
{
|
||||
pg_command '
|
||||
if [ -z "$PGDATA" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "%s\n" "$PGDATA"
|
||||
'
|
||||
}
|
||||
|
||||
pg_running()
|
||||
{
|
||||
pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
"$BINDIR/pg_ctl" status -D "$PGDATA" >/dev/null 2>&1
|
||||
'
|
||||
}
|
||||
|
||||
pg_ready()
|
||||
{
|
||||
as_service_user \
|
||||
podman exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
pg_isready -q \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}"
|
||||
}
|
||||
|
||||
pg_role()
|
||||
{
|
||||
as_service_user \
|
||||
podman exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}" \
|
||||
-Atqc 'SELECT pg_is_in_recovery();'
|
||||
}
|
||||
|
||||
pg_wal_receiver_status()
|
||||
{
|
||||
as_service_user \
|
||||
podman exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}" \
|
||||
-Atqc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;"
|
||||
}
|
||||
|
||||
managed_monitor()
|
||||
{
|
||||
local role
|
||||
|
||||
if ! service_running; then
|
||||
ocf_log debug "User service ${SERVICE_NAME} is not active"
|
||||
return "${OCF_NOT_RUNNING}"
|
||||
fi
|
||||
|
||||
if ! container_running; then
|
||||
ocf_log warn \
|
||||
"Service ${SERVICE_NAME} is active but container ${CONTAINER_NAME} is not running"
|
||||
return "${OCF_NOT_RUNNING}"
|
||||
fi
|
||||
|
||||
if ! pg_running; then
|
||||
ocf_log debug \
|
||||
"Container ${CONTAINER_NAME} is running but PostgreSQL is stopped"
|
||||
return "${OCF_NOT_RUNNING}"
|
||||
fi
|
||||
|
||||
if ! pg_ready; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL process exists but pg_isready failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL is accepting connections but role query failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${role}" in
|
||||
f)
|
||||
ocf_log debug \
|
||||
"PostgreSQL healthy: role=primary (promoted)"
|
||||
return "${OCF_RUNNING_PROMOTED}"
|
||||
;;
|
||||
t)
|
||||
ocf_log debug \
|
||||
"PostgreSQL healthy: role=standby (unpromoted)"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Unexpected pg_is_in_recovery() result: '${role}'"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
}
|
||||
|
||||
managed_start()
|
||||
{
|
||||
local i monitor_rc
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL resource already running unpromoted"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
ocf_exit_reason \
|
||||
"Refusing start: PostgreSQL is already primary; promotion must occur through promote()"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
"${OCF_NOT_RUNNING}")
|
||||
# Expected for a stopped resource. Continue.
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot start unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
ocf_log info \
|
||||
"Ensuring container service ${SERVICE_NAME} is running"
|
||||
|
||||
if ! service_running; then
|
||||
if ! as_service_user \
|
||||
systemctl --user start "${SERVICE_NAME}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to start ${SERVICE_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
for i in $(seq 1 20); do
|
||||
container_running && break
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Container ${CONTAINER_NAME} did not become running"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! pg_running; then
|
||||
#
|
||||
# OCF promotable-resource rule:
|
||||
# start() must produce an Unpromoted instance.
|
||||
#
|
||||
# A PostgreSQL data directory without standby.signal would start
|
||||
# writable/primary, so never start it from this action.
|
||||
#
|
||||
if ! pg_command '
|
||||
test -f "$PGDATA/standby.signal"
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Refusing start: PGDATA is not configured as a standby (standby.signal missing)"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL as an unpromoted standby inside ${CONTAINER_NAME}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "pg_ctl start failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL start completed in unpromoted state"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL became primary during start; stopping it to preserve OCF role semantics"
|
||||
|
||||
pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
' >/dev/null 2>&1 || true
|
||||
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
*)
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
managed_stop()
|
||||
{
|
||||
local i
|
||||
|
||||
if container_running && pg_running; then
|
||||
ocf_log info \
|
||||
"Stopping PostgreSQL cleanly inside ${CONTAINER_NAME}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "pg_ctl stop failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if service_running; then
|
||||
ocf_log info \
|
||||
"Stopping container service ${SERVICE_NAME}"
|
||||
|
||||
if ! as_service_user \
|
||||
systemctl --user stop "${SERVICE_NAME}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to stop ${SERVICE_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
for i in $(seq 1 20); do
|
||||
if ! service_running && ! container_running; then
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if service_running; then
|
||||
ocf_exit_reason \
|
||||
"${SERVICE_NAME} remained active after stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if container_running; then
|
||||
ocf_exit_reason \
|
||||
"${CONTAINER_NAME} remained running after stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
managed_promote()
|
||||
{
|
||||
local monitor_rc role i
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
ocf_log info \
|
||||
"PostgreSQL is already primary; reconciling primary invariants"
|
||||
|
||||
if ! ensure_local_ha_slots; then
|
||||
ocf_exit_reason \
|
||||
"HA slot reconciliation failed on existing primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_rewind_privileges; then
|
||||
ocf_exit_reason \
|
||||
"pg_rewind privilege reconciliation failed on existing primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
"${OCF_SUCCESS}")
|
||||
# Healthy standby: continue with promotion.
|
||||
;;
|
||||
|
||||
"${OCF_NOT_RUNNING}")
|
||||
ocf_exit_reason \
|
||||
"Cannot promote PostgreSQL because the resource is not running"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot promote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
ocf_log info \
|
||||
"Promoting PostgreSQL standby inside ${CONTAINER_NAME}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
promote -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "pg_ctl promote failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
sleep 1
|
||||
continue
|
||||
}
|
||||
|
||||
if [ "${role}" = "f" ]; then
|
||||
ocf_log info \
|
||||
"PostgreSQL became primary; reconciling primary invariants"
|
||||
|
||||
if ! ensure_local_ha_slots; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL promoted but HA slot reconciliation failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_rewind_privileges; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL promoted but pg_rewind privilege reconciliation failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"PostgreSQL promotion completed successfully"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL did not become primary after promotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
managed_rejoin()
|
||||
{
|
||||
local source_role role receiver_status pgdata i
|
||||
|
||||
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
||||
true|false|1|0|yes|no)
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason "rejoin_allow_basebackup must be true or false"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "${REJOIN_SOURCE_HOST}" ]; then
|
||||
ocf_exit_reason \
|
||||
"rejoin_source_host is required for rejoin"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
if [ -z "${REPLICATION_SLOT}" ]; then
|
||||
ocf_exit_reason \
|
||||
"replication_slot is required for rejoin"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
if ! service_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service ${SERVICE_NAME} must be running before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance container ${CONTAINER_NAME} must be running before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL must be stopped before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}"
|
||||
|
||||
source_role="$(pg_remote_query \
|
||||
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${source_role}" != "f" ]; then
|
||||
ocf_exit_reason \
|
||||
"Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Running pg_rewind against ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
pgdata="$(pgdata_path 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ -z "${pgdata}" ]; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! pg_replication_exec \
|
||||
pg_rewind \
|
||||
--target-pgdata="${pgdata}" \
|
||||
--source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
||||
--progress
|
||||
then
|
||||
local fallback_enabled source_bytes free_bytes required_bytes
|
||||
local timestamp preserved_pgdata failed_pgdata source_role_retry
|
||||
|
||||
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
||||
true|1|yes)
|
||||
fallback_enabled=true
|
||||
;;
|
||||
*)
|
||||
fallback_enabled=false
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "${fallback_enabled}" != "true" ]; then
|
||||
ocf_exit_reason \
|
||||
"pg_rewind failed and rejoin_allow_basebackup is not enabled"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"pg_rewind failed; evaluating pg_basebackup fallback"
|
||||
|
||||
# Revalidate the source immediately before replacing local PGDATA.
|
||||
source_role_retry="$(pg_remote_query \
|
||||
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to revalidate rejoin source before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${source_role_retry}" != "f" ]; then
|
||||
ocf_exit_reason \
|
||||
"Rejoin source is no longer a writable primary; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL became active during rejoin; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! service_running || ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service/container is unavailable; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_remote_slot \
|
||||
"${REJOIN_SOURCE_HOST}" \
|
||||
"${REPLICATION_SLOT}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
# Estimate source database payload and require 2x that amount free.
|
||||
# This is deliberately conservative and does not pretend to be an
|
||||
# exact prediction of the resulting PGDATA size.
|
||||
source_bytes="$(pg_remote_query \
|
||||
"SELECT COALESCE(sum(pg_database_size(datname)),0)
|
||||
FROM pg_database
|
||||
WHERE datallowconn;" 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to estimate source database size before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${source_bytes}" in
|
||||
""|*[!0-9]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid source database size returned before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
free_bytes="$(
|
||||
pg_command '
|
||||
blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")"
|
||||
case "$blocks" in
|
||||
""|*[!0-9]*)
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
printf "%s\n" "$((blocks * 1024))"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to determine free space for pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${free_bytes}" in
|
||||
""|*[!0-9]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid free-space value before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
required_bytes=$((source_bytes * 2))
|
||||
|
||||
if [ "${free_bytes}" -lt "${required_bytes}" ]; then
|
||||
ocf_exit_reason \
|
||||
"Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
timestamp="$(date -u +%Y%m%d-%H%M%S)"
|
||||
preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}"
|
||||
failed_pgdata="${pgdata}.failed-basebackup-${timestamp}"
|
||||
|
||||
ocf_log warn \
|
||||
"Preserving existing PGDATA as ${preserved_pgdata}"
|
||||
|
||||
if ! pg_command "
|
||||
set -e
|
||||
|
||||
[ -d '${pgdata}' ]
|
||||
[ ! -e '${preserved_pgdata}' ]
|
||||
[ ! -e '${failed_pgdata}' ]
|
||||
|
||||
mv '${pgdata}' '${preserved_pgdata}'
|
||||
install -d -m 0700 '${pgdata}'
|
||||
"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to preserve existing PGDATA for pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"Running pg_basebackup from ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
if ! pg_replication_exec \
|
||||
pg_basebackup \
|
||||
--dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \
|
||||
--pgdata="${pgdata}" \
|
||||
--format=plain \
|
||||
--wal-method=stream \
|
||||
--slot="${REPLICATION_SLOT}" \
|
||||
--write-recovery-conf \
|
||||
--progress
|
||||
then
|
||||
ocf_log err \
|
||||
"pg_basebackup failed; preserving failed copy and restoring previous PGDATA"
|
||||
|
||||
if ! pg_command "
|
||||
set -e
|
||||
|
||||
if [ -e '${failed_pgdata}' ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -e '${pgdata}' ]; then
|
||||
mv '${pgdata}' '${failed_pgdata}'
|
||||
fi
|
||||
|
||||
mv '${preserved_pgdata}' '${pgdata}'
|
||||
"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_exit_reason \
|
||||
"pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
if ! as_service_user \
|
||||
podman exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
--env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \
|
||||
--env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \
|
||||
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
||||
--env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \
|
||||
--env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
set -e
|
||||
|
||||
sed -i \
|
||||
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \
|
||||
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \
|
||||
"$PGDATA/postgresql.auto.conf"
|
||||
|
||||
printf "%s\n" \
|
||||
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \
|
||||
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \
|
||||
>> "$PGDATA/postgresql.auto.conf"
|
||||
|
||||
touch "$PGDATA/standby.signal"
|
||||
|
||||
chmod 600 \
|
||||
"$PGDATA/postgresql.auto.conf" \
|
||||
"$PGDATA/standby.signal"
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to configure standby settings"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL as standby"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL failed to start after rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
sleep 1
|
||||
continue
|
||||
}
|
||||
|
||||
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
||||
|
||||
if [ "${role}" = "t" ] &&
|
||||
[ "${receiver_status}" = "streaming" ]
|
||||
then
|
||||
ocf_log info \
|
||||
"PostgreSQL successfully rejoined as streaming standby"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL started but did not become a streaming standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
pg_remote_query()
|
||||
{
|
||||
local sql="$1"
|
||||
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
||||
-Atqc "${sql}"
|
||||
}
|
||||
|
||||
ensure_local_ha_slots()
|
||||
{
|
||||
local slots raw_slot slot slot_type
|
||||
|
||||
#
|
||||
# POSTGRES_HA_SLOTS lives in the container environment supplied by
|
||||
# postgres.env. It is the cluster-wide list of physical slots that
|
||||
# every writable primary must provide.
|
||||
#
|
||||
slots="$(
|
||||
pg_command '
|
||||
printf "%s" "${POSTGRES_HA_SLOTS:-}"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to read POSTGRES_HA_SLOTS from PostgreSQL container"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ -z "${slots}" ]; then
|
||||
ocf_log warn \
|
||||
"POSTGRES_HA_SLOTS is empty; no local HA slots will be reconciled"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
while IFS= read -r raw_slot; do
|
||||
slot="$(printf '%s' "${raw_slot}" | tr -d '[:space:]')"
|
||||
|
||||
[ -n "${slot}" ] || continue
|
||||
|
||||
case "${slot}" in
|
||||
*[!a-z0-9_]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid physical replication slot name: ${slot}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
slot_type="$(
|
||||
pg_command "
|
||||
psql \
|
||||
-U '${DB_USER}' \
|
||||
-d postgres \
|
||||
-At \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-c \"
|
||||
SELECT slot_type
|
||||
FROM pg_replication_slots
|
||||
WHERE slot_name = '${slot}';
|
||||
\"
|
||||
" 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Failed querying replication slot ${slot}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
slot_type="$(printf '%s' "${slot_type}" | tr -d '[:space:]')"
|
||||
|
||||
case "${slot_type}" in
|
||||
physical)
|
||||
ocf_log info \
|
||||
"Physical HA slot already exists: ${slot}"
|
||||
;;
|
||||
|
||||
"")
|
||||
ocf_log info \
|
||||
"Creating physical HA slot: ${slot}"
|
||||
|
||||
if ! pg_command "
|
||||
psql \
|
||||
-U '${DB_USER}' \
|
||||
-d postgres \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-c \"
|
||||
SELECT pg_create_physical_replication_slot('${slot}');
|
||||
\"
|
||||
" >/dev/null
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed creating physical replication slot ${slot}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Replication slot ${slot} exists but is type ${slot_type}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
done <<EOF
|
||||
$(printf '%s\n' "${slots}" | tr ',' '\n')
|
||||
EOF
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
ensure_remote_slot()
|
||||
{
|
||||
local host="${1:-}"
|
||||
local slot="${2:-}"
|
||||
local result
|
||||
|
||||
[ -n "${host}" ] || return 1
|
||||
[ -n "${slot}" ] || return 1
|
||||
|
||||
result="$(
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \
|
||||
-Atc "
|
||||
SELECT slot_type
|
||||
FROM pg_replication_slots
|
||||
WHERE slot_name = '${slot}';
|
||||
" 2>/dev/null
|
||||
)" || return 1
|
||||
|
||||
case "${result}" in
|
||||
physical)
|
||||
return 0
|
||||
;;
|
||||
|
||||
"")
|
||||
if ! pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \
|
||||
-Atc "
|
||||
SELECT slot_name
|
||||
FROM pg_create_physical_replication_slot('${slot}');
|
||||
" >/dev/null
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_log err \
|
||||
"Replication slot ${slot} exists on ${host} but is not physical"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
candidate_caught_up()
|
||||
{
|
||||
local candidate="${1:-}"
|
||||
local result
|
||||
|
||||
[ -n "${candidate}" ] || return 1
|
||||
|
||||
result="$(pg_command "
|
||||
psql -U '${DB_USER}' -d '${DB_NAME}' -Atc \"
|
||||
SELECT CASE
|
||||
WHEN state = 'streaming'
|
||||
AND replay_lsn >= pg_current_wal_flush_lsn()
|
||||
THEN 'yes'
|
||||
ELSE 'no'
|
||||
END
|
||||
FROM pg_stat_replication
|
||||
WHERE application_name = '${candidate}'
|
||||
LIMIT 1;
|
||||
\"
|
||||
" 2>/dev/null)" || return 1
|
||||
|
||||
[ "${result}" = "yes" ]
|
||||
}
|
||||
|
||||
candidate_is_standby()
|
||||
{
|
||||
local host="${1:-}"
|
||||
local result
|
||||
local port="${REJOIN_SOURCE_PORT:-5432}"
|
||||
|
||||
[ -n "${host}" ] || return 1
|
||||
|
||||
result="$(
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres" \
|
||||
-Atc "SELECT pg_is_in_recovery();" \
|
||||
2>/dev/null
|
||||
)" || return 1
|
||||
|
||||
[ "${result}" = "t" ]
|
||||
}
|
||||
Reference in New Issue
Block a user