Freeze lab-tested PostgreSQL plugin engine
Extract PostgreSQL, Podman runtime and systemd lifecycle adapters while preserving existing lifecycle and scoring behavior. Update RPM layout and add guarded operator-run lab harnesses. Record successful manual stop/start, promotion, demotion and rewind-only rejoin with pinned installed hashes. The unfenced lab does not validate automatic failover, Pacemaker-managed transitions or divergent-data rewind. Validation: 103 offline tests, original-function differential checks, checksum manifest and whitespace checks passed.
This commit is contained in:
Lyncoln
@@ -0,0 +1,88 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
pg_replication_exec()
|
||||
{
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
if [ -z "${POSTGRES_REPLICATION_PASSWORD:-}" ]; then
|
||||
printf "%s\n" \
|
||||
"POSTGRES_REPLICATION_PASSWORD is missing from container environment" \
|
||||
>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD}"
|
||||
|
||||
exec "$@"
|
||||
' sh "$@"
|
||||
}
|
||||
|
||||
ensure_rewind_privileges()
|
||||
{
|
||||
local repl_user
|
||||
|
||||
repl_user="$(
|
||||
pg_command '
|
||||
printf "%s" "${POSTGRES_REPLICATION_USER:-}"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to read POSTGRES_REPLICATION_USER from container"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ -z "${repl_user}" ]; then
|
||||
ocf_exit_reason \
|
||||
"POSTGRES_REPLICATION_USER is empty"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
case "${repl_user}" in
|
||||
*[!A-Za-z0-9_]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid replication role name: ${repl_user}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
ocf_log info \
|
||||
"Ensuring pg_rewind source privileges for role ${repl_user}"
|
||||
|
||||
if ! pg_command "
|
||||
psql \
|
||||
-U '${DB_USER}' \
|
||||
-d postgres \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-c \"
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean)
|
||||
TO ${repl_user};
|
||||
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_stat_file(text, boolean)
|
||||
TO ${repl_user};
|
||||
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_read_binary_file(text)
|
||||
TO ${repl_user};
|
||||
|
||||
GRANT EXECUTE
|
||||
ON FUNCTION pg_catalog.pg_read_binary_file(
|
||||
text,
|
||||
bigint,
|
||||
bigint,
|
||||
boolean
|
||||
)
|
||||
TO ${repl_user};
|
||||
\"
|
||||
" >/dev/null
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed ensuring pg_rewind privileges for ${repl_user}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
node_to_addr()
|
||||
{
|
||||
local target="${1:-}"
|
||||
|
||||
[ -n "${target}" ] || return 1
|
||||
|
||||
corosync-cmapctl 2>/dev/null |
|
||||
awk -v target="${target}" '
|
||||
/nodelist\.node\.[0-9]+\.name / {
|
||||
key=$1
|
||||
sub(/\.name$/, "", key)
|
||||
names[key]=$NF
|
||||
}
|
||||
|
||||
/nodelist\.node\.[0-9]+\.ring0_addr / {
|
||||
key=$1
|
||||
sub(/\.ring0_addr$/, "", key)
|
||||
addrs[key]=$NF
|
||||
}
|
||||
|
||||
END {
|
||||
for (key in names) {
|
||||
if (names[key] == target && addrs[key] != "") {
|
||||
print addrs[key]
|
||||
exit 0
|
||||
}
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
'
|
||||
}
|
||||
|
||||
managed_demote()
|
||||
{
|
||||
local monitor_rc
|
||||
local target_node target_addr
|
||||
local local_node slot_name target_port
|
||||
local role receiver_status i
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
;;
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL is already unpromoted"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
"${OCF_NOT_RUNNING}")
|
||||
ocf_exit_reason \
|
||||
"Cannot complete demotion because PostgreSQL is stopped"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
target_node="$(demote_target)" || {
|
||||
ocf_exit_reason \
|
||||
"No fresh valid Pacemaker pre-demote target is available"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
target_addr="$(node_to_addr "${target_node}")" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to resolve Pacemaker promotion target ${target_node} through Corosync"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
target_port="${REJOIN_SOURCE_PORT:-5432}"
|
||||
local_node="$(hostname)"
|
||||
slot_name="ha_$(printf '%s' "${local_node}" |
|
||||
tr '[:upper:]-' '[:lower:]_')"
|
||||
|
||||
ocf_log info \
|
||||
"Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})"
|
||||
|
||||
if ! candidate_is_standby "${target_addr}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not a reachable PostgreSQL standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! candidate_caught_up "${target_node}"; then
|
||||
ocf_exit_reason \
|
||||
"Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then
|
||||
ocf_exit_reason \
|
||||
"Unable to ensure physical replication slot ${slot_name} on ${target_node}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Demotion safety checks passed; stopping PostgreSQL primary"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"pg_ctl stop failed during controlled demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL remained running after pg_ctl stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! service_running || ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service/container stopped unexpectedly during demotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Configuring ${local_node} as cascading standby of ${target_node}"
|
||||
|
||||
if ! mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
--env "RA_PRIMARY_HOST=${target_addr}" \
|
||||
--env "RA_PRIMARY_PORT=${target_port}" \
|
||||
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
||||
--env "RA_REPLICATION_SLOT=${slot_name}" \
|
||||
--env "RA_APPLICATION_NAME=${local_node}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
set -e
|
||||
|
||||
sed -i \
|
||||
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \
|
||||
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \
|
||||
"$PGDATA/postgresql.auto.conf"
|
||||
|
||||
printf "%s
|
||||
" \
|
||||
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \
|
||||
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \
|
||||
>> "$PGDATA/postgresql.auto.conf"
|
||||
|
||||
touch "$PGDATA/standby.signal"
|
||||
|
||||
chmod 600 \
|
||||
"$PGDATA/postgresql.auto.conf" \
|
||||
"$PGDATA/standby.signal"
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to configure PostgreSQL as standby of ${target_node}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL as cascading standby of ${target_node}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL failed to start as cascading standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null || true)"
|
||||
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
||||
|
||||
if [ "${role}" = "t" ] &&
|
||||
[ "${receiver_status}" = "streaming" ]
|
||||
then
|
||||
ocf_log info \
|
||||
"Controlled demotion complete: ${local_node} is streaming from ${target_node}"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL started after demotion but did not become a streaming standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
demote_target()
|
||||
{
|
||||
local state_file
|
||||
local type operation promote_uname demote_uname local_node
|
||||
local now mtime age
|
||||
|
||||
state_file="/run/loxthelion/${OCF_RESOURCE_INSTANCE:-managed-container}.notify"
|
||||
local_node="$(hostname)"
|
||||
|
||||
[ -r "${state_file}" ] || return 1
|
||||
|
||||
now="$(date +%s)" || return 1
|
||||
mtime="$(stat -c %Y "${state_file}" 2>/dev/null)" || return 1
|
||||
age=$((now - mtime))
|
||||
|
||||
[ "${age}" -ge 0 ] || return 1
|
||||
[ "${age}" -le 120 ] || return 1
|
||||
|
||||
type="$(
|
||||
sed -n 's/^type=//p' "${state_file}" | head -n1
|
||||
)"
|
||||
|
||||
operation="$(
|
||||
sed -n 's/^operation=//p' "${state_file}" | head -n1
|
||||
)"
|
||||
|
||||
promote_uname="$(
|
||||
sed -n 's/^promote_uname=//p' "${state_file}" | head -n1
|
||||
)"
|
||||
|
||||
demote_uname="$(
|
||||
sed -n 's/^demote_uname=//p' "${state_file}" | head -n1
|
||||
)"
|
||||
|
||||
[ "${type}" = "pre" ] || return 1
|
||||
[ "${operation}" = "demote" ] || return 1
|
||||
[ "${demote_uname}" = "${local_node}" ] || return 1
|
||||
[ -n "${promote_uname}" ] || return 1
|
||||
[ "${promote_uname}" != "${local_node}" ] || return 1
|
||||
|
||||
case "${promote_uname}" in
|
||||
*" "*) return 1 ;;
|
||||
esac
|
||||
|
||||
printf '%s
|
||||
' "${promote_uname}"
|
||||
}
|
||||
|
||||
managed_notify()
|
||||
{
|
||||
local state_dir state_file
|
||||
|
||||
state_dir="/run/loxthelion"
|
||||
state_file="${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify"
|
||||
|
||||
ocf_log info \
|
||||
"notify: type=${OCF_RESKEY_CRM_meta_notify_type:-unset} operation=${OCF_RESKEY_CRM_meta_notify_operation:-unset}"
|
||||
|
||||
ocf_log info \
|
||||
"notify: promote_uname=${OCF_RESKEY_CRM_meta_notify_promote_uname:-none} demote_uname=${OCF_RESKEY_CRM_meta_notify_demote_uname:-none}"
|
||||
|
||||
ocf_log info \
|
||||
"notify: promoted_uname=${OCF_RESKEY_CRM_meta_notify_promoted_uname:-none} unpromoted_uname=${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-none}"
|
||||
|
||||
if ! mkdir -p -m 0755 "${state_dir}"; then
|
||||
ocf_exit_reason \
|
||||
"Unable to create notification state directory ${state_dir}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! {
|
||||
printf 'timestamp=%s\n' "$(date --iso-8601=seconds)"
|
||||
printf 'node=%s\n' "$(hostname)"
|
||||
printf 'type=%s\n' "${OCF_RESKEY_CRM_meta_notify_type:-}"
|
||||
printf 'operation=%s\n' "${OCF_RESKEY_CRM_meta_notify_operation:-}"
|
||||
printf 'promote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}"
|
||||
printf 'demote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}"
|
||||
printf 'promoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}"
|
||||
printf 'unpromoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}"
|
||||
} > "${state_file}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to write notification state ${state_file}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
chmod 0600 "${state_file}" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to secure notification state ${state_file}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
{
|
||||
printf '%s node=%s type=%s operation=%s promote=[%s] demote=[%s] promoted=[%s] unpromoted=[%s]\n' \
|
||||
"$(date --iso-8601=seconds)" \
|
||||
"$(hostname)" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_type:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_operation:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" \
|
||||
"${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" \
|
||||
>> "${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify.log"
|
||||
} || true
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
OCF_RESKEY_service_user_default="david"
|
||||
OCF_RESKEY_service_name_default="postgres.service"
|
||||
OCF_RESKEY_container_name_default="postgres"
|
||||
OCF_RESKEY_pg_os_user_default="postgres"
|
||||
OCF_RESKEY_db_user_default="david"
|
||||
OCF_RESKEY_db_name_default="david"
|
||||
OCF_RESKEY_rejoin_source_host_default=""
|
||||
OCF_RESKEY_rejoin_source_port_default="5432"
|
||||
OCF_RESKEY_replication_user_default="replicator"
|
||||
OCF_RESKEY_replication_slot_default=""
|
||||
OCF_RESKEY_application_name_default=""
|
||||
OCF_RESKEY_rejoin_allow_basebackup_default="false"
|
||||
|
||||
: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}"
|
||||
: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}"
|
||||
: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}"
|
||||
: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}"
|
||||
: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}"
|
||||
: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}"
|
||||
: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}"
|
||||
: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}"
|
||||
: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}"
|
||||
: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}"
|
||||
: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}"
|
||||
: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}"
|
||||
|
||||
SERVICE_USER="${OCF_RESKEY_service_user}"
|
||||
SERVICE_NAME="${OCF_RESKEY_service_name}"
|
||||
CONTAINER_NAME="${OCF_RESKEY_container_name}"
|
||||
PG_OS_USER="${OCF_RESKEY_pg_os_user}"
|
||||
DB_USER="${OCF_RESKEY_db_user}"
|
||||
DB_NAME="${OCF_RESKEY_db_name}"
|
||||
REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}"
|
||||
REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}"
|
||||
REPLICATION_USER="${OCF_RESKEY_replication_user}"
|
||||
REPLICATION_SLOT="${OCF_RESKEY_replication_slot}"
|
||||
APPLICATION_NAME="${OCF_RESKEY_application_name}"
|
||||
REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}"
|
||||
|
||||
if [ -z "${APPLICATION_NAME}" ]; then
|
||||
APPLICATION_NAME="$(hostname -s)"
|
||||
fi
|
||||
|
||||
meta_data()
|
||||
{
|
||||
cat <<EOF_META
|
||||
<?xml version="1.0"?>
|
||||
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
|
||||
<resource-agent name="managed-container" version="0.4">
|
||||
<version>1.0</version>
|
||||
|
||||
<longdesc lang="en">
|
||||
Manage an application inside a persistent rootless container.
|
||||
|
||||
Version 0.4 implements role-aware monitoring, promotion, and demotion.
|
||||
The systemd user Quadlet owns the container lifecycle while pg_ctl owns
|
||||
the PostgreSQL lifecycle inside the container.
|
||||
</longdesc>
|
||||
|
||||
<shortdesc lang="en">
|
||||
Manage PostgreSQL independently inside a persistent container
|
||||
</shortdesc>
|
||||
|
||||
<parameters>
|
||||
|
||||
<parameter name="service_user" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
Linux user owning the rootless systemd/Podman service.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Rootless service owner</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_service_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="service_name" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
systemd user service generated by Quadlet.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Quadlet service name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_service_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="container_name" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
Podman container containing PostgreSQL.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Container name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_container_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="pg_os_user" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
Operating-system user running PostgreSQL inside the container.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">PostgreSQL OS user</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_pg_os_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="db_user" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL database user used for health queries.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Database user</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_db_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="db_name" unique="0" required="1">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL database used for health queries.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Database name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_db_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
|
||||
<parameter name="rejoin_source_host" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL primary host used as the source for rejoin operations.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Rejoin source host</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_rejoin_source_host_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="rejoin_source_port" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL port on the rejoin source.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Rejoin source port</shortdesc>
|
||||
<content type="integer" default="${OCF_RESKEY_rejoin_source_port_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="replication_user" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL role used for replication and pg_rewind.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Replication user</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_replication_user_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="replication_slot" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
Physical replication slot used by this standby.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Physical replication slot</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_replication_slot_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="application_name" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
PostgreSQL replication application_name. Defaults to the local hostname.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Replication application name</shortdesc>
|
||||
<content type="string" default="${OCF_RESKEY_application_name_default}"/>
|
||||
</parameter>
|
||||
|
||||
<parameter name="rejoin_allow_basebackup" unique="0" required="0">
|
||||
<longdesc lang="en">
|
||||
Allow rejoin to fall back to pg_basebackup when pg_rewind cannot
|
||||
repair the local PostgreSQL data directory. The old PGDATA is
|
||||
preserved before replacement. This must only be enabled when the
|
||||
operator has established that this node may safely discard its
|
||||
divergent PostgreSQL history.
|
||||
</longdesc>
|
||||
<shortdesc lang="en">Allow pg_basebackup rejoin fallback</shortdesc>
|
||||
<content type="boolean" default="${OCF_RESKEY_rejoin_allow_basebackup_default}"/>
|
||||
</parameter>
|
||||
|
||||
</parameters>
|
||||
|
||||
<actions>
|
||||
<action name="start" timeout="60s"/>
|
||||
<action name="stop" timeout="60s"/>
|
||||
<action name="promote" timeout="60s"/>
|
||||
<action name="demote" timeout="60s"/>
|
||||
<action name="notify" timeout="20s"/>
|
||||
<action name="monitor" timeout="20s" interval="10s"/>
|
||||
<action name="validate-all" timeout="20s"/>
|
||||
<action name="meta-data" timeout="5s"/>
|
||||
</actions>
|
||||
</resource-agent>
|
||||
EOF_META
|
||||
}
|
||||
|
||||
validate_all()
|
||||
{
|
||||
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
|
||||
ocf_exit_reason "Service user '${SERVICE_USER}' does not exist"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
for cmd in runuser systemctl podman getent; do
|
||||
if ! command -v "${cmd}" >/dev/null 2>&1; then
|
||||
ocf_exit_reason "Required command '${cmd}' is not installed"
|
||||
return "${OCF_ERR_INSTALLED}"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ! -d "/run/user/$(service_uid)" ]; then
|
||||
ocf_exit_reason \
|
||||
"Runtime directory /run/user/$(service_uid) does not exist"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! mc_lifecycle_cat "${SERVICE_NAME}" >/dev/null 2>&1
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
local podman_error
|
||||
|
||||
podman_error="$(
|
||||
mc_runtime_info 2>&1 >/dev/null
|
||||
)"
|
||||
local podman_rc=$?
|
||||
|
||||
if [ "${podman_rc}" -ne 0 ]; then
|
||||
ocf_exit_reason \
|
||||
"Rootless Podman failed for ${SERVICE_USER}: ${podman_error}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
|
||||
# Keep the original metadata function intact; advertise the added policy here.
|
||||
mc_promotion_metadata()
|
||||
{
|
||||
meta_data | sed '/<\/parameters>/i\
|
||||
<parameter name="disconnected_max_age" unique="0" required="0">\
|
||||
<longdesc lang="en">Maximum age in seconds of a verified streaming observation for disconnected promotion eligibility. Default 300; 0 disables disconnected eligibility. Requires fencing managed by Pacemaker. PostgreSQL restart requires fresh streaming verification.</longdesc>\
|
||||
<shortdesc lang="en">Disconnected standby observation lifetime</shortdesc>\
|
||||
<content type="integer" default="300"/>\
|
||||
</parameter>'
|
||||
}
|
||||
|
||||
MC_APPLICATION_API=1
|
||||
MC_APPLICATION_DIR="${MC_MODULE_DIR}/applications/postgresql"
|
||||
for MC_PG_MODULE in postgres auth replication notify promote demote rejoin; do
|
||||
if ! . "${MC_APPLICATION_DIR}/${MC_PG_MODULE}.sh"; then
|
||||
ocf_exit_reason "Unable to load resource-agent module ${MC_APPLICATION_DIR}/${MC_PG_MODULE}.sh"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
done
|
||||
unset MC_PG_MODULE
|
||||
|
||||
mc_application_monitor()
|
||||
{
|
||||
managed_monitor
|
||||
}
|
||||
|
||||
mc_application_standby_score()
|
||||
{
|
||||
mc_pg_standby_score
|
||||
}
|
||||
|
||||
mc_application_validate()
|
||||
{
|
||||
validate_all
|
||||
}
|
||||
|
||||
mc_application_metadata()
|
||||
{
|
||||
mc_promotion_metadata
|
||||
}
|
||||
|
||||
mc_application_notify()
|
||||
{
|
||||
managed_notify
|
||||
}
|
||||
|
||||
mc_application_action()
|
||||
{
|
||||
case "$1" in
|
||||
start|stop|promote|demote|rejoin) "managed_$1" ;;
|
||||
*) return "${OCF_ERR_UNIMPLEMENTED}" ;;
|
||||
esac
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
pg_command()
|
||||
{
|
||||
local command="$1"
|
||||
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c "${command}"
|
||||
}
|
||||
|
||||
pgdata_path()
|
||||
{
|
||||
pg_command '
|
||||
if [ -z "$PGDATA" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "%s\n" "$PGDATA"
|
||||
'
|
||||
}
|
||||
|
||||
pg_running()
|
||||
{
|
||||
pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
"$BINDIR/pg_ctl" status -D "$PGDATA" >/dev/null 2>&1
|
||||
'
|
||||
}
|
||||
|
||||
pg_ready()
|
||||
{
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
pg_isready -q \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}"
|
||||
}
|
||||
|
||||
pg_role()
|
||||
{
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}" \
|
||||
-Atqc 'SELECT pg_is_in_recovery();'
|
||||
}
|
||||
|
||||
pg_wal_receiver_status()
|
||||
{
|
||||
mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql \
|
||||
-U "${DB_USER}" \
|
||||
-d "${DB_NAME}" \
|
||||
-Atqc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;"
|
||||
}
|
||||
|
||||
managed_monitor()
|
||||
{
|
||||
local role
|
||||
|
||||
if ! service_running; then
|
||||
ocf_log debug "User service ${SERVICE_NAME} is not active"
|
||||
return "${OCF_NOT_RUNNING}"
|
||||
fi
|
||||
|
||||
if ! container_running; then
|
||||
ocf_log warn \
|
||||
"Service ${SERVICE_NAME} is active but container ${CONTAINER_NAME} is not running"
|
||||
return "${OCF_NOT_RUNNING}"
|
||||
fi
|
||||
|
||||
if ! pg_running; then
|
||||
ocf_log debug \
|
||||
"Container ${CONTAINER_NAME} is running but PostgreSQL is stopped"
|
||||
return "${OCF_NOT_RUNNING}"
|
||||
fi
|
||||
|
||||
if ! pg_ready; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL process exists but pg_isready failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL is accepting connections but role query failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${role}" in
|
||||
f)
|
||||
ocf_log debug \
|
||||
"PostgreSQL healthy: role=primary (promoted)"
|
||||
return "${OCF_RUNNING_PROMOTED}"
|
||||
;;
|
||||
t)
|
||||
ocf_log debug \
|
||||
"PostgreSQL healthy: role=standby (unpromoted)"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Unexpected pg_is_in_recovery() result: '${role}'"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
}
|
||||
|
||||
managed_start()
|
||||
{
|
||||
local i monitor_rc
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL resource already running unpromoted"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
ocf_exit_reason \
|
||||
"Refusing start: PostgreSQL is already primary; promotion must occur through promote()"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
"${OCF_NOT_RUNNING}")
|
||||
# Expected for a stopped resource. Continue.
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot start unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
ocf_log info \
|
||||
"Ensuring container service ${SERVICE_NAME} is running"
|
||||
|
||||
if ! service_running; then
|
||||
if ! mc_lifecycle_start "${SERVICE_NAME}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to start ${SERVICE_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
for i in $(seq 1 20); do
|
||||
container_running && break
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Container ${CONTAINER_NAME} did not become running"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! pg_running; then
|
||||
#
|
||||
# OCF promotable-resource rule:
|
||||
# start() must produce an Unpromoted instance.
|
||||
#
|
||||
# A PostgreSQL data directory without standby.signal would start
|
||||
# writable/primary, so never start it from this action.
|
||||
#
|
||||
if ! pg_command '
|
||||
test -f "$PGDATA/standby.signal"
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Refusing start: PGDATA is not configured as a standby (standby.signal missing)"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL as an unpromoted standby inside ${CONTAINER_NAME}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "pg_ctl start failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_SUCCESS}")
|
||||
ocf_log info \
|
||||
"PostgreSQL start completed in unpromoted state"
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL became primary during start; stopping it to preserve OCF role semantics"
|
||||
|
||||
pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
' >/dev/null 2>&1 || true
|
||||
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
*)
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
managed_stop()
|
||||
{
|
||||
local i
|
||||
|
||||
if container_running && pg_running; then
|
||||
ocf_log info \
|
||||
"Stopping PostgreSQL cleanly inside ${CONTAINER_NAME}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
stop -m fast -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "pg_ctl stop failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if service_running; then
|
||||
ocf_log info \
|
||||
"Stopping container service ${SERVICE_NAME}"
|
||||
|
||||
if ! mc_lifecycle_stop "${SERVICE_NAME}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to stop ${SERVICE_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
for i in $(seq 1 20); do
|
||||
if ! service_running && ! container_running; then
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if service_running; then
|
||||
ocf_exit_reason \
|
||||
"${SERVICE_NAME} remained active after stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if container_running; then
|
||||
ocf_exit_reason \
|
||||
"${CONTAINER_NAME} remained running after stop"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
# Conservative scoring eligibility only; this does not replace role detection,
|
||||
# fencing, the promotion action, or the existing controlled-demote WAL checks.
|
||||
# This strict predicate remains unchanged; disconnected policy is separate below.
|
||||
mc_pg_promotion_eligible()
|
||||
{
|
||||
local eligible
|
||||
|
||||
eligible="$(
|
||||
mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" \
|
||||
--env 'PGOPTIONS=-c statement_timeout=2000' \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc "
|
||||
SELECT CASE WHEN pg_is_in_recovery() THEN
|
||||
NOT pg_is_wal_replay_paused()
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM pg_stat_wal_receiver
|
||||
WHERE status = 'streaming'
|
||||
AND flushed_lsn IS NOT NULL
|
||||
AND pg_last_wal_replay_lsn() >= flushed_lsn
|
||||
)
|
||||
ELSE false END;
|
||||
" 2>/dev/null
|
||||
)" || return 1
|
||||
[ "${eligible}" = t ]
|
||||
}
|
||||
|
||||
# One observation supplies both strict connected eligibility and disconnected
|
||||
# evidence. PostgreSQL restart time deliberately invalidates prior observations,
|
||||
# including a new basebackup sharing the same PostgreSQL system identifier.
|
||||
mc_pg_standby_observation()
|
||||
{
|
||||
mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" \
|
||||
--env 'PGOPTIONS=-c statement_timeout=2000' \
|
||||
"${CONTAINER_NAME}" \
|
||||
psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc "
|
||||
WITH s AS MATERIALIZED (
|
||||
SELECT pg_is_in_recovery() AS recovery,
|
||||
pg_last_wal_replay_lsn() AS replay,
|
||||
r.status AS receiver, r.flushed_lsn AS flush
|
||||
FROM (SELECT 1) AS singleton
|
||||
LEFT JOIN pg_stat_wal_receiver r ON true
|
||||
)
|
||||
SELECT json_build_object(
|
||||
'identity', json_build_array(
|
||||
(SELECT system_identifier::text FROM pg_control_system()),
|
||||
current_setting('data_directory'),
|
||||
extract(epoch FROM pg_postmaster_start_time())::text),
|
||||
'recovery', recovery,
|
||||
'paused', CASE WHEN recovery THEN pg_is_wal_replay_paused() ELSE true END,
|
||||
'receiver', receiver, 'replay', replay::text, 'flush', flush::text,
|
||||
'strict', CASE WHEN recovery THEN
|
||||
NOT pg_is_wal_replay_paused() AND receiver = 'streaming'
|
||||
AND flush IS NOT NULL AND replay >= flush
|
||||
ELSE false END)
|
||||
FROM s;
|
||||
" 2>/dev/null
|
||||
}
|
||||
|
||||
mc_pg_standby_score()
|
||||
{
|
||||
local observation
|
||||
observation="$(mc_pg_standby_observation)" || return 1
|
||||
# The helper is root-owned alongside the modules; state never enters the
|
||||
# rootless container. A failed/slow helper always withdraws eligibility.
|
||||
printf '%s' "${observation}" | timeout --kill-after=1s 1s python3 \
|
||||
"${MC_APPLICATION_DIR}/promotion_state.py" \
|
||||
"${MC_PROMOTION_RESOURCE_INSTANCE:-}" \
|
||||
"${SERVICE_USER}" "${SERVICE_NAME}" "${CONTAINER_NAME}" \
|
||||
"${OCF_RESKEY_disconnected_max_age:-300}"
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
managed_promote()
|
||||
{
|
||||
local monitor_rc role i
|
||||
|
||||
managed_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_RUNNING_PROMOTED}")
|
||||
ocf_log info \
|
||||
"PostgreSQL is already primary; reconciling primary invariants"
|
||||
|
||||
if ! ensure_local_ha_slots; then
|
||||
ocf_exit_reason \
|
||||
"HA slot reconciliation failed on existing primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_rewind_privileges; then
|
||||
ocf_exit_reason \
|
||||
"pg_rewind privilege reconciliation failed on existing primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
;;
|
||||
|
||||
"${OCF_SUCCESS}")
|
||||
# Healthy standby: continue with promotion.
|
||||
;;
|
||||
|
||||
"${OCF_NOT_RUNNING}")
|
||||
ocf_exit_reason \
|
||||
"Cannot promote PostgreSQL because the resource is not running"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Cannot promote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})"
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
ocf_log info \
|
||||
"Promoting PostgreSQL standby inside ${CONTAINER_NAME}"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
promote -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason "pg_ctl promote failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
sleep 1
|
||||
continue
|
||||
}
|
||||
|
||||
if [ "${role}" = "f" ]; then
|
||||
ocf_log info \
|
||||
"PostgreSQL became primary; reconciling primary invariants"
|
||||
|
||||
if ! ensure_local_ha_slots; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL promoted but HA slot reconciliation failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_rewind_privileges; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL promoted but pg_rewind privilege reconciliation failed"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"PostgreSQL promotion completed successfully"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL did not become primary after promotion"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Local promotion evidence. Invoked by the root RA; never invokes cluster tools.
|
||||
|
||||
State is a bounded JSON record, never executable input. The public CLI uses a
|
||||
fixed root-owned /run directory and CLOCK_BOOTTIME, including suspend duration.
|
||||
"""
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
STATE_DIR = Path('/run/managed-container/promotion')
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError('duplicate JSON key')
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def lsn(value):
|
||||
if not isinstance(value, str) or not re.fullmatch(r'[0-9A-F]{1,8}/[0-9A-F]{1,8}', value):
|
||||
raise ValueError('invalid LSN')
|
||||
high, low = value.split('/')
|
||||
return (int(high, 16) << 32) | int(low, 16)
|
||||
|
||||
|
||||
def number(value):
|
||||
return type(value) in (int, float) and math.isfinite(value) and value >= 0
|
||||
|
||||
|
||||
def validate_observation(obs):
|
||||
if set(obs) != {'identity', 'recovery', 'paused', 'receiver', 'replay', 'flush', 'strict'}:
|
||||
raise ValueError('invalid observation fields')
|
||||
identity = obs['identity']
|
||||
if not isinstance(identity, list) or len(identity) != 3:
|
||||
raise ValueError('invalid identity')
|
||||
system, path, started = identity
|
||||
if (not isinstance(system, str) or not re.fullmatch(r'[0-9]{1,20}', system)
|
||||
or not isinstance(path, str) or not path.startswith('/') or len(path) > 2048
|
||||
or not isinstance(started, str) or not re.fullmatch(r'[0-9]{1,12}\.[0-9]{1,6}', started)):
|
||||
raise ValueError('invalid identity values')
|
||||
if type(obs['recovery']) is not bool or type(obs['paused']) is not bool:
|
||||
raise ValueError('invalid role')
|
||||
if obs['strict'] is not None and type(obs['strict']) is not bool:
|
||||
raise ValueError('invalid strict flag')
|
||||
if obs['receiver'] is not None and obs['receiver'] not in (
|
||||
'starting', 'streaming', 'waiting', 'restarting', 'stopping'):
|
||||
raise ValueError('invalid receiver status')
|
||||
replay = lsn(obs['replay']) if obs['replay'] is not None else None
|
||||
flush = lsn(obs['flush']) if obs['flush'] is not None else None
|
||||
strict = (obs['recovery'] and not obs['paused'] and obs['receiver'] == 'streaming'
|
||||
and replay is not None and flush is not None and replay >= flush)
|
||||
if (obs['strict'] is True) != strict:
|
||||
raise ValueError('inconsistent strict observation')
|
||||
return strict
|
||||
|
||||
|
||||
class Evidence:
|
||||
def __init__(self, directory, context, boot, now, wall, owner=0):
|
||||
self.directory, self.context = Path(directory), context
|
||||
self.boot, self.now, self.wall, self.owner = boot, now, wall, owner
|
||||
key = hashlib.sha256(json.dumps(context, separators=(',', ':')).encode()).hexdigest()
|
||||
self.path = self.directory / (key + '.json')
|
||||
self.lock = self.directory / (key + '.lock')
|
||||
|
||||
def secure(self, path, mode):
|
||||
info = path.lstat()
|
||||
if (info.st_uid != self.owner or stat.S_IMODE(info.st_mode) != mode
|
||||
or (not stat.S_ISREG(info.st_mode) if mode == 0o600 else not stat.S_ISDIR(info.st_mode))):
|
||||
raise ValueError('unsafe state ownership/type/mode')
|
||||
|
||||
def discard(self):
|
||||
self.path.unlink(missing_ok=True)
|
||||
|
||||
def evaluate(self, obs, max_age):
|
||||
if type(max_age) is not int or not 0 <= max_age <= 99999:
|
||||
raise ValueError('invalid freshness threshold')
|
||||
self.directory.mkdir(mode=0o700, exist_ok=True)
|
||||
self.secure(self.directory, 0o700)
|
||||
fd = os.open(self.lock, os.O_CREAT | os.O_NOFOLLOW | os.O_RDWR, 0o600)
|
||||
with os.fdopen(fd, 'r+') as lock:
|
||||
self.secure(self.lock, 0o600)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
try:
|
||||
return self._evaluate(obs, max_age)
|
||||
except (ValueError, TypeError, KeyError, OverflowError):
|
||||
self.discard()
|
||||
return '-INFINITY'
|
||||
|
||||
def _evaluate(self, obs, max_age):
|
||||
strict = validate_observation(obs)
|
||||
if strict:
|
||||
record = dict(version=1, context=self.context, boot=self.boot,
|
||||
identity=obs['identity'], observed_boottime=self.now,
|
||||
observed_unix=self.wall, replay=obs['replay'], flush=obs['flush'])
|
||||
# Same-directory atomic replace; failed persistence cannot authorize
|
||||
# subsequent disconnected promotion. No secrets are recorded.
|
||||
fd, temporary = tempfile.mkstemp(prefix='.observation-', dir=self.directory)
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as out:
|
||||
json.dump(record, out, separators=(',', ':'))
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
os.replace(temporary, self.path)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
return '100'
|
||||
if (not obs['recovery'] or obs['paused'] or obs['receiver'] == 'streaming'
|
||||
or obs['replay'] is None or max_age == 0):
|
||||
self.discard()
|
||||
return '-INFINITY'
|
||||
if not self.path.exists():
|
||||
return '-INFINITY'
|
||||
self.secure(self.path, 0o600)
|
||||
if self.path.stat().st_size > 4096:
|
||||
raise ValueError('oversized state')
|
||||
with self.path.open() as source:
|
||||
record = json.load(source, object_pairs_hook=unique_object)
|
||||
if set(record) != {'version', 'context', 'boot', 'identity', 'observed_boottime',
|
||||
'observed_unix', 'replay', 'flush'}:
|
||||
raise ValueError('invalid record fields')
|
||||
if (type(record['version']) is not int or record['version'] != 1
|
||||
or record['context'] != self.context or record['boot'] != self.boot
|
||||
or record['identity'] != obs['identity']
|
||||
or not number(record['observed_boottime']) or not number(record['observed_unix'])
|
||||
or not 0 <= self.now - record['observed_boottime'] <= max_age
|
||||
or lsn(record['replay']) < lsn(record['flush'])
|
||||
or lsn(obs['replay']) < lsn(record['replay'])):
|
||||
raise ValueError('expired or incompatible observation')
|
||||
return '50'
|
||||
|
||||
|
||||
def main():
|
||||
if os.geteuid() != 0 or len(sys.argv) != 6 or not all(sys.argv[1:5]):
|
||||
raise ValueError('root and explicit resource context required')
|
||||
age = sys.argv[5]
|
||||
if not re.fullmatch(r'(0|[1-9][0-9]{0,4})', age):
|
||||
raise ValueError('invalid freshness threshold')
|
||||
# /run is trusted root-owned; validate our parent before using a child.
|
||||
parent = STATE_DIR.parent
|
||||
parent.mkdir(mode=0o700, exist_ok=True)
|
||||
info = parent.lstat()
|
||||
if info.st_uid != 0 or not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700:
|
||||
raise ValueError('unsafe state parent')
|
||||
raw = sys.stdin.read(8193)
|
||||
if len(raw) > 8192:
|
||||
raise ValueError('oversized observation')
|
||||
evidence = Evidence(STATE_DIR, sys.argv[1:5],
|
||||
Path('/proc/sys/kernel/random/boot_id').read_text().strip(),
|
||||
time.clock_gettime(time.CLOCK_BOOTTIME), time.time())
|
||||
try:
|
||||
obs = json.loads(raw, object_pairs_hook=unique_object)
|
||||
except ValueError:
|
||||
obs = {} # Invalid query output withdraws and removes prior evidence.
|
||||
print(evidence.evaluate(obs, int(age)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (OSError, ValueError, TypeError, KeyError, OverflowError):
|
||||
print('-INFINITY')
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,342 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
managed_rejoin()
|
||||
{
|
||||
local source_role role receiver_status pgdata i
|
||||
|
||||
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
||||
true|false|1|0|yes|no)
|
||||
;;
|
||||
*)
|
||||
ocf_exit_reason "rejoin_allow_basebackup must be true or false"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "${REJOIN_SOURCE_HOST}" ]; then
|
||||
ocf_exit_reason \
|
||||
"rejoin_source_host is required for rejoin"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
if [ -z "${REPLICATION_SLOT}" ]; then
|
||||
ocf_exit_reason \
|
||||
"replication_slot is required for rejoin"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
fi
|
||||
|
||||
if ! service_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service ${SERVICE_NAME} must be running before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance container ${CONTAINER_NAME} must be running before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL must be stopped before rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}"
|
||||
|
||||
source_role="$(pg_remote_query \
|
||||
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${source_role}" != "f" ]; then
|
||||
ocf_exit_reason \
|
||||
"Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Running pg_rewind against ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
pgdata="$(pgdata_path 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ -z "${pgdata}" ]; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! pg_replication_exec \
|
||||
pg_rewind \
|
||||
--target-pgdata="${pgdata}" \
|
||||
--source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
||||
--progress
|
||||
then
|
||||
local fallback_enabled source_bytes free_bytes required_bytes
|
||||
local timestamp preserved_pgdata failed_pgdata source_role_retry
|
||||
|
||||
case "${REJOIN_ALLOW_BASEBACKUP}" in
|
||||
true|1|yes)
|
||||
fallback_enabled=true
|
||||
;;
|
||||
*)
|
||||
fallback_enabled=false
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "${fallback_enabled}" != "true" ]; then
|
||||
ocf_exit_reason \
|
||||
"pg_rewind failed and rejoin_allow_basebackup is not enabled"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"pg_rewind failed; evaluating pg_basebackup fallback"
|
||||
|
||||
# Revalidate the source immediately before replacing local PGDATA.
|
||||
source_role_retry="$(pg_remote_query \
|
||||
'SELECT pg_is_in_recovery();' 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to revalidate rejoin source before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ "${source_role_retry}" != "f" ]; then
|
||||
ocf_exit_reason \
|
||||
"Rejoin source is no longer a writable primary; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if pg_running; then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL became active during rejoin; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! service_running || ! container_running; then
|
||||
ocf_exit_reason \
|
||||
"Maintenance service/container is unavailable; refusing pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
if ! ensure_remote_slot \
|
||||
"${REJOIN_SOURCE_HOST}" \
|
||||
"${REPLICATION_SLOT}"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
# Estimate source database payload and require 2x that amount free.
|
||||
# This is deliberately conservative and does not pretend to be an
|
||||
# exact prediction of the resulting PGDATA size.
|
||||
source_bytes="$(pg_remote_query \
|
||||
"SELECT COALESCE(sum(pg_database_size(datname)),0)
|
||||
FROM pg_database
|
||||
WHERE datallowconn;" 2>/dev/null)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to estimate source database size before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${source_bytes}" in
|
||||
""|*[!0-9]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid source database size returned before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
free_bytes="$(
|
||||
pg_command '
|
||||
blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")"
|
||||
case "$blocks" in
|
||||
""|*[!0-9]*)
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
printf "%s\n" "$((blocks * 1024))"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to determine free space for pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
case "${free_bytes}" in
|
||||
""|*[!0-9]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid free-space value before pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
required_bytes=$((source_bytes * 2))
|
||||
|
||||
if [ "${free_bytes}" -lt "${required_bytes}" ]; then
|
||||
ocf_exit_reason \
|
||||
"Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
timestamp="$(date -u +%Y%m%d-%H%M%S)"
|
||||
preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}"
|
||||
failed_pgdata="${pgdata}.failed-basebackup-${timestamp}"
|
||||
|
||||
ocf_log warn \
|
||||
"Preserving existing PGDATA as ${preserved_pgdata}"
|
||||
|
||||
if ! pg_command "
|
||||
set -e
|
||||
|
||||
[ -d '${pgdata}' ]
|
||||
[ ! -e '${preserved_pgdata}' ]
|
||||
[ ! -e '${failed_pgdata}' ]
|
||||
|
||||
mv '${pgdata}' '${preserved_pgdata}'
|
||||
install -d -m 0700 '${pgdata}'
|
||||
"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Unable to preserve existing PGDATA for pg_basebackup fallback"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log warn \
|
||||
"Running pg_basebackup from ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
if ! pg_replication_exec \
|
||||
pg_basebackup \
|
||||
--dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \
|
||||
--pgdata="${pgdata}" \
|
||||
--format=plain \
|
||||
--wal-method=stream \
|
||||
--slot="${REPLICATION_SLOT}" \
|
||||
--write-recovery-conf \
|
||||
--progress
|
||||
then
|
||||
ocf_log err \
|
||||
"pg_basebackup failed; preserving failed copy and restoring previous PGDATA"
|
||||
|
||||
if ! pg_command "
|
||||
set -e
|
||||
|
||||
if [ -e '${failed_pgdata}' ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -e '${pgdata}' ]; then
|
||||
mv '${pgdata}' '${failed_pgdata}'
|
||||
fi
|
||||
|
||||
mv '${preserved_pgdata}' '${pgdata}'
|
||||
"
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_exit_reason \
|
||||
"pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}"
|
||||
|
||||
if ! mc_runtime_exec \
|
||||
--user "${PG_OS_USER}" \
|
||||
--env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \
|
||||
--env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \
|
||||
--env "RA_REPLICATION_USER=${REPLICATION_USER}" \
|
||||
--env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \
|
||||
--env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \
|
||||
"${CONTAINER_NAME}" \
|
||||
sh -c '
|
||||
set -e
|
||||
|
||||
sed -i \
|
||||
-e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \
|
||||
-e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \
|
||||
"$PGDATA/postgresql.auto.conf"
|
||||
|
||||
printf "%s\n" \
|
||||
"primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \
|
||||
"primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \
|
||||
>> "$PGDATA/postgresql.auto.conf"
|
||||
|
||||
touch "$PGDATA/standby.signal"
|
||||
|
||||
chmod 600 \
|
||||
"$PGDATA/postgresql.auto.conf" \
|
||||
"$PGDATA/standby.signal"
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed to configure standby settings"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info \
|
||||
"Starting PostgreSQL as standby"
|
||||
|
||||
if ! pg_command '
|
||||
BINDIR="$(pg_config --bindir)" || exit 1
|
||||
exec "$BINDIR/pg_ctl" \
|
||||
-D "$PGDATA" \
|
||||
-l "$PGDATA/postgresql.log" \
|
||||
start -w
|
||||
'
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL failed to start after rejoin"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
role="$(pg_role 2>/dev/null)" || {
|
||||
sleep 1
|
||||
continue
|
||||
}
|
||||
|
||||
receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)"
|
||||
|
||||
if [ "${role}" = "t" ] &&
|
||||
[ "${receiver_status}" = "streaming" ]
|
||||
then
|
||||
ocf_log info \
|
||||
"PostgreSQL successfully rejoined as streaming standby"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
ocf_exit_reason \
|
||||
"PostgreSQL started but did not become a streaming standby"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
# Sourced by managed-container; uses the shared OCF and parameter environment.
|
||||
|
||||
pg_remote_query()
|
||||
{
|
||||
local sql="$1"
|
||||
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \
|
||||
-Atqc "${sql}"
|
||||
}
|
||||
|
||||
ensure_local_ha_slots()
|
||||
{
|
||||
local slots raw_slot slot slot_type
|
||||
|
||||
#
|
||||
# POSTGRES_HA_SLOTS lives in the container environment supplied by
|
||||
# postgres.env. It is the cluster-wide list of physical slots that
|
||||
# every writable primary must provide.
|
||||
#
|
||||
slots="$(
|
||||
pg_command '
|
||||
printf "%s" "${POSTGRES_HA_SLOTS:-}"
|
||||
' 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Unable to read POSTGRES_HA_SLOTS from PostgreSQL container"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
if [ -z "${slots}" ]; then
|
||||
ocf_log warn \
|
||||
"POSTGRES_HA_SLOTS is empty; no local HA slots will be reconciled"
|
||||
return "${OCF_SUCCESS}"
|
||||
fi
|
||||
|
||||
while IFS= read -r raw_slot; do
|
||||
slot="$(printf '%s' "${raw_slot}" | tr -d '[:space:]')"
|
||||
|
||||
[ -n "${slot}" ] || continue
|
||||
|
||||
case "${slot}" in
|
||||
*[!a-z0-9_]*)
|
||||
ocf_exit_reason \
|
||||
"Invalid physical replication slot name: ${slot}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
slot_type="$(
|
||||
pg_command "
|
||||
psql \
|
||||
-U '${DB_USER}' \
|
||||
-d postgres \
|
||||
-At \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-c \"
|
||||
SELECT slot_type
|
||||
FROM pg_replication_slots
|
||||
WHERE slot_name = '${slot}';
|
||||
\"
|
||||
" 2>/dev/null
|
||||
)" || {
|
||||
ocf_exit_reason \
|
||||
"Failed querying replication slot ${slot}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
}
|
||||
|
||||
slot_type="$(printf '%s' "${slot_type}" | tr -d '[:space:]')"
|
||||
|
||||
case "${slot_type}" in
|
||||
physical)
|
||||
ocf_log info \
|
||||
"Physical HA slot already exists: ${slot}"
|
||||
;;
|
||||
|
||||
"")
|
||||
ocf_log info \
|
||||
"Creating physical HA slot: ${slot}"
|
||||
|
||||
if ! pg_command "
|
||||
psql \
|
||||
-U '${DB_USER}' \
|
||||
-d postgres \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-c \"
|
||||
SELECT pg_create_physical_replication_slot('${slot}');
|
||||
\"
|
||||
" >/dev/null
|
||||
then
|
||||
ocf_exit_reason \
|
||||
"Failed creating physical replication slot ${slot}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_exit_reason \
|
||||
"Replication slot ${slot} exists but is type ${slot_type}"
|
||||
return "${OCF_ERR_CONFIGURED}"
|
||||
;;
|
||||
esac
|
||||
|
||||
done <<EOF
|
||||
$(printf '%s\n' "${slots}" | tr ',' '\n')
|
||||
EOF
|
||||
|
||||
return "${OCF_SUCCESS}"
|
||||
}
|
||||
|
||||
ensure_remote_slot()
|
||||
{
|
||||
local host="${1:-}"
|
||||
local slot="${2:-}"
|
||||
local result
|
||||
|
||||
[ -n "${host}" ] || return 1
|
||||
[ -n "${slot}" ] || return 1
|
||||
|
||||
result="$(
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \
|
||||
-Atc "
|
||||
SELECT slot_type
|
||||
FROM pg_replication_slots
|
||||
WHERE slot_name = '${slot}';
|
||||
" 2>/dev/null
|
||||
)" || return 1
|
||||
|
||||
case "${result}" in
|
||||
physical)
|
||||
return 0
|
||||
;;
|
||||
|
||||
"")
|
||||
if ! pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \
|
||||
-Atc "
|
||||
SELECT slot_name
|
||||
FROM pg_create_physical_replication_slot('${slot}');
|
||||
" >/dev/null
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_log err \
|
||||
"Replication slot ${slot} exists on ${host} but is not physical"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
candidate_caught_up()
|
||||
{
|
||||
local candidate="${1:-}"
|
||||
local result
|
||||
|
||||
[ -n "${candidate}" ] || return 1
|
||||
|
||||
result="$(pg_command "
|
||||
psql -U '${DB_USER}' -d '${DB_NAME}' -Atc \"
|
||||
SELECT CASE
|
||||
WHEN state = 'streaming'
|
||||
AND replay_lsn >= pg_current_wal_flush_lsn()
|
||||
THEN 'yes'
|
||||
ELSE 'no'
|
||||
END
|
||||
FROM pg_stat_replication
|
||||
WHERE application_name = '${candidate}'
|
||||
LIMIT 1;
|
||||
\"
|
||||
" 2>/dev/null)" || return 1
|
||||
|
||||
[ "${result}" = "yes" ]
|
||||
}
|
||||
|
||||
candidate_is_standby()
|
||||
{
|
||||
local host="${1:-}"
|
||||
local result
|
||||
local port="${REJOIN_SOURCE_PORT:-5432}"
|
||||
|
||||
[ -n "${host}" ] || return 1
|
||||
|
||||
result="$(
|
||||
pg_replication_exec \
|
||||
psql \
|
||||
"host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres" \
|
||||
-Atc "SELECT pg_is_in_recovery();" \
|
||||
2>/dev/null
|
||||
)" || return 1
|
||||
|
||||
[ "${result}" = "t" ]
|
||||
}
|
||||
Reference in New Issue
Block a user