#!/bin/bash # OCF Resource Agent: # loxthelion:managed-container # # v0.4 # Container lifecycle: systemd user Quadlet # Application profile: PostgreSQL # # Version 0.4 adds explicit PostgreSQL demotion scaffolding. # Healthy primaries report OCF_RUNNING_PROMOTED while healthy standbys # report OCF_SUCCESS. : "${OCF_ROOT:=/usr/lib/ocf}" OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat" . "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs" # Pacemaker uses "Promoted" terminology, while many OCF shell libraries # still expose the historical OCF_RUNNING_MASTER constant for return code 8. # Normalize both forms here so the agent remains portable. : "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}" ############################################################################### # Defaults ############################################################################### OCF_RESKEY_service_user_default="david" OCF_RESKEY_service_name_default="postgres.service" OCF_RESKEY_container_name_default="postgres" OCF_RESKEY_pg_os_user_default="postgres" OCF_RESKEY_db_user_default="david" OCF_RESKEY_db_name_default="david" OCF_RESKEY_rejoin_source_host_default="" OCF_RESKEY_rejoin_source_port_default="5432" OCF_RESKEY_replication_user_default="replicator" OCF_RESKEY_replication_slot_default="" OCF_RESKEY_application_name_default="" OCF_RESKEY_rejoin_allow_basebackup_default="false" : "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}" : "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}" : "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}" : "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}" : "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}" : "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}" : "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}" : "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}" : "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}" : "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}" : "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}" : "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}" SERVICE_USER="${OCF_RESKEY_service_user}" SERVICE_NAME="${OCF_RESKEY_service_name}" CONTAINER_NAME="${OCF_RESKEY_container_name}" PG_OS_USER="${OCF_RESKEY_pg_os_user}" DB_USER="${OCF_RESKEY_db_user}" DB_NAME="${OCF_RESKEY_db_name}" REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}" REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}" REPLICATION_USER="${OCF_RESKEY_replication_user}" REPLICATION_SLOT="${OCF_RESKEY_replication_slot}" APPLICATION_NAME="${OCF_RESKEY_application_name}" REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}" if [ -z "${APPLICATION_NAME}" ]; then APPLICATION_NAME="$(hostname -s)" fi ############################################################################### # Metadata ############################################################################### meta_data() { cat < 1.0 Manage an application inside a persistent rootless container. Version 0.4 implements role-aware monitoring, promotion, and demotion. The systemd user Quadlet owns the container lifecycle while pg_ctl owns the PostgreSQL lifecycle inside the container. Manage PostgreSQL independently inside a persistent container Linux user owning the rootless systemd/Podman service. Rootless service owner systemd user service generated by Quadlet. Quadlet service name Podman container containing PostgreSQL. Container name Operating-system user running PostgreSQL inside the container. PostgreSQL OS user PostgreSQL database user used for health queries. Database user PostgreSQL database used for health queries. Database name PostgreSQL primary host used as the source for rejoin operations. Rejoin source host PostgreSQL port on the rejoin source. Rejoin source port PostgreSQL role used for replication and pg_rewind. Replication user Physical replication slot used by this standby. Physical replication slot PostgreSQL replication application_name. Defaults to the local hostname. Replication application name Allow rejoin to fall back to pg_basebackup when pg_rewind cannot repair the local PostgreSQL data directory. The old PGDATA is preserved before replacement. This must only be enabled when the operator has established that this node may safely discard its divergent PostgreSQL history. Allow pg_basebackup rejoin fallback EOF_META } ############################################################################### # Rootless-user helpers ############################################################################### service_uid() { id -u "${SERVICE_USER}" } service_home() { getent passwd "${SERVICE_USER}" | cut -d: -f6 } as_service_user() { local uid home uid="$(id -u "$SERVICE_USER")" || return 1 home="$(getent passwd "$SERVICE_USER" | cut -d: -f6)" || return 1 runuser -u "$SERVICE_USER" -- \ sh -c ' home="$1" runtime="$2" shift 2 cd "$home" || exit 1 export HOME="$home" export XDG_RUNTIME_DIR="$runtime" exec "$@" ' sh \ "$home" \ "/run/user/$uid" \ "$@" } ############################################################################### # Container / application status ############################################################################### service_running() { as_service_user \ systemctl --user is-active --quiet "${SERVICE_NAME}" } container_running() { local state state="$( as_service_user podman inspect \ --format '{{.State.Running}}' \ "${CONTAINER_NAME}" 2>/dev/null )" || return 1 [ "${state}" = "true" ] } pg_command() { local command="$1" as_service_user \ podman exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ sh -c "${command}" } pg_replication_exec() { as_service_user \ podman exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ sh -c ' if [ -z "${POSTGRES_REPLICATION_PASSWORD:-}" ]; then printf "%s\n" \ "POSTGRES_REPLICATION_PASSWORD is missing from container environment" \ >&2 exit 1 fi export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD}" exec "$@" ' sh "$@" } pgdata_path() { pg_command ' if [ -z "$PGDATA" ]; then exit 1 fi printf "%s\n" "$PGDATA" ' } pg_running() { pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 "$BINDIR/pg_ctl" status -D "$PGDATA" >/dev/null 2>&1 ' } pg_ready() { as_service_user \ podman exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ pg_isready -q \ -U "${DB_USER}" \ -d "${DB_NAME}" } pg_role() { as_service_user \ podman exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ psql \ -U "${DB_USER}" \ -d "${DB_NAME}" \ -Atqc 'SELECT pg_is_in_recovery();' } pg_remote_query() { local sql="$1" pg_replication_exec \ psql \ "host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ -Atqc "${sql}" } pg_wal_receiver_status() { as_service_user \ podman exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ psql \ -U "${DB_USER}" \ -d "${DB_NAME}" \ -Atqc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;" } ############################################################################### # Validation ############################################################################### validate_all() { if ! id "${SERVICE_USER}" >/dev/null 2>&1; then ocf_exit_reason "Service user '${SERVICE_USER}' does not exist" return "${OCF_ERR_CONFIGURED}" fi for cmd in runuser systemctl podman getent; do if ! command -v "${cmd}" >/dev/null 2>&1; then ocf_exit_reason "Required command '${cmd}' is not installed" return "${OCF_ERR_INSTALLED}" fi done if [ ! -d "/run/user/$(service_uid)" ]; then ocf_exit_reason \ "Runtime directory /run/user/$(service_uid) does not exist" return "${OCF_ERR_GENERIC}" fi if ! as_service_user \ systemctl --user cat "${SERVICE_NAME}" >/dev/null 2>&1 then ocf_exit_reason \ "User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}" return "${OCF_ERR_CONFIGURED}" fi local podman_error podman_error="$( as_service_user podman info 2>&1 >/dev/null )" local podman_rc=$? if [ "${podman_rc}" -ne 0 ]; then ocf_exit_reason \ "Rootless Podman failed for ${SERVICE_USER}: ${podman_error}" return "${OCF_ERR_GENERIC}" fi return "${OCF_SUCCESS}" } ############################################################################### # Monitor ############################################################################### managed_monitor() { local role if ! service_running; then ocf_log debug "User service ${SERVICE_NAME} is not active" return "${OCF_NOT_RUNNING}" fi if ! container_running; then ocf_log warn \ "Service ${SERVICE_NAME} is active but container ${CONTAINER_NAME} is not running" return "${OCF_NOT_RUNNING}" fi if ! pg_running; then ocf_log debug \ "Container ${CONTAINER_NAME} is running but PostgreSQL is stopped" return "${OCF_NOT_RUNNING}" fi if ! pg_ready; then ocf_exit_reason \ "PostgreSQL process exists but pg_isready failed" return "${OCF_ERR_GENERIC}" fi role="$(pg_role 2>/dev/null)" || { ocf_exit_reason \ "PostgreSQL is accepting connections but role query failed" return "${OCF_ERR_GENERIC}" } case "${role}" in f) ocf_log debug \ "PostgreSQL healthy: role=primary (promoted)" return "${OCF_RUNNING_PROMOTED}" ;; t) ocf_log debug \ "PostgreSQL healthy: role=standby (unpromoted)" return "${OCF_SUCCESS}" ;; *) ocf_exit_reason \ "Unexpected pg_is_in_recovery() result: '${role}'" return "${OCF_ERR_GENERIC}" ;; esac } ############################################################################### # Start ############################################################################### managed_start() { local i monitor_rc managed_monitor monitor_rc=$? case "${monitor_rc}" in "${OCF_SUCCESS}") ocf_log info \ "PostgreSQL resource already running unpromoted" return "${OCF_SUCCESS}" ;; "${OCF_RUNNING_PROMOTED}") ocf_exit_reason \ "Refusing start: PostgreSQL is already primary; promotion must occur through promote()" return "${OCF_ERR_GENERIC}" ;; "${OCF_NOT_RUNNING}") # Expected for a stopped resource. Continue. ;; *) ocf_exit_reason \ "Cannot start unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" return "${monitor_rc}" ;; esac ocf_log info \ "Ensuring container service ${SERVICE_NAME} is running" if ! service_running; then if ! as_service_user \ systemctl --user start "${SERVICE_NAME}" then ocf_exit_reason \ "Failed to start ${SERVICE_NAME}" return "${OCF_ERR_GENERIC}" fi fi for i in $(seq 1 20); do container_running && break sleep 0.5 done if ! container_running; then ocf_exit_reason \ "Container ${CONTAINER_NAME} did not become running" return "${OCF_ERR_GENERIC}" fi if ! pg_running; then # # OCF promotable-resource rule: # start() must produce an Unpromoted instance. # # A PostgreSQL data directory without standby.signal would start # writable/primary, so never start it from this action. # if ! pg_command ' test -f "$PGDATA/standby.signal" ' then ocf_exit_reason \ "Refusing start: PGDATA is not configured as a standby (standby.signal missing)" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Starting PostgreSQL as an unpromoted standby inside ${CONTAINER_NAME}" if ! pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 exec "$BINDIR/pg_ctl" \ -D "$PGDATA" \ -l "$PGDATA/postgresql.log" \ start -w ' then ocf_exit_reason "pg_ctl start failed" return "${OCF_ERR_GENERIC}" fi fi managed_monitor monitor_rc=$? case "${monitor_rc}" in "${OCF_SUCCESS}") ocf_log info \ "PostgreSQL start completed in unpromoted state" return "${OCF_SUCCESS}" ;; "${OCF_RUNNING_PROMOTED}") ocf_exit_reason \ "PostgreSQL became primary during start; stopping it to preserve OCF role semantics" pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 exec "$BINDIR/pg_ctl" \ -D "$PGDATA" \ stop -m fast -w ' >/dev/null 2>&1 || true return "${OCF_ERR_GENERIC}" ;; *) return "${monitor_rc}" ;; esac } ensure_rewind_privileges() { local repl_user repl_user="$( pg_command ' printf "%s" "${POSTGRES_REPLICATION_USER:-}" ' 2>/dev/null )" || { ocf_exit_reason \ "Unable to read POSTGRES_REPLICATION_USER from container" return "${OCF_ERR_GENERIC}" } if [ -z "${repl_user}" ]; then ocf_exit_reason \ "POSTGRES_REPLICATION_USER is empty" return "${OCF_ERR_CONFIGURED}" fi case "${repl_user}" in *[!A-Za-z0-9_]*) ocf_exit_reason \ "Invalid replication role name: ${repl_user}" return "${OCF_ERR_CONFIGURED}" ;; esac ocf_log info \ "Ensuring pg_rewind source privileges for role ${repl_user}" if ! pg_command " psql \ -U '${DB_USER}' \ -d postgres \ -v ON_ERROR_STOP=1 \ -c \" GRANT EXECUTE ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean) TO ${repl_user}; GRANT EXECUTE ON FUNCTION pg_catalog.pg_stat_file(text, boolean) TO ${repl_user}; GRANT EXECUTE ON FUNCTION pg_catalog.pg_read_binary_file(text) TO ${repl_user}; GRANT EXECUTE ON FUNCTION pg_catalog.pg_read_binary_file( text, bigint, bigint, boolean ) TO ${repl_user}; \" " >/dev/null then ocf_exit_reason \ "Failed ensuring pg_rewind privileges for ${repl_user}" return "${OCF_ERR_GENERIC}" fi return "${OCF_SUCCESS}" } ensure_local_ha_slots() { local slots raw_slot slot slot_type # # POSTGRES_HA_SLOTS lives in the container environment supplied by # postgres.env. It is the cluster-wide list of physical slots that # every writable primary must provide. # slots="$( pg_command ' printf "%s" "${POSTGRES_HA_SLOTS:-}" ' 2>/dev/null )" || { ocf_exit_reason \ "Unable to read POSTGRES_HA_SLOTS from PostgreSQL container" return "${OCF_ERR_GENERIC}" } if [ -z "${slots}" ]; then ocf_log warn \ "POSTGRES_HA_SLOTS is empty; no local HA slots will be reconciled" return "${OCF_SUCCESS}" fi while IFS= read -r raw_slot; do slot="$(printf '%s' "${raw_slot}" | tr -d '[:space:]')" [ -n "${slot}" ] || continue case "${slot}" in *[!a-z0-9_]*) ocf_exit_reason \ "Invalid physical replication slot name: ${slot}" return "${OCF_ERR_CONFIGURED}" ;; esac slot_type="$( pg_command " psql \ -U '${DB_USER}' \ -d postgres \ -At \ -v ON_ERROR_STOP=1 \ -c \" SELECT slot_type FROM pg_replication_slots WHERE slot_name = '${slot}'; \" " 2>/dev/null )" || { ocf_exit_reason \ "Failed querying replication slot ${slot}" return "${OCF_ERR_GENERIC}" } slot_type="$(printf '%s' "${slot_type}" | tr -d '[:space:]')" case "${slot_type}" in physical) ocf_log info \ "Physical HA slot already exists: ${slot}" ;; "") ocf_log info \ "Creating physical HA slot: ${slot}" if ! pg_command " psql \ -U '${DB_USER}' \ -d postgres \ -v ON_ERROR_STOP=1 \ -c \" SELECT pg_create_physical_replication_slot('${slot}'); \" " >/dev/null then ocf_exit_reason \ "Failed creating physical replication slot ${slot}" return "${OCF_ERR_GENERIC}" fi ;; *) ocf_exit_reason \ "Replication slot ${slot} exists but is type ${slot_type}" return "${OCF_ERR_CONFIGURED}" ;; esac done </dev/null | awk -v target="${target}" ' /nodelist\.node\.[0-9]+\.name / { key=$1 sub(/\.name$/, "", key) names[key]=$NF } /nodelist\.node\.[0-9]+\.ring0_addr / { key=$1 sub(/\.ring0_addr$/, "", key) addrs[key]=$NF } END { for (key in names) { if (names[key] == target && addrs[key] != "") { print addrs[key] exit 0 } } exit 1 } ' } candidate_caught_up() { local candidate="${1:-}" local result [ -n "${candidate}" ] || return 1 result="$(pg_command " psql -U '${DB_USER}' -d '${DB_NAME}' -Atc \" SELECT CASE WHEN state = 'streaming' AND replay_lsn >= pg_current_wal_flush_lsn() THEN 'yes' ELSE 'no' END FROM pg_stat_replication WHERE application_name = '${candidate}' LIMIT 1; \" " 2>/dev/null)" || return 1 [ "${result}" = "yes" ] } demote_target() { local state_file local type operation promote_uname demote_uname local_node local now mtime age state_file="/run/loxthelion/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" local_node="$(hostname)" [ -r "${state_file}" ] || return 1 now="$(date +%s)" || return 1 mtime="$(stat -c %Y "${state_file}" 2>/dev/null)" || return 1 age=$((now - mtime)) [ "${age}" -ge 0 ] || return 1 [ "${age}" -le 120 ] || return 1 type="$( sed -n 's/^type=//p' "${state_file}" | head -n1 )" operation="$( sed -n 's/^operation=//p' "${state_file}" | head -n1 )" promote_uname="$( sed -n 's/^promote_uname=//p' "${state_file}" | head -n1 )" demote_uname="$( sed -n 's/^demote_uname=//p' "${state_file}" | head -n1 )" [ "${type}" = "pre" ] || return 1 [ "${operation}" = "demote" ] || return 1 [ "${demote_uname}" = "${local_node}" ] || return 1 [ -n "${promote_uname}" ] || return 1 [ "${promote_uname}" != "${local_node}" ] || return 1 case "${promote_uname}" in *" "*) return 1 ;; esac printf '%s ' "${promote_uname}" } ensure_remote_slot() { local host="${1:-}" local slot="${2:-}" local result [ -n "${host}" ] || return 1 [ -n "${slot}" ] || return 1 result="$( pg_replication_exec \ psql \ "host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \ -Atc " SELECT slot_type FROM pg_replication_slots WHERE slot_name = '${slot}'; " 2>/dev/null )" || return 1 case "${result}" in physical) return 0 ;; "") if ! pg_replication_exec \ psql \ "host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \ -Atc " SELECT slot_name FROM pg_create_physical_replication_slot('${slot}'); " >/dev/null then return 1 fi return 0 ;; *) ocf_log err \ "Replication slot ${slot} exists on ${host} but is not physical" return 1 ;; esac } candidate_is_standby() { local host="${1:-}" local result local port="${REJOIN_SOURCE_PORT:-5432}" [ -n "${host}" ] || return 1 result="$( pg_replication_exec \ psql \ "host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres" \ -Atc "SELECT pg_is_in_recovery();" \ 2>/dev/null )" || return 1 [ "${result}" = "t" ] } managed_demote() { local monitor_rc local target_node target_addr local local_node slot_name target_port local role receiver_status i managed_monitor monitor_rc=$? case "${monitor_rc}" in "${OCF_RUNNING_PROMOTED}") ;; "${OCF_SUCCESS}") ocf_log info \ "PostgreSQL is already unpromoted" return "${OCF_SUCCESS}" ;; "${OCF_NOT_RUNNING}") ocf_exit_reason \ "Cannot complete demotion because PostgreSQL is stopped" return "${OCF_ERR_GENERIC}" ;; *) ocf_exit_reason \ "Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" return "${monitor_rc}" ;; esac target_node="$(demote_target)" || { ocf_exit_reason \ "No fresh valid Pacemaker pre-demote target is available" return "${OCF_ERR_GENERIC}" } target_addr="$(node_to_addr "${target_node}")" || { ocf_exit_reason \ "Unable to resolve Pacemaker promotion target ${target_node} through Corosync" return "${OCF_ERR_GENERIC}" } target_port="${REJOIN_SOURCE_PORT:-5432}" local_node="$(hostname)" slot_name="ha_$(printf '%s' "${local_node}" | tr '[:upper:]-' '[:lower:]_')" ocf_log info \ "Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})" if ! candidate_is_standby "${target_addr}"; then ocf_exit_reason \ "Promotion target ${target_node} is not a reachable PostgreSQL standby" return "${OCF_ERR_GENERIC}" fi if ! candidate_caught_up "${target_node}"; then ocf_exit_reason \ "Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position" return "${OCF_ERR_GENERIC}" fi if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then ocf_exit_reason \ "Unable to ensure physical replication slot ${slot_name} on ${target_node}" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Demotion safety checks passed; stopping PostgreSQL primary" if ! pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 exec "$BINDIR/pg_ctl" \ -D "$PGDATA" \ stop -m fast -w ' then ocf_exit_reason \ "pg_ctl stop failed during controlled demotion" return "${OCF_ERR_GENERIC}" fi if pg_running; then ocf_exit_reason \ "PostgreSQL remained running after pg_ctl stop" return "${OCF_ERR_GENERIC}" fi if ! service_running || ! container_running; then ocf_exit_reason \ "Maintenance service/container stopped unexpectedly during demotion" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Configuring ${local_node} as cascading standby of ${target_node}" if ! as_service_user \ podman exec \ --user "${PG_OS_USER}" \ --env "RA_PRIMARY_HOST=${target_addr}" \ --env "RA_PRIMARY_PORT=${target_port}" \ --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ --env "RA_REPLICATION_SLOT=${slot_name}" \ --env "RA_APPLICATION_NAME=${local_node}" \ "${CONTAINER_NAME}" \ sh -c ' set -e sed -i \ -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \ -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \ "$PGDATA/postgresql.auto.conf" printf "%s " \ "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \ "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \ >> "$PGDATA/postgresql.auto.conf" touch "$PGDATA/standby.signal" chmod 600 \ "$PGDATA/postgresql.auto.conf" \ "$PGDATA/standby.signal" ' then ocf_exit_reason \ "Failed to configure PostgreSQL as standby of ${target_node}" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Starting PostgreSQL as cascading standby of ${target_node}" if ! pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 exec "$BINDIR/pg_ctl" \ -D "$PGDATA" \ -l "$PGDATA/postgresql.log" \ start -w ' then ocf_exit_reason \ "PostgreSQL failed to start as cascading standby" return "${OCF_ERR_GENERIC}" fi for i in $(seq 1 30); do role="$(pg_role 2>/dev/null || true)" receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" if [ "${role}" = "t" ] && [ "${receiver_status}" = "streaming" ] then ocf_log info \ "Controlled demotion complete: ${local_node} is streaming from ${target_node}" return "${OCF_SUCCESS}" fi sleep 1 done ocf_exit_reason \ "PostgreSQL started after demotion but did not become a streaming standby" return "${OCF_ERR_GENERIC}" } managed_rejoin() { local source_role role receiver_status pgdata i case "${REJOIN_ALLOW_BASEBACKUP}" in true|false|1|0|yes|no) ;; *) ocf_exit_reason "rejoin_allow_basebackup must be true or false" return "${OCF_ERR_CONFIGURED}" ;; esac if [ -z "${REJOIN_SOURCE_HOST}" ]; then ocf_exit_reason \ "rejoin_source_host is required for rejoin" return "${OCF_ERR_CONFIGURED}" fi if [ -z "${REPLICATION_SLOT}" ]; then ocf_exit_reason \ "replication_slot is required for rejoin" return "${OCF_ERR_CONFIGURED}" fi if ! service_running; then ocf_exit_reason \ "Maintenance service ${SERVICE_NAME} must be running before rejoin" return "${OCF_ERR_GENERIC}" fi if ! container_running; then ocf_exit_reason \ "Maintenance container ${CONTAINER_NAME} must be running before rejoin" return "${OCF_ERR_GENERIC}" fi if pg_running; then ocf_exit_reason \ "PostgreSQL must be stopped before rejoin" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}" source_role="$(pg_remote_query \ 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { ocf_exit_reason \ "Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}" return "${OCF_ERR_GENERIC}" } if [ "${source_role}" != "f" ]; then ocf_exit_reason \ "Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}" then ocf_exit_reason \ "Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Running pg_rewind against ${REJOIN_SOURCE_HOST}" pgdata="$(pgdata_path 2>/dev/null)" || { ocf_exit_reason \ "Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}" return "${OCF_ERR_GENERIC}" } if [ -z "${pgdata}" ]; then ocf_exit_reason \ "PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}" return "${OCF_ERR_GENERIC}" fi if ! pg_replication_exec \ pg_rewind \ --target-pgdata="${pgdata}" \ --source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ --progress then local fallback_enabled source_bytes free_bytes required_bytes local timestamp preserved_pgdata failed_pgdata source_role_retry case "${REJOIN_ALLOW_BASEBACKUP}" in true|1|yes) fallback_enabled=true ;; *) fallback_enabled=false ;; esac if [ "${fallback_enabled}" != "true" ]; then ocf_exit_reason \ "pg_rewind failed and rejoin_allow_basebackup is not enabled" return "${OCF_ERR_GENERIC}" fi ocf_log warn \ "pg_rewind failed; evaluating pg_basebackup fallback" # Revalidate the source immediately before replacing local PGDATA. source_role_retry="$(pg_remote_query \ 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { ocf_exit_reason \ "Unable to revalidate rejoin source before pg_basebackup fallback" return "${OCF_ERR_GENERIC}" } if [ "${source_role_retry}" != "f" ]; then ocf_exit_reason \ "Rejoin source is no longer a writable primary; refusing pg_basebackup fallback" return "${OCF_ERR_GENERIC}" fi if pg_running; then ocf_exit_reason \ "PostgreSQL became active during rejoin; refusing pg_basebackup fallback" return "${OCF_ERR_GENERIC}" fi if ! service_running || ! container_running; then ocf_exit_reason \ "Maintenance service/container is unavailable; refusing pg_basebackup fallback" return "${OCF_ERR_GENERIC}" fi if ! ensure_remote_slot \ "${REJOIN_SOURCE_HOST}" \ "${REPLICATION_SLOT}" then ocf_exit_reason \ "Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback" return "${OCF_ERR_GENERIC}" fi # Estimate source database payload and require 2x that amount free. # This is deliberately conservative and does not pretend to be an # exact prediction of the resulting PGDATA size. source_bytes="$(pg_remote_query \ "SELECT COALESCE(sum(pg_database_size(datname)),0) FROM pg_database WHERE datallowconn;" 2>/dev/null)" || { ocf_exit_reason \ "Unable to estimate source database size before pg_basebackup fallback" return "${OCF_ERR_GENERIC}" } case "${source_bytes}" in ""|*[!0-9]*) ocf_exit_reason \ "Invalid source database size returned before pg_basebackup fallback" return "${OCF_ERR_GENERIC}" ;; esac free_bytes="$( pg_command ' blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")" case "$blocks" in ""|*[!0-9]*) exit 1 ;; esac printf "%s\n" "$((blocks * 1024))" ' 2>/dev/null )" || { ocf_exit_reason \ "Unable to determine free space for pg_basebackup fallback" return "${OCF_ERR_GENERIC}" } case "${free_bytes}" in ""|*[!0-9]*) ocf_exit_reason \ "Invalid free-space value before pg_basebackup fallback" return "${OCF_ERR_GENERIC}" ;; esac required_bytes=$((source_bytes * 2)) if [ "${free_bytes}" -lt "${required_bytes}" ]; then ocf_exit_reason \ "Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes}" return "${OCF_ERR_GENERIC}" fi timestamp="$(date -u +%Y%m%d-%H%M%S)" preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}" failed_pgdata="${pgdata}.failed-basebackup-${timestamp}" ocf_log warn \ "Preserving existing PGDATA as ${preserved_pgdata}" if ! pg_command " set -e [ -d '${pgdata}' ] [ ! -e '${preserved_pgdata}' ] [ ! -e '${failed_pgdata}' ] mv '${pgdata}' '${preserved_pgdata}' install -d -m 0700 '${pgdata}' " then ocf_exit_reason \ "Unable to preserve existing PGDATA for pg_basebackup fallback" return "${OCF_ERR_GENERIC}" fi ocf_log warn \ "Running pg_basebackup from ${REJOIN_SOURCE_HOST}" if ! pg_replication_exec \ pg_basebackup \ --dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \ --pgdata="${pgdata}" \ --format=plain \ --wal-method=stream \ --slot="${REPLICATION_SLOT}" \ --write-recovery-conf \ --progress then ocf_log err \ "pg_basebackup failed; preserving failed copy and restoring previous PGDATA" if ! pg_command " set -e if [ -e '${failed_pgdata}' ]; then exit 1 fi if [ -e '${pgdata}' ]; then mv '${pgdata}' '${failed_pgdata}' fi mv '${preserved_pgdata}' '${pgdata}' " then ocf_exit_reason \ "pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required" return "${OCF_ERR_GENERIC}" fi ocf_exit_reason \ "pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}" fi ocf_log info \ "Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}" if ! as_service_user \ podman exec \ --user "${PG_OS_USER}" \ --env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \ --env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \ --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ --env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \ --env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \ "${CONTAINER_NAME}" \ sh -c ' set -e sed -i \ -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \ -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \ "$PGDATA/postgresql.auto.conf" printf "%s\n" \ "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \ "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \ >> "$PGDATA/postgresql.auto.conf" touch "$PGDATA/standby.signal" chmod 600 \ "$PGDATA/postgresql.auto.conf" \ "$PGDATA/standby.signal" ' then ocf_exit_reason \ "Failed to configure standby settings" return "${OCF_ERR_GENERIC}" fi ocf_log info \ "Starting PostgreSQL as standby" if ! pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 exec "$BINDIR/pg_ctl" \ -D "$PGDATA" \ -l "$PGDATA/postgresql.log" \ start -w ' then ocf_exit_reason \ "PostgreSQL failed to start after rejoin" return "${OCF_ERR_GENERIC}" fi for i in $(seq 1 30); do role="$(pg_role 2>/dev/null)" || { sleep 1 continue } receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" if [ "${role}" = "t" ] && [ "${receiver_status}" = "streaming" ] then ocf_log info \ "PostgreSQL successfully rejoined as streaming standby" return "${OCF_SUCCESS}" fi sleep 1 done ocf_exit_reason \ "PostgreSQL started but did not become a streaming standby" return "${OCF_ERR_GENERIC}" } managed_stop() { local i if container_running && pg_running; then ocf_log info \ "Stopping PostgreSQL cleanly inside ${CONTAINER_NAME}" if ! pg_command ' BINDIR="$(pg_config --bindir)" || exit 1 exec "$BINDIR/pg_ctl" \ -D "$PGDATA" \ stop -m fast -w ' then ocf_exit_reason "pg_ctl stop failed" return "${OCF_ERR_GENERIC}" fi fi if service_running; then ocf_log info \ "Stopping container service ${SERVICE_NAME}" if ! as_service_user \ systemctl --user stop "${SERVICE_NAME}" then ocf_exit_reason \ "Failed to stop ${SERVICE_NAME}" return "${OCF_ERR_GENERIC}" fi fi for i in $(seq 1 20); do if ! service_running && ! container_running; then return "${OCF_SUCCESS}" fi sleep 0.5 done if service_running; then ocf_exit_reason \ "${SERVICE_NAME} remained active after stop" return "${OCF_ERR_GENERIC}" fi if container_running; then ocf_exit_reason \ "${CONTAINER_NAME} remained running after stop" return "${OCF_ERR_GENERIC}" fi return "${OCF_SUCCESS}" } managed_notify() { local state_dir state_file state_dir="/run/loxthelion" state_file="${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" ocf_log info \ "notify: type=${OCF_RESKEY_CRM_meta_notify_type:-unset} operation=${OCF_RESKEY_CRM_meta_notify_operation:-unset}" ocf_log info \ "notify: promote_uname=${OCF_RESKEY_CRM_meta_notify_promote_uname:-none} demote_uname=${OCF_RESKEY_CRM_meta_notify_demote_uname:-none}" ocf_log info \ "notify: promoted_uname=${OCF_RESKEY_CRM_meta_notify_promoted_uname:-none} unpromoted_uname=${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-none}" if ! mkdir -p -m 0755 "${state_dir}"; then ocf_exit_reason \ "Unable to create notification state directory ${state_dir}" return "${OCF_ERR_GENERIC}" fi if ! { printf 'timestamp=%s\n' "$(date --iso-8601=seconds)" printf 'node=%s\n' "$(hostname)" printf 'type=%s\n' "${OCF_RESKEY_CRM_meta_notify_type:-}" printf 'operation=%s\n' "${OCF_RESKEY_CRM_meta_notify_operation:-}" printf 'promote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" printf 'demote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" printf 'promoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" printf 'unpromoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" } > "${state_file}" then ocf_exit_reason \ "Unable to write notification state ${state_file}" return "${OCF_ERR_GENERIC}" fi chmod 0600 "${state_file}" || { ocf_exit_reason \ "Unable to secure notification state ${state_file}" return "${OCF_ERR_GENERIC}" } { printf '%s node=%s type=%s operation=%s promote=[%s] demote=[%s] promoted=[%s] unpromoted=[%s]\n' \ "$(date --iso-8601=seconds)" \ "$(hostname)" \ "${OCF_RESKEY_CRM_meta_notify_type:-}" \ "${OCF_RESKEY_CRM_meta_notify_operation:-}" \ "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" \ "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" \ "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" \ "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" \ >> "${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify.log" } || true return "${OCF_SUCCESS}" } ############################################################################### # Dispatcher ############################################################################### case "$1" in meta-data) meta_data exit "${OCF_SUCCESS}" ;; validate-all) validate_all exit $? ;; monitor|status) managed_monitor exit $? ;; start) validate_all || exit $? managed_start exit $? ;; stop) managed_stop exit $? ;; promote) validate_all || exit $? managed_promote exit $? ;; demote) validate_all || exit $? managed_demote exit $? ;; notify) managed_notify exit $? ;; rejoin) validate_all || exit $? managed_rejoin exit $? ;; *) echo "Usage: $0 {start|stop|promote|demote|notify|rejoin|monitor|status|validate-all|meta-data}" exit "${OCF_ERR_UNIMPLEMENTED}" ;; esac