From b6c4fc848fce328d633ab9e58b17a0bd208a7d25 Mon Sep 17 00:00:00 2001 From: Lyncoln Date: Thu, 17 Sep 2026 10:54:08 -0400 Subject: [PATCH] Initial managed-container HA resource agent baseline --- .gitignore | 20 + ocf/loxthelion/ipvs-sync | 372 ++++++++++ ocf/loxthelion/managed-container | 113 +++ .../applications/postgresql/auth.sh | 88 +++ .../applications/postgresql/demote.sh | 278 +++++++ .../applications/postgresql/native_peers.sh | 228 ++++++ .../applications/postgresql/notify.sh | 120 +++ .../applications/postgresql/peer_config.sh | 54 ++ .../applications/postgresql/plugin.sh | 391 ++++++++++ .../applications/postgresql/postgres.sh | 696 ++++++++++++++++++ .../applications/postgresql/promote.sh | 110 +++ .../postgresql/promotion_state.py | 173 +++++ .../applications/postgresql/rejoin.sh | 391 ++++++++++ .../applications/postgresql/replication.sh | 256 +++++++ ocf/loxthelion/managed-container.d/common.sh | 209 ++++++ .../common.sh.named-rc.new | 209 ++++++ .../managed-container.d/container.sh | 54 ++ ocf/loxthelion/managed-container.d/engine.sh | 64 ++ .../managed-container.d/lifecycles/systemd.sh | 23 + ocf/loxthelion/managed-container.d/proxy.sh | 464 ++++++++++++ .../managed-container.d/proxy.sh.named-rc.new | 464 ++++++++++++ .../managed-container.d/runtimes/podman.sh | 35 + ocf/loxthelion/managed-container.named-rc.new | 93 +++ tests/test-entrypoint-structure.sh | 46 ++ 24 files changed, 4951 insertions(+) create mode 100644 .gitignore create mode 100755 ocf/loxthelion/ipvs-sync create mode 100755 ocf/loxthelion/managed-container create mode 100644 ocf/loxthelion/managed-container.d/applications/postgresql/auth.sh create mode 100644 ocf/loxthelion/managed-container.d/applications/postgresql/demote.sh create mode 100755 ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh create mode 100755 ocf/loxthelion/managed-container.d/applications/postgresql/notify.sh create mode 100644 ocf/loxthelion/managed-container.d/applications/postgresql/peer_config.sh create mode 100755 ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh create mode 100755 ocf/loxthelion/managed-container.d/applications/postgresql/postgres.sh create mode 100644 ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh create mode 100644 ocf/loxthelion/managed-container.d/applications/postgresql/promotion_state.py create mode 100755 ocf/loxthelion/managed-container.d/applications/postgresql/rejoin.sh create mode 100644 ocf/loxthelion/managed-container.d/applications/postgresql/replication.sh create mode 100755 ocf/loxthelion/managed-container.d/common.sh create mode 100755 ocf/loxthelion/managed-container.d/common.sh.named-rc.new create mode 100644 ocf/loxthelion/managed-container.d/container.sh create mode 100755 ocf/loxthelion/managed-container.d/engine.sh create mode 100755 ocf/loxthelion/managed-container.d/lifecycles/systemd.sh create mode 100755 ocf/loxthelion/managed-container.d/proxy.sh create mode 100755 ocf/loxthelion/managed-container.d/proxy.sh.named-rc.new create mode 100755 ocf/loxthelion/managed-container.d/runtimes/podman.sh create mode 100755 ocf/loxthelion/managed-container.named-rc.new create mode 100755 tests/test-entrypoint-structure.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8c732b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# Python +__pycache__/ +*.py[cod] + +# Editor / OS +*~ +*.swp +*.swo +.DS_Store + +# Development backups +*.bak +*.orig +*.rej +*.pre-* +*.pre-*-* + +# Temporary files +*.tmp +*.temp diff --git a/ocf/loxthelion/ipvs-sync b/ocf/loxthelion/ipvs-sync new file mode 100755 index 0000000..7b1439f --- /dev/null +++ b/ocf/loxthelion/ipvs-sync @@ -0,0 +1,372 @@ +#!/bin/bash +# +# OCF resource agent for Linux IPVS connection synchronization +# +# Promoted = IPVS sync master +# Unpromoted = IPVS sync backup +# + +OCF_SUCCESS=0 +OCF_ERR_GENERIC=1 +OCF_ERR_ARGS=2 +OCF_ERR_UNIMPLEMENTED=3 +OCF_ERR_PERM=4 +OCF_ERR_INSTALLED=5 +OCF_ERR_CONFIGURED=6 +OCF_NOT_RUNNING=7 +OCF_RUNNING_PROMOTED=8 +OCF_FAILED_PROMOTED=9 + +IPVSADM="${IPVSADM:-/usr/sbin/ipvsadm}" +IP="${IP:-/usr/sbin/ip}" + +# The IP is used only to discover the LAN interface. +# Different nodes may have different interface names. +OCF_RESKEY_probe_ip="${OCF_RESKEY_probe_ip:-192.168.1.1}" +OCF_RESKEY_interface="${OCF_RESKEY_interface:-}" + +meta_data() +{ + cat <<'XML' + + + + 1.0 + + + Manages the Linux IPVS connection synchronization daemon as a + Pacemaker promotable resource. + + Unpromoted instances run the IPVS backup synchronization daemon. + The promoted instance runs the IPVS master synchronization daemon. + + + + IPVS connection synchronization + + + + + + An IPv4 address reachable through the interface that should + carry IPVS synchronization traffic. Used only for automatic + interface discovery. + + Interface discovery IP + + + + + + Optional explicit synchronization interface. If omitted, + the interface is discovered using the route to probe_ip. + + Synchronization interface + + + + + + + + + + + + + + +XML +} + +get_iface() +{ + if [ -n "$OCF_RESKEY_interface" ]; then + printf '%s\n' "$OCF_RESKEY_interface" + return 0 + fi + + "$IP" route get "$OCF_RESKEY_probe_ip" 2>/dev/null | + awk ' + { + for (i=1; i<=NF; i++) { + if ($i == "dev" && (i+1) <= NF) { + print $(i+1) + exit + } + } + }' +} + +daemon_status() +{ + "$IPVSADM" -L --daemon 2>/dev/null +} + +set_promotion_score() +{ + /usr/sbin/crm_attribute --promotion --update 100 >/dev/null 2>&1 || true +} + +monitor_resource() +{ + local output + local have_master=0 + local have_backup=0 + + output="$(daemon_status)" + + printf '%s\n' "$output" | + grep -q '^master sync daemon' && + have_master=1 + + printf '%s\n' "$output" | + grep -q '^backup sync daemon' && + have_backup=1 + + # Both roles simultaneously is an invalid state. + if [ "$have_master" -eq 1 ] && + [ "$have_backup" -eq 1 ]; then + return "$OCF_ERR_GENERIC" + fi + + if [ "$have_master" -eq 1 ]; then + set_promotion_score + return "$OCF_RUNNING_PROMOTED" + fi + + if [ "$have_backup" -eq 1 ]; then + set_promotion_score + return "$OCF_SUCCESS" + fi + + return "$OCF_NOT_RUNNING" +} + +stop_master() +{ + "$IPVSADM" --stop-daemon master >/dev/null 2>&1 || true +} + +stop_backup() +{ + "$IPVSADM" --stop-daemon backup >/dev/null 2>&1 || true +} + +start_master() +{ + local iface + + iface="$(get_iface)" + + if [ -z "$iface" ]; then + echo "ERROR: unable to determine synchronization interface" >&2 + return "$OCF_ERR_GENERIC" + fi + + "$IPVSADM" \ + --start-daemon master \ + --mcast-interface "$iface" +} + +start_backup() +{ + local iface + + iface="$(get_iface)" + + if [ -z "$iface" ]; then + echo "ERROR: unable to determine synchronization interface" >&2 + return "$OCF_ERR_GENERIC" + fi + + "$IPVSADM" \ + --start-daemon backup \ + --mcast-interface "$iface" +} + +start_resource() +{ + monitor_resource + rc=$? + + case "$rc" in + "$OCF_SUCCESS") + # Already running correctly as backup. + return "$OCF_SUCCESS" + ;; + + "$OCF_RUNNING_PROMOTED") + # A start operation must leave a promotable resource + # in its unpromoted state. + stop_master + start_backup || return "$OCF_ERR_GENERIC" + ;; + + "$OCF_NOT_RUNNING") + start_backup || return "$OCF_ERR_GENERIC" + ;; + + *) + return "$OCF_ERR_GENERIC" + ;; + esac + + monitor_resource + [ $? -eq "$OCF_SUCCESS" ] && + return "$OCF_SUCCESS" + + return "$OCF_ERR_GENERIC" +} + +stop_resource() +{ + stop_master + stop_backup + + monitor_resource + rc=$? + + [ "$rc" -eq "$OCF_NOT_RUNNING" ] && + return "$OCF_SUCCESS" + + return "$OCF_ERR_GENERIC" +} + +promote_resource() +{ + monitor_resource + rc=$? + + case "$rc" in + "$OCF_RUNNING_PROMOTED") + return "$OCF_SUCCESS" + ;; + + "$OCF_SUCCESS") + stop_backup + + if ! start_master; then + # Try to return to a safe unpromoted state. + start_backup >/dev/null 2>&1 || true + return "$OCF_ERR_GENERIC" + fi + ;; + + *) + echo "ERROR: cannot promote IPVS sync resource from state rc=$rc" >&2 + return "$OCF_ERR_GENERIC" + ;; + esac + + monitor_resource + [ $? -eq "$OCF_RUNNING_PROMOTED" ] && + return "$OCF_SUCCESS" + + return "$OCF_ERR_GENERIC" +} + +demote_resource() +{ + monitor_resource + rc=$? + + case "$rc" in + "$OCF_SUCCESS") + return "$OCF_SUCCESS" + ;; + + "$OCF_RUNNING_PROMOTED") + stop_master + + if ! start_backup; then + return "$OCF_ERR_GENERIC" + fi + ;; + + "$OCF_NOT_RUNNING") + # Be defensive: demote should result in a running + # unpromoted instance. + start_backup || return "$OCF_ERR_GENERIC" + ;; + + *) + return "$OCF_ERR_GENERIC" + ;; + esac + + monitor_resource + [ $? -eq "$OCF_SUCCESS" ] && + return "$OCF_SUCCESS" + + return "$OCF_ERR_GENERIC" +} + +validate_all() +{ + if [ ! -x "$IPVSADM" ]; then + echo "ERROR: ipvsadm not found at $IPVSADM" >&2 + return "$OCF_ERR_INSTALLED" + fi + + if [ ! -x "$IP" ]; then + echo "ERROR: ip command not found at $IP" >&2 + return "$OCF_ERR_INSTALLED" + fi + + iface="$(get_iface)" + + if [ -z "$iface" ]; then + echo "ERROR: unable to determine synchronization interface" >&2 + return "$OCF_ERR_ARGS" + fi + + if ! "$IP" link show dev "$iface" >/dev/null 2>&1; then + echo "ERROR: interface '$iface' does not exist" >&2 + return "$OCF_ERR_ARGS" + fi + + return "$OCF_SUCCESS" +} + +case "$1" in + meta-data) + meta_data + exit "$OCF_SUCCESS" + ;; + + validate-all) + validate_all + exit $? + ;; + + start) + start_resource + exit $? + ;; + + stop) + stop_resource + exit $? + ;; + + monitor|status) + monitor_resource + exit $? + ;; + + promote) + promote_resource + exit $? + ;; + + demote) + demote_resource + exit $? + ;; + + *) + echo "Usage: $0 {start|stop|monitor|promote|demote|validate-all|meta-data}" >&2 + exit "$OCF_ERR_UNIMPLEMENTED" + ;; +esac diff --git a/ocf/loxthelion/managed-container b/ocf/loxthelion/managed-container new file mode 100755 index 0000000..f48fec9 --- /dev/null +++ b/ocf/loxthelion/managed-container @@ -0,0 +1,113 @@ +#!/bin/bash + +# OCF Resource Agent: +# loxthelion:managed-container +# +# v0.4 +# Container lifecycle: systemd user Quadlet +# Application profile: PostgreSQL +# +# Version 0.4 adds explicit PostgreSQL demotion scaffolding. +# Healthy primaries report OCF_RUNNING_PROMOTED while healthy standbys +# report OCF_SUCCESS. + +: "${OCF_ROOT:=/usr/lib/ocf}" +OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat" +# Preserve explicit resource identity before ocf-shellfuncs supplies its +# standalone-invocation default; never publish a score for that dummy resource. +MC_PROMOTION_RESOURCE_INSTANCE="${OCF_RESOURCE_INSTANCE:-}" +. "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs" + +# Pacemaker uses "Promoted" terminology, while many OCF shell libraries +# still expose the historical OCF_RUNNING_MASTER constant for return code 8. +# Normalize both forms here so the agent remains portable. +: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}" + +# Internal managed-containerd client return classes. +# Private RA implementation statuses, not OCF action return codes. +readonly MC_PROXY_RC_TRANSPORT=10 +readonly MC_PROXY_RC_HARD_FAILURE=11 + + +# Resolve modules relative to this entrypoint, independent of Pacemaker's cwd. +MC_MODULE_DIR="$(CDPATH= cd -- "$(command dirname -- "${BASH_SOURCE[0]}")" && pwd)/managed-container.d" +for MC_MODULE in proxy common container engine; do + if ! . "${MC_MODULE_DIR}/${MC_MODULE}.sh"; then + ocf_exit_reason "Unable to load resource-agent module ${MC_MODULE_DIR}/${MC_MODULE}.sh" + exit "${OCF_ERR_CONFIGURED}" + fi +done + +# Structural self-test for release/deployment validation. +# This deliberately runs through the real entrypoint module-loading path. +if [ "${MC_ENTRYPOINT_SELFTEST:-0}" = "1" ]; then + for MC_REQUIRED_FUNCTION in \ + mc_proxy_monitor \ + mc_proxy_monitor_cache_clear \ + mc_promotion_monitor + do + if ! command -V "${MC_REQUIRED_FUNCTION}" >/dev/null 2>&1; then + printf 'ENTRYPOINT_SELFTEST_MISSING=%s\n' \ + "${MC_REQUIRED_FUNCTION}" >&2 + exit 1 + fi + done + + printf '%s\n' 'ENTRYPOINT_STRUCTURE_OK=1' + exit 0 +fi + +unset MC_MODULE +mc_engine_load || exit $? + +case "$1" in + meta-data) + mc_engine_metadata + exit $? + ;; + + validate-all) + mc_promotion_validate + exit $? + ;; + + monitor|status) + mc_promotion_monitor + exit $? + ;; + + start) + mc_promotion_action start + exit $? + ;; + + stop) + mc_promotion_action stop + exit $? + ;; + + promote) + mc_promotion_action promote + exit $? + ;; + + demote) + mc_promotion_action demote + exit $? + ;; + + notify) + mc_application_notify + exit $? + ;; + + rejoin) + mc_promotion_action rejoin + exit $? + ;; + + *) + echo "Usage: $0 {start|stop|promote|demote|notify|rejoin|monitor|status|validate-all|meta-data}" + exit "${OCF_ERR_UNIMPLEMENTED}" + ;; +esac diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/auth.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/auth.sh new file mode 100644 index 0000000..ae0a533 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/auth.sh @@ -0,0 +1,88 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +pg_replication_exec() +{ + mc_runtime_exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + sh -c ' + if [ -z "${POSTGRES_REPLICATION_PASSWORD:-}" ]; then + printf "%s\n" \ + "POSTGRES_REPLICATION_PASSWORD is missing from container environment" \ + >&2 + exit 1 + fi + + export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD}" + + exec "$@" + ' sh "$@" +} + +ensure_rewind_privileges() +{ + local repl_user + + repl_user="$( + pg_command ' + printf "%s" "${POSTGRES_REPLICATION_USER:-}" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to read POSTGRES_REPLICATION_USER from container" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${repl_user}" ]; then + ocf_exit_reason \ + "POSTGRES_REPLICATION_USER is empty" + return "${OCF_ERR_CONFIGURED}" + fi + + case "${repl_user}" in + *[!A-Za-z0-9_]*) + ocf_exit_reason \ + "Invalid replication role name: ${repl_user}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + ocf_log info \ + "Ensuring pg_rewind source privileges for role ${repl_user}" + + if ! pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -v ON_ERROR_STOP=1 \ + -c \" + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_stat_file(text, boolean) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_read_binary_file(text) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_read_binary_file( + text, + bigint, + bigint, + boolean + ) + TO ${repl_user}; + \" + " >/dev/null + then + ocf_exit_reason \ + "Failed ensuring pg_rewind privileges for ${repl_user}" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/demote.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/demote.sh new file mode 100644 index 0000000..1dee141 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/demote.sh @@ -0,0 +1,278 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +node_to_addr() +{ + local target="${1:-}" + + [ -n "${target}" ] || return 1 + + corosync-cmapctl 2>/dev/null | + awk -v target="${target}" ' + /nodelist\.node\.[0-9]+\.name / { + key=$1 + sub(/\.name$/, "", key) + names[key]=$NF + } + + /nodelist\.node\.[0-9]+\.ring0_addr / { + key=$1 + sub(/\.ring0_addr$/, "", key) + addrs[key]=$NF + } + + END { + for (key in names) { + if (names[key] == target && addrs[key] != "") { + print addrs[key] + exit 0 + } + } + exit 1 + } + ' +} + +# Read the final checkpoint only after pg_ctl has confirmed clean primary shutdown. +mc_pg_shutdown_checkpoint() +{ + local control summary + control="$(mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" --env LC_ALL=C \ + "${CONTAINER_NAME}" sh -c \ + 'exec "$(pg_config --bindir)/pg_controldata" "$PGDATA"')" || return 1 + summary="$(printf '%s\n' "${control}" | awk -F ': *' ' + /^Database cluster state:/ { state=$2; ns++ } + /^Database system identifier:/ { sid=$2; ni++ } + /^Latest checkpoint location:/ { checkpoint=$2; nc++ } + /^Latest checkpoint.s TimeLineID:/ { timeline=$2; nt++ } + END { + if (ns != 1 || ni != 1 || nc != 1 || nt != 1 || state != "shut down") exit 1; + print sid "|" timeline "|" checkpoint; + }')" || return 1 + [[ "${summary}" =~ ^[0-9]+\|[1-9][0-9]*\|[0-9A-F]+/[0-9A-F]+$ ]] || return 1 + printf '%s\n' "${summary}" +} + +# A checkpoint LSN is its record START. Replay must be strictly beyond it to +# establish that the complete shutdown checkpoint has been replayed. +mc_pg_wait_shutdown_replay() +{ + local host="$1" port="$2" observation="$3" deadline="$4" + local system timeline checkpoint response ready identity rest got_system got_timeline + local remaining duration + IFS='|' read -r system timeline checkpoint <<< "${observation}" + [[ "${observation}" =~ ^[0-9]+\|[1-9][0-9]*\|[0-9A-F]+/[0-9A-F]+$ ]] || return 1 + while (( SECONDS < deadline )); do + remaining=$((deadline - SECONDS)) + duration=3 + (( remaining < duration )) && duration=${remaining} + response="$(mc_runtime_exec_bounded "${duration}s" --user "${PG_OS_USER}" \ + --env PGCONNECT_TIMEOUT=2 --env 'PGOPTIONS=-c statement_timeout=1000' \ + "${CONTAINER_NAME}" sh -c ' + export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD:?}" + psql -X -w -v ON_ERROR_STOP=1 -h "$1" -p "$2" -U "$3" -d postgres -Atqc \ + "SELECT CASE WHEN pg_is_in_recovery() THEN + NOT pg_is_wal_replay_paused() AND + pg_last_wal_replay_lsn() > '\''$4'\''::pg_lsn + ELSE false END;" || exit 1 + exec psql -X -w -v ON_ERROR_STOP=1 -h "$1" -p "$2" -U "$3" \ + -d replication=true -Atqc IDENTIFY_SYSTEM + ' sh "${host}" "${port}" "${REPLICATION_USER}" "${checkpoint}" 2>/dev/null)" || response='' + ready="${response%%$'\n'*}" + identity="${response#*$'\n'}" + IFS='|' read -r got_system got_timeline rest <<< "${identity}" + if [ "${ready}" = t ] && [ "${got_system}" = "${system}" ] && + [ "${got_timeline}" = "${timeline}" ]; then + return 0 + fi + (( SECONDS < deadline )) && sleep 1 + done + return 1 +} + +managed_demote() +{ + local monitor_rc + local target_node target_addr + local local_node slot_name target_port + local role receiver_status i checkpoint handoff_deadline + local action_started=${SECONDS} action_budget=${OCF_RESKEY_CRM_meta_timeout:-60000} + [[ "${action_budget}" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}" + action_budget=$((10#${action_budget} / 1000)) + if (( action_budget < 15 )); then + ocf_exit_reason "Demotion timeout is too short for a verified shutdown-WAL handoff" + return "${OCF_ERR_CONFIGURED}" + fi + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") + ;; + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL is already unpromoted" + return "${OCF_SUCCESS}" + ;; + "${OCF_NOT_RUNNING}") + ocf_exit_reason \ + "Cannot complete demotion because PostgreSQL is stopped" + return "${OCF_ERR_GENERIC}" + ;; + *) + ocf_exit_reason \ + "Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + target_node="$(demote_target)" || { + ocf_exit_reason \ + "No fresh valid Pacemaker pre-demote target is available" + return "${OCF_ERR_GENERIC}" + } + + target_addr="$(node_to_addr "${target_node}")" || { + ocf_exit_reason \ + "Unable to resolve Pacemaker promotion target ${target_node} through Corosync" + return "${OCF_ERR_GENERIC}" + } + + target_port="${REJOIN_SOURCE_PORT:-5432}" + local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" + slot_name="ha_$(printf '%s' "${local_node}" | + tr '[:upper:]-' '[:lower:]_')" + + ocf_log info \ + "Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})" + + if ! candidate_is_standby "${target_addr}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not a reachable PostgreSQL standby" + return "${OCF_ERR_GENERIC}" + fi + + if ! candidate_caught_up "${target_node}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then + ocf_exit_reason \ + "Unable to ensure physical replication slot ${slot_name} on ${target_node}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Demotion safety checks passed; stopping PostgreSQL primary" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason \ + "pg_ctl stop failed during controlled demotion" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL remained running after pg_ctl stop" + return "${OCF_ERR_GENERIC}" + fi + + if ! service_running || ! container_running; then + ocf_exit_reason \ + "Maintenance service/container stopped unexpectedly during demotion" + return "${OCF_ERR_GENERIC}" + fi + + # Keep at most 20 seconds for the handoff, reserving 10 seconds of the + # action budget for subsequent work. Pacemaker remains the overall timeout. + checkpoint="$(mc_pg_shutdown_checkpoint)" || { + ocf_exit_reason "Unable to read a valid clean-shutdown checkpoint; PostgreSQL remains stopped" + return "${OCF_ERR_GENERIC}" + } + handoff_deadline=$((action_started + action_budget - 10)) + (( handoff_deadline > SECONDS + 20 )) && handoff_deadline=$((SECONDS + 20)) + if ! mc_pg_wait_shutdown_replay "${target_addr}" "${target_port}" "${checkpoint}" "${handoff_deadline}"; then + ocf_exit_reason "Successor has not verified final shutdown-checkpoint replay; PostgreSQL remains stopped" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Configuring ${local_node} as cascading standby of ${target_node}" + + if ! mc_runtime_exec \ + --user "${PG_OS_USER}" \ + --env "RA_PRIMARY_HOST=${target_addr}" \ + --env "RA_PRIMARY_PORT=${target_port}" \ + --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ + --env "RA_REPLICATION_SLOT=${slot_name}" \ + --env "RA_APPLICATION_NAME=${local_node}" \ + "${CONTAINER_NAME}" \ + sh -c ' + set -e + + sed -i \ + -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \ + -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \ + "$PGDATA/postgresql.auto.conf" + + printf "%s +" \ + "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \ + "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \ + >> "$PGDATA/postgresql.auto.conf" + + touch "$PGDATA/standby.signal" + + chmod 600 \ + "$PGDATA/postgresql.auto.conf" \ + "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Failed to configure PostgreSQL as standby of ${target_node}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as cascading standby of ${target_node}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason \ + "PostgreSQL failed to start as cascading standby" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null || true)" + receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" + + if [ "${role}" = "t" ] && + [ "${receiver_status}" = "streaming" ] + then + ocf_log info \ + "Controlled demotion complete: ${local_node} is streaming from ${target_node}" + return "${OCF_SUCCESS}" + fi + + sleep 1 + done + + ocf_exit_reason \ + "PostgreSQL started after demotion but did not become a streaming standby" + return "${OCF_ERR_GENERIC}" +} diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh new file mode 100755 index 0000000..7b6b314 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/native_peers.sh @@ -0,0 +1,228 @@ +# Optional native peer lifecycle; defaults continue to use the legacy path. +mc_pg_peer_validate() +{ + [ -n "${OCF_RESKEY_replication_peers:-}" ] || return 0 + mc_peer_conninfo "${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" >/dev/null || { + ocf_exit_reason "Invalid replication_peers map or local identity" + return "${OCF_ERR_CONFIGURED}" + } + # The container may legitimately not exist yet during validate/start. + # Validate an explicitly supplied slot now, but defer runtime discovery + # until an operation actually requires the slot. + if [ -n "${REPLICATION_SLOT}" ]; then + case "${REPLICATION_SLOT}" in ''|*[!a-z0-9_]*) + ocf_exit_reason "Configured replication slot is invalid; expected lowercase letters, digits, and underscore" + return "${OCF_ERR_CONFIGURED}";; esac + [ ${#REPLICATION_SLOT} -le 63 ] || return "${OCF_ERR_CONFIGURED}" + fi +} + +mc_pg_write_peer_recovery() +{ + local code + # Same canonical POSIX helper is mounted into bootstrap; no duplicated writer. + code="$(cat "${MC_APPLICATION_DIR}/peer_config.sh")" || return 1 + mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" sh -c \ + "${code} +mc_peer_write_recovery \"\$PGDATA\" \"\$1\" \"\$2\" \"\$3\" \"\$4\" \"\$5\"" sh \ + "${OCF_RESKEY_replication_peers}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" \ + "${REPLICATION_USER}" "${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" "${REPLICATION_SLOT}" +} + +mc_pg_native_demote() +( + local mc_native_started=${SECONDS} + local mc_native_ms=${OCF_RESKEY_CRM_meta_timeout:-60000} + [[ "$mc_native_ms" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}" + local mc_native_deadline=$((mc_native_started + 10#$mc_native_ms / 1000 - 5)) + local mc_original_bounded + mc_original_bounded=$(declare -f mc_runtime_exec_bounded) + eval "${mc_original_bounded/mc_runtime_exec_bounded/mc_native_original_bounded}" + mc_runtime_exec_bounded() { + local requested=$1 remaining=$((mc_native_deadline - SECONDS)) + shift + (( remaining > 0 )) || return 124 + # Existing bounded call sites use integer seconds. + [[ "$requested" =~ ^[0-9]+s$ ]] || return 124 + (( ${requested%s} < remaining )) && remaining=${requested%s} + mc_native_original_bounded "${remaining}s" "$@" + } + mc_runtime_exec() { + local remaining=$((mc_native_deadline - SECONDS)) + (( remaining > 0 )) || return 124 + mc_native_original_bounded "${remaining}s" "$@" + } + local monitor_rc + local target_node target_addr + local local_node slot_name target_port + local role receiver_status i checkpoint handoff_deadline + local self_restart=false + local action_started=${mc_native_started} action_budget=${OCF_RESKEY_CRM_meta_timeout:-60000} + [[ "${action_budget}" =~ ^[0-9]{1,9}$ ]] || return "${OCF_ERR_CONFIGURED}" + action_budget=$((10#${action_budget} / 1000)) + if (( action_budget < 15 )); then + ocf_exit_reason "Demotion timeout is too short for a verified shutdown-WAL handoff" + return "${OCF_ERR_CONFIGURED}" + fi + + mc_pg_peer_validate || return "$?" + mc_pg_require_replication_slot || return "$?" + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") + ;; + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL is already unpromoted" + return "${OCF_SUCCESS}" + ;; + "${OCF_NOT_RUNNING}") + ocf_exit_reason \ + "Cannot complete demotion because PostgreSQL is stopped" + return "${OCF_ERR_GENERIC}" + ;; + *) + ocf_exit_reason \ + "Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" + slot_name=${REPLICATION_SLOT} + local native_deadline=$((action_started + action_budget - 5)) + + target_node="$(mc_pg_native_demote_target)" || { + ocf_exit_reason \ + "No fresh valid Pacemaker pre-demote target is available" + return "${OCF_ERR_GENERIC}" + } + + if [ "$(printf '%s' "${target_node}" | tr '[:upper:]' '[:lower:]')" = \ + "$(printf '%s' "${local_node}" | tr '[:upper:]' '[:lower:]')" ]; then + self_restart=true + ocf_log info \ + "Pacemaker requested a self-restart demotion of ${local_node}; performing clean local demotion without remote handoff" + else + local peer_pair + peer_pair=$(mc_peer_endpoints "${OCF_RESKEY_replication_peers}" "${local_node}" "${target_node}") || { + ocf_exit_reason "Successor is missing from the validated peer map" + return "${OCF_ERR_CONFIGURED}" + } + target_addr=${peer_pair%|*} + target_port=${peer_pair#*|} + # Existing candidate/slot SQL helpers use REJOIN_SOURCE_PORT. + local REJOIN_SOURCE_PORT=${target_port} + + ocf_log info \ + "Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})" + + if ! candidate_is_standby "${target_addr}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not a reachable PostgreSQL standby" + return "${OCF_ERR_GENERIC}" + fi + + if ! candidate_caught_up "${target_node}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then + ocf_exit_reason \ + "Unable to ensure physical replication slot ${slot_name} on ${target_node}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Demotion safety checks passed; stopping PostgreSQL primary" + fi + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason \ + "pg_ctl stop failed during controlled demotion" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL remained running after pg_ctl stop" + return "${OCF_ERR_GENERIC}" + fi + + if ! service_running || ! container_running; then + ocf_exit_reason \ + "Maintenance service/container stopped unexpectedly during demotion" + return "${OCF_ERR_GENERIC}" + fi + + checkpoint="$(mc_pg_shutdown_checkpoint)" || { + ocf_exit_reason "Unable to read a valid clean-shutdown checkpoint; PostgreSQL remains stopped" + return "${OCF_ERR_GENERIC}" + } + + if [ "${self_restart}" != true ]; then + # Keep at most 20 seconds for the handoff, reserving 10 seconds of the + # action budget for subsequent work. Pacemaker remains the overall timeout. + handoff_deadline=$((action_started + action_budget - 10)) + (( handoff_deadline > SECONDS + 20 )) && handoff_deadline=$((SECONDS + 20)) + if ! mc_pg_wait_shutdown_replay "${target_addr}" "${target_port}" "${checkpoint}" "${handoff_deadline}"; then + ocf_exit_reason "Successor has not verified final shutdown-checkpoint replay; PostgreSQL remains stopped" + return "${OCF_ERR_GENERIC}" + fi + else + ocf_log info \ + "Clean shutdown checkpoint verified locally for Pacemaker self-restart demotion" + fi + + ocf_log info \ + "Configuring ${local_node} with stable native replication peers" + + if ! mc_pg_write_peer_recovery; then + ocf_exit_reason "Failed to write native peer recovery configuration" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL in recovery with native primary selection" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason \ + "PostgreSQL failed to start as cascading standby" + return "${OCF_ERR_GENERIC}" + fi + + while (( SECONDS < native_deadline )); do + if pg_ready && [ "$(pg_role 2>/dev/null)" = t ]; then + local state + state=$(mc_runtime_exec --user "${PG_OS_USER}" "${CONTAINER_NAME}" psql -X -w -v ON_ERROR_STOP=1 -U "${DB_USER}" -d postgres -Atqc "SELECT system_identifier::text || '|' || (NOT pg_is_wal_replay_paused())::text FROM pg_control_system()" 2>/dev/null) || state='' + if [ "${state}" = "${checkpoint%%|*}|true" ]; then + if [ "${self_restart}" = true ]; then + ocf_log info "Pacemaker self-restart demotion complete: clean former primary is ready in recovery for possible re-promotion" + else + ocf_log info "Native peer demotion complete: final WAL verified; old primary ready in recovery awaiting successor promotion" + fi + return "${OCF_SUCCESS}" + fi + fi + sleep 1 + done + ocf_exit_reason "Old primary did not become verified ready recovery within demotion budget" + return "${OCF_ERR_GENERIC}" +) diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/notify.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/notify.sh new file mode 100755 index 0000000..135994e --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/notify.sh @@ -0,0 +1,120 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +mc_pg_demote_notification_target() +{ + local allow_self="${1:-false}" + local state_file + local type operation promote_uname demote_uname local_node + local now mtime age + local promote_norm local_norm demote_norm + + state_file="/run/loxthelion/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" + local_node="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" + + [ -r "${state_file}" ] || return 1 + + now="$(date +%s)" || return 1 + mtime="$(stat -c %Y "${state_file}" 2>/dev/null)" || return 1 + age=$((now - mtime)) + + [ "${age}" -ge 0 ] || return 1 + [ "${age}" -le 120 ] || return 1 + + type="$(sed -n 's/^type=//p' "${state_file}" | head -n1)" + operation="$(sed -n 's/^operation=//p' "${state_file}" | head -n1)" + promote_uname="$(sed -n 's/^promote_uname=//p' "${state_file}" | head -n1)" + demote_uname="$(sed -n 's/^demote_uname=//p' "${state_file}" | head -n1)" + + [ "${type}" = "pre" ] || return 1 + [ "${operation}" = "demote" ] || return 1 + + local_norm="$(printf '%s' "${local_node}" | tr '[:upper:]' '[:lower:]')" + demote_norm="$(printf '%s' "${demote_uname}" | tr '[:upper:]' '[:lower:]')" + promote_norm="$(printf '%s' "${promote_uname}" | tr '[:upper:]' '[:lower:]')" + + [ "${demote_norm}" = "${local_norm}" ] || return 1 + [ -n "${promote_uname}" ] || return 1 + + case "${promote_uname}" in + *" "*) return 1 ;; + esac + + if [ "${allow_self}" != true ] && [ "${promote_norm}" = "${local_norm}" ]; then + return 1 + fi + + printf '%s\n' "${promote_uname}" +} + +# Legacy callers retain the historical requirement for a distinct successor. +demote_target() +{ + mc_pg_demote_notification_target false +} + +# Native-peer demotion can distinguish a true handoff from Pacemaker's +# demote/restart/re-promote of the same node after a resource-definition change. +mc_pg_native_demote_target() +{ + mc_pg_demote_notification_target true +} + +managed_notify() +{ + local state_dir state_file + + state_dir="/run/loxthelion" + state_file="${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" + + ocf_log info \ + "notify: type=${OCF_RESKEY_CRM_meta_notify_type:-unset} operation=${OCF_RESKEY_CRM_meta_notify_operation:-unset}" + + ocf_log info \ + "notify: promote_uname=${OCF_RESKEY_CRM_meta_notify_promote_uname:-none} demote_uname=${OCF_RESKEY_CRM_meta_notify_demote_uname:-none}" + + ocf_log info \ + "notify: promoted_uname=${OCF_RESKEY_CRM_meta_notify_promoted_uname:-none} unpromoted_uname=${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-none}" + + if ! mkdir -p -m 0755 "${state_dir}"; then + ocf_exit_reason \ + "Unable to create notification state directory ${state_dir}" + return "${OCF_ERR_GENERIC}" + fi + + if ! { + printf 'timestamp=%s\n' "$(date --iso-8601=seconds)" + printf 'node=%s\n' "$(hostname)" + printf 'type=%s\n' "${OCF_RESKEY_CRM_meta_notify_type:-}" + printf 'operation=%s\n' "${OCF_RESKEY_CRM_meta_notify_operation:-}" + printf 'promote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" + printf 'demote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" + printf 'promoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" + printf 'unpromoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" + } > "${state_file}" + then + ocf_exit_reason \ + "Unable to write notification state ${state_file}" + return "${OCF_ERR_GENERIC}" + fi + + chmod 0600 "${state_file}" || { + ocf_exit_reason \ + "Unable to secure notification state ${state_file}" + return "${OCF_ERR_GENERIC}" + } + + { + printf '%s node=%s type=%s operation=%s promote=[%s] demote=[%s] promoted=[%s] unpromoted=[%s]\n' \ + "$(date --iso-8601=seconds)" \ + "$(hostname)" \ + "${OCF_RESKEY_CRM_meta_notify_type:-}" \ + "${OCF_RESKEY_CRM_meta_notify_operation:-}" \ + "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" \ + >> "${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify.log" + } || true + + return "${OCF_SUCCESS}" +} diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/peer_config.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/peer_config.sh new file mode 100644 index 0000000..165e326 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/peer_config.sh @@ -0,0 +1,54 @@ +# POSIX-compatible shared normalizer. Also mounted beside the bootstrap script. +# Inputs are data, never evaluated shell. IPv4/DNS endpoints only in API 1. +mc_peer_endpoints() +{ + LC_ALL=C awk -v peers="$1" -v self="$2" -v select_node="${3:-}" ' + BEGIN { + n=split(peers, entries, ","); self=tolower(self); select_node=tolower(select_node); + if(n<2 || n>32 || self=="") exit 1; + for(i=1;i<=n;i++) { + if(split(entries[i],pair,"=")!=2 || pair[1]!~/^[A-Za-z0-9_][A-Za-z0-9_.-]*$/) exit 1; + node=tolower(pair[1]); if(seen[node]++) exit 1; + if(split(pair[2],addr,":")!=2 || addr[1]!~/^[A-Za-z0-9][A-Za-z0-9.-]*$/ || addr[2]!~/^[0-9]+$/ || addr[2]+0<1 || addr[2]+0>65535) exit 1; + endpoint=tolower(addr[1]) ":" (addr[2]+0); if(addresses[endpoint]++) exit 1; + if(node==self) { own++; continue; } + if(select_node!="" && node!=select_node) continue; + hosts=hosts (count?",":"") addr[1]; ports=ports (count?",":"") (addr[2]+0); count++; + } + if(own!=1 || count<1 || (select_node!="" && count!=1)) exit 1; + print hosts "|" ports; + }' +} + +mc_peer_conninfo() +{ + # peers, self, replication user, application name + case "$3" in ''|*[!a-zA-Z0-9_.-]*) return 1;; esac + case "$4" in ''|*[!a-zA-Z0-9_.-]*) return 1;; esac + mc_peer_pair=$(mc_peer_endpoints "$1" "$2") || return 1 + printf 'host=%s port=%s user=%s application_name=%s target_session_attrs=primary connect_timeout=2\n' \ + "${mc_peer_pair%|*}" "${mc_peer_pair#*|}" "$3" "$4" +} + +mc_peer_write_recovery() +( + # PGDATA, peers, self, user, application, slot. No credentials in output. + set -eu + data=$1 + conn=$(mc_peer_conninfo "$2" "$3" "$4" "$5") || exit 1 + case "$6" in ''|*[!a-z0-9_]*) exit 1;; esac + [ ${#6} -le 63 ] || exit 1 + [ -d "$data" ] && [ -f "$data/postgresql.auto.conf" ] && [ ! -L "$data/postgresql.auto.conf" ] || exit 1 + [ ! -L "$data/standby.signal" ] || exit 1 + umask 077 + tmp=$(mktemp "$data/.peer-recovery.XXXXXXXX") + trap 'rm -f "$tmp"' EXIT HUP INT TERM + sed -e '/^[[:space:]]*primary_conninfo[[:space:]]*=/d' \ + -e '/^[[:space:]]*primary_slot_name[[:space:]]*=/d' "$data/postgresql.auto.conf" > "$tmp" + printf "primary_conninfo = '%s'\nprimary_slot_name = '%s'\n" "$conn" "$6" >> "$tmp" + chmod 600 "$tmp" + mv -f "$tmp" "$data/postgresql.auto.conf" + [ ! -L "$data/standby.signal" ] || exit 1 + touch "$data/standby.signal" + chmod 600 "$data/standby.signal" +) diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh new file mode 100755 index 0000000..74e51cb --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/plugin.sh @@ -0,0 +1,391 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +OCF_RESKEY_service_user_default="david" +OCF_RESKEY_service_name_default="postgres.service" +OCF_RESKEY_container_name_default="postgres" +OCF_RESKEY_pg_os_user_default="postgres" +OCF_RESKEY_db_user_default="david" +OCF_RESKEY_db_name_default="david" +OCF_RESKEY_rejoin_source_host_default="" +OCF_RESKEY_rejoin_source_port_default="5432" +OCF_RESKEY_replication_user_default="replicator" +OCF_RESKEY_replication_slot_default="" +OCF_RESKEY_application_name_default="" +OCF_RESKEY_rejoin_allow_basebackup_default="false" +OCF_RESKEY_rejoin_basebackup_headroom_percent_default="50" + +: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}" +: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}" +: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}" +: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}" +: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}" +: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}" +: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}" +: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}" +: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}" +: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}" +: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}" +: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}" +: "${OCF_RESKEY_rejoin_basebackup_headroom_percent:=${OCF_RESKEY_rejoin_basebackup_headroom_percent_default}}" + +SERVICE_USER="${OCF_RESKEY_service_user}" +SERVICE_NAME="${OCF_RESKEY_service_name}" +CONTAINER_NAME="${OCF_RESKEY_container_name}" +PG_OS_USER="${OCF_RESKEY_pg_os_user}" +DB_USER="${OCF_RESKEY_db_user}" +DB_NAME="${OCF_RESKEY_db_name}" +REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}" +REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}" +REPLICATION_USER="${OCF_RESKEY_replication_user}" +REPLICATION_SLOT="${OCF_RESKEY_replication_slot}" +APPLICATION_NAME="${OCF_RESKEY_application_name}" +REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}" +REJOIN_BASEBACKUP_HEADROOM_PERCENT="${OCF_RESKEY_rejoin_basebackup_headroom_percent}" + +# Resolve PostgreSQL configuration from the managed service when Pacemaker has +# not explicitly supplied it. The application configuration remains the source +# of truth; no slot naming convention is inferred. +mc_pg_resolve_replication_slot() +{ + local resolved + + if [ -n "${REPLICATION_SLOT}" ]; then + return 0 + fi + + # Prefer PostgreSQL's effective setting when the server is already up. + # This asks PostgreSQL itself rather than reconstructing configuration. + if declare -F pg_running >/dev/null 2>&1 && pg_running; then + resolved="$( + mc_runtime_exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql -X -U "${DB_USER}" -d "${DB_NAME}" -Atqc \ + 'SHOW primary_slot_name;' 2>/dev/null + )" || resolved= + if [ -n "${resolved}" ]; then + REPLICATION_SLOT=${resolved} + OCF_RESKEY_replication_slot=${resolved} + ocf_log debug "Resolved PostgreSQL replication slot from PostgreSQL effective configuration" + return 0 + fi + fi + + # When PostgreSQL is stopped but its maintenance container is running, + # obtain the exact value injected into the managed container. Runtime + # adapters own this lookup; the application plugin does not parse Quadlets. + if declare -F mc_runtime_environment >/dev/null 2>&1; then + resolved="$( + mc_runtime_environment \ + "${CONTAINER_NAME}" \ + POSTGRES_REPLICATION_SLOT + )" || resolved= + if [ -n "${resolved}" ]; then + REPLICATION_SLOT=${resolved} + OCF_RESKEY_replication_slot=${resolved} + ocf_log debug "Resolved PostgreSQL replication slot from managed container environment" + return 0 + fi + fi + + return 1 +} + +mc_pg_require_replication_slot() +{ + mc_pg_resolve_replication_slot || { + ocf_exit_reason "PostgreSQL replication slot is required but could not be resolved from replication_slot, PostgreSQL, or the managed container environment" + return "${OCF_ERR_CONFIGURED}" + } + + case "${REPLICATION_SLOT}" in + ''|*[!a-z0-9_]*) + ocf_exit_reason "Resolved replication slot is invalid; expected lowercase letters, digits, and underscore" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + if [ ${#REPLICATION_SLOT} -gt 63 ]; then + ocf_exit_reason "Resolved replication slot exceeds PostgreSQL's 63-character identifier limit" + return "${OCF_ERR_CONFIGURED}" + fi + + return 0 +} + +if [ -z "${APPLICATION_NAME}" ]; then + APPLICATION_NAME="$(hostname -s)" +fi + +meta_data() +{ + cat < + + + 1.0 + + + Manage an application inside a persistent rootless container. + + Version 0.4 implements role-aware monitoring, promotion, and demotion. + The systemd user Quadlet owns the container lifecycle while pg_ctl owns + the PostgreSQL lifecycle inside the container. + + + + Manage PostgreSQL independently inside a persistent container + + + + + + + Linux user owning the rootless systemd/Podman service. + + Rootless service owner + + + + + + systemd user service generated by Quadlet. + + Quadlet service name + + + + + + Podman container containing PostgreSQL. + + Container name + + + + + + Operating-system user running PostgreSQL inside the container. + + PostgreSQL OS user + + + + + + PostgreSQL database user used for health queries. + + Database user + + + + + + PostgreSQL database used for health queries. + + Database name + + + + + + + PostgreSQL primary host used as the source for rejoin operations. + + Rejoin source host + + + + + + PostgreSQL port on the rejoin source. + + Rejoin source port + + + + + + PostgreSQL role used for replication and pg_rewind. + + Replication user + + + + + + Physical replication slot used by this standby. If unset, the agent + resolves the effective PostgreSQL primary_slot_name when available, + otherwise POSTGRES_REPLICATION_SLOT from the running managed container. + No slot name is derived or guessed. + + Physical replication slot + + + + + + PostgreSQL replication application_name. Defaults to the local hostname. + + Replication application name + + + + + + Allow rejoin to fall back to pg_basebackup when pg_rewind cannot + repair the local PostgreSQL data directory. The old PGDATA is + preserved before replacement. This must only be enabled when the + operator has established that this node may safely discard its + divergent PostgreSQL history. + + Allow pg_basebackup rejoin fallback + + + + + + Additional free-space headroom, as a percentage of the estimated + source database payload, required before pg_basebackup fallback. + The old PGDATA is renamed in place and therefore is not counted as + another free-space copy. A minimum 1 GiB headroom is always required. + + pg_basebackup free-space headroom percent + + + + + + + + + + + + + + + + +EOF_META +} + +validate_all() +{ + if ! id "${SERVICE_USER}" >/dev/null 2>&1; then + ocf_exit_reason "Service user '${SERVICE_USER}' does not exist" + return "${OCF_ERR_CONFIGURED}" + fi + + for cmd in runuser systemctl podman getent; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + ocf_exit_reason "Required command '${cmd}' is not installed" + return "${OCF_ERR_INSTALLED}" + fi + done + + if [ ! -d "/run/user/$(service_uid)" ]; then + ocf_exit_reason \ + "Runtime directory /run/user/$(service_uid) does not exist" + return "${OCF_ERR_GENERIC}" + fi + + case "${REJOIN_BASEBACKUP_HEADROOM_PERCENT}" in + ""|*[!0-9]*) + ocf_exit_reason "rejoin_basebackup_headroom_percent must be a non-negative integer" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + if [ "${REJOIN_BASEBACKUP_HEADROOM_PERCENT}" -gt 500 ]; then + ocf_exit_reason "rejoin_basebackup_headroom_percent must be between 0 and 500" + return "${OCF_ERR_CONFIGURED}" + fi + + if ! mc_lifecycle_cat "${SERVICE_NAME}" >/dev/null 2>&1 + then + ocf_exit_reason \ + "User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}" + return "${OCF_ERR_CONFIGURED}" + fi + + local podman_error + + podman_error="$( + mc_runtime_info 2>&1 >/dev/null + )" + local podman_rc=$? + + if [ "${podman_rc}" -ne 0 ]; then + ocf_exit_reason \ + "Rootless Podman failed for ${SERVICE_USER}: ${podman_error}" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + + +# Keep the original metadata function intact; advertise the added policy here. +mc_promotion_metadata() +{ + meta_data | sed '/<\/parameters>/i\ + \ + Maximum age in seconds of a verified streaming observation for disconnected promotion eligibility. Default 300; 0 disables disconnected eligibility. Requires fencing managed by Pacemaker. PostgreSQL restart requires fresh streaming verification.\ + Disconnected standby observation lifetime\ + \ + \ + \ + Opt-in stable peer map: node=host:port,node=host:port. Include the local node; exclude the client VIP. IPv4 or DNS hosts only. Requires a configured per-node replication slot, supplied explicitly or resolved from PostgreSQL/the managed container environment, plus peer authentication and slots on every upstream. Uses native primary-only selection; demotion verifies final WAL then ready recovery before successor promotion.\ + Native replication peer map\ + \ + ' +} + +MC_APPLICATION_API=1 +MC_APPLICATION_DIR="${MC_MODULE_DIR}/applications/postgresql" +for MC_PG_MODULE in peer_config native_peers postgres auth replication notify promote demote rejoin; do + if ! . "${MC_APPLICATION_DIR}/${MC_PG_MODULE}.sh"; then + ocf_exit_reason "Unable to load resource-agent module ${MC_APPLICATION_DIR}/${MC_PG_MODULE}.sh" + return "${OCF_ERR_CONFIGURED}" + fi +done +unset MC_PG_MODULE + +mc_application_monitor() +{ + managed_monitor +} + +mc_application_standby_score() +{ + mc_pg_standby_score +} + +mc_application_validate() +{ + mc_pg_peer_validate || return "$?" + validate_all +} + +mc_application_metadata() +{ + mc_promotion_metadata +} + +mc_application_notify() +{ + managed_notify +} + +mc_application_action() +{ + mc_pg_peer_validate || return "$?" + if [ "$1" = demote ] && [ -n "${OCF_RESKEY_replication_peers:-}" ]; then + mc_pg_native_demote + return "$?" + fi + case "$1" in + start|stop|promote|demote|rejoin) "managed_$1" ;; + *) return "${OCF_ERR_UNIMPLEMENTED}" ;; + esac +} diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/postgres.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/postgres.sh new file mode 100755 index 0000000..b92a7c9 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/postgres.sh @@ -0,0 +1,696 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +pg_command() +{ + local command="$1" + + mc_runtime_exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + sh -c "${command}" +} + +pgdata_path() +{ + pg_command ' + if [ -z "$PGDATA" ]; then + exit 1 + fi + + printf "%s\n" "$PGDATA" + ' +} + +pg_running() +{ + pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + "$BINDIR/pg_ctl" status -D "$PGDATA" >/dev/null 2>&1 + ' +} + +pg_ready() +{ + mc_runtime_exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + pg_isready -q \ + -U "${DB_USER}" \ + -d "${DB_NAME}" +} + +pg_role() +{ + mc_runtime_exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql \ + -U "${DB_USER}" \ + -d "${DB_NAME}" \ + -Atqc 'SELECT pg_is_in_recovery();' +} + +pg_wal_receiver_status() +{ + mc_runtime_exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql \ + -U "${DB_USER}" \ + -d "${DB_NAME}" \ + -Atqc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;" +} + +pg_streaming() +{ + [ "$(pg_wal_receiver_status 2>/dev/null || true)" = "streaming" ] +} + +pg_wal_receiver_sender() +{ + mc_runtime_exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql \ + -U "${DB_USER}" \ + -d "${DB_NAME}" \ + -Atqc " + SELECT sender_host || '|' || sender_port + FROM pg_stat_wal_receiver + WHERE status = 'streaming' + LIMIT 1; + " +} + +mc_pg_rejoin_running_standby() +{ + local primary_info primary_rc primary_node primary_host primary_port + + [ -n "${OCF_RESKEY_replication_peers:-}" ] || return 1 + + primary_info="$(mc_pg_find_remote_primary 2>/dev/null)" + primary_rc=$? + + case "${primary_rc}" in + 0) + primary_node="${primary_info%%|*}" + primary_info="${primary_info#*|}" + primary_host="${primary_info%%|*}" + primary_port="${primary_info#*|}" + + ocf_log warn \ + "Standby is not streaming while remote primary ${primary_node} (${primary_host}:${primary_port}) is available; forcing managed rejoin" + + if pg_running; then + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason \ + "Unable to stop non-streaming standby before managed rejoin" + return "${OCF_ERR_GENERIC}" + fi + fi + + mc_pg_require_replication_slot || return "$?" + + REJOIN_SOURCE_HOST="${primary_host}" + REJOIN_SOURCE_PORT="${primary_port}" + OCF_RESKEY_rejoin_source_host="${primary_host}" + OCF_RESKEY_rejoin_source_port="${primary_port}" + + if ! managed_rejoin; then + ocf_exit_reason \ + "Managed rejoin of non-streaming standby against ${primary_node} failed" + return "${OCF_ERR_GENERIC}" + fi + + pg_clear_rejoin_pending >/dev/null 2>&1 || true + return "${OCF_SUCCESS}" + ;; + + 1) + # No writable peer exists. This is a normal failover-election + # condition; leave the standby running Unpromoted and let the + # promotion scorer decide whether it is eligible. + return 1 + ;; + + 2) + ocf_exit_reason \ + "Multiple writable PostgreSQL peers detected while repairing non-streaming standby" + return "${OCF_ERR_GENERIC}" + ;; + + *) + ocf_exit_reason \ + "Unable to determine PostgreSQL peer-primary state while repairing non-streaming standby" + return "${OCF_ERR_GENERIC}" + ;; + esac +} + + +pg_rejoin_pending() +{ + pg_command 'test -f "$PGDATA/.loxthelion-ha-rejoin-required"' +} + +pg_mark_rejoin_pending() +{ + pg_command ' + umask 077 + : > "$PGDATA/.loxthelion-ha-rejoin-required" + chmod 600 "$PGDATA/.loxthelion-ha-rejoin-required" + ' +} + +pg_clear_rejoin_pending() +{ + pg_command 'rm -f "$PGDATA/.loxthelion-ha-rejoin-required"' +} + +mc_pg_reconcile_rejoin_marker() +{ + local primary_info primary_rc primary_node primary_host primary_port + local receiver_sender + + [ -n "${OCF_RESKEY_replication_peers:-}" ] || return 1 + + primary_info="$(mc_pg_find_remote_primary 2>/dev/null)" + primary_rc=$? + + [ "${primary_rc}" -eq 0 ] || return 1 + + primary_node="${primary_info%%|*}" + primary_info="${primary_info#*|}" + primary_host="${primary_info%%|*}" + primary_port="${primary_info#*|}" + + receiver_sender="$(pg_wal_receiver_sender 2>/dev/null || true)" + + if ! pg_streaming; then + return 1 + fi + + if [ "${receiver_sender}" != "${primary_host}|${primary_port}" ]; then + return 1 + fi + + if ! pg_clear_rejoin_pending; then + ocf_log warn \ + "Standby is streaming from elected primary ${primary_node}, but rejoin-required marker could not be cleared" + return 1 + fi + + ocf_log info \ + "Standby is verified streaming from elected primary ${primary_node} (${primary_host}:${primary_port}); cleared provisional rejoin marker" + + return 0 +} + +pg_has_standby_signal() +{ + pg_command 'test -f "$PGDATA/standby.signal"' +} + +managed_monitor() +{ + local role + + if ! service_running; then + ocf_log debug "User service ${SERVICE_NAME} is not active" + return "${OCF_NOT_RUNNING}" + fi + + if ! container_running; then + ocf_log warn \ + "Service ${SERVICE_NAME} is active but container ${CONTAINER_NAME} is not running" + return "${OCF_NOT_RUNNING}" + fi + + if ! pg_running; then + ocf_log debug \ + "Container ${CONTAINER_NAME} is running but PostgreSQL is stopped" + return "${OCF_NOT_RUNNING}" + fi + + if ! pg_ready; then + ocf_exit_reason \ + "PostgreSQL process exists but pg_isready failed" + return "${OCF_ERR_GENERIC}" + fi + + role="$(pg_role 2>/dev/null)" || { + ocf_exit_reason \ + "PostgreSQL is accepting connections but role query failed" + return "${OCF_ERR_GENERIC}" + } + + case "${role}" in + f) + ocf_log debug \ + "PostgreSQL healthy: role=primary (promoted)" + return "${OCF_RUNNING_PROMOTED}" + ;; + t) + # A former primary may be started provisionally in recovery when + # no writable peer exists yet, solely so Pacemaker can decide + # which Unpromoted instance to promote. Once another node becomes + # primary, force recovery of this provisional instance through a + # stop/start cycle so start() can pg_rewind it against the winner. + if pg_rejoin_pending && [ -n "${OCF_RESKEY_replication_peers:-}" ]; then + local primary_info primary_rc + primary_info="$(mc_pg_find_remote_primary 2>/dev/null)" + primary_rc=$? + + case "${primary_rc}" in + 0) + ocf_exit_reason \ + "A remote primary is now available (${primary_info%%|*}); provisional former-primary standby requires rejoin" + return "${OCF_ERR_GENERIC}" + ;; + 2) + ocf_exit_reason \ + "Multiple writable PostgreSQL peers detected while monitoring provisional standby" + return "${OCF_ERR_GENERIC}" + ;; + esac + fi + + if pg_streaming; then + ocf_log debug \ + "PostgreSQL healthy: role=standby (unpromoted), WAL receiver=streaming" + return "${OCF_SUCCESS}" + fi + + # A standby can legitimately lose its receiver while the cluster + # has no writable primary (for example during failover election). + # Do not declare that condition unhealthy: promotion eligibility + # is decided separately by the promotion-scoring evidence. If a + # writable primary already exists, however, a non-streaming + # standby is broken and must be repaired rather than reported OK. + if [ -n "${OCF_RESKEY_replication_peers:-}" ]; then + local primary_info primary_rc + primary_info="$(mc_pg_find_remote_primary 2>/dev/null)" + primary_rc=$? + + case "${primary_rc}" in + 0) + ocf_exit_reason \ + "PostgreSQL standby is in recovery but WAL receiver is not streaming while remote primary ${primary_info%%|*} is available" + return "${OCF_ERR_GENERIC}" + ;; + 1) + ocf_log debug \ + "PostgreSQL standby is disconnected and no remote primary is currently identifiable; leaving promotion eligibility to Pacemaker scoring" + return "${OCF_SUCCESS}" + ;; + 2) + ocf_exit_reason \ + "Multiple writable PostgreSQL peers detected while monitoring standby" + return "${OCF_ERR_GENERIC}" + ;; + *) + ocf_exit_reason \ + "Unable to determine PostgreSQL peer-primary state while monitoring standby" + return "${OCF_ERR_GENERIC}" + ;; + esac + fi + + # Backward-compatible behavior when native peer discovery is not + # configured: role-aware monitoring cannot prove an upstream. + ocf_log debug \ + "PostgreSQL healthy: role=standby (unpromoted); native peer discovery is not configured" + return "${OCF_SUCCESS}" + ;; + *) + ocf_exit_reason \ + "Unexpected pg_is_in_recovery() result: '${role}'" + return "${OCF_ERR_GENERIC}" + ;; + esac + +} + +managed_start() +{ + local i monitor_rc + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL resource already running unpromoted" + return "${OCF_SUCCESS}" + ;; + + "${OCF_RUNNING_PROMOTED}") + ocf_exit_reason \ + "Refusing start: PostgreSQL is already primary; promotion must occur through promote()" + return "${OCF_ERR_GENERIC}" + ;; + + "${OCF_NOT_RUNNING}") + # Expected for a stopped resource. Continue. + ;; + + *) + # If start() is called for an already-running standby that is in + # recovery but cannot stream from an existing primary, repair it + # instead of simply returning the monitor failure. This catches a + # previous partial rejoin whose standby.signal survived restart. + if service_running && container_running && pg_running; then + local existing_role + existing_role="$(pg_role 2>/dev/null || true)" + if [ "${existing_role}" = "t" ] && ! pg_streaming; then + if mc_pg_rejoin_running_standby; then + managed_monitor + return $? + fi + fi + fi + + ocf_exit_reason \ + "Cannot start unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + ocf_log info \ + "Ensuring container service ${SERVICE_NAME} is running" + + if ! service_running; then + if ! mc_lifecycle_start "${SERVICE_NAME}" + then + ocf_exit_reason \ + "Failed to start ${SERVICE_NAME}" + return "${OCF_ERR_GENERIC}" + fi + fi + + for i in $(seq 1 20); do + container_running && break + sleep 0.5 + done + + if ! container_running; then + ocf_exit_reason \ + "Container ${CONTAINER_NAME} did not become running" + return "${OCF_ERR_GENERIC}" + fi + + if ! pg_running; then + # + # OCF promotable-resource rule: + # start() must produce an Unpromoted instance. Missing standby.signal + # therefore never authorizes a primary start. It means either: + # * a former primary can be rewound against an existing winner, or + # * no winner exists yet, in which case place this PGDATA into + # recovery and let Pacemaker decide whether to promote it. + # + if pg_rejoin_pending || ! pg_has_standby_signal; then + local primary_info primary_rc primary_node primary_host primary_port + + if [ -z "${OCF_RESKEY_replication_peers:-}" ]; then + ocf_exit_reason \ + "Cannot safely recover PGDATA without standby.signal: replication_peers is required for automatic primary discovery" + return "${OCF_ERR_CONFIGURED}" + fi + + # At this point the maintenance container is running, so resolve + # the exact configured slot from PostgreSQL/container state before + # any rewind or recovery configuration is written. + mc_pg_require_replication_slot || return "$?" + + primary_info="$(mc_pg_find_remote_primary 2>/dev/null)" + primary_rc=$? + + case "${primary_rc}" in + 0) + primary_node="${primary_info%%|*}" + primary_info="${primary_info#*|}" + primary_host="${primary_info%%|*}" + primary_port="${primary_info#*|}" + + ocf_log info \ + "Existing remote primary ${primary_node} (${primary_host}:${primary_port}) found; rejoining local PGDATA before start" + + REJOIN_SOURCE_HOST="${primary_host}" + REJOIN_SOURCE_PORT="${primary_port}" + OCF_RESKEY_rejoin_source_host="${primary_host}" + OCF_RESKEY_rejoin_source_port="${primary_port}" + + if ! managed_rejoin; then + ocf_exit_reason \ + "Automatic rejoin against ${primary_node} failed" + return "${OCF_ERR_GENERIC}" + fi + + pg_clear_rejoin_pending >/dev/null 2>&1 || true + managed_monitor + return $? + ;; + + 1) + ocf_log info \ + "No remote primary is currently identifiable; preparing local PGDATA as Unpromoted and leaving promotion exclusively to Pacemaker" + + if ! mc_pg_write_peer_recovery; then + ocf_exit_reason \ + "Unable to place PGDATA into native standby recovery while no primary exists" + return "${OCF_ERR_GENERIC}" + fi + + if ! pg_mark_rejoin_pending; then + ocf_exit_reason \ + "Unable to mark provisional standby as requiring rejoin if another node wins promotion" + return "${OCF_ERR_GENERIC}" + fi + ;; + + 2) + ocf_exit_reason \ + "Multiple writable PostgreSQL peers detected; refusing start" + return "${OCF_ERR_GENERIC}" + ;; + + *) + ocf_exit_reason \ + "Unable to determine PostgreSQL peer-primary state" + return "${OCF_ERR_GENERIC}" + ;; + esac + fi + + ocf_log info \ + "Starting PostgreSQL as an unpromoted standby inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason "pg_ctl start failed" + return "${OCF_ERR_GENERIC}" + fi + fi + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL start completed in unpromoted state" + return "${OCF_SUCCESS}" + ;; + + "${OCF_RUNNING_PROMOTED}") + ocf_exit_reason \ + "PostgreSQL became primary during start; stopping it to preserve OCF role semantics" + + pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' >/dev/null 2>&1 || true + + return "${OCF_ERR_GENERIC}" + ;; + + *) + # standby.signal can exist while the standby is still unusable + # (for example, its required WAL was removed after a failed + # primary repair). If a writable peer exists, route the instance + # through managed_rejoin(), whose guarded basebackup fallback can + # rebuild PGDATA when rewind alone cannot establish streaming. + if pg_running; then + local final_role + final_role="$(pg_role 2>/dev/null || true)" + if [ "${final_role}" = "t" ] && ! pg_streaming; then + if mc_pg_rejoin_running_standby; then + managed_monitor + return $? + fi + fi + fi + + return "${monitor_rc}" + ;; + esac +} + +managed_stop() +{ + local i + + if container_running && pg_running; then + ocf_log info \ + "Stopping PostgreSQL cleanly inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason "pg_ctl stop failed" + return "${OCF_ERR_GENERIC}" + fi + fi + + if service_running; then + ocf_log info \ + "Stopping container service ${SERVICE_NAME}" + + if ! mc_lifecycle_stop "${SERVICE_NAME}" + then + ocf_exit_reason \ + "Failed to stop ${SERVICE_NAME}" + return "${OCF_ERR_GENERIC}" + fi + fi + + for i in $(seq 1 20); do + if ! service_running && ! container_running; then + return "${OCF_SUCCESS}" + fi + sleep 0.5 + done + + if service_running; then + ocf_exit_reason \ + "${SERVICE_NAME} remained active after stop" + return "${OCF_ERR_GENERIC}" + fi + + if container_running; then + ocf_exit_reason \ + "${CONTAINER_NAME} remained running after stop" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + +# Conservative scoring eligibility only; this does not replace role detection, +# fencing, the promotion action, or the existing controlled-demote WAL checks. +# This strict predicate remains unchanged; disconnected policy is separate below. +mc_pg_promotion_eligible() +{ + local eligible + + # A provisional former primary must never become promotion-eligible. + # It may run in recovery so Pacemaker can observe it, but it must first + # complete a real rejoin/rewind against the elected primary. + if pg_rejoin_pending; then + return 1 + fi + + eligible="$( + mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" \ + --env 'PGOPTIONS=-c statement_timeout=2000' \ + "${CONTAINER_NAME}" \ + psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc " + SELECT CASE WHEN pg_is_in_recovery() THEN + NOT pg_is_wal_replay_paused() + AND EXISTS ( + SELECT 1 FROM pg_stat_wal_receiver + WHERE status = 'streaming' + AND flushed_lsn IS NOT NULL + AND pg_last_wal_replay_lsn() >= flushed_lsn + ) + ELSE false END; + " 2>/dev/null + )" || return 1 + [ "${eligible}" = t ] +} + +# One observation supplies both strict connected eligibility and disconnected +# evidence. PostgreSQL restart time deliberately invalidates prior observations, +# including a new basebackup sharing the same PostgreSQL system identifier. +mc_pg_standby_observation() +{ + mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" \ + --env 'PGOPTIONS=-c statement_timeout=2000' \ + "${CONTAINER_NAME}" \ + psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc " + WITH s AS MATERIALIZED ( + SELECT pg_is_in_recovery() AS recovery, + pg_last_wal_replay_lsn() AS replay, + r.status AS receiver, r.flushed_lsn AS flush + FROM (SELECT 1) AS singleton + LEFT JOIN pg_stat_wal_receiver r ON true + ) + SELECT json_build_object( + 'identity', json_build_array( + (SELECT system_identifier::text FROM pg_control_system()), + current_setting('data_directory'), + extract(epoch FROM pg_postmaster_start_time())::text), + 'recovery', recovery, + 'paused', CASE WHEN recovery THEN pg_is_wal_replay_paused() ELSE true END, + 'receiver', receiver, 'replay', replay::text, 'flush', flush::text, + 'strict', CASE WHEN recovery THEN + NOT pg_is_wal_replay_paused() AND receiver = 'streaming' + AND flush IS NOT NULL AND replay >= flush + ELSE false END) + FROM s; + " 2>/dev/null +} + +mc_pg_standby_score() +{ + local observation + + # Hard promotion veto for provisional former-primary PGDATA. Returning + # -INFINITY here ensures Pacemaker cannot choose this instance while the + # rejoin-required marker exists, even though monitor may report it as a + # healthy Unpromoted standby. + if pg_rejoin_pending; then + if ! mc_pg_reconcile_rejoin_marker; then + printf '%s\n' '-INFINITY' + return 0 + fi + fi + + observation="$(mc_pg_standby_observation)" || return 1 + # The helper is root-owned alongside the modules; state never enters the + # rootless container. A failed/slow helper always withdraws eligibility. + printf '%s' "${observation}" | timeout --kill-after=1s 1s python3 \ + "${MC_APPLICATION_DIR}/promotion_state.py" \ + "${MC_PROMOTION_RESOURCE_INSTANCE:-}" \ + "${SERVICE_USER}" "${SERVICE_NAME}" "${CONTAINER_NAME}" \ + "${OCF_RESKEY_disconnected_max_age:-300}" +} diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh new file mode 100644 index 0000000..0068a6f --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/promote.sh @@ -0,0 +1,110 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +managed_promote() +{ + local monitor_rc role i + + # Belt-and-suspenders safety: promotion scoring already returns + # -INFINITY while this marker exists, but refuse the action itself too. + # This protects against an administrative/manual promotion request or a + # future scoring regression. + if pg_rejoin_pending; then + ocf_exit_reason \ + "Cannot promote PostgreSQL while rejoin-required is set; instance must rejoin the elected primary first" + return "${OCF_ERR_GENERIC}" + fi + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") + ocf_log info \ + "PostgreSQL is already primary; reconciling primary invariants" + + if ! ensure_local_ha_slots; then + ocf_exit_reason \ + "HA slot reconciliation failed on existing primary" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_rewind_privileges; then + ocf_exit_reason \ + "pg_rewind privilege reconciliation failed on existing primary" + return "${OCF_ERR_GENERIC}" + fi + + pg_clear_rejoin_pending >/dev/null 2>&1 || true + return "${OCF_SUCCESS}" + ;; + + "${OCF_SUCCESS}") + # Healthy standby: continue with promotion. + ;; + + "${OCF_NOT_RUNNING}") + ocf_exit_reason \ + "Cannot promote PostgreSQL because the resource is not running" + return "${OCF_ERR_GENERIC}" + ;; + + *) + ocf_exit_reason \ + "Cannot promote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + ocf_log info \ + "Promoting PostgreSQL standby inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + promote -w + ' + then + ocf_exit_reason "pg_ctl promote failed" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null)" || { + sleep 1 + continue + } + + if [ "${role}" = "f" ]; then + ocf_log info \ + "PostgreSQL became primary; reconciling primary invariants" + + if ! ensure_local_ha_slots; then + ocf_exit_reason \ + "PostgreSQL promoted but HA slot reconciliation failed" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_rewind_privileges; then + ocf_exit_reason \ + "PostgreSQL promoted but pg_rewind privilege reconciliation failed" + return "${OCF_ERR_GENERIC}" + fi + + if ! pg_clear_rejoin_pending; then + ocf_log warn \ + "PostgreSQL promoted successfully but provisional rejoin marker could not be cleared" + fi + + ocf_log info \ + "PostgreSQL promotion completed successfully" + return "${OCF_SUCCESS}" + fi + + sleep 1 + done + + ocf_exit_reason \ + "PostgreSQL did not become primary after promotion" + return "${OCF_ERR_GENERIC}" +} diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/promotion_state.py b/ocf/loxthelion/managed-container.d/applications/postgresql/promotion_state.py new file mode 100644 index 0000000..f9c8113 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/promotion_state.py @@ -0,0 +1,173 @@ +"""Local promotion evidence. Invoked by the root RA; never invokes cluster tools. + +State is a bounded JSON record, never executable input. The public CLI uses a +fixed root-owned /run directory and CLOCK_BOOTTIME, including suspend duration. +""" +import fcntl +import hashlib +import json +import math +import os +from pathlib import Path +import re +import stat +import sys +import tempfile +import time + +STATE_DIR = Path('/run/managed-container/promotion') + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate JSON key') + result[key] = value + return result + + +def lsn(value): + if not isinstance(value, str) or not re.fullmatch(r'[0-9A-F]{1,8}/[0-9A-F]{1,8}', value): + raise ValueError('invalid LSN') + high, low = value.split('/') + return (int(high, 16) << 32) | int(low, 16) + + +def number(value): + return type(value) in (int, float) and math.isfinite(value) and value >= 0 + + +def validate_observation(obs): + if set(obs) != {'identity', 'recovery', 'paused', 'receiver', 'replay', 'flush', 'strict'}: + raise ValueError('invalid observation fields') + identity = obs['identity'] + if not isinstance(identity, list) or len(identity) != 3: + raise ValueError('invalid identity') + system, path, started = identity + if (not isinstance(system, str) or not re.fullmatch(r'[0-9]{1,20}', system) + or not isinstance(path, str) or not path.startswith('/') or len(path) > 2048 + or not isinstance(started, str) or not re.fullmatch(r'[0-9]{1,12}\.[0-9]{1,6}', started)): + raise ValueError('invalid identity values') + if type(obs['recovery']) is not bool or type(obs['paused']) is not bool: + raise ValueError('invalid role') + if obs['strict'] is not None and type(obs['strict']) is not bool: + raise ValueError('invalid strict flag') + if obs['receiver'] is not None and obs['receiver'] not in ( + 'starting', 'streaming', 'waiting', 'restarting', 'stopping'): + raise ValueError('invalid receiver status') + replay = lsn(obs['replay']) if obs['replay'] is not None else None + flush = lsn(obs['flush']) if obs['flush'] is not None else None + strict = (obs['recovery'] and not obs['paused'] and obs['receiver'] == 'streaming' + and replay is not None and flush is not None and replay >= flush) + if (obs['strict'] is True) != strict: + raise ValueError('inconsistent strict observation') + return strict + + +class Evidence: + def __init__(self, directory, context, boot, now, wall, owner=0): + self.directory, self.context = Path(directory), context + self.boot, self.now, self.wall, self.owner = boot, now, wall, owner + key = hashlib.sha256(json.dumps(context, separators=(',', ':')).encode()).hexdigest() + self.path = self.directory / (key + '.json') + self.lock = self.directory / (key + '.lock') + + def secure(self, path, mode): + info = path.lstat() + if (info.st_uid != self.owner or stat.S_IMODE(info.st_mode) != mode + or (not stat.S_ISREG(info.st_mode) if mode == 0o600 else not stat.S_ISDIR(info.st_mode))): + raise ValueError('unsafe state ownership/type/mode') + + def discard(self): + self.path.unlink(missing_ok=True) + + def evaluate(self, obs, max_age): + if type(max_age) is not int or not 0 <= max_age <= 99999: + raise ValueError('invalid freshness threshold') + self.directory.mkdir(mode=0o700, exist_ok=True) + self.secure(self.directory, 0o700) + fd = os.open(self.lock, os.O_CREAT | os.O_NOFOLLOW | os.O_RDWR, 0o600) + with os.fdopen(fd, 'r+') as lock: + self.secure(self.lock, 0o600) + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + return self._evaluate(obs, max_age) + except (ValueError, TypeError, KeyError, OverflowError): + self.discard() + return '-INFINITY' + + def _evaluate(self, obs, max_age): + strict = validate_observation(obs) + if strict: + record = dict(version=1, context=self.context, boot=self.boot, + identity=obs['identity'], observed_boottime=self.now, + observed_unix=self.wall, replay=obs['replay'], flush=obs['flush']) + # Same-directory atomic replace; failed persistence cannot authorize + # subsequent disconnected promotion. No secrets are recorded. + fd, temporary = tempfile.mkstemp(prefix='.observation-', dir=self.directory) + try: + with os.fdopen(fd, 'w') as out: + json.dump(record, out, separators=(',', ':')) + out.flush() + os.fsync(out.fileno()) + os.replace(temporary, self.path) + finally: + Path(temporary).unlink(missing_ok=True) + return '100' + if (not obs['recovery'] or obs['paused'] or obs['receiver'] == 'streaming' + or obs['replay'] is None or max_age == 0): + self.discard() + return '-INFINITY' + if not self.path.exists(): + return '-INFINITY' + self.secure(self.path, 0o600) + if self.path.stat().st_size > 4096: + raise ValueError('oversized state') + with self.path.open() as source: + record = json.load(source, object_pairs_hook=unique_object) + if set(record) != {'version', 'context', 'boot', 'identity', 'observed_boottime', + 'observed_unix', 'replay', 'flush'}: + raise ValueError('invalid record fields') + if (type(record['version']) is not int or record['version'] != 1 + or record['context'] != self.context or record['boot'] != self.boot + or record['identity'] != obs['identity'] + or not number(record['observed_boottime']) or not number(record['observed_unix']) + or not 0 <= self.now - record['observed_boottime'] <= max_age + or lsn(record['replay']) < lsn(record['flush']) + or lsn(obs['replay']) < lsn(record['replay'])): + raise ValueError('expired or incompatible observation') + return '50' + + +def main(): + if os.geteuid() != 0 or len(sys.argv) != 6 or not all(sys.argv[1:5]): + raise ValueError('root and explicit resource context required') + age = sys.argv[5] + if not re.fullmatch(r'(0|[1-9][0-9]{0,4})', age): + raise ValueError('invalid freshness threshold') + # /run is trusted root-owned; validate our parent before using a child. + parent = STATE_DIR.parent + parent.mkdir(mode=0o700, exist_ok=True) + info = parent.lstat() + if info.st_uid != 0 or not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700: + raise ValueError('unsafe state parent') + raw = sys.stdin.read(8193) + if len(raw) > 8192: + raise ValueError('oversized observation') + evidence = Evidence(STATE_DIR, sys.argv[1:5], + Path('/proc/sys/kernel/random/boot_id').read_text().strip(), + time.clock_gettime(time.CLOCK_BOOTTIME), time.time()) + try: + obs = json.loads(raw, object_pairs_hook=unique_object) + except ValueError: + obs = {} # Invalid query output withdraws and removes prior evidence. + print(evidence.evaluate(obs, int(age))) + + +if __name__ == '__main__': + try: + main() + except (OSError, ValueError, TypeError, KeyError, OverflowError): + print('-INFINITY') + sys.exit(1) diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/rejoin.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/rejoin.sh new file mode 100755 index 0000000..917a055 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/rejoin.sh @@ -0,0 +1,391 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +mc_pg_rejoin_basebackup_enabled() +{ + case "${REJOIN_ALLOW_BASEBACKUP}" in + true|1|yes) + return 0 + ;; + *) + return 1 + ;; + esac +} + +mc_pg_rejoin_basebackup_fallback() +{ + local pgdata="$1" + local reason="${2:-rejoin fallback requested}" + local source_bytes free_bytes required_bytes headroom_bytes min_headroom_bytes + local timestamp preserved_pgdata failed_pgdata source_role_retry + + if ! mc_pg_rejoin_basebackup_enabled; then + ocf_exit_reason \ + "${reason}; rejoin_allow_basebackup is not enabled" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log warn \ + "${reason}; evaluating pg_basebackup fallback" + + # Revalidate the source immediately before replacing local PGDATA. + source_role_retry="$(pg_remote_query \ + 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to revalidate rejoin source before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + if [ "${source_role_retry}" != "f" ]; then + ocf_exit_reason \ + "Rejoin source is no longer a writable primary; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL is still active; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if ! service_running || ! container_running; then + ocf_exit_reason \ + "Maintenance service/container is unavailable; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_remote_slot \ + "${REJOIN_SOURCE_HOST}" \ + "${REPLICATION_SLOT}" + then + ocf_exit_reason \ + "Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + # Estimate the replacement payload. Existing PGDATA is renamed in place on + # the same filesystem, so it consumes no additional free space. Require one + # replacement copy plus configurable safety headroom. + source_bytes="$(pg_remote_query \ + "SELECT COALESCE(sum(pg_database_size(datname)),0) + FROM pg_database + WHERE datallowconn;" 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to estimate source database size before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + case "${source_bytes}" in + ""|*[!0-9]*) + ocf_exit_reason \ + "Invalid source database size returned before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + ;; + esac + + free_bytes="$( + pg_command ' + blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")" + case "$blocks" in + ""|*[!0-9]*) exit 1 ;; + esac + printf "%s\\n" "$((blocks * 1024))" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to determine free space for pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + case "${free_bytes}" in + ""|*[!0-9]*) + ocf_exit_reason \ + "Invalid free-space value before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + ;; + esac + + min_headroom_bytes=$((1024 * 1024 * 1024)) + headroom_bytes=$((source_bytes * REJOIN_BASEBACKUP_HEADROOM_PERCENT / 100)) + if [ "${headroom_bytes}" -lt "${min_headroom_bytes}" ]; then + headroom_bytes=${min_headroom_bytes} + fi + required_bytes=$((source_bytes + headroom_bytes)) + + ocf_log info \ + "pg_basebackup space check: source-estimate=${source_bytes} headroom=${headroom_bytes} required=${required_bytes} free=${free_bytes}" + + if [ "${free_bytes}" -lt "${required_bytes}" ]; then + ocf_exit_reason \ + "Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes} source-estimate=${source_bytes} headroom=${headroom_bytes}" + return "${OCF_ERR_GENERIC}" + fi + + timestamp="$(date -u +%Y%m%d-%H%M%S)" + preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}" + failed_pgdata="${pgdata}.failed-basebackup-${timestamp}" + + ocf_log warn \ + "Preserving existing PGDATA as ${preserved_pgdata}" + + if ! pg_command " + set -e + [ -d '${pgdata}' ] + [ ! -e '${preserved_pgdata}' ] + [ ! -e '${failed_pgdata}' ] + mv '${pgdata}' '${preserved_pgdata}' + install -d -m 0700 '${pgdata}' + " + then + ocf_exit_reason \ + "Unable to preserve existing PGDATA for pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log warn \ + "Running pg_basebackup from ${REJOIN_SOURCE_HOST}" + + if ! pg_replication_exec \ + pg_basebackup \ + --dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \ + --pgdata="${pgdata}" \ + --format=plain \ + --wal-method=stream \ + --slot="${REPLICATION_SLOT}" \ + --write-recovery-conf \ + --progress + then + ocf_log err \ + "pg_basebackup failed; preserving failed copy and restoring previous PGDATA" + + if ! pg_command " + set -e + [ ! -e '${failed_pgdata}' ] + if [ -e '${pgdata}' ]; then + mv '${pgdata}' '${failed_pgdata}' + fi + mv '${preserved_pgdata}' '${pgdata}' + " + then + ocf_exit_reason \ + "pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required" + return "${OCF_ERR_GENERIC}" + fi + + ocf_exit_reason \ + "pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}" + return "${OCF_SUCCESS}" +} + +mc_pg_rejoin_configure_standby() +{ + if [ -n "${OCF_RESKEY_replication_peers:-}" ]; then + mc_pg_write_peer_recovery || { + ocf_exit_reason "Failed to configure native peer recovery" + return "${OCF_ERR_GENERIC}" + } + return "${OCF_SUCCESS}" + fi + + if ! mc_runtime_exec \ + --user "${PG_OS_USER}" \ + --env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \ + --env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \ + --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ + --env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \ + --env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \ + "${CONTAINER_NAME}" \ + sh -c ' + set -e + sed -i \\ + -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \\ + -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \\ + "$PGDATA/postgresql.auto.conf" + printf "%s\\n" \\ + "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \\ + "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \\ + >> "$PGDATA/postgresql.auto.conf" + touch "$PGDATA/standby.signal" + chmod 600 "$PGDATA/postgresql.auto.conf" "$PGDATA/standby.signal" + ' + then + ocf_exit_reason "Failed to configure standby settings" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + +mc_pg_rejoin_start_and_wait_streaming() +{ + local role receiver_status i + + ocf_log info "Starting PostgreSQL as standby" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason "PostgreSQL failed to start after rejoin" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null)" || { + sleep 1 + continue + } + receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" + + if [ "${role}" = "t" ] && [ "${receiver_status}" = "streaming" ]; then + return "${OCF_SUCCESS}" + fi + sleep 1 + done + + return "${OCF_ERR_GENERIC}" +} + +managed_rejoin() +{ + local source_role pgdata + local fallback_used=false + + case "${REJOIN_ALLOW_BASEBACKUP}" in + true|false|1|0|yes|no) ;; + *) + ocf_exit_reason "rejoin_allow_basebackup must be true or false" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + if [ -z "${REJOIN_SOURCE_HOST}" ]; then + ocf_exit_reason "rejoin_source_host is required for rejoin" + return "${OCF_ERR_CONFIGURED}" + fi + + if ! service_running; then + ocf_exit_reason "Maintenance service ${SERVICE_NAME} must be running before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + if ! container_running; then + ocf_exit_reason "Maintenance container ${CONTAINER_NAME} must be running before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + mc_pg_require_replication_slot || return "$?" + + if pg_running; then + ocf_exit_reason "PostgreSQL must be stopped before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info "Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}" + source_role="$(pg_remote_query 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { + ocf_exit_reason "Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}" + return "${OCF_ERR_GENERIC}" + } + + if [ "${source_role}" != "f" ]; then + ocf_exit_reason "Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info "Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" + if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}"; then + ocf_exit_reason "Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" + return "${OCF_ERR_GENERIC}" + fi + + pgdata="$(pgdata_path 2>/dev/null)" || { + ocf_exit_reason "Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}" + return "${OCF_ERR_GENERIC}" + } + if [ -z "${pgdata}" ]; then + ocf_exit_reason "PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info "Running pg_rewind against ${REJOIN_SOURCE_HOST}" + if ! pg_replication_exec \ + pg_rewind \ + --target-pgdata="${pgdata}" \ + --source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ + --progress + then + mc_pg_rejoin_basebackup_fallback "${pgdata}" "pg_rewind failed" || return "$?" + fallback_used=true + fi + + ocf_log info "Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}" + mc_pg_rejoin_configure_standby || return "$?" + + if mc_pg_rejoin_start_and_wait_streaming; then + pg_clear_rejoin_pending >/dev/null 2>&1 || true + ocf_log info "PostgreSQL successfully rejoined as streaming standby" + return "${OCF_SUCCESS}" + fi + + # A successful pg_rewind does not guarantee that the WAL required to join + # the new timeline is still available. If the standby cannot reach + # streaming, stop it and perform the same guarded full-resync fallback. + if [ "${fallback_used}" = "true" ]; then + ocf_exit_reason \ + "PostgreSQL did not become a streaming standby even after pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if ! mc_pg_rejoin_basebackup_enabled; then + ocf_exit_reason \ + "PostgreSQL started in recovery but did not become streaming; rejoin_allow_basebackup is not enabled" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log warn \ + "PostgreSQL entered recovery but did not reach streaming after pg_rewind; stopping it for pg_basebackup fallback" + + # The failed post-rewind start may already have exited. Stopping an + # already-stopped server is success for fallback purposes. + if pg_running; then + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" -D "$PGDATA" stop -m fast -w + ' + then + ocf_exit_reason \ + "Unable to stop non-streaming standby before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + else + ocf_log info \ + "PostgreSQL is already stopped; proceeding directly to pg_basebackup fallback" + fi + + mc_pg_rejoin_basebackup_fallback \ + "${pgdata}" \ + "pg_rewind completed but standby could not establish WAL streaming" || return "$?" + + ocf_log info "Reconfiguring PostgreSQL standby after pg_basebackup fallback" + mc_pg_rejoin_configure_standby || return "$?" + + if ! mc_pg_rejoin_start_and_wait_streaming; then + ocf_exit_reason \ + "PostgreSQL did not become a streaming standby after pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + pg_clear_rejoin_pending >/dev/null 2>&1 || true + ocf_log info \ + "PostgreSQL successfully rejoined as streaming standby after pg_basebackup fallback" + return "${OCF_SUCCESS}" +} diff --git a/ocf/loxthelion/managed-container.d/applications/postgresql/replication.sh b/ocf/loxthelion/managed-container.d/applications/postgresql/replication.sh new file mode 100644 index 0000000..8872387 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/applications/postgresql/replication.sh @@ -0,0 +1,256 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +pg_remote_query() +{ + local sql="$1" + + pg_replication_exec \ + psql \ + "host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ + -Atqc "${sql}" +} + + +# Return exactly one reachable remote writable PostgreSQL primary from the +# validated native peer map as: node|host|port. +# Return 1 when no remote primary is currently reachable, and 2 when more +# than one writable peer is observed (unsafe / split-brain indication). +mc_pg_find_remote_primary() +{ + local self peers line node host port role + local found_count=0 found_node='' found_host='' found_port='' + + peers="${OCF_RESKEY_replication_peers:-}" + self="${OCF_RESKEY_CRM_meta_on_node:-$(hostname)}" + + [ -n "${peers}" ] || return 1 + + while IFS='|' read -r node host port; do + [ -n "${node}" ] || continue + + role="$( + pg_replication_exec \ + psql \ + "host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres connect_timeout=2" \ + -X -w -Atqc 'SELECT pg_is_in_recovery();' \ + 2>/dev/null + )" || continue + + if [ "${role}" = "f" ]; then + found_count=$((found_count + 1)) + found_node="${node}" + found_host="${host}" + found_port="${port}" + fi + done </dev/null + )" || { + ocf_exit_reason \ + "Unable to read POSTGRES_HA_SLOTS from PostgreSQL container" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${slots}" ]; then + ocf_log warn \ + "POSTGRES_HA_SLOTS is empty; no local HA slots will be reconciled" + return "${OCF_SUCCESS}" + fi + + while IFS= read -r raw_slot; do + slot="$(printf '%s' "${raw_slot}" | tr -d '[:space:]')" + + [ -n "${slot}" ] || continue + + case "${slot}" in + *[!a-z0-9_]*) + ocf_exit_reason \ + "Invalid physical replication slot name: ${slot}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + slot_type="$( + pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -At \ + -v ON_ERROR_STOP=1 \ + -c \" + SELECT slot_type + FROM pg_replication_slots + WHERE slot_name = '${slot}'; + \" + " 2>/dev/null + )" || { + ocf_exit_reason \ + "Failed querying replication slot ${slot}" + return "${OCF_ERR_GENERIC}" + } + + slot_type="$(printf '%s' "${slot_type}" | tr -d '[:space:]')" + + case "${slot_type}" in + physical) + ocf_log info \ + "Physical HA slot already exists: ${slot}" + ;; + + "") + ocf_log info \ + "Creating physical HA slot: ${slot}" + + if ! pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -v ON_ERROR_STOP=1 \ + -c \" + SELECT pg_create_physical_replication_slot('${slot}'); + \" + " >/dev/null + then + ocf_exit_reason \ + "Failed creating physical replication slot ${slot}" + return "${OCF_ERR_GENERIC}" + fi + ;; + + *) + ocf_exit_reason \ + "Replication slot ${slot} exists but is type ${slot_type}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + done </dev/null + )" || return 1 + + case "${result}" in + physical) + return 0 + ;; + + "") + if ! pg_replication_exec \ + psql \ + "host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \ + -Atc " + SELECT slot_name + FROM pg_create_physical_replication_slot('${slot}'); + " >/dev/null + then + return 1 + fi + + return 0 + ;; + + *) + ocf_log err \ + "Replication slot ${slot} exists on ${host} but is not physical" + return 1 + ;; + esac +} + +candidate_caught_up() +{ + local candidate="${1:-}" + local result + + [ -n "${candidate}" ] || return 1 + + result="$(pg_command " + psql -U '${DB_USER}' -d '${DB_NAME}' -Atc \" + SELECT CASE + WHEN state = 'streaming' + AND replay_lsn >= pg_current_wal_flush_lsn() + THEN 'yes' + ELSE 'no' + END + FROM pg_stat_replication + WHERE application_name = '${candidate}' + LIMIT 1; + \" + " 2>/dev/null)" || return 1 + + [ "${result}" = "yes" ] +} + +candidate_is_standby() +{ + local host="${1:-}" + local result + local port="${REJOIN_SOURCE_PORT:-5432}" + + [ -n "${host}" ] || return 1 + + result="$( + pg_replication_exec \ + psql \ + "host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres" \ + -Atc "SELECT pg_is_in_recovery();" \ + 2>/dev/null + )" || return 1 + + [ "${result}" = "t" ] +} diff --git a/ocf/loxthelion/managed-container.d/common.sh b/ocf/loxthelion/managed-container.d/common.sh new file mode 100755 index 0000000..e1fa717 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/common.sh @@ -0,0 +1,209 @@ +# Shared OCF scoring wrappers; application policy is supplied by the plugin. + +# Promotion scores are scoped by Pacemaker to OCF_RESOURCE_INSTANCE and the +# executing node. Do not construct a legacy attribute name or cache locally: +# querying attrd on every observation also repairs a lost transient attribute. +mc_promotion_score_set() +{ + local score="$1" current + local resource="${MC_PROMOTION_RESOURCE_INSTANCE:-}" + + if [ -z "${resource}" ]; then + ocf_log warn "Cannot publish promotion score without OCF_RESOURCE_INSTANCE" + return 1 + fi + + case "${score}" in + 1000|100|50|-INFINITY) ;; + *) return 1 ;; + esac + + current="$( + timeout --kill-after=1s 2s crm_attribute \ + --promotion="${resource}" --lifetime=reboot \ + --query --quiet 2>/dev/null + )" || current="" + + [ "${current}" = "${score}" ] && return 0 + + if ! timeout --kill-after=1s 2s crm_attribute \ + --promotion="${resource}" --lifetime=reboot \ + --update="${score}" >/dev/null 2>&1 + then + ocf_log err "Unable to publish promotion score ${score} for ${resource}; prior score may remain" + return 1 + fi + return 0 +} + +mc_promotion_score_from_monitor() +{ + local monitor_rc="$1" score=-INFINITY + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") score=1000 ;; + "${OCF_SUCCESS}") + score="$(mc_application_standby_score)" || score=-INFINITY + ;; + esac + case "${score}" in + 1000|100|50|-INFINITY) ;; + *) score=-INFINITY ;; + esac + mc_promotion_score_set "${score}" +} + +# managed-containerd is authoritative for recurring monitor/status results. +# The legacy application monitor remains temporarily available as a shadow +# comparison only. Promotion scoring is still supplied by the existing +# application scoring hook during this migration stage. +mc_promotion_monitor() +{ + local monitor_rc legacy_rc proxy_result proxy_rc cached_rc + + proxy_result="$(mc_proxy_monitor)" + proxy_rc=$? + + if [ "${proxy_rc}" -ne 0 ]; then + + case "${proxy_rc}" in + "${MC_PROXY_RC_TRANSPORT}") + # Transport loss is a control-plane failure, not proof that + # the managed application itself has failed. + cached_rc="$(mc_proxy_cached_monitor 2>/dev/null)" || cached_rc="" + + if [ -n "${cached_rc}" ]; then + monitor_rc="${cached_rc}" + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + # A standby with an unavailable control plane must + # never remain promotion-eligible during grace. + mc_promotion_score_set -INFINITY || true + ;; + + "${OCF_RUNNING_PROMOTED}") + # Preserve the established primary during a brief + # bounded transport outage. + ;; + esac + + ocf_log warn \ + "managed-containerd unavailable; using recent cached authoritative rc=${monitor_rc} for bounded control-plane grace" + + mc_application_monitor + legacy_rc=$? + + mc_proxy_legacy_shadow_monitor \ + "${monitor_rc}" \ + "${legacy_rc}" || true + + return "${monitor_rc}" + fi + + mc_promotion_score_set -INFINITY || true + ocf_exit_reason \ + "managed-containerd authoritative monitor transport failure and no valid control-plane grace observation exists" + return "${OCF_ERR_GENERIC}" + ;; + + *) + # Protocol/result/configuration/client failures are hard. + # Never conceal them behind cached authoritative health. + mc_proxy_monitor_cache_clear || true + mc_promotion_score_set -INFINITY || true + ocf_exit_reason \ + "managed-containerd authoritative monitor hard failure; cached grace is not permitted" + return "${OCF_ERR_GENERIC}" + ;; + esac + fi + + monitor_rc="${proxy_result%%|*}" + + case "${monitor_rc}" in + "${OCF_SUCCESS}"|\ + "${OCF_ERR_GENERIC}"|\ + "${OCF_NOT_RUNNING}"|\ + "${OCF_RUNNING_PROMOTED}") + ;; + *) + mc_promotion_score_set -INFINITY || true + ocf_exit_reason "managed-containerd returned invalid monitor rc ${monitor_rc}" + return "${OCF_ERR_GENERIC}" + ;; + esac + + ocf_log info "managed-containerd authoritative monitor: resource=${OCF_RESOURCE_INSTANCE:-unknown} proxy=${proxy_result}" + + # Legacy PostgreSQL monitoring remains observational only. + mc_application_monitor + legacy_rc=$? + + mc_proxy_legacy_shadow_monitor "${monitor_rc}" "${legacy_rc}" || true + + mc_promotion_score_from_monitor "${monitor_rc}" || true + return "${monitor_rc}" +} + +# Lifecycle functions retain their original checks, commands and return codes. +# Withdraw the previous score before any transition (including validation +# failure), then reassess only after success. Their internal monitor calls do +# not publish scores, so they cannot restore a positive score mid-transition. +mc_promotion_action() +{ + local action="$1" action_rc monitor_rc + + case "${action}" in + start|stop|promote|demote|rejoin) ;; + *) return "${OCF_ERR_UNIMPLEMENTED}" ;; + esac + + # Any lifecycle mutation invalidates the previous daemon-observed role. + # A fresh daemon monitor must establish the next cache entry. + mc_proxy_monitor_cache_clear + + mc_promotion_score_set -INFINITY || true + if [ "${action}" != stop ]; then + mc_promotion_validate + action_rc=$? + [ "${action_rc}" -eq "${OCF_SUCCESS}" ] || return "${action_rc}" + fi + + mc_application_action "${action}" + action_rc=$? + if [ "${action_rc}" -eq "${OCF_SUCCESS}" ] && [ "${action}" != stop ]; then + mc_application_monitor + monitor_rc=$? + mc_promotion_score_from_monitor "${monitor_rc}" || true + fi + return "${action_rc}" +} + +mc_promotion_validate() +{ + local utility + + if ! [[ "${OCF_RESKEY_proxy_failure_grace_seconds:-15}" =~ ^(0|[1-9][0-9]{0,2})$ ]]; then + ocf_exit_reason "proxy_failure_grace_seconds must be an integer from 0 to 999 seconds" + return "${OCF_ERR_CONFIGURED}" + fi + + if [ "${OCF_RESKEY_proxy_failure_grace_seconds:-15}" -gt 300 ]; then + ocf_exit_reason "proxy_failure_grace_seconds must be between 0 and 300 seconds" + return "${OCF_ERR_CONFIGURED}" + fi + + if ! [[ "${OCF_RESKEY_disconnected_max_age:-300}" =~ ^(0|[1-9][0-9]{0,4})$ ]]; then + ocf_exit_reason "disconnected_max_age must be an integer from 0 to 99999 seconds" + return "${OCF_ERR_CONFIGURED}" + fi + + for utility in crm_attribute timeout python3; do + if ! command -v "${utility}" >/dev/null 2>&1; then + ocf_exit_reason "Required promotion scoring command '${utility}' is not installed" + return "${OCF_ERR_INSTALLED}" + fi + done + mc_application_validate +} diff --git a/ocf/loxthelion/managed-container.d/common.sh.named-rc.new b/ocf/loxthelion/managed-container.d/common.sh.named-rc.new new file mode 100755 index 0000000..e1fa717 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/common.sh.named-rc.new @@ -0,0 +1,209 @@ +# Shared OCF scoring wrappers; application policy is supplied by the plugin. + +# Promotion scores are scoped by Pacemaker to OCF_RESOURCE_INSTANCE and the +# executing node. Do not construct a legacy attribute name or cache locally: +# querying attrd on every observation also repairs a lost transient attribute. +mc_promotion_score_set() +{ + local score="$1" current + local resource="${MC_PROMOTION_RESOURCE_INSTANCE:-}" + + if [ -z "${resource}" ]; then + ocf_log warn "Cannot publish promotion score without OCF_RESOURCE_INSTANCE" + return 1 + fi + + case "${score}" in + 1000|100|50|-INFINITY) ;; + *) return 1 ;; + esac + + current="$( + timeout --kill-after=1s 2s crm_attribute \ + --promotion="${resource}" --lifetime=reboot \ + --query --quiet 2>/dev/null + )" || current="" + + [ "${current}" = "${score}" ] && return 0 + + if ! timeout --kill-after=1s 2s crm_attribute \ + --promotion="${resource}" --lifetime=reboot \ + --update="${score}" >/dev/null 2>&1 + then + ocf_log err "Unable to publish promotion score ${score} for ${resource}; prior score may remain" + return 1 + fi + return 0 +} + +mc_promotion_score_from_monitor() +{ + local monitor_rc="$1" score=-INFINITY + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") score=1000 ;; + "${OCF_SUCCESS}") + score="$(mc_application_standby_score)" || score=-INFINITY + ;; + esac + case "${score}" in + 1000|100|50|-INFINITY) ;; + *) score=-INFINITY ;; + esac + mc_promotion_score_set "${score}" +} + +# managed-containerd is authoritative for recurring monitor/status results. +# The legacy application monitor remains temporarily available as a shadow +# comparison only. Promotion scoring is still supplied by the existing +# application scoring hook during this migration stage. +mc_promotion_monitor() +{ + local monitor_rc legacy_rc proxy_result proxy_rc cached_rc + + proxy_result="$(mc_proxy_monitor)" + proxy_rc=$? + + if [ "${proxy_rc}" -ne 0 ]; then + + case "${proxy_rc}" in + "${MC_PROXY_RC_TRANSPORT}") + # Transport loss is a control-plane failure, not proof that + # the managed application itself has failed. + cached_rc="$(mc_proxy_cached_monitor 2>/dev/null)" || cached_rc="" + + if [ -n "${cached_rc}" ]; then + monitor_rc="${cached_rc}" + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + # A standby with an unavailable control plane must + # never remain promotion-eligible during grace. + mc_promotion_score_set -INFINITY || true + ;; + + "${OCF_RUNNING_PROMOTED}") + # Preserve the established primary during a brief + # bounded transport outage. + ;; + esac + + ocf_log warn \ + "managed-containerd unavailable; using recent cached authoritative rc=${monitor_rc} for bounded control-plane grace" + + mc_application_monitor + legacy_rc=$? + + mc_proxy_legacy_shadow_monitor \ + "${monitor_rc}" \ + "${legacy_rc}" || true + + return "${monitor_rc}" + fi + + mc_promotion_score_set -INFINITY || true + ocf_exit_reason \ + "managed-containerd authoritative monitor transport failure and no valid control-plane grace observation exists" + return "${OCF_ERR_GENERIC}" + ;; + + *) + # Protocol/result/configuration/client failures are hard. + # Never conceal them behind cached authoritative health. + mc_proxy_monitor_cache_clear || true + mc_promotion_score_set -INFINITY || true + ocf_exit_reason \ + "managed-containerd authoritative monitor hard failure; cached grace is not permitted" + return "${OCF_ERR_GENERIC}" + ;; + esac + fi + + monitor_rc="${proxy_result%%|*}" + + case "${monitor_rc}" in + "${OCF_SUCCESS}"|\ + "${OCF_ERR_GENERIC}"|\ + "${OCF_NOT_RUNNING}"|\ + "${OCF_RUNNING_PROMOTED}") + ;; + *) + mc_promotion_score_set -INFINITY || true + ocf_exit_reason "managed-containerd returned invalid monitor rc ${monitor_rc}" + return "${OCF_ERR_GENERIC}" + ;; + esac + + ocf_log info "managed-containerd authoritative monitor: resource=${OCF_RESOURCE_INSTANCE:-unknown} proxy=${proxy_result}" + + # Legacy PostgreSQL monitoring remains observational only. + mc_application_monitor + legacy_rc=$? + + mc_proxy_legacy_shadow_monitor "${monitor_rc}" "${legacy_rc}" || true + + mc_promotion_score_from_monitor "${monitor_rc}" || true + return "${monitor_rc}" +} + +# Lifecycle functions retain their original checks, commands and return codes. +# Withdraw the previous score before any transition (including validation +# failure), then reassess only after success. Their internal monitor calls do +# not publish scores, so they cannot restore a positive score mid-transition. +mc_promotion_action() +{ + local action="$1" action_rc monitor_rc + + case "${action}" in + start|stop|promote|demote|rejoin) ;; + *) return "${OCF_ERR_UNIMPLEMENTED}" ;; + esac + + # Any lifecycle mutation invalidates the previous daemon-observed role. + # A fresh daemon monitor must establish the next cache entry. + mc_proxy_monitor_cache_clear + + mc_promotion_score_set -INFINITY || true + if [ "${action}" != stop ]; then + mc_promotion_validate + action_rc=$? + [ "${action_rc}" -eq "${OCF_SUCCESS}" ] || return "${action_rc}" + fi + + mc_application_action "${action}" + action_rc=$? + if [ "${action_rc}" -eq "${OCF_SUCCESS}" ] && [ "${action}" != stop ]; then + mc_application_monitor + monitor_rc=$? + mc_promotion_score_from_monitor "${monitor_rc}" || true + fi + return "${action_rc}" +} + +mc_promotion_validate() +{ + local utility + + if ! [[ "${OCF_RESKEY_proxy_failure_grace_seconds:-15}" =~ ^(0|[1-9][0-9]{0,2})$ ]]; then + ocf_exit_reason "proxy_failure_grace_seconds must be an integer from 0 to 999 seconds" + return "${OCF_ERR_CONFIGURED}" + fi + + if [ "${OCF_RESKEY_proxy_failure_grace_seconds:-15}" -gt 300 ]; then + ocf_exit_reason "proxy_failure_grace_seconds must be between 0 and 300 seconds" + return "${OCF_ERR_CONFIGURED}" + fi + + if ! [[ "${OCF_RESKEY_disconnected_max_age:-300}" =~ ^(0|[1-9][0-9]{0,4})$ ]]; then + ocf_exit_reason "disconnected_max_age must be an integer from 0 to 99999 seconds" + return "${OCF_ERR_CONFIGURED}" + fi + + for utility in crm_attribute timeout python3; do + if ! command -v "${utility}" >/dev/null 2>&1; then + ocf_exit_reason "Required promotion scoring command '${utility}' is not installed" + return "${OCF_ERR_INSTALLED}" + fi + done + mc_application_validate +} diff --git a/ocf/loxthelion/managed-container.d/container.sh b/ocf/loxthelion/managed-container.d/container.sh new file mode 100644 index 0000000..208b477 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/container.sh @@ -0,0 +1,54 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +service_uid() +{ + id -u "${SERVICE_USER}" +} + +service_home() +{ + getent passwd "${SERVICE_USER}" | cut -d: -f6 +} + +as_service_user() +{ + local uid home + + uid="$(id -u "$SERVICE_USER")" || return 1 + home="$(getent passwd "$SERVICE_USER" | cut -d: -f6)" || return 1 + + runuser -u "$SERVICE_USER" -- \ + sh -c ' + home="$1" + runtime="$2" + shift 2 + + cd "$home" || exit 1 + + export HOME="$home" + export XDG_RUNTIME_DIR="$runtime" + + exec "$@" + ' sh \ + "$home" \ + "/run/user/$uid" \ + "$@" +} + +service_running() +{ + mc_lifecycle_running "${SERVICE_NAME}" +} + +container_running() +{ + local state + + state="$( + mc_runtime_inspect \ + --format '{{.State.Running}}' \ + "${CONTAINER_NAME}" 2>/dev/null + )" || return 1 + + [ "${state}" = "true" ] +} diff --git a/ocf/loxthelion/managed-container.d/engine.sh b/ocf/loxthelion/managed-container.d/engine.sh new file mode 100755 index 0000000..2c1c126 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/engine.sh @@ -0,0 +1,64 @@ +# Engine API 1. Selection is an allowlist of packaged implementations, never a +# filename, shell fragment or a plugin downloaded from cluster configuration. +mc_engine_load() +{ + local plugin hook + if [ "${OCF_RESKEY_application:-postgresql}" != postgresql ] || + [ "${OCF_RESKEY_container_runtime:-podman}" != podman ] || + [ "${OCF_RESKEY_lifecycle_manager:-systemd}" != systemd ]; then + ocf_exit_reason "Supported plugin combination: application=postgresql container_runtime=podman lifecycle_manager=systemd" + return "${OCF_ERR_CONFIGURED}" + fi + + unset MC_RUNTIME_API MC_LIFECYCLE_API MC_APPLICATION_API + for plugin in runtimes/podman lifecycles/systemd applications/postgresql/plugin; do + if ! . "${MC_MODULE_DIR}/${plugin}.sh"; then + ocf_exit_reason "Unable to load resource-agent module ${MC_MODULE_DIR}/${plugin}.sh" + return "${OCF_ERR_CONFIGURED}" + fi + done + if [ "${MC_RUNTIME_API:-}" != 1 ] || [ "${MC_LIFECYCLE_API:-}" != 1 ] || + [ "${MC_APPLICATION_API:-}" != 1 ]; then + ocf_exit_reason "Incompatible plugin interface; API 1 required" + return "${OCF_ERR_CONFIGURED}" + fi + for hook in mc_runtime_exec mc_runtime_exec_bounded mc_runtime_inspect mc_runtime_info \ + mc_lifecycle_start mc_lifecycle_stop mc_lifecycle_running mc_lifecycle_cat \ + mc_application_monitor mc_application_standby_score mc_application_validate \ + mc_application_metadata mc_application_notify mc_application_action; do + if ! declare -F "${hook}" >/dev/null; then + ocf_exit_reason "Required plugin hook is missing: ${hook}" + return "${OCF_ERR_CONFIGURED}" + fi + done +} + +mc_engine_metadata() +{ + # Metadata discovery works without configured resource instance attributes. + # Capture the plugin's status before transforming XML; avoid a pipeline + # masking a metadata failure. No runtime/service/cluster access is required. + local metadata + metadata="$(mc_application_metadata)" || return "$?" + printf '%s\n' "${metadata}" | sed '/<\/parameters>/i\ + \ + Packaged application plugin. Only postgresql is supported.\ + Application plugin\ + \ + \ + \ + Packaged runtime adapter. Only podman is supported.\ + Container runtime\ + \ + \ + \ + Container lifecycle owner. Only systemd with the existing user service is supported.\ + Lifecycle manager\ + \ + \ + \ + Bounded control-plane grace after managed-containerd becomes unavailable. A recent healthy daemon-authoritative role may be temporarily preserved. Unpromoted instances have promotion eligibility withdrawn during the grace. Default 15 seconds; 0 disables the grace.\ + managed-containerd failure grace\ + \ + ' +} diff --git a/ocf/loxthelion/managed-container.d/lifecycles/systemd.sh b/ocf/loxthelion/managed-container.d/lifecycles/systemd.sh new file mode 100755 index 0000000..016caaf --- /dev/null +++ b/ocf/loxthelion/managed-container.d/lifecycles/systemd.sh @@ -0,0 +1,23 @@ +# Lifecycle adapter API 1. Quadlet remains the owner of container construction, +# init/maintenance command, signal settings and service restart/exit policies. +MC_LIFECYCLE_API=1 + +mc_lifecycle_start() +{ + as_service_user systemctl --user start "$1" +} + +mc_lifecycle_stop() +{ + as_service_user systemctl --user stop "$1" +} + +mc_lifecycle_running() +{ + as_service_user systemctl --user is-active --quiet "$1" +} + +mc_lifecycle_cat() +{ + as_service_user systemctl --user cat "$1" +} diff --git a/ocf/loxthelion/managed-container.d/proxy.sh b/ocf/loxthelion/managed-container.d/proxy.sh new file mode 100755 index 0000000..bf09682 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/proxy.sh @@ -0,0 +1,464 @@ +# Generic managed-containerd client helpers. +# Shadow calls are observational only and must never affect OCF action results. + +MC_PROXY_SOCKET_DEFAULT="/run/user/1000/managed-containerd/managed-containerd.sock" + +mc_proxy_shadow_monitor() +{ + local authoritative_rc="$1" + local socket_path="${MC_PROXY_SOCKET:-${MC_PROXY_SOCKET_DEFAULT}}" + local resource="${OCF_RESOURCE_INSTANCE:-}" + local response + + resource="${resource%%:*}" + + [ -n "${resource}" ] || return 0 + [ -S "${socket_path}" ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + + response="$( + timeout --kill-after=1s 3s python3 - "${socket_path}" "${resource}" "${authoritative_rc}" <<'PY' +import json +import socket +import sys + +path, resource, authoritative_raw = sys.argv[1], sys.argv[2], sys.argv[3] +authoritative_rc = int(authoritative_raw) +request = {"action": "monitor", "resource": resource} + +s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +s.settimeout(2.0) +s.connect(path) +s.sendall(json.dumps(request).encode("utf-8") + b"\n") + +raw = b"" +while b"\n" not in raw: + chunk = s.recv(65536) + if not chunk: + break + raw += chunk +s.close() + +obj = json.loads(raw.decode("utf-8").strip()) +if not obj.get("ok"): + print("ERROR|" + str(obj.get("error", "unknown proxy error"))) + raise SystemExit(0) + +result = obj.get("result") or {} +ha = result.get("ha") or {} +proxy_rc = ha.get("ocf_rc") + +if proxy_rc is None: + comparison = "UNCLASSIFIED" +else: + comparison = "MATCH" if int(proxy_rc) == authoritative_rc else "MISMATCH" + +receiver = result.get("wal_receiver") or {} +print("OK|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}".format( + comparison, + "null" if proxy_rc is None else proxy_rc, + ha.get("health", "unknown"), + ha.get("ha_role", "unknown"), + ha.get("state", "unknown"), + str(ha.get("promotion_eligible", "unknown")).lower(), + result.get("role", "unknown"), + str(result.get("streaming", "unknown")).lower(), + receiver.get("slot_name") or "", + receiver.get("sender_host") or "", +)) +PY + )" || response="TRANSPORT_ERROR" + + case "${response}" in + OK\|MATCH\|*) + ocf_log info \ + "managed-containerd shadow monitor: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}" + ;; + OK\|UNCLASSIFIED\|*) + ocf_log warn \ + "managed-containerd shadow monitor unclassified: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}" + ;; + OK\|MISMATCH\|*) + ocf_log err \ + "managed-containerd shadow monitor MISMATCH: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}" + ;; + ERROR\|*) + ocf_log warn \ + "managed-containerd shadow monitor failed: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}" + ;; + *) + ocf_log warn \ + "managed-containerd shadow monitor transport failure: resource=${resource} authoritative_rc=${authoritative_rc}" + ;; + esac + + return 0 +} + + +# Query managed-containerd for the authoritative monitor result. +# +# stdout: +# | +# +# return: +# 0 valid classified daemon result +# MC_PROXY_RC_TRANSPORT daemon transport/control-plane failure; bounded cache grace allowed +# MC_PROXY_RC_HARD_FAILURE configuration/protocol/result failure; fail closed, no cache grace +mc_proxy_monitor() +{ + local socket_path="${MC_PROXY_SOCKET:-${MC_PROXY_SOCKET_DEFAULT}}" + local resource="${OCF_RESOURCE_INSTANCE:-}" + local response rc + + resource="${resource%%:*}" + + if [ -z "${resource}" ]; then + ocf_log err \ + "managed-containerd monitor requires OCF_RESOURCE_INSTANCE" + return "${MC_PROXY_RC_HARD_FAILURE}" + fi + + if [ ! -S "${socket_path}" ]; then + ocf_log warn \ + "managed-containerd socket is unavailable: ${socket_path}" + return "${MC_PROXY_RC_TRANSPORT}" + fi + + if ! command -v python3 >/dev/null 2>&1; then + ocf_log err \ + "python3 is required for managed-containerd monitor transport" + return "${MC_PROXY_RC_HARD_FAILURE}" + fi + + response="$( + timeout --kill-after=1s 3s \ + python3 - \ + "${socket_path}" \ + "${resource}" \ + "${MC_PROXY_RC_TRANSPORT}" \ + "${MC_PROXY_RC_HARD_FAILURE}" <<'PY' +import json +import socket +import sys + +path, resource = sys.argv[1], sys.argv[2] +TRANSPORT_FAILURE = int(sys.argv[3]) +PROTOCOL_FAILURE = int(sys.argv[4]) + +request = { + "action": "monitor", + "resource": resource, +} + +try: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.settimeout(2.0) + + try: + s.connect(path) + s.sendall(json.dumps(request).encode("utf-8") + b"\n") + + raw = b"" + + while b"\n" not in raw: + chunk = s.recv(65536) + + if not chunk: + break + + raw += chunk + finally: + s.close() + +except (socket.timeout, ConnectionError, OSError): + # Local daemon/control-plane reachability failure. + # Recent authoritative state may be used for bounded grace. + raise SystemExit(TRANSPORT_FAILURE) + +# Transport succeeded. Invalid daemon output is therefore a +# protocol/result failure and must never be hidden by cached health. +if not raw: + raise SystemExit(PROTOCOL_FAILURE) + +try: + obj = json.loads(raw.decode("utf-8").strip()) +except (UnicodeDecodeError, json.JSONDecodeError): + raise SystemExit(PROTOCOL_FAILURE) + +if not isinstance(obj, dict): + raise SystemExit(PROTOCOL_FAILURE) + +if not obj.get("ok"): + raise SystemExit(PROTOCOL_FAILURE) + +result = obj.get("result") +if not isinstance(result, dict): + raise SystemExit(PROTOCOL_FAILURE) + +ha = result.get("ha") +if not isinstance(ha, dict): + raise SystemExit(PROTOCOL_FAILURE) + +rc = ha.get("ocf_rc") + +try: + rc = int(rc) +except (TypeError, ValueError): + raise SystemExit(PROTOCOL_FAILURE) + +if rc not in (0, 1, 7, 8): + raise SystemExit(PROTOCOL_FAILURE) + +receiver = result.get("wal_receiver") or {} + +if not isinstance(receiver, dict): + raise SystemExit(PROTOCOL_FAILURE) + +payload = "|".join([ + str(rc), + str(ha.get("health", "unknown")), + str(ha.get("ha_role", "unknown")), + str(ha.get("state", "unknown")), + str(ha.get("promotion_eligible", "unknown")).lower(), + str(result.get("role", "unknown")), + str(result.get("streaming", "unknown")).lower(), + str(receiver.get("slot_name") or ""), + str(receiver.get("sender_host") or ""), +]) + +print(payload) +PY + )" + rc=$? + + case "${rc}" in + 0) + ;; + + "${MC_PROXY_RC_TRANSPORT}"|124|137) + # MC_PROXY_RC_TRANSPORT = classified Python transport failure. + # 124/137 = timeout(1) expiry / forced termination. + ocf_log warn \ + "managed-containerd authoritative monitor transport failure" + return "${MC_PROXY_RC_TRANSPORT}" + ;; + + "${MC_PROXY_RC_HARD_FAILURE}") + ocf_log err \ + "managed-containerd authoritative monitor protocol or result failure" + return "${MC_PROXY_RC_HARD_FAILURE}" + ;; + + *) + ocf_log err \ + "managed-containerd authoritative monitor client failure rc=${rc}" + return "${MC_PROXY_RC_HARD_FAILURE}" + ;; + esac + + if [ -z "${response}" ]; then + ocf_log err \ + "managed-containerd authoritative monitor returned an empty result" + return "${MC_PROXY_RC_HARD_FAILURE}" + fi + + rc="${response%%|*}" + + # Cache only healthy daemon-authoritative role observations. An + # application-level failure automatically invalidates any previous cache. + mc_proxy_monitor_cache_write "${rc}" || { + ocf_log warn \ + "Unable to update managed-containerd authoritative monitor cache" + } + + printf '%s\n' "${response}" + return 0 +} + + +# Compare the legacy application monitor against a daemon-authoritative rc. +# The legacy result is diagnostic only and can never change the returned +# Pacemaker monitor status. +mc_proxy_legacy_shadow_monitor() +{ + local authoritative_rc="$1" + local legacy_rc="$2" + local resource="${OCF_RESOURCE_INSTANCE:-}" + + resource="${resource%%:*}" + + if [ "${legacy_rc}" -eq "${authoritative_rc}" ]; then + ocf_log info \ + "legacy application shadow monitor: resource=${resource} authoritative_rc=${authoritative_rc} legacy_rc=${legacy_rc} comparison=MATCH" + else + ocf_log err \ + "legacy application shadow monitor MISMATCH: resource=${resource} authoritative_rc=${authoritative_rc} legacy_rc=${legacy_rc}" + fi + + return 0 +} + + +# Generic short-lived cache of the last healthy daemon-authoritative OCF role. +# This is control-plane state only; it contains no application-specific data. +# +# Use HA_RSCTMP when available because it is the standard resource-agent +# runtime location and is preferable to inventing a persistent filesystem path. +MC_PROXY_STATE_DIR_DEFAULT="${HA_RSCTMP:-/run/resource-agents}/managed-container" + + +mc_proxy_state_file() +{ + local resource safe_resource + resource="${OCF_RESOURCE_INSTANCE:-}" + resource="${resource%%:*}" + + [ -n "${resource}" ] || return 1 + + safe_resource="$( + printf '%s' "${resource}" | + tr -c 'A-Za-z0-9_.-' '_' + )" + + printf '%s/%s.monitor\n' \ + "${MC_PROXY_STATE_DIR:-${MC_PROXY_STATE_DIR_DEFAULT}}" \ + "${safe_resource}" +} + + +mc_proxy_monitor_cache_clear() +{ + local state_file + + state_file="$(mc_proxy_state_file)" || return 0 + rm -f "${state_file}" 2>/dev/null || true +} + + +mc_proxy_monitor_cache_write() +{ + local rc="$1" + local state_file state_dir tmp now + + # Only healthy authoritative role observations are eligible for + # temporary control-plane grace. + case "${rc}" in + "${OCF_SUCCESS}"|"${OCF_RUNNING_PROMOTED}") + ;; + *) + mc_proxy_monitor_cache_clear + return 0 + ;; + esac + + state_file="$(mc_proxy_state_file)" || return 1 + state_dir="${state_file%/*}" + + mkdir -p -m 700 "${state_dir}" || return 1 + + now="$(date +%s)" + tmp="${state_file}.$$" + + umask 077 + printf '%s|%s|0 +' "${now}" "${rc}" > "${tmp}" || { + rm -f "${tmp}" 2>/dev/null || true + return 1 + } + + chmod 600 "${tmp}" 2>/dev/null || true + mv -f "${tmp}" "${state_file}" +} + + +# Print a cached healthy OCF rc when its observation is still inside the +# configured daemon control-plane failure grace. +mc_proxy_cached_monitor() +{ + local state_file observed rc failure_started + local now failure_age observation_age grace max_observation_age + local tmp + + grace="${OCF_RESKEY_proxy_failure_grace_seconds:-15}" + + case "${grace}" in + ""|*[!0-9]*) + return 1 + ;; + esac + + [ "${grace}" -gt 0 ] || return 1 + + state_file="$(mc_proxy_state_file)" || return 1 + [ -f "${state_file}" ] || return 1 + + IFS='|' read -r observed rc failure_started < "${state_file}" || { + mc_proxy_monitor_cache_clear + return 1 + } + + case "${observed}" in + ""|*[!0-9]*) + mc_proxy_monitor_cache_clear + return 1 + ;; + esac + + case "${failure_started}" in + ""|*[!0-9]*) + mc_proxy_monitor_cache_clear + return 1 + ;; + esac + + case "${rc}" in + "${OCF_SUCCESS}"|"${OCF_RUNNING_PROMOTED}") + ;; + *) + mc_proxy_monitor_cache_clear + return 1 + ;; + esac + + now="$(date +%s)" + observation_age=$((now - observed)) + + # A cached role must come from a reasonably recent successful daemon + # observation. This protects against resurrecting stale runtime state. + max_observation_age=$((grace + 60)) + + if [ "${observation_age}" -lt 0 ] || + [ "${observation_age}" -gt "${max_observation_age}" ]; then + mc_proxy_monitor_cache_clear + return 1 + fi + + # First failed daemon observation starts the bounded grace period. + if [ "${failure_started}" -eq 0 ]; then + failure_started="${now}" + tmp="${state_file}.$$" + + umask 077 + printf '%s|%s|%s +' "${observed}" "${rc}" "${failure_started}" > "${tmp}" || { + rm -f "${tmp}" 2>/dev/null || true + return 1 + } + + chmod 600 "${tmp}" 2>/dev/null || true + mv -f "${tmp}" "${state_file}" || return 1 + fi + + failure_age=$((now - failure_started)) + + if [ "${failure_age}" -lt 0 ] || + [ "${failure_age}" -ge "${grace}" ]; then + mc_proxy_monitor_cache_clear + return 1 + fi + + printf '%s +' "${rc}" + return 0 +} diff --git a/ocf/loxthelion/managed-container.d/proxy.sh.named-rc.new b/ocf/loxthelion/managed-container.d/proxy.sh.named-rc.new new file mode 100755 index 0000000..bf09682 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/proxy.sh.named-rc.new @@ -0,0 +1,464 @@ +# Generic managed-containerd client helpers. +# Shadow calls are observational only and must never affect OCF action results. + +MC_PROXY_SOCKET_DEFAULT="/run/user/1000/managed-containerd/managed-containerd.sock" + +mc_proxy_shadow_monitor() +{ + local authoritative_rc="$1" + local socket_path="${MC_PROXY_SOCKET:-${MC_PROXY_SOCKET_DEFAULT}}" + local resource="${OCF_RESOURCE_INSTANCE:-}" + local response + + resource="${resource%%:*}" + + [ -n "${resource}" ] || return 0 + [ -S "${socket_path}" ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + + response="$( + timeout --kill-after=1s 3s python3 - "${socket_path}" "${resource}" "${authoritative_rc}" <<'PY' +import json +import socket +import sys + +path, resource, authoritative_raw = sys.argv[1], sys.argv[2], sys.argv[3] +authoritative_rc = int(authoritative_raw) +request = {"action": "monitor", "resource": resource} + +s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +s.settimeout(2.0) +s.connect(path) +s.sendall(json.dumps(request).encode("utf-8") + b"\n") + +raw = b"" +while b"\n" not in raw: + chunk = s.recv(65536) + if not chunk: + break + raw += chunk +s.close() + +obj = json.loads(raw.decode("utf-8").strip()) +if not obj.get("ok"): + print("ERROR|" + str(obj.get("error", "unknown proxy error"))) + raise SystemExit(0) + +result = obj.get("result") or {} +ha = result.get("ha") or {} +proxy_rc = ha.get("ocf_rc") + +if proxy_rc is None: + comparison = "UNCLASSIFIED" +else: + comparison = "MATCH" if int(proxy_rc) == authoritative_rc else "MISMATCH" + +receiver = result.get("wal_receiver") or {} +print("OK|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}".format( + comparison, + "null" if proxy_rc is None else proxy_rc, + ha.get("health", "unknown"), + ha.get("ha_role", "unknown"), + ha.get("state", "unknown"), + str(ha.get("promotion_eligible", "unknown")).lower(), + result.get("role", "unknown"), + str(result.get("streaming", "unknown")).lower(), + receiver.get("slot_name") or "", + receiver.get("sender_host") or "", +)) +PY + )" || response="TRANSPORT_ERROR" + + case "${response}" in + OK\|MATCH\|*) + ocf_log info \ + "managed-containerd shadow monitor: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}" + ;; + OK\|UNCLASSIFIED\|*) + ocf_log warn \ + "managed-containerd shadow monitor unclassified: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}" + ;; + OK\|MISMATCH\|*) + ocf_log err \ + "managed-containerd shadow monitor MISMATCH: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}" + ;; + ERROR\|*) + ocf_log warn \ + "managed-containerd shadow monitor failed: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}" + ;; + *) + ocf_log warn \ + "managed-containerd shadow monitor transport failure: resource=${resource} authoritative_rc=${authoritative_rc}" + ;; + esac + + return 0 +} + + +# Query managed-containerd for the authoritative monitor result. +# +# stdout: +# | +# +# return: +# 0 valid classified daemon result +# MC_PROXY_RC_TRANSPORT daemon transport/control-plane failure; bounded cache grace allowed +# MC_PROXY_RC_HARD_FAILURE configuration/protocol/result failure; fail closed, no cache grace +mc_proxy_monitor() +{ + local socket_path="${MC_PROXY_SOCKET:-${MC_PROXY_SOCKET_DEFAULT}}" + local resource="${OCF_RESOURCE_INSTANCE:-}" + local response rc + + resource="${resource%%:*}" + + if [ -z "${resource}" ]; then + ocf_log err \ + "managed-containerd monitor requires OCF_RESOURCE_INSTANCE" + return "${MC_PROXY_RC_HARD_FAILURE}" + fi + + if [ ! -S "${socket_path}" ]; then + ocf_log warn \ + "managed-containerd socket is unavailable: ${socket_path}" + return "${MC_PROXY_RC_TRANSPORT}" + fi + + if ! command -v python3 >/dev/null 2>&1; then + ocf_log err \ + "python3 is required for managed-containerd monitor transport" + return "${MC_PROXY_RC_HARD_FAILURE}" + fi + + response="$( + timeout --kill-after=1s 3s \ + python3 - \ + "${socket_path}" \ + "${resource}" \ + "${MC_PROXY_RC_TRANSPORT}" \ + "${MC_PROXY_RC_HARD_FAILURE}" <<'PY' +import json +import socket +import sys + +path, resource = sys.argv[1], sys.argv[2] +TRANSPORT_FAILURE = int(sys.argv[3]) +PROTOCOL_FAILURE = int(sys.argv[4]) + +request = { + "action": "monitor", + "resource": resource, +} + +try: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.settimeout(2.0) + + try: + s.connect(path) + s.sendall(json.dumps(request).encode("utf-8") + b"\n") + + raw = b"" + + while b"\n" not in raw: + chunk = s.recv(65536) + + if not chunk: + break + + raw += chunk + finally: + s.close() + +except (socket.timeout, ConnectionError, OSError): + # Local daemon/control-plane reachability failure. + # Recent authoritative state may be used for bounded grace. + raise SystemExit(TRANSPORT_FAILURE) + +# Transport succeeded. Invalid daemon output is therefore a +# protocol/result failure and must never be hidden by cached health. +if not raw: + raise SystemExit(PROTOCOL_FAILURE) + +try: + obj = json.loads(raw.decode("utf-8").strip()) +except (UnicodeDecodeError, json.JSONDecodeError): + raise SystemExit(PROTOCOL_FAILURE) + +if not isinstance(obj, dict): + raise SystemExit(PROTOCOL_FAILURE) + +if not obj.get("ok"): + raise SystemExit(PROTOCOL_FAILURE) + +result = obj.get("result") +if not isinstance(result, dict): + raise SystemExit(PROTOCOL_FAILURE) + +ha = result.get("ha") +if not isinstance(ha, dict): + raise SystemExit(PROTOCOL_FAILURE) + +rc = ha.get("ocf_rc") + +try: + rc = int(rc) +except (TypeError, ValueError): + raise SystemExit(PROTOCOL_FAILURE) + +if rc not in (0, 1, 7, 8): + raise SystemExit(PROTOCOL_FAILURE) + +receiver = result.get("wal_receiver") or {} + +if not isinstance(receiver, dict): + raise SystemExit(PROTOCOL_FAILURE) + +payload = "|".join([ + str(rc), + str(ha.get("health", "unknown")), + str(ha.get("ha_role", "unknown")), + str(ha.get("state", "unknown")), + str(ha.get("promotion_eligible", "unknown")).lower(), + str(result.get("role", "unknown")), + str(result.get("streaming", "unknown")).lower(), + str(receiver.get("slot_name") or ""), + str(receiver.get("sender_host") or ""), +]) + +print(payload) +PY + )" + rc=$? + + case "${rc}" in + 0) + ;; + + "${MC_PROXY_RC_TRANSPORT}"|124|137) + # MC_PROXY_RC_TRANSPORT = classified Python transport failure. + # 124/137 = timeout(1) expiry / forced termination. + ocf_log warn \ + "managed-containerd authoritative monitor transport failure" + return "${MC_PROXY_RC_TRANSPORT}" + ;; + + "${MC_PROXY_RC_HARD_FAILURE}") + ocf_log err \ + "managed-containerd authoritative monitor protocol or result failure" + return "${MC_PROXY_RC_HARD_FAILURE}" + ;; + + *) + ocf_log err \ + "managed-containerd authoritative monitor client failure rc=${rc}" + return "${MC_PROXY_RC_HARD_FAILURE}" + ;; + esac + + if [ -z "${response}" ]; then + ocf_log err \ + "managed-containerd authoritative monitor returned an empty result" + return "${MC_PROXY_RC_HARD_FAILURE}" + fi + + rc="${response%%|*}" + + # Cache only healthy daemon-authoritative role observations. An + # application-level failure automatically invalidates any previous cache. + mc_proxy_monitor_cache_write "${rc}" || { + ocf_log warn \ + "Unable to update managed-containerd authoritative monitor cache" + } + + printf '%s\n' "${response}" + return 0 +} + + +# Compare the legacy application monitor against a daemon-authoritative rc. +# The legacy result is diagnostic only and can never change the returned +# Pacemaker monitor status. +mc_proxy_legacy_shadow_monitor() +{ + local authoritative_rc="$1" + local legacy_rc="$2" + local resource="${OCF_RESOURCE_INSTANCE:-}" + + resource="${resource%%:*}" + + if [ "${legacy_rc}" -eq "${authoritative_rc}" ]; then + ocf_log info \ + "legacy application shadow monitor: resource=${resource} authoritative_rc=${authoritative_rc} legacy_rc=${legacy_rc} comparison=MATCH" + else + ocf_log err \ + "legacy application shadow monitor MISMATCH: resource=${resource} authoritative_rc=${authoritative_rc} legacy_rc=${legacy_rc}" + fi + + return 0 +} + + +# Generic short-lived cache of the last healthy daemon-authoritative OCF role. +# This is control-plane state only; it contains no application-specific data. +# +# Use HA_RSCTMP when available because it is the standard resource-agent +# runtime location and is preferable to inventing a persistent filesystem path. +MC_PROXY_STATE_DIR_DEFAULT="${HA_RSCTMP:-/run/resource-agents}/managed-container" + + +mc_proxy_state_file() +{ + local resource safe_resource + resource="${OCF_RESOURCE_INSTANCE:-}" + resource="${resource%%:*}" + + [ -n "${resource}" ] || return 1 + + safe_resource="$( + printf '%s' "${resource}" | + tr -c 'A-Za-z0-9_.-' '_' + )" + + printf '%s/%s.monitor\n' \ + "${MC_PROXY_STATE_DIR:-${MC_PROXY_STATE_DIR_DEFAULT}}" \ + "${safe_resource}" +} + + +mc_proxy_monitor_cache_clear() +{ + local state_file + + state_file="$(mc_proxy_state_file)" || return 0 + rm -f "${state_file}" 2>/dev/null || true +} + + +mc_proxy_monitor_cache_write() +{ + local rc="$1" + local state_file state_dir tmp now + + # Only healthy authoritative role observations are eligible for + # temporary control-plane grace. + case "${rc}" in + "${OCF_SUCCESS}"|"${OCF_RUNNING_PROMOTED}") + ;; + *) + mc_proxy_monitor_cache_clear + return 0 + ;; + esac + + state_file="$(mc_proxy_state_file)" || return 1 + state_dir="${state_file%/*}" + + mkdir -p -m 700 "${state_dir}" || return 1 + + now="$(date +%s)" + tmp="${state_file}.$$" + + umask 077 + printf '%s|%s|0 +' "${now}" "${rc}" > "${tmp}" || { + rm -f "${tmp}" 2>/dev/null || true + return 1 + } + + chmod 600 "${tmp}" 2>/dev/null || true + mv -f "${tmp}" "${state_file}" +} + + +# Print a cached healthy OCF rc when its observation is still inside the +# configured daemon control-plane failure grace. +mc_proxy_cached_monitor() +{ + local state_file observed rc failure_started + local now failure_age observation_age grace max_observation_age + local tmp + + grace="${OCF_RESKEY_proxy_failure_grace_seconds:-15}" + + case "${grace}" in + ""|*[!0-9]*) + return 1 + ;; + esac + + [ "${grace}" -gt 0 ] || return 1 + + state_file="$(mc_proxy_state_file)" || return 1 + [ -f "${state_file}" ] || return 1 + + IFS='|' read -r observed rc failure_started < "${state_file}" || { + mc_proxy_monitor_cache_clear + return 1 + } + + case "${observed}" in + ""|*[!0-9]*) + mc_proxy_monitor_cache_clear + return 1 + ;; + esac + + case "${failure_started}" in + ""|*[!0-9]*) + mc_proxy_monitor_cache_clear + return 1 + ;; + esac + + case "${rc}" in + "${OCF_SUCCESS}"|"${OCF_RUNNING_PROMOTED}") + ;; + *) + mc_proxy_monitor_cache_clear + return 1 + ;; + esac + + now="$(date +%s)" + observation_age=$((now - observed)) + + # A cached role must come from a reasonably recent successful daemon + # observation. This protects against resurrecting stale runtime state. + max_observation_age=$((grace + 60)) + + if [ "${observation_age}" -lt 0 ] || + [ "${observation_age}" -gt "${max_observation_age}" ]; then + mc_proxy_monitor_cache_clear + return 1 + fi + + # First failed daemon observation starts the bounded grace period. + if [ "${failure_started}" -eq 0 ]; then + failure_started="${now}" + tmp="${state_file}.$$" + + umask 077 + printf '%s|%s|%s +' "${observed}" "${rc}" "${failure_started}" > "${tmp}" || { + rm -f "${tmp}" 2>/dev/null || true + return 1 + } + + chmod 600 "${tmp}" 2>/dev/null || true + mv -f "${tmp}" "${state_file}" || return 1 + fi + + failure_age=$((now - failure_started)) + + if [ "${failure_age}" -lt 0 ] || + [ "${failure_age}" -ge "${grace}" ]; then + mc_proxy_monitor_cache_clear + return 1 + fi + + printf '%s +' "${rc}" + return 0 +} diff --git a/ocf/loxthelion/managed-container.d/runtimes/podman.sh b/ocf/loxthelion/managed-container.d/runtimes/podman.sh new file mode 100755 index 0000000..6d686f4 --- /dev/null +++ b/ocf/loxthelion/managed-container.d/runtimes/podman.sh @@ -0,0 +1,35 @@ +# Runtime adapter API 1. The existing rootless privilege boundary is unchanged. +MC_RUNTIME_API=1 + +mc_runtime_exec() +{ + as_service_user podman exec "$@" +} + +mc_runtime_exec_bounded() +{ + local duration="$1" + shift + as_service_user timeout --kill-after=1s "${duration}" podman exec "$@" +} + +mc_runtime_inspect() +{ + as_service_user podman inspect "$@" +} + +mc_runtime_info() +{ + as_service_user podman info "$@" +} + +# Return one environment variable from a running managed container. +# Application plugins use this rather than parsing lifecycle-manager config. +mc_runtime_environment() +{ + local container=$1 key=$2 + case "$key" in + ''|[!A-Za-z_]*|*[!A-Za-z0-9_]*) return 2 ;; + esac + mc_runtime_exec "$container" printenv "$key" 2>/dev/null +} diff --git a/ocf/loxthelion/managed-container.named-rc.new b/ocf/loxthelion/managed-container.named-rc.new new file mode 100755 index 0000000..650e613 --- /dev/null +++ b/ocf/loxthelion/managed-container.named-rc.new @@ -0,0 +1,93 @@ +#!/bin/bash + +# OCF Resource Agent: +# loxthelion:managed-container +# +# v0.4 +# Container lifecycle: systemd user Quadlet +# Application profile: PostgreSQL +# +# Version 0.4 adds explicit PostgreSQL demotion scaffolding. +# Healthy primaries report OCF_RUNNING_PROMOTED while healthy standbys +# report OCF_SUCCESS. + +: "${OCF_ROOT:=/usr/lib/ocf}" +OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat" +# Preserve explicit resource identity before ocf-shellfuncs supplies its +# standalone-invocation default; never publish a score for that dummy resource. +MC_PROMOTION_RESOURCE_INSTANCE="${OCF_RESOURCE_INSTANCE:-}" +. "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs" + +# Pacemaker uses "Promoted" terminology, while many OCF shell libraries +# still expose the historical OCF_RUNNING_MASTER constant for return code 8. +# Normalize both forms here so the agent remains portable. +: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}" + +# Internal managed-containerd client return classes. +# Private RA implementation statuses, not OCF action return codes. +readonly MC_PROXY_RC_TRANSPORT=10 +readonly MC_PROXY_RC_HARD_FAILURE=11 + + +# Resolve modules relative to this entrypoint, independent of Pacemaker's cwd. +MC_MODULE_DIR="$(CDPATH= cd -- "$(command dirname -- "${BASH_SOURCE[0]}")" && pwd)/managed-container.d" +for MC_MODULE in common container engine; do + if ! . "${MC_MODULE_DIR}/${MC_MODULE}.sh"; then + ocf_exit_reason "Unable to load resource-agent module ${MC_MODULE_DIR}/${MC_MODULE}.sh" + exit "${OCF_ERR_CONFIGURED}" + fi +done +unset MC_MODULE +mc_engine_load || exit $? + +case "$1" in + meta-data) + mc_engine_metadata + exit $? + ;; + + validate-all) + mc_promotion_validate + exit $? + ;; + + monitor|status) + mc_promotion_monitor + exit $? + ;; + + start) + mc_promotion_action start + exit $? + ;; + + stop) + mc_promotion_action stop + exit $? + ;; + + promote) + mc_promotion_action promote + exit $? + ;; + + demote) + mc_promotion_action demote + exit $? + ;; + + notify) + mc_application_notify + exit $? + ;; + + rejoin) + mc_promotion_action rejoin + exit $? + ;; + + *) + echo "Usage: $0 {start|stop|promote|demote|notify|rejoin|monitor|status|validate-all|meta-data}" + exit "${OCF_ERR_UNIMPLEMENTED}" + ;; +esac diff --git a/tests/test-entrypoint-structure.sh b/tests/test-entrypoint-structure.sh new file mode 100755 index 0000000..0fc7766 --- /dev/null +++ b/tests/test-entrypoint-structure.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$( + cd "$(dirname "${BASH_SOURCE[0]}")/.." && + pwd +)" + +ENTRYPOINT="$ROOT/ocf/loxthelion/managed-container" + +if [[ ! -x "$ENTRYPOINT" ]]; then + echo "ERROR: managed-container entrypoint is missing or not executable" >&2 + exit 1 +fi + +echo "===== SYNTAX =====" + +bash -n "$ENTRYPOINT" + +while IFS= read -r -d '' file; do + bash -n "$file" +done < <( + find "$ROOT/ocf/loxthelion/managed-container.d" \ + -type f \ + -name '*.sh' \ + -print0 +) + +echo "SYNTAX_OK=1" + +echo +echo "===== REAL ENTRYPOINT LOAD TEST =====" + +output="$( + env \ + OCF_ROOT="${OCF_ROOT:-/usr/lib/ocf}" \ + MC_ENTRYPOINT_SELFTEST=1 \ + "$ENTRYPOINT" +)" + +printf '%s\n' "$output" + +grep -qx 'ENTRYPOINT_STRUCTURE_OK=1' <<<"$output" + +echo +echo "ENTRYPOINT_TEST_PASS=1"