From 4a358ae0f6e101fe01c511f1c6abede25d14efe2 Mon Sep 17 00:00:00 2001 From: Lyncoln Date: Sat, 12 Sep 2026 01:28:33 -0400 Subject: [PATCH] Add PostgreSQL HA OCF agent modular baseline --- .gitignore | 15 + HANDOFF.md | 352 ++++ README.md | 44 + cluster-context/corosync.conf | 34 + cluster-context/pcs-properties.txt | 8 + cluster-context/pcs-resource-pg-ha-clone.txt | 29 + cluster-context/pcs-resource-pg-ha.txt | 21 + cluster-context/pcs-status.txt | 24 + notes/REFACTOR-PLAN.md | 101 + notes/REFACTOR-REPORT.md | 137 ++ notes/REFACTOR-SHA256SUMS | 13 + notes/SHA256SUMS | 5 + refactored/managed-container | 88 + refactored/managed-container.d/auth.sh | 89 + refactored/managed-container.d/common.sh | 227 +++ refactored/managed-container.d/container.sh | 55 + refactored/managed-container.d/demote.sh | 202 ++ refactored/managed-container.d/notify.sh | 109 ++ refactored/managed-container.d/postgres.sh | 292 +++ refactored/managed-container.d/promote.sh | 94 + refactored/managed-container.d/rejoin.sh | 343 ++++ refactored/managed-container.d/replication.sh | 201 ++ reference/managed-container | 1721 +++++++++++++++++ .../managed-container.pre-refactor-5eae1f3 | 1721 +++++++++++++++++ runtime-context/current-state.txt | 16 + runtime-context/postgres-ha-bootstrap | 533 +++++ .../postgres-ha-bootstrap.pre-v0.7 | 441 +++++ runtime-context/postgres.container | 26 + runtime-context/postgres.env.sanitized | 44 + tests/verify_refactor.py | 134 ++ 30 files changed, 7119 insertions(+) create mode 100644 .gitignore create mode 100644 HANDOFF.md create mode 100644 README.md create mode 100644 cluster-context/corosync.conf create mode 100644 cluster-context/pcs-properties.txt create mode 100644 cluster-context/pcs-resource-pg-ha-clone.txt create mode 100644 cluster-context/pcs-resource-pg-ha.txt create mode 100644 cluster-context/pcs-status.txt create mode 100644 notes/REFACTOR-PLAN.md create mode 100644 notes/REFACTOR-REPORT.md create mode 100644 notes/REFACTOR-SHA256SUMS create mode 100644 notes/SHA256SUMS create mode 100755 refactored/managed-container create mode 100644 refactored/managed-container.d/auth.sh create mode 100644 refactored/managed-container.d/common.sh create mode 100644 refactored/managed-container.d/container.sh create mode 100644 refactored/managed-container.d/demote.sh create mode 100644 refactored/managed-container.d/notify.sh create mode 100644 refactored/managed-container.d/postgres.sh create mode 100644 refactored/managed-container.d/promote.sh create mode 100644 refactored/managed-container.d/rejoin.sh create mode 100644 refactored/managed-container.d/replication.sh create mode 100755 reference/managed-container create mode 100755 reference/managed-container.pre-refactor-5eae1f3 create mode 100644 runtime-context/current-state.txt create mode 100755 runtime-context/postgres-ha-bootstrap create mode 100755 runtime-context/postgres-ha-bootstrap.pre-v0.7 create mode 100644 runtime-context/postgres.container create mode 100644 runtime-context/postgres.env.sanitized create mode 100644 tests/verify_refactor.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2f6680d --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# Local agent configuration +.agents/ +.codex/ + +# Python caches +__pycache__/ +*.py[cod] + +# Local credentials; retain explicitly sanitized environment snapshots +.env +.env.* +*.env +secrets/ +*.pem +*.key diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 0000000..ca4fac5 --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,352 @@ +# PostgreSQL HA OCF Resource Agent — Codex Refactor Handoff + +## Objective + +Perform a behavior-preserving modular refactor of the proven PostgreSQL HA +OCF resource agent contained in: + + reference/managed-container + +Known-good SHA256: + + 5eae1f3dba0586dc06105e4dd7a9b8809a5c9961452abe7654dcfce77c3c5ce2 + +A frozen identical reference copy is: + + reference/managed-container.pre-refactor-5eae1f3 + +DO NOT MODIFY THE FROZEN REFERENCE COPY. + +The objective is maintainability and modularity, not redesign. + +## Architecture + +Control path: + + Pacemaker + -> OCF resource agent + -> runuser david + -> rootless systemd / Podman + -> persistent PostgreSQL maintenance container + -> PostgreSQL + +The maintenance container PID 1 is intentionally `sleep infinity`. + +PostgreSQL itself is independently started/stopped with `pg_ctl`. + +The maintenance container must remain alive during demote/rejoin operations so +the RA can perform: + +- SQL checks +- pg_rewind +- pg_basebackup +- replication slot management +- recovery configuration + +PostgreSQL storage is NOT DRBD-backed. + +Quadlet owns container construction and environment injection. + +The RA may consume environment variables from the already-running container, +but must not gain a dependency on parsing a host-side env file. + +## Public OCF entrypoint + +The public executable must remain: + + /usr/lib/ocf/resource.d/loxthelion/managed-container + +Preferred modular layout: + + managed-container + managed-container.d/ + common.sh + container.sh + postgres.sh + auth.sh + replication.sh + promote.sh + demote.sh + rejoin.sh + notify.sh + +Helper files should normally be sourced shell modules rather than separate +executables. + +Namespaced helper functions such as these are preferred: + + mc_container_* + mc_pg_* + mc_auth_* + mc_replication_* + mc_promote_* + mc_demote_* + mc_rejoin_* + +## Preserve these actions + +- start +- stop +- monitor +- promote +- demote +- notify +- rejoin +- meta-data +- validate-all + +Do not change OCF return codes or action semantics. + +## start() + +start must NOT promote PostgreSQL. + +It should: + +- succeed for an already healthy standby +- refuse an already-running primary +- start/ensure the maintenance container where appropriate +- require standby.signal for stopped PostgreSQL +- start PostgreSQL with pg_ctl +- succeed only as Unpromoted + +Promotion belongs exclusively to promote(). + +## stop() + +stop performs a full resource shutdown: + +1. stop PostgreSQL cleanly +2. stop the rootless postgres.service +3. verify service/container disappear + +Demote and rejoin intentionally behave differently and preserve the maintenance +container. + +## promote() + +Preserve current behavior. + +Promotion must: + +- promote a standby +- verify primary role +- ensure configured local physical HA slots +- ensure pg_rewind privileges + +POSTGRES_HA_SLOTS is obtained from the running container environment. + +Do not automatically delete inactive replication slots. + +## pg_rewind privileges + +Preserve grants for: + +- pg_catalog.pg_ls_dir(text, boolean, boolean) +- pg_catalog.pg_stat_file(text, boolean) +- pg_catalog.pg_read_binary_file(text) +- pg_catalog.pg_read_binary_file(text, bigint,bigint,boolean) + +## demote() + +The proven controlled demotion logic must remain behaviorally identical. + +It: + +1. determines the current role +2. retrieves the persisted pre-demote target from Pacemaker notify state +3. resolves that Corosync node to its ring0 address +4. determines the local physical replication slot +5. verifies the target is a standby +6. verifies the target is caught up +7. ensures this node's slot on the destination +8. stops the old primary +9. keeps the maintenance container alive +10. rewrites recovery settings +11. creates standby.signal +12. starts PostgreSQL +13. verifies recovery mode +14. verifies WAL receiver is streaming + +Do not replace Corosync ring0 addressing with normal hostname resolution. + +## notify() + +Pacemaker notification state is persisted because notification environment +variables may no longer exist by demote time. + +Preserve this mechanism. + +## Replication authentication + +Runtime replication authentication comes from the running container environment. + +POSTGRES_REPLICATION_PASSWORD is exposed inside the running container. + +The RA sets PGPASSWORD only inside the container process executing libpq tools. + +Preserve this design. + +Do NOT: + +- introduce .pgpass +- place passwords in connection strings +- parse a host environment file for runtime credentials + +## rejoin() + +Preconditions include: + +- maintenance service alive +- maintenance container alive +- PostgreSQL stopped +- source reachable +- source confirmed primary +- physical replication slot ensured on source + +Repair order: + +1. pg_rewind +2. optional pg_basebackup fallback + +## pg_basebackup fallback + +Parameter: + + rejoin_allow_basebackup + +Default: + + false + +If rewind fails and fallback is disabled, fail safely. + +If enabled: + +1. revalidate source primary +2. revalidate PostgreSQL stopped +3. revalidate maintenance container alive +4. reensure replication slot +5. calculate approximate source size +6. check conservative free-space requirement +7. rename old canonical PGDATA to: + + $PGDATA.pre-basebackup-TIMESTAMP + +8. create new canonical PGDATA +9. perform pg_basebackup + +If basebackup fails: + +- quarantine partial copy as: + + $PGDATA.failed-basebackup-TIMESTAMP + +- restore the old PGDATA to its canonical pathname +- leave PostgreSQL stopped +- return failure + +Never blindly: + + rm -rf "$PGDATA" + +## Recovery normalization + +After successful rewind or basebackup: + +- normalize primary_conninfo +- normalize primary_slot_name +- create standby.signal +- apply expected permissions +- start PostgreSQL +- verify pg_is_in_recovery() = true +- verify WAL receiver state = streaming + +Prefer extracting common recovery-configuration logic rather than duplicating it. + +## Behavior already proven + +The reference build has successfully passed: + +- standby start +- promotion +- controlled demotion +- bidirectional switchover +- slot reconciliation after promotion +- rewind privilege reconciliation +- successful pg_rewind rejoin +- pg_rewind failure with fallback=false +- pg_rewind failure followed by successful pg_basebackup +- pg_rewind failure followed by failed pg_basebackup and successful rollback +- refusal to run rejoin while PostgreSQL is running + +Do not change these semantics. + +## Current lab + +DAVID-DV6: +- primary + +DAVID-ELITEDESK: +- standby +- streaming from 192.168.1.2 +- slot ha_david_elitedesk + +Pacemaker is intentionally in: + + maintenance-mode=true + +Do NOT disable maintenance mode. + +Do NOT touch production systems. + +## Refactor procedure + +Before editing: + +1. read the complete reference implementation +2. inventory global variables +3. inventory OCF parameters +4. inventory action entrypoints +5. map helper/function dependencies +6. produce a proposed module dependency map + +Then refactor incrementally. + +Avoid changing shell semantics unless required to modularize the code. + +Preserve quoting, SQL, command invocation, OCF error handling and logging. + +## Validation + +At minimum: + +- bash -n main entrypoint +- bash -n every module +- meta-data action works +- validate-all works +- all modules resolve correctly in OCF execution environment +- no missing functions +- no missing variables +- same metadata parameters +- same OCF actions +- no .pgpass usage +- no host env-file parsing dependency +- frozen reference remains unchanged + +Do NOT automatically perform destructive HA tests. + +When finished, report: + +1. final file tree +2. SHA256 of every file +3. functions assigned to each module +4. syntax validation results +5. metadata validation results +6. validate-all results +7. any behavior differences, ideally none +8. exact commands to install the refactored version on the second lab node +9. recommended behavioral comparison test sequence + +The frozen source implementation is authoritative whenever this document and +the implementation differ. diff --git a/README.md b/README.md new file mode 100644 index 0000000..9698342 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# PostgreSQL HA OCF resource agent + +A behavior-preserving modular refactor of `ocf:loxthelion:managed-container`. +Pacemaker invokes the agent to manage a rootless systemd/Quadlet maintenance +container. PostgreSQL is started and stopped independently with `pg_ctl`. + +## Project layout + +- `reference/`: original agent and immutable known-good reference copy. +- `refactored/`: executable agent and nine sourced shell modules. +- `tests/`: offline structural and behavioral comparison checks. +- `cluster-context/` and `runtime-context/`: saved lab configuration and state; + these snapshots are not live status. Environment credentials are redacted. +- `notes/`: refactor plan, validation report, and file hashes. +- `HANDOFF.md`: scope, architecture, and behavior that must be preserved. + +## Validation + +Requires Bash, Python 3, and the OCF shell library (normally +`/usr/lib/ocf/lib/heartbeat/ocf-shellfuncs`). From the repository root: + +```bash +python3 tests/verify_refactor.py +sha256sum -c notes/REFACTOR-SHA256SUMS +``` + +The tests compare the refactor against the frozen reference without operating +the live cluster. Target-node validation and controlled HA transition tests +remain outstanding. Existing function names and recovery configuration blocks +are retained in this initial modular baseline. + +Read [the refactor report](notes/REFACTOR-REPORT.md) for installation commands, +validation results, limitations, and the recommended lab comparison sequence. +The public entrypoint remains +`/usr/lib/ocf/resource.d/loxthelion/managed-container`. + +Keep lab Pacemaker maintenance mode enabled during this work. Do not modify +the frozen reference or automatically run destructive HA tests. + +## Clone + +```bash +git clone ssh://git@gitea.loxthelion.com:2222/david/ocf-managed-container.git +``` diff --git a/cluster-context/corosync.conf b/cluster-context/corosync.conf new file mode 100644 index 0000000..f420976 --- /dev/null +++ b/cluster-context/corosync.conf @@ -0,0 +1,34 @@ +totem { + version: 2 + cluster_name: postgres-ra-test + transport: knet + crypto_cipher: aes256 + crypto_hash: sha256 + cluster_uuid: 58bac4b5a81c4bae904d5f719bdb88b7 +} + +nodelist { + node { + ring0_addr: 192.168.1.6 + name: DAVID-ELITEDESK + nodeid: 1 + } + + node { + ring0_addr: 192.168.1.2 + name: DAVID-DV6 + nodeid: 2 + } +} + +quorum { + provider: corosync_votequorum + two_node: 1 +} + +logging { + to_logfile: yes + logfile: /var/log/cluster/corosync.log + to_syslog: yes + timestamp: on +} diff --git a/cluster-context/pcs-properties.txt b/cluster-context/pcs-properties.txt new file mode 100644 index 0000000..681e9cd --- /dev/null +++ b/cluster-context/pcs-properties.txt @@ -0,0 +1,8 @@ +Cluster Properties: cib-bootstrap-options + cluster-infrastructure=corosync + cluster-name=postgres-ra-test + dc-version=3.0.2-1.fc43-61983e8 + fencing-enabled=false + have-watchdog=false + last-lrm-refresh=1789180342 + maintenance-mode=true diff --git a/cluster-context/pcs-resource-pg-ha-clone.txt b/cluster-context/pcs-resource-pg-ha-clone.txt new file mode 100644 index 0000000..55a6efd --- /dev/null +++ b/cluster-context/pcs-resource-pg-ha-clone.txt @@ -0,0 +1,29 @@ +Clone: pg-ha-clone + Meta Attributes: pg-ha-clone-meta_attributes + clone-max=2 + clone-node-max=1 + notify=true + promotable=true + promoted-max=1 + promoted-node-max=1 + Resource: pg-ha (class=ocf provider=loxthelion type=managed-container) + Attributes: pg-ha-instance_attributes + container_name=postgres + db_name=david + db_user=david + pg_os_user=postgres + service_name=postgres.service + service_user=david + Operations: + demote: pg-ha-demote-interval-0s + interval=0s timeout=60s + monitor: pg-ha-monitor-interval-11s + interval=11s timeout=20s role=Promoted + promote: pg-ha-promote-interval-0s + interval=0s timeout=60s + start: pg-ha-start-interval-0s + interval=0s timeout=60s + stop: pg-ha-stop-interval-0s + interval=0s timeout=60s + monitor: pg-ha-monitor-interval-10s + interval=10s timeout=20s role=Unpromoted diff --git a/cluster-context/pcs-resource-pg-ha.txt b/cluster-context/pcs-resource-pg-ha.txt new file mode 100644 index 0000000..55fbfe2 --- /dev/null +++ b/cluster-context/pcs-resource-pg-ha.txt @@ -0,0 +1,21 @@ +Resource: pg-ha (class=ocf provider=loxthelion type=managed-container) + Attributes: pg-ha-instance_attributes + container_name=postgres + db_name=david + db_user=david + pg_os_user=postgres + service_name=postgres.service + service_user=david + Operations: + demote: pg-ha-demote-interval-0s + interval=0s timeout=60s + monitor: pg-ha-monitor-interval-11s + interval=11s timeout=20s role=Promoted + promote: pg-ha-promote-interval-0s + interval=0s timeout=60s + start: pg-ha-start-interval-0s + interval=0s timeout=60s + stop: pg-ha-stop-interval-0s + interval=0s timeout=60s + monitor: pg-ha-monitor-interval-10s + interval=10s timeout=20s role=Unpromoted diff --git a/cluster-context/pcs-status.txt b/cluster-context/pcs-status.txt new file mode 100644 index 0000000..feeea48 --- /dev/null +++ b/cluster-context/pcs-status.txt @@ -0,0 +1,24 @@ +Cluster name: postgres-ra-test +Cluster Summary: + * Stack: corosync (Pacemaker is running) + * Current DC: DAVID-DV6 (version 3.0.2-1.fc43-61983e8) - partition with quorum + * Last updated: Sat Sep 12 01:12:28 2026 on DAVID-ELITEDESK + * Last change: Sat Sep 12 00:02:03 2026 by root via cibadmin on DAVID-ELITEDESK + * 2 nodes configured + * 2 resource instances configured + + *** Resource management is DISABLED *** + The cluster will not attempt to start, stop or recover services + +Node List: + * Online: [ DAVID-DV6 DAVID-ELITEDESK ] + +Full List of Resources: + * Clone Set: pg-ha-clone [pg-ha] (promotable, maintenance): + * pg-ha (ocf:loxthelion:managed-container): Promoted DAVID-DV6 (maintenance) + * pg-ha (ocf:loxthelion:managed-container): Unpromoted DAVID-ELITEDESK (maintenance) + +Daemon Status: + corosync: active/disabled + pacemaker: active/disabled + pcsd: active/enabled diff --git a/notes/REFACTOR-PLAN.md b/notes/REFACTOR-PLAN.md new file mode 100644 index 0000000..6807307 --- /dev/null +++ b/notes/REFACTOR-PLAN.md @@ -0,0 +1,101 @@ +# Refactor inventory and module plan + +Prepared before implementation. The frozen reference is authoritative. + +## Parameters and globals + +| OCF parameter | Default | Runtime global | +| --- | --- | --- | +| `service_user` | `david` | `SERVICE_USER` | +| `service_name` | `postgres.service` | `SERVICE_NAME` | +| `container_name` | `postgres` | `CONTAINER_NAME` | +| `pg_os_user` | `postgres` | `PG_OS_USER` | +| `db_user` | `david` | `DB_USER` | +| `db_name` | `david` | `DB_NAME` | +| `rejoin_source_host` | `` | `REJOIN_SOURCE_HOST` | +| `rejoin_source_port` | `5432` | `REJOIN_SOURCE_PORT` | +| `replication_user` | `replicator` | `REPLICATION_USER` | +| `replication_slot` | `` | `REPLICATION_SLOT` | +| `application_name` | `` | `APPLICATION_NAME` | +| `rejoin_allow_basebackup` | `false` | `REJOIN_ALLOW_BASEBACKUP` | + +`application_name` falls back to `hostname -s`. Each parameter has an OCF_RESKEY input and an OCF_RESKEY_*_default global. Other framework globals: OCF_ROOT, OCF_FUNCTIONS_DIR, OCF_RUNNING_PROMOTED (normalized from OCF_RUNNING_MASTER), OCF_SUCCESS, OCF_NOT_RUNNING, OCF_ERR_GENERIC, OCF_ERR_CONFIGURED, OCF_ERR_INSTALLED, OCF_ERR_UNIMPLEMENTED, OCF_RESOURCE_INSTANCE. The validate_all loop also assigns the existing nonlocal variable cmd. + +Notification inputs: OCF_RESKEY_CRM_meta_notify_{type,operation,promote_uname,demote_uname,promoted_uname,unpromoted_uname}. State persists under /run/loxthelion; demote accepts matching pre-demote state aged 0–120 seconds. + +Container inputs: PGDATA, POSTGRES_REPLICATION_PASSWORD, POSTGRES_REPLICATION_USER, POSTGRES_HA_SLOTS. Container command locals include BINDIR, blocks and the explicitly injected RA_PRIMARY_HOST, RA_PRIMARY_PORT, RA_REPLICATION_USER, RA_REPLICATION_SLOT, RA_APPLICATION_NAME. PGPASSWORD is exported only in the container authentication subprocess. The rootless wrapper sets HOME and XDG_RUNTIME_DIR for that subprocess. + +## Actions + +Dispatcher: start, stop, monitor (status alias), promote, demote, notify, rejoin, meta-data, validate-all. Start/promote/demote/rejoin call validate_all first. Metadata advertises eight actions and omits rejoin/status; preserve this exactly. + +## Module ownership and dependencies + +All modules are sourced once by the entrypoint before dispatch. common initializes original defaults. Source order: common, container, postgres, auth, replication, notify, promote, demote, rejoin. Functions resolve dependencies when invoked; common validation can therefore call container functions without a source cycle. + +Initial extraction retains original function names and bodies to enable exact comparison, including existing unused service_home. Namespacing and recovery deduplication are deferred from this first mechanical extraction so it can serve as a verified modular baseline. No shell option, SQL, quoting, log, or error-handling cleanup is included. + +### common.sh + +- `meta_data` → external commands / OCF library only +- `validate_all` → `service_uid`, `as_service_user` + +### container.sh + +- `service_uid` → external commands / OCF library only +- `service_home` → external commands / OCF library only +- `as_service_user` → external commands / OCF library only +- `service_running` → `as_service_user` +- `container_running` → `as_service_user` + +### postgres.sh + +- `pg_command` → `as_service_user` +- `pgdata_path` → `pg_command` +- `pg_running` → `pg_command` +- `pg_ready` → `as_service_user` +- `pg_role` → `as_service_user` +- `pg_wal_receiver_status` → `as_service_user` +- `managed_monitor` → `service_running`, `container_running`, `pg_running`, `pg_ready`, `pg_role` +- `managed_start` → `as_service_user`, `service_running`, `container_running`, `pg_command`, `pg_running`, `managed_monitor` +- `managed_stop` → `as_service_user`, `service_running`, `container_running`, `pg_command`, `pg_running` + +### auth.sh + +- `pg_replication_exec` → `as_service_user` +- `ensure_rewind_privileges` → `pg_command` + +### replication.sh + +- `pg_remote_query` → `pg_replication_exec` +- `ensure_local_ha_slots` → `pg_command` +- `ensure_remote_slot` → `pg_replication_exec` +- `candidate_caught_up` → `pg_command` +- `candidate_is_standby` → `pg_replication_exec` + +### notify.sh + +- `demote_target` → external commands / OCF library only +- `managed_notify` → external commands / OCF library only + +### promote.sh + +- `managed_promote` → `pg_command`, `pg_role`, `managed_monitor`, `ensure_rewind_privileges`, `ensure_local_ha_slots` + +### demote.sh + +- `node_to_addr` → external commands / OCF library only +- `managed_demote` → `as_service_user`, `service_running`, `container_running`, `pg_command`, `pg_running`, `pg_role`, `pg_wal_receiver_status`, `managed_monitor`, `node_to_addr`, `candidate_caught_up`, `demote_target`, `ensure_remote_slot`, `candidate_is_standby` + +### rejoin.sh + +- `managed_rejoin` → `as_service_user`, `service_running`, `container_running`, `pg_command`, `pg_replication_exec`, `pgdata_path`, `pg_running`, `pg_role`, `pg_remote_query`, `pg_wal_receiver_status`, `ensure_remote_slot` + +## Incremental implementation and validation + +1. Extract verbatim functions to the modules above in a new refactored/ tree; preserve both reference files. +2. Keep original OCF initialization and dispatcher; add script-relative module loading with a configured-error failure if any module cannot load. +3. Compare every function body, original defaults and dispatcher against the reference; run bash -n on every shell file. +4. Compare real meta-data output and parsed XML; exercise validate-all and record environmental limitations. +5. Exercise isolated action scenarios using mocked external boundaries; do not operate the live cluster. +6. Report hashes, install commands, and the operator-run lab comparison sequence. diff --git a/notes/REFACTOR-REPORT.md b/notes/REFACTOR-REPORT.md new file mode 100644 index 0000000..97df765 --- /dev/null +++ b/notes/REFACTOR-REPORT.md @@ -0,0 +1,137 @@ +# PostgreSQL HA resource-agent modular refactor + +The refactored agent is in `refactored/managed-container`, with nine sourced modules beside it. Both reference copies remain unchanged at SHA256 `5eae1f3dba0586dc06105e4dd7a9b8809a5c9961452abe7654dcfce77c3c5ce2`. + +## Final file tree + +```text +refactored/ + managed-container + managed-container.d/ + common.sh + container.sh + postgres.sh + auth.sh + replication.sh + notify.sh + promote.sh + demote.sh + rejoin.sh +notes/ + REFACTOR-PLAN.md + REFACTOR-REPORT.md + REFACTOR-SHA256SUMS +tests/ + verify_refactor.py +``` + +The SHA256 manifest lists every delivered code, test, and documentation file (excluding the manifest itself). Existing context files are unchanged. + +## Function assignments + +| Module | Functions | +| --- | --- | +| common.sh | meta_data, validate_all; original parameter defaults and runtime globals | +| container.sh | service_uid, service_home, as_service_user, service_running, container_running | +| postgres.sh | pg_command, pgdata_path, pg_running, pg_ready, pg_role, pg_wal_receiver_status, managed_monitor, managed_start, managed_stop | +| auth.sh | pg_replication_exec, ensure_rewind_privileges | +| replication.sh | pg_remote_query, ensure_local_ha_slots, ensure_remote_slot, candidate_caught_up, candidate_is_standby | +| notify.sh | demote_target, managed_notify | +| promote.sh | managed_promote | +| demote.sh | node_to_addr, managed_demote | +| rejoin.sh | managed_rejoin | + +The entrypoint retains OCF initialization and the original dispatcher. See REFACTOR-PLAN.md for parameters, globals, function dependencies, and source order. + +## Validation results + +Run from the workspace root: + +```bash +python3 tests/verify_refactor.py +sha256sum -c notes/REFACTOR-SHA256SUMS +``` + +- `bash -n`: entrypoint and all nine modules pass. +- All 29 original function definitions are byte-identical. Original defaults/global initialization and dispatcher are unchanged. This also preserves function/variable references inside original code. +- Real OCF-library metadata execution succeeds from `/tmp`; output is byte-identical to the reference and parses as XML with the same 12 parameters and eight advertised actions. +- All modules load in the installed OCF-library environment. The loader uses `command dirname` because this OCF library defines a dirname shell function that does not support `--`. +- Missing module directory produces OCF_ERR_CONFIGURED (6). +- 22 isolated differential scenarios pass, comparing return codes, stdout, and stderr: eight monitor states; three start cases; two demote cases; rejoin while PostgreSQL is running; existing-primary promotion reconciliation; already-stopped stop; and six validate-all paths. +- Direct `validate-all` on both original and refactored agents returns 6 with `User service 'postgres.service' cannot be found for david`. The execution environment cannot validate the target service. Stubbed validation success and missing user/command/runtime/service/Podman failure paths all match. +- No `.pgpass`, PGPASSFILE, or host environment-file sourcing was introduced into the RA. +- Both frozen and working reference hashes still match the handoff. + +No live cluster actions or destructive HA tests were executed. Offline comparisons establish extraction equivalence; they do not replace controlled live validation of a multi-file installation. + +## Behavior differences and limits + +No intended action behavior differences. Function bodies, SQL, quoting, log messages, OCF return paths, authentication, and recovery flow are preserved exactly. The new packaging requires all nine modules beside the executable. Missing/unloadable modules now fail explicitly with configured error. The only added persistent shell global is MC_MODULE_DIR; the loader unsets its loop variable MC_MODULE. + +Original function names and duplicated demote/rejoin recovery configuration remain in this first modular extraction. Namespacing and deduplication were preferences in the handoff; retaining them makes exact body equivalence checkable. The existing unused service_home helper remains too. No global strict-mode options were added. The executable should be installed as a regular file at the public OCF path, with modules in its sibling managed-container.d directory; a symlink elsewhere does not automatically locate modules at its target. + +The bootstrap script's existing pgpass handling and the Quadlet host environment-file injection are outside this RA refactor and were not changed. The metadata continues to omit the dispatched rejoin action and status alias, matching the reference. + +## Installation commands for the second lab node + +Target the handoff's standby, DAVID-ELITEDESK (192.168.1.6). These commands are provided for operator execution and have not been run. Run the first block from the workspace root on the machine holding these files. The staging path must be unused; it is created with mkdir so an existing staging directory stops the sequence. + +```bash +set -e +ssh david@192.168.1.6 'mkdir /tmp/managed-container-refactor' +scp -r refactored david@192.168.1.6:/tmp/managed-container-refactor/ +``` + +Then log into DAVID-ELITEDESK and run the following block. Confirm the saved lab roles still hold before applying. Leave maintenance mode enabled throughout. + +```bash +ssh david@192.168.1.6 +``` + +On DAVID-ELITEDESK: + +```bash +set -e +sudo pcs property config maintenance-mode +# Confirm maintenance-mode=true in the output before continuing. +sudo bash -n /tmp/managed-container-refactor/refactored/managed-container +for module in /tmp/managed-container-refactor/refactored/managed-container.d/*.sh; do + sudo bash -n "$module" +done + +mc_backup_dir="/var/backups/managed-container-$(date -u +%Y%m%d-%H%M%S)" +sudo mkdir -m 0700 "$mc_backup_dir" +sudo cp -a /usr/lib/ocf/resource.d/loxthelion/managed-container "$mc_backup_dir/" +if sudo test -d /usr/lib/ocf/resource.d/loxthelion/managed-container.d; then + sudo cp -a /usr/lib/ocf/resource.d/loxthelion/managed-container.d "$mc_backup_dir/" +fi +sudo install -d -o root -g root -m 0755 /usr/lib/ocf/resource.d/loxthelion/managed-container.d +sudo install -o root -g root -m 0644 \ + /tmp/managed-container-refactor/refactored/managed-container.d/*.sh \ + /usr/lib/ocf/resource.d/loxthelion/managed-container.d/ +sudo install -o root -g root -m 0755 \ + /tmp/managed-container-refactor/refactored/managed-container \ + /usr/lib/ocf/resource.d/loxthelion/managed-container +sudo restorecon -RF /usr/lib/ocf/resource.d/loxthelion/managed-container \ + /usr/lib/ocf/resource.d/loxthelion/managed-container.d +sudo /usr/lib/ocf/resource.d/loxthelion/managed-container meta-data +sudo /usr/lib/ocf/resource.d/loxthelion/managed-container validate-all +printf 'Original installation saved at %s\n' "$mc_backup_dir" +``` + +Module files must be root-owned and not writable by the service user because the RA sources them as root. Install modules before the entrypoint, with resource management still disabled. No service restart is required just to replace the agent files. No commands here disable maintenance mode. + +## Recommended behavioral comparison sequence + +Perform in the isolated lab with maintenance mode retained and a recoverable data baseline. Compare original and refactored agents from equivalent initial states, recording OCF return codes, logs, service/container state, PostgreSQL role, receiver status, slot state and data-directory names. Do not invoke both implementations sequentially against already-mutated state and call that an equivalence test. + +1. Validate metadata/parameters and validate-all on the target node. Run monitor on healthy primary and standby (8 and 0); verify stopped cases return 7. +2. Verify standby start succeeds, primary start is refused, and missing standby.signal prevents startup of stopped PostgreSQL. +3. Verify full stop shuts down PostgreSQL and the maintenance service/container, then restore the standby setup. +4. Promote a healthy standby and repeat promotion on the primary; check configured physical slots and all four rewind grants without deleting inactive slots. +5. Reproduce controlled switchover using the proven lab orchestration: persist a fresh matching pre-demote notification, resolve the successor through Corosync ring0, verify standby/catch-up checks, and coordinate successor promotion so streaming can complete. Repeat in the opposite direction. Keep the maintenance container alive throughout demote. +6. Rejoin a stopped divergent node using successful pg_rewind; verify normalized recovery settings, permissions, recovery=true and streaming. Verify rejoin refuses running PostgreSQL. +7. With a recoverable lab baseline, force rewind failure with fallback disabled; verify failure and no basebackup replacement. +8. Enable fallback for the explicit lab test; force rewind failure and successful basebackup. Verify old PGDATA preservation and streaming standby recovery. +9. Force basebackup failure; verify partial-copy quarantine, restoration of canonical old PGDATA, PostgreSQL stopped, and failure returned. Also exercise source-role/slot/container/free-space preflight failures. +10. Restore the intended DAVID-DV6 primary / DAVID-ELITEDESK standby topology and verify maintenance mode remains true. Decide separately when to deploy beyond this lab. diff --git a/notes/REFACTOR-SHA256SUMS b/notes/REFACTOR-SHA256SUMS new file mode 100644 index 0000000..b7b2942 --- /dev/null +++ b/notes/REFACTOR-SHA256SUMS @@ -0,0 +1,13 @@ +6bdc385dff0e66de372f802a577f67d852f7a2a0e45610865f558ee3b93adf20 notes/REFACTOR-PLAN.md +08135e93b355d2f8239d9abbe0297eafdd6109d025a27d13ab35ccd74811e03a notes/REFACTOR-REPORT.md +c91f09adfdeb45a3be1a72df3f5ca5b02fd4555dec4397ad944eab439ace0b89 refactored/managed-container +93191771d3cb8aefadb2a16d2f89d64a8b6c584071a473439be818237bdf7f17 refactored/managed-container.d/auth.sh +e0bb13e9fda84ba7b657b2e774c08fa64988d463f45c699681e0e6ce49109440 refactored/managed-container.d/common.sh +ac6bc363a2301f1491d162ef609103dbfbceb3b099a3556614c6165952309374 refactored/managed-container.d/container.sh +0af3bb362a303017372c631354ba94ef9e723bf14e64d2eff91b6662e54a5be0 refactored/managed-container.d/demote.sh +6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670 refactored/managed-container.d/notify.sh +310132d8032891727076295f39548b1cc0e51a10301a0ee4d934161f4f903e3e refactored/managed-container.d/postgres.sh +3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58 refactored/managed-container.d/promote.sh +21beea72bcb7f24a51c17ea48666ac87df9436702ad77b166b45f4647e28e8ad refactored/managed-container.d/rejoin.sh +6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547 refactored/managed-container.d/replication.sh +e3bc103b61b317e91d28260cc7bb86a3cc8390854762d6c5b9f325a1a8b1c362 tests/verify_refactor.py diff --git a/notes/SHA256SUMS b/notes/SHA256SUMS new file mode 100644 index 0000000..df99c06 --- /dev/null +++ b/notes/SHA256SUMS @@ -0,0 +1,5 @@ +5eae1f3dba0586dc06105e4dd7a9b8809a5c9961452abe7654dcfce77c3c5ce2 /home/david/codex-postgres-ha-refactor/reference/managed-container +5eae1f3dba0586dc06105e4dd7a9b8809a5c9961452abe7654dcfce77c3c5ce2 /home/david/codex-postgres-ha-refactor/reference/managed-container.pre-refactor-5eae1f3 + +3f1a15f7054fb69b9aa92204de30306fd6053c63560310f4231f314b148e2dcf /home/david/codex-postgres-ha-refactor/runtime-context/postgres-ha-bootstrap +21d9c28704d0dc4a0cb49b8aff553b337455ccb1bd708518042419b85f6ecf12 /home/david/codex-postgres-ha-refactor/runtime-context/postgres.container diff --git a/refactored/managed-container b/refactored/managed-container new file mode 100755 index 0000000..9a99297 --- /dev/null +++ b/refactored/managed-container @@ -0,0 +1,88 @@ +#!/bin/bash + +# OCF Resource Agent: +# loxthelion:managed-container +# +# v0.4 +# Container lifecycle: systemd user Quadlet +# Application profile: PostgreSQL +# +# Version 0.4 adds explicit PostgreSQL demotion scaffolding. +# Healthy primaries report OCF_RUNNING_PROMOTED while healthy standbys +# report OCF_SUCCESS. + +: "${OCF_ROOT:=/usr/lib/ocf}" +OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat" +. "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs" + +# Pacemaker uses "Promoted" terminology, while many OCF shell libraries +# still expose the historical OCF_RUNNING_MASTER constant for return code 8. +# Normalize both forms here so the agent remains portable. +: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}" + + +# Resolve modules relative to this entrypoint, independent of Pacemaker's cwd. +MC_MODULE_DIR="$(CDPATH= cd -- "$(command dirname -- "${BASH_SOURCE[0]}")" && pwd)/managed-container.d" +for MC_MODULE in common container postgres auth replication notify promote demote rejoin; do + if ! . "${MC_MODULE_DIR}/${MC_MODULE}.sh"; then + ocf_exit_reason "Unable to load resource-agent module ${MC_MODULE_DIR}/${MC_MODULE}.sh" + exit "${OCF_ERR_CONFIGURED}" + fi +done +unset MC_MODULE + +case "$1" in + meta-data) + meta_data + exit "${OCF_SUCCESS}" + ;; + + validate-all) + validate_all + exit $? + ;; + + monitor|status) + managed_monitor + exit $? + ;; + + start) + validate_all || exit $? + managed_start + exit $? + ;; + + stop) + managed_stop + exit $? + ;; + + promote) + validate_all || exit $? + managed_promote + exit $? + ;; + + demote) + validate_all || exit $? + managed_demote + exit $? + ;; + + notify) + managed_notify + exit $? + ;; + + rejoin) + validate_all || exit $? + managed_rejoin + exit $? + ;; + + *) + echo "Usage: $0 {start|stop|promote|demote|notify|rejoin|monitor|status|validate-all|meta-data}" + exit "${OCF_ERR_UNIMPLEMENTED}" + ;; +esac diff --git a/refactored/managed-container.d/auth.sh b/refactored/managed-container.d/auth.sh new file mode 100644 index 0000000..8dc19a1 --- /dev/null +++ b/refactored/managed-container.d/auth.sh @@ -0,0 +1,89 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +pg_replication_exec() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + sh -c ' + if [ -z "${POSTGRES_REPLICATION_PASSWORD:-}" ]; then + printf "%s\n" \ + "POSTGRES_REPLICATION_PASSWORD is missing from container environment" \ + >&2 + exit 1 + fi + + export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD}" + + exec "$@" + ' sh "$@" +} + +ensure_rewind_privileges() +{ + local repl_user + + repl_user="$( + pg_command ' + printf "%s" "${POSTGRES_REPLICATION_USER:-}" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to read POSTGRES_REPLICATION_USER from container" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${repl_user}" ]; then + ocf_exit_reason \ + "POSTGRES_REPLICATION_USER is empty" + return "${OCF_ERR_CONFIGURED}" + fi + + case "${repl_user}" in + *[!A-Za-z0-9_]*) + ocf_exit_reason \ + "Invalid replication role name: ${repl_user}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + ocf_log info \ + "Ensuring pg_rewind source privileges for role ${repl_user}" + + if ! pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -v ON_ERROR_STOP=1 \ + -c \" + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_stat_file(text, boolean) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_read_binary_file(text) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_read_binary_file( + text, + bigint, + bigint, + boolean + ) + TO ${repl_user}; + \" + " >/dev/null + then + ocf_exit_reason \ + "Failed ensuring pg_rewind privileges for ${repl_user}" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} diff --git a/refactored/managed-container.d/common.sh b/refactored/managed-container.d/common.sh new file mode 100644 index 0000000..06f45d4 --- /dev/null +++ b/refactored/managed-container.d/common.sh @@ -0,0 +1,227 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +OCF_RESKEY_service_user_default="david" +OCF_RESKEY_service_name_default="postgres.service" +OCF_RESKEY_container_name_default="postgres" +OCF_RESKEY_pg_os_user_default="postgres" +OCF_RESKEY_db_user_default="david" +OCF_RESKEY_db_name_default="david" +OCF_RESKEY_rejoin_source_host_default="" +OCF_RESKEY_rejoin_source_port_default="5432" +OCF_RESKEY_replication_user_default="replicator" +OCF_RESKEY_replication_slot_default="" +OCF_RESKEY_application_name_default="" +OCF_RESKEY_rejoin_allow_basebackup_default="false" + +: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}" +: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}" +: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}" +: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}" +: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}" +: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}" +: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}" +: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}" +: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}" +: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}" +: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}" +: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}" + +SERVICE_USER="${OCF_RESKEY_service_user}" +SERVICE_NAME="${OCF_RESKEY_service_name}" +CONTAINER_NAME="${OCF_RESKEY_container_name}" +PG_OS_USER="${OCF_RESKEY_pg_os_user}" +DB_USER="${OCF_RESKEY_db_user}" +DB_NAME="${OCF_RESKEY_db_name}" +REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}" +REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}" +REPLICATION_USER="${OCF_RESKEY_replication_user}" +REPLICATION_SLOT="${OCF_RESKEY_replication_slot}" +APPLICATION_NAME="${OCF_RESKEY_application_name}" +REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}" + +if [ -z "${APPLICATION_NAME}" ]; then + APPLICATION_NAME="$(hostname -s)" +fi + +meta_data() +{ + cat < + + + 1.0 + + + Manage an application inside a persistent rootless container. + + Version 0.4 implements role-aware monitoring, promotion, and demotion. + The systemd user Quadlet owns the container lifecycle while pg_ctl owns + the PostgreSQL lifecycle inside the container. + + + + Manage PostgreSQL independently inside a persistent container + + + + + + + Linux user owning the rootless systemd/Podman service. + + Rootless service owner + + + + + + systemd user service generated by Quadlet. + + Quadlet service name + + + + + + Podman container containing PostgreSQL. + + Container name + + + + + + Operating-system user running PostgreSQL inside the container. + + PostgreSQL OS user + + + + + + PostgreSQL database user used for health queries. + + Database user + + + + + + PostgreSQL database used for health queries. + + Database name + + + + + + + PostgreSQL primary host used as the source for rejoin operations. + + Rejoin source host + + + + + + PostgreSQL port on the rejoin source. + + Rejoin source port + + + + + + PostgreSQL role used for replication and pg_rewind. + + Replication user + + + + + + Physical replication slot used by this standby. + + Physical replication slot + + + + + + PostgreSQL replication application_name. Defaults to the local hostname. + + Replication application name + + + + + + Allow rejoin to fall back to pg_basebackup when pg_rewind cannot + repair the local PostgreSQL data directory. The old PGDATA is + preserved before replacement. This must only be enabled when the + operator has established that this node may safely discard its + divergent PostgreSQL history. + + Allow pg_basebackup rejoin fallback + + + + + + + + + + + + + + + + +EOF_META +} + +validate_all() +{ + if ! id "${SERVICE_USER}" >/dev/null 2>&1; then + ocf_exit_reason "Service user '${SERVICE_USER}' does not exist" + return "${OCF_ERR_CONFIGURED}" + fi + + for cmd in runuser systemctl podman getent; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + ocf_exit_reason "Required command '${cmd}' is not installed" + return "${OCF_ERR_INSTALLED}" + fi + done + + if [ ! -d "/run/user/$(service_uid)" ]; then + ocf_exit_reason \ + "Runtime directory /run/user/$(service_uid) does not exist" + return "${OCF_ERR_GENERIC}" + fi + + if ! as_service_user \ + systemctl --user cat "${SERVICE_NAME}" >/dev/null 2>&1 + then + ocf_exit_reason \ + "User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}" + return "${OCF_ERR_CONFIGURED}" + fi + + local podman_error + + podman_error="$( + as_service_user podman info 2>&1 >/dev/null + )" + local podman_rc=$? + + if [ "${podman_rc}" -ne 0 ]; then + ocf_exit_reason \ + "Rootless Podman failed for ${SERVICE_USER}: ${podman_error}" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} diff --git a/refactored/managed-container.d/container.sh b/refactored/managed-container.d/container.sh new file mode 100644 index 0000000..0610a70 --- /dev/null +++ b/refactored/managed-container.d/container.sh @@ -0,0 +1,55 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +service_uid() +{ + id -u "${SERVICE_USER}" +} + +service_home() +{ + getent passwd "${SERVICE_USER}" | cut -d: -f6 +} + +as_service_user() +{ + local uid home + + uid="$(id -u "$SERVICE_USER")" || return 1 + home="$(getent passwd "$SERVICE_USER" | cut -d: -f6)" || return 1 + + runuser -u "$SERVICE_USER" -- \ + sh -c ' + home="$1" + runtime="$2" + shift 2 + + cd "$home" || exit 1 + + export HOME="$home" + export XDG_RUNTIME_DIR="$runtime" + + exec "$@" + ' sh \ + "$home" \ + "/run/user/$uid" \ + "$@" +} + +service_running() +{ + as_service_user \ + systemctl --user is-active --quiet "${SERVICE_NAME}" +} + +container_running() +{ + local state + + state="$( + as_service_user podman inspect \ + --format '{{.State.Running}}' \ + "${CONTAINER_NAME}" 2>/dev/null + )" || return 1 + + [ "${state}" = "true" ] +} diff --git a/refactored/managed-container.d/demote.sh b/refactored/managed-container.d/demote.sh new file mode 100644 index 0000000..4aecfa7 --- /dev/null +++ b/refactored/managed-container.d/demote.sh @@ -0,0 +1,202 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +node_to_addr() +{ + local target="${1:-}" + + [ -n "${target}" ] || return 1 + + corosync-cmapctl 2>/dev/null | + awk -v target="${target}" ' + /nodelist\.node\.[0-9]+\.name / { + key=$1 + sub(/\.name$/, "", key) + names[key]=$NF + } + + /nodelist\.node\.[0-9]+\.ring0_addr / { + key=$1 + sub(/\.ring0_addr$/, "", key) + addrs[key]=$NF + } + + END { + for (key in names) { + if (names[key] == target && addrs[key] != "") { + print addrs[key] + exit 0 + } + } + exit 1 + } + ' +} + +managed_demote() +{ + local monitor_rc + local target_node target_addr + local local_node slot_name target_port + local role receiver_status i + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") + ;; + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL is already unpromoted" + return "${OCF_SUCCESS}" + ;; + "${OCF_NOT_RUNNING}") + ocf_exit_reason \ + "Cannot complete demotion because PostgreSQL is stopped" + return "${OCF_ERR_GENERIC}" + ;; + *) + ocf_exit_reason \ + "Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + target_node="$(demote_target)" || { + ocf_exit_reason \ + "No fresh valid Pacemaker pre-demote target is available" + return "${OCF_ERR_GENERIC}" + } + + target_addr="$(node_to_addr "${target_node}")" || { + ocf_exit_reason \ + "Unable to resolve Pacemaker promotion target ${target_node} through Corosync" + return "${OCF_ERR_GENERIC}" + } + + target_port="${REJOIN_SOURCE_PORT:-5432}" + local_node="$(hostname)" + slot_name="ha_$(printf '%s' "${local_node}" | + tr '[:upper:]-' '[:lower:]_')" + + ocf_log info \ + "Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})" + + if ! candidate_is_standby "${target_addr}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not a reachable PostgreSQL standby" + return "${OCF_ERR_GENERIC}" + fi + + if ! candidate_caught_up "${target_node}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then + ocf_exit_reason \ + "Unable to ensure physical replication slot ${slot_name} on ${target_node}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Demotion safety checks passed; stopping PostgreSQL primary" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason \ + "pg_ctl stop failed during controlled demotion" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL remained running after pg_ctl stop" + return "${OCF_ERR_GENERIC}" + fi + + if ! service_running || ! container_running; then + ocf_exit_reason \ + "Maintenance service/container stopped unexpectedly during demotion" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Configuring ${local_node} as cascading standby of ${target_node}" + + if ! as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + --env "RA_PRIMARY_HOST=${target_addr}" \ + --env "RA_PRIMARY_PORT=${target_port}" \ + --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ + --env "RA_REPLICATION_SLOT=${slot_name}" \ + --env "RA_APPLICATION_NAME=${local_node}" \ + "${CONTAINER_NAME}" \ + sh -c ' + set -e + + sed -i \ + -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \ + -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \ + "$PGDATA/postgresql.auto.conf" + + printf "%s +" \ + "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \ + "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \ + >> "$PGDATA/postgresql.auto.conf" + + touch "$PGDATA/standby.signal" + + chmod 600 \ + "$PGDATA/postgresql.auto.conf" \ + "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Failed to configure PostgreSQL as standby of ${target_node}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as cascading standby of ${target_node}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason \ + "PostgreSQL failed to start as cascading standby" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null || true)" + receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" + + if [ "${role}" = "t" ] && + [ "${receiver_status}" = "streaming" ] + then + ocf_log info \ + "Controlled demotion complete: ${local_node} is streaming from ${target_node}" + return "${OCF_SUCCESS}" + fi + + sleep 1 + done + + ocf_exit_reason \ + "PostgreSQL started after demotion but did not become a streaming standby" + return "${OCF_ERR_GENERIC}" +} diff --git a/refactored/managed-container.d/notify.sh b/refactored/managed-container.d/notify.sh new file mode 100644 index 0000000..4702a66 --- /dev/null +++ b/refactored/managed-container.d/notify.sh @@ -0,0 +1,109 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +demote_target() +{ + local state_file + local type operation promote_uname demote_uname local_node + local now mtime age + + state_file="/run/loxthelion/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" + local_node="$(hostname)" + + [ -r "${state_file}" ] || return 1 + + now="$(date +%s)" || return 1 + mtime="$(stat -c %Y "${state_file}" 2>/dev/null)" || return 1 + age=$((now - mtime)) + + [ "${age}" -ge 0 ] || return 1 + [ "${age}" -le 120 ] || return 1 + + type="$( + sed -n 's/^type=//p' "${state_file}" | head -n1 + )" + + operation="$( + sed -n 's/^operation=//p' "${state_file}" | head -n1 + )" + + promote_uname="$( + sed -n 's/^promote_uname=//p' "${state_file}" | head -n1 + )" + + demote_uname="$( + sed -n 's/^demote_uname=//p' "${state_file}" | head -n1 + )" + + [ "${type}" = "pre" ] || return 1 + [ "${operation}" = "demote" ] || return 1 + [ "${demote_uname}" = "${local_node}" ] || return 1 + [ -n "${promote_uname}" ] || return 1 + [ "${promote_uname}" != "${local_node}" ] || return 1 + + case "${promote_uname}" in + *" "*) return 1 ;; + esac + + printf '%s +' "${promote_uname}" +} + +managed_notify() +{ + local state_dir state_file + + state_dir="/run/loxthelion" + state_file="${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" + + ocf_log info \ + "notify: type=${OCF_RESKEY_CRM_meta_notify_type:-unset} operation=${OCF_RESKEY_CRM_meta_notify_operation:-unset}" + + ocf_log info \ + "notify: promote_uname=${OCF_RESKEY_CRM_meta_notify_promote_uname:-none} demote_uname=${OCF_RESKEY_CRM_meta_notify_demote_uname:-none}" + + ocf_log info \ + "notify: promoted_uname=${OCF_RESKEY_CRM_meta_notify_promoted_uname:-none} unpromoted_uname=${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-none}" + + if ! mkdir -p -m 0755 "${state_dir}"; then + ocf_exit_reason \ + "Unable to create notification state directory ${state_dir}" + return "${OCF_ERR_GENERIC}" + fi + + if ! { + printf 'timestamp=%s\n' "$(date --iso-8601=seconds)" + printf 'node=%s\n' "$(hostname)" + printf 'type=%s\n' "${OCF_RESKEY_CRM_meta_notify_type:-}" + printf 'operation=%s\n' "${OCF_RESKEY_CRM_meta_notify_operation:-}" + printf 'promote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" + printf 'demote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" + printf 'promoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" + printf 'unpromoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" + } > "${state_file}" + then + ocf_exit_reason \ + "Unable to write notification state ${state_file}" + return "${OCF_ERR_GENERIC}" + fi + + chmod 0600 "${state_file}" || { + ocf_exit_reason \ + "Unable to secure notification state ${state_file}" + return "${OCF_ERR_GENERIC}" + } + + { + printf '%s node=%s type=%s operation=%s promote=[%s] demote=[%s] promoted=[%s] unpromoted=[%s]\n' \ + "$(date --iso-8601=seconds)" \ + "$(hostname)" \ + "${OCF_RESKEY_CRM_meta_notify_type:-}" \ + "${OCF_RESKEY_CRM_meta_notify_operation:-}" \ + "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" \ + >> "${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify.log" + } || true + + return "${OCF_SUCCESS}" +} diff --git a/refactored/managed-container.d/postgres.sh b/refactored/managed-container.d/postgres.sh new file mode 100644 index 0000000..a77c805 --- /dev/null +++ b/refactored/managed-container.d/postgres.sh @@ -0,0 +1,292 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +pg_command() +{ + local command="$1" + + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + sh -c "${command}" +} + +pgdata_path() +{ + pg_command ' + if [ -z "$PGDATA" ]; then + exit 1 + fi + + printf "%s\n" "$PGDATA" + ' +} + +pg_running() +{ + pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + "$BINDIR/pg_ctl" status -D "$PGDATA" >/dev/null 2>&1 + ' +} + +pg_ready() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + pg_isready -q \ + -U "${DB_USER}" \ + -d "${DB_NAME}" +} + +pg_role() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql \ + -U "${DB_USER}" \ + -d "${DB_NAME}" \ + -Atqc 'SELECT pg_is_in_recovery();' +} + +pg_wal_receiver_status() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql \ + -U "${DB_USER}" \ + -d "${DB_NAME}" \ + -Atqc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;" +} + +managed_monitor() +{ + local role + + if ! service_running; then + ocf_log debug "User service ${SERVICE_NAME} is not active" + return "${OCF_NOT_RUNNING}" + fi + + if ! container_running; then + ocf_log warn \ + "Service ${SERVICE_NAME} is active but container ${CONTAINER_NAME} is not running" + return "${OCF_NOT_RUNNING}" + fi + + if ! pg_running; then + ocf_log debug \ + "Container ${CONTAINER_NAME} is running but PostgreSQL is stopped" + return "${OCF_NOT_RUNNING}" + fi + + if ! pg_ready; then + ocf_exit_reason \ + "PostgreSQL process exists but pg_isready failed" + return "${OCF_ERR_GENERIC}" + fi + + role="$(pg_role 2>/dev/null)" || { + ocf_exit_reason \ + "PostgreSQL is accepting connections but role query failed" + return "${OCF_ERR_GENERIC}" + } + + case "${role}" in + f) + ocf_log debug \ + "PostgreSQL healthy: role=primary (promoted)" + return "${OCF_RUNNING_PROMOTED}" + ;; + t) + ocf_log debug \ + "PostgreSQL healthy: role=standby (unpromoted)" + return "${OCF_SUCCESS}" + ;; + *) + ocf_exit_reason \ + "Unexpected pg_is_in_recovery() result: '${role}'" + return "${OCF_ERR_GENERIC}" + ;; + esac + +} + +managed_start() +{ + local i monitor_rc + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL resource already running unpromoted" + return "${OCF_SUCCESS}" + ;; + + "${OCF_RUNNING_PROMOTED}") + ocf_exit_reason \ + "Refusing start: PostgreSQL is already primary; promotion must occur through promote()" + return "${OCF_ERR_GENERIC}" + ;; + + "${OCF_NOT_RUNNING}") + # Expected for a stopped resource. Continue. + ;; + + *) + ocf_exit_reason \ + "Cannot start unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + ocf_log info \ + "Ensuring container service ${SERVICE_NAME} is running" + + if ! service_running; then + if ! as_service_user \ + systemctl --user start "${SERVICE_NAME}" + then + ocf_exit_reason \ + "Failed to start ${SERVICE_NAME}" + return "${OCF_ERR_GENERIC}" + fi + fi + + for i in $(seq 1 20); do + container_running && break + sleep 0.5 + done + + if ! container_running; then + ocf_exit_reason \ + "Container ${CONTAINER_NAME} did not become running" + return "${OCF_ERR_GENERIC}" + fi + + if ! pg_running; then + # + # OCF promotable-resource rule: + # start() must produce an Unpromoted instance. + # + # A PostgreSQL data directory without standby.signal would start + # writable/primary, so never start it from this action. + # + if ! pg_command ' + test -f "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Refusing start: PGDATA is not configured as a standby (standby.signal missing)" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as an unpromoted standby inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason "pg_ctl start failed" + return "${OCF_ERR_GENERIC}" + fi + fi + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL start completed in unpromoted state" + return "${OCF_SUCCESS}" + ;; + + "${OCF_RUNNING_PROMOTED}") + ocf_exit_reason \ + "PostgreSQL became primary during start; stopping it to preserve OCF role semantics" + + pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' >/dev/null 2>&1 || true + + return "${OCF_ERR_GENERIC}" + ;; + + *) + return "${monitor_rc}" + ;; + esac +} + +managed_stop() +{ + local i + + if container_running && pg_running; then + ocf_log info \ + "Stopping PostgreSQL cleanly inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason "pg_ctl stop failed" + return "${OCF_ERR_GENERIC}" + fi + fi + + if service_running; then + ocf_log info \ + "Stopping container service ${SERVICE_NAME}" + + if ! as_service_user \ + systemctl --user stop "${SERVICE_NAME}" + then + ocf_exit_reason \ + "Failed to stop ${SERVICE_NAME}" + return "${OCF_ERR_GENERIC}" + fi + fi + + for i in $(seq 1 20); do + if ! service_running && ! container_running; then + return "${OCF_SUCCESS}" + fi + sleep 0.5 + done + + if service_running; then + ocf_exit_reason \ + "${SERVICE_NAME} remained active after stop" + return "${OCF_ERR_GENERIC}" + fi + + if container_running; then + ocf_exit_reason \ + "${CONTAINER_NAME} remained running after stop" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} diff --git a/refactored/managed-container.d/promote.sh b/refactored/managed-container.d/promote.sh new file mode 100644 index 0000000..4461f6a --- /dev/null +++ b/refactored/managed-container.d/promote.sh @@ -0,0 +1,94 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +managed_promote() +{ + local monitor_rc role i + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") + ocf_log info \ + "PostgreSQL is already primary; reconciling primary invariants" + + if ! ensure_local_ha_slots; then + ocf_exit_reason \ + "HA slot reconciliation failed on existing primary" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_rewind_privileges; then + ocf_exit_reason \ + "pg_rewind privilege reconciliation failed on existing primary" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" + ;; + + "${OCF_SUCCESS}") + # Healthy standby: continue with promotion. + ;; + + "${OCF_NOT_RUNNING}") + ocf_exit_reason \ + "Cannot promote PostgreSQL because the resource is not running" + return "${OCF_ERR_GENERIC}" + ;; + + *) + ocf_exit_reason \ + "Cannot promote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + ocf_log info \ + "Promoting PostgreSQL standby inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + promote -w + ' + then + ocf_exit_reason "pg_ctl promote failed" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null)" || { + sleep 1 + continue + } + + if [ "${role}" = "f" ]; then + ocf_log info \ + "PostgreSQL became primary; reconciling primary invariants" + + if ! ensure_local_ha_slots; then + ocf_exit_reason \ + "PostgreSQL promoted but HA slot reconciliation failed" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_rewind_privileges; then + ocf_exit_reason \ + "PostgreSQL promoted but pg_rewind privilege reconciliation failed" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "PostgreSQL promotion completed successfully" + return "${OCF_SUCCESS}" + fi + + sleep 1 + done + + ocf_exit_reason \ + "PostgreSQL did not become primary after promotion" + return "${OCF_ERR_GENERIC}" +} diff --git a/refactored/managed-container.d/rejoin.sh b/refactored/managed-container.d/rejoin.sh new file mode 100644 index 0000000..7c500b9 --- /dev/null +++ b/refactored/managed-container.d/rejoin.sh @@ -0,0 +1,343 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +managed_rejoin() +{ + local source_role role receiver_status pgdata i + + case "${REJOIN_ALLOW_BASEBACKUP}" in + true|false|1|0|yes|no) + ;; + *) + ocf_exit_reason "rejoin_allow_basebackup must be true or false" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + if [ -z "${REJOIN_SOURCE_HOST}" ]; then + ocf_exit_reason \ + "rejoin_source_host is required for rejoin" + return "${OCF_ERR_CONFIGURED}" + fi + + if [ -z "${REPLICATION_SLOT}" ]; then + ocf_exit_reason \ + "replication_slot is required for rejoin" + return "${OCF_ERR_CONFIGURED}" + fi + + if ! service_running; then + ocf_exit_reason \ + "Maintenance service ${SERVICE_NAME} must be running before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + if ! container_running; then + ocf_exit_reason \ + "Maintenance container ${CONTAINER_NAME} must be running before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL must be stopped before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}" + + source_role="$(pg_remote_query \ + 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}" + return "${OCF_ERR_GENERIC}" + } + + if [ "${source_role}" != "f" ]; then + ocf_exit_reason \ + "Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" + + if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}" + then + ocf_exit_reason \ + "Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Running pg_rewind against ${REJOIN_SOURCE_HOST}" + + pgdata="$(pgdata_path 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${pgdata}" ]; then + ocf_exit_reason \ + "PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}" + return "${OCF_ERR_GENERIC}" + fi + + if ! pg_replication_exec \ + pg_rewind \ + --target-pgdata="${pgdata}" \ + --source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ + --progress + then + local fallback_enabled source_bytes free_bytes required_bytes + local timestamp preserved_pgdata failed_pgdata source_role_retry + + case "${REJOIN_ALLOW_BASEBACKUP}" in + true|1|yes) + fallback_enabled=true + ;; + *) + fallback_enabled=false + ;; + esac + + if [ "${fallback_enabled}" != "true" ]; then + ocf_exit_reason \ + "pg_rewind failed and rejoin_allow_basebackup is not enabled" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log warn \ + "pg_rewind failed; evaluating pg_basebackup fallback" + + # Revalidate the source immediately before replacing local PGDATA. + source_role_retry="$(pg_remote_query \ + 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to revalidate rejoin source before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + if [ "${source_role_retry}" != "f" ]; then + ocf_exit_reason \ + "Rejoin source is no longer a writable primary; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL became active during rejoin; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if ! service_running || ! container_running; then + ocf_exit_reason \ + "Maintenance service/container is unavailable; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_remote_slot \ + "${REJOIN_SOURCE_HOST}" \ + "${REPLICATION_SLOT}" + then + ocf_exit_reason \ + "Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + # Estimate source database payload and require 2x that amount free. + # This is deliberately conservative and does not pretend to be an + # exact prediction of the resulting PGDATA size. + source_bytes="$(pg_remote_query \ + "SELECT COALESCE(sum(pg_database_size(datname)),0) + FROM pg_database + WHERE datallowconn;" 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to estimate source database size before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + case "${source_bytes}" in + ""|*[!0-9]*) + ocf_exit_reason \ + "Invalid source database size returned before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + ;; + esac + + free_bytes="$( + pg_command ' + blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")" + case "$blocks" in + ""|*[!0-9]*) + exit 1 + ;; + esac + printf "%s\n" "$((blocks * 1024))" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to determine free space for pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + case "${free_bytes}" in + ""|*[!0-9]*) + ocf_exit_reason \ + "Invalid free-space value before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + ;; + esac + + required_bytes=$((source_bytes * 2)) + + if [ "${free_bytes}" -lt "${required_bytes}" ]; then + ocf_exit_reason \ + "Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes}" + return "${OCF_ERR_GENERIC}" + fi + + timestamp="$(date -u +%Y%m%d-%H%M%S)" + preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}" + failed_pgdata="${pgdata}.failed-basebackup-${timestamp}" + + ocf_log warn \ + "Preserving existing PGDATA as ${preserved_pgdata}" + + if ! pg_command " + set -e + + [ -d '${pgdata}' ] + [ ! -e '${preserved_pgdata}' ] + [ ! -e '${failed_pgdata}' ] + + mv '${pgdata}' '${preserved_pgdata}' + install -d -m 0700 '${pgdata}' + " + then + ocf_exit_reason \ + "Unable to preserve existing PGDATA for pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log warn \ + "Running pg_basebackup from ${REJOIN_SOURCE_HOST}" + + if ! pg_replication_exec \ + pg_basebackup \ + --dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \ + --pgdata="${pgdata}" \ + --format=plain \ + --wal-method=stream \ + --slot="${REPLICATION_SLOT}" \ + --write-recovery-conf \ + --progress + then + ocf_log err \ + "pg_basebackup failed; preserving failed copy and restoring previous PGDATA" + + if ! pg_command " + set -e + + if [ -e '${failed_pgdata}' ]; then + exit 1 + fi + + if [ -e '${pgdata}' ]; then + mv '${pgdata}' '${failed_pgdata}' + fi + + mv '${preserved_pgdata}' '${pgdata}' + " + then + ocf_exit_reason \ + "pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required" + return "${OCF_ERR_GENERIC}" + fi + + ocf_exit_reason \ + "pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}" + fi + + ocf_log info \ + "Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}" + + if ! as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + --env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \ + --env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \ + --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ + --env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \ + --env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \ + "${CONTAINER_NAME}" \ + sh -c ' + set -e + + sed -i \ + -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \ + -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \ + "$PGDATA/postgresql.auto.conf" + + printf "%s\n" \ + "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \ + "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \ + >> "$PGDATA/postgresql.auto.conf" + + touch "$PGDATA/standby.signal" + + chmod 600 \ + "$PGDATA/postgresql.auto.conf" \ + "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Failed to configure standby settings" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as standby" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason \ + "PostgreSQL failed to start after rejoin" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null)" || { + sleep 1 + continue + } + + receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" + + if [ "${role}" = "t" ] && + [ "${receiver_status}" = "streaming" ] + then + ocf_log info \ + "PostgreSQL successfully rejoined as streaming standby" + return "${OCF_SUCCESS}" + fi + + sleep 1 + done + + ocf_exit_reason \ + "PostgreSQL started but did not become a streaming standby" + return "${OCF_ERR_GENERIC}" +} diff --git a/refactored/managed-container.d/replication.sh b/refactored/managed-container.d/replication.sh new file mode 100644 index 0000000..e6a7e2f --- /dev/null +++ b/refactored/managed-container.d/replication.sh @@ -0,0 +1,201 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +pg_remote_query() +{ + local sql="$1" + + pg_replication_exec \ + psql \ + "host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ + -Atqc "${sql}" +} + +ensure_local_ha_slots() +{ + local slots raw_slot slot slot_type + + # + # POSTGRES_HA_SLOTS lives in the container environment supplied by + # postgres.env. It is the cluster-wide list of physical slots that + # every writable primary must provide. + # + slots="$( + pg_command ' + printf "%s" "${POSTGRES_HA_SLOTS:-}" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to read POSTGRES_HA_SLOTS from PostgreSQL container" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${slots}" ]; then + ocf_log warn \ + "POSTGRES_HA_SLOTS is empty; no local HA slots will be reconciled" + return "${OCF_SUCCESS}" + fi + + while IFS= read -r raw_slot; do + slot="$(printf '%s' "${raw_slot}" | tr -d '[:space:]')" + + [ -n "${slot}" ] || continue + + case "${slot}" in + *[!a-z0-9_]*) + ocf_exit_reason \ + "Invalid physical replication slot name: ${slot}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + slot_type="$( + pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -At \ + -v ON_ERROR_STOP=1 \ + -c \" + SELECT slot_type + FROM pg_replication_slots + WHERE slot_name = '${slot}'; + \" + " 2>/dev/null + )" || { + ocf_exit_reason \ + "Failed querying replication slot ${slot}" + return "${OCF_ERR_GENERIC}" + } + + slot_type="$(printf '%s' "${slot_type}" | tr -d '[:space:]')" + + case "${slot_type}" in + physical) + ocf_log info \ + "Physical HA slot already exists: ${slot}" + ;; + + "") + ocf_log info \ + "Creating physical HA slot: ${slot}" + + if ! pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -v ON_ERROR_STOP=1 \ + -c \" + SELECT pg_create_physical_replication_slot('${slot}'); + \" + " >/dev/null + then + ocf_exit_reason \ + "Failed creating physical replication slot ${slot}" + return "${OCF_ERR_GENERIC}" + fi + ;; + + *) + ocf_exit_reason \ + "Replication slot ${slot} exists but is type ${slot_type}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + done </dev/null + )" || return 1 + + case "${result}" in + physical) + return 0 + ;; + + "") + if ! pg_replication_exec \ + psql \ + "host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \ + -Atc " + SELECT slot_name + FROM pg_create_physical_replication_slot('${slot}'); + " >/dev/null + then + return 1 + fi + + return 0 + ;; + + *) + ocf_log err \ + "Replication slot ${slot} exists on ${host} but is not physical" + return 1 + ;; + esac +} + +candidate_caught_up() +{ + local candidate="${1:-}" + local result + + [ -n "${candidate}" ] || return 1 + + result="$(pg_command " + psql -U '${DB_USER}' -d '${DB_NAME}' -Atc \" + SELECT CASE + WHEN state = 'streaming' + AND replay_lsn >= pg_current_wal_flush_lsn() + THEN 'yes' + ELSE 'no' + END + FROM pg_stat_replication + WHERE application_name = '${candidate}' + LIMIT 1; + \" + " 2>/dev/null)" || return 1 + + [ "${result}" = "yes" ] +} + +candidate_is_standby() +{ + local host="${1:-}" + local result + local port="${REJOIN_SOURCE_PORT:-5432}" + + [ -n "${host}" ] || return 1 + + result="$( + pg_replication_exec \ + psql \ + "host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres" \ + -Atc "SELECT pg_is_in_recovery();" \ + 2>/dev/null + )" || return 1 + + [ "${result}" = "t" ] +} diff --git a/reference/managed-container b/reference/managed-container new file mode 100755 index 0000000..6c6dc09 --- /dev/null +++ b/reference/managed-container @@ -0,0 +1,1721 @@ +#!/bin/bash + +# OCF Resource Agent: +# loxthelion:managed-container +# +# v0.4 +# Container lifecycle: systemd user Quadlet +# Application profile: PostgreSQL +# +# Version 0.4 adds explicit PostgreSQL demotion scaffolding. +# Healthy primaries report OCF_RUNNING_PROMOTED while healthy standbys +# report OCF_SUCCESS. + +: "${OCF_ROOT:=/usr/lib/ocf}" +OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat" +. "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs" + +# Pacemaker uses "Promoted" terminology, while many OCF shell libraries +# still expose the historical OCF_RUNNING_MASTER constant for return code 8. +# Normalize both forms here so the agent remains portable. +: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}" + +############################################################################### +# Defaults +############################################################################### + +OCF_RESKEY_service_user_default="david" +OCF_RESKEY_service_name_default="postgres.service" +OCF_RESKEY_container_name_default="postgres" +OCF_RESKEY_pg_os_user_default="postgres" +OCF_RESKEY_db_user_default="david" +OCF_RESKEY_db_name_default="david" +OCF_RESKEY_rejoin_source_host_default="" +OCF_RESKEY_rejoin_source_port_default="5432" +OCF_RESKEY_replication_user_default="replicator" +OCF_RESKEY_replication_slot_default="" +OCF_RESKEY_application_name_default="" +OCF_RESKEY_rejoin_allow_basebackup_default="false" + +: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}" +: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}" +: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}" +: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}" +: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}" +: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}" +: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}" +: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}" +: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}" +: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}" +: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}" +: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}" + +SERVICE_USER="${OCF_RESKEY_service_user}" +SERVICE_NAME="${OCF_RESKEY_service_name}" +CONTAINER_NAME="${OCF_RESKEY_container_name}" +PG_OS_USER="${OCF_RESKEY_pg_os_user}" +DB_USER="${OCF_RESKEY_db_user}" +DB_NAME="${OCF_RESKEY_db_name}" +REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}" +REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}" +REPLICATION_USER="${OCF_RESKEY_replication_user}" +REPLICATION_SLOT="${OCF_RESKEY_replication_slot}" +APPLICATION_NAME="${OCF_RESKEY_application_name}" +REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}" + +if [ -z "${APPLICATION_NAME}" ]; then + APPLICATION_NAME="$(hostname -s)" +fi + +############################################################################### +# Metadata +############################################################################### + +meta_data() +{ + cat < + + + 1.0 + + + Manage an application inside a persistent rootless container. + + Version 0.4 implements role-aware monitoring, promotion, and demotion. + The systemd user Quadlet owns the container lifecycle while pg_ctl owns + the PostgreSQL lifecycle inside the container. + + + + Manage PostgreSQL independently inside a persistent container + + + + + + + Linux user owning the rootless systemd/Podman service. + + Rootless service owner + + + + + + systemd user service generated by Quadlet. + + Quadlet service name + + + + + + Podman container containing PostgreSQL. + + Container name + + + + + + Operating-system user running PostgreSQL inside the container. + + PostgreSQL OS user + + + + + + PostgreSQL database user used for health queries. + + Database user + + + + + + PostgreSQL database used for health queries. + + Database name + + + + + + + PostgreSQL primary host used as the source for rejoin operations. + + Rejoin source host + + + + + + PostgreSQL port on the rejoin source. + + Rejoin source port + + + + + + PostgreSQL role used for replication and pg_rewind. + + Replication user + + + + + + Physical replication slot used by this standby. + + Physical replication slot + + + + + + PostgreSQL replication application_name. Defaults to the local hostname. + + Replication application name + + + + + + Allow rejoin to fall back to pg_basebackup when pg_rewind cannot + repair the local PostgreSQL data directory. The old PGDATA is + preserved before replacement. This must only be enabled when the + operator has established that this node may safely discard its + divergent PostgreSQL history. + + Allow pg_basebackup rejoin fallback + + + + + + + + + + + + + + + + +EOF_META +} + +############################################################################### +# Rootless-user helpers +############################################################################### + +service_uid() +{ + id -u "${SERVICE_USER}" +} + +service_home() +{ + getent passwd "${SERVICE_USER}" | cut -d: -f6 +} + +as_service_user() +{ + local uid home + + uid="$(id -u "$SERVICE_USER")" || return 1 + home="$(getent passwd "$SERVICE_USER" | cut -d: -f6)" || return 1 + + runuser -u "$SERVICE_USER" -- \ + sh -c ' + home="$1" + runtime="$2" + shift 2 + + cd "$home" || exit 1 + + export HOME="$home" + export XDG_RUNTIME_DIR="$runtime" + + exec "$@" + ' sh \ + "$home" \ + "/run/user/$uid" \ + "$@" +} + +############################################################################### +# Container / application status +############################################################################### + +service_running() +{ + as_service_user \ + systemctl --user is-active --quiet "${SERVICE_NAME}" +} + +container_running() +{ + local state + + state="$( + as_service_user podman inspect \ + --format '{{.State.Running}}' \ + "${CONTAINER_NAME}" 2>/dev/null + )" || return 1 + + [ "${state}" = "true" ] +} + +pg_command() +{ + local command="$1" + + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + sh -c "${command}" +} + +pg_replication_exec() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + sh -c ' + if [ -z "${POSTGRES_REPLICATION_PASSWORD:-}" ]; then + printf "%s\n" \ + "POSTGRES_REPLICATION_PASSWORD is missing from container environment" \ + >&2 + exit 1 + fi + + export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD}" + + exec "$@" + ' sh "$@" +} + +pgdata_path() +{ + pg_command ' + if [ -z "$PGDATA" ]; then + exit 1 + fi + + printf "%s\n" "$PGDATA" + ' +} + +pg_running() +{ + pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + "$BINDIR/pg_ctl" status -D "$PGDATA" >/dev/null 2>&1 + ' +} + +pg_ready() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + pg_isready -q \ + -U "${DB_USER}" \ + -d "${DB_NAME}" +} + +pg_role() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql \ + -U "${DB_USER}" \ + -d "${DB_NAME}" \ + -Atqc 'SELECT pg_is_in_recovery();' +} + + +pg_remote_query() +{ + local sql="$1" + + pg_replication_exec \ + psql \ + "host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ + -Atqc "${sql}" +} + +pg_wal_receiver_status() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql \ + -U "${DB_USER}" \ + -d "${DB_NAME}" \ + -Atqc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;" +} + +############################################################################### +# Validation +############################################################################### + +validate_all() +{ + if ! id "${SERVICE_USER}" >/dev/null 2>&1; then + ocf_exit_reason "Service user '${SERVICE_USER}' does not exist" + return "${OCF_ERR_CONFIGURED}" + fi + + for cmd in runuser systemctl podman getent; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + ocf_exit_reason "Required command '${cmd}' is not installed" + return "${OCF_ERR_INSTALLED}" + fi + done + + if [ ! -d "/run/user/$(service_uid)" ]; then + ocf_exit_reason \ + "Runtime directory /run/user/$(service_uid) does not exist" + return "${OCF_ERR_GENERIC}" + fi + + if ! as_service_user \ + systemctl --user cat "${SERVICE_NAME}" >/dev/null 2>&1 + then + ocf_exit_reason \ + "User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}" + return "${OCF_ERR_CONFIGURED}" + fi + + local podman_error + + podman_error="$( + as_service_user podman info 2>&1 >/dev/null + )" + local podman_rc=$? + + if [ "${podman_rc}" -ne 0 ]; then + ocf_exit_reason \ + "Rootless Podman failed for ${SERVICE_USER}: ${podman_error}" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + +############################################################################### +# Monitor +############################################################################### + +managed_monitor() +{ + local role + + if ! service_running; then + ocf_log debug "User service ${SERVICE_NAME} is not active" + return "${OCF_NOT_RUNNING}" + fi + + if ! container_running; then + ocf_log warn \ + "Service ${SERVICE_NAME} is active but container ${CONTAINER_NAME} is not running" + return "${OCF_NOT_RUNNING}" + fi + + if ! pg_running; then + ocf_log debug \ + "Container ${CONTAINER_NAME} is running but PostgreSQL is stopped" + return "${OCF_NOT_RUNNING}" + fi + + if ! pg_ready; then + ocf_exit_reason \ + "PostgreSQL process exists but pg_isready failed" + return "${OCF_ERR_GENERIC}" + fi + + role="$(pg_role 2>/dev/null)" || { + ocf_exit_reason \ + "PostgreSQL is accepting connections but role query failed" + return "${OCF_ERR_GENERIC}" + } + + case "${role}" in + f) + ocf_log debug \ + "PostgreSQL healthy: role=primary (promoted)" + return "${OCF_RUNNING_PROMOTED}" + ;; + t) + ocf_log debug \ + "PostgreSQL healthy: role=standby (unpromoted)" + return "${OCF_SUCCESS}" + ;; + *) + ocf_exit_reason \ + "Unexpected pg_is_in_recovery() result: '${role}'" + return "${OCF_ERR_GENERIC}" + ;; + esac + +} + +############################################################################### +# Start +############################################################################### + +managed_start() +{ + local i monitor_rc + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL resource already running unpromoted" + return "${OCF_SUCCESS}" + ;; + + "${OCF_RUNNING_PROMOTED}") + ocf_exit_reason \ + "Refusing start: PostgreSQL is already primary; promotion must occur through promote()" + return "${OCF_ERR_GENERIC}" + ;; + + "${OCF_NOT_RUNNING}") + # Expected for a stopped resource. Continue. + ;; + + *) + ocf_exit_reason \ + "Cannot start unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + ocf_log info \ + "Ensuring container service ${SERVICE_NAME} is running" + + if ! service_running; then + if ! as_service_user \ + systemctl --user start "${SERVICE_NAME}" + then + ocf_exit_reason \ + "Failed to start ${SERVICE_NAME}" + return "${OCF_ERR_GENERIC}" + fi + fi + + for i in $(seq 1 20); do + container_running && break + sleep 0.5 + done + + if ! container_running; then + ocf_exit_reason \ + "Container ${CONTAINER_NAME} did not become running" + return "${OCF_ERR_GENERIC}" + fi + + if ! pg_running; then + # + # OCF promotable-resource rule: + # start() must produce an Unpromoted instance. + # + # A PostgreSQL data directory without standby.signal would start + # writable/primary, so never start it from this action. + # + if ! pg_command ' + test -f "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Refusing start: PGDATA is not configured as a standby (standby.signal missing)" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as an unpromoted standby inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason "pg_ctl start failed" + return "${OCF_ERR_GENERIC}" + fi + fi + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL start completed in unpromoted state" + return "${OCF_SUCCESS}" + ;; + + "${OCF_RUNNING_PROMOTED}") + ocf_exit_reason \ + "PostgreSQL became primary during start; stopping it to preserve OCF role semantics" + + pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' >/dev/null 2>&1 || true + + return "${OCF_ERR_GENERIC}" + ;; + + *) + return "${monitor_rc}" + ;; + esac +} + +ensure_rewind_privileges() +{ + local repl_user + + repl_user="$( + pg_command ' + printf "%s" "${POSTGRES_REPLICATION_USER:-}" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to read POSTGRES_REPLICATION_USER from container" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${repl_user}" ]; then + ocf_exit_reason \ + "POSTGRES_REPLICATION_USER is empty" + return "${OCF_ERR_CONFIGURED}" + fi + + case "${repl_user}" in + *[!A-Za-z0-9_]*) + ocf_exit_reason \ + "Invalid replication role name: ${repl_user}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + ocf_log info \ + "Ensuring pg_rewind source privileges for role ${repl_user}" + + if ! pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -v ON_ERROR_STOP=1 \ + -c \" + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_stat_file(text, boolean) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_read_binary_file(text) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_read_binary_file( + text, + bigint, + bigint, + boolean + ) + TO ${repl_user}; + \" + " >/dev/null + then + ocf_exit_reason \ + "Failed ensuring pg_rewind privileges for ${repl_user}" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + + +ensure_local_ha_slots() +{ + local slots raw_slot slot slot_type + + # + # POSTGRES_HA_SLOTS lives in the container environment supplied by + # postgres.env. It is the cluster-wide list of physical slots that + # every writable primary must provide. + # + slots="$( + pg_command ' + printf "%s" "${POSTGRES_HA_SLOTS:-}" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to read POSTGRES_HA_SLOTS from PostgreSQL container" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${slots}" ]; then + ocf_log warn \ + "POSTGRES_HA_SLOTS is empty; no local HA slots will be reconciled" + return "${OCF_SUCCESS}" + fi + + while IFS= read -r raw_slot; do + slot="$(printf '%s' "${raw_slot}" | tr -d '[:space:]')" + + [ -n "${slot}" ] || continue + + case "${slot}" in + *[!a-z0-9_]*) + ocf_exit_reason \ + "Invalid physical replication slot name: ${slot}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + slot_type="$( + pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -At \ + -v ON_ERROR_STOP=1 \ + -c \" + SELECT slot_type + FROM pg_replication_slots + WHERE slot_name = '${slot}'; + \" + " 2>/dev/null + )" || { + ocf_exit_reason \ + "Failed querying replication slot ${slot}" + return "${OCF_ERR_GENERIC}" + } + + slot_type="$(printf '%s' "${slot_type}" | tr -d '[:space:]')" + + case "${slot_type}" in + physical) + ocf_log info \ + "Physical HA slot already exists: ${slot}" + ;; + + "") + ocf_log info \ + "Creating physical HA slot: ${slot}" + + if ! pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -v ON_ERROR_STOP=1 \ + -c \" + SELECT pg_create_physical_replication_slot('${slot}'); + \" + " >/dev/null + then + ocf_exit_reason \ + "Failed creating physical replication slot ${slot}" + return "${OCF_ERR_GENERIC}" + fi + ;; + + *) + ocf_exit_reason \ + "Replication slot ${slot} exists but is type ${slot_type}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + done </dev/null | + awk -v target="${target}" ' + /nodelist\.node\.[0-9]+\.name / { + key=$1 + sub(/\.name$/, "", key) + names[key]=$NF + } + + /nodelist\.node\.[0-9]+\.ring0_addr / { + key=$1 + sub(/\.ring0_addr$/, "", key) + addrs[key]=$NF + } + + END { + for (key in names) { + if (names[key] == target && addrs[key] != "") { + print addrs[key] + exit 0 + } + } + exit 1 + } + ' +} + +candidate_caught_up() +{ + local candidate="${1:-}" + local result + + [ -n "${candidate}" ] || return 1 + + result="$(pg_command " + psql -U '${DB_USER}' -d '${DB_NAME}' -Atc \" + SELECT CASE + WHEN state = 'streaming' + AND replay_lsn >= pg_current_wal_flush_lsn() + THEN 'yes' + ELSE 'no' + END + FROM pg_stat_replication + WHERE application_name = '${candidate}' + LIMIT 1; + \" + " 2>/dev/null)" || return 1 + + [ "${result}" = "yes" ] +} + +demote_target() +{ + local state_file + local type operation promote_uname demote_uname local_node + local now mtime age + + state_file="/run/loxthelion/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" + local_node="$(hostname)" + + [ -r "${state_file}" ] || return 1 + + now="$(date +%s)" || return 1 + mtime="$(stat -c %Y "${state_file}" 2>/dev/null)" || return 1 + age=$((now - mtime)) + + [ "${age}" -ge 0 ] || return 1 + [ "${age}" -le 120 ] || return 1 + + type="$( + sed -n 's/^type=//p' "${state_file}" | head -n1 + )" + + operation="$( + sed -n 's/^operation=//p' "${state_file}" | head -n1 + )" + + promote_uname="$( + sed -n 's/^promote_uname=//p' "${state_file}" | head -n1 + )" + + demote_uname="$( + sed -n 's/^demote_uname=//p' "${state_file}" | head -n1 + )" + + [ "${type}" = "pre" ] || return 1 + [ "${operation}" = "demote" ] || return 1 + [ "${demote_uname}" = "${local_node}" ] || return 1 + [ -n "${promote_uname}" ] || return 1 + [ "${promote_uname}" != "${local_node}" ] || return 1 + + case "${promote_uname}" in + *" "*) return 1 ;; + esac + + printf '%s +' "${promote_uname}" +} + + +ensure_remote_slot() +{ + local host="${1:-}" + local slot="${2:-}" + local result + + [ -n "${host}" ] || return 1 + [ -n "${slot}" ] || return 1 + + result="$( + pg_replication_exec \ + psql \ + "host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \ + -Atc " + SELECT slot_type + FROM pg_replication_slots + WHERE slot_name = '${slot}'; + " 2>/dev/null + )" || return 1 + + case "${result}" in + physical) + return 0 + ;; + + "") + if ! pg_replication_exec \ + psql \ + "host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \ + -Atc " + SELECT slot_name + FROM pg_create_physical_replication_slot('${slot}'); + " >/dev/null + then + return 1 + fi + + return 0 + ;; + + *) + ocf_log err \ + "Replication slot ${slot} exists on ${host} but is not physical" + return 1 + ;; + esac +} + +candidate_is_standby() +{ + local host="${1:-}" + local result + local port="${REJOIN_SOURCE_PORT:-5432}" + + [ -n "${host}" ] || return 1 + + result="$( + pg_replication_exec \ + psql \ + "host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres" \ + -Atc "SELECT pg_is_in_recovery();" \ + 2>/dev/null + )" || return 1 + + [ "${result}" = "t" ] +} + +managed_demote() +{ + local monitor_rc + local target_node target_addr + local local_node slot_name target_port + local role receiver_status i + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") + ;; + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL is already unpromoted" + return "${OCF_SUCCESS}" + ;; + "${OCF_NOT_RUNNING}") + ocf_exit_reason \ + "Cannot complete demotion because PostgreSQL is stopped" + return "${OCF_ERR_GENERIC}" + ;; + *) + ocf_exit_reason \ + "Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + target_node="$(demote_target)" || { + ocf_exit_reason \ + "No fresh valid Pacemaker pre-demote target is available" + return "${OCF_ERR_GENERIC}" + } + + target_addr="$(node_to_addr "${target_node}")" || { + ocf_exit_reason \ + "Unable to resolve Pacemaker promotion target ${target_node} through Corosync" + return "${OCF_ERR_GENERIC}" + } + + target_port="${REJOIN_SOURCE_PORT:-5432}" + local_node="$(hostname)" + slot_name="ha_$(printf '%s' "${local_node}" | + tr '[:upper:]-' '[:lower:]_')" + + ocf_log info \ + "Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})" + + if ! candidate_is_standby "${target_addr}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not a reachable PostgreSQL standby" + return "${OCF_ERR_GENERIC}" + fi + + if ! candidate_caught_up "${target_node}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then + ocf_exit_reason \ + "Unable to ensure physical replication slot ${slot_name} on ${target_node}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Demotion safety checks passed; stopping PostgreSQL primary" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason \ + "pg_ctl stop failed during controlled demotion" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL remained running after pg_ctl stop" + return "${OCF_ERR_GENERIC}" + fi + + if ! service_running || ! container_running; then + ocf_exit_reason \ + "Maintenance service/container stopped unexpectedly during demotion" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Configuring ${local_node} as cascading standby of ${target_node}" + + if ! as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + --env "RA_PRIMARY_HOST=${target_addr}" \ + --env "RA_PRIMARY_PORT=${target_port}" \ + --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ + --env "RA_REPLICATION_SLOT=${slot_name}" \ + --env "RA_APPLICATION_NAME=${local_node}" \ + "${CONTAINER_NAME}" \ + sh -c ' + set -e + + sed -i \ + -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \ + -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \ + "$PGDATA/postgresql.auto.conf" + + printf "%s +" \ + "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \ + "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \ + >> "$PGDATA/postgresql.auto.conf" + + touch "$PGDATA/standby.signal" + + chmod 600 \ + "$PGDATA/postgresql.auto.conf" \ + "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Failed to configure PostgreSQL as standby of ${target_node}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as cascading standby of ${target_node}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason \ + "PostgreSQL failed to start as cascading standby" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null || true)" + receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" + + if [ "${role}" = "t" ] && + [ "${receiver_status}" = "streaming" ] + then + ocf_log info \ + "Controlled demotion complete: ${local_node} is streaming from ${target_node}" + return "${OCF_SUCCESS}" + fi + + sleep 1 + done + + ocf_exit_reason \ + "PostgreSQL started after demotion but did not become a streaming standby" + return "${OCF_ERR_GENERIC}" +} + + +managed_rejoin() +{ + local source_role role receiver_status pgdata i + + case "${REJOIN_ALLOW_BASEBACKUP}" in + true|false|1|0|yes|no) + ;; + *) + ocf_exit_reason "rejoin_allow_basebackup must be true or false" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + if [ -z "${REJOIN_SOURCE_HOST}" ]; then + ocf_exit_reason \ + "rejoin_source_host is required for rejoin" + return "${OCF_ERR_CONFIGURED}" + fi + + if [ -z "${REPLICATION_SLOT}" ]; then + ocf_exit_reason \ + "replication_slot is required for rejoin" + return "${OCF_ERR_CONFIGURED}" + fi + + if ! service_running; then + ocf_exit_reason \ + "Maintenance service ${SERVICE_NAME} must be running before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + if ! container_running; then + ocf_exit_reason \ + "Maintenance container ${CONTAINER_NAME} must be running before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL must be stopped before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}" + + source_role="$(pg_remote_query \ + 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}" + return "${OCF_ERR_GENERIC}" + } + + if [ "${source_role}" != "f" ]; then + ocf_exit_reason \ + "Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" + + if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}" + then + ocf_exit_reason \ + "Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Running pg_rewind against ${REJOIN_SOURCE_HOST}" + + pgdata="$(pgdata_path 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${pgdata}" ]; then + ocf_exit_reason \ + "PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}" + return "${OCF_ERR_GENERIC}" + fi + + if ! pg_replication_exec \ + pg_rewind \ + --target-pgdata="${pgdata}" \ + --source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ + --progress + then + local fallback_enabled source_bytes free_bytes required_bytes + local timestamp preserved_pgdata failed_pgdata source_role_retry + + case "${REJOIN_ALLOW_BASEBACKUP}" in + true|1|yes) + fallback_enabled=true + ;; + *) + fallback_enabled=false + ;; + esac + + if [ "${fallback_enabled}" != "true" ]; then + ocf_exit_reason \ + "pg_rewind failed and rejoin_allow_basebackup is not enabled" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log warn \ + "pg_rewind failed; evaluating pg_basebackup fallback" + + # Revalidate the source immediately before replacing local PGDATA. + source_role_retry="$(pg_remote_query \ + 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to revalidate rejoin source before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + if [ "${source_role_retry}" != "f" ]; then + ocf_exit_reason \ + "Rejoin source is no longer a writable primary; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL became active during rejoin; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if ! service_running || ! container_running; then + ocf_exit_reason \ + "Maintenance service/container is unavailable; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_remote_slot \ + "${REJOIN_SOURCE_HOST}" \ + "${REPLICATION_SLOT}" + then + ocf_exit_reason \ + "Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + # Estimate source database payload and require 2x that amount free. + # This is deliberately conservative and does not pretend to be an + # exact prediction of the resulting PGDATA size. + source_bytes="$(pg_remote_query \ + "SELECT COALESCE(sum(pg_database_size(datname)),0) + FROM pg_database + WHERE datallowconn;" 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to estimate source database size before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + case "${source_bytes}" in + ""|*[!0-9]*) + ocf_exit_reason \ + "Invalid source database size returned before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + ;; + esac + + free_bytes="$( + pg_command ' + blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")" + case "$blocks" in + ""|*[!0-9]*) + exit 1 + ;; + esac + printf "%s\n" "$((blocks * 1024))" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to determine free space for pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + case "${free_bytes}" in + ""|*[!0-9]*) + ocf_exit_reason \ + "Invalid free-space value before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + ;; + esac + + required_bytes=$((source_bytes * 2)) + + if [ "${free_bytes}" -lt "${required_bytes}" ]; then + ocf_exit_reason \ + "Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes}" + return "${OCF_ERR_GENERIC}" + fi + + timestamp="$(date -u +%Y%m%d-%H%M%S)" + preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}" + failed_pgdata="${pgdata}.failed-basebackup-${timestamp}" + + ocf_log warn \ + "Preserving existing PGDATA as ${preserved_pgdata}" + + if ! pg_command " + set -e + + [ -d '${pgdata}' ] + [ ! -e '${preserved_pgdata}' ] + [ ! -e '${failed_pgdata}' ] + + mv '${pgdata}' '${preserved_pgdata}' + install -d -m 0700 '${pgdata}' + " + then + ocf_exit_reason \ + "Unable to preserve existing PGDATA for pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log warn \ + "Running pg_basebackup from ${REJOIN_SOURCE_HOST}" + + if ! pg_replication_exec \ + pg_basebackup \ + --dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \ + --pgdata="${pgdata}" \ + --format=plain \ + --wal-method=stream \ + --slot="${REPLICATION_SLOT}" \ + --write-recovery-conf \ + --progress + then + ocf_log err \ + "pg_basebackup failed; preserving failed copy and restoring previous PGDATA" + + if ! pg_command " + set -e + + if [ -e '${failed_pgdata}' ]; then + exit 1 + fi + + if [ -e '${pgdata}' ]; then + mv '${pgdata}' '${failed_pgdata}' + fi + + mv '${preserved_pgdata}' '${pgdata}' + " + then + ocf_exit_reason \ + "pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required" + return "${OCF_ERR_GENERIC}" + fi + + ocf_exit_reason \ + "pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}" + fi + + ocf_log info \ + "Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}" + + if ! as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + --env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \ + --env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \ + --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ + --env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \ + --env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \ + "${CONTAINER_NAME}" \ + sh -c ' + set -e + + sed -i \ + -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \ + -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \ + "$PGDATA/postgresql.auto.conf" + + printf "%s\n" \ + "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \ + "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \ + >> "$PGDATA/postgresql.auto.conf" + + touch "$PGDATA/standby.signal" + + chmod 600 \ + "$PGDATA/postgresql.auto.conf" \ + "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Failed to configure standby settings" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as standby" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason \ + "PostgreSQL failed to start after rejoin" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null)" || { + sleep 1 + continue + } + + receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" + + if [ "${role}" = "t" ] && + [ "${receiver_status}" = "streaming" ] + then + ocf_log info \ + "PostgreSQL successfully rejoined as streaming standby" + return "${OCF_SUCCESS}" + fi + + sleep 1 + done + + ocf_exit_reason \ + "PostgreSQL started but did not become a streaming standby" + return "${OCF_ERR_GENERIC}" +} + +managed_stop() +{ + local i + + if container_running && pg_running; then + ocf_log info \ + "Stopping PostgreSQL cleanly inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason "pg_ctl stop failed" + return "${OCF_ERR_GENERIC}" + fi + fi + + if service_running; then + ocf_log info \ + "Stopping container service ${SERVICE_NAME}" + + if ! as_service_user \ + systemctl --user stop "${SERVICE_NAME}" + then + ocf_exit_reason \ + "Failed to stop ${SERVICE_NAME}" + return "${OCF_ERR_GENERIC}" + fi + fi + + for i in $(seq 1 20); do + if ! service_running && ! container_running; then + return "${OCF_SUCCESS}" + fi + sleep 0.5 + done + + if service_running; then + ocf_exit_reason \ + "${SERVICE_NAME} remained active after stop" + return "${OCF_ERR_GENERIC}" + fi + + if container_running; then + ocf_exit_reason \ + "${CONTAINER_NAME} remained running after stop" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + +managed_notify() +{ + local state_dir state_file + + state_dir="/run/loxthelion" + state_file="${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" + + ocf_log info \ + "notify: type=${OCF_RESKEY_CRM_meta_notify_type:-unset} operation=${OCF_RESKEY_CRM_meta_notify_operation:-unset}" + + ocf_log info \ + "notify: promote_uname=${OCF_RESKEY_CRM_meta_notify_promote_uname:-none} demote_uname=${OCF_RESKEY_CRM_meta_notify_demote_uname:-none}" + + ocf_log info \ + "notify: promoted_uname=${OCF_RESKEY_CRM_meta_notify_promoted_uname:-none} unpromoted_uname=${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-none}" + + if ! mkdir -p -m 0755 "${state_dir}"; then + ocf_exit_reason \ + "Unable to create notification state directory ${state_dir}" + return "${OCF_ERR_GENERIC}" + fi + + if ! { + printf 'timestamp=%s\n' "$(date --iso-8601=seconds)" + printf 'node=%s\n' "$(hostname)" + printf 'type=%s\n' "${OCF_RESKEY_CRM_meta_notify_type:-}" + printf 'operation=%s\n' "${OCF_RESKEY_CRM_meta_notify_operation:-}" + printf 'promote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" + printf 'demote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" + printf 'promoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" + printf 'unpromoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" + } > "${state_file}" + then + ocf_exit_reason \ + "Unable to write notification state ${state_file}" + return "${OCF_ERR_GENERIC}" + fi + + chmod 0600 "${state_file}" || { + ocf_exit_reason \ + "Unable to secure notification state ${state_file}" + return "${OCF_ERR_GENERIC}" + } + + { + printf '%s node=%s type=%s operation=%s promote=[%s] demote=[%s] promoted=[%s] unpromoted=[%s]\n' \ + "$(date --iso-8601=seconds)" \ + "$(hostname)" \ + "${OCF_RESKEY_CRM_meta_notify_type:-}" \ + "${OCF_RESKEY_CRM_meta_notify_operation:-}" \ + "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" \ + >> "${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify.log" + } || true + + return "${OCF_SUCCESS}" +} + +############################################################################### +# Dispatcher +############################################################################### + +case "$1" in + meta-data) + meta_data + exit "${OCF_SUCCESS}" + ;; + + validate-all) + validate_all + exit $? + ;; + + monitor|status) + managed_monitor + exit $? + ;; + + start) + validate_all || exit $? + managed_start + exit $? + ;; + + stop) + managed_stop + exit $? + ;; + + promote) + validate_all || exit $? + managed_promote + exit $? + ;; + + demote) + validate_all || exit $? + managed_demote + exit $? + ;; + + notify) + managed_notify + exit $? + ;; + + rejoin) + validate_all || exit $? + managed_rejoin + exit $? + ;; + + *) + echo "Usage: $0 {start|stop|promote|demote|notify|rejoin|monitor|status|validate-all|meta-data}" + exit "${OCF_ERR_UNIMPLEMENTED}" + ;; +esac diff --git a/reference/managed-container.pre-refactor-5eae1f3 b/reference/managed-container.pre-refactor-5eae1f3 new file mode 100755 index 0000000..6c6dc09 --- /dev/null +++ b/reference/managed-container.pre-refactor-5eae1f3 @@ -0,0 +1,1721 @@ +#!/bin/bash + +# OCF Resource Agent: +# loxthelion:managed-container +# +# v0.4 +# Container lifecycle: systemd user Quadlet +# Application profile: PostgreSQL +# +# Version 0.4 adds explicit PostgreSQL demotion scaffolding. +# Healthy primaries report OCF_RUNNING_PROMOTED while healthy standbys +# report OCF_SUCCESS. + +: "${OCF_ROOT:=/usr/lib/ocf}" +OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat" +. "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs" + +# Pacemaker uses "Promoted" terminology, while many OCF shell libraries +# still expose the historical OCF_RUNNING_MASTER constant for return code 8. +# Normalize both forms here so the agent remains portable. +: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}" + +############################################################################### +# Defaults +############################################################################### + +OCF_RESKEY_service_user_default="david" +OCF_RESKEY_service_name_default="postgres.service" +OCF_RESKEY_container_name_default="postgres" +OCF_RESKEY_pg_os_user_default="postgres" +OCF_RESKEY_db_user_default="david" +OCF_RESKEY_db_name_default="david" +OCF_RESKEY_rejoin_source_host_default="" +OCF_RESKEY_rejoin_source_port_default="5432" +OCF_RESKEY_replication_user_default="replicator" +OCF_RESKEY_replication_slot_default="" +OCF_RESKEY_application_name_default="" +OCF_RESKEY_rejoin_allow_basebackup_default="false" + +: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}" +: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}" +: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}" +: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}" +: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}" +: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}" +: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}" +: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}" +: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}" +: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}" +: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}" +: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}" + +SERVICE_USER="${OCF_RESKEY_service_user}" +SERVICE_NAME="${OCF_RESKEY_service_name}" +CONTAINER_NAME="${OCF_RESKEY_container_name}" +PG_OS_USER="${OCF_RESKEY_pg_os_user}" +DB_USER="${OCF_RESKEY_db_user}" +DB_NAME="${OCF_RESKEY_db_name}" +REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}" +REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}" +REPLICATION_USER="${OCF_RESKEY_replication_user}" +REPLICATION_SLOT="${OCF_RESKEY_replication_slot}" +APPLICATION_NAME="${OCF_RESKEY_application_name}" +REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}" + +if [ -z "${APPLICATION_NAME}" ]; then + APPLICATION_NAME="$(hostname -s)" +fi + +############################################################################### +# Metadata +############################################################################### + +meta_data() +{ + cat < + + + 1.0 + + + Manage an application inside a persistent rootless container. + + Version 0.4 implements role-aware monitoring, promotion, and demotion. + The systemd user Quadlet owns the container lifecycle while pg_ctl owns + the PostgreSQL lifecycle inside the container. + + + + Manage PostgreSQL independently inside a persistent container + + + + + + + Linux user owning the rootless systemd/Podman service. + + Rootless service owner + + + + + + systemd user service generated by Quadlet. + + Quadlet service name + + + + + + Podman container containing PostgreSQL. + + Container name + + + + + + Operating-system user running PostgreSQL inside the container. + + PostgreSQL OS user + + + + + + PostgreSQL database user used for health queries. + + Database user + + + + + + PostgreSQL database used for health queries. + + Database name + + + + + + + PostgreSQL primary host used as the source for rejoin operations. + + Rejoin source host + + + + + + PostgreSQL port on the rejoin source. + + Rejoin source port + + + + + + PostgreSQL role used for replication and pg_rewind. + + Replication user + + + + + + Physical replication slot used by this standby. + + Physical replication slot + + + + + + PostgreSQL replication application_name. Defaults to the local hostname. + + Replication application name + + + + + + Allow rejoin to fall back to pg_basebackup when pg_rewind cannot + repair the local PostgreSQL data directory. The old PGDATA is + preserved before replacement. This must only be enabled when the + operator has established that this node may safely discard its + divergent PostgreSQL history. + + Allow pg_basebackup rejoin fallback + + + + + + + + + + + + + + + + +EOF_META +} + +############################################################################### +# Rootless-user helpers +############################################################################### + +service_uid() +{ + id -u "${SERVICE_USER}" +} + +service_home() +{ + getent passwd "${SERVICE_USER}" | cut -d: -f6 +} + +as_service_user() +{ + local uid home + + uid="$(id -u "$SERVICE_USER")" || return 1 + home="$(getent passwd "$SERVICE_USER" | cut -d: -f6)" || return 1 + + runuser -u "$SERVICE_USER" -- \ + sh -c ' + home="$1" + runtime="$2" + shift 2 + + cd "$home" || exit 1 + + export HOME="$home" + export XDG_RUNTIME_DIR="$runtime" + + exec "$@" + ' sh \ + "$home" \ + "/run/user/$uid" \ + "$@" +} + +############################################################################### +# Container / application status +############################################################################### + +service_running() +{ + as_service_user \ + systemctl --user is-active --quiet "${SERVICE_NAME}" +} + +container_running() +{ + local state + + state="$( + as_service_user podman inspect \ + --format '{{.State.Running}}' \ + "${CONTAINER_NAME}" 2>/dev/null + )" || return 1 + + [ "${state}" = "true" ] +} + +pg_command() +{ + local command="$1" + + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + sh -c "${command}" +} + +pg_replication_exec() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + sh -c ' + if [ -z "${POSTGRES_REPLICATION_PASSWORD:-}" ]; then + printf "%s\n" \ + "POSTGRES_REPLICATION_PASSWORD is missing from container environment" \ + >&2 + exit 1 + fi + + export PGPASSWORD="${POSTGRES_REPLICATION_PASSWORD}" + + exec "$@" + ' sh "$@" +} + +pgdata_path() +{ + pg_command ' + if [ -z "$PGDATA" ]; then + exit 1 + fi + + printf "%s\n" "$PGDATA" + ' +} + +pg_running() +{ + pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + "$BINDIR/pg_ctl" status -D "$PGDATA" >/dev/null 2>&1 + ' +} + +pg_ready() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + pg_isready -q \ + -U "${DB_USER}" \ + -d "${DB_NAME}" +} + +pg_role() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql \ + -U "${DB_USER}" \ + -d "${DB_NAME}" \ + -Atqc 'SELECT pg_is_in_recovery();' +} + + +pg_remote_query() +{ + local sql="$1" + + pg_replication_exec \ + psql \ + "host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ + -Atqc "${sql}" +} + +pg_wal_receiver_status() +{ + as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + "${CONTAINER_NAME}" \ + psql \ + -U "${DB_USER}" \ + -d "${DB_NAME}" \ + -Atqc "SELECT status FROM pg_stat_wal_receiver LIMIT 1;" +} + +############################################################################### +# Validation +############################################################################### + +validate_all() +{ + if ! id "${SERVICE_USER}" >/dev/null 2>&1; then + ocf_exit_reason "Service user '${SERVICE_USER}' does not exist" + return "${OCF_ERR_CONFIGURED}" + fi + + for cmd in runuser systemctl podman getent; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + ocf_exit_reason "Required command '${cmd}' is not installed" + return "${OCF_ERR_INSTALLED}" + fi + done + + if [ ! -d "/run/user/$(service_uid)" ]; then + ocf_exit_reason \ + "Runtime directory /run/user/$(service_uid) does not exist" + return "${OCF_ERR_GENERIC}" + fi + + if ! as_service_user \ + systemctl --user cat "${SERVICE_NAME}" >/dev/null 2>&1 + then + ocf_exit_reason \ + "User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}" + return "${OCF_ERR_CONFIGURED}" + fi + + local podman_error + + podman_error="$( + as_service_user podman info 2>&1 >/dev/null + )" + local podman_rc=$? + + if [ "${podman_rc}" -ne 0 ]; then + ocf_exit_reason \ + "Rootless Podman failed for ${SERVICE_USER}: ${podman_error}" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + +############################################################################### +# Monitor +############################################################################### + +managed_monitor() +{ + local role + + if ! service_running; then + ocf_log debug "User service ${SERVICE_NAME} is not active" + return "${OCF_NOT_RUNNING}" + fi + + if ! container_running; then + ocf_log warn \ + "Service ${SERVICE_NAME} is active but container ${CONTAINER_NAME} is not running" + return "${OCF_NOT_RUNNING}" + fi + + if ! pg_running; then + ocf_log debug \ + "Container ${CONTAINER_NAME} is running but PostgreSQL is stopped" + return "${OCF_NOT_RUNNING}" + fi + + if ! pg_ready; then + ocf_exit_reason \ + "PostgreSQL process exists but pg_isready failed" + return "${OCF_ERR_GENERIC}" + fi + + role="$(pg_role 2>/dev/null)" || { + ocf_exit_reason \ + "PostgreSQL is accepting connections but role query failed" + return "${OCF_ERR_GENERIC}" + } + + case "${role}" in + f) + ocf_log debug \ + "PostgreSQL healthy: role=primary (promoted)" + return "${OCF_RUNNING_PROMOTED}" + ;; + t) + ocf_log debug \ + "PostgreSQL healthy: role=standby (unpromoted)" + return "${OCF_SUCCESS}" + ;; + *) + ocf_exit_reason \ + "Unexpected pg_is_in_recovery() result: '${role}'" + return "${OCF_ERR_GENERIC}" + ;; + esac + +} + +############################################################################### +# Start +############################################################################### + +managed_start() +{ + local i monitor_rc + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL resource already running unpromoted" + return "${OCF_SUCCESS}" + ;; + + "${OCF_RUNNING_PROMOTED}") + ocf_exit_reason \ + "Refusing start: PostgreSQL is already primary; promotion must occur through promote()" + return "${OCF_ERR_GENERIC}" + ;; + + "${OCF_NOT_RUNNING}") + # Expected for a stopped resource. Continue. + ;; + + *) + ocf_exit_reason \ + "Cannot start unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + ocf_log info \ + "Ensuring container service ${SERVICE_NAME} is running" + + if ! service_running; then + if ! as_service_user \ + systemctl --user start "${SERVICE_NAME}" + then + ocf_exit_reason \ + "Failed to start ${SERVICE_NAME}" + return "${OCF_ERR_GENERIC}" + fi + fi + + for i in $(seq 1 20); do + container_running && break + sleep 0.5 + done + + if ! container_running; then + ocf_exit_reason \ + "Container ${CONTAINER_NAME} did not become running" + return "${OCF_ERR_GENERIC}" + fi + + if ! pg_running; then + # + # OCF promotable-resource rule: + # start() must produce an Unpromoted instance. + # + # A PostgreSQL data directory without standby.signal would start + # writable/primary, so never start it from this action. + # + if ! pg_command ' + test -f "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Refusing start: PGDATA is not configured as a standby (standby.signal missing)" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as an unpromoted standby inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason "pg_ctl start failed" + return "${OCF_ERR_GENERIC}" + fi + fi + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL start completed in unpromoted state" + return "${OCF_SUCCESS}" + ;; + + "${OCF_RUNNING_PROMOTED}") + ocf_exit_reason \ + "PostgreSQL became primary during start; stopping it to preserve OCF role semantics" + + pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' >/dev/null 2>&1 || true + + return "${OCF_ERR_GENERIC}" + ;; + + *) + return "${monitor_rc}" + ;; + esac +} + +ensure_rewind_privileges() +{ + local repl_user + + repl_user="$( + pg_command ' + printf "%s" "${POSTGRES_REPLICATION_USER:-}" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to read POSTGRES_REPLICATION_USER from container" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${repl_user}" ]; then + ocf_exit_reason \ + "POSTGRES_REPLICATION_USER is empty" + return "${OCF_ERR_CONFIGURED}" + fi + + case "${repl_user}" in + *[!A-Za-z0-9_]*) + ocf_exit_reason \ + "Invalid replication role name: ${repl_user}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + ocf_log info \ + "Ensuring pg_rewind source privileges for role ${repl_user}" + + if ! pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -v ON_ERROR_STOP=1 \ + -c \" + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_stat_file(text, boolean) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_read_binary_file(text) + TO ${repl_user}; + + GRANT EXECUTE + ON FUNCTION pg_catalog.pg_read_binary_file( + text, + bigint, + bigint, + boolean + ) + TO ${repl_user}; + \" + " >/dev/null + then + ocf_exit_reason \ + "Failed ensuring pg_rewind privileges for ${repl_user}" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + + +ensure_local_ha_slots() +{ + local slots raw_slot slot slot_type + + # + # POSTGRES_HA_SLOTS lives in the container environment supplied by + # postgres.env. It is the cluster-wide list of physical slots that + # every writable primary must provide. + # + slots="$( + pg_command ' + printf "%s" "${POSTGRES_HA_SLOTS:-}" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to read POSTGRES_HA_SLOTS from PostgreSQL container" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${slots}" ]; then + ocf_log warn \ + "POSTGRES_HA_SLOTS is empty; no local HA slots will be reconciled" + return "${OCF_SUCCESS}" + fi + + while IFS= read -r raw_slot; do + slot="$(printf '%s' "${raw_slot}" | tr -d '[:space:]')" + + [ -n "${slot}" ] || continue + + case "${slot}" in + *[!a-z0-9_]*) + ocf_exit_reason \ + "Invalid physical replication slot name: ${slot}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + slot_type="$( + pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -At \ + -v ON_ERROR_STOP=1 \ + -c \" + SELECT slot_type + FROM pg_replication_slots + WHERE slot_name = '${slot}'; + \" + " 2>/dev/null + )" || { + ocf_exit_reason \ + "Failed querying replication slot ${slot}" + return "${OCF_ERR_GENERIC}" + } + + slot_type="$(printf '%s' "${slot_type}" | tr -d '[:space:]')" + + case "${slot_type}" in + physical) + ocf_log info \ + "Physical HA slot already exists: ${slot}" + ;; + + "") + ocf_log info \ + "Creating physical HA slot: ${slot}" + + if ! pg_command " + psql \ + -U '${DB_USER}' \ + -d postgres \ + -v ON_ERROR_STOP=1 \ + -c \" + SELECT pg_create_physical_replication_slot('${slot}'); + \" + " >/dev/null + then + ocf_exit_reason \ + "Failed creating physical replication slot ${slot}" + return "${OCF_ERR_GENERIC}" + fi + ;; + + *) + ocf_exit_reason \ + "Replication slot ${slot} exists but is type ${slot_type}" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + done </dev/null | + awk -v target="${target}" ' + /nodelist\.node\.[0-9]+\.name / { + key=$1 + sub(/\.name$/, "", key) + names[key]=$NF + } + + /nodelist\.node\.[0-9]+\.ring0_addr / { + key=$1 + sub(/\.ring0_addr$/, "", key) + addrs[key]=$NF + } + + END { + for (key in names) { + if (names[key] == target && addrs[key] != "") { + print addrs[key] + exit 0 + } + } + exit 1 + } + ' +} + +candidate_caught_up() +{ + local candidate="${1:-}" + local result + + [ -n "${candidate}" ] || return 1 + + result="$(pg_command " + psql -U '${DB_USER}' -d '${DB_NAME}' -Atc \" + SELECT CASE + WHEN state = 'streaming' + AND replay_lsn >= pg_current_wal_flush_lsn() + THEN 'yes' + ELSE 'no' + END + FROM pg_stat_replication + WHERE application_name = '${candidate}' + LIMIT 1; + \" + " 2>/dev/null)" || return 1 + + [ "${result}" = "yes" ] +} + +demote_target() +{ + local state_file + local type operation promote_uname demote_uname local_node + local now mtime age + + state_file="/run/loxthelion/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" + local_node="$(hostname)" + + [ -r "${state_file}" ] || return 1 + + now="$(date +%s)" || return 1 + mtime="$(stat -c %Y "${state_file}" 2>/dev/null)" || return 1 + age=$((now - mtime)) + + [ "${age}" -ge 0 ] || return 1 + [ "${age}" -le 120 ] || return 1 + + type="$( + sed -n 's/^type=//p' "${state_file}" | head -n1 + )" + + operation="$( + sed -n 's/^operation=//p' "${state_file}" | head -n1 + )" + + promote_uname="$( + sed -n 's/^promote_uname=//p' "${state_file}" | head -n1 + )" + + demote_uname="$( + sed -n 's/^demote_uname=//p' "${state_file}" | head -n1 + )" + + [ "${type}" = "pre" ] || return 1 + [ "${operation}" = "demote" ] || return 1 + [ "${demote_uname}" = "${local_node}" ] || return 1 + [ -n "${promote_uname}" ] || return 1 + [ "${promote_uname}" != "${local_node}" ] || return 1 + + case "${promote_uname}" in + *" "*) return 1 ;; + esac + + printf '%s +' "${promote_uname}" +} + + +ensure_remote_slot() +{ + local host="${1:-}" + local slot="${2:-}" + local result + + [ -n "${host}" ] || return 1 + [ -n "${slot}" ] || return 1 + + result="$( + pg_replication_exec \ + psql \ + "host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \ + -Atc " + SELECT slot_type + FROM pg_replication_slots + WHERE slot_name = '${slot}'; + " 2>/dev/null + )" || return 1 + + case "${result}" in + physical) + return 0 + ;; + + "") + if ! pg_replication_exec \ + psql \ + "host=${host} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} dbname=postgres" \ + -Atc " + SELECT slot_name + FROM pg_create_physical_replication_slot('${slot}'); + " >/dev/null + then + return 1 + fi + + return 0 + ;; + + *) + ocf_log err \ + "Replication slot ${slot} exists on ${host} but is not physical" + return 1 + ;; + esac +} + +candidate_is_standby() +{ + local host="${1:-}" + local result + local port="${REJOIN_SOURCE_PORT:-5432}" + + [ -n "${host}" ] || return 1 + + result="$( + pg_replication_exec \ + psql \ + "host=${host} port=${port} user=${REPLICATION_USER} dbname=postgres" \ + -Atc "SELECT pg_is_in_recovery();" \ + 2>/dev/null + )" || return 1 + + [ "${result}" = "t" ] +} + +managed_demote() +{ + local monitor_rc + local target_node target_addr + local local_node slot_name target_port + local role receiver_status i + + managed_monitor + monitor_rc=$? + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") + ;; + "${OCF_SUCCESS}") + ocf_log info \ + "PostgreSQL is already unpromoted" + return "${OCF_SUCCESS}" + ;; + "${OCF_NOT_RUNNING}") + ocf_exit_reason \ + "Cannot complete demotion because PostgreSQL is stopped" + return "${OCF_ERR_GENERIC}" + ;; + *) + ocf_exit_reason \ + "Cannot demote unhealthy PostgreSQL instance (monitor rc=${monitor_rc})" + return "${monitor_rc}" + ;; + esac + + target_node="$(demote_target)" || { + ocf_exit_reason \ + "No fresh valid Pacemaker pre-demote target is available" + return "${OCF_ERR_GENERIC}" + } + + target_addr="$(node_to_addr "${target_node}")" || { + ocf_exit_reason \ + "Unable to resolve Pacemaker promotion target ${target_node} through Corosync" + return "${OCF_ERR_GENERIC}" + } + + target_port="${REJOIN_SOURCE_PORT:-5432}" + local_node="$(hostname)" + slot_name="ha_$(printf '%s' "${local_node}" | + tr '[:upper:]-' '[:lower:]_')" + + ocf_log info \ + "Preparing controlled demotion of ${local_node}; successor=${target_node} (${target_addr})" + + if ! candidate_is_standby "${target_addr}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not a reachable PostgreSQL standby" + return "${OCF_ERR_GENERIC}" + fi + + if ! candidate_caught_up "${target_node}"; then + ocf_exit_reason \ + "Promotion target ${target_node} is not streaming and fully replayed to the current flushed WAL position" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_remote_slot "${target_addr}" "${slot_name}"; then + ocf_exit_reason \ + "Unable to ensure physical replication slot ${slot_name} on ${target_node}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Demotion safety checks passed; stopping PostgreSQL primary" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason \ + "pg_ctl stop failed during controlled demotion" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL remained running after pg_ctl stop" + return "${OCF_ERR_GENERIC}" + fi + + if ! service_running || ! container_running; then + ocf_exit_reason \ + "Maintenance service/container stopped unexpectedly during demotion" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Configuring ${local_node} as cascading standby of ${target_node}" + + if ! as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + --env "RA_PRIMARY_HOST=${target_addr}" \ + --env "RA_PRIMARY_PORT=${target_port}" \ + --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ + --env "RA_REPLICATION_SLOT=${slot_name}" \ + --env "RA_APPLICATION_NAME=${local_node}" \ + "${CONTAINER_NAME}" \ + sh -c ' + set -e + + sed -i \ + -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \ + -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \ + "$PGDATA/postgresql.auto.conf" + + printf "%s +" \ + "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \ + "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \ + >> "$PGDATA/postgresql.auto.conf" + + touch "$PGDATA/standby.signal" + + chmod 600 \ + "$PGDATA/postgresql.auto.conf" \ + "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Failed to configure PostgreSQL as standby of ${target_node}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as cascading standby of ${target_node}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason \ + "PostgreSQL failed to start as cascading standby" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null || true)" + receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" + + if [ "${role}" = "t" ] && + [ "${receiver_status}" = "streaming" ] + then + ocf_log info \ + "Controlled demotion complete: ${local_node} is streaming from ${target_node}" + return "${OCF_SUCCESS}" + fi + + sleep 1 + done + + ocf_exit_reason \ + "PostgreSQL started after demotion but did not become a streaming standby" + return "${OCF_ERR_GENERIC}" +} + + +managed_rejoin() +{ + local source_role role receiver_status pgdata i + + case "${REJOIN_ALLOW_BASEBACKUP}" in + true|false|1|0|yes|no) + ;; + *) + ocf_exit_reason "rejoin_allow_basebackup must be true or false" + return "${OCF_ERR_CONFIGURED}" + ;; + esac + + if [ -z "${REJOIN_SOURCE_HOST}" ]; then + ocf_exit_reason \ + "rejoin_source_host is required for rejoin" + return "${OCF_ERR_CONFIGURED}" + fi + + if [ -z "${REPLICATION_SLOT}" ]; then + ocf_exit_reason \ + "replication_slot is required for rejoin" + return "${OCF_ERR_CONFIGURED}" + fi + + if ! service_running; then + ocf_exit_reason \ + "Maintenance service ${SERVICE_NAME} must be running before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + if ! container_running; then + ocf_exit_reason \ + "Maintenance container ${CONTAINER_NAME} must be running before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL must be stopped before rejoin" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Verifying rejoin source ${REJOIN_SOURCE_HOST}:${REJOIN_SOURCE_PORT}" + + source_role="$(pg_remote_query \ + 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to connect to rejoin source ${REJOIN_SOURCE_HOST}" + return "${OCF_ERR_GENERIC}" + } + + if [ "${source_role}" != "f" ]; then + ocf_exit_reason \ + "Rejoin source ${REJOIN_SOURCE_HOST} is not a writable primary" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Ensuring replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" + + if ! ensure_remote_slot "${REJOIN_SOURCE_HOST}" "${REPLICATION_SLOT}" + then + ocf_exit_reason \ + "Unable to ensure physical replication slot ${REPLICATION_SLOT} on rejoin source ${REJOIN_SOURCE_HOST}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Running pg_rewind against ${REJOIN_SOURCE_HOST}" + + pgdata="$(pgdata_path 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to determine PostgreSQL PGDATA inside ${CONTAINER_NAME}" + return "${OCF_ERR_GENERIC}" + } + + if [ -z "${pgdata}" ]; then + ocf_exit_reason \ + "PostgreSQL PGDATA is empty inside ${CONTAINER_NAME}" + return "${OCF_ERR_GENERIC}" + fi + + if ! pg_replication_exec \ + pg_rewind \ + --target-pgdata="${pgdata}" \ + --source-server="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} dbname=postgres user=${REPLICATION_USER}" \ + --progress + then + local fallback_enabled source_bytes free_bytes required_bytes + local timestamp preserved_pgdata failed_pgdata source_role_retry + + case "${REJOIN_ALLOW_BASEBACKUP}" in + true|1|yes) + fallback_enabled=true + ;; + *) + fallback_enabled=false + ;; + esac + + if [ "${fallback_enabled}" != "true" ]; then + ocf_exit_reason \ + "pg_rewind failed and rejoin_allow_basebackup is not enabled" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log warn \ + "pg_rewind failed; evaluating pg_basebackup fallback" + + # Revalidate the source immediately before replacing local PGDATA. + source_role_retry="$(pg_remote_query \ + 'SELECT pg_is_in_recovery();' 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to revalidate rejoin source before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + if [ "${source_role_retry}" != "f" ]; then + ocf_exit_reason \ + "Rejoin source is no longer a writable primary; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if pg_running; then + ocf_exit_reason \ + "PostgreSQL became active during rejoin; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if ! service_running || ! container_running; then + ocf_exit_reason \ + "Maintenance service/container is unavailable; refusing pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + if ! ensure_remote_slot \ + "${REJOIN_SOURCE_HOST}" \ + "${REPLICATION_SLOT}" + then + ocf_exit_reason \ + "Unable to revalidate replication slot ${REPLICATION_SLOT} before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + # Estimate source database payload and require 2x that amount free. + # This is deliberately conservative and does not pretend to be an + # exact prediction of the resulting PGDATA size. + source_bytes="$(pg_remote_query \ + "SELECT COALESCE(sum(pg_database_size(datname)),0) + FROM pg_database + WHERE datallowconn;" 2>/dev/null)" || { + ocf_exit_reason \ + "Unable to estimate source database size before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + case "${source_bytes}" in + ""|*[!0-9]*) + ocf_exit_reason \ + "Invalid source database size returned before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + ;; + esac + + free_bytes="$( + pg_command ' + blocks="$(df -k --output=avail "$PGDATA" | tail -n 1 | tr -d "[:space:]")" + case "$blocks" in + ""|*[!0-9]*) + exit 1 + ;; + esac + printf "%s\n" "$((blocks * 1024))" + ' 2>/dev/null + )" || { + ocf_exit_reason \ + "Unable to determine free space for pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + } + + case "${free_bytes}" in + ""|*[!0-9]*) + ocf_exit_reason \ + "Invalid free-space value before pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + ;; + esac + + required_bytes=$((source_bytes * 2)) + + if [ "${free_bytes}" -lt "${required_bytes}" ]; then + ocf_exit_reason \ + "Insufficient free space for pg_basebackup fallback: free=${free_bytes} estimated-required=${required_bytes}" + return "${OCF_ERR_GENERIC}" + fi + + timestamp="$(date -u +%Y%m%d-%H%M%S)" + preserved_pgdata="${pgdata}.pre-basebackup-${timestamp}" + failed_pgdata="${pgdata}.failed-basebackup-${timestamp}" + + ocf_log warn \ + "Preserving existing PGDATA as ${preserved_pgdata}" + + if ! pg_command " + set -e + + [ -d '${pgdata}' ] + [ ! -e '${preserved_pgdata}' ] + [ ! -e '${failed_pgdata}' ] + + mv '${pgdata}' '${preserved_pgdata}' + install -d -m 0700 '${pgdata}' + " + then + ocf_exit_reason \ + "Unable to preserve existing PGDATA for pg_basebackup fallback" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log warn \ + "Running pg_basebackup from ${REJOIN_SOURCE_HOST}" + + if ! pg_replication_exec \ + pg_basebackup \ + --dbname="host=${REJOIN_SOURCE_HOST} port=${REJOIN_SOURCE_PORT} user=${REPLICATION_USER} application_name=${APPLICATION_NAME}" \ + --pgdata="${pgdata}" \ + --format=plain \ + --wal-method=stream \ + --slot="${REPLICATION_SLOT}" \ + --write-recovery-conf \ + --progress + then + ocf_log err \ + "pg_basebackup failed; preserving failed copy and restoring previous PGDATA" + + if ! pg_command " + set -e + + if [ -e '${failed_pgdata}' ]; then + exit 1 + fi + + if [ -e '${pgdata}' ]; then + mv '${pgdata}' '${failed_pgdata}' + fi + + mv '${preserved_pgdata}' '${pgdata}' + " + then + ocf_exit_reason \ + "pg_basebackup failed and automatic PGDATA rollback also failed; manual recovery required" + return "${OCF_ERR_GENERIC}" + fi + + ocf_exit_reason \ + "pg_basebackup failed; original PGDATA restored and failed copy preserved as ${failed_pgdata}" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "pg_basebackup fallback completed; previous PGDATA preserved as ${preserved_pgdata}" + fi + + ocf_log info \ + "Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}" + + if ! as_service_user \ + podman exec \ + --user "${PG_OS_USER}" \ + --env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \ + --env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \ + --env "RA_REPLICATION_USER=${REPLICATION_USER}" \ + --env "RA_REPLICATION_SLOT=${REPLICATION_SLOT}" \ + --env "RA_APPLICATION_NAME=${APPLICATION_NAME}" \ + "${CONTAINER_NAME}" \ + sh -c ' + set -e + + sed -i \ + -e "/^[[:space:]]*primary_conninfo[[:space:]]*=/d" \ + -e "/^[[:space:]]*primary_slot_name[[:space:]]*=/d" \ + "$PGDATA/postgresql.auto.conf" + + printf "%s\n" \ + "primary_conninfo = '\''host=${RA_PRIMARY_HOST} port=${RA_PRIMARY_PORT} user=${RA_REPLICATION_USER} application_name=${RA_APPLICATION_NAME}'\''" \ + "primary_slot_name = '\''${RA_REPLICATION_SLOT}'\''" \ + >> "$PGDATA/postgresql.auto.conf" + + touch "$PGDATA/standby.signal" + + chmod 600 \ + "$PGDATA/postgresql.auto.conf" \ + "$PGDATA/standby.signal" + ' + then + ocf_exit_reason \ + "Failed to configure standby settings" + return "${OCF_ERR_GENERIC}" + fi + + ocf_log info \ + "Starting PostgreSQL as standby" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + -l "$PGDATA/postgresql.log" \ + start -w + ' + then + ocf_exit_reason \ + "PostgreSQL failed to start after rejoin" + return "${OCF_ERR_GENERIC}" + fi + + for i in $(seq 1 30); do + role="$(pg_role 2>/dev/null)" || { + sleep 1 + continue + } + + receiver_status="$(pg_wal_receiver_status 2>/dev/null || true)" + + if [ "${role}" = "t" ] && + [ "${receiver_status}" = "streaming" ] + then + ocf_log info \ + "PostgreSQL successfully rejoined as streaming standby" + return "${OCF_SUCCESS}" + fi + + sleep 1 + done + + ocf_exit_reason \ + "PostgreSQL started but did not become a streaming standby" + return "${OCF_ERR_GENERIC}" +} + +managed_stop() +{ + local i + + if container_running && pg_running; then + ocf_log info \ + "Stopping PostgreSQL cleanly inside ${CONTAINER_NAME}" + + if ! pg_command ' + BINDIR="$(pg_config --bindir)" || exit 1 + exec "$BINDIR/pg_ctl" \ + -D "$PGDATA" \ + stop -m fast -w + ' + then + ocf_exit_reason "pg_ctl stop failed" + return "${OCF_ERR_GENERIC}" + fi + fi + + if service_running; then + ocf_log info \ + "Stopping container service ${SERVICE_NAME}" + + if ! as_service_user \ + systemctl --user stop "${SERVICE_NAME}" + then + ocf_exit_reason \ + "Failed to stop ${SERVICE_NAME}" + return "${OCF_ERR_GENERIC}" + fi + fi + + for i in $(seq 1 20); do + if ! service_running && ! container_running; then + return "${OCF_SUCCESS}" + fi + sleep 0.5 + done + + if service_running; then + ocf_exit_reason \ + "${SERVICE_NAME} remained active after stop" + return "${OCF_ERR_GENERIC}" + fi + + if container_running; then + ocf_exit_reason \ + "${CONTAINER_NAME} remained running after stop" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + +managed_notify() +{ + local state_dir state_file + + state_dir="/run/loxthelion" + state_file="${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify" + + ocf_log info \ + "notify: type=${OCF_RESKEY_CRM_meta_notify_type:-unset} operation=${OCF_RESKEY_CRM_meta_notify_operation:-unset}" + + ocf_log info \ + "notify: promote_uname=${OCF_RESKEY_CRM_meta_notify_promote_uname:-none} demote_uname=${OCF_RESKEY_CRM_meta_notify_demote_uname:-none}" + + ocf_log info \ + "notify: promoted_uname=${OCF_RESKEY_CRM_meta_notify_promoted_uname:-none} unpromoted_uname=${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-none}" + + if ! mkdir -p -m 0755 "${state_dir}"; then + ocf_exit_reason \ + "Unable to create notification state directory ${state_dir}" + return "${OCF_ERR_GENERIC}" + fi + + if ! { + printf 'timestamp=%s\n' "$(date --iso-8601=seconds)" + printf 'node=%s\n' "$(hostname)" + printf 'type=%s\n' "${OCF_RESKEY_CRM_meta_notify_type:-}" + printf 'operation=%s\n' "${OCF_RESKEY_CRM_meta_notify_operation:-}" + printf 'promote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" + printf 'demote_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" + printf 'promoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" + printf 'unpromoted_uname=%s\n' "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" + } > "${state_file}" + then + ocf_exit_reason \ + "Unable to write notification state ${state_file}" + return "${OCF_ERR_GENERIC}" + fi + + chmod 0600 "${state_file}" || { + ocf_exit_reason \ + "Unable to secure notification state ${state_file}" + return "${OCF_ERR_GENERIC}" + } + + { + printf '%s node=%s type=%s operation=%s promote=[%s] demote=[%s] promoted=[%s] unpromoted=[%s]\n' \ + "$(date --iso-8601=seconds)" \ + "$(hostname)" \ + "${OCF_RESKEY_CRM_meta_notify_type:-}" \ + "${OCF_RESKEY_CRM_meta_notify_operation:-}" \ + "${OCF_RESKEY_CRM_meta_notify_promote_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_demote_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_promoted_uname:-}" \ + "${OCF_RESKEY_CRM_meta_notify_unpromoted_uname:-}" \ + >> "${state_dir}/${OCF_RESOURCE_INSTANCE:-managed-container}.notify.log" + } || true + + return "${OCF_SUCCESS}" +} + +############################################################################### +# Dispatcher +############################################################################### + +case "$1" in + meta-data) + meta_data + exit "${OCF_SUCCESS}" + ;; + + validate-all) + validate_all + exit $? + ;; + + monitor|status) + managed_monitor + exit $? + ;; + + start) + validate_all || exit $? + managed_start + exit $? + ;; + + stop) + managed_stop + exit $? + ;; + + promote) + validate_all || exit $? + managed_promote + exit $? + ;; + + demote) + validate_all || exit $? + managed_demote + exit $? + ;; + + notify) + managed_notify + exit $? + ;; + + rejoin) + validate_all || exit $? + managed_rejoin + exit $? + ;; + + *) + echo "Usage: $0 {start|stop|promote|demote|notify|rejoin|monitor|status|validate-all|meta-data}" + exit "${OCF_ERR_UNIMPLEMENTED}" + ;; +esac diff --git a/runtime-context/current-state.txt b/runtime-context/current-state.txt new file mode 100644 index 0000000..a3666f3 --- /dev/null +++ b/runtime-context/current-state.txt @@ -0,0 +1,16 @@ +===== HOST ===== +DAVID-ELITEDESK + +===== CONTAINER ===== +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +8fe2467e2f68 docker.io/library/postgres@sha256:4ef4dbc939d61acea57712655ddb4b4ab27419c913f94cca0cd57cb3ea3c2280 sleep infinity About an hour ago Up About an hour 5432/tcp postgres + +===== POSTGRES ROLE ===== +t + +===== WAL RECEIVER ===== +streaming|192.168.1.2|5432|ha_david_elitedesk|0/A000130|0/A000000|0/A000130 + +===== PGDATA ===== +PGDATA=/var/lib/postgresql/18/docker +device=49 inode=20681933 owner=postgres:postgres mode=700 path=/var/lib/postgresql/18/docker diff --git a/runtime-context/postgres-ha-bootstrap b/runtime-context/postgres-ha-bootstrap new file mode 100755 index 0000000..e99a0c5 --- /dev/null +++ b/runtime-context/postgres-ha-bootstrap @@ -0,0 +1,533 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +log() { + printf '[postgres-ha-bootstrap] %s\n' "$*" >&2 +} + +die() { + log "ERROR: $*" + exit 1 +} + +require_var() { + local name="$1" + [[ -n "${!name:-}" ]] || die "Required environment variable $name is not set" +} + +require_file() { + local path="$1" + [[ -r "$path" ]] || die "Required file is not readable: $path" +} + +as_postgres() { + runuser -u postgres -- "$@" +} + +# PostgreSQL replication slot names are intentionally restricted here +# so values can safely be used in SQL and configuration. +validate_slot_name() { + [[ "$1" =~ ^[a-z0-9_]+$ ]] || + die "Invalid replication slot name: $1" +} + +install_secret_for_postgres() { + local source="$1" + local destination="$2" + + require_file "$source" + + install \ + -o postgres \ + -g postgres \ + -m 0600 \ + "$source" \ + "$destination" +} + +pgdata_is_initialized() { + [[ -s "$PGDATA/PG_VERSION" ]] && + [[ -s "$PGDATA/global/pg_control" ]] +} + +pgdata_is_empty() { + [[ ! -d "$PGDATA" ]] && return 0 + + local first + first="$(find "$PGDATA" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null || true)" + [[ -z "$first" ]] +} + +marker_path() { + printf '%s/%s\n' \ + "$PGDATA" \ + "${POSTGRES_BOOTSTRAP_MARKER:-.loxthelion-ha-initialized}" +} + +write_managed_config() { + local config="$PGDATA/postgres-ha.conf" + + cat > "$config" <> "$PGDATA/postgresql.conf" <<'EOF_CONFIG' + +# Loxthelion PostgreSQL HA configuration +include_if_exists = 'postgres-ha.conf' +EOF_CONFIG + fi +} + +write_managed_hba() { + local hba="$PGDATA/pg_hba.conf" + local tmp + + tmp="$(mktemp)" + + # Remove a previous block generated by this script. + awk ' + /^# BEGIN LOXTHELION POSTGRES HA$/ { skip=1; next } + /^# END LOXTHELION POSTGRES HA$/ { skip=0; next } + !skip { print } + ' "$hba" > "$tmp" + + cat >> "$tmp" < "$hba" + rm -f "$tmp" + + chown postgres:postgres "$hba" + chmod 0600 "$hba" +} + +create_persistent_pgpass() { + require_var POSTGRES_PRIMARY_HOST + require_var POSTGRES_PRIMARY_PORT + require_var POSTGRES_REPLICATION_USER + require_var POSTGRES_REPLICATION_PASSWORD_FILE + + require_file "$POSTGRES_REPLICATION_PASSWORD_FILE" + + local password + password="$(cat "$POSTGRES_REPLICATION_PASSWORD_FILE")" + + mkdir -p /var/lib/postgresql + chown postgres:postgres /var/lib/postgresql + + printf '%s:%s:*:%s:%s\n' \ + "$POSTGRES_PRIMARY_HOST" \ + "$POSTGRES_PRIMARY_PORT" \ + "$POSTGRES_REPLICATION_USER" \ + "$password" \ + > /var/lib/postgresql/.pgpass + + chown postgres:postgres /var/lib/postgresql/.pgpass + chmod 0600 /var/lib/postgresql/.pgpass +} + +grant_rewind_privileges() { + require_var POSTGRES_USER + require_var POSTGRES_REPLICATION_USER + + log "Ensuring pg_rewind privileges for $POSTGRES_REPLICATION_USER" + + as_postgres psql \ + --username="$POSTGRES_USER" \ + --dbname=postgres \ + --set=ON_ERROR_STOP=1 \ + --set=repl_user="$POSTGRES_REPLICATION_USER" <<'SQL' +GRANT EXECUTE +ON FUNCTION pg_catalog.pg_ls_dir(text, boolean, boolean) +TO :"repl_user"; + +GRANT EXECUTE +ON FUNCTION pg_catalog.pg_stat_file(text, boolean) +TO :"repl_user"; + +GRANT EXECUTE +ON FUNCTION pg_catalog.pg_read_binary_file(text) +TO :"repl_user"; + +GRANT EXECUTE +ON FUNCTION pg_catalog.pg_read_binary_file(text, bigint, bigint, boolean) +TO :"repl_user"; +SQL +} + + +ensure_local_ha_slots() { + [[ -n "${POSTGRES_HA_SLOTS:-}" ]] || { + log "POSTGRES_HA_SLOTS is empty; no local HA slots requested" + return 0 + } + + local raw_slot slot slot_type + + while IFS= read -r raw_slot; do + slot="${raw_slot//[[:space:]]/}" + + [[ -n "$slot" ]] || continue + + validate_slot_name "$slot" + + slot_type="$( + as_postgres psql \ + --username="$POSTGRES_USER" \ + --dbname=postgres \ + --tuples-only \ + --no-align \ + --set=ON_ERROR_STOP=1 \ + --set=slot="$slot" <<'SQL' +SELECT slot_type +FROM pg_replication_slots +WHERE slot_name = :'slot'; +SQL + )" + + slot_type="${slot_type//[[:space:]]/}" + + case "$slot_type" in + physical) + log "Physical HA slot already exists: $slot" + ;; + + "") + log "Creating physical HA slot: $slot" + + as_postgres psql \ + --username="$POSTGRES_USER" \ + --dbname=postgres \ + --set=ON_ERROR_STOP=1 \ + --set=slot="$slot" <<'SQL' >/dev/null +SELECT pg_create_physical_replication_slot(:'slot'); +SQL + ;; + + *) + die "HA slot $slot exists but is type $slot_type" + ;; + esac + done < <( + printf '%s\n' "$POSTGRES_HA_SLOTS" | + tr ',' '\n' + ) +} + + +bootstrap_primary() { + require_var POSTGRES_DB + require_var POSTGRES_USER + require_var POSTGRES_REPLICATION_USER + require_var POSTGRES_HA_CIDR + require_var POSTGRES_PASSWORD_FILE + require_var POSTGRES_REPLICATION_PASSWORD_FILE + + require_file "$POSTGRES_PASSWORD_FILE" + require_file "$POSTGRES_REPLICATION_PASSWORD_FILE" + + pgdata_is_empty || + die "primary bootstrap requires an empty PGDATA" + + log "Initializing fresh PostgreSQL primary" + + mkdir -p "$PGDATA" + chown -R postgres:postgres /var/lib/postgresql + chmod 0700 "$PGDATA" + + local pwfile + pwfile="/tmp/postgres-bootstrap-password" + + install_secret_for_postgres \ + "$POSTGRES_PASSWORD_FILE" \ + "$pwfile" + + as_postgres initdb \ + -D "$PGDATA" \ + --username="$POSTGRES_USER" \ + --pwfile="$pwfile" \ + --auth-host=scram-sha-256 \ + --auth-local=trust \ + --data-checksums + + rm -f "$pwfile" + + write_managed_config + write_managed_hba + + log "Starting PostgreSQL temporarily for initial role/database setup" + + as_postgres pg_ctl \ + -D "$PGDATA" \ + -w \ + start + + trap 'as_postgres pg_ctl -D "$PGDATA" -m fast -w stop >/dev/null 2>&1 || true' RETURN + + local repl_password + repl_password="$(cat "$POSTGRES_REPLICATION_PASSWORD_FILE")" + + # psql variables keep the password out of SQL construction itself. + as_postgres psql \ + --username="$POSTGRES_USER" \ + --dbname=postgres \ + -v ON_ERROR_STOP=1 \ + --set=repl_user="$POSTGRES_REPLICATION_USER" \ + --set=repl_password="$repl_password" <<'SQL' +SELECT format( + 'CREATE ROLE %I WITH LOGIN REPLICATION PASSWORD %L', + :'repl_user', + :'repl_password' +) +WHERE NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = :'repl_user' +) +\gexec +SQL + + grant_rewind_privileges + ensure_local_ha_slots + + if ! as_postgres psql \ + --username="$POSTGRES_USER" \ + --dbname=postgres \ + --tuples-only \ + --no-align \ + --set=ON_ERROR_STOP=1 \ + --set=db="$POSTGRES_DB" <<'SQL' | grep -qx 1 +SELECT 1 +FROM pg_database +WHERE datname = :'db'; +SQL + then + as_postgres createdb \ + --username="$POSTGRES_USER" \ + --owner="$POSTGRES_USER" \ + "$POSTGRES_DB" + fi + + as_postgres pg_ctl \ + -D "$PGDATA" \ + -m fast \ + -w \ + stop + + trap - RETURN + + touch "$(marker_path)" + chown postgres:postgres "$(marker_path)" + chmod 0600 "$(marker_path)" + + log "Fresh primary bootstrap complete" +} + +ensure_remote_replication_slot() { + validate_slot_name "$POSTGRES_REPLICATION_SLOT" + + local slot_type + + slot_type="$( + as_postgres env \ + PGPASSFILE=/var/lib/postgresql/.pgpass \ + psql \ + --host="$POSTGRES_PRIMARY_HOST" \ + --port="$POSTGRES_PRIMARY_PORT" \ + --username="$POSTGRES_REPLICATION_USER" \ + --dbname=postgres \ + --tuples-only \ + --no-align \ + --set=ON_ERROR_STOP=1 \ + --command=" + SELECT slot_type + FROM pg_replication_slots + WHERE slot_name = '$POSTGRES_REPLICATION_SLOT'; + " | + tr -d '[:space:]' + )" + + case "$slot_type" in + physical) + log "Replication slot already exists: $POSTGRES_REPLICATION_SLOT" + ;; + + "") + log "Creating physical replication slot: $POSTGRES_REPLICATION_SLOT" + + as_postgres env \ + PGPASSFILE=/var/lib/postgresql/.pgpass \ + psql \ + --host="$POSTGRES_PRIMARY_HOST" \ + --port="$POSTGRES_PRIMARY_PORT" \ + --username="$POSTGRES_REPLICATION_USER" \ + --dbname=postgres \ + --set=ON_ERROR_STOP=1 \ + --command=" + SELECT pg_create_physical_replication_slot( + '$POSTGRES_REPLICATION_SLOT' + ); + " + ;; + + *) + die "Slot $POSTGRES_REPLICATION_SLOT exists but is not physical" + ;; + esac +} + +bootstrap_standby() { + require_var POSTGRES_PRIMARY_HOST + require_var POSTGRES_PRIMARY_PORT + require_var POSTGRES_REPLICATION_USER + require_var POSTGRES_REPLICATION_SLOT + require_var POSTGRES_APPLICATION_NAME + require_var POSTGRES_REPLICATION_PASSWORD_FILE + + pgdata_is_empty || + die "standby bootstrap requires an empty PGDATA" + + validate_slot_name "$POSTGRES_REPLICATION_SLOT" + + log "Preparing fresh standby from $POSTGRES_PRIMARY_HOST" + + mkdir -p "$PGDATA" + chmod 0700 "$PGDATA" + chown -R postgres:postgres /var/lib/postgresql + + create_persistent_pgpass + ensure_remote_replication_slot + + log "Running pg_basebackup" + + as_postgres env \ + PGPASSFILE=/var/lib/postgresql/.pgpass \ + pg_basebackup \ + --dbname="host=$POSTGRES_PRIMARY_HOST port=$POSTGRES_PRIMARY_PORT user=$POSTGRES_REPLICATION_USER application_name=$POSTGRES_APPLICATION_NAME passfile=/var/lib/postgresql/.pgpass" \ + --pgdata="$PGDATA" \ + --format=plain \ + --wal-method=stream \ + --slot="$POSTGRES_REPLICATION_SLOT" \ + --write-recovery-conf \ + --progress + + [[ -e "$PGDATA/standby.signal" ]] || + die "pg_basebackup completed without standby.signal" + + pgdata_is_initialized || + die "pg_basebackup did not produce a valid PostgreSQL data directory" + + touch "$(marker_path)" + chown postgres:postgres "$(marker_path)" + chmod 0600 "$(marker_path)" + + log "Fresh standby bootstrap complete" +} + +adopt_existing() { + pgdata_is_initialized || + die "POSTGRES_HA_MODE=existing requires a valid existing PostgreSQL cluster" + + log "Existing PostgreSQL cluster detected" + + write_managed_config + + # HBA management requires these only when supplied. + if [[ -n "${POSTGRES_REPLICATION_USER:-}" && + -n "${POSTGRES_HA_CIDR:-}" ]]; then + write_managed_hba + fi + + touch "$(marker_path)" + chown postgres:postgres "$(marker_path)" + chmod 0600 "$(marker_path)" + + log "Existing cluster adopted; no initdb or basebackup performed" +} + +validate_bootstrapped_cluster() { + pgdata_is_initialized || + die "Bootstrap marker exists but PGDATA is not a valid PostgreSQL cluster" + + local version + version="$(cat "$PGDATA/PG_VERSION")" + + [[ "$version" == "18" ]] || + die "Expected PostgreSQL major version 18, found $version" + + as_postgres pg_controldata "$PGDATA" >/dev/null || + die "pg_controldata validation failed" + + local pgdata_mode + pgdata_mode="$(stat -c '%a' "$PGDATA")" + + [[ "$pgdata_mode" == "700" || "$pgdata_mode" == "750" ]] || + die "PGDATA has invalid permissions: $pgdata_mode (expected 700 or 750)" + + [[ "$(stat -c '%U' "$PGDATA")" == "postgres" ]] || + die "PGDATA is not owned by postgres" + + log "Existing bootstrapped PostgreSQL 18 cluster validated" +} + +main() { + require_var PGDATA + + local marker + marker="$(marker_path)" + + log "Node: ${POSTGRES_NODE_NAME:-unknown}" + log "Mode: ${POSTGRES_HA_MODE:-unset}" + log "PGDATA: $PGDATA" + + if [[ -e "$marker" ]]; then + validate_bootstrapped_cluster + else + case "${POSTGRES_HA_MODE:-}" in + primary) + bootstrap_primary + ;; + + standby) + bootstrap_standby + ;; + + existing) + adopt_existing + ;; + + *) + die "POSTGRES_HA_MODE must be primary, standby, or existing" + ;; + esac + fi + + [[ "$#" -gt 0 ]] || + die "No long-running container command supplied" + + log "Bootstrap gate passed; executing: $*" + exec "$@" +} + +main "$@" diff --git a/runtime-context/postgres-ha-bootstrap.pre-v0.7 b/runtime-context/postgres-ha-bootstrap.pre-v0.7 new file mode 100755 index 0000000..f1175ed --- /dev/null +++ b/runtime-context/postgres-ha-bootstrap.pre-v0.7 @@ -0,0 +1,441 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +log() { + printf '[postgres-ha-bootstrap] %s\n' "$*" >&2 +} + +die() { + log "ERROR: $*" + exit 1 +} + +require_var() { + local name="$1" + [[ -n "${!name:-}" ]] || die "Required environment variable $name is not set" +} + +require_file() { + local path="$1" + [[ -r "$path" ]] || die "Required file is not readable: $path" +} + +as_postgres() { + runuser -u postgres -- "$@" +} + +# PostgreSQL replication slot names are intentionally restricted here +# so values can safely be used in SQL and configuration. +validate_slot_name() { + [[ "$1" =~ ^[a-z0-9_]+$ ]] || + die "Invalid replication slot name: $1" +} + +install_secret_for_postgres() { + local source="$1" + local destination="$2" + + require_file "$source" + + install \ + -o postgres \ + -g postgres \ + -m 0600 \ + "$source" \ + "$destination" +} + +pgdata_is_initialized() { + [[ -s "$PGDATA/PG_VERSION" ]] && + [[ -s "$PGDATA/global/pg_control" ]] +} + +pgdata_is_empty() { + [[ ! -d "$PGDATA" ]] && return 0 + + local first + first="$(find "$PGDATA" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null || true)" + [[ -z "$first" ]] +} + +marker_path() { + printf '%s/%s\n' \ + "$PGDATA" \ + "${POSTGRES_BOOTSTRAP_MARKER:-.loxthelion-ha-initialized}" +} + +write_managed_config() { + local config="$PGDATA/postgres-ha.conf" + + cat > "$config" <> "$PGDATA/postgresql.conf" <<'EOF_CONFIG' + +# Loxthelion PostgreSQL HA configuration +include_if_exists = 'postgres-ha.conf' +EOF_CONFIG + fi +} + +write_managed_hba() { + local hba="$PGDATA/pg_hba.conf" + local tmp + + tmp="$(mktemp)" + + # Remove a previous block generated by this script. + awk ' + /^# BEGIN LOXTHELION POSTGRES HA$/ { skip=1; next } + /^# END LOXTHELION POSTGRES HA$/ { skip=0; next } + !skip { print } + ' "$hba" > "$tmp" + + cat >> "$tmp" < "$hba" + rm -f "$tmp" + + chown postgres:postgres "$hba" + chmod 0600 "$hba" +} + +create_persistent_pgpass() { + require_var POSTGRES_PRIMARY_HOST + require_var POSTGRES_PRIMARY_PORT + require_var POSTGRES_REPLICATION_USER + require_var POSTGRES_REPLICATION_PASSWORD_FILE + + require_file "$POSTGRES_REPLICATION_PASSWORD_FILE" + + local password + password="$(cat "$POSTGRES_REPLICATION_PASSWORD_FILE")" + + mkdir -p /var/lib/postgresql + chown postgres:postgres /var/lib/postgresql + + printf '%s:%s:*:%s:%s\n' \ + "$POSTGRES_PRIMARY_HOST" \ + "$POSTGRES_PRIMARY_PORT" \ + "$POSTGRES_REPLICATION_USER" \ + "$password" \ + > /var/lib/postgresql/.pgpass + + chown postgres:postgres /var/lib/postgresql/.pgpass + chmod 0600 /var/lib/postgresql/.pgpass +} + +bootstrap_primary() { + require_var POSTGRES_DB + require_var POSTGRES_USER + require_var POSTGRES_REPLICATION_USER + require_var POSTGRES_HA_CIDR + require_var POSTGRES_PASSWORD_FILE + require_var POSTGRES_REPLICATION_PASSWORD_FILE + + require_file "$POSTGRES_PASSWORD_FILE" + require_file "$POSTGRES_REPLICATION_PASSWORD_FILE" + + pgdata_is_empty || + die "primary bootstrap requires an empty PGDATA" + + log "Initializing fresh PostgreSQL primary" + + mkdir -p "$PGDATA" + chown -R postgres:postgres /var/lib/postgresql + chmod 0700 "$PGDATA" + + local pwfile + pwfile="/tmp/postgres-bootstrap-password" + + install_secret_for_postgres \ + "$POSTGRES_PASSWORD_FILE" \ + "$pwfile" + + as_postgres initdb \ + -D "$PGDATA" \ + --username="$POSTGRES_USER" \ + --pwfile="$pwfile" \ + --auth-host=scram-sha-256 \ + --auth-local=trust \ + --data-checksums + + rm -f "$pwfile" + + write_managed_config + write_managed_hba + + log "Starting PostgreSQL temporarily for initial role/database setup" + + as_postgres pg_ctl \ + -D "$PGDATA" \ + -w \ + start + + trap 'as_postgres pg_ctl -D "$PGDATA" -m fast -w stop >/dev/null 2>&1 || true' RETURN + + local repl_password + repl_password="$(cat "$POSTGRES_REPLICATION_PASSWORD_FILE")" + + # psql variables keep the password out of SQL construction itself. + as_postgres psql \ + --username="$POSTGRES_USER" \ + --dbname=postgres \ + -v ON_ERROR_STOP=1 \ + --set=repl_user="$POSTGRES_REPLICATION_USER" \ + --set=repl_password="$repl_password" <<'SQL' +SELECT format( + 'CREATE ROLE %I WITH LOGIN REPLICATION PASSWORD %L', + :'repl_user', + :'repl_password' +) +WHERE NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = :'repl_user' +) +\gexec +SQL + + if ! as_postgres psql \ + --username="$POSTGRES_USER" \ + --dbname=postgres \ + --tuples-only \ + --no-align \ + --set=ON_ERROR_STOP=1 \ + --set=db="$POSTGRES_DB" <<'SQL' | grep -qx 1 +SELECT 1 +FROM pg_database +WHERE datname = :'db'; +SQL + then + as_postgres createdb \ + --username="$POSTGRES_USER" \ + --owner="$POSTGRES_USER" \ + "$POSTGRES_DB" + fi + + as_postgres pg_ctl \ + -D "$PGDATA" \ + -m fast \ + -w \ + stop + + trap - RETURN + + touch "$(marker_path)" + chown postgres:postgres "$(marker_path)" + chmod 0600 "$(marker_path)" + + log "Fresh primary bootstrap complete" +} + +ensure_remote_replication_slot() { + validate_slot_name "$POSTGRES_REPLICATION_SLOT" + + local slot_type + + slot_type="$( + as_postgres env \ + PGPASSFILE=/var/lib/postgresql/.pgpass \ + psql \ + --host="$POSTGRES_PRIMARY_HOST" \ + --port="$POSTGRES_PRIMARY_PORT" \ + --username="$POSTGRES_REPLICATION_USER" \ + --dbname=postgres \ + --tuples-only \ + --no-align \ + --set=ON_ERROR_STOP=1 \ + --command=" + SELECT slot_type + FROM pg_replication_slots + WHERE slot_name = '$POSTGRES_REPLICATION_SLOT'; + " | + tr -d '[:space:]' + )" + + case "$slot_type" in + physical) + log "Replication slot already exists: $POSTGRES_REPLICATION_SLOT" + ;; + + "") + log "Creating physical replication slot: $POSTGRES_REPLICATION_SLOT" + + as_postgres env \ + PGPASSFILE=/var/lib/postgresql/.pgpass \ + psql \ + --host="$POSTGRES_PRIMARY_HOST" \ + --port="$POSTGRES_PRIMARY_PORT" \ + --username="$POSTGRES_REPLICATION_USER" \ + --dbname=postgres \ + --set=ON_ERROR_STOP=1 \ + --command=" + SELECT pg_create_physical_replication_slot( + '$POSTGRES_REPLICATION_SLOT' + ); + " + ;; + + *) + die "Slot $POSTGRES_REPLICATION_SLOT exists but is not physical" + ;; + esac +} + +bootstrap_standby() { + require_var POSTGRES_PRIMARY_HOST + require_var POSTGRES_PRIMARY_PORT + require_var POSTGRES_REPLICATION_USER + require_var POSTGRES_REPLICATION_SLOT + require_var POSTGRES_APPLICATION_NAME + require_var POSTGRES_REPLICATION_PASSWORD_FILE + + pgdata_is_empty || + die "standby bootstrap requires an empty PGDATA" + + validate_slot_name "$POSTGRES_REPLICATION_SLOT" + + log "Preparing fresh standby from $POSTGRES_PRIMARY_HOST" + + mkdir -p "$PGDATA" + chmod 0700 "$PGDATA" + chown -R postgres:postgres /var/lib/postgresql + + create_persistent_pgpass + ensure_remote_replication_slot + + log "Running pg_basebackup" + + as_postgres env \ + PGPASSFILE=/var/lib/postgresql/.pgpass \ + pg_basebackup \ + --dbname="host=$POSTGRES_PRIMARY_HOST port=$POSTGRES_PRIMARY_PORT user=$POSTGRES_REPLICATION_USER application_name=$POSTGRES_APPLICATION_NAME passfile=/var/lib/postgresql/.pgpass" \ + --pgdata="$PGDATA" \ + --format=plain \ + --wal-method=stream \ + --slot="$POSTGRES_REPLICATION_SLOT" \ + --write-recovery-conf \ + --progress + + [[ -e "$PGDATA/standby.signal" ]] || + die "pg_basebackup completed without standby.signal" + + pgdata_is_initialized || + die "pg_basebackup did not produce a valid PostgreSQL data directory" + + touch "$(marker_path)" + chown postgres:postgres "$(marker_path)" + chmod 0600 "$(marker_path)" + + log "Fresh standby bootstrap complete" +} + +adopt_existing() { + pgdata_is_initialized || + die "POSTGRES_HA_MODE=existing requires a valid existing PostgreSQL cluster" + + log "Existing PostgreSQL cluster detected" + + write_managed_config + + # HBA management requires these only when supplied. + if [[ -n "${POSTGRES_REPLICATION_USER:-}" && + -n "${POSTGRES_HA_CIDR:-}" ]]; then + write_managed_hba + fi + + touch "$(marker_path)" + chown postgres:postgres "$(marker_path)" + chmod 0600 "$(marker_path)" + + log "Existing cluster adopted; no initdb or basebackup performed" +} + +validate_bootstrapped_cluster() { + pgdata_is_initialized || + die "Bootstrap marker exists but PGDATA is not a valid PostgreSQL cluster" + + local version + version="$(cat "$PGDATA/PG_VERSION")" + + [[ "$version" == "18" ]] || + die "Expected PostgreSQL major version 18, found $version" + + as_postgres pg_controldata "$PGDATA" >/dev/null || + die "pg_controldata validation failed" + + local pgdata_mode + pgdata_mode="$(stat -c '%a' "$PGDATA")" + + [[ "$pgdata_mode" == "700" || "$pgdata_mode" == "750" ]] || + die "PGDATA has invalid permissions: $pgdata_mode (expected 700 or 750)" + + [[ "$(stat -c '%U' "$PGDATA")" == "postgres" ]] || + die "PGDATA is not owned by postgres" + + log "Existing bootstrapped PostgreSQL 18 cluster validated" +} + +main() { + require_var PGDATA + + local marker + marker="$(marker_path)" + + log "Node: ${POSTGRES_NODE_NAME:-unknown}" + log "Mode: ${POSTGRES_HA_MODE:-unset}" + log "PGDATA: $PGDATA" + + if [[ -e "$marker" ]]; then + validate_bootstrapped_cluster + else + case "${POSTGRES_HA_MODE:-}" in + primary) + bootstrap_primary + ;; + + standby) + bootstrap_standby + ;; + + existing) + adopt_existing + ;; + + *) + die "POSTGRES_HA_MODE must be primary, standby, or existing" + ;; + esac + fi + + [[ "$#" -gt 0 ]] || + die "No long-running container command supplied" + + log "Bootstrap gate passed; executing: $*" + exec "$@" +} + +main "$@" diff --git a/runtime-context/postgres.container b/runtime-context/postgres.container new file mode 100644 index 0000000..0221687 --- /dev/null +++ b/runtime-context/postgres.container @@ -0,0 +1,26 @@ +[Unit] +Description=PostgreSQL HA Maintenance Container + +[Container] +ContainerName=postgres +Image=docker.io/library/postgres@sha256:4ef4dbc939d61acea57712655ddb4b4ab27419c913f94cca0cd57cb3ea3c2280 + +Entrypoint=/usr/local/sbin/postgres-ha-bootstrap +Exec=sleep infinity + +RunInit=true +StopSignal=SIGTERM +StopTimeout=5 + +Network=host + +Volume=postgres-bootstrap-test:/var/lib/postgresql:Z +Volume=/home/david/.local/libexec/postgres-ha-bootstrap:/usr/local/sbin/postgres-ha-bootstrap:ro,Z +Volume=/home/david/.config/postgres-ha/secrets:/run/secrets/postgres-ha:ro,Z + +EnvironmentFile=/home/david/.config/postgres-ha/postgres.env + +[Service] +SuccessExitStatus=143 +Restart=on-abnormal +TimeoutStartSec=0 diff --git a/runtime-context/postgres.env.sanitized b/runtime-context/postgres.env.sanitized new file mode 100644 index 0000000..50c7dd7 --- /dev/null +++ b/runtime-context/postgres.env.sanitized @@ -0,0 +1,44 @@ +# PostgreSQL data +PGDATA=/var/lib/postgresql/18/docker + +# Bootstrap behavior: +# primary = initialize a new primary +# standby = clone a new standby +# existing = adopt an already-existing PostgreSQL data directory +POSTGRES_HA_MODE=primary + +POSTGRES_NODE_NAME=DAVID-ELITEDESK + +# Initial database/admin account +POSTGRES_DB=david +POSTGRES_USER=david + +# Replication +POSTGRES_REPLICATION_USER=replicator +POSTGRES_REPLICATION_SLOT=ha_david_elitedesk +POSTGRES_APPLICATION_NAME=DAVID-ELITEDESK + +# Used by standby mode +POSTGRES_PRIMARY_HOST=192.168.1.6 +POSTGRES_PRIMARY_PORT=5432 + +# Networks allowed to perform streaming replication +POSTGRES_HA_CIDR=192.168.1.0/24 + +# PostgreSQL HA settings +POSTGRES_WAL_LEVEL=replica +POSTGRES_MAX_WAL_SENDERS=10 +POSTGRES_MAX_REPLICATION_SLOTS=10 +POSTGRES_HOT_STANDBY=on + +# Bootstrap state +POSTGRES_BOOTSTRAP_MARKER=.loxthelion-ha-initialized + +# Mounted secret locations +POSTGRES_PASSWORD_FILE=/run/secrets/postgres-ha/postgres_password +POSTGRES_REPLICATION_PASSWORD_FILE=/run/secrets/postgres-ha/replication_password + +# Physical replication slots expected on every writable primary +POSTGRES_HA_SLOTS=ha_david_elitedesk,ha_david_dv6 +POSTGRES_PASSWORD= +POSTGRES_REPLICATION_PASSWORD= diff --git a/tests/verify_refactor.py b/tests/verify_refactor.py new file mode 100644 index 0000000..c3fbc89 --- /dev/null +++ b/tests/verify_refactor.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Offline structural and behavior comparisons. Never invokes live HA actions.""" +from pathlib import Path +import hashlib +import re +import subprocess +import tempfile +import xml.etree.ElementTree as ET + +ROOT = Path(__file__).resolve().parents[1] +REFERENCE = ROOT / 'reference/managed-container' +AGENT = ROOT / 'refactored/managed-container' +MODULES = sorted((AGENT.parent / 'managed-container.d').glob('*.sh')) +SOURCE = REFERENCE.read_text() +PATTERN = r'^([a-z_]+)\(\)\n\{\n.*?^\}' + + +def functions(source): + matches = list(re.finditer(PATTERN, source, re.M | re.S)) + result = {m[1]: m[0] for m in matches} + assert len(result) == len(matches), 'Duplicate functions' + return result + + +def run(code): + return subprocess.run(['bash', '-c', code], capture_output=True, text=True) + + +def compare_case(name, setup, action, expected): + results = [] + for definitions in (SOURCE[:SOURCE.index('case "$1" in')], modular_definitions): + result = run(definitions + '\n' + setup + '\n' + action) + assert result.returncode == expected, (name, result.returncode, result.stderr) + results.append((result.returncode, result.stdout, result.stderr)) + assert results[0] == results[1], (name, results) + print('PASS', name) + + +expected_hash = '5eae1f3dba0586dc06105e4dd7a9b8809a5c9961452abe7654dcfce77c3c5ce2' +for path in (REFERENCE, ROOT / 'reference/managed-container.pre-refactor-5eae1f3'): + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_hash +print('PASS both reference hashes') + +assert functions(SOURCE) == functions('\n'.join(p.read_text() for p in MODULES)) +assert len(functions(SOURCE)) == 29 +common = (AGENT.parent / 'managed-container.d/common.sh').read_text() +defaults = SOURCE[SOURCE.index('OCF_RESKEY_service_user_default='):SOURCE.index('\n###############################################################################\n# Metadata')].strip() +assert defaults in common +assert SOURCE[SOURCE.index('case "$1" in'):] == AGENT.read_text()[AGENT.read_text().index('case "$1" in'):] +print('PASS all 29 function bodies, defaults/globals and dispatcher identical') + +for path in [AGENT, *MODULES]: + subprocess.run(['bash', '-n', str(path)], check=True) +print('PASS syntax for entrypoint and all nine modules') + +metadata = [] +for path in (REFERENCE, AGENT): + result = subprocess.run([str(path), 'meta-data'], cwd='/tmp', capture_output=True, text=True) + assert result.returncode == 0 and not result.stderr, result + metadata.append(result.stdout) +assert metadata[0] == metadata[1] +xml = ET.fromstring(metadata[0]) +assert len(xml.findall('./parameters/parameter')) == 12 +assert len(xml.findall('./actions/action')) == 8 +print('PASS metadata byte equality and XML parsing from unrelated cwd') + +# Sourcing production modules in a fresh shell tests their real loading order. +# No dispatcher executes. All operational boundaries are overridden below. +modular_definitions = SOURCE[:SOURCE.index('\n###############################################################################')] +for name in ('common', 'container', 'postgres', 'auth', 'replication', 'notify', 'promote', 'demote', 'rejoin'): + modular_definitions += '\n. ' + "'" + str(AGENT.parent / 'managed-container.d' / (name + '.sh')).replace("'", "'\\''") + "'" +base = ''' +ocf_log() { printf 'log %s\\n' "$*"; } +ocf_exit_reason() { printf 'reason %s\\n' "$*" >&2; } +service_running() { return 0; } +container_running() { return 0; } +pg_running() { return 0; } +pg_ready() { return 0; } +pg_role() { printf 't\\n'; } +# Unexpected access to the real rootless boundary fails the scenario. +as_service_user() { printf 'UNEXPECTED external access\\n' >&2; return 99; } +''' +for name, override, rc in [ + ('healthy standby', '', 0), + ('healthy primary', "pg_role() { printf 'f\\n'; }", 8), + ('service stopped', 'service_running() { return 1; }', 7), + ('container stopped', 'container_running() { return 1; }', 7), + ('postgres stopped', 'pg_running() { return 1; }', 7), + ('postgres unready', 'pg_ready() { return 1; }', 1), + ('role query failure', 'pg_role() { return 1; }', 1), + ('invalid role', "pg_role() { printf 'unknown\\n'; }", 1), +]: + compare_case('monitor: ' + name, base + override, 'managed_monitor', rc) +compare_case('start: healthy standby', base, 'managed_start', 0) +compare_case('start: refuse primary', base + "pg_role() { printf 'f\\n'; }", 'managed_start', 1) +compare_case('start: refuse missing standby.signal', base + '\npg_running() { return 1; }\npg_command() { return 1; }', 'managed_start', 1) +compare_case('demote: already standby', base, 'managed_demote', 0) +compare_case('demote: no fresh target', base + '\npg_role() { echo f; }\ndemote_target() { return 1; }', 'managed_demote', 1) +compare_case('rejoin: reject running postgres', base + '\nREJOIN_SOURCE_HOST=192.0.2.1\nREPLICATION_SLOT=ha_test', 'managed_rejoin', 1) +compare_case('promote: existing primary reconciles', base + ''' +pg_role() { echo f; } +ensure_local_ha_slots() { echo slots; } +ensure_rewind_privileges() { echo grants; } +''', 'managed_promote', 0) +compare_case('stop: already stopped', base + '\nservice_running() { return 1; }\ncontainer_running() { return 1; }', 'managed_stop', 0) + +# validate_all invokes only these stubbed system boundaries. test() redirects +# only the /run/user directory predicate; all other checks use the builtin. +validation = base + ''' +id() { echo 1000; } +service_uid() { echo 1000; } +command() { if [ "$1" = -v ]; then return 0; fi; builtin command "$@"; } +[() { if [[ "$1" == ! && "$2" == -d && "$3" == /run/user/* ]]; then return 1; fi; builtin [ "$@"; } +as_service_user() { return 0; } +''' +compare_case('validate-all: success', validation, 'validate_all', 0) +compare_case('validate-all: missing user', validation + '\nid() { return 1; }', 'validate_all', 6) +compare_case('validate-all: missing command', validation + '\ncommand() { return 1; }', 'validate_all', 5) +compare_case('validate-all: missing runtime dir', validation + '\n[() { if [[ "$1" == ! && "$2" == -d ]]; then return 0; fi; builtin [ "$@"; }', 'validate_all', 1) +compare_case('validate-all: missing service', validation + '\nas_service_user() { return 1; }', 'validate_all', 6) +compare_case('validate-all: podman failure', validation + '\nas_service_user() { if [[ "$1" == podman ]]; then echo failed >&2; return 1; fi; return 0; }', 'validate_all', 1) + +with tempfile.TemporaryDirectory(prefix='mc-loader-') as directory: + copied = Path(directory) / 'managed-container' + copied.write_text(AGENT.read_text()) + result = subprocess.run(['bash', str(copied), 'meta-data'], capture_output=True, text=True) + assert result.returncode == 6 and 'Unable to load resource-agent module' in result.stderr +print('PASS missing module fails with configured error') + +text = AGENT.read_text() + '\n'.join(p.read_text() for p in MODULES) +assert '.pgpass' not in text and 'PGPASSFILE' not in text +assert not re.search(r'^\s*(?:source|\.)\s+.*(?:\.env|env-file)', text, re.M) +print('PASS no pgpass or host env-file sourcing introduced') +print('All offline comparisons passed; live HA transitions were not exercised.')