From ec2077f979e4f66c4c28cf95fe614cb0d81c9c1c Mon Sep 17 00:00:00 2001 From: Lyncoln Date: Sun, 13 Sep 2026 01:57:04 -0400 Subject: [PATCH] Freeze lab-tested PostgreSQL plugin engine Extract PostgreSQL, Podman runtime and systemd lifecycle adapters while preserving existing lifecycle and scoring behavior. Update RPM layout and add guarded operator-run lab harnesses. Record successful manual stop/start, promotion, demotion and rewind-only rejoin with pinned installed hashes. The unfenced lab does not validate automatic failover, Pacemaker-managed transitions or divergent-data rewind. Validation: 103 offline tests, original-function differential checks, checksum manifest and whitespace checks passed. --- .gitea/workflows/build-rpm.yml | 2 +- README.md | 14 +- docs/lab-installed-before-plugin.sha256 | 11 + docs/lab-installed-plugin.sha256 | 15 + docs/lab-plugin-basic-test.md | 119 ++++++++ docs/lab-role-tests.md | 113 +++++++ docs/operations.md | 5 +- docs/plugin-engine.md | 210 +++++++++++++ docs/testing.md | 7 +- notes/REFACTOR-SHA256SUMS | 63 ++-- packaging/README.md | 6 +- packaging/ocf-managed-container.spec | 18 +- refactored/managed-container | 9 +- .../{ => applications/postgresql}/auth.sh | 3 +- .../{ => applications/postgresql}/demote.sh | 3 +- .../{ => applications/postgresql}/notify.sh | 0 .../applications/postgresql/plugin.sh | 281 ++++++++++++++++++ .../{ => applications/postgresql}/postgres.sh | 26 +- .../{ => applications/postgresql}/promote.sh | 0 .../postgresql}/promotion_state.py | 0 .../{ => applications/postgresql}/rejoin.sh | 3 +- .../postgresql}/replication.sh | 0 refactored/managed-container.d/common.sh | 249 +--------------- refactored/managed-container.d/container.sh | 5 +- refactored/managed-container.d/engine.sh | 59 ++++ .../managed-container.d/lifecycles/systemd.sh | 23 ++ .../managed-container.d/runtimes/podman.sh | 24 ++ scripts/test-elitedesk-cycle.py | 134 +++++++++ scripts/test-elitedesk-plugin.py | 203 +++++++++++++ scripts/test-lab-roles.py | 250 ++++++++++++++++ tests/plugin_test_support.py | 33 ++ tests/test_disconnected_state.py | 2 +- tests/test_lab_cycle.py | 65 ++++ tests/test_lab_installer.py | 78 +++++ tests/test_lab_roles.py | 55 ++++ tests/test_plugin_engine.py | 121 ++++++++ tests/test_promotion_scores.py | 6 +- tests/verify_installed_rpm.py | 13 +- tests/verify_promotion_sql.py | 2 +- tests/verify_promotion_state_cli.py | 2 +- tests/verify_refactor.py | 24 +- 41 files changed, 1925 insertions(+), 331 deletions(-) create mode 100644 docs/lab-installed-before-plugin.sha256 create mode 100644 docs/lab-installed-plugin.sha256 create mode 100644 docs/lab-plugin-basic-test.md create mode 100644 docs/lab-role-tests.md create mode 100644 docs/plugin-engine.md rename refactored/managed-container.d/{ => applications/postgresql}/auth.sh (98%) rename refactored/managed-container.d/{ => applications/postgresql}/demote.sh (99%) rename refactored/managed-container.d/{ => applications/postgresql}/notify.sh (100%) create mode 100644 refactored/managed-container.d/applications/postgresql/plugin.sh rename refactored/managed-container.d/{ => applications/postgresql}/postgres.sh (94%) rename refactored/managed-container.d/{ => applications/postgresql}/promote.sh (100%) rename refactored/managed-container.d/{ => applications/postgresql}/promotion_state.py (100%) rename refactored/managed-container.d/{ => applications/postgresql}/rejoin.sh (99%) rename refactored/managed-container.d/{ => applications/postgresql}/replication.sh (100%) create mode 100644 refactored/managed-container.d/engine.sh create mode 100644 refactored/managed-container.d/lifecycles/systemd.sh create mode 100644 refactored/managed-container.d/runtimes/podman.sh create mode 100644 scripts/test-elitedesk-cycle.py create mode 100644 scripts/test-elitedesk-plugin.py create mode 100644 scripts/test-lab-roles.py create mode 100644 tests/plugin_test_support.py create mode 100644 tests/test_lab_cycle.py create mode 100644 tests/test_lab_installer.py create mode 100644 tests/test_lab_roles.py create mode 100644 tests/test_plugin_engine.py diff --git a/.gitea/workflows/build-rpm.yml b/.gitea/workflows/build-rpm.yml index 0734563..56a365b 100644 --- a/.gitea/workflows/build-rpm.yml +++ b/.gitea/workflows/build-rpm.yml @@ -25,7 +25,7 @@ jobs: run: | dnf -y install ./dist/*.noarch.rpm python3 tests/verify_installed_rpm.py - MC_DISPOSABLE_STATE_TEST=1 python3 tests/verify_promotion_state_cli.py /usr/lib/ocf/resource.d/loxthelion/managed-container.d/promotion_state.py + MC_DISPOSABLE_STATE_TEST=1 python3 tests/verify_promotion_state_cli.py /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/promotion_state.py # v3 uses the artifact protocol supported by older act_runner instances. - name: Save development packages uses: https://github.com/actions/upload-artifact@v3 diff --git a/README.md b/README.md index 6a0f4d5..c79d36d 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,18 @@ A modular PostgreSQL HA implementation of `ocf:loxthelion:managed-container`. Pacemaker invokes the agent to manage a rootless systemd/Quadlet maintenance container. PostgreSQL is started and stopped independently with `pg_ctl`. +This branch extracts the frozen implementation into a PostgreSQL plugin, Podman +runtime adapter and systemd lifecycle adapter. It is **installed on ELITEDESK +in the test cluster only**. [Basic lifecycle checks](docs/lab-plugin-basic-test.md) +and [manual promotion, demotion and rewind-only rejoin tests](docs/lab-role-tests.md) +passed according to operator-supplied output. The lab has no fencing. See the +[engine contract and validation plan](docs/plugin-engine.md). The production test +report applies to the earlier freeze, not automatically to this extraction. + ## Project layout - `reference/`: original agent and immutable known-good reference copy. -- `refactored/`: executable agent, nine sourced shell modules, and a Python state helper. +- `refactored/`: executable agent, application/runtime/lifecycle modules, and a Python state helper. - `tests/`: offline structural and behavioral comparison checks. - `cluster-context/` and `runtime-context/`: saved lab configuration and state; these snapshots are not live status. Environment credentials are redacted. @@ -21,6 +29,7 @@ Requires Bash, Python 3, and the OCF shell library (normally ```bash python3 tests/verify_refactor.py +python3 tests/test_plugin_engine.py python3 tests/test_promotion_scores.py python3 tests/test_promotion_scheduler.py python3 tests/test_disconnected_state.py @@ -28,7 +37,8 @@ python3 tests/test_promotion_cleanup.py sha256sum -c notes/REFACTOR-SHA256SUMS ``` -The tests preserve the original 29 lifecycle function bodies and exercise the +The tests retain all 29 original functions and verify exact equivalence after +reversing only adapter-call substitutions. They exercise the added promotion-scoring layer without operating the live cluster. The operator has reported successful production SBD fencing, automatic promotion, VIP movement, and former-primary rewind/rejoin in a three-node cluster. diff --git a/docs/lab-installed-before-plugin.sha256 b/docs/lab-installed-before-plugin.sha256 new file mode 100644 index 0000000..54ad638 --- /dev/null +++ b/docs/lab-installed-before-plugin.sha256 @@ -0,0 +1,11 @@ +a4b775a0ddcabdb0558da9b34c7fe2a883c07d0ef308c1c7224526abf53f63a0 managed-container +93191771d3cb8aefadb2a16d2f89d64a8b6c584071a473439be818237bdf7f17 managed-container.d/auth.sh +97e93d9164e854e5f3688cf9d3cc656b001eb30ef3c2073c907450ca43588f3d managed-container.d/common.sh +ac6bc363a2301f1491d162ef609103dbfbceb3b099a3556614c6165952309374 managed-container.d/container.sh +0af3bb362a303017372c631354ba94ef9e723bf14e64d2eff91b6662e54a5be0 managed-container.d/demote.sh +6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670 managed-container.d/notify.sh +d2779ce6e7d1f426ffca2e41bbb8de77503005dc38d658d69f184020365796eb managed-container.d/postgres.sh +3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58 managed-container.d/promote.sh +21beea72bcb7f24a51c17ea48666ac87df9436702ad77b166b45f4647e28e8ad managed-container.d/rejoin.sh +6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547 managed-container.d/replication.sh +5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7 managed-container.d/promotion_state.py diff --git a/docs/lab-installed-plugin.sha256 b/docs/lab-installed-plugin.sha256 new file mode 100644 index 0000000..6aff47b --- /dev/null +++ b/docs/lab-installed-plugin.sha256 @@ -0,0 +1,15 @@ +01eb9b161c8e3db289f09bf63953be779921e15bdf8b1a4d8ce7e60032b826e4 /usr/lib/ocf/resource.d/loxthelion/managed-container +10932919c4339b553c7c9dbf30b45a917c51c0a08f362e12678003d58b12fa82 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/auth.sh +da4e58cadf1df3953cb97a0512e3931ee1b9d3ee61c39a1ad1a6047ef537addc /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/demote.sh +6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/notify.sh +d3a989de97ceb61a56828f006d5ccb87b7c34fc155a944585bedf928785021fc /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/plugin.sh +fa222e6767249999767eb82fac2d85e7e35404da240517a8b630571e77282053 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/postgres.sh +3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/promote.sh +5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/promotion_state.py +28b0e44554a88ccb87cd16a68315c045dd37d354aceb2fa25da7c7399a659b23 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/rejoin.sh +6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/applications/postgresql/replication.sh +a482ef3cc1048e6a4f427e76dd2dce173ffd00760dbed1afa601a3fd05b0ad99 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/common.sh +3a27ac9c63c61310da8b8711fa2c4400b37ea8eae2ee2a79e28bc63180b06c7c /usr/lib/ocf/resource.d/loxthelion/managed-container.d/container.sh +53978e2367394695d61c6fb38f2ee46814474c5a4e0448d2692f1a3ce583af93 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/engine.sh +6c92c095ce10ac95d843fdc1c49e31c19491ca22fdac960affd8649606d76d82 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/lifecycles/systemd.sh +dc5820067a78c4a1c531426902793aae97a03fd64912b017b3b102031dcc1e04 /usr/lib/ocf/resource.d/loxthelion/managed-container.d/runtimes/podman.sh diff --git a/docs/lab-plugin-basic-test.md b/docs/lab-plugin-basic-test.md new file mode 100644 index 0000000..422e888 --- /dev/null +++ b/docs/lab-plugin-basic-test.md @@ -0,0 +1,119 @@ +# ELITEDESK plugin installation and basic checks + +Status: installed on DAVID-ELITEDESK; operator-reported basic checks passed. +All 15 installed code files were subsequently verified read-only against the +working candidate and matched byte-for-byte. +No production or peer-node actions are part of this procedure. + +Read-only preflight on DAVID-ELITEDESK found its PostgreSQL container running as +an asynchronous streaming standby of 192.168.1.2 using ha_david_elitedesk. +All 11 installed RA/module/helper files match source freeze 488a167; their exact +hashes are captured in lab-installed-before-plugin.sha256. This is a lab snapshot, +not the outstanding production snapshot. Current cluster maintenance state could +not be read without sudo; the installer verifies it live before any replacement. + +Run in the operator's terminal: + +```sh +sudo python3 /home/david/codex-postgres-ha-refactor/scripts/test-elitedesk-plugin.py +``` + +The command requires the existing postgres-ra-test cluster, exactly the two lab +nodes, pg-ha under pg-ha-clone, expected local container/database settings, and +maintenance-mode already true. It does not enable maintenance mode or modify CIB +configuration. It copies parameters from the live resource and rejects ambiguous +parameter sets. Native sudo prompts stay in the terminal; never share a password. + +It verifies source checksums, stages root-owned code, backs up the installed agent +and modules under /var/backups/managed-container-plugin-lab-*, validates staged +metadata and configuration, then replaces the complete module tree and entrypoint. +The installation remains in place after success. A failed post-install check +restores the old agent and module tree and retains diagnostic files. + +Basic checks are metadata (16 parameters), validate-all, monitor, status, an +idempotent start on the already-running standby, and another monitor. Before and +after checks require the same PostgreSQL system identifier, postmaster start time, +recovery role, streaming sender and slot. No stop, promote, demote, rejoin or +fencing is invoked. Normal monitors/start may refresh local promotion evidence +and the pg-ha promotion score; those expected side effects are part of testing. +A score of -INFINITY is recorded rather than mistaken for a failed role monitor; +strict catch-up can fluctuate. Investigate that result before broader HA testing. + +The script prints its backup/result directory and action results. It records +results.json and installed-SHA256SUMS there with root-only access. Return the +terminal output before progressing to disruptive stop/start or switchover tests. + +Preparation validation: current structural/differential tests passed; two +isolated installer tests passed, covering basic-action-only success and restoration +of both agent and modules on an injected installed-monitor failure. These tests +mock privilege/cluster boundaries and touch only temporary directories; they do +not claim live installation success. + +## Completed basic lab run + +The operator supplied successful installer output for backup/result directory +`/var/backups/managed-container-plugin-lab-mow_t6zi`: + +- Staged metadata and validate-all: rc=0. +- Installed validate-all, monitor, status, idempotent start, monitor: rc=0. +- Final promotion score: 100. +- The installer reported PostgreSQL was not restarted and maintenance remained true. +- No stop, promote, demote, rejoin, fencing, or peer-node actions ran. + +Codex independently compared every installed entrypoint/module/helper file with +the working candidate using read-only filesystem access; all 15 matched. Exact +installed SHA-256 values are recorded in [lab-installed-plugin.sha256](lab-installed-plugin.sha256). +The root-only results.json was not independently read; the action results above +are attributed to the operator-supplied output. This validates the installation +and non-disruptive standby path, not actual stop/start, switchover or failover. + +Installed-file comparison captured at 2026-09-13T03:01:11.114949+00:00 UTC. + +## Controlled standby stop/start — operator-reported pass + +The operator ran: + +```sh +sudo python3 /home/david/codex-postgres-ha-refactor/scripts/test-elitedesk-cycle.py +``` + +This briefly interrupts only the ELITEDESK standby. The script requires the same +lab identity, maintenance=true, streaming sender/slot and exact installed hashes. +It validates/monitors, runs RA stop, checks systemd inactive and the container +stopped/absent, expects monitor=7 and score=-INFINITY, then runs RA start. It +requires the same database identifier, a new postmaster start time, recovery and +streaming from DV6, monitor/status=0 and score=100 within the verification window. +Each RA action has a 60-second timeout with a 5-second kill grace. Post-start +streaming/eligibility checks have a 60-second retry window (an in-flight bounded +check can finish after the deadline). No source files or cluster configuration +are replaced, and no promotion, demotion, rejoin, fencing or peer action runs. + +Failures halt without automatic recovery and may leave the standby stopped. +Inspect the printed /var/backups/managed-container-cycle-lab-* result directory +before proceeding. It contains root-only results.json and per-action output. +Three mocked harness tests passed: successful cycle, stop failure without start, +and start failure without retry. Those are preparation checks, not live results. + + +### Completed cycle result + +The operator supplied successful output with evidence directory +`/var/backups/managed-container-cycle-lab-b1048r6q`: + +| Check | Result | +| --- | --- | +| Initial validate-all and monitor | rc=0 | +| RA stop | rc=0 | +| Monitor while stopped | rc=7 | +| Stopped lifecycle state | systemd inactive; container stopped/absent | +| Stopped promotion score | -INFINITY | +| RA start, subsequent monitor and status | rc=0 | +| Database identity | Same system identifier; new postmaster start time | +| Replication after restart | Streaming from DV6; score 100 | +| Cluster maintenance | Remained true | + +These results are attributed to the operator-supplied script output; Codex has +not independently read the root-only results.json. No promotion, demotion, +rejoin, fencing or peer-node actions ran. The basic standby lifecycle now has +live lab coverage; this does not establish switchover/failover behavior for the +plugin extraction or constitute production deployment validation. diff --git a/docs/lab-role-tests.md b/docs/lab-role-tests.md new file mode 100644 index 0000000..554ce75 --- /dev/null +++ b/docs/lab-role-tests.md @@ -0,0 +1,113 @@ +# Staged lab role tests + +**All five live stages passed according to operator-supplied terminal output.** +The sequence followed the successful ELITEDESK stop/start cycle. Only the two-node postgres-ra-test lab is supported; maintenance +must remain true throughout. DV6 retains the frozen RA and ELITEDESK has the +plugin candidate. Neither implementation is installed or changed by this tool. + +The operator confirmed that this lab has no fencing; fencing is available only +in production. This checkpoint covers controlled, manual role transitions with +a reachable primary. Abrupt-primary-loss automatic failover is not validated +for the plugin candidate. The earlier production fencing report applies to the +earlier implementation, not this extraction. + +During maintenance, Pacemaker's recurring role monitors are paused. The harness +invokes the RA directly, so its action results do not update Pacemaker's recorded +operation history. A Cockpit cluster role display can therefore remain stale +even while the RA explicitly updates promotion scores. Database roles were +verified using direct SQL; systemd/container running state remains distinct +from PostgreSQL role. Maintenance was not disabled to refresh the display. + +`scripts/test-lab-roles.py` is a portable, operator-run harness with embedded +hashes from both lab snapshots. It needs root locally and noninteractive SSH +as david to the peer for read-only observations. SSH host keys must already be +verified. Passwords are entered in the operator's terminal, never in chat. + +Run `sudo python3 /path/to/test-lab-roles.py inspect` on each node first. +Inspection checks cluster name, membership, maintenance, resource parameters, +installed hashes, SQL readiness, roles, system identity and maintenance service. +It changes no database or cluster state. Do not proceed if either check fails. + +Each stage requires a separate invocation and review of its successful output: + +| Order | Host | Stage argument | Expected result | +| --- | --- | --- | --- | +| 1 | DV6 | demote | DV6 streams from ELITEDESK; both are standbys | +| 2 | ELITEDESK | promote | ELITEDESK primary, DV6 streaming standby | +| 3 | ELITEDESK | demote | ELITEDESK streams from DV6; both are standbys | +| 4 | DV6 | promote | DV6 primary, ELITEDESK streaming standby | +| 5 | ELITEDESK | rejoin | Stop PostgreSQL only, run rewind-only rejoin to DV6, verify streaming | + +The demote stage submits a **synthetic lab pre-demote notification** through the +RA notify action, then invokes demote. This tests the notification consumer and +controlled demotion; it does not prove Pacemaker generates/orders notifications. +The RA retains its own catch-up and successor checks. The harness refuses promote +unless the peer is already in recovery and streaming from the local standby. +Between demote and promote there is deliberately no writable primary. Arrange +both terminals before starting; do not perform other manual database operations +during the sequence. Scores on both standbys can fluctuate during this interval. + +The rejoin stage is restricted to the restored ELITEDESK standby. It explicitly +sets basebackup fallback=false. It tests the stopped-PostgreSQL precondition, +maintenance container survival, source authentication, pg_rewind invocation, +recovery configuration, PostgreSQL restart, streaming and score restoration. +Because it does not create divergent data, it does **not** validate divergent +former-primary rewind repair. No fencing, data replacement fallback, VIP moves, +Pacemaker-managed switchover or automatic failover are tested here. + +Each modifying invocation writes root-only results/action logs beneath +`/var/backups/managed-container-roles-lab-*`. Failures halt without automatic +retry/recovery. A failure after demotion can leave no primary; a rejoin failure +can leave ELITEDESK stopped. Stop the sequence and inspect the evidence rather +than running the next stage. Do not clear failed actions or change maintenance. + +The checks sample state; they cannot prevent a separate administrator from +changing roles concurrently. Controlled demotion of a reachable primary is the +basis for these manual lab promotions. This is not a procedure for promoting +after an unreachable primary failure; that requires fencing and Pacemaker's +ordering under the operating policy. + +## Completed live sequence + +The operator supplied successful output for each stage below. Evidence directories +are on the host named in the table, under `/var/backups/`. Codex has not read the +root-only results.json or action logs; these are operator-reported passes, not +independently witnessed live actions. + +| Host | Stage | Result | Evidence directory | +| --- | --- | --- | --- | +| DV6 | demote | rc=0; standby monitor=0; score=100 | managed-container-roles-lab-o1xjobi2 | +| ELITEDESK | promote | rc=0; primary monitor=8; score=1000 | managed-container-roles-lab-qdz57g70 | +| ELITEDESK | demote | rc=0; standby monitor=0; score=100 | managed-container-roles-lab-nwbzdxnl | +| DV6 | promote | rc=0; primary monitor=8; score=1000 | managed-container-roles-lab-r69amyxg | +| ELITEDESK | rejoin | stopped monitor=7; rejoin=0; standby monitor=0; score=100 | managed-container-roles-lab-7c4vw7_p | + +Each stage reported preserved database identity and maintenance container, +verified roles and streaming, and maintenance=true. The final reported state is +DV6 primary and ELITEDESK streaming standby. The rejoin harness also checks the +stopped score is -INFINITY before invoking rejoin, with basebackup disabled. +This establishes successful pg_rewind invocation, not repair of deliberately +divergent timelines or a measured amount of rewound data. + +The supplied inspection after ELITEDESK demotion showed both nodes in recovery, +streaming from each other, with replay LSN 0/A02B560 and system identifier +7684451577990111248. An earlier inspection in the same paste showed DV6 primary; +the later observation was used to resolve the role confusion before restoring DV6. + +Both-way SSH access was established and independently checked before the role +sequence. The installed code was pinned to the existing lab snapshots. No RA +installation, maintenance change, fencing, VIP move or failure cleanup was part +of these tests. The limitations above continue to apply. + +Offline original-function comparisons and 97 tests passed before the live +sequence (the 11 scheduler tests required execution outside the sandbox). +Six additional role-harness safety-gate tests passed before staging the harness. + +## Commit checkpoint verification + +On 2026-09-13, all 103 tests in `python3 -m unittest discover -s tests -p +'test_*.py'` passed, including offline scheduler simulations. +`python3 tests/verify_refactor.py` also passed: frozen reference hashes, all 29 +original functions after exact adapter-call reversal, syntax, metadata and +22 differential lifecycle cases. No additional live actions were run for this +commit checkpoint. diff --git a/docs/operations.md b/docs/operations.md index 38fbf61..df06b80 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -46,9 +46,10 @@ and set PGPASSWORD only for the child process. Lifecycle success is 0; monitor's 8 is a role report, not a generic lifecycle success. Promoted monitoring must expect the correct role. Metadata advertises -13 parameters and eight actions; status is an alias and rejoin stays private. +16 parameters and eight actions; status is an alias and rejoin stays private. The scoring wrappers withdraw scores before lifecycle actions and reassess after -successful actions. They do not change the original 29 function bodies. +successful actions. The engine extraction preserves the original 29 functions with explicit +adapter-call substitutions; see plugin-engine.md. ## Scoring and fencing diff --git a/docs/plugin-engine.md b/docs/plugin-engine.md new file mode 100644 index 0000000..36cb0c4 --- /dev/null +++ b/docs/plugin-engine.md @@ -0,0 +1,210 @@ +# Podman plugin engine: first extraction + +Development baseline: freeze commit `488a167`. This branch extracts the existing PostgreSQL/Podman/systemd implementation. +It is installed on DAVID-ELITEDESK in the lab only; see lab-plugin-basic-test.md. Production +hash provenance is still pending as described in testing.md. Quadlet, bootstrap and cluster configuration remain unchanged. The lab RA files +were replaced, and basic monitor/start checks refreshed standby scoring without +restarting PostgreSQL. No production installation has occurred. + +## Scope and supported configuration + +The default and only supported combination is: + +```text +application=postgresql +container_runtime=podman +lifecycle_manager=systemd +``` + +These are advertised OCF instance parameters, not Pacemaker meta-attributes. +Existing resource configurations need no new parameters. Missing or empty +selectors use the defaults. Explicit unsupported values, including Docker or +direct-runtime lifecycle, fail with OCF_ERR_CONFIGURED (6) before operations. +The entrypoint/provider and every existing parameter default remain unchanged. + +The engine loads a fixed allowlist of packaged files beside the entrypoint. +There is no arbitrary plugin pathname, dynamic shell evaluation, network plugin +installation, or fallback to another runtime. Every implementation declares API +version 1; missing files, incompatible versions and missing required hooks fail +configuration. Packaged code is trusted/root-owned, as was the original RA. + +## Boundaries + +```text +Pacemaker OCF action + -> shared scoring/action wrappers + -> applications/postgresql/plugin.sh + -> existing PostgreSQL lifecycle and recovery modules + -> runtimes/podman.sh (inspect / exec / info) + -> lifecycles/systemd.sh (user service lifecycle) + -> unchanged as_service_user privilege boundary +``` + +`engine.sh` selects and checks plugins and adds selector metadata. `common.sh` +contains the existing score-publication and action wrappers, now calling the +application interface. `container.sh` retains service-user access plus +compatibility health helpers that delegate to the selected adapters. + +`applications/postgresql/plugin.sh` owns the existing PostgreSQL defaults, metadata, +validation and seven PostgreSQL/HA modules. Application modules and the state +helper live together under +`applications/postgresql/`; runtime and lifecycle adapters occupy their own +`runtimes/` and `lifecycles/` directories. The loader uses fixed relative paths. +Future applications will have their own +metadata and dependency validation rather than inherit PostgreSQL assumptions. + +The packaged layout is: + +```text +managed-container.d/ + engine.sh + common.sh + container.sh + applications/postgresql/ + plugin.sh + postgres.sh, auth.sh, replication.sh, notify.sh + promote.sh, demote.sh, rejoin.sh + promotion_state.py + lifecycles/systemd.sh + runtimes/podman.sh +``` + +The PostgreSQL plugin sets its own `MC_APPLICATION_DIR` for sibling modules and +the Python state helper. Host evidence remains at its existing `/run` location; +only packaged code paths changed. The public RA entrypoint is unchanged. + +## Interface contract, API 1 + +All interfaces are sourced Bash functions in a single RA invocation. Arguments +are passed as arrays using `"$@"`; adapters do not concatenate or evaluate command +strings. Existing PostgreSQL shell snippets remain explicit arguments to `sh -c`; +this extraction does not redesign or broaden their trusted configuration inputs. + +| Interface | Contract | +| --- | --- | +| mc_runtime_exec ARGS... | Execute in the container through the service owner; preserve stdout, stderr, and exact exit status. | +| mc_runtime_exec_bounded DURATION ARGS... | Same execution with timeout placed inside the service-user boundary; retain kill-after=1s. Current scoring callers pass 3s. | +| mc_runtime_inspect ARGS... | Read container properties through the same runtime/user boundary. | +| mc_runtime_info ARGS... | Validate runtime access through the service owner. | +| mc_lifecycle_start SERVICE | Start the configured user-systemd service. | +| mc_lifecycle_stop SERVICE | Stop that service, preserving failures. | +| mc_lifecycle_running SERVICE | systemctl --user is-active --quiet. | +| mc_lifecycle_cat SERVICE | Inspect service definition for existing validation. | +| mc_application_monitor | Return the application's normalized OCF health/role code. | +| mc_application_standby_score | Emit the eligible standby score, or fail; PostgreSQL owns evidence and freshness. | +| mc_application_action ACTION | Dispatch only start/stop/promote/demote/rejoin to the existing application functions. | +| mc_application_validate | Run existing application/configuration validation. | +| mc_application_metadata | Return application metadata without runtime or CIB access. | +| mc_application_notify | Process the existing transition notification. | + +The runtime exec argument grammar currently follows the existing callers: +`--user USER`, optional `--env KEY=VALUE` entries, container name, command and +arguments. Inspect retains the existing Go-template argument. This is a narrow +Podman adapter contract, not a claim that every future runtime accepts it. +A second runtime must explicitly implement and test those semantics. + +## Normalization and state + +For this extraction the existing PostgreSQL monitor is the normalizer: it turns +PostgreSQL's role/health observations into OCF codes. No extra JSON protocol or +new independently maintained role state is introduced. + +| Observation | Monitor code | +| --- | ---: | +| Healthy standby | 0 (Unpromoted) | +| Healthy primary | 8 (Promoted) | +| Container absent/stopped, or PostgreSQL stopped | 7 (not running) | +| PostgreSQL unready or failed/invalid role query | 1 (generic error) | + +Container availability, application health/role and promotion eligibility remain +separate concepts. Container-running/PostgreSQL-stopped is a legitimate +maintenance condition, but is not OCF start success. A healthy standby can have +score -INFINITY without losing its Unpromoted role report. Core score validation +still replaces malformed/failed score output with -INFINITY. + +All existing scores, freshness checks, persisted identity/LSNs, timeout locations, +notification handling, fencing assumptions, standby-only start and manual fenced +rejoin rules are preserved. Direct application-plugin calls are internal; +Pacemaker dispatch continues through the score/lifecycle wrappers. + +## Maintenance and lifecycle ownership + +Quadlet remains responsible for the bootstrap entrypoint, `Exec=sleep infinity`, +`RunInit=true`, stop signal/timeout, mounts, image, environment and systemd service +policies. `SuccessExitStatus=143` is systemd's classification of its managed +service; the runtime exec adapter must still return 143 as an application-command +failure. `Restart=on-abnormal` remains solely in the existing unit. No new +restart loop, container creation, container replacement or supervision is added. + +During demote/rejoin, PostgreSQL may be stopped while its container remains +available. Ordinary stop still stops PostgreSQL before asking systemd to stop +the container service. These ordering rules remain in the PostgreSQL lifecycle +functions; the adapters do not reorder them. + +## Preservation and validation + +The original 29 functions are retained by name. Some now call adapter functions, +so their literal hashes differ. `tests/verify_refactor.py` reverses only the +explicit command-boundary substitutions and compares every remaining function +byte with the immutable reference. It also retains all 22 differential lifecycle +cases and verifies original metadata plus the four accumulated added parameters +(freshness plus three selectors). Existing scoring tests use the actual engine +loader and still exercise the original handlers against fake external boundaries. + +`tests/test_plugin_engine.py` checks argument fidelity (including quotes, spaces, +newlines and literal shell syntax), lifecycle ownership, exit-code propagation, +timeout placement, application dispatch, metadata discovery, missing/incompatible +plugins, and unsupported/path-like selections. It also checks that PostgreSQL +modules no longer invoke Podman/systemd or service-user execution directly. + +Run: + +```sh +python3 tests/verify_refactor.py +python3 tests/test_plugin_engine.py +python3 tests/test_promotion_scores.py +python3 tests/test_disconnected_state.py +python3 tests/test_promotion_scheduler.py +python3 tests/test_promotion_cleanup.py +python3 tests/verify_promotion_sql.py +``` + +The last command uses only its own disposable PostgreSQL container. RPM checks +include the plugin tests and verify discovery from the staged installation and +an unrelated working directory. No live HA transitions are part of this branch's +validation. Live lab authorization and actual configuration comparison remain +necessary before deployment. + +## Follow-on work + +After this extraction passes lab validation, a separate change can add direct +Podman lifecycle management for pre-created maintenance containers. That work +must define exit/restart behavior and sole lifecycle ownership. Container creation, +RA-coordinated standby provisioning, a second runtime or another application are +separate extensions. The current PostgreSQL plugin is intentionally still the +only supported application; this is not yet a general multi-application framework. + +## Results for this extraction + +Passed: 22 original differential lifecycle cases, 25 scoring tests, 16 state/monitor +tests, 11 scheduler tests, 4 preference-planner tests, 11 engine contract tests, +and all 9 PostgreSQL 18 SQL scenarios. Fedora 43 RPM build, disposable installation, +15-file ownership/mode/payload checks, and the installed state-helper CLI passed. +Folder-layout build log: `/tmp/mc-plugin-folders-rpm.log`; development artifacts: +`dist/plugin-folders/`. Those build and offline checks did not run live HA transitions; subsequent +operator-run lab transitions are recorded below. + +The core still retains the existing score vocabulary and freshness-parameter +validation for this sole supported PostgreSQL profile. API 1 is an internal +extraction interface, not a stable third-party plugin SDK. Generalizing these +remaining policy assumptions should be driven by a second real application, +not assumed complete by this milestone. + +Basic ELITEDESK installation checks subsequently passed with score 100; all 15 +installed files match the candidate. A controlled ELITEDESK standby stop/start subsequently passed according to +operator-supplied output: stopped score -INFINITY, resumed streaming score 100, +same database and new postmaster. The operator subsequently reported successful controlled switchovers in both +directions and a rewind-only rejoin on ELITEDESK; see [role test results](lab-role-tests.md). +Fencing, Pacemaker-managed transitions and divergent-data rewind remain untested +with this extraction. +See [the lab record](lab-plugin-basic-test.md) for evidence attribution. diff --git a/docs/testing.md b/docs/testing.md index e5d00a3..7bc3c9a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,3 +1,6 @@ +> These production reports and source hashes describe the 488a167 freeze. +> The current undeployed extraction is described in [plugin-engine.md](plugin-engine.md). + # Validation and freeze-point evidence ## Reported production fencing and recovery test @@ -69,7 +72,7 @@ whether any code changed since the test. Hash the running container's bootstrap; a host snapshot or image tag alone is not equivalent. Do not dump environment, secret files, or pgpass. Compare every node individually, preserving mismatches. -## Repository freeze snapshot +## Repository freeze snapshot (488a167) The exact repository code hashes appear below and in the reviewed checksum manifest. These identify the source freeze only, pending production comparison. @@ -94,7 +97,7 @@ not yet confirmed identical to the production bootstrap. ## Non-production checks -The source retains all 29 original lifecycle function bodies and original +The frozen source retained all 29 original lifecycle function bodies and original parameter defaults. Added scoring changes public dispatch and metadata; it is not a claim of identical total behavior to the monolithic reference. diff --git a/notes/REFACTOR-SHA256SUMS b/notes/REFACTOR-SHA256SUMS index 9d34045..8f58e81 100644 --- a/notes/REFACTOR-SHA256SUMS +++ b/notes/REFACTOR-SHA256SUMS @@ -1,39 +1,56 @@ -e2df02e66e120481eb62894b2356f40030c63d5bcf734de2cb7ee64bda8b3376 .gitea/workflows/build-rpm.yml +175ccb8e4b851fe1c06e3490dbd1bb0ab12f9b1c8374dd001effd88256b9dde2 .gitea/workflows/build-rpm.yml bb9026b0c528721ab80038344929a8bcd0ab8c5c8e438a22978bc362572d6112 .gitignore 4351008e41de847f0dd8323843a28ea3b0fbe100d0268faa77183cdb4f7c7da0 HANDOFF.md -6e3cf9e32b85808ff503ac4d07f1dff69e002f0dbcb1630ad3c36ca32e530afb README.md -13b689315ab43e830473aa3c97e37390a5417b4681fab2774c6072b52b3266b4 docs/operations.md +b43b3c36a6a039002b40bf655d7af06d25f240059466ed457a97f8ed841bb6a5 README.md +6832fdee40cf85b006dfd4b542d8a2ee4f9498775f0e7cea310df765f8b00203 docs/lab-installed-before-plugin.sha256 +ab6ffa63105281fd4cb010f0b954367d8dfd4163528eb5679a1c25f499c5ed29 docs/lab-installed-plugin.sha256 +9747927dfbe8a87db803e4f3139ce2386e8cb849726741b6a595158667fefa13 docs/lab-plugin-basic-test.md +e3224fd761f46a518a705380ea867f4b8e55086e55a47f8272d0dce2afdc3e0d docs/operations.md +5678ef595be40be398c51ccb630884cb1ecdb7431922600eba0b80aa1f4ab8bf docs/plugin-engine.md 3e5f075baa7863bee3d12dd50319466291e30e7be895dfe16e9724da9ae4f021 docs/review.md -6e0ce0054b478dcae608b6f35c6614bbfae8ef8bb978144cfc6d560de6d57d0f docs/testing.md +8b7455b45f27dda61071be549abaffa1ac033ac7f652c0831402bde647b8be4c docs/testing.md bfdeecfb1791c4f1d8ec08f28774f94769fffeb51b3268342fd6c63e2322eb23 notes/DISCONNECTED-STANDBY.md 0a6af2a5fbf1ea1d9551893578df094442ee10e032e1b2de7c92f9662b133ad3 notes/PROMOTION-SCORING.md 6bdc385dff0e66de372f802a577f67d852f7a2a0e45610865f558ee3b93adf20 notes/REFACTOR-PLAN.md 05b860269ba536361f7b806f415ffb6c2f6632ecf8374bfcd02e7bb7ad1069bb notes/REFACTOR-REPORT.md 2d728176c301e24f673b8d36a9ae0fd3bd3816d37b1f945100b37a74779887ef packaging/Containerfile 676455c64d130eb1ad36e007f023f0f30c98beedf87f18be74ae33202b646ffa packaging/PROVENANCE.md -ba3fe4e1ee60ec9445c4c2cc6591e90b674c7113474ec8b33edb4f1016dc0ae2 packaging/README.md +1a05f3153b0bd8c83dd01b925670193ca763caceee9dc07bf63ac2e71eff9d7b packaging/README.md e9dd8507f4bf0c6f42458e41aea833ad0bd3f6127272335eee9bf4d58541ed67 packaging/VERSION -4b9e13339a1eacbced38c69edf2a8c518997160104aab1bbfe6d2f019a93d6fa packaging/ocf-managed-container.spec -a4b775a0ddcabdb0558da9b34c7fe2a883c07d0ef308c1c7224526abf53f63a0 refactored/managed-container -93191771d3cb8aefadb2a16d2f89d64a8b6c584071a473439be818237bdf7f17 refactored/managed-container.d/auth.sh -97e93d9164e854e5f3688cf9d3cc656b001eb30ef3c2073c907450ca43588f3d refactored/managed-container.d/common.sh -ac6bc363a2301f1491d162ef609103dbfbceb3b099a3556614c6165952309374 refactored/managed-container.d/container.sh -0af3bb362a303017372c631354ba94ef9e723bf14e64d2eff91b6662e54a5be0 refactored/managed-container.d/demote.sh -6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670 refactored/managed-container.d/notify.sh -d2779ce6e7d1f426ffca2e41bbb8de77503005dc38d658d69f184020365796eb refactored/managed-container.d/postgres.sh -3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58 refactored/managed-container.d/promote.sh -5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7 refactored/managed-container.d/promotion_state.py -21beea72bcb7f24a51c17ea48666ac87df9436702ad77b166b45f4647e28e8ad refactored/managed-container.d/rejoin.sh -6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547 refactored/managed-container.d/replication.sh +93b5a897b8a6b8f92687c242df353fae9e6a3b24872fadd1553252fb8a227512 packaging/ocf-managed-container.spec +01eb9b161c8e3db289f09bf63953be779921e15bdf8b1a4d8ce7e60032b826e4 refactored/managed-container +10932919c4339b553c7c9dbf30b45a917c51c0a08f362e12678003d58b12fa82 refactored/managed-container.d/applications/postgresql/auth.sh +da4e58cadf1df3953cb97a0512e3931ee1b9d3ee61c39a1ad1a6047ef537addc refactored/managed-container.d/applications/postgresql/demote.sh +6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670 refactored/managed-container.d/applications/postgresql/notify.sh +d3a989de97ceb61a56828f006d5ccb87b7c34fc155a944585bedf928785021fc refactored/managed-container.d/applications/postgresql/plugin.sh +fa222e6767249999767eb82fac2d85e7e35404da240517a8b630571e77282053 refactored/managed-container.d/applications/postgresql/postgres.sh +3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58 refactored/managed-container.d/applications/postgresql/promote.sh +5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7 refactored/managed-container.d/applications/postgresql/promotion_state.py +28b0e44554a88ccb87cd16a68315c045dd37d354aceb2fa25da7c7399a659b23 refactored/managed-container.d/applications/postgresql/rejoin.sh +6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547 refactored/managed-container.d/applications/postgresql/replication.sh +a482ef3cc1048e6a4f427e76dd2dce173ffd00760dbed1afa601a3fd05b0ad99 refactored/managed-container.d/common.sh +3a27ac9c63c61310da8b8711fa2c4400b37ea8eae2ee2a79e28bc63180b06c7c refactored/managed-container.d/container.sh +53978e2367394695d61c6fb38f2ee46814474c5a4e0448d2692f1a3ce583af93 refactored/managed-container.d/engine.sh +6c92c095ce10ac95d843fdc1c49e31c19491ca22fdac960affd8649606d76d82 refactored/managed-container.d/lifecycles/systemd.sh +dc5820067a78c4a1c531426902793aae97a03fd64912b017b3b102031dcc1e04 refactored/managed-container.d/runtimes/podman.sh 3f1a15f7054fb69b9aa92204de30306fd6053c63560310f4231f314b148e2dcf runtime-context/postgres-ha-bootstrap 17ce35741b0ef55a6e4785a093a5097f13139edd428fd4c65e8f50d222161b65 scripts/build-rpm.sh 42744b1b407ab4a79725bd90d7d067f46dd8b36728f104c52544d464b2e323c3 scripts/install-elitedesk-ra.sh 470fd174e42efef876e540dfbfa42b4f36bf70b29a26ab3e5f0c10c813f3b6ae scripts/plan-promotion-cleanup.py -52162183beb2eb258b2336c4fd6392319e2ac7415372b623226bd1dd4ff0720d tests/test_disconnected_state.py +7c426f3924cdaa92252987c784844d7a93675bee3259b536fba9a9347fcdc600 scripts/test-elitedesk-cycle.py +5d9ddb7d554508e02bd0c4ab4aa208998d6b54b489727aa901b946d2990e8460 scripts/test-elitedesk-plugin.py +7e2ec6e9201b239607c5119674543a5d8ae53e9bb54bc5d02bb1119d1769cee1 tests/plugin_test_support.py +9e4db508ebabc077a99299c3df1f6881948fc5766d326cb4bccb70fddbd84889 tests/test_disconnected_state.py +b6356d055861fa014b6210a193f565fec23f08935dab80e1320df77d558d1bbd tests/test_lab_cycle.py +347b05d4bcd4df9e0f8fcbd0cda10d3d54113c88635e54f7d94d91c74d4cfe05 tests/test_lab_installer.py +81ce2770d872a064d0ac0ff0c53ad3851650d9002249d79eac882558d2b077c8 tests/test_plugin_engine.py e44ac0e6fc322d7e314814c69d9c3b1ab582b43afbf8b66bd30a07713682ae1c tests/test_promotion_cleanup.py eb88cdaffeccb00c92e1c43725112ae16aa15faf923b84db62ae5950c5c26d2e tests/test_promotion_scheduler.py -29bac1cc5939aaf772259bb08f3d2cfae2186e081fcb203c4979a32d0a60e0c0 tests/test_promotion_scores.py -40211e06e4b23f0e1842759cfeb4508a7dbd2ec985da9bdef090efa017f4e02d tests/verify_installed_rpm.py -d2280256e451afca90d035e85998ed9ebab8ec21aa7c0aa4605fed7619ba24b9 tests/verify_promotion_sql.py -c7744c533055dac5744fcb54a56ff1dd3f95940185ddc860a69dc2dd88669819 tests/verify_promotion_state_cli.py -dc7d1c8b47c4f0d4edbfacd38714405d6da8d3ae6b183f363afd264e4a0229e5 tests/verify_refactor.py +ac3976873c018d0c5aa6eaf496fb3356df01fb26bc1e782a5fcb72aeb3baa1c3 tests/test_promotion_scores.py +6d865d4abaee977646b5ee51ea940ee15e06ca5fb46010ec48fb8a94d4303711 tests/verify_installed_rpm.py +6bae43d0b5b702078bdd8a1fb6afc7ac3eb7b9f6d05c94128fdab45bd28f05c2 tests/verify_promotion_sql.py +d51bc222bddf861dddb3b6dd7f2694666c38d3d46b4cddbe13ce86dcc2836994 tests/verify_promotion_state_cli.py +0226e88f0c997b7a1c99c810534e24e48efa14210bb5183bb1ac6c3425398e2b tests/verify_refactor.py +4d6036b4b8aa45f3b297fafc868b045ab904e421b12da1fea1f57ef4785ca93f scripts/test-lab-roles.py +e39a786f3fb8d67b8f52faf74650b2a63cfe090567a1117b60fd338add31cadb tests/test_lab_roles.py +0c41d7db4094db56093efeae2dd5bdc25b5076d0c6ec86feb1ea98f922a1f924 docs/lab-role-tests.md diff --git a/packaging/README.md b/packaging/README.md index 886f276..3efa492 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -5,7 +5,7 @@ Fedora 43, matching the saved lab package context. The workstation currently reports Fedora 44; this first workflow does not claim a Fedora 44 build test. It runs on main pushes, pull requests and manual dispatch. -The build installs only the OCF executable and nine shell modules, one Python state helper, plus documentation. +The build installs only the OCF executable and 13 shell modules, one Python state helper, plus documentation. It includes no Quadlet, secrets, databases or cluster configuration. There are no RPM lifecycle scriptlets. Installation does not start, stop or restart a service or change maintenance mode. Host PostgreSQL packages are not required. @@ -85,3 +85,7 @@ The disconnected candidate adds a runtime Python 3 dependency and advertises Current operating policy and operator-reported production validation are packaged from `docs/`. The source freeze is distinct from verified deployed-byte identity; see [testing](../docs/testing.md) for production snapshot status. + +The plugin extraction adds three selectors, defaulting to PostgreSQL/Podman/systemd. +The package includes all API-1 plugin files and `plugin-engine.md`; Docker and +direct-runtime lifecycle management are not implemented. diff --git a/packaging/ocf-managed-container.spec b/packaging/ocf-managed-container.spec index e97247c..6adef26 100644 --- a/packaging/ocf-managed-container.spec +++ b/packaging/ocf-managed-container.spec @@ -51,7 +51,11 @@ cp packaging/README.md RPM-BUILD.md %install install -d %{buildroot}%{ocf_provider_dir}/managed-container.d install -m 0755 refactored/managed-container %{buildroot}%{ocf_provider_dir}/managed-container -install -m 0644 refactored/managed-container.d/*.sh refactored/managed-container.d/*.py %{buildroot}%{ocf_provider_dir}/managed-container.d/ +for subdir in . applications/postgresql lifecycles runtimes; do + install -d %{buildroot}%{ocf_provider_dir}/managed-container.d/$subdir + install -m 0644 refactored/managed-container.d/$subdir/*.sh %{buildroot}%{ocf_provider_dir}/managed-container.d/$subdir/ +done +install -m 0644 refactored/managed-container.d/applications/postgresql/promotion_state.py %{buildroot}%{ocf_provider_dir}/managed-container.d/applications/postgresql/ %check python3 tests/verify_refactor.py @@ -59,6 +63,7 @@ python3 tests/test_promotion_scores.py python3 tests/test_promotion_scheduler.py python3 tests/test_promotion_cleanup.py python3 tests/test_disconnected_state.py +python3 tests/test_plugin_engine.py # Check the actual staged layout, including resolution from an unrelated cwd. agent="%{buildroot}%{ocf_provider_dir}/managed-container" "$agent" meta-data > staged-metadata.xml @@ -70,9 +75,16 @@ cmp reference-metadata.xml other-cwd-metadata.xml %files %defattr(-,root,root,-) %doc README.md packaging/PROVENANCE.md RPM-BUILD.md notes/PROMOTION-SCORING.md notes/DISCONNECTED-STANDBY.md -%doc docs/operations.md docs/testing.md docs/review.md +%doc docs/operations.md docs/testing.md docs/review.md docs/plugin-engine.md %dir %{ocf_provider_dir} %attr(0755,root,root) %{ocf_provider_dir}/managed-container %dir %{ocf_provider_dir}/managed-container.d %attr(0644,root,root) %{ocf_provider_dir}/managed-container.d/*.sh -%attr(0644,root,root) %{ocf_provider_dir}/managed-container.d/*.py +%dir %{ocf_provider_dir}/managed-container.d/applications +%dir %{ocf_provider_dir}/managed-container.d/applications/postgresql +%dir %{ocf_provider_dir}/managed-container.d/lifecycles +%dir %{ocf_provider_dir}/managed-container.d/runtimes +%attr(0644,root,root) %{ocf_provider_dir}/managed-container.d/applications/postgresql/*.sh +%attr(0644,root,root) %{ocf_provider_dir}/managed-container.d/applications/postgresql/*.py +%attr(0644,root,root) %{ocf_provider_dir}/managed-container.d/lifecycles/*.sh +%attr(0644,root,root) %{ocf_provider_dir}/managed-container.d/runtimes/*.sh diff --git a/refactored/managed-container b/refactored/managed-container index 42435a9..3ac9fad 100755 --- a/refactored/managed-container +++ b/refactored/managed-container @@ -26,18 +26,19 @@ MC_PROMOTION_RESOURCE_INSTANCE="${OCF_RESOURCE_INSTANCE:-}" # Resolve modules relative to this entrypoint, independent of Pacemaker's cwd. MC_MODULE_DIR="$(CDPATH= cd -- "$(command dirname -- "${BASH_SOURCE[0]}")" && pwd)/managed-container.d" -for MC_MODULE in common container postgres auth replication notify promote demote rejoin; do +for MC_MODULE in common container engine; do if ! . "${MC_MODULE_DIR}/${MC_MODULE}.sh"; then ocf_exit_reason "Unable to load resource-agent module ${MC_MODULE_DIR}/${MC_MODULE}.sh" exit "${OCF_ERR_CONFIGURED}" fi done unset MC_MODULE +mc_engine_load || exit $? case "$1" in meta-data) - mc_promotion_metadata - exit "${OCF_SUCCESS}" + mc_engine_metadata + exit $? ;; validate-all) @@ -71,7 +72,7 @@ case "$1" in ;; notify) - managed_notify + mc_application_notify exit $? ;; diff --git a/refactored/managed-container.d/auth.sh b/refactored/managed-container.d/applications/postgresql/auth.sh similarity index 98% rename from refactored/managed-container.d/auth.sh rename to refactored/managed-container.d/applications/postgresql/auth.sh index 8dc19a1..ae0a533 100644 --- a/refactored/managed-container.d/auth.sh +++ b/refactored/managed-container.d/applications/postgresql/auth.sh @@ -2,8 +2,7 @@ pg_replication_exec() { - as_service_user \ - podman exec \ + mc_runtime_exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ sh -c ' diff --git a/refactored/managed-container.d/demote.sh b/refactored/managed-container.d/applications/postgresql/demote.sh similarity index 99% rename from refactored/managed-container.d/demote.sh rename to refactored/managed-container.d/applications/postgresql/demote.sh index 4aecfa7..c4f465a 100644 --- a/refactored/managed-container.d/demote.sh +++ b/refactored/managed-container.d/applications/postgresql/demote.sh @@ -130,8 +130,7 @@ managed_demote() ocf_log info \ "Configuring ${local_node} as cascading standby of ${target_node}" - if ! as_service_user \ - podman exec \ + if ! mc_runtime_exec \ --user "${PG_OS_USER}" \ --env "RA_PRIMARY_HOST=${target_addr}" \ --env "RA_PRIMARY_PORT=${target_port}" \ diff --git a/refactored/managed-container.d/notify.sh b/refactored/managed-container.d/applications/postgresql/notify.sh similarity index 100% rename from refactored/managed-container.d/notify.sh rename to refactored/managed-container.d/applications/postgresql/notify.sh diff --git a/refactored/managed-container.d/applications/postgresql/plugin.sh b/refactored/managed-container.d/applications/postgresql/plugin.sh new file mode 100644 index 0000000..19cb6db --- /dev/null +++ b/refactored/managed-container.d/applications/postgresql/plugin.sh @@ -0,0 +1,281 @@ +# Sourced by managed-container; uses the shared OCF and parameter environment. + +OCF_RESKEY_service_user_default="david" +OCF_RESKEY_service_name_default="postgres.service" +OCF_RESKEY_container_name_default="postgres" +OCF_RESKEY_pg_os_user_default="postgres" +OCF_RESKEY_db_user_default="david" +OCF_RESKEY_db_name_default="david" +OCF_RESKEY_rejoin_source_host_default="" +OCF_RESKEY_rejoin_source_port_default="5432" +OCF_RESKEY_replication_user_default="replicator" +OCF_RESKEY_replication_slot_default="" +OCF_RESKEY_application_name_default="" +OCF_RESKEY_rejoin_allow_basebackup_default="false" + +: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}" +: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}" +: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}" +: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}" +: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}" +: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}" +: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}" +: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}" +: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}" +: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}" +: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}" +: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}" + +SERVICE_USER="${OCF_RESKEY_service_user}" +SERVICE_NAME="${OCF_RESKEY_service_name}" +CONTAINER_NAME="${OCF_RESKEY_container_name}" +PG_OS_USER="${OCF_RESKEY_pg_os_user}" +DB_USER="${OCF_RESKEY_db_user}" +DB_NAME="${OCF_RESKEY_db_name}" +REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}" +REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}" +REPLICATION_USER="${OCF_RESKEY_replication_user}" +REPLICATION_SLOT="${OCF_RESKEY_replication_slot}" +APPLICATION_NAME="${OCF_RESKEY_application_name}" +REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}" + +if [ -z "${APPLICATION_NAME}" ]; then + APPLICATION_NAME="$(hostname -s)" +fi + +meta_data() +{ + cat < + + + 1.0 + + + Manage an application inside a persistent rootless container. + + Version 0.4 implements role-aware monitoring, promotion, and demotion. + The systemd user Quadlet owns the container lifecycle while pg_ctl owns + the PostgreSQL lifecycle inside the container. + + + + Manage PostgreSQL independently inside a persistent container + + + + + + + Linux user owning the rootless systemd/Podman service. + + Rootless service owner + + + + + + systemd user service generated by Quadlet. + + Quadlet service name + + + + + + Podman container containing PostgreSQL. + + Container name + + + + + + Operating-system user running PostgreSQL inside the container. + + PostgreSQL OS user + + + + + + PostgreSQL database user used for health queries. + + Database user + + + + + + PostgreSQL database used for health queries. + + Database name + + + + + + + PostgreSQL primary host used as the source for rejoin operations. + + Rejoin source host + + + + + + PostgreSQL port on the rejoin source. + + Rejoin source port + + + + + + PostgreSQL role used for replication and pg_rewind. + + Replication user + + + + + + Physical replication slot used by this standby. + + Physical replication slot + + + + + + PostgreSQL replication application_name. Defaults to the local hostname. + + Replication application name + + + + + + Allow rejoin to fall back to pg_basebackup when pg_rewind cannot + repair the local PostgreSQL data directory. The old PGDATA is + preserved before replacement. This must only be enabled when the + operator has established that this node may safely discard its + divergent PostgreSQL history. + + Allow pg_basebackup rejoin fallback + + + + + + + + + + + + + + + + +EOF_META +} + +validate_all() +{ + if ! id "${SERVICE_USER}" >/dev/null 2>&1; then + ocf_exit_reason "Service user '${SERVICE_USER}' does not exist" + return "${OCF_ERR_CONFIGURED}" + fi + + for cmd in runuser systemctl podman getent; do + if ! command -v "${cmd}" >/dev/null 2>&1; then + ocf_exit_reason "Required command '${cmd}' is not installed" + return "${OCF_ERR_INSTALLED}" + fi + done + + if [ ! -d "/run/user/$(service_uid)" ]; then + ocf_exit_reason \ + "Runtime directory /run/user/$(service_uid) does not exist" + return "${OCF_ERR_GENERIC}" + fi + + if ! mc_lifecycle_cat "${SERVICE_NAME}" >/dev/null 2>&1 + then + ocf_exit_reason \ + "User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}" + return "${OCF_ERR_CONFIGURED}" + fi + + local podman_error + + podman_error="$( + mc_runtime_info 2>&1 >/dev/null + )" + local podman_rc=$? + + if [ "${podman_rc}" -ne 0 ]; then + ocf_exit_reason \ + "Rootless Podman failed for ${SERVICE_USER}: ${podman_error}" + return "${OCF_ERR_GENERIC}" + fi + + return "${OCF_SUCCESS}" +} + + +# Keep the original metadata function intact; advertise the added policy here. +mc_promotion_metadata() +{ + meta_data | sed '/<\/parameters>/i\ + \ + Maximum age in seconds of a verified streaming observation for disconnected promotion eligibility. Default 300; 0 disables disconnected eligibility. Requires fencing managed by Pacemaker. PostgreSQL restart requires fresh streaming verification.\ + Disconnected standby observation lifetime\ + \ + ' +} + +MC_APPLICATION_API=1 +MC_APPLICATION_DIR="${MC_MODULE_DIR}/applications/postgresql" +for MC_PG_MODULE in postgres auth replication notify promote demote rejoin; do + if ! . "${MC_APPLICATION_DIR}/${MC_PG_MODULE}.sh"; then + ocf_exit_reason "Unable to load resource-agent module ${MC_APPLICATION_DIR}/${MC_PG_MODULE}.sh" + return "${OCF_ERR_CONFIGURED}" + fi +done +unset MC_PG_MODULE + +mc_application_monitor() +{ + managed_monitor +} + +mc_application_standby_score() +{ + mc_pg_standby_score +} + +mc_application_validate() +{ + validate_all +} + +mc_application_metadata() +{ + mc_promotion_metadata +} + +mc_application_notify() +{ + managed_notify +} + +mc_application_action() +{ + case "$1" in + start|stop|promote|demote|rejoin) "managed_$1" ;; + *) return "${OCF_ERR_UNIMPLEMENTED}" ;; + esac +} diff --git a/refactored/managed-container.d/postgres.sh b/refactored/managed-container.d/applications/postgresql/postgres.sh similarity index 94% rename from refactored/managed-container.d/postgres.sh rename to refactored/managed-container.d/applications/postgresql/postgres.sh index 9b3dc2c..1740e86 100644 --- a/refactored/managed-container.d/postgres.sh +++ b/refactored/managed-container.d/applications/postgresql/postgres.sh @@ -4,8 +4,7 @@ pg_command() { local command="$1" - as_service_user \ - podman exec \ + mc_runtime_exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ sh -c "${command}" @@ -32,8 +31,7 @@ pg_running() pg_ready() { - as_service_user \ - podman exec \ + mc_runtime_exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ pg_isready -q \ @@ -43,8 +41,7 @@ pg_ready() pg_role() { - as_service_user \ - podman exec \ + mc_runtime_exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ psql \ @@ -55,8 +52,7 @@ pg_role() pg_wal_receiver_status() { - as_service_user \ - podman exec \ + mc_runtime_exec \ --user "${PG_OS_USER}" \ "${CONTAINER_NAME}" \ psql \ @@ -153,8 +149,7 @@ managed_start() "Ensuring container service ${SERVICE_NAME} is running" if ! service_running; then - if ! as_service_user \ - systemctl --user start "${SERVICE_NAME}" + if ! mc_lifecycle_start "${SERVICE_NAME}" then ocf_exit_reason \ "Failed to start ${SERVICE_NAME}" @@ -260,8 +255,7 @@ managed_stop() ocf_log info \ "Stopping container service ${SERVICE_NAME}" - if ! as_service_user \ - systemctl --user stop "${SERVICE_NAME}" + if ! mc_lifecycle_stop "${SERVICE_NAME}" then ocf_exit_reason \ "Failed to stop ${SERVICE_NAME}" @@ -299,8 +293,7 @@ mc_pg_promotion_eligible() local eligible eligible="$( - as_service_user timeout --kill-after=1s 3s \ - podman exec --user "${PG_OS_USER}" \ + mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" \ --env 'PGOPTIONS=-c statement_timeout=2000' \ "${CONTAINER_NAME}" \ psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc " @@ -323,8 +316,7 @@ mc_pg_promotion_eligible() # including a new basebackup sharing the same PostgreSQL system identifier. mc_pg_standby_observation() { - as_service_user timeout --kill-after=1s 3s \ - podman exec --user "${PG_OS_USER}" \ + mc_runtime_exec_bounded 3s --user "${PG_OS_USER}" \ --env 'PGOPTIONS=-c statement_timeout=2000' \ "${CONTAINER_NAME}" \ psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc " @@ -358,7 +350,7 @@ mc_pg_standby_score() # The helper is root-owned alongside the modules; state never enters the # rootless container. A failed/slow helper always withdraws eligibility. printf '%s' "${observation}" | timeout --kill-after=1s 1s python3 \ - "${MC_MODULE_DIR}/promotion_state.py" \ + "${MC_APPLICATION_DIR}/promotion_state.py" \ "${MC_PROMOTION_RESOURCE_INSTANCE:-}" \ "${SERVICE_USER}" "${SERVICE_NAME}" "${CONTAINER_NAME}" \ "${OCF_RESKEY_disconnected_max_age:-300}" diff --git a/refactored/managed-container.d/promote.sh b/refactored/managed-container.d/applications/postgresql/promote.sh similarity index 100% rename from refactored/managed-container.d/promote.sh rename to refactored/managed-container.d/applications/postgresql/promote.sh diff --git a/refactored/managed-container.d/promotion_state.py b/refactored/managed-container.d/applications/postgresql/promotion_state.py similarity index 100% rename from refactored/managed-container.d/promotion_state.py rename to refactored/managed-container.d/applications/postgresql/promotion_state.py diff --git a/refactored/managed-container.d/rejoin.sh b/refactored/managed-container.d/applications/postgresql/rejoin.sh similarity index 99% rename from refactored/managed-container.d/rejoin.sh rename to refactored/managed-container.d/applications/postgresql/rejoin.sh index 7c500b9..dc39c39 100644 --- a/refactored/managed-container.d/rejoin.sh +++ b/refactored/managed-container.d/applications/postgresql/rejoin.sh @@ -268,8 +268,7 @@ managed_rejoin() ocf_log info \ "Configuring PostgreSQL as standby of ${REJOIN_SOURCE_HOST}" - if ! as_service_user \ - podman exec \ + if ! mc_runtime_exec \ --user "${PG_OS_USER}" \ --env "RA_PRIMARY_HOST=${REJOIN_SOURCE_HOST}" \ --env "RA_PRIMARY_PORT=${REJOIN_SOURCE_PORT}" \ diff --git a/refactored/managed-container.d/replication.sh b/refactored/managed-container.d/applications/postgresql/replication.sh similarity index 100% rename from refactored/managed-container.d/replication.sh rename to refactored/managed-container.d/applications/postgresql/replication.sh diff --git a/refactored/managed-container.d/common.sh b/refactored/managed-container.d/common.sh index 70f2a08..6f1e7b7 100644 --- a/refactored/managed-container.d/common.sh +++ b/refactored/managed-container.d/common.sh @@ -1,230 +1,4 @@ -# Sourced by managed-container; uses the shared OCF and parameter environment. - -OCF_RESKEY_service_user_default="david" -OCF_RESKEY_service_name_default="postgres.service" -OCF_RESKEY_container_name_default="postgres" -OCF_RESKEY_pg_os_user_default="postgres" -OCF_RESKEY_db_user_default="david" -OCF_RESKEY_db_name_default="david" -OCF_RESKEY_rejoin_source_host_default="" -OCF_RESKEY_rejoin_source_port_default="5432" -OCF_RESKEY_replication_user_default="replicator" -OCF_RESKEY_replication_slot_default="" -OCF_RESKEY_application_name_default="" -OCF_RESKEY_rejoin_allow_basebackup_default="false" - -: "${OCF_RESKEY_service_user:=${OCF_RESKEY_service_user_default}}" -: "${OCF_RESKEY_service_name:=${OCF_RESKEY_service_name_default}}" -: "${OCF_RESKEY_container_name:=${OCF_RESKEY_container_name_default}}" -: "${OCF_RESKEY_pg_os_user:=${OCF_RESKEY_pg_os_user_default}}" -: "${OCF_RESKEY_db_user:=${OCF_RESKEY_db_user_default}}" -: "${OCF_RESKEY_db_name:=${OCF_RESKEY_db_name_default}}" -: "${OCF_RESKEY_rejoin_source_host:=${OCF_RESKEY_rejoin_source_host_default}}" -: "${OCF_RESKEY_rejoin_source_port:=${OCF_RESKEY_rejoin_source_port_default}}" -: "${OCF_RESKEY_replication_user:=${OCF_RESKEY_replication_user_default}}" -: "${OCF_RESKEY_replication_slot:=${OCF_RESKEY_replication_slot_default}}" -: "${OCF_RESKEY_application_name:=${OCF_RESKEY_application_name_default}}" -: "${OCF_RESKEY_rejoin_allow_basebackup:=${OCF_RESKEY_rejoin_allow_basebackup_default}}" - -SERVICE_USER="${OCF_RESKEY_service_user}" -SERVICE_NAME="${OCF_RESKEY_service_name}" -CONTAINER_NAME="${OCF_RESKEY_container_name}" -PG_OS_USER="${OCF_RESKEY_pg_os_user}" -DB_USER="${OCF_RESKEY_db_user}" -DB_NAME="${OCF_RESKEY_db_name}" -REJOIN_SOURCE_HOST="${OCF_RESKEY_rejoin_source_host}" -REJOIN_SOURCE_PORT="${OCF_RESKEY_rejoin_source_port}" -REPLICATION_USER="${OCF_RESKEY_replication_user}" -REPLICATION_SLOT="${OCF_RESKEY_replication_slot}" -APPLICATION_NAME="${OCF_RESKEY_application_name}" -REJOIN_ALLOW_BASEBACKUP="${OCF_RESKEY_rejoin_allow_basebackup}" - -if [ -z "${APPLICATION_NAME}" ]; then - APPLICATION_NAME="$(hostname -s)" -fi - -meta_data() -{ - cat < - - - 1.0 - - - Manage an application inside a persistent rootless container. - - Version 0.4 implements role-aware monitoring, promotion, and demotion. - The systemd user Quadlet owns the container lifecycle while pg_ctl owns - the PostgreSQL lifecycle inside the container. - - - - Manage PostgreSQL independently inside a persistent container - - - - - - - Linux user owning the rootless systemd/Podman service. - - Rootless service owner - - - - - - systemd user service generated by Quadlet. - - Quadlet service name - - - - - - Podman container containing PostgreSQL. - - Container name - - - - - - Operating-system user running PostgreSQL inside the container. - - PostgreSQL OS user - - - - - - PostgreSQL database user used for health queries. - - Database user - - - - - - PostgreSQL database used for health queries. - - Database name - - - - - - - PostgreSQL primary host used as the source for rejoin operations. - - Rejoin source host - - - - - - PostgreSQL port on the rejoin source. - - Rejoin source port - - - - - - PostgreSQL role used for replication and pg_rewind. - - Replication user - - - - - - Physical replication slot used by this standby. - - Physical replication slot - - - - - - PostgreSQL replication application_name. Defaults to the local hostname. - - Replication application name - - - - - - Allow rejoin to fall back to pg_basebackup when pg_rewind cannot - repair the local PostgreSQL data directory. The old PGDATA is - preserved before replacement. This must only be enabled when the - operator has established that this node may safely discard its - divergent PostgreSQL history. - - Allow pg_basebackup rejoin fallback - - - - - - - - - - - - - - - - -EOF_META -} - -validate_all() -{ - if ! id "${SERVICE_USER}" >/dev/null 2>&1; then - ocf_exit_reason "Service user '${SERVICE_USER}' does not exist" - return "${OCF_ERR_CONFIGURED}" - fi - - for cmd in runuser systemctl podman getent; do - if ! command -v "${cmd}" >/dev/null 2>&1; then - ocf_exit_reason "Required command '${cmd}' is not installed" - return "${OCF_ERR_INSTALLED}" - fi - done - - if [ ! -d "/run/user/$(service_uid)" ]; then - ocf_exit_reason \ - "Runtime directory /run/user/$(service_uid) does not exist" - return "${OCF_ERR_GENERIC}" - fi - - if ! as_service_user \ - systemctl --user cat "${SERVICE_NAME}" >/dev/null 2>&1 - then - ocf_exit_reason \ - "User service '${SERVICE_NAME}' cannot be found for ${SERVICE_USER}" - return "${OCF_ERR_CONFIGURED}" - fi - - local podman_error - - podman_error="$( - as_service_user podman info 2>&1 >/dev/null - )" - local podman_rc=$? - - if [ "${podman_rc}" -ne 0 ]; then - ocf_exit_reason \ - "Rootless Podman failed for ${SERVICE_USER}: ${podman_error}" - return "${OCF_ERR_GENERIC}" - fi - - return "${OCF_SUCCESS}" -} +# Shared OCF scoring wrappers; application policy is supplied by the plugin. # Promotion scores are scoped by Pacemaker to OCF_RESOURCE_INSTANCE and the # executing node. Do not construct a legacy attribute name or cache locally: @@ -269,7 +43,7 @@ mc_promotion_score_from_monitor() case "${monitor_rc}" in "${OCF_RUNNING_PROMOTED}") score=1000 ;; "${OCF_SUCCESS}") - score="$(mc_pg_standby_score)" || score=-INFINITY + score="$(mc_application_standby_score)" || score=-INFINITY ;; esac case "${score}" in @@ -284,7 +58,7 @@ mc_promotion_monitor() { local monitor_rc - managed_monitor + mc_application_monitor monitor_rc=$? mc_promotion_score_from_monitor "${monitor_rc}" || true return "${monitor_rc}" @@ -310,10 +84,10 @@ mc_promotion_action() [ "${action_rc}" -eq "${OCF_SUCCESS}" ] || return "${action_rc}" fi - "managed_${action}" + mc_application_action "${action}" action_rc=$? if [ "${action_rc}" -eq "${OCF_SUCCESS}" ] && [ "${action}" != stop ]; then - managed_monitor + mc_application_monitor monitor_rc=$? mc_promotion_score_from_monitor "${monitor_rc}" || true fi @@ -335,16 +109,5 @@ mc_promotion_validate() return "${OCF_ERR_INSTALLED}" fi done - validate_all -} - -# Keep the original metadata function intact; advertise the added policy here. -mc_promotion_metadata() -{ - meta_data | sed '/<\/parameters>/i\ - \ - Maximum age in seconds of a verified streaming observation for disconnected promotion eligibility. Default 300; 0 disables disconnected eligibility. Requires fencing managed by Pacemaker. PostgreSQL restart requires fresh streaming verification.\ - Disconnected standby observation lifetime\ - \ - ' + mc_application_validate } diff --git a/refactored/managed-container.d/container.sh b/refactored/managed-container.d/container.sh index 0610a70..208b477 100644 --- a/refactored/managed-container.d/container.sh +++ b/refactored/managed-container.d/container.sh @@ -37,8 +37,7 @@ as_service_user() service_running() { - as_service_user \ - systemctl --user is-active --quiet "${SERVICE_NAME}" + mc_lifecycle_running "${SERVICE_NAME}" } container_running() @@ -46,7 +45,7 @@ container_running() local state state="$( - as_service_user podman inspect \ + mc_runtime_inspect \ --format '{{.State.Running}}' \ "${CONTAINER_NAME}" 2>/dev/null )" || return 1 diff --git a/refactored/managed-container.d/engine.sh b/refactored/managed-container.d/engine.sh new file mode 100644 index 0000000..8e4b694 --- /dev/null +++ b/refactored/managed-container.d/engine.sh @@ -0,0 +1,59 @@ +# Engine API 1. Selection is an allowlist of packaged implementations, never a +# filename, shell fragment or a plugin downloaded from cluster configuration. +mc_engine_load() +{ + local plugin hook + if [ "${OCF_RESKEY_application:-postgresql}" != postgresql ] || + [ "${OCF_RESKEY_container_runtime:-podman}" != podman ] || + [ "${OCF_RESKEY_lifecycle_manager:-systemd}" != systemd ]; then + ocf_exit_reason "Supported plugin combination: application=postgresql container_runtime=podman lifecycle_manager=systemd" + return "${OCF_ERR_CONFIGURED}" + fi + + unset MC_RUNTIME_API MC_LIFECYCLE_API MC_APPLICATION_API + for plugin in runtimes/podman lifecycles/systemd applications/postgresql/plugin; do + if ! . "${MC_MODULE_DIR}/${plugin}.sh"; then + ocf_exit_reason "Unable to load resource-agent module ${MC_MODULE_DIR}/${plugin}.sh" + return "${OCF_ERR_CONFIGURED}" + fi + done + if [ "${MC_RUNTIME_API:-}" != 1 ] || [ "${MC_LIFECYCLE_API:-}" != 1 ] || + [ "${MC_APPLICATION_API:-}" != 1 ]; then + ocf_exit_reason "Incompatible plugin interface; API 1 required" + return "${OCF_ERR_CONFIGURED}" + fi + for hook in mc_runtime_exec mc_runtime_exec_bounded mc_runtime_inspect mc_runtime_info \ + mc_lifecycle_start mc_lifecycle_stop mc_lifecycle_running mc_lifecycle_cat \ + mc_application_monitor mc_application_standby_score mc_application_validate \ + mc_application_metadata mc_application_notify mc_application_action; do + if ! declare -F "${hook}" >/dev/null; then + ocf_exit_reason "Required plugin hook is missing: ${hook}" + return "${OCF_ERR_CONFIGURED}" + fi + done +} + +mc_engine_metadata() +{ + # Metadata discovery works without configured resource instance attributes. + # Capture the plugin's status before transforming XML; avoid a pipeline + # masking a metadata failure. No runtime/service/cluster access is required. + local metadata + metadata="$(mc_application_metadata)" || return "$?" + printf '%s\n' "${metadata}" | sed '/<\/parameters>/i\ + \ + Packaged application plugin. Only postgresql is supported.\ + Application plugin\ + \ + \ + \ + Packaged runtime adapter. Only podman is supported.\ + Container runtime\ + \ + \ + \ + Container lifecycle owner. Only systemd with the existing user service is supported.\ + Lifecycle manager\ + \ + ' +} diff --git a/refactored/managed-container.d/lifecycles/systemd.sh b/refactored/managed-container.d/lifecycles/systemd.sh new file mode 100644 index 0000000..016caaf --- /dev/null +++ b/refactored/managed-container.d/lifecycles/systemd.sh @@ -0,0 +1,23 @@ +# Lifecycle adapter API 1. Quadlet remains the owner of container construction, +# init/maintenance command, signal settings and service restart/exit policies. +MC_LIFECYCLE_API=1 + +mc_lifecycle_start() +{ + as_service_user systemctl --user start "$1" +} + +mc_lifecycle_stop() +{ + as_service_user systemctl --user stop "$1" +} + +mc_lifecycle_running() +{ + as_service_user systemctl --user is-active --quiet "$1" +} + +mc_lifecycle_cat() +{ + as_service_user systemctl --user cat "$1" +} diff --git a/refactored/managed-container.d/runtimes/podman.sh b/refactored/managed-container.d/runtimes/podman.sh new file mode 100644 index 0000000..cd8c2d4 --- /dev/null +++ b/refactored/managed-container.d/runtimes/podman.sh @@ -0,0 +1,24 @@ +# Runtime adapter API 1. The existing rootless privilege boundary is unchanged. +MC_RUNTIME_API=1 + +mc_runtime_exec() +{ + as_service_user podman exec "$@" +} + +mc_runtime_exec_bounded() +{ + local duration="$1" + shift + as_service_user timeout --kill-after=1s "${duration}" podman exec "$@" +} + +mc_runtime_inspect() +{ + as_service_user podman inspect "$@" +} + +mc_runtime_info() +{ + as_service_user podman info "$@" +} diff --git a/scripts/test-elitedesk-cycle.py b/scripts/test-elitedesk-cycle.py new file mode 100644 index 0000000..dc31cdd --- /dev/null +++ b/scripts/test-elitedesk-cycle.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""One guarded RA stop/start cycle on the ELITEDESK lab standby only.""" +import importlib.util +import json +import os +from pathlib import Path +import pwd +import socket +import subprocess +import sys +import tempfile +import time +import hashlib + +spec = importlib.util.spec_from_file_location('lab', Path(__file__).with_name('test-elitedesk-plugin.py')) +lab = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lab) + + +def owner_command(*args): + account = pwd.getpwnam('david') + return lab.run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir, + 'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid), + 'timeout', '--kill-after=1s', '5s', *args], cwd=account.pw_dir) + + +def score(): + return lab.run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK', + '--lifetime=reboot', '--query', '--quiet']) + + +def verify_payload(): + entries = lab.hashes(lab.ROOT / 'docs/lab-installed-plugin.sha256') + lab.require(len(entries) == 15, 'Expected the 15-file installed plugin snapshot') + for name, digest in entries.items(): + path = Path(name) + lab.require(lab.PROVIDER in path.parents and not path.is_symlink() + and hashlib.sha256(path.read_bytes()).hexdigest() == digest, + 'Installed code changed; inspect before testing: ' + name) + + +def main(): + lab.require(os.geteuid() == 0, 'Run with sudo in your terminal; do not send passwords in chat') + lab.require(socket.gethostname() == 'DAVID-ELITEDESK', 'Only the ELITEDESK lab standby is supported') + os.umask(0o077) + attributes = lab.cluster() # Exact lab membership/resource; maintenance=true. + before = lab.observe() # Running standby streaming from DV6 with its own slot. + verify_payload() + evidence = Path(tempfile.mkdtemp(prefix='managed-container-cycle-lab-', dir=lab.BACKUP_ROOT)) + print('Results:', evidence, flush=True) + env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', OCF_ROOT='/usr/lib/ocf', + OCF_RESOURCE_INSTANCE='pg-ha', OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK', + **{'OCF_RESKEY_' + k: v for k, v in attributes.items()}) + results = dict(before=before, actions=[], status='running') + + def save(): + (evidence / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + + def unchanged_cluster(): + lab.require(lab.cluster() == attributes, 'Lab configuration or maintenance state changed; halt') + + def action(name, expected=0): + unchanged_cluster() + # GNU timeout bounds the RA process group, not just its parent process. + result = subprocess.run(['timeout', '--kill-after=5s', '60s', str(lab.AGENT), name], + env=env, cwd='/', text=True, capture_output=True) + results['actions'].append(dict(action=name, rc=result.returncode, expected=expected)) + (evidence / f'{len(results["actions"]):02d}-{name}.log').write_text(result.stdout + result.stderr) + save() + lab.require(result.returncode == expected, + f'{name}: expected rc={expected}, got {result.returncode}; see {evidence}') + print(f'PASS {name}: rc={result.returncode}', flush=True) + + save() + try: + action('validate-all') + action('monitor') + lab.require(lab.observe() == before, 'Standby changed before stop') + action('stop') + active = owner_command('systemctl', '--user', 'show', 'postgres.service', '--property=ActiveState', '--value') + containers = owner_command('podman', 'ps', '--all', '--filter', 'name=^postgres$', '--format', '{{.State}}') + results['stopped'] = dict(service=active, container=containers or 'absent') + lab.require(active == 'inactive' and containers in ('', 'exited', 'stopped'), + 'Service/container did not reach the expected stopped state') + action('monitor', 7) + results['stopped']['score'] = score() + lab.require(results['stopped']['score'] == '-INFINITY', 'Stopped standby retained an eligible score') + save() + print('PASS stopped state: service inactive, container stopped/absent, score=-INFINITY', flush=True) + verify_payload() + action('start') # Existing standby.signal guard remains in the RA. + deadline = time.monotonic() + 60 + last_error = 'No streaming observation yet' + while time.monotonic() < deadline: + unchanged_cluster() + try: + after = lab.observe() + except RuntimeError as error: + last_error = str(error) + time.sleep(2) + continue + lab.require(after['system_id'] == before['system_id'], 'PostgreSQL system identifier changed') + lab.require(after['started'] != before['started'], 'Postmaster start time did not change') + results['after'] = after + action('monitor') + results['score'] = score() + save() + if results['score'] == '100': + break + last_error = 'Streaming resumed but strict standby score has not reached 100' + time.sleep(2) + else: + raise RuntimeError('Post-start verification deadline expired: ' + last_error) + action('status') + unchanged_cluster() + verify_payload() + results['status'] = 'passed' + save() + print('PASS cycle: same database, new postmaster, streaming from DV6, score=100.', flush=True) + print('Maintenance remains true. No promotion, demotion, rejoin, fencing, or peer actions ran.', flush=True) + except BaseException as error: + results['status'] = 'failed' + results['error'] = str(error) + save() + print('HALTED: no automatic retry, recovery, or data replacement. Inspect results:', evidence, flush=True) + raise + + +if __name__ == '__main__': + try: + main() + except Exception as error: + print('STOP:', error, file=sys.stderr) + sys.exit(1) diff --git a/scripts/test-elitedesk-plugin.py b/scripts/test-elitedesk-plugin.py new file mode 100644 index 0000000..87c42f4 --- /dev/null +++ b/scripts/test-elitedesk-plugin.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Install the plugin candidate and run non-disruptive checks on the lab standby. + +Run with sudo from the operator's terminal. Never supplies a password, changes +maintenance mode, or runs stop/promote/demote/rejoin. Logs are root-only. +""" +import hashlib +import json +import os +from pathlib import Path +import pwd +import shutil +import socket +import signal +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET + +ROOT = Path(__file__).resolve().parents[1] +PROVIDER = Path('/usr/lib/ocf/resource.d/loxthelion') +AGENT = PROVIDER / 'managed-container' +MODULES = PROVIDER / 'managed-container.d' +BASELINE = ROOT / 'docs/lab-installed-before-plugin.sha256' +BACKUP_ROOT = Path('/var/backups') + + +def run(args, env=None, cwd=None): + clean_env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', HOME='/root', LC_ALL='C') + result = subprocess.run(args, env=env if env is not None else clean_env, cwd=cwd, + text=True, capture_output=True, timeout=90) + if result.returncode: + raise RuntimeError(f'{Path(args[0]).name} failed (rc={result.returncode}): {result.stderr.strip()}') + return result.stdout.strip() + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def hashes(manifest): + return dict((name, digest) for digest, name in + (line.split(' ', 1) for line in manifest.read_text().splitlines() if line)) + + +def cluster(): + cib = ET.fromstring(run(['cibadmin', '--query'])) + require(run(['crm_attribute', '--type', 'crm_config', '--name', 'cluster-name', '--query', '--quiet']) + == 'postgres-ra-test', 'This is not the postgres-ra-test lab') + require(run(['crm_attribute', '--type', 'crm_config', '--name', 'maintenance-mode', '--query', '--quiet']) + == 'true', 'Lab maintenance-mode must already be true; nothing changed') + require({n.get('uname') for n in cib.findall('./configuration/nodes/node')} == + {'DAVID-ELITEDESK', 'DAVID-DV6'}, 'Unexpected cluster membership; inspect before proceeding') + resources = cib.findall("./configuration/resources/clone[@id='pg-ha-clone']/primitive[@id='pg-ha']") + require(len(resources) == 1, 'Expected pg-ha under pg-ha-clone') + resource = resources[0] + require((resource.get('class'), resource.get('provider'), resource.get('type')) == + ('ocf', 'loxthelion', 'managed-container'), 'Unexpected resource agent') + attributes = {} + for group in resource.findall('instance_attributes'): + require(not group.get('id-ref') and group.find('rule') is None, + 'Referenced/rule-based parameters require manual review') + for pair in group.findall('nvpair'): + name, value = pair.get('name'), pair.get('value') + require(name not in attributes and name and value is not None, 'Ambiguous resource parameters') + attributes[name] = value + expected = dict(service_user='david', service_name='postgres.service', container_name='postgres', + pg_os_user='postgres', db_user='david', db_name='david') + require(all(attributes.get(k, v) == v for k, v in expected.items()), 'Lab service/database parameters changed') + return dict(expected, **{k: v for k, v in attributes.items() if k not in expected}) + + +def observe(): + account = pwd.getpwnam('david') + query = """SELECT json_build_object('recovery',pg_is_in_recovery(), + 'started',pg_postmaster_start_time()::text, + 'system_id',(SELECT system_identifier::text FROM pg_control_system()), + 'receiver',(SELECT status FROM pg_stat_wal_receiver), + 'sender',(SELECT sender_host FROM pg_stat_wal_receiver), + 'slot',(SELECT slot_name FROM pg_stat_wal_receiver));""" + output = run(['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir, + 'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid), + 'timeout', '--kill-after=1s', '5s', 'podman', 'exec', '--user', 'postgres', + '--env', 'PGOPTIONS=-c statement_timeout=2000', 'postgres', + 'psql', '-X', '-v', 'ON_ERROR_STOP=1', '-U', 'david', '-d', 'david', '-Atqc', query], + cwd=account.pw_dir) + state = json.loads(output) + require(state['recovery'] is True and state['receiver'] == 'streaming' + and state['sender'] == '192.168.1.2' and state['slot'] == 'ha_david_elitedesk', + 'Expected a streaming ELITEDESK standby of lab DV6; refusing installation/start') + return state + + +def main(): + require(os.geteuid() == 0, 'Run this script with sudo in your terminal; do not send passwords in chat') + require(socket.gethostname() == 'DAVID-ELITEDESK', 'This script is only for DAVID-ELITEDESK') + def interrupted(signum, frame): + raise RuntimeError(f'Interrupted by signal {signum}') + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + os.umask(0o077) + os.environ['PATH'] = '/usr/sbin:/usr/bin:/sbin:/bin' + attributes = cluster() + before = observe() + for name, digest in hashes(BASELINE).items(): + path = PROVIDER / name + require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == digest, + f'Installed baseline changed: {name}; review before replacing') + manifest = hashes(ROOT / 'notes/REFACTOR-SHA256SUMS') + candidates = [ROOT / 'refactored/managed-container'] + sorted( + p for p in (ROOT / 'refactored/managed-container.d').rglob('*') if p.suffix in ('.sh', '.py') and p.is_file()) + require(len(candidates) == 15, 'Unexpected candidate file layout') + for path in candidates: + require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() + == manifest.get(str(path.relative_to(ROOT))), f'Candidate checksum mismatch: {path.name}') + # Copy verified code into a root-owned staging directory before executing it. + backup = Path(tempfile.mkdtemp(prefix='managed-container-plugin-lab-', dir=BACKUP_ROOT)) + print('Backup and results:', backup, flush=True) + shutil.copy2(AGENT, backup / 'managed-container') + shutil.copytree(MODULES, backup / 'managed-container.d', symlinks=True) + stage = Path(tempfile.mkdtemp(prefix='.mc-plugin-stage-', dir=PROVIDER)) + for source in candidates: + destination = stage / source.relative_to(ROOT / 'refactored') + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + destination.chmod(0o755 if destination.name == 'managed-container' else 0o644) + require(hashlib.sha256(destination.read_bytes()).hexdigest() == + manifest[str(source.relative_to(ROOT))], 'Source changed while staging') + for directory in (stage / 'managed-container.d').rglob('*'): + if directory.is_dir(): directory.chmod(0o755) + (stage / 'managed-container.d').chmod(0o755) + env = dict(PATH=os.environ['PATH'], OCF_ROOT='/usr/lib/ocf', OCF_RESOURCE_INSTANCE='pg-ha', + OCF_RESKEY_CRM_meta_on_node='DAVID-ELITEDESK', + **{'OCF_RESKEY_' + k: v for k, v in attributes.items()}) + results = dict(before=before, actions=[]) + + def action(executable, name): + output = run([str(executable), name], env=env, cwd='/') + results['actions'].append(dict(agent=str(executable), action=name, rc=0)) + print(f'PASS {executable.parent.name}/{name}: rc=0', flush=True) + return output + + staged = stage / 'managed-container' + metadata = action(staged, 'meta-data') + require(len(ET.fromstring(metadata).findall('./parameters/parameter')) == 16, 'Unexpected plugin metadata') + (backup / 'candidate-metadata.xml').write_text(metadata + '\n') + action(staged, 'validate-all') + require(cluster() == attributes, 'Cluster configuration changed before installation') + require(observe() == before, 'PostgreSQL state changed before installation') + retired = stage / 'previous-modules' + modules_moved = False + try: + os.replace(MODULES, retired) + modules_moved = True + os.replace(stage / 'managed-container.d', MODULES) + os.replace(staged, AGENT) + run(['restorecon', '-RF', str(AGENT), str(MODULES)]) + action(AGENT, 'validate-all') + action(AGENT, 'monitor') + action(AGENT, 'status') + require(cluster() == attributes and observe() == before, 'Standby/cluster changed before idempotent start') + action(AGENT, 'start') # Already healthy standby: exercises the no-op path. + action(AGENT, 'monitor') + require(cluster() == attributes, 'Cluster configuration changed during checks') + results['after'] = observe() + require(results['after'] == before, 'Database identity/start time or replication changed during checks') + results['score'] = run(['crm_attribute', '--promotion=pg-ha', '--node', 'DAVID-ELITEDESK', + '--lifetime=reboot', '--query', '--quiet']) + require(results['score'] in ('100', '-INFINITY'), 'Unexpected standby score') + # Under write load a connected standby can legitimately fail strict catch-up. + for source in candidates: + installed = PROVIDER / source.relative_to(ROOT / 'refactored') + require(hashlib.sha256(installed.read_bytes()).hexdigest() == + manifest[str(source.relative_to(ROOT))], 'Installed payload mismatch') + (backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + (backup / 'installed-SHA256SUMS').write_text(''.join( + manifest[str(p.relative_to(ROOT))] + ' ' + str(PROVIDER / p.relative_to(ROOT / 'refactored')) + '\n' + for p in candidates)) + except BaseException: + if modules_moved: + print('Check failed: restoring previous agent and modules; maintenance mode is not changed.', flush=True) + if MODULES.exists(): os.replace(MODULES, stage / 'failed-modules') + os.replace(retired, MODULES) + shutil.copyfile(backup / 'managed-container', stage / 'restore-agent') + (stage / 'restore-agent').chmod(0o755) + os.replace(stage / 'restore-agent', AGENT) + run(['restorecon', '-RF', str(AGENT), str(MODULES)]) + (backup / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + print('Staging/evidence retained:', stage, flush=True) + raise + shutil.rmtree(stage) + print('PASS: installed; PostgreSQL was not restarted; maintenance remains true.', flush=True) + print('Current score:', results['score'], '(strict standby catch-up can fluctuate)', flush=True) + print('No stop, promote, demote, rejoin, fencing, or peer-node actions were run.', flush=True) + + +if __name__ == '__main__': + try: + main() + except Exception as error: + print('STOP:', error, file=sys.stderr) + sys.exit(1) diff --git a/scripts/test-lab-roles.py b/scripts/test-lab-roles.py new file mode 100644 index 0000000..bc96291 --- /dev/null +++ b/scripts/test-lab-roles.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Operator-run, one-stage-at-a-time role tests for the two-node lab. + +No installation, automatic recovery, fencing, or maintenance-mode changes. +See docs/lab-role-tests.md before running a mutating stage. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import pwd +import shlex +import socket +import subprocess +import tempfile +import time +import xml.etree.ElementTree as ET + +PROVIDER = Path('/usr/lib/ocf/resource.d/loxthelion') +AGENT = PROVIDER / 'managed-container' +NODES = {'DAVID-ELITEDESK': '192.168.1.6', 'DAVID-DV6': '192.168.1.2'} +PINNED = {'DAVID-DV6': {'managed-container': 'a4b775a0ddcabdb0558da9b34c7fe2a883c07d0ef308c1c7224526abf53f63a0', + 'managed-container.d/auth.sh': '93191771d3cb8aefadb2a16d2f89d64a8b6c584071a473439be818237bdf7f17', + 'managed-container.d/common.sh': '97e93d9164e854e5f3688cf9d3cc656b001eb30ef3c2073c907450ca43588f3d', + 'managed-container.d/container.sh': 'ac6bc363a2301f1491d162ef609103dbfbceb3b099a3556614c6165952309374', + 'managed-container.d/demote.sh': '0af3bb362a303017372c631354ba94ef9e723bf14e64d2eff91b6662e54a5be0', + 'managed-container.d/notify.sh': '6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670', + 'managed-container.d/postgres.sh': 'd2779ce6e7d1f426ffca2e41bbb8de77503005dc38d658d69f184020365796eb', + 'managed-container.d/promote.sh': '3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58', + 'managed-container.d/promotion_state.py': '5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7', + 'managed-container.d/rejoin.sh': '21beea72bcb7f24a51c17ea48666ac87df9436702ad77b166b45f4647e28e8ad', + 'managed-container.d/replication.sh': '6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547'}, + 'DAVID-ELITEDESK': {'managed-container': '01eb9b161c8e3db289f09bf63953be779921e15bdf8b1a4d8ce7e60032b826e4', + 'managed-container.d/applications/postgresql/auth.sh': '10932919c4339b553c7c9dbf30b45a917c51c0a08f362e12678003d58b12fa82', + 'managed-container.d/applications/postgresql/demote.sh': 'da4e58cadf1df3953cb97a0512e3931ee1b9d3ee61c39a1ad1a6047ef537addc', + 'managed-container.d/applications/postgresql/notify.sh': '6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670', + 'managed-container.d/applications/postgresql/plugin.sh': 'd3a989de97ceb61a56828f006d5ccb87b7c34fc155a944585bedf928785021fc', + 'managed-container.d/applications/postgresql/postgres.sh': 'fa222e6767249999767eb82fac2d85e7e35404da240517a8b630571e77282053', + 'managed-container.d/applications/postgresql/promote.sh': '3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58', + 'managed-container.d/applications/postgresql/promotion_state.py': '5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7', + 'managed-container.d/applications/postgresql/rejoin.sh': '28b0e44554a88ccb87cd16a68315c045dd37d354aceb2fa25da7c7399a659b23', + 'managed-container.d/applications/postgresql/replication.sh': '6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547', + 'managed-container.d/common.sh': 'a482ef3cc1048e6a4f427e76dd2dce173ffd00760dbed1afa601a3fd05b0ad99', + 'managed-container.d/container.sh': '3a27ac9c63c61310da8b8711fa2c4400b37ea8eae2ee2a79e28bc63180b06c7c', + 'managed-container.d/engine.sh': '53978e2367394695d61c6fb38f2ee46814474c5a4e0448d2692f1a3ce583af93', + 'managed-container.d/lifecycles/systemd.sh': '6c92c095ce10ac95d843fdc1c49e31c19491ca22fdac960affd8649606d76d82', + 'managed-container.d/runtimes/podman.sh': 'dc5820067a78c4a1c531426902793aae97a03fd64912b017b3b102031dcc1e04'}} + +QUERY = """SELECT json_build_object( + 'recovery',pg_is_in_recovery(), 'paused',pg_is_wal_replay_paused(), + 'system_id',(SELECT system_identifier::text FROM pg_control_system()), + 'started',pg_postmaster_start_time()::text, + 'receiver',(SELECT status FROM pg_stat_wal_receiver), + 'sender',(SELECT sender_host FROM pg_stat_wal_receiver), + 'slot',(SELECT slot_name FROM pg_stat_wal_receiver), + 'replay',pg_last_wal_replay_lsn()::text);""" +# pg_is_wal_replay_paused() is only callable during recovery. +QUERY = QUERY.replace('pg_is_wal_replay_paused()', + 'CASE WHEN pg_is_in_recovery() THEN pg_is_wal_replay_paused() ELSE false END') + + +def require(ok, message): + if not ok: + raise RuntimeError(message) + + +def run(args, **kwargs): + result = subprocess.run(args, text=True, capture_output=True, timeout=90, + env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', LC_ALL='C'), **kwargs) + require(result.returncode == 0, f'{args[0]} failed: {result.stderr.strip()}') + return result.stdout.strip() + + +def owner_args(*args): + account = pwd.getpwnam('david') + return ['runuser', '-u', 'david', '--', 'env', 'HOME=' + account.pw_dir, + 'XDG_RUNTIME_DIR=/run/user/' + str(account.pw_uid), *args] + + +def cluster(): + cib = ET.fromstring(run(['cibadmin', '--query'])) + def prop(name): + return run(['crm_attribute', '--type', 'crm_config', '--name', name, '--query', '--quiet']) + require(prop('cluster-name') == 'postgres-ra-test' and prop('maintenance-mode') == 'true', + 'Expected postgres-ra-test with maintenance=true; nothing changed') + require({n.get('uname') for n in cib.findall('./configuration/nodes/node')} == set(NODES), + 'Unexpected cluster membership') + resource = cib.find("./configuration/resources/clone[@id='pg-ha-clone']/primitive[@id='pg-ha']") + require(resource is not None and (resource.get('class'), resource.get('provider'), resource.get('type')) + == ('ocf', 'loxthelion', 'managed-container'), 'Unexpected resource') + attrs = {} + for group in resource.findall('instance_attributes'): + require(not group.get('id-ref') and group.find('rule') is None, 'Unsupported parameter rules') + for pair in group.findall('nvpair'): + key, value = pair.get('name'), pair.get('value') + require(key and value is not None and key not in attrs, 'Ambiguous parameters') + attrs[key] = value + defaults = dict(service_user='david', service_name='postgres.service', container_name='postgres', + pg_os_user='postgres', db_user='david', db_name='david', + application='postgresql', container_runtime='podman', lifecycle_manager='systemd') + require(all(attrs.get(k, v) == v for k, v in defaults.items()), 'Unexpected service/application settings') + return dict(defaults, **attrs) + + +def snapshot(node): + # Executes only read operations as david on either host. Never inspects container environment. + code = """import hashlib,json,pathlib,socket,subprocess +def cmd(args): + p=subprocess.run(['timeout','--kill-after=1s','5s']+args,text=True,capture_output=True) + if p.returncode: raise RuntimeError('Read-only observation failed: '+args[0]) + return p.stdout.strip() +files=FILES +base=pathlib.Path('/usr/lib/ocf/resource.d/loxthelion') +assert all(not (base/n).is_symlink() for n in files) +print(json.dumps(dict(node=socket.gethostname(), + hashes={n:hashlib.sha256((base/n).read_bytes()).hexdigest() for n in files}, + service=cmd(['systemctl','--user','show','postgres.service','--property=ActiveState','--value']), + container=cmd(['podman','inspect','--format','{{.Id}}','postgres']), + db=json.loads(cmd(['podman','exec','--user','postgres','--env','PGOPTIONS=-c statement_timeout=2000', + 'postgres','psql','-X','-v','ON_ERROR_STOP=1','-U','david','-d','david','-Atqc',QUERY]))))) +""".replace('FILES', repr(list(PINNED[node]))).replace('QUERY', repr(QUERY)) + if node == socket.gethostname(): + args = owner_args('python3', '-c', code) + else: + args = owner_args('ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=5', + '-o', 'StrictHostKeyChecking=yes', 'david@' + NODES[node], + 'python3 -c ' + shlex.quote(code)) + value = json.loads(run(args, cwd=pwd.getpwnam('david').pw_dir)) + require(value['node'] == node and value['hashes'] == PINNED[node], 'Node or installed code differs: ' + node) + require(value['service'] == 'active', 'Maintenance service not active: ' + node) + return value + + +def standby(db, source): + return (db['recovery'] is True and db['paused'] is False and db['receiver'] == 'streaming' + and db['sender'] == NODES[source]) + + +def guard(stage, node, local, peer): + require(local['system_id'] == peer['system_id'], 'Database system identifiers differ') + other = next(n for n in NODES if n != node) + if stage == 'demote': + require(local['recovery'] is False and standby(peer, node), 'Need local primary and streaming peer standby') + elif stage == 'promote': + require(local['recovery'] is True and local['paused'] is False and standby(peer, node), + 'Both nodes must be in recovery and the demoted peer must stream from this node') + elif stage == 'rejoin': + require(node == 'DAVID-ELITEDESK' and standby(local, other) and peer['recovery'] is False, + 'Rejoin test requires ELITEDESK streaming from primary DV6') + + +def main(stage): + node = socket.gethostname() + require(os.geteuid() == 0 and node in NODES, 'Run with sudo on an expected lab node') + os.umask(0o077) + peer = next(n for n in NODES if n != node) + attrs = cluster() + before, remote = snapshot(node), snapshot(peer) + guard(stage, node, before['db'], remote['db']) + if stage == 'inspect': + print(json.dumps(dict(local=before, peer=remote, maintenance=True), indent=2)) + return + evidence = Path(tempfile.mkdtemp(prefix='managed-container-roles-lab-', dir='/var/backups')) + print('Results:', evidence, flush=True) + result = dict(stage=stage, node=node, before=before, peer_before=remote, actions=[], status='running') + env = dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin', OCF_ROOT='/usr/lib/ocf', OCF_RESOURCE_INSTANCE='pg-ha', + OCF_RESKEY_CRM_meta_on_node=node, **{'OCF_RESKEY_' + k: v for k, v in attrs.items()}) + def save(): + (evidence / 'results.json').write_text(json.dumps(result, indent=2) + '\n') + def action(name, expected=0, extra=None): + require(cluster() == attrs, 'Cluster configuration or maintenance changed') + response = subprocess.run(['timeout', '--kill-after=5s', '120s', str(AGENT), name], + env=dict(env, **(extra or {})), cwd='/', text=True, capture_output=True) + result['actions'].append(dict(action=name, rc=response.returncode, expected=expected)) + (evidence / f'{len(result["actions"]):02d}-{name}.log').write_text(response.stdout + response.stderr) + save() + require(response.returncode == expected, f'{name} returned {response.returncode}; expected {expected}') + print(f'PASS {name}: rc={response.returncode}', flush=True) + save() + try: + action('validate-all') + action('monitor', 8 if before['db']['recovery'] is False else 0) + # Reobserve immediately before mutation. No action retry or inferred fencing certificate. + current, remote = snapshot(node), snapshot(peer) + require(current['container'] == before['container'] and + all(current['db'][k] == before['db'][k] for k in ('system_id', 'started', 'recovery')), + 'Local incarnation/role changed during preflight') + guard(stage, node, current['db'], remote['db']) + if stage == 'demote': + action('notify', extra=dict(OCF_RESKEY_CRM_meta_notify_type='pre', + OCF_RESKEY_CRM_meta_notify_operation='demote', + OCF_RESKEY_CRM_meta_notify_demote_uname=node, + OCF_RESKEY_CRM_meta_notify_promote_uname=peer)) + action('demote') + elif stage == 'promote': + action('promote') + elif stage == 'rejoin': + # This stage tests rewind/recovery of a healthy standby, not divergent former-primary repair. + run(owner_args('timeout', '--kill-after=5s', '60s', 'podman', 'exec', '--user', 'postgres', + 'postgres', 'sh', '-c', + 'exec "$(pg_config --bindir)/pg_ctl" -D "$PGDATA" stop -m fast -w')) + action('monitor', 7) + require(run(['crm_attribute', '--promotion=pg-ha', '--node', node, + '--lifetime=reboot', '--query', '--quiet']) == '-INFINITY', + 'Stopped PostgreSQL retained an eligible score') + require(snapshot(peer)['db']['recovery'] is False, 'DV6 is no longer primary') + action('rejoin', extra=dict(OCF_RESKEY_rejoin_source_host=NODES[peer], + OCF_RESKEY_replication_slot='ha_david_elitedesk', + OCF_RESKEY_application_name=node, + OCF_RESKEY_rejoin_allow_basebackup='false')) + deadline = time.monotonic() + 60 + while True: + after, remote = snapshot(node), snapshot(peer) + require(after['db']['system_id'] == before['db']['system_id'], 'Local database identity changed') + require(after['container'] == before['container'], 'Maintenance container was replaced') + require(remote['db']['system_id'] == before['db']['system_id'], 'Peer database identity changed') + if stage == 'promote': + require(after['db']['recovery'] is False and standby(remote['db'], node), 'Unexpected roles after promotion') + expected_score = '1000' + else: + require(standby(after['db'], peer), 'Local node did not return to streaming recovery') + require(after['db']['started'] != before['db']['started'], 'Expected a new postmaster') + require(remote['db']['recovery'] is (stage == 'demote'), 'Unexpected peer role') + expected_score = '100' + action('monitor', 8 if stage == 'promote' else 0) + value = run(['crm_attribute', '--promotion=pg-ha', '--node', node, '--lifetime=reboot', '--query', '--quiet']) + if value == expected_score: + break + require(time.monotonic() < deadline, 'Expected promotion score did not converge') + time.sleep(2) + result.update(after=after, peer_after=remote, score=value, status='passed') + require(cluster() == attrs, 'Cluster configuration changed') + save() + print(f'PASS {stage}: verified roles, streaming, same database/container, score={value}.', flush=True) + print('Maintenance remains true. No fencing or automatic recovery ran.', flush=True) + except BaseException as error: + result.update(status='failed', error=str(error)) + save() + print('HALTED. Do not continue to the next stage. Inspect:', evidence, flush=True) + raise + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('stage', choices=['inspect', 'demote', 'promote', 'rejoin']) + try: + main(parser.parse_args().stage) + except Exception as error: + raise SystemExit('STOP: ' + str(error)) diff --git a/tests/plugin_test_support.py b/tests/plugin_test_support.py new file mode 100644 index 0000000..43d7284 --- /dev/null +++ b/tests/plugin_test_support.py @@ -0,0 +1,33 @@ +"""Shared test loader and narrowly reversible command-boundary substitutions.""" +from pathlib import Path +import re +import shlex + +ROOT = Path(__file__).resolve().parents[1] +MODULE_DIR = ROOT / 'refactored/managed-container.d' +LOAD = '''MC_PROMOTION_RESOURCE_INSTANCE="${OCF_RESOURCE_INSTANCE:-}" +: "${OCF_ROOT:=/usr/lib/ocf}" +. "$OCF_ROOT/lib/heartbeat/ocf-shellfuncs" +: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}" +MC_MODULE_DIR=''' + shlex.quote(str(MODULE_DIR)) + ''' +. "$MC_MODULE_DIR/common.sh" +. "$MC_MODULE_DIR/container.sh" +. "$MC_MODULE_DIR/engine.sh" +mc_engine_load || exit $? +''' + + +def restore_boundaries(source): + """Project only adapter calls back to their frozen original spelling. + + This is not whitespace normalization or a general semantic equivalence + claim. All remaining function bytes must match the immutable reference. + """ + source = source.replace('mc_runtime_inspect', 'as_service_user podman inspect') + source = source.replace('mc_runtime_info', 'as_service_user podman info') + pattern = r'^([ \t]*)(.*?)mc_runtime_exec \\\n' + source = re.sub(pattern, lambda m: m[1] + m[2] + 'as_service_user \\\n' + m[1] + ' podman exec \\\n', source, flags=re.M) + pattern = r'^([ \t]*)(.*?)mc_lifecycle_(start|stop|cat|running) ' + source = re.sub(pattern, lambda m: m[1] + m[2] + 'as_service_user \\\n' + m[1] + ' systemctl --user ' + + ('is-active --quiet' if m[3] == 'running' else m[3]) + ' ', source, flags=re.M) + return source diff --git a/tests/test_disconnected_state.py b/tests/test_disconnected_state.py index a0b5572..f1f6f24 100644 --- a/tests/test_disconnected_state.py +++ b/tests/test_disconnected_state.py @@ -12,7 +12,7 @@ import unittest from test_promotion_scores import Scores, LOAD, FIXTURE ROOT = Path(__file__).resolve().parents[1] -HELPER = ROOT / 'refactored/managed-container.d/promotion_state.py' +HELPER = ROOT / 'refactored/managed-container.d/applications/postgresql/promotion_state.py' spec = importlib.util.spec_from_file_location('promotion_state', HELPER) policy = importlib.util.module_from_spec(spec) spec.loader.exec_module(policy) diff --git a/tests/test_lab_cycle.py b/tests/test_lab_cycle.py new file mode 100644 index 0000000..32ba005 --- /dev/null +++ b/tests/test_lab_cycle.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Stop/start harness tests with mocked commands and temporary evidence only.""" +import contextlib +import importlib.util +import io +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('cycle', ROOT / 'scripts/test-elitedesk-cycle.py') +cycle = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cycle) + + +class Cycle(unittest.TestCase): + def scenario(self, failure=None): + with tempfile.TemporaryDirectory() as directory, contextlib.ExitStack() as stack: + state = {'running': True, 'started': 'old'} + calls = [] + def command(args, **kwargs): + action = args[-1]; calls.append(action) + self.assertIn(action, ['validate-all', 'monitor', 'stop', 'start', 'status']) + if action == failure: + return SimpleNamespace(returncode=1, stdout='', stderr='injected failure') + if action == 'stop': state['running'] = False + if action == 'start': state.update(running=True, started='new') + rc = 7 if action == 'monitor' and not state['running'] else 0 + return SimpleNamespace(returncode=rc, stdout='', stderr='') + stack.enter_context(patch.object(cycle.os, 'geteuid', return_value=0)) + stack.enter_context(patch.object(cycle.socket, 'gethostname', return_value='DAVID-ELITEDESK')) + stack.enter_context(patch.object(cycle.lab, 'BACKUP_ROOT', Path(directory))) + stack.enter_context(patch.object(cycle.lab, 'cluster', return_value={})) + stack.enter_context(patch.object(cycle.lab, 'observe', side_effect=lambda: + dict(system_id='same', started=state['started']))) + stack.enter_context(patch.object(cycle, 'verify_payload')) + stack.enter_context(patch.object(cycle, 'owner_command', side_effect=lambda *args: + 'inactive' if args[0] == 'systemctl' else '')) + stack.enter_context(patch.object(cycle, 'score', side_effect=lambda: + '100' if state['running'] else '-INFINITY')) + stack.enter_context(patch.object(cycle.subprocess, 'run', side_effect=command)) + old_mask = os.umask(0o022) + try: + with contextlib.redirect_stdout(io.StringIO()): + if failure: + with self.assertRaises(RuntimeError): cycle.main() + self.assertEqual(calls[-1], failure) + self.assertEqual(calls.count(failure), 1) + else: + cycle.main() + self.assertEqual(calls, ['validate-all','monitor','stop','monitor','start','monitor','status']) + finally: + os.umask(old_mask) + self.assertEqual(len(list(Path(directory).glob('*/results.json'))), 1) + + def test_successful_cycle(self): self.scenario() + def test_stop_failure_halts_without_start(self): self.scenario('stop') + def test_start_failure_halts_without_retry(self): self.scenario('start') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_lab_installer.py b/tests/test_lab_installer.py new file mode 100644 index 0000000..901c1d5 --- /dev/null +++ b/tests/test_lab_installer.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Exercise publication and rollback using temp files and mocked lab boundaries.""" +import contextlib +import hashlib +import importlib.util +import io +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('installer', ROOT / 'scripts/test-elitedesk-plugin.py') +installer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(installer) + + +class Installer(unittest.TestCase): + def scenario(self, fail_monitor=False): + with tempfile.TemporaryDirectory() as directory, contextlib.ExitStack() as stack: + directory = Path(directory) + provider = directory / 'provider'; provider.mkdir() + agent = provider / 'managed-container'; agent.write_text('previous agent') + modules = provider / 'managed-container.d'; modules.mkdir() + old_module = modules / 'old.sh'; old_module.write_text('previous module') + baseline = directory / 'baseline' + baseline.write_text(''.join(hashlib.sha256(p.read_bytes()).hexdigest() + ' ' + str(p.relative_to(provider)) + '\n' + for p in [agent, old_module])) + backup = directory / 'backups'; backup.mkdir() + for name, value in dict(PROVIDER=provider, AGENT=agent, MODULES=modules, + BASELINE=baseline, BACKUP_ROOT=backup).items(): + stack.enter_context(patch.object(installer, name, value)) + stack.enter_context(patch.object(installer.os, 'geteuid', return_value=0)) + stack.enter_context(patch.object(installer.socket, 'gethostname', return_value='DAVID-ELITEDESK')) + stack.enter_context(patch.object(installer.signal, 'signal')) + stack.enter_context(patch.object(installer, 'cluster', return_value={})) + stack.enter_context(patch.object(installer, 'observe', return_value={'unchanged': True})) + stack.enter_context(patch.dict(os.environ)) + calls = [] + def fake_run(args, **kwargs): + calls.append(args) + if args[0] == 'restorecon': return '' + if args[0] == 'crm_attribute': return '100' + require_action = args[1] + self.assertIn(require_action, ['meta-data', 'validate-all', 'monitor', 'status', 'start']) + if require_action == 'meta-data': + return '' + '' * 16 + '' + if fail_monitor and Path(args[0]) == agent and require_action == 'monitor': + raise RuntimeError('injected installed monitor failure') + return '' + stack.enter_context(patch.object(installer, 'run', side_effect=fake_run)) + old_mask = os.umask(0o022) + try: + with contextlib.redirect_stdout(io.StringIO()): + if fail_monitor: + with self.assertRaisesRegex(RuntimeError, 'injected'): + installer.main() + self.assertEqual(agent.read_text(), 'previous agent') + self.assertEqual(old_module.read_text(), 'previous module') + else: + installer.main() + self.assertEqual(agent.read_bytes(), (ROOT / 'refactored/managed-container').read_bytes()) + self.assertTrue((modules / 'applications/postgresql/plugin.sh').is_file()) + self.assertFalse(old_module.exists()) + finally: + os.umask(old_mask) + self.assertEqual(len(list(backup.glob('*/results.json'))), 1) + + def test_success_only_runs_basic_actions(self): + self.scenario() + + def test_failed_installed_monitor_restores_agent_and_modules(self): + self.scenario(fail_monitor=True) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_lab_roles.py b/tests/test_lab_roles.py new file mode 100644 index 0000000..ad4da5a --- /dev/null +++ b/tests/test_lab_roles.py @@ -0,0 +1,55 @@ +"""Exercise the safety gates without running cluster or PostgreSQL commands.""" +import importlib.util +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location('roles', Path(__file__).resolve().parents[1] / 'scripts/test-lab-roles.py') +roles = importlib.util.module_from_spec(spec) +spec.loader.exec_module(roles) + + +def db(recovery, source=None, **changes): + return dict(dict(system_id='same', recovery=recovery, paused=False, + receiver='streaming' if source else None, + sender=roles.NODES.get(source)), **changes) + + +class Gates(unittest.TestCase): + def test_demote_requires_streaming_successor(self): + roles.guard('demote', 'DAVID-DV6', db(False), db(True, 'DAVID-DV6')) + for peer in [db(False), db(True), db(True, 'DAVID-DV6', paused=True)]: + with self.assertRaises(RuntimeError): + roles.guard('demote', 'DAVID-DV6', db(False), peer) + + def test_promote_refuses_live_peer_primary(self): + with self.assertRaises(RuntimeError): + roles.guard('promote', 'DAVID-ELITEDESK', db(True, 'DAVID-DV6'), db(False)) + + def test_promote_requires_demoted_peer_streaming_back(self): + roles.guard('promote', 'DAVID-ELITEDESK', db(True), db(True, 'DAVID-ELITEDESK')) + with self.assertRaises(RuntimeError): + roles.guard('promote', 'DAVID-ELITEDESK', db(True), db(True)) + + def test_identity_mismatch_refused(self): + with self.assertRaises(RuntimeError): + roles.guard('demote', 'DAVID-DV6', db(False), db(True, 'DAVID-DV6', system_id='other')) + + def test_rejoin_only_healthy_elitedesk_standby(self): + roles.guard('rejoin', 'DAVID-ELITEDESK', db(True, 'DAVID-DV6'), db(False)) + for node, local, peer in [('DAVID-DV6', db(True, 'DAVID-ELITEDESK'), db(False)), + ('DAVID-ELITEDESK', db(False), db(True)), + ('DAVID-ELITEDESK', db(True, 'DAVID-DV6'), db(True))]: + with self.assertRaises(RuntimeError): + roles.guard('rejoin', node, local, peer) + + def test_pinned_versions_match_lab_snapshots(self): + root = Path(__file__).resolve().parents[1] + for node, name in [('DAVID-DV6', 'lab-installed-before-plugin.sha256'), + ('DAVID-ELITEDESK', 'lab-installed-plugin.sha256')]: + expected = {p.removeprefix(str(roles.PROVIDER) + '/'): h for h, p in + (line.split(' ', 1) for line in (root / 'docs' / name).read_text().splitlines())} + self.assertEqual(roles.PINNED[node], expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_plugin_engine.py b/tests/test_plugin_engine.py new file mode 100644 index 0000000..ca35215 --- /dev/null +++ b/tests/test_plugin_engine.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Plugin contract tests: no real runtime, systemd, or cluster operations.""" +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import unittest +import xml.etree.ElementTree as ET + +from plugin_test_support import LOAD, ROOT + +AGENT = ROOT / 'refactored/managed-container' + + +class Engine(unittest.TestCase): + def run_shell(self, code, rc=0, env=None): + result = subprocess.run(['bash', '-c', LOAD + '\n' + code], text=True, + capture_output=True, timeout=10, env=env) + self.assertEqual(result.returncode, rc, result.stderr) + return result + + def trace(self, command, arguments, expected, rc=0): + args = ' '.join(shlex.quote(x) for x in arguments) + fixture = ''' +as_service_user() { + python3 -c 'import json,sys; print(json.dumps(sys.argv[1:]))' "$@" + return ''' + str(rc) + ''' +} +''' + result = self.run_shell(fixture + command + ' ' + args, rc) + self.assertEqual(json.loads(result.stdout), expected) + + def test_exec_preserves_argument_boundaries(self): + args = ['--user', 'postgres', '--env', 'VALUE=a b\n$(touch /UNEXPECTED)', + 'container', 'sh', '-c', 'printf "%s" "$1"', 'sh', "quotes'\" $HOME ;"] + self.trace('mc_runtime_exec', args, ['podman', 'exec', *args]) + + def test_exec_error_is_not_systemd_success_exit_status(self): + self.trace('mc_runtime_exec', ['container', 'false'], ['podman', 'exec', 'container', 'false'], 143) + + def test_bounded_exec_preserves_timeout_location_and_status(self): + self.trace('mc_runtime_exec_bounded', ['3s', '--user', 'postgres', 'container', 'psql'], + ['timeout', '--kill-after=1s', '3s', 'podman', 'exec', '--user', 'postgres', 'container', 'psql'], 124) + + def test_inspect_and_info(self): + self.trace('mc_runtime_inspect', ['--format', '{{.State.Running}}', 'container'], + ['podman', 'inspect', '--format', '{{.State.Running}}', 'container']) + self.trace('mc_runtime_info', [], ['podman', 'info'], 125) + + def test_lifecycle_uses_systemd_only(self): + for action, args in [('start', ['start']), ('stop', ['stop']), + ('running', ['is-active', '--quiet']), ('cat', ['cat'])]: + self.trace('mc_lifecycle_' + action, ['postgres.service'], + ['systemctl', '--user', *args, 'postgres.service']) + self.trace('mc_lifecycle_stop', ['postgres.service'], + ['systemctl', '--user', 'stop', 'postgres.service'], 1) + + def test_application_dispatch_and_return_codes(self): + for action in ['start', 'stop', 'promote', 'demote', 'rejoin']: + result = self.run_shell('managed_' + action + '() { echo ' + action + '; return 6; }; ' + 'mc_application_action ' + action, 6) + self.assertEqual(result.stdout.strip(), action) + self.run_shell('mc_application_action invalid', 3) + self.run_shell('managed_monitor() { return 8; }; mc_application_monitor', 8) + self.run_shell('managed_notify() { return 1; }; mc_application_notify', 1) + + def test_unsupported_or_path_like_selection_rejected_before_commands(self): + for key, value in [('application', 'mysql'), ('application', '../postgresql'), + ('container_runtime', 'docker'), ('lifecycle_manager', 'runtime'), + ('application', '$(touch /UNEXPECTED)')]: + env = dict(os.environ, **{'OCF_RESKEY_' + key: value}) + result = subprocess.run([str(AGENT), 'monitor'], env=env, text=True, capture_output=True) + self.assertEqual(result.returncode, 6) + self.assertIn('Supported plugin combination', result.stderr) + + def test_missing_and_incompatible_plugins(self): + for kind in ['missing', 'version', 'hook']: + with tempfile.TemporaryDirectory(prefix='mc-plugin-loader-') as directory: + target = Path(directory) / 'refactored' + shutil.copytree(AGENT.parent, target, ignore=shutil.ignore_patterns('__pycache__')) + plugin = target / 'managed-container.d/runtimes/podman.sh' + if kind == 'missing': + plugin.unlink() + elif kind == 'version': + plugin.write_text(plugin.read_text().replace('MC_RUNTIME_API=1', 'MC_RUNTIME_API=2')) + else: + plugin.write_text(plugin.read_text().replace('mc_runtime_exec()', 'missing_hook()')) + result = subprocess.run([str(target / 'managed-container'), 'meta-data'], + text=True, capture_output=True, cwd='/tmp') + self.assertEqual(result.returncode, 6, result.stderr) + self.assertEqual(result.stdout, '') + + def test_metadata_without_instance_and_with_explicit_defaults(self): + env = {k: v for k, v in os.environ.items() if not k.startswith('OCF_')} + baseline = subprocess.check_output([str(AGENT), 'meta-data'], env=env, cwd='/tmp', text=True) + tree = ET.fromstring(baseline) + params = {p.get('name'): p.find('content').get('default') for p in tree.findall('./parameters/parameter')} + self.assertEqual(len(params), 16) + for key, value in [('application', 'postgresql'), ('container_runtime', 'podman'), ('lifecycle_manager', 'systemd')]: + self.assertEqual(params[key], value) + env['OCF_RESKEY_' + key] = value + self.assertEqual(subprocess.check_output([str(AGENT), 'meta-data'], env=env, text=True), baseline) + + def test_metadata_failure_is_not_hidden(self): + self.run_shell('mc_application_metadata() { return 1; }; mc_engine_metadata', 1) + + def test_application_cannot_call_runtime_or_systemd_directly(self): + # All execution crosses an explicit adapter; command dependency names + # remain part of PostgreSQL's compatibility validation in this revision. + for name in ['postgres', 'auth', 'replication', 'notify', 'promote', 'demote', 'rejoin']: + text = (AGENT.parent / 'managed-container.d/applications/postgresql' / (name + '.sh')).read_text() + self.assertNotIn('as_service_user', text, name) + self.assertNotIn('systemctl --user', text, name) + self.assertNotIn('podman exec', text, name) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_promotion_scores.py b/tests/test_promotion_scores.py index 4a07726..a8592f0 100644 --- a/tests/test_promotion_scores.py +++ b/tests/test_promotion_scores.py @@ -12,9 +12,7 @@ import unittest ROOT = Path(__file__).resolve().parents[1] AGENT = ROOT / 'refactored/managed-container' -MODULES = ('common', 'container', 'postgres', 'auth', 'replication', 'notify', 'promote', 'demote', 'rejoin') -LOAD = 'MC_PROMOTION_RESOURCE_INSTANCE="${OCF_RESOURCE_INSTANCE:-}"\n: "${OCF_ROOT:=/usr/lib/ocf}"\n. "$OCF_ROOT/lib/heartbeat/ocf-shellfuncs"\n: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}"\n' -LOAD += '\n'.join('. ' + shlex.quote(str(AGENT.parent / 'managed-container.d' / (m + '.sh'))) for m in MODULES) +from plugin_test_support import LOAD CRM = '''#!/usr/bin/env python3 import json, os, sys @@ -282,7 +280,7 @@ class Scores(unittest.TestCase): for action in ('monitor', 'status'): code = 'mc_promotion_monitor() { echo scored-monitor; return 8; }; set -- ' + action + '\ncase "$1" in' + dispatch self.assertEqual(self.run_shell(code, 8).stdout.strip(), 'scored-monitor') - for action, handler in [('meta-data', 'mc_promotion_metadata'), ('validate-all', 'mc_promotion_validate'), ('notify', 'managed_notify')]: + for action, handler in [('meta-data', 'mc_engine_metadata'), ('validate-all', 'mc_promotion_validate'), ('notify', 'mc_application_notify')]: code = handler + '() { echo plain; return 0; }; set -- ' + action + '\ncase "$1" in' + dispatch self.assertEqual(self.run_shell(code).stdout.strip(), 'plain') self.assertFalse(self.state()['calls']) diff --git a/tests/verify_installed_rpm.py b/tests/verify_installed_rpm.py index 7689feb..fc2ca5e 100644 --- a/tests/verify_installed_rpm.py +++ b/tests/verify_installed_rpm.py @@ -18,11 +18,11 @@ assert output('rpm', '-q', '--qf', '%{ARCH}', PACKAGE) == 'noarch' assert output('rpm', '-q', '--scripts', PACKAGE) == '' subprocess.run(['rpm', '-V', PACKAGE], check=True) expected = {AGENT: ROOT / 'refactored/managed-container'} -for path in (ROOT / 'refactored/managed-container.d').glob('*'): - if not path.is_file(): +for path in (ROOT / 'refactored/managed-container.d').rglob('*'): + if not path.is_file() or path.suffix not in ('.sh', '.py'): continue - expected[PROVIDER / 'managed-container.d' / path.name] = path -assert len(expected) == 11 + expected[PROVIDER / path.relative_to(ROOT / 'refactored')] = path +assert len(expected) == 15 for path, source in expected.items(): assert path.read_bytes() == source.read_bytes(), path info = path.stat() @@ -31,6 +31,11 @@ for path, source in expected.items(): owned = set(output('rpm', '-ql', PACKAGE).splitlines()) allowed = {str(path) for path in expected} | {str(PROVIDER), str(PROVIDER / 'managed-container.d')} +for path in expected: + allowed.update(str(parent) for parent in path.parents if parent == PROVIDER or PROVIDER in parent.parents) +for directory in allowed - {str(path) for path in expected}: + info = Path(directory).stat() + assert (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode)) == (0, 0, 0o755), directory unexpected = {path for path in owned - allowed if not path.startswith('/usr/share/doc/' + PACKAGE)} assert not unexpected, unexpected diff --git a/tests/verify_promotion_sql.py b/tests/verify_promotion_sql.py index 9e8c234..93d554a 100644 --- a/tests/verify_promotion_sql.py +++ b/tests/verify_promotion_sql.py @@ -18,7 +18,7 @@ ROOT = Path(__file__).resolve().parents[1] with tempfile.TemporaryDirectory(prefix='mc-score-sql-') as directory: query_file = Path(directory) / 'query.sql' env = dict(os.environ, QUERY_FILE=str(query_file)) - extract = '. ' + shlex.quote(str(ROOT / 'refactored/managed-container.d/postgres.sh')) + ''' + extract = '. ' + shlex.quote(str(ROOT / 'refactored/managed-container.d/runtimes/podman.sh')) + '\n. ' + shlex.quote(str(ROOT / 'refactored/managed-container.d/applications/postgresql/postgres.sh')) + ''' as_service_user() { printf '%s' "${@: -1}" > "$QUERY_FILE"; echo t; } mc_pg_promotion_eligible ''' diff --git a/tests/verify_promotion_state_cli.py b/tests/verify_promotion_state_cli.py index 34fe357..2fba517 100644 --- a/tests/verify_promotion_state_cli.py +++ b/tests/verify_promotion_state_cli.py @@ -12,7 +12,7 @@ from test_disconnected_state import CONNECTED, DISCONNECTED assert os.geteuid() == 0 and os.environ.get('MC_DISPOSABLE_STATE_TEST') == '1', ( 'Run only in a disposable container with MC_DISPOSABLE_STATE_TEST=1') helper = (Path(sys.argv[1]) if len(sys.argv) > 1 else - Path(__file__).resolve().parents[1] / 'refactored/managed-container.d/promotion_state.py') + Path(__file__).resolve().parents[1] / 'refactored/managed-container.d/applications/postgresql/promotion_state.py') resource = 'mc-disposable-' + uuid.uuid4().hex args = ['python3', str(helper), resource, 'test-owner', 'test.service', 'test-container', '300'] diff --git a/tests/verify_refactor.py b/tests/verify_refactor.py index 3611bbe..8be3b60 100644 --- a/tests/verify_refactor.py +++ b/tests/verify_refactor.py @@ -6,11 +6,12 @@ import re import subprocess import tempfile import xml.etree.ElementTree as ET +from plugin_test_support import LOAD, restore_boundaries ROOT = Path(__file__).resolve().parents[1] REFERENCE = ROOT / 'reference/managed-container' AGENT = ROOT / 'refactored/managed-container' -MODULES = sorted((AGENT.parent / 'managed-container.d').glob('*.sh')) +MODULES = sorted((AGENT.parent / 'managed-container.d').rglob('*.sh')) SOURCE = REFERENCE.read_text() PATTERN = r'^([a-z_]+)\(\)\n\{\n.*?^\}' @@ -43,21 +44,16 @@ print('PASS both reference hashes') current_functions = functions('\n'.join(p.read_text() for p in MODULES)) for name, definition in functions(SOURCE).items(): - assert current_functions[name] == definition, name -assert set(current_functions) - set(functions(SOURCE)) == { - 'mc_promotion_score_set', 'mc_promotion_score_from_monitor', - 'mc_promotion_monitor', 'mc_promotion_action', 'mc_promotion_validate', 'mc_pg_promotion_eligible', - 'mc_promotion_metadata', 'mc_pg_standby_observation', 'mc_pg_standby_score', -} + assert restore_boundaries(current_functions[name]) == definition, name assert len(functions(SOURCE)) == 29 -common = (AGENT.parent / 'managed-container.d/common.sh').read_text() +common = (AGENT.parent / 'managed-container.d/applications/postgresql/plugin.sh').read_text() defaults = SOURCE[SOURCE.index('OCF_RESKEY_service_user_default='):SOURCE.index('\n###############################################################################\n# Metadata')].strip() assert defaults in common -print('PASS all 29 original function bodies and defaults/globals identical; nine scoring helpers added') +print('PASS all 29 original function bodies identical after exact adapter-call reversal; defaults unchanged') for path in [AGENT, *MODULES]: subprocess.run(['bash', '-n', str(path)], check=True) -print('PASS syntax for entrypoint and all nine modules') +print('PASS syntax for entrypoint and all modules') metadata = [] for path in (REFERENCE, AGENT): @@ -69,6 +65,10 @@ parameters = extended.find('parameters') added = parameters.find("parameter[@name='disconnected_max_age']") assert added is not None and added.find('content').get('default') == '300' parameters.remove(added) +for name, default in [('application', 'postgresql'), ('container_runtime', 'podman'), ('lifecycle_manager', 'systemd')]: + added = parameters.find("parameter[@name='" + name + "']") + assert added is not None and added.find('content').get('default') == default + parameters.remove(added) def normalize(element): return (element.tag, element.attrib, (element.text or '').strip(), [normalize(child) for child in element]) @@ -80,9 +80,7 @@ print('PASS original metadata preserved plus freshness parameter and XML parsing # Sourcing production modules in a fresh shell tests their real loading order. # No dispatcher executes. All operational boundaries are overridden below. -modular_definitions = SOURCE[:SOURCE.index('\n###############################################################################')] -for name in ('common', 'container', 'postgres', 'auth', 'replication', 'notify', 'promote', 'demote', 'rejoin'): - modular_definitions += '\n. ' + "'" + str(AGENT.parent / 'managed-container.d' / (name + '.sh')).replace("'", "'\\''") + "'" +modular_definitions = LOAD base = ''' ocf_log() { printf 'log %s\\n' "$*"; } ocf_exit_reason() { printf 'reason %s\\n' "$*" >&2; }