diff --git a/.gitea/workflows/build-rpm.yml b/.gitea/workflows/build-rpm.yml index 67b4fec..0734563 100644 --- a/.gitea/workflows/build-rpm.yml +++ b/.gitea/workflows/build-rpm.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Install build tools run: | - dnf -y install rpm-build python3 resource-agents git tar gzip diffutils findutils coreutils bash nodejs + dnf -y install rpm-build python3 resource-agents pacemaker-cli git tar gzip diffutils findutils coreutils bash nodejs - name: Check out source uses: https://github.com/actions/checkout@v4 with: @@ -25,6 +25,7 @@ jobs: run: | dnf -y install ./dist/*.noarch.rpm python3 tests/verify_installed_rpm.py + MC_DISPOSABLE_STATE_TEST=1 python3 tests/verify_promotion_state_cli.py /usr/lib/ocf/resource.d/loxthelion/managed-container.d/promotion_state.py # v3 uses the artifact protocol supported by older act_runner instances. - name: Save development packages uses: https://github.com/actions/upload-artifact@v3 diff --git a/.gitignore b/.gitignore index a4ac404..d96ad92 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ secrets/ # RPM build trees and development artifacts .build/ dist/ + +# Generated review diffs; Git stores the reviewed source itself +/notes/*.patch diff --git a/HANDOFF.md b/HANDOFF.md index ca4fac5..cc1a207 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,3 +1,7 @@ +> Historical refactor brief. Current production operating policy and validation +> are documented in [docs/operations.md](docs/operations.md) and +> [docs/testing.md](docs/testing.md). + # PostgreSQL HA OCF Resource Agent — Codex Refactor Handoff ## Objective @@ -30,7 +34,9 @@ Control path: -> persistent PostgreSQL maintenance container -> PostgreSQL -The maintenance container PID 1 is intentionally `sleep infinity`. +The container entrypoint runs the bootstrap wrapper, which executes +`sleep infinity` after its bootstrap gate. Podman may place its init shim at +PID 1; PostgreSQL is not the container lifetime process. PostgreSQL itself is independently started/stopped with `pg_ctl`. diff --git a/README.md b/README.md index 55265d2..6a0f4d5 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # PostgreSQL HA OCF resource agent -A behavior-preserving modular refactor of `ocf:loxthelion:managed-container`. +A modular PostgreSQL HA implementation of `ocf:loxthelion:managed-container`. Pacemaker invokes the agent to manage a rootless systemd/Quadlet maintenance container. PostgreSQL is started and stopped independently with `pg_ctl`. ## Project layout - `reference/`: original agent and immutable known-good reference copy. -- `refactored/`: executable agent and nine sourced shell modules. +- `refactored/`: executable agent, nine sourced shell modules, and a Python state helper. - `tests/`: offline structural and behavioral comparison checks. - `cluster-context/` and `runtime-context/`: saved lab configuration and state; these snapshots are not live status. Environment credentials are redacted. @@ -21,21 +21,25 @@ Requires Bash, Python 3, and the OCF shell library (normally ```bash python3 tests/verify_refactor.py +python3 tests/test_promotion_scores.py +python3 tests/test_promotion_scheduler.py +python3 tests/test_disconnected_state.py +python3 tests/test_promotion_cleanup.py sha256sum -c notes/REFACTOR-SHA256SUMS ``` -The tests compare the refactor against the frozen reference without operating -the live cluster. Target-node validation and controlled HA transition tests -remain outstanding. Existing function names and recovery configuration blocks -are retained in this initial modular baseline. +The tests preserve the original 29 lifecycle function bodies and exercise the +added promotion-scoring layer without operating the live cluster. The operator +has reported successful production SBD fencing, automatic promotion, VIP movement, +and former-primary rewind/rejoin in a three-node cluster. -Read [the refactor report](notes/REFACTOR-REPORT.md) for installation commands, -validation results, limitations, and the recommended lab comparison sequence. -The public entrypoint remains -`/usr/lib/ocf/resource.d/loxthelion/managed-container`. +Read [operations](docs/operations.md), [testing and snapshot provenance](docs/testing.md), +and [the review findings](docs/review.md). Historical lab snapshots and notes are +not current production configuration. Keep `target-role=Promoted`, single-primary +limits, and the VIP's Promoted-role colocation and promotion-before-start ordering. -Keep lab Pacemaker maintenance mode enabled during this work. Do not modify -the frozen reference or automatically run destructive HA tests. +The public entrypoint remains `/usr/lib/ocf/resource.d/loxthelion/managed-container`. +No production changes are part of this documentation freeze. ## Clone @@ -49,4 +53,12 @@ Gitea Actions builds development RPMs on `main` pushes and pull requests using an `ubuntu-latest` runner with a Fedora 43 job container. See [packaging instructions](packaging/README.md) for local builds and artifacts. [License provenance remains pending](packaging/PROVENANCE.md); release publishing -and deployment are not configured. +and automated deployment are not configured. + +## Promotion scoring + +Scores are 1000 for a healthy primary, 100 for a strictly verified streaming +standby, 50 for a recently verified eligible disconnected standby, and -INFINITY +otherwise. See [the scoring design](notes/DISCONNECTED-STANDBY.md) for identity, +local state ownership, the 300-second default freshness policy and limitations. +Fencing remains mandatory; scores do not authorize unfenced manual promotion. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..38fbf61 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,130 @@ +# PostgreSQL HA operating policy + +This document describes the intentional production design. Historical two-node +lab snapshots and old installation notes are not production runbooks. Keep the +three-node production clone's **target-role=Promoted** and **promoted-max=1**. +Promotion scores select an eligible node; the target role expresses the desired +promoted service. This project requires that explicit configuration even though +Pacemaker can permit promotion with other default target-role settings. +The VIP must be colocated with the Promoted instance and ordered to start after +promotion, with reverse shutdown ordering. Scores do not create those constraints. + +## Ownership and lifecycles + +Pacemaker invokes the root OCF RA. Its `runuser` boundary enters the rootless +service owner's home and user-systemd runtime, then invokes systemd/Podman with +argument arrays. Quadlet constructs the maintenance container and supplies its +environment, mounts and bootstrap. The bootstrap gates initialization/adoption, +then executes `sleep infinity`; Podman may use an init shim as PID 1. PostgreSQL +is not the container lifetime process. + +The RA independently controls PostgreSQL with `pg_ctl`. During demotion and +rejoin, the maintenance container stays available even when PostgreSQL is +stopped. This is required for inspection, `pg_rewind`, and optional basebackup. +The OCF stop action stops both PostgreSQL and the maintenance service/container. +Starting the container alone is not equivalent to starting the OCF resource. + +The RA reads PGDATA and intentional replication settings from the running +container. It does not parse or source host PostgreSQL env/config files. Quadlet +may intentionally consume a host env-file when constructing that container. +The bootstrap manages its secret-file/pgpass interfaces separately; the RA's +replication commands read `POSTGRES_REPLICATION_PASSWORD` inside the container +and set PGPASSWORD only for the child process. + +## Actions and role reporting + +| Action | Implementation contract | +| --- | --- | +| start | Starts or verifies an Unpromoted standby. Refuses an already-primary instance; when PostgreSQL is stopped, refuses PGDATA without standby.signal. | +| monitor/status | Reports healthy standby 0, healthy primary 8, stopped service/container/PostgreSQL 7, and unhealthy/failed query 1. Promotion eligibility does not change these role codes. | +| promote | Promotes a healthy standby and reconciles local HA slots/rewind grants; an existing primary reconciles those invariants. | +| demote | Controlled switchover: uses fresh pre-demote notification and Corosync successor address, verifies successor standby/catch-up, stops local PostgreSQL and rejoins it as a cascading standby. | +| stop | Stops PostgreSQL, then the maintenance service/container. | +| notify | Persists Pacemaker transition context; not promotion eligibility or fencing proof. | +| rejoin | Private/manual recovery action, deliberately absent from advertised OCF actions. Requires a running maintenance container, stopped PostgreSQL, source and slot. Rewinds then verifies streaming recovery. | +| validate-all/meta-data | Validate dependencies/configuration and report metadata. Validation can return 5 for missing utilities or 6 for invalid configuration. | + +Lifecycle success is 0; monitor's 8 is a role report, not a generic lifecycle +success. Promoted monitoring must expect the correct role. Metadata advertises +13 parameters and eight actions; status is an alias and rejoin stays private. +The scoring wrappers withdraw scores before lifecycle actions and reassess after +successful actions. They do not change the original 29 function bodies. + +## Scoring and fencing + +Healthy primary: 1000. Strictly verified streaming standby: 100. Recently verified +eligible disconnected standby: 50. Unsafe or missing evidence: -INFINITY. +Disconnected evidence is bounded by `disconnected_max_age` (default 300 seconds), +bound to the PostgreSQL incarnation and local boot, and recalculated each monitor. +See [the detailed state model](../notes/DISCONNECTED-STANDBY.md). + +An eligible score is not a fencing certificate. Pacemaker/SBD must isolate the +failed primary and order promotion safely. Never bypass fencing by manually +calling promote, creating standby.signal on divergent data, or deleting data. +Asynchronous replication accepts possible loss of acknowledged transactions; +the freshness window does not bound the number of lost transactions. + +Controlled switchover differs from abrupt failover: demote requires a reachable, +caught-up successor and transition notification. Former-primary rejoin is a +separate recovery procedure after fencing, not an automatic start workaround. + +## Former-primary recovery order + +1. Establish that fencing completed and identify the surviving writable primary + and its VIP. Coordinate cluster management so Pacemaker cannot race the manual + repair. Do not clear the former primary's failed start yet. +2. Bring up only its maintenance container using the existing Quadlet as the + service owner. Verify PostgreSQL is stopped. Preserve the existing data. +3. Confirm the running container's replication credentials are correct without + printing them. Supply the existing resource instance/parameters and a unique + local replication slot to the private rejoin action, using the PostgreSQL VIP + as `rejoin_source_host`. This documentation does not invent missing live IDs. +4. Rejoin first attempts pg_rewind. If it fails, stop and inspect the error. + Basebackup fallback remains disabled unless explicitly authorized to replace + divergent data. It preserves the old directory but is still a destructive + recovery decision; a failed rewind is not guaranteed to leave usable data. +5. Require rejoin success AND verify `pg_is_in_recovery() = true`, a streaming WAL + receiver connected to the intended primary, and the expected slot/application + identity. Verify the maintenance container remained available. +6. **Only now** clear the relevant failed actions for that resource on that node, + and allow Pacemaker to adopt it as Unpromoted. Recheck roles, scores, streaming, + VIP placement, and failed actions. Do not use broad cleanup before recovery. + +Safe inspection commands, run on the appropriate node, include: + +```sh +sudo pcs status --full +sudo pcs resource config postgresql-ha-clone +sudo pcs constraint config --full +``` + +Inspect output locally; raw CIB, environment dumps and logs may contain sensitive +configuration and should not be committed. The optional preference planner reads +an operator-supplied CIB and prints only a narrowly matched temporary preference +removal. It preserves target-role=Promoted and all unrelated constraints. It does +not perform resource failure cleanup or make a node safe to rejoin. + +## Podman env-file incident + +The production test exposed a quoting mismatch. Podman's env-file parser retained +surrounding quote characters, while the generated pgpass had the intended value. +Streaming worked, but rejoin's direct environment password failed authentication. +These are intentionally fake examples, not deployment values: + +```text +# Incorrect if the intended value contains no quotation marks: +POSTGRES_REPLICATION_PASSWORD="EXAMPLE_ONLY_NOT_A_SECRET" +# Correct representation of that same intended value: +POSTGRES_REPLICATION_PASSWORD=EXAMPLE_ONLY_NOT_A_SECRET +``` + +Podman env-file syntax is not shell assignment syntax; do not source it to +validate container behavior. Correct the secret locally and apply it through the +planned Quadlet/container lifecycle: editing a file alone does not alter an +already-running container's environment. Do not blindly restart a primary to +apply this change. Keep real values, .pgpass contents, secret files, and raw +container environment inspection out of Git and shared logs. + +References: [OCF role semantics](https://github.com/ClusterLabs/pacemaker/blob/main/doc/sphinx/Pacemaker_Administration/agents.rst), +[Pacemaker configuration and ordering](https://clusterlabs.org/projects/pacemaker/doc/3.0/Pacemaker_Explained/pdf/Pacemaker_Explained.pdf), +and [Podman run options](https://docs.podman.io/en/latest/markdown/podman-run.1.html). diff --git a/docs/review.md b/docs/review.md new file mode 100644 index 0000000..d778418 --- /dev/null +++ b/docs/review.md @@ -0,0 +1,115 @@ +# Freeze review — 2026-09-12 + +This review preserves the implementation reported as production-tested by the +operator. No production systems were changed. It is a source/documentation +freeze, not certification of every failure mode or a byte-level deployment +attestation. Production hash collection remains outstanding in [testing.md](testing.md). + +## Corrections and their reasons + +- Added operations.md to distinguish container lifecycle, PostgreSQL lifecycle, + Pacemaker state, scores, fencing, controlled switchover and former-primary rejoin. +- Added testing.md to record the operator's SBD failover/VIP/rewind/rejoin sequence + and final three-node roles/scores, separately from local automated checks. + Recorded exact repository RA/module/helper/bootstrap hashes without inventing + deployed hashes. Included read-only per-node hash collection commands. +- Required target-role=Promoted, promoted-max=1, Promoted-role VIP colocation and + promotion-before-VIP-start ordering in the operating policy. Compatibility + tests for absent/Started target roles are not recommendations to change production. +- Fixed plan-promotion-cleanup.py to emit only temporary node-preference removal + and preserve target-role=Promoted. Removed its obsolete inherited-Started + check because it no longer clears the direct target role. Updated its exact + expected-output test; ambiguity and unrelated-rule protections remain. Added + an explicit Promoted-target scheduler case and used that target in fencing + scenarios; compatibility cases remain. +- Explicitly prohibited clearing former-primary failed actions before successful + rejoin and verification of streaming recovery. Documented rewind before any + explicitly authorized basebackup fallback, and that failed rewind may require + further recovery rather than a normal restart. +- Documented literal env-file quotes and the mismatch between generated pgpass + and the RA's direct container environment password. All examples use obvious + fake values; no actual password from the handoff was copied into the repository. +- Updated README and historical handoffs/notes to point to current operations + and evidence. Corrected the PID 1 simplification: bootstrap executes the + maintenance command, possibly behind Podman's init shim. +- Included current operating/testing/review documents in source/RPM documentation. + Retained the old ELITEDESK installer as a clearly marked historical lab tool: + its hostname/cluster/hash guards reject the scoring implementation. It is not + a production installer and was not executed. +- Ignored generated review patches; the commit records the actual files instead + of duplicating obsolete proposed commands in patch attachments. Refreshed the + checksum manifest. Existing local patch files remain available locally. + +The staged scoring, state helper, tests and RPM workflow from the preceding +implementation work are included in this freeze commit. This review did not +modify RA/module/scoring-helper or bootstrap code. The frozen original remains +unchanged, and all 29 original lifecycle bodies still compare byte-for-byte. + +## Checks completed in this review + +- Structural/default/metadata verification and all 22 differential lifecycle cases. +- 25 promotion-scoring tests, 16 disconnected-state/monitor tests, 11 synthetic + Pacemaker scheduler tests and 4 corrected preference-planner tests: all pass. +- Nine SQL scenarios against a disposable PostgreSQL 18 container: all pass for + the original strict predicate and current observation query. Container removed. +- Bash syntax and Python AST parsing across repository code and historical installer: + pass. ShellCheck and Gitleaks were not installed; no claim of running them is made. +- Repository/diff review for private keys, recognizable tokens, credential-bearing + URLs, and credential assignments, plus manual inspection of matches: no actual + secrets found. Matches were variable references, redacted snapshots, or explicit + fake examples. No CIB, secret file, .pgpass content or raw environment dump was added. +- A disposable Podman test reproduced literal env-file quotes using only a fake + value. No production credentials or environments were inspected. +- Fedora 43 development RPM build with the new docs passed; artifacts are in + dist/documentation-freeze/ and the log is /tmp/mc-freeze-build.log. The extra + Promoted-target scheduler case was subsequently verified separately. +- Checksum manifest and git diff whitespace checks are verified immediately before + commit. + +## Privilege and safety boundaries reviewed + +The RA does not source host-side PostgreSQL environment/configuration. Its +root-to-service-user boundary preserves argument arrays, and container commands +consume intentional injected settings. Authentication secrets are not placed in +RA command arguments or host-side pgpass files. The stored bootstrap is separate +and has a different provisioning interface. + +The scoring helper uses bounded subprocesses, a root-only runtime directory, +strictly parsed JSON, per-resource locks and atomic replacement. Its identity +includes postmaster start time, so a restarted/replaced data instance must stream +successfully before receiving disconnected eligibility. Notification state uses +a separate root-owned /run directory and relies on Pacemaker serializing actions +for the same resource; it is not a distributed locking or fencing mechanism. + +Original SQL/shell construction in some lifecycle helpers assumes trusted, +conventional resource configuration (identifiers, paths, connection fields). +It is not a general escaping layer for arbitrary user-supplied text. Restrict CIB +write access and container ownership; do not expose RA parameters to untrusted +input. No new untrusted input interface was introduced by this review. + +## Remaining concerns and explicit limits + +- The three production nodes' exact deployed hashes and confirmation that those + bytes were used in the fencing test are missing. Source hashes are not a + substitute. The runtime-context bootstrap may differ from deployed production. +- The saved bootstrap's fresh-primary provisioning path passes a replication + password through a psql --set argument. That can expose it to process-argument + inspection under local /proc permissions. This is a concrete existing exposure, + not a claim that the RA rejoin path leaks a password. It must be reviewed against + the actual deployed bootstrap before calling provisioning secret-safe; no + bootstrap rewrite or production secret change was made in this freeze. +- Existing bootstrap pgpass generation assumes values needing no additional + colon/backslash escaping. Do not treat the documented quote correction as a + universal password-encoding solution. Inspect the deployed bootstrap before + changing provisioning behavior. +- Scores are observations, not fencing authorization or atomic promotion guards. + IPC failure can leave a prior score; monitor cadence/fencing latency can exceed + freshness. Two disconnected candidates are not ranked by cross-node WAL history. +- Asynchronous replication can lose acknowledged transactions. Neither the test + nor the freshness threshold proves a numerical data-loss bound. +- The reported fenced-failure/rejoin success does not prove every concurrent-write + controlled-switchover scenario or optional destructive basebackup fallback. +- License provenance remains unresolved as documented in packaging/PROVENANCE.md. + This source freeze is not a license grant or public-release authorization. + +No push or production rollout is part of this review. diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..e5d00a3 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,122 @@ +# Validation and freeze-point evidence + +## Reported production fencing and recovery test + +Evidence source: operator-supplied production handoff received during the +2026-09-12 documentation review. The operator reports a successful real +three-node Pacemaker/Corosync test with SBD fencing. Codex did not execute or +independently witness this production test. Its exact start/end timestamps and +raw cluster logs were not supplied; no timestamps or measurements are invented. + +1. david-eve was primary and was fenced successfully through SBD. +2. david-prodesk promoted automatically; the PostgreSQL VIP moved with it. +3. david-macbookpro remained/reconnected as a streaming standby. +4. david-eve rebooted with former-primary PGDATA. Ordinary resource start + correctly refused it because standby.signal was absent. +5. The operator manually started its maintenance container with PostgreSQL stopped. +6. The private rejoin action used the PostgreSQL VIP as rewind source. + pg_rewind detected divergence, found the common checkpoint and completed. +7. david-eve started as a streaming standby. Only after recovery, Pacemaker + cleanup allowed the cluster to adopt it as Unpromoted. + +Reported final state: + +| Node | Role | Score | +| --- | --- | ---: | +| david-prodesk | Promoted; PostgreSQL VIP local | 1000 | +| david-macbookpro | Unpromoted; asynchronous streaming | 100 | +| david-eve | Unpromoted; asynchronous streaming | 100 | + +No PostgreSQL failed actions remained. The successful run demonstrates the +reported fencing/failover/VIP/rejoin sequence, not every possible partition, +latency, concurrent-write, or recovery failure. The intermediate disconnected +score and lost-transaction count were not supplied. The quoting/authentication +incident and required fix are documented in [operations](operations.md). + +## Validated production snapshot — exact deployed hashes pending + +The production outcome is operator-reported validated; **the byte-level production +snapshot is not yet established**. Exact hashes from david-prodesk, +david-macbookpro and david-eve have not been supplied or collected. Repository +hashes below must not be represented as deployed hashes. A later capture also +requires operator confirmation that those bytes are the ones used during the test. + +| Artifact on each production node | Snapshot status | +| --- | --- | +| /usr/lib/ocf/resource.d/loxthelion/managed-container | Pending production SHA-256 | +| managed-container.d/*.sh (all nine modules) | Pending production SHA-256 | +| managed-container.d/promotion_state.py | Pending production SHA-256 | +| Running container /usr/local/sbin/postgres-ha-bootstrap | Pending production SHA-256 | + +Run the following read-only commands on **each production node**, using the +rootless container owner for podman. They hash code, not environment or secrets: + +```sh +hostname +date -u '+%Y-%m-%dT%H:%M:%SZ' +sha256sum /usr/lib/ocf/resource.d/loxthelion/managed-container \ + /usr/lib/ocf/resource.d/loxthelion/managed-container.d/*.sh \ + /usr/lib/ocf/resource.d/loxthelion/managed-container.d/promotion_state.py +podman exec postgres sha256sum /usr/local/sbin/postgres-ha-bootstrap +podman inspect --format '{{.Image}}' postgres +``` + +If permissions require it, run only the host sha256sum command with sudo. Do not +run rootful podman instead of the actual container owner. If the installed +bootstrap path/container name differs, record the actual path/name. Capture +node identity, UTC capture time, all filenames/digests, immutable image ID, and +whether any code changed since the test. Hash the running container's bootstrap; +a host snapshot or image tag alone is not equivalent. Do not dump environment, +secret files, or pgpass. Compare every node individually, preserving mismatches. + +## Repository freeze snapshot + +The exact repository code hashes appear below and in the reviewed checksum +manifest. These identify the source freeze only, pending production comparison. +The bootstrap is a saved runtime-context artifact, not part of the RA RPM and +not yet confirmed identical to the production bootstrap. + +| Repository file | SHA-256 | +| --- | --- | +| `refactored/managed-container` | `a4b775a0ddcabdb0558da9b34c7fe2a883c07d0ef308c1c7224526abf53f63a0` | +| `refactored/managed-container.d/auth.sh` | `93191771d3cb8aefadb2a16d2f89d64a8b6c584071a473439be818237bdf7f17` | +| `refactored/managed-container.d/common.sh` | `97e93d9164e854e5f3688cf9d3cc656b001eb30ef3c2073c907450ca43588f3d` | +| `refactored/managed-container.d/container.sh` | `ac6bc363a2301f1491d162ef609103dbfbceb3b099a3556614c6165952309374` | +| `refactored/managed-container.d/demote.sh` | `0af3bb362a303017372c631354ba94ef9e723bf14e64d2eff91b6662e54a5be0` | +| `refactored/managed-container.d/notify.sh` | `6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670` | +| `refactored/managed-container.d/postgres.sh` | `d2779ce6e7d1f426ffca2e41bbb8de77503005dc38d658d69f184020365796eb` | +| `refactored/managed-container.d/promote.sh` | `3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58` | +| `refactored/managed-container.d/rejoin.sh` | `21beea72bcb7f24a51c17ea48666ac87df9436702ad77b166b45f4647e28e8ad` | +| `refactored/managed-container.d/replication.sh` | `6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547` | +| `refactored/managed-container.d/promotion_state.py` | `5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7` | +| `runtime-context/postgres-ha-bootstrap` | `3f1a15f7054fb69b9aa92204de30306fd6053c63560310f4231f314b148e2dcf` | + + +## Non-production checks + +The source retains all 29 original lifecycle function bodies and original +parameter defaults. Added scoring changes public dispatch and metadata; it is +not a claim of identical total behavior to the monolithic reference. + +Run from the repository root: + +```sh +python3 tests/verify_refactor.py +python3 tests/test_promotion_scores.py +python3 tests/test_disconnected_state.py +python3 tests/test_promotion_scheduler.py +python3 tests/test_promotion_cleanup.py +sha256sum -c notes/REFACTOR-SHA256SUMS +git diff --check +``` + +The baseline suite covers 22 differential lifecycle cases, 25 scoring tests, +16 state/monitor tests, 11 real scheduler simulations and 4 planner tests. +Scheduler failure simulations use synthetic CIBs, enable fencing, and assert +fencing prerequisites for promotion. They do not operate a live cluster. + +`tests/verify_promotion_sql.py` explicitly creates/removes a disposable PostgreSQL +18 container and checks nine strict/observation SQL scenarios. Previous candidate +RPM build/install and root-helper CLI verification passed in disposable Fedora +containers. They are separate from the operator's production validation. +Current freeze review results are recorded in [review.md](review.md). diff --git a/notes/DISCONNECTED-STANDBY.md b/notes/DISCONNECTED-STANDBY.md new file mode 100644 index 0000000..fb39b74 --- /dev/null +++ b/notes/DISCONNECTED-STANDBY.md @@ -0,0 +1,198 @@ +> Production fencing/rejoin was subsequently reported successful by the operator. +> See [../docs/testing.md](../docs/testing.md) for evidence and snapshot status, +> and [../docs/operations.md](../docs/operations.md) for current operating policy. +> Deployment statements below describe the original development session only. + +# Disconnected standby implementation and original candidate validation + +This revision uses the accepted promotion-scoring candidate as its baseline. +The user explicitly accepts asynchronous replication data-loss risk after an +unexpected primary failure, provided Pacemaker fences the failed primary before +promotion. No installed RA, live cluster attributes, constraints, fencing +configuration, or PostgreSQL instance was changed during this revision. + +## State model and scores + +Every monitor first runs the original role/health monitor. Primary remains +1000 with OCF return 8; stopped, unready, and failed role detection publish +-INFINITY with their original return codes. Healthy standbys report OCF return +0 regardless of eligibility. Each standby monitor collects a new local SQL +observation and evaluates one of these states: + +| State | Evidence | Score | +| --- | --- | --- | +| Connected and verified | In recovery; replay unpaused; receiver streaming; flush non-NULL; replay >= flush | 100 | +| Disconnected and verified recently | Same instance; recovery and replay healthy; no streaming receiver; valid fresh stored observation; current replay >= saved replay | 50 | +| Ineligible | Missing, expired, invalid or incompatible evidence; replay paused/regressed; connected strict check failed; query/helper failure | -INFINITY | + +The strict connected predicate has not been relaxed. A streaming receiver with +lag or NULL flush cannot fall back to disconnected eligibility. A missing +receiver, or a PostgreSQL receiver in starting/waiting/restarting/stopping state, +can use the disconnected path. A valid connected observation replaces the +record atomically; disconnected observations never renew its timestamp. +Reconnection must pass the strict checks to create fresh evidence. + +Score publication still uses bounded `crm_attribute --promotion` query/update +calls. A missing transient attribute is recreated from current eligibility, +including disconnected eligibility. This is not retention of an old +100. +Lifecycle wrappers continue to withdraw scores before actions and re-evaluate +after successful actions. The original 29 function bodies, defaults, and all +original safety checks remain byte-identical. An added metadata wrapper +advertises the new parameter without modifying the original metadata function. + +## Exact persistence and ownership + +Host-side directory: `/run/managed-container/promotion/`. +Both `/run/managed-container` and its `promotion` child are root-owned mode 0700. +Each record is `.json`, root-owned mode 0600, with a corresponding +root-owned mode 0600 `.lock`. The key is SHA-256 of the UTF-8 compact JSON +array `[OCF_RESOURCE_INSTANCE, service_user, service_name, container_name]`. +An instance suffix change conservatively selects a different record. + +The RA's root process runs `managed-container.d/promotion_state.py` with Python +3; the rootless service user cannot read/write the host state directory. There +are no credentials, connection strings, passwords, or upstream secrets in the +record. Contents are schema version, resource context, host boot ID, PostgreSQL +system identifier, actual SQL `data_directory`, postmaster start timestamp, +verified replay and flush LSNs, wall-clock observation timestamp for audit, +and boot-relative observation time for expiry. + +A nonblocking per-resource lock serializes evaluation and same-directory atomic +replacement. Temporary files use mode 0600; JSON is bounded and parsed as data, +with duplicate keys, invalid types, invalid LSNs, and extra/missing fields +rejected. Unsafe directory/file modes, ownership or symlinks fail closed. +Failed persistence withdraws eligibility. Runtime state survives separate RA +invocations and Pacemaker transient score loss, but not host reboot; boot ID +also guards against restored state from a previous boot. + +## Freshness and identity policy + +OCF parameter `disconnected_max_age`: integer seconds 0–99999, default **300**. +Zero disables disconnected eligibility. Ages from zero through the threshold +are accepted; negative, future, or greater ages are rejected. Invalid parameter +values fail validation and cannot yield a positive standby score. + +Expiry uses Linux CLOCK_BOOTTIME, which includes suspended time. Changing the +wall clock does not extend or prematurely expire an observation. Choose the +threshold to cover the monitoring, failure detection, fencing, and scheduling +time needed by the cluster. Slow fencing may outlast this window and leave no +eligible candidate; freshness is deliberately not extended in response. + +The data identity is `(system_identifier, data_directory, postmaster_start_time)` +plus host boot/resource context. A new PostgreSQL system ID or data path +invalidates the observation. **Any PostgreSQL restart also requires a new strict +streaming verification**, even when restarting the same data. This is deliberate: +a basebackup or same-cluster replacement can preserve the system identifier, +but cannot preserve the running postmaster incarnation. Normal reinitialization, +rewind/rejoin restart, or container restart therefore cannot inherit eligibility. +Current replay must also be at least the stored replay LSN. + +Malformed, stale or mismatching records are deleted when observed by the +state evaluator. A subsequent strict streaming verification creates a new +record. A stopped PostgreSQL process is immediately scored -INFINITY; its file +may remain until the next standby evaluation, but its replacement/restarted +process cannot use it. A healthy primary keeps 1000 and does not consume standby +evidence. Stale unused resource-key files disappear at host reboot. + +## Candidate selection and fencing + +100 ranks currently verified connected evidence above recent disconnected +proof at 50, while preserving primary 1000. Two disconnected candidates receive +equal 50. Pacemaker resolves ties using its configured scheduling rules; no +preferred hostname is hardcoded. Tests show repeatable selection for an +identical synthetic CIB, not a guarantee of a particular host in every cluster. + +We intentionally do not map LSNs or local observation timestamps into unequal +scores. A 64-bit LSN cannot be represented faithfully in Pacemaker's bounded +score range, and comparison needs compatible timeline/history context. Local +observation age is not a measure of WAL completeness: a more recent observation +can contain less WAL than an older observation on another node. Cross-node +clock timestamps introduce further uncertainty. Replay and flush LSNs are +stored for a future explicit comparison policy; this revision has no cross-node +shell access or invented distributed ranking mechanism. + +Fencing remains mandatory and is entirely Pacemaker's responsibility. The RA +does not inspect fencing completion or create/modify constraints. A score of 50 +means locally eligible under the accepted asynchronous policy; it is not +permission to run the promote action manually against an unfenced primary. +Configured fencing, quorum, and action ordering must be validated in the lab. +The synthetic failure tests use fencing enabled and check that every promotion +has a fencing action as a transitive prerequisite in the scheduler graph. + +## Verification and review artifacts + +- Original frozen hashes and all 29 original function bodies/defaults: unchanged. +- All 22 original differential lifecycle cases pass. +- All 25 accepted scoring/action tests pass; mock boundary and metadata handler + were adjusted for the added helper without removing existing assertions. +- 16 state/monitor tests cover streaming-disconnect, never verified, exact + expiry/disabled policy, malformed and unsafe state, system/path/postmaster + changes, reinitialization replacement, replay pause/regression, connected + failures, reconnect, clock changes, stopped PostgreSQL, score loss/reprobe, and query/helper failures. +- 10 real `crm_simulate` synthetic-CIB tests pass, including the original six, + unexpected primary failure with one/two disconnected candidates, mandatory + fencing dependencies, all-stale refusal, and connected preference. +- All 4 cleanup-planner tests pass. No generated cleanup commands were executed. +- PostgreSQL 18 integration: all nine original strict SQL cases pass, and the + new JSON observation query agrees with each. Only a disposable container with + tmpfs data and no network/host ports was used. +- The root helper CLI also passed in a disposable Fedora container, using the + actual fixed /run path: never-verified refusal, 100 -> 50, ownership/modes, + unchanged disconnected timestamp, malformed-state deletion and identity change. +- Fedora 43 RPM build and disposable installation validation passed, recorded in + `/tmp/mc-disconnected-rpm.log`; artifacts are in `dist/disconnected-review/`. +- Incremental diff from the accepted candidate: `DISCONNECTED-STANDBY.patch`. + Full diff from committed baseline: `PROMOTION-SCORING.patch`. + +Run the offline suite with: + +```sh +python3 tests/verify_refactor.py +python3 tests/test_promotion_scores.py +python3 tests/test_disconnected_state.py +python3 tests/test_promotion_scheduler.py +python3 tests/test_promotion_cleanup.py +``` + +The optional disposable SQL test is `python3 tests/verify_promotion_sql.py`. +During the original candidate-development session, no live HA transitions or +deployment were performed. Subsequent operator-reported production results are +recorded in ../docs/testing.md. The previous scoring +report is retained as historical context; its strict-disconnection blocker is +superseded by this policy. Existing lab/production cleanup prerequisites remain. + +## Remaining limitations + +Asynchronous replication does not guarantee zero data loss. The 300-second +window bounds observation age, **not lost bytes or transactions**, and does not +prove that a standby has the failed primary's last acknowledged transaction. +Nor does this design guarantee selection of the most advanced of two eligible +standbys. It does not establish common timelines across peers. + +The SQL user must be allowed to execute `pg_control_system()` and read the +required monitoring fields. Insufficient privileges fail closed; no grants are +changed automatically. Restarting a disconnected standby forfeits eligibility +until it streams again. Loss of /run state likewise requires fresh verification. +Replacing files underneath a running PostgreSQL process or deliberately restoring +fabricated identity/state is outside supported PostgreSQL operation; root and +the database monitoring boundary are trusted. + +The scoring SQL retains its 3-second external bound (1-second kill grace) and +2-second statement timeout. State evaluation adds a 1-second external bound +(1-second kill grace); crm_attribute retains each 2-second bound and 1-second +grace. Original lifecycle/role queries and operation timeouts are unchanged; +validate the aggregate monitor budget on actual hardware. + +Eligibility is sampled, not an atomic transaction with future promotion. +Publication/IPC failure can leave an old Pacemaker score until the next +successful update, as in the accepted baseline. Scheduling and fencing can +also outlast an observation's expiry before the next monitor. This revision +preserves the existing promote implementation rather than adding a new final +promotion guard. Lab testing must cover detection/fencing latency, failover in +both directions, reconnect/rejoin, monitoring load, and score updates during +failure before a production rollout is approved. + +Sources: PostgreSQL documents the system identifier and server start timestamp +in [system information functions](https://www.postgresql.org/docs/18/functions-info.html). +Pacemaker's fencing, resource requirements and promotion scheduling are described +in [Pacemaker Explained](https://clusterlabs.org/projects/pacemaker/doc/3.0/Pacemaker_Explained/pdf/Pacemaker_Explained.pdf). diff --git a/notes/PROMOTION-SCORING.md b/notes/PROMOTION-SCORING.md new file mode 100644 index 0000000..b85ea8c --- /dev/null +++ b/notes/PROMOTION-SCORING.md @@ -0,0 +1,280 @@ +> Current production report and required operating policy: [../docs/testing.md](../docs/testing.md) +> and [../docs/operations.md](../docs/operations.md). The historical lab plan below +> is not a production runbook. + +> Historical accepted scoring baseline. The current disconnected policy, tests, +> and limitations are in [DISCONNECTED-STANDBY.md](DISCONNECTED-STANDBY.md). +> The pending decision and strict-disconnection blocker below are superseded. + +# Promotion scoring — undeployed review candidate + +This enhancement is based on the canonical modular source at c8636ed. This task changed no +installed agent, production configuration, fencing setting or live cluster +attribute. The earlier baseline installer is now pinned to that +baseline entrypoint and refuses this candidate. Do not deploy it using the old +installation instructions. + +## Decision still required before rollout + +**Strict live-streaming eligibility can prevent failover when the primary is +lost.** A previously healthy standby can lose its WAL receiver before Pacemaker +chooses a successor. This candidate then replaces its positive score with +-INFINITY. With all candidates ineligible, no instance is promoted. The SQL and +scheduler tests confirm this outcome; it is not a claim of preserved automatic +failover. Controlled switchovers and abrupt primary-loss failovers must both be +reviewed. The user's choice between this conservative first candidate and a +separate disconnected-standby eligibility policy is pending. + +There is no inferred grace period, persisted last-known-good eligibility, or +cross-node WAL ranking. Retaining an old positive score merely because a +standby once streamed would contradict the requested stale-score cleanup. +Resolving disconnected-standby promotion safely requires an explicit data-loss +policy and validated fencing/recovery assumptions. Do not remove the production +preference or reset production metadata until this decision and lab validation +are complete. + +## Scoring policy + +| Observed state | Published score | Monitor result | +| --- | --- | --- | +| Original monitor reports healthy primary | 1000 | 8 (Promoted) | +| Healthy standby; in recovery, replay unpaused, streaming receiver, replay LSN at least receiver's flushed LSN | 100 | 0 (Unpromoted) | +| Healthy standby but eligibility SQL false, empty, malformed, failed or timed out | -INFINITY | 0 (Unpromoted) | +| Service/container/PostgreSQL stopped | -INFINITY | Original result, normally 7 | +| Unready, failed role query or unexpected role | -INFINITY | Original failure code | + +The eligibility query is local to the running container. It uses the durable +`flushed_lsn`, not `written_lsn`. SQL NULL values cannot yield eligibility. +A CASE expression avoids invoking recovery-only functions on a primary. +Standbys receive equal scores when eligible; no preferred hostname is encoded. +Requiring replay to catch up to locally flushed WAL is conservative and can +make eligibility fluctuate under sustained write load. This is a real state +change, not artificial periodic CIB rewriting. + +A positive score is an observation of local eligibility, not proof that a node +has every acknowledged transaction from the primary, is freshest among peers, +or is safe to promote without fencing. Asynchronous replication can lose +transactions. These scores do not implement consensus or replace PostgreSQL +recovery checks, Pacemaker constraints, quorum, or fencing. + +## Placement and scope + +All 29 original function definitions, defaults and metadata remain byte-for-byte +identical. The existing nine-module layout remains. Six helpers were added: + +- common.sh: mc_promotion_score_set, mc_promotion_score_from_monitor, + mc_promotion_monitor, mc_promotion_action, mc_promotion_validate. +- postgres.sh: mc_pg_promotion_eligible. + +The dispatcher now routes monitor/status through mc_promotion_monitor, which +calls the original managed_monitor once, publishes a score and returns the +original role code. Internal lifecycle monitor calls remain unchanged and do +not publish scores mid-transition. + +Start/promote/demote/stop/rejoin publish -INFINITY before validation or mutation. +After a successful non-stop action, the original monitor is called again and +eligibility is reassessed. Failure leaves the negative score until a subsequent +fresh monitor observation. Successful stop leaves -INFINITY. Stop still skips +validate_all and attempts shutdown even if publication fails. Notify retains +its original notification persistence behavior and does not affect scores. +Metadata and validate-all publish no scores. validate-all additionally checks +that crm_attribute and timeout exist; deployments require Pacemaker >=2.1.0 +for the modern --promotion option, enforced by the RPM dependency. + +The entrypoint captures the explicitly supplied OCF_RESOURCE_INSTANCE before +ocf-shellfuncs can supply a dummy standalone default. If none was supplied, +publication logs a warning and performs no write. Pacemaker resolves the node +from its execution context (including OCF_RESKEY_CRM_meta_on_node); there are no +node-name literals or manual node-resolution rules in the RA scoring code. + +The helper uses: + +```text +crm_attribute --promotion= --lifetime=reboot --query --quiet +crm_attribute --promotion= --lifetime=reboot --update= +``` + +Only a missing or different value causes an update. No local cache or durable +eligibility file is used, so reprobes restore missing transient scores. Values +are replaced, never incremented. A negative value is used instead of deletion: +a missing score alone is not an explicit promotion prohibition. + +Pacemaker currently implements this API with an internal master- +attribute name. Seeing that prefix for the NEW resource is expected. The RA +never constructs, reads, deletes or migrates master-postgresql, +postgresql-data-status, or postgresql-status. Stale attributes for the old +primitive are not inputs to this code. Independently configured location rules +that reference old attributes remain cluster configuration and require their +own operator review. + +Each score query/write is bounded by a two-second timeout plus a one-second +kill grace. The new eligibility command has a three-second outer timeout plus +a one-second kill grace and a two-second SQL statement timeout. The existing +monitor's own command timeouts are unchanged. Account for these additions in +lab operation timing before production rollout. + +If attribute publication fails, the helper logs an error explicitly warning +that the prior score may remain. It retries on the next observation. Role +return codes and existing lifecycle results are preserved; shutdown is never +blocked by a score-write failure. **No RA can guarantee stale-score removal +while the cluster attribute service is unreachable.** This failure mode needs +lab validation and is not hidden by reporting a false PostgreSQL role. +Pacemaker normally serializes actions for a resource; parallel direct/manual +calls that race score updates are not supported by this implementation. + +## Tests and evidence + +Commands run from the source tree: + +```bash +bash -n refactored/managed-container +for module in refactored/managed-container.d/*.sh; do bash -n "$module"; done +python3 tests/verify_refactor.py +python3 tests/test_promotion_scores.py +python3 tests/test_promotion_scheduler.py +python3 tests/test_promotion_cleanup.py +python3 tests/verify_promotion_sql.py +``` + +- Original 22 differential scenarios pass. All 29 original definitions and the + same 12 metadata parameters / eight advertised actions are preserved. +- 25 scoring tests pass, including failures, stop/start, promote/demote/rejoin, + stale positive replacement, repeated monitors, loss/recreation of transient + attributes, missing explicit resource identity and score-publication errors. +- A real crm_attribute test uses CIB_file with a synthetic CIB. Repeated equal + observations do not change the file; negative replacement and recreation + after loss of transient attributes work. +- Six real crm_simulate tests on synthetic CIBs pass: absent and explicit + Started target-role each permit one promotion; Unpromoted prevents it; + larger scores win, ineligible nodes are excluded, and all-negative scores + permit no promotion. Old resource attributes are seeded and do not alter + this new resource's election. These are scheduler simulations, not live HA + transitions. The synthetic CIB disables fencing only inside its test file; + the live lab/production fencing configuration was not touched. +- Nine SQL cases pass in a separate ephemeral PostgreSQL 18 container with no + host ports or data mounts, evaluating the actual production SQL against + controlled state fixtures. Includes replay pause/lag, missing positions, + stopped/absent receiver, and recovery-only function handling on a primary. +- Four cleanup-command planner tests pass, including ambiguous-rule rejection + and preservation of unrelated rules. + +The RPM %check now runs the regression, scoring and scheduler suites. PostgreSQL +integration is an explicit opt-in local test needing Podman and a local +postgres:18 image; it does not touch the lab postgres container. + +## Specific disposable two-node lab sequence (after review) + +Keep maintenance-mode=true as required by the lab handoff. Use direct RA +operations only after installing the reviewed version under a separate approved +installation procedure. Automatic scheduling claims must first be exercised +with a copied CIB and crm_simulate; do not disable maintenance mode as part of +these instructions. Do not use this sequence on production. + +1. Save the lab CIB, installed executable/modules, and current state on both + nodes. Verify the cluster is postgres-ra-test, the roles and streaming source, + current PGDATA backup/recovery baseline, and that production is not targeted. + Establish timing and monitor outputs from the existing implementation. +2. Run metadata and validate-all for the candidate on each lab node. For direct + actions always supply the actual primitive ID (pg-ha in saved lab context), + not the clone ID. Example observation commands on each lab node: + + ```bash + sudo env OCF_RESOURCE_INSTANCE=pg-ha \ + /usr/lib/ocf/resource.d/loxthelion/managed-container monitor + # Interpret rc=8 as healthy primary, rc=0 as healthy standby. + sudo crm_attribute --promotion=pg-ha --lifetime=reboot --query --quiet + ``` + + Expect primary=1000 and a caught-up streaming standby=100. Repeat monitors; + verify no writes for identical scores and no unrelated attribute changes. +3. On the standby, test full stop and start: stopped service/container and + PostgreSQL must yield monitor=7 and score=-INFINITY; standby start restores + monitor=0 and score=100 after streaming/catch-up. Verify primary start refusal + remains intact. Do not use a successful role-return code as a score assertion. +4. Separately stop only standby PostgreSQL while keeping the maintenance + container alive; observe -INFINITY, then start it through the proven standby + path. Verify no rejoin is attempted while PostgreSQL is running. +5. Pause replay on the standby (lab only), observe monitor=0 and -INFINITY, + resume replay and await catch-up/100. Test receiver interruption, SQL errors, + stopped container, and a seeded prior positive score becoming ineligible. +6. Exercise the existing controlled switchover sequence in both directions: + persist the fresh matching pre-demote notification on the old primary, + demote through the Corosync-resolved successor, and promote the successor. + Verify -INFINITY during transitions, 100 for a successful eligible demotion, + 1000 after promotion/invariant reconciliation, and persistent container + identity throughout demotion. Return to DV6 primary / ELITEDESK standby. +7. Test two eligible standbys using the existing cascading-standby intermediate + state. Both should have the same positive score; a copied-CIB simulation + should select one with promoted-max=1. Do not promote both manually. +8. Reproduce the abrupt-primary-loss case in the isolated lab only after a + recoverable baseline exists. Allow the surviving receiver to disappear, + run its monitor, and inspect the score. This candidate is EXPECTED to deny + promotion then. That is a rollout blocker unless the strict policy is + explicitly chosen; it cannot be counted as a passed failover test. +9. Remove only the new resource's transient score in the lab, then reprobe: + fresh eligibility must recreate it. Exercise score-service/query/update + failure and recovery without confusing role codes. Repeated successful + monitoring must replace any stale positive score after ineligibility. +10. With a saved CIB, simulate removal of only the temporary node preference. + Retain target-role=Promoted, promoted-max=1, and the VIP constraints. + Earlier absent/Started scheduler cases are compatibility tests, not + production configuration recommendations. +11. Restore the baseline topology, streaming, test-induced settings and intended + agent version. Confirm maintenance mode still true. Record both nodes' + installed hashes, OCF codes, scores, receiver status and action durations. + +## Production cleanup and rollout: gated on lab validation + +The conceptual description does not include the actual preference constraint +ID, so inventing a deletion command would be unsafe. Read-only discovery is: + +```bash +sudo pcs constraint config --full +sudo pcs resource config postgresql-ha-clone +sudo cibadmin --query > production-before-scoring.xml +``` + +Do not commit that CIB snapshot: it may contain sensitive deployment settings. +After lab validation and policy approval, scripts/plan-promotion-cleanup.py can +read the supplied snapshot and PRINT the exact commands using the real IDs: + +```bash +python3 scripts/plan-promotion-cleanup.py production-before-scoring.xml \ + --clone postgresql-ha-clone --preferred-node david-eve --lab-validated +``` + +The planner never executes commands. It matches only the simple rule described +in the task (Promoted, score=200, #uname eq david-eve), refuses zero/multiple +matches, preserves other rules, and emits only the preference constraint/rule +removal. It requires and preserves direct clone-level target-role=Promoted. +Rule-based/inherited settings require manual inspection. Do not confuse +preference removal with clearing failed actions: former-primary failure state +must remain until rejoin succeeds and streaming standby status is verified. + +There is no RA code that deletes constraints or changes clone metadata. No +production installation, cleanup, or restart was executed during that original +source-review session. Subsequent production validation is in ../docs/testing.md. Exact production +rollout commands are intentionally deferred until the policy, real constraint +ID, candidate hashes and successful lab results are known, as requested. + +## Sources used for API semantics + +- https://clusterlabs.org/projects/pacemaker/man/crm_attribute.8.html +- https://www.postgresql.org/docs/18/monitoring-stats.html#MONITORING-PG-STAT-WAL-RECEIVER-VIEW +- https://www.postgresql.org/docs/18/functions-admin.html +- Installed pcs help: `pcs constraint remove --help`, `pcs resource meta --help`. + +The source diff is provided in PROMOTION-SCORING.patch. Source and test file +hashes are recorded in REFACTOR-SHA256SUMS, which is refreshed for this candidate. +The frozen reference hash remains unchanged. + +RPM validation of this candidate also passed in the disposable Fedora 43 build +container: binary/source packages built, %check ran the regression, scoring, +scheduler and cleanup tests, DNF resolved the new Pacemaker dependency, and the +installed-RPM integrity/permissions/metadata checks passed. Local artifacts are +under dist/scoring-review/ and are not releases or deployment approval. + +A final read-only inspection found that ELITEDESK's installed executable and +all nine modules match the earlier modular baseline at c8636ed exactly. Its +entrypoint SHA256 is c91f09adfdeb45a3be1a72df3f5ca5b02fd4555dec4397ad944eab439ace0b89. +The promotion-scoring candidate is not installed there. diff --git a/notes/REFACTOR-REPORT.md b/notes/REFACTOR-REPORT.md index 97df765..b628240 100644 --- a/notes/REFACTOR-REPORT.md +++ b/notes/REFACTOR-REPORT.md @@ -1,5 +1,10 @@ # PostgreSQL HA resource-agent modular refactor +Historical baseline report (commit c8636ed and earlier). The later promotion +scoring candidate changes dispatcher behavior and adds helpers; see +[DISCONNECTED-STANDBY.md](DISCONNECTED-STANDBY.md). Do not use the installation +commands below to deploy that unreviewed candidate. + The refactored agent is in `refactored/managed-container`, with nine sourced modules beside it. Both reference copies remain unchanged at SHA256 `5eae1f3dba0586dc06105e4dd7a9b8809a5c9961452abe7654dcfce77c3c5ce2`. ## Final file tree diff --git a/notes/REFACTOR-SHA256SUMS b/notes/REFACTOR-SHA256SUMS index b7b2942..9d34045 100644 --- a/notes/REFACTOR-SHA256SUMS +++ b/notes/REFACTOR-SHA256SUMS @@ -1,13 +1,39 @@ +e2df02e66e120481eb62894b2356f40030c63d5bcf734de2cb7ee64bda8b3376 .gitea/workflows/build-rpm.yml +bb9026b0c528721ab80038344929a8bcd0ab8c5c8e438a22978bc362572d6112 .gitignore +4351008e41de847f0dd8323843a28ea3b0fbe100d0268faa77183cdb4f7c7da0 HANDOFF.md +6e3cf9e32b85808ff503ac4d07f1dff69e002f0dbcb1630ad3c36ca32e530afb README.md +13b689315ab43e830473aa3c97e37390a5417b4681fab2774c6072b52b3266b4 docs/operations.md +3e5f075baa7863bee3d12dd50319466291e30e7be895dfe16e9724da9ae4f021 docs/review.md +6e0ce0054b478dcae608b6f35c6614bbfae8ef8bb978144cfc6d560de6d57d0f docs/testing.md +bfdeecfb1791c4f1d8ec08f28774f94769fffeb51b3268342fd6c63e2322eb23 notes/DISCONNECTED-STANDBY.md +0a6af2a5fbf1ea1d9551893578df094442ee10e032e1b2de7c92f9662b133ad3 notes/PROMOTION-SCORING.md 6bdc385dff0e66de372f802a577f67d852f7a2a0e45610865f558ee3b93adf20 notes/REFACTOR-PLAN.md -08135e93b355d2f8239d9abbe0297eafdd6109d025a27d13ab35ccd74811e03a notes/REFACTOR-REPORT.md -c91f09adfdeb45a3be1a72df3f5ca5b02fd4555dec4397ad944eab439ace0b89 refactored/managed-container +05b860269ba536361f7b806f415ffb6c2f6632ecf8374bfcd02e7bb7ad1069bb notes/REFACTOR-REPORT.md +2d728176c301e24f673b8d36a9ae0fd3bd3816d37b1f945100b37a74779887ef packaging/Containerfile +676455c64d130eb1ad36e007f023f0f30c98beedf87f18be74ae33202b646ffa packaging/PROVENANCE.md +ba3fe4e1ee60ec9445c4c2cc6591e90b674c7113474ec8b33edb4f1016dc0ae2 packaging/README.md +e9dd8507f4bf0c6f42458e41aea833ad0bd3f6127272335eee9bf4d58541ed67 packaging/VERSION +4b9e13339a1eacbced38c69edf2a8c518997160104aab1bbfe6d2f019a93d6fa packaging/ocf-managed-container.spec +a4b775a0ddcabdb0558da9b34c7fe2a883c07d0ef308c1c7224526abf53f63a0 refactored/managed-container 93191771d3cb8aefadb2a16d2f89d64a8b6c584071a473439be818237bdf7f17 refactored/managed-container.d/auth.sh -e0bb13e9fda84ba7b657b2e774c08fa64988d463f45c699681e0e6ce49109440 refactored/managed-container.d/common.sh +97e93d9164e854e5f3688cf9d3cc656b001eb30ef3c2073c907450ca43588f3d refactored/managed-container.d/common.sh ac6bc363a2301f1491d162ef609103dbfbceb3b099a3556614c6165952309374 refactored/managed-container.d/container.sh 0af3bb362a303017372c631354ba94ef9e723bf14e64d2eff91b6662e54a5be0 refactored/managed-container.d/demote.sh 6a98fcdda82cea9365db95b4ee779be5c567b3d846f48b90fbc8c254af697670 refactored/managed-container.d/notify.sh -310132d8032891727076295f39548b1cc0e51a10301a0ee4d934161f4f903e3e refactored/managed-container.d/postgres.sh +d2779ce6e7d1f426ffca2e41bbb8de77503005dc38d658d69f184020365796eb refactored/managed-container.d/postgres.sh 3085ec9b4410905ab9d4370786ed776eac3470a0415aa1b4f608b083945aeb58 refactored/managed-container.d/promote.sh +5166fc3370f49ab14cd975ded4b03a74b9bd7dbff5762b41a15552c42f934ea7 refactored/managed-container.d/promotion_state.py 21beea72bcb7f24a51c17ea48666ac87df9436702ad77b166b45f4647e28e8ad refactored/managed-container.d/rejoin.sh 6b2021887e3b4cd6a12490e7a8dff98320b2133b472ab25a22a0eead46137547 refactored/managed-container.d/replication.sh -e3bc103b61b317e91d28260cc7bb86a3cc8390854762d6c5b9f325a1a8b1c362 tests/verify_refactor.py +3f1a15f7054fb69b9aa92204de30306fd6053c63560310f4231f314b148e2dcf runtime-context/postgres-ha-bootstrap +17ce35741b0ef55a6e4785a093a5097f13139edd428fd4c65e8f50d222161b65 scripts/build-rpm.sh +42744b1b407ab4a79725bd90d7d067f46dd8b36728f104c52544d464b2e323c3 scripts/install-elitedesk-ra.sh +470fd174e42efef876e540dfbfa42b4f36bf70b29a26ab3e5f0c10c813f3b6ae scripts/plan-promotion-cleanup.py +52162183beb2eb258b2336c4fd6392319e2ac7415372b623226bd1dd4ff0720d tests/test_disconnected_state.py +e44ac0e6fc322d7e314814c69d9c3b1ab582b43afbf8b66bd30a07713682ae1c tests/test_promotion_cleanup.py +eb88cdaffeccb00c92e1c43725112ae16aa15faf923b84db62ae5950c5c26d2e tests/test_promotion_scheduler.py +29bac1cc5939aaf772259bb08f3d2cfae2186e081fcb203c4979a32d0a60e0c0 tests/test_promotion_scores.py +40211e06e4b23f0e1842759cfeb4508a7dbd2ec985da9bdef090efa017f4e02d tests/verify_installed_rpm.py +d2280256e451afca90d035e85998ed9ebab8ec21aa7c0aa4605fed7619ba24b9 tests/verify_promotion_sql.py +c7744c533055dac5744fcb54a56ff1dd3f95940185ddc860a69dc2dd88669819 tests/verify_promotion_state_cli.py +dc7d1c8b47c4f0d4edbfacd38714405d6da8d3ae6b183f363afd264e4a0229e5 tests/verify_refactor.py diff --git a/packaging/Containerfile b/packaging/Containerfile index eea3a1a..3874011 100644 --- a/packaging/Containerfile +++ b/packaging/Containerfile @@ -1,3 +1,3 @@ FROM registry.fedoraproject.org/fedora:43 -RUN dnf -y install rpm-build python3 resource-agents git tar gzip diffutils findutils \ +RUN dnf -y install rpm-build python3 resource-agents pacemaker-cli git tar gzip diffutils findutils \ coreutils bash nodejs && dnf clean all diff --git a/packaging/README.md b/packaging/README.md index d36b290..886f276 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -5,7 +5,7 @@ Fedora 43, matching the saved lab package context. The workstation currently reports Fedora 44; this first workflow does not claim a Fedora 44 build test. It runs on main pushes, pull requests and manual dispatch. -The build installs only the OCF executable and nine modules, plus documentation. +The build installs only the OCF executable and nine shell modules, one Python state helper, plus documentation. It includes no Quadlet, secrets, databases or cluster configuration. There are no RPM lifecycle scriptlets. Installation does not start, stop or restart a service or change maintenance mode. Host PostgreSQL packages are not required. @@ -72,3 +72,16 @@ on Fedora 43). No package was installed on either lab host. The Gitea server reports 1.27.3. Its Actions API returned HTTP 401 without authentication, so a successful local build does not establish runner checkout or artifact upload success. Check the first run in the repository's Actions tab. + +The promotion-scoring review candidate additionally requires pacemaker-cli +2.1.0 or later. Its %check includes scoring and synthetic-CIB scheduler tests. +See notes/PROMOTION-SCORING.md in the source checkout for the unresolved +disconnected-standby policy; do not deploy the candidate from CI artifacts. + +The disconnected candidate adds a runtime Python 3 dependency and advertises +`disconnected_max_age` (default 300 seconds). See +[the current policy](../notes/DISCONNECTED-STANDBY.md) before any installation. + +Current operating policy and operator-reported production validation are packaged +from `docs/`. The source freeze is distinct from verified deployed-byte identity; +see [testing](../docs/testing.md) for production snapshot status. diff --git a/packaging/ocf-managed-container.spec b/packaging/ocf-managed-container.spec index 5f5b0d2..e97247c 100644 --- a/packaging/ocf-managed-container.spec +++ b/packaging/ocf-managed-container.spec @@ -20,6 +20,8 @@ BuildRequires: resource-agents BuildRequires: hostname BuildRequires: diffutils BuildRequires: coreutils +BuildRequires: pacemaker-cli >= 2.1.0 +Requires: python3 Requires: bash Requires: /usr/lib/ocf/lib/heartbeat/ocf-shellfuncs Requires: podman @@ -31,6 +33,7 @@ Requires: coreutils Requires: gawk Requires: sed Requires: hostname +Requires: pacemaker-cli >= 2.1.0 %description OCF agent managing PostgreSQL through a persistent rootless systemd/Quadlet @@ -48,22 +51,28 @@ cp packaging/README.md RPM-BUILD.md %install install -d %{buildroot}%{ocf_provider_dir}/managed-container.d install -m 0755 refactored/managed-container %{buildroot}%{ocf_provider_dir}/managed-container -install -m 0644 refactored/managed-container.d/*.sh %{buildroot}%{ocf_provider_dir}/managed-container.d/ +install -m 0644 refactored/managed-container.d/*.sh refactored/managed-container.d/*.py %{buildroot}%{ocf_provider_dir}/managed-container.d/ %check python3 tests/verify_refactor.py +python3 tests/test_promotion_scores.py +python3 tests/test_promotion_scheduler.py +python3 tests/test_promotion_cleanup.py +python3 tests/test_disconnected_state.py # Check the actual staged layout, including resolution from an unrelated cwd. agent="%{buildroot}%{ocf_provider_dir}/managed-container" "$agent" meta-data > staged-metadata.xml -reference/managed-container meta-data > reference-metadata.xml +refactored/managed-container meta-data > reference-metadata.xml cmp reference-metadata.xml staged-metadata.xml (cd /tmp && "$agent" meta-data) > other-cwd-metadata.xml cmp reference-metadata.xml other-cwd-metadata.xml %files %defattr(-,root,root,-) -%doc README.md packaging/PROVENANCE.md RPM-BUILD.md +%doc README.md packaging/PROVENANCE.md RPM-BUILD.md notes/PROMOTION-SCORING.md notes/DISCONNECTED-STANDBY.md +%doc docs/operations.md docs/testing.md docs/review.md %dir %{ocf_provider_dir} %attr(0755,root,root) %{ocf_provider_dir}/managed-container %dir %{ocf_provider_dir}/managed-container.d %attr(0644,root,root) %{ocf_provider_dir}/managed-container.d/*.sh +%attr(0644,root,root) %{ocf_provider_dir}/managed-container.d/*.py diff --git a/refactored/managed-container b/refactored/managed-container index 9a99297..42435a9 100755 --- a/refactored/managed-container +++ b/refactored/managed-container @@ -13,6 +13,9 @@ : "${OCF_ROOT:=/usr/lib/ocf}" OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat" +# Preserve explicit resource identity before ocf-shellfuncs supplies its +# standalone-invocation default; never publish a score for that dummy resource. +MC_PROMOTION_RESOURCE_INSTANCE="${OCF_RESOURCE_INSTANCE:-}" . "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs" # Pacemaker uses "Promoted" terminology, while many OCF shell libraries @@ -33,40 +36,37 @@ unset MC_MODULE case "$1" in meta-data) - meta_data + mc_promotion_metadata exit "${OCF_SUCCESS}" ;; validate-all) - validate_all + mc_promotion_validate exit $? ;; monitor|status) - managed_monitor + mc_promotion_monitor exit $? ;; start) - validate_all || exit $? - managed_start + mc_promotion_action start exit $? ;; stop) - managed_stop + mc_promotion_action stop exit $? ;; promote) - validate_all || exit $? - managed_promote + mc_promotion_action promote exit $? ;; demote) - validate_all || exit $? - managed_demote + mc_promotion_action demote exit $? ;; @@ -76,8 +76,7 @@ case "$1" in ;; rejoin) - validate_all || exit $? - managed_rejoin + mc_promotion_action rejoin exit $? ;; diff --git a/refactored/managed-container.d/common.sh b/refactored/managed-container.d/common.sh index 06f45d4..70f2a08 100644 --- a/refactored/managed-container.d/common.sh +++ b/refactored/managed-container.d/common.sh @@ -225,3 +225,126 @@ validate_all() return "${OCF_SUCCESS}" } + +# Promotion scores are scoped by Pacemaker to OCF_RESOURCE_INSTANCE and the +# executing node. Do not construct a legacy attribute name or cache locally: +# querying attrd on every observation also repairs a lost transient attribute. +mc_promotion_score_set() +{ + local score="$1" current + local resource="${MC_PROMOTION_RESOURCE_INSTANCE:-}" + + if [ -z "${resource}" ]; then + ocf_log warn "Cannot publish promotion score without OCF_RESOURCE_INSTANCE" + return 1 + fi + + case "${score}" in + 1000|100|50|-INFINITY) ;; + *) return 1 ;; + esac + + current="$( + timeout --kill-after=1s 2s crm_attribute \ + --promotion="${resource}" --lifetime=reboot \ + --query --quiet 2>/dev/null + )" || current="" + + [ "${current}" = "${score}" ] && return 0 + + if ! timeout --kill-after=1s 2s crm_attribute \ + --promotion="${resource}" --lifetime=reboot \ + --update="${score}" >/dev/null 2>&1 + then + ocf_log err "Unable to publish promotion score ${score} for ${resource}; prior score may remain" + return 1 + fi + return 0 +} + +mc_promotion_score_from_monitor() +{ + local monitor_rc="$1" score=-INFINITY + + case "${monitor_rc}" in + "${OCF_RUNNING_PROMOTED}") score=1000 ;; + "${OCF_SUCCESS}") + score="$(mc_pg_standby_score)" || score=-INFINITY + ;; + esac + case "${score}" in + 1000|100|50|-INFINITY) ;; + *) score=-INFINITY ;; + esac + mc_promotion_score_set "${score}" +} + +# Score publication must not change OCF role detection or its return code. +mc_promotion_monitor() +{ + local monitor_rc + + managed_monitor + monitor_rc=$? + mc_promotion_score_from_monitor "${monitor_rc}" || true + return "${monitor_rc}" +} + +# Lifecycle functions retain their original checks, commands and return codes. +# Withdraw the previous score before any transition (including validation +# failure), then reassess only after success. Their internal monitor calls do +# not publish scores, so they cannot restore a positive score mid-transition. +mc_promotion_action() +{ + local action="$1" action_rc monitor_rc + + case "${action}" in + start|stop|promote|demote|rejoin) ;; + *) return "${OCF_ERR_UNIMPLEMENTED}" ;; + esac + + mc_promotion_score_set -INFINITY || true + if [ "${action}" != stop ]; then + mc_promotion_validate + action_rc=$? + [ "${action_rc}" -eq "${OCF_SUCCESS}" ] || return "${action_rc}" + fi + + "managed_${action}" + action_rc=$? + if [ "${action_rc}" -eq "${OCF_SUCCESS}" ] && [ "${action}" != stop ]; then + managed_monitor + monitor_rc=$? + mc_promotion_score_from_monitor "${monitor_rc}" || true + fi + return "${action_rc}" +} + +mc_promotion_validate() +{ + local utility + + if ! [[ "${OCF_RESKEY_disconnected_max_age:-300}" =~ ^(0|[1-9][0-9]{0,4})$ ]]; then + ocf_exit_reason "disconnected_max_age must be an integer from 0 to 99999 seconds" + return "${OCF_ERR_CONFIGURED}" + fi + + for utility in crm_attribute timeout python3; do + if ! command -v "${utility}" >/dev/null 2>&1; then + ocf_exit_reason "Required promotion scoring command '${utility}' is not installed" + return "${OCF_ERR_INSTALLED}" + fi + done + validate_all +} + +# Keep the original metadata function intact; advertise the added policy here. +mc_promotion_metadata() +{ + meta_data | sed '/<\/parameters>/i\ + \ + Maximum age in seconds of a verified streaming observation for disconnected promotion eligibility. Default 300; 0 disables disconnected eligibility. Requires fencing managed by Pacemaker. PostgreSQL restart requires fresh streaming verification.\ + Disconnected standby observation lifetime\ + \ + ' +} diff --git a/refactored/managed-container.d/postgres.sh b/refactored/managed-container.d/postgres.sh index a77c805..9b3dc2c 100644 --- a/refactored/managed-container.d/postgres.sh +++ b/refactored/managed-container.d/postgres.sh @@ -290,3 +290,76 @@ managed_stop() return "${OCF_SUCCESS}" } + +# Conservative scoring eligibility only; this does not replace role detection, +# fencing, the promotion action, or the existing controlled-demote WAL checks. +# This strict predicate remains unchanged; disconnected policy is separate below. +mc_pg_promotion_eligible() +{ + local eligible + + eligible="$( + as_service_user timeout --kill-after=1s 3s \ + podman exec --user "${PG_OS_USER}" \ + --env 'PGOPTIONS=-c statement_timeout=2000' \ + "${CONTAINER_NAME}" \ + psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc " + SELECT CASE WHEN pg_is_in_recovery() THEN + NOT pg_is_wal_replay_paused() + AND EXISTS ( + SELECT 1 FROM pg_stat_wal_receiver + WHERE status = 'streaming' + AND flushed_lsn IS NOT NULL + AND pg_last_wal_replay_lsn() >= flushed_lsn + ) + ELSE false END; + " 2>/dev/null + )" || return 1 + [ "${eligible}" = t ] +} + +# One observation supplies both strict connected eligibility and disconnected +# evidence. PostgreSQL restart time deliberately invalidates prior observations, +# including a new basebackup sharing the same PostgreSQL system identifier. +mc_pg_standby_observation() +{ + as_service_user timeout --kill-after=1s 3s \ + podman exec --user "${PG_OS_USER}" \ + --env 'PGOPTIONS=-c statement_timeout=2000' \ + "${CONTAINER_NAME}" \ + psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc " + WITH s AS MATERIALIZED ( + SELECT pg_is_in_recovery() AS recovery, + pg_last_wal_replay_lsn() AS replay, + r.status AS receiver, r.flushed_lsn AS flush + FROM (SELECT 1) AS singleton + LEFT JOIN pg_stat_wal_receiver r ON true + ) + SELECT json_build_object( + 'identity', json_build_array( + (SELECT system_identifier::text FROM pg_control_system()), + current_setting('data_directory'), + extract(epoch FROM pg_postmaster_start_time())::text), + 'recovery', recovery, + 'paused', CASE WHEN recovery THEN pg_is_wal_replay_paused() ELSE true END, + 'receiver', receiver, 'replay', replay::text, 'flush', flush::text, + 'strict', CASE WHEN recovery THEN + NOT pg_is_wal_replay_paused() AND receiver = 'streaming' + AND flush IS NOT NULL AND replay >= flush + ELSE false END) + FROM s; + " 2>/dev/null +} + +mc_pg_standby_score() +{ + local observation + observation="$(mc_pg_standby_observation)" || return 1 + # The helper is root-owned alongside the modules; state never enters the + # rootless container. A failed/slow helper always withdraws eligibility. + printf '%s' "${observation}" | timeout --kill-after=1s 1s python3 \ + "${MC_MODULE_DIR}/promotion_state.py" \ + "${MC_PROMOTION_RESOURCE_INSTANCE:-}" \ + "${SERVICE_USER}" "${SERVICE_NAME}" "${CONTAINER_NAME}" \ + "${OCF_RESKEY_disconnected_max_age:-300}" +} diff --git a/refactored/managed-container.d/promotion_state.py b/refactored/managed-container.d/promotion_state.py new file mode 100644 index 0000000..f9c8113 --- /dev/null +++ b/refactored/managed-container.d/promotion_state.py @@ -0,0 +1,173 @@ +"""Local promotion evidence. Invoked by the root RA; never invokes cluster tools. + +State is a bounded JSON record, never executable input. The public CLI uses a +fixed root-owned /run directory and CLOCK_BOOTTIME, including suspend duration. +""" +import fcntl +import hashlib +import json +import math +import os +from pathlib import Path +import re +import stat +import sys +import tempfile +import time + +STATE_DIR = Path('/run/managed-container/promotion') + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate JSON key') + result[key] = value + return result + + +def lsn(value): + if not isinstance(value, str) or not re.fullmatch(r'[0-9A-F]{1,8}/[0-9A-F]{1,8}', value): + raise ValueError('invalid LSN') + high, low = value.split('/') + return (int(high, 16) << 32) | int(low, 16) + + +def number(value): + return type(value) in (int, float) and math.isfinite(value) and value >= 0 + + +def validate_observation(obs): + if set(obs) != {'identity', 'recovery', 'paused', 'receiver', 'replay', 'flush', 'strict'}: + raise ValueError('invalid observation fields') + identity = obs['identity'] + if not isinstance(identity, list) or len(identity) != 3: + raise ValueError('invalid identity') + system, path, started = identity + if (not isinstance(system, str) or not re.fullmatch(r'[0-9]{1,20}', system) + or not isinstance(path, str) or not path.startswith('/') or len(path) > 2048 + or not isinstance(started, str) or not re.fullmatch(r'[0-9]{1,12}\.[0-9]{1,6}', started)): + raise ValueError('invalid identity values') + if type(obs['recovery']) is not bool or type(obs['paused']) is not bool: + raise ValueError('invalid role') + if obs['strict'] is not None and type(obs['strict']) is not bool: + raise ValueError('invalid strict flag') + if obs['receiver'] is not None and obs['receiver'] not in ( + 'starting', 'streaming', 'waiting', 'restarting', 'stopping'): + raise ValueError('invalid receiver status') + replay = lsn(obs['replay']) if obs['replay'] is not None else None + flush = lsn(obs['flush']) if obs['flush'] is not None else None + strict = (obs['recovery'] and not obs['paused'] and obs['receiver'] == 'streaming' + and replay is not None and flush is not None and replay >= flush) + if (obs['strict'] is True) != strict: + raise ValueError('inconsistent strict observation') + return strict + + +class Evidence: + def __init__(self, directory, context, boot, now, wall, owner=0): + self.directory, self.context = Path(directory), context + self.boot, self.now, self.wall, self.owner = boot, now, wall, owner + key = hashlib.sha256(json.dumps(context, separators=(',', ':')).encode()).hexdigest() + self.path = self.directory / (key + '.json') + self.lock = self.directory / (key + '.lock') + + def secure(self, path, mode): + info = path.lstat() + if (info.st_uid != self.owner or stat.S_IMODE(info.st_mode) != mode + or (not stat.S_ISREG(info.st_mode) if mode == 0o600 else not stat.S_ISDIR(info.st_mode))): + raise ValueError('unsafe state ownership/type/mode') + + def discard(self): + self.path.unlink(missing_ok=True) + + def evaluate(self, obs, max_age): + if type(max_age) is not int or not 0 <= max_age <= 99999: + raise ValueError('invalid freshness threshold') + self.directory.mkdir(mode=0o700, exist_ok=True) + self.secure(self.directory, 0o700) + fd = os.open(self.lock, os.O_CREAT | os.O_NOFOLLOW | os.O_RDWR, 0o600) + with os.fdopen(fd, 'r+') as lock: + self.secure(self.lock, 0o600) + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + return self._evaluate(obs, max_age) + except (ValueError, TypeError, KeyError, OverflowError): + self.discard() + return '-INFINITY' + + def _evaluate(self, obs, max_age): + strict = validate_observation(obs) + if strict: + record = dict(version=1, context=self.context, boot=self.boot, + identity=obs['identity'], observed_boottime=self.now, + observed_unix=self.wall, replay=obs['replay'], flush=obs['flush']) + # Same-directory atomic replace; failed persistence cannot authorize + # subsequent disconnected promotion. No secrets are recorded. + fd, temporary = tempfile.mkstemp(prefix='.observation-', dir=self.directory) + try: + with os.fdopen(fd, 'w') as out: + json.dump(record, out, separators=(',', ':')) + out.flush() + os.fsync(out.fileno()) + os.replace(temporary, self.path) + finally: + Path(temporary).unlink(missing_ok=True) + return '100' + if (not obs['recovery'] or obs['paused'] or obs['receiver'] == 'streaming' + or obs['replay'] is None or max_age == 0): + self.discard() + return '-INFINITY' + if not self.path.exists(): + return '-INFINITY' + self.secure(self.path, 0o600) + if self.path.stat().st_size > 4096: + raise ValueError('oversized state') + with self.path.open() as source: + record = json.load(source, object_pairs_hook=unique_object) + if set(record) != {'version', 'context', 'boot', 'identity', 'observed_boottime', + 'observed_unix', 'replay', 'flush'}: + raise ValueError('invalid record fields') + if (type(record['version']) is not int or record['version'] != 1 + or record['context'] != self.context or record['boot'] != self.boot + or record['identity'] != obs['identity'] + or not number(record['observed_boottime']) or not number(record['observed_unix']) + or not 0 <= self.now - record['observed_boottime'] <= max_age + or lsn(record['replay']) < lsn(record['flush']) + or lsn(obs['replay']) < lsn(record['replay'])): + raise ValueError('expired or incompatible observation') + return '50' + + +def main(): + if os.geteuid() != 0 or len(sys.argv) != 6 or not all(sys.argv[1:5]): + raise ValueError('root and explicit resource context required') + age = sys.argv[5] + if not re.fullmatch(r'(0|[1-9][0-9]{0,4})', age): + raise ValueError('invalid freshness threshold') + # /run is trusted root-owned; validate our parent before using a child. + parent = STATE_DIR.parent + parent.mkdir(mode=0o700, exist_ok=True) + info = parent.lstat() + if info.st_uid != 0 or not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700: + raise ValueError('unsafe state parent') + raw = sys.stdin.read(8193) + if len(raw) > 8192: + raise ValueError('oversized observation') + evidence = Evidence(STATE_DIR, sys.argv[1:5], + Path('/proc/sys/kernel/random/boot_id').read_text().strip(), + time.clock_gettime(time.CLOCK_BOOTTIME), time.time()) + try: + obs = json.loads(raw, object_pairs_hook=unique_object) + except ValueError: + obs = {} # Invalid query output withdraws and removes prior evidence. + print(evidence.evaluate(obs, int(age))) + + +if __name__ == '__main__': + try: + main() + except (OSError, ValueError, TypeError, KeyError, OverflowError): + print('-INFINITY') + sys.exit(1) diff --git a/scripts/build-rpm.sh b/scripts/build-rpm.sh index 588e004..2d273d3 100755 --- a/scripts/build-rpm.sh +++ b/scripts/build-rpm.sh @@ -22,7 +22,7 @@ tar --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --numeric-owne --exclude='__pycache__' --exclude='*.pyc' \ --transform="s,^,ocf-managed-container-$package_version/," \ -czf "$build_dir/SOURCES/ocf-managed-container-$package_version.tar.gz" \ - README.md HANDOFF.md refactored reference tests packaging scripts + README.md HANDOFF.md docs refactored reference tests packaging scripts notes/PROMOTION-SCORING.md notes/DISCONNECTED-STANDBY.md # Freeze identity into the SRPM spec so rpmbuild --rebuild needs no Git metadata # or external macro definitions and retains the same version and release. sed -e "s/^Version:.*/Version: $package_version/" \ diff --git a/scripts/install-elitedesk-ra.sh b/scripts/install-elitedesk-ra.sh new file mode 100755 index 0000000..3faa483 --- /dev/null +++ b/scripts/install-elitedesk-ra.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# HISTORICAL LAB TOOL: not a production/scoring installer; see docs/operations.md. +# One-time installation of the modular baseline on the confirmed lab standby. +# Run with sudo in your terminal. No service or PostgreSQL lifecycle actions. +set -euo pipefail +export PATH=/usr/sbin:/usr/bin:/sbin:/bin + +project_dir="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +provider_dir=/usr/lib/ocf/resource.d/loxthelion +agent="$provider_dir/managed-container" +expected_hash=5eae1f3dba0586dc06105e4dd7a9b8809a5c9961452abe7654dcfce77c3c5ce2 + +fail() { printf 'STOP: %s\n' "$*" >&2; exit 1; } +[[ "$EUID" == 0 ]] || fail 'Run this installer with sudo in your terminal.' +[[ "$(hostname -s)" == DAVID-ELITEDESK ]] || fail 'This installer is only for DAVID-ELITEDESK.' +[[ "$(crm_attribute --type crm_config --name cluster-name --query --quiet)" == postgres-ra-test ]] || fail 'Unexpected cluster name.' +[[ "$(crm_attribute --type crm_config --name maintenance-mode --query --quiet)" == true ]] || fail 'Pacemaker maintenance-mode must already be true.' +[[ "$(sha256sum "$agent" | cut -d ' ' -f1)" == "$expected_hash" ]] || fail 'Installed agent differs from the confirmed reference.' +[[ ! -e "$provider_dir/managed-container.d" ]] || fail 'A module directory already exists; review it before installing.' + +cd "$project_dir" +[[ "$(sha256sum refactored/managed-container | cut -d ' ' -f1)" == "c91f09adfdeb45a3be1a72df3f5ca5b02fd4555dec4397ad944eab439ace0b89" ]] || fail 'This installer is for the original modular baseline only; the scoring enhancement requires separate review.' +sha256sum -c notes/REFACTOR-SHA256SUMS +for file in refactored/managed-container refactored/managed-container.d/*.sh; do + bash -n "$file" +done + +# Match the recorded pg-ha resource parameters and avoid inherited test overrides. +run_ra() { + env -i PATH="$PATH" OCF_ROOT=/usr/lib/ocf OCF_RESOURCE_INSTANCE=pg-ha \ + OCF_RESKEY_service_user=david OCF_RESKEY_service_name=postgres.service \ + OCF_RESKEY_container_name=postgres OCF_RESKEY_pg_os_user=postgres \ + OCF_RESKEY_db_user=david OCF_RESKEY_db_name=david \ + "$1" "$2" +} +run_ra "$agent" validate-all +run_ra "$agent" monitor || fail 'The original agent must report a healthy unpromoted standby (rc=0).' + +mkdir -p /var/backups +backup_dir="$(mktemp -d /var/backups/managed-container-ELITEDESK.XXXXXXXX)" +cp -a "$agent" "$backup_dir/managed-container" +printf 'Original agent backed up to: %s\n' "$backup_dir" +stage_dir="$(mktemp -d "$provider_dir/.managed-container-install.XXXXXXXX")" +install -d -o root -g root -m 0755 "$stage_dir/managed-container.d" +install -o root -g root -m 0755 refactored/managed-container "$stage_dir/managed-container" +install -o root -g root -m 0644 refactored/managed-container.d/*.sh "$stage_dir/managed-container.d/" +run_ra "$agent" meta-data > "$backup_dir/original-metadata.xml" +run_ra "$stage_dir/managed-container" meta-data > "$backup_dir/staged-metadata.xml" +cmp "$backup_dir/original-metadata.xml" "$backup_dir/staged-metadata.xml" +run_ra "$stage_dir/managed-container" validate-all +run_ra "$stage_dir/managed-container" monitor || fail 'Staged agent must report a healthy standby.' + +published=false +rollback_on_error() { + local rc=$? + trap - EXIT + if [[ "$rc" != 0 && "$published" == true ]]; then + printf 'Installation/check failed; restoring the original entrypoint.\n' >&2 + if install -o root -g root -m 0755 "$backup_dir/managed-container" "$stage_dir/rollback-agent" && + mv -f "$stage_dir/rollback-agent" "$agent" && restorecon "$agent"; then + printf 'Original restored. New modules remain unused beside it for inspection.\n' >&2 + else + printf 'Automatic restoration FAILED. Original backup: %s/managed-container\n' "$backup_dir" >&2 + fi + fi + exit "$rc" +} +trap rollback_on_error EXIT +# Recheck immediately before changing the installed files. +[[ "$(crm_attribute --type crm_config --name maintenance-mode --query --quiet)" == true ]] || fail 'Maintenance mode changed; refusing installation.' +published=true +mv "$stage_dir/managed-container.d" "$provider_dir/managed-container.d" +mv -f "$stage_dir/managed-container" "$agent" +restorecon -RF "$agent" "$provider_dir/managed-container.d" + +run_ra "$agent" meta-data > "$backup_dir/installed-metadata.xml" +cmp "$backup_dir/original-metadata.xml" "$backup_dir/installed-metadata.xml" +run_ra "$agent" validate-all +run_ra "$agent" monitor || fail 'Installed agent did not report a healthy standby.' +[[ "$(crm_attribute --type crm_config --name maintenance-mode --query --quiet)" == true ]] || fail 'Maintenance mode is no longer true.' +for file in refactored/managed-container refactored/managed-container.d/*.sh; do + cmp "$file" "$provider_dir/${file#refactored/}" +done +sha256sum "$agent" "$provider_dir/managed-container.d/"*.sh > "$backup_dir/installed-SHA256SUMS" +cat "$backup_dir/installed-SHA256SUMS" +published=false +trap - EXIT +rmdir "$stage_dir" +printf '\nPASS: refactored RA installed; metadata identical, validate-all=0, monitor=0.\n' +printf 'Maintenance mode remains true. No service restart or HA transition was requested.\n' +printf 'Backup and validation evidence: %s\n' "$backup_dir" diff --git a/scripts/plan-promotion-cleanup.py b/scripts/plan-promotion-cleanup.py new file mode 100644 index 0000000..7d8a4bf --- /dev/null +++ b/scripts/plan-promotion-cleanup.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Print (never execute) exact cleanup commands from a supplied CIB snapshot.""" +import argparse +from pathlib import Path +import shlex +import sys +import xml.etree.ElementTree as ET + + +def plan(path, clone_id, preferred_node): + root = ET.parse(path).getroot() + clones = [c for c in root.findall('./configuration/resources/clone') if c.get('id') == clone_id] + if len(clones) != 1: + raise ValueError('Expected exactly one clone with the supplied ID') + for meta in clones[0].findall('meta_attributes'): + if meta.get('id-ref') or meta.find('rule') is not None: + raise ValueError('Clone metadata uses references/rules; inspect manually') + matches = [] + for location in root.findall('./configuration/constraints/rsc_location'): + if location.get('rsc') != clone_id: + continue + for rule in location.findall('rule'): + children = list(rule) + if (rule.get('role') == 'Promoted' and rule.get('score') == '200' + and len(children) == 1 and children[0].tag == 'expression' + and children[0].get('attribute') == '#uname' + and children[0].get('operation') == 'eq' + and children[0].get('value') == preferred_node): + # Preserve unrelated rules if the location has more than one. + target = location.get('id') if len(list(location)) == 1 else rule.get('id') + if not target: + raise ValueError('Matching constraint/rule lacks an ID') + matches.append(target) + if len(matches) != 1: + raise ValueError('Expected exactly one simple Promoted/200/#uname preference; inspect the full CIB manually') + target_roles = [p.get('value') for m in clones[0].findall('meta_attributes') + for p in m.findall('nvpair') if p.get('name') == 'target-role'] + if target_roles != ['Promoted']: + raise ValueError('Expected one direct clone target-role=Promoted; inspect inheritance/rules manually') + # target-role=Promoted is an intentional production invariant, not a + # temporary node preference. Never remove or rewrite it here. + return [shlex.join(['sudo', 'pcs', 'constraint', 'remove', matches[0]])] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('cib', type=Path) + parser.add_argument('--clone', required=True) + parser.add_argument('--preferred-node', required=True) + parser.add_argument('--lab-validated', action='store_true', help='operator acknowledgement that this candidate passed lab validation') + args = parser.parse_args() + if not args.lab_validated: + parser.error('No rollout commands emitted: lab validation must be completed and acknowledged first') + try: + commands = plan(args.cib, args.clone, args.preferred_node) + except (ValueError, ET.ParseError, OSError) as error: + print('No commands emitted:', error, file=sys.stderr) + return 1 + print('# Review against the current CIB before running. No commands have been executed.') + print('\n'.join(commands)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/test_disconnected_state.py b/tests/test_disconnected_state.py new file mode 100644 index 0000000..a0b5572 --- /dev/null +++ b/tests/test_disconnected_state.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""State and monitor integration tests: temporary files and fake boundaries only.""" +import copy +import importlib.util +import json +import os +from pathlib import Path +import shlex +import tempfile +import unittest + +from test_promotion_scores import Scores, LOAD, FIXTURE + +ROOT = Path(__file__).resolve().parents[1] +HELPER = ROOT / 'refactored/managed-container.d/promotion_state.py' +spec = importlib.util.spec_from_file_location('promotion_state', HELPER) +policy = importlib.util.module_from_spec(spec) +spec.loader.exec_module(policy) +CONNECTED = dict(identity=['7684451577990111248', '/data/pg18', '1789200000.123456'], + recovery=True, paused=False, receiver='streaming', replay='0/30', + flush='0/20', strict=True) +DISCONNECTED = dict(CONNECTED, receiver=None, flush=None, strict=None) + + +class State(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix='mc-evidence-') + self.addCleanup(self.tmp.cleanup) + self.evidence = policy.Evidence(Path(self.tmp.name) / 'promotion', + ['pg-ha:0', 'owner', 'postgres.service', 'postgres'], + 'test-boot', 1000.0, 1789201000.0, os.getuid()) + + def check(self, obs=DISCONNECTED, age=300): + return self.evidence.evaluate(copy.deepcopy(obs), age) + + def seed(self): + self.assertEqual(self.check(CONNECTED), '100') + return json.loads(self.evidence.path.read_text()) + + def test_streaming_disconnected_eligible_and_persisted_lsn(self): + record = self.seed() + self.assertEqual(record['replay'], '0/30') + self.assertEqual(record['flush'], '0/20') + self.evidence.now += 50 + self.assertEqual(self.check(), '50') + self.assertEqual(json.loads(self.evidence.path.read_text()), record) + self.assertEqual(self.evidence.path.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.evidence.directory.stat().st_mode & 0o777, 0o700) + + def test_never_verified(self): + self.assertEqual(self.check(), '-INFINITY') + + def test_stale_and_exact_boundary(self): + self.seed(); self.evidence.now += 300 + self.assertEqual(self.check(), '50') + self.evidence.now += .001 + self.assertEqual(self.check(), '-INFINITY') + self.assertFalse(self.evidence.path.exists()) + + def test_disabled_and_invalid_age(self): + self.seed() + self.assertEqual(self.check(age=0), '-INFINITY') + for age in (-1, True, 100000, '300'): + with self.assertRaises(ValueError): self.check(age=age) + + def test_malformed_state(self): + for value in ('{', '[]', '{}', 'null', '{"version":true}', '{"version":1,"version":1}', 'x' * 4097): + self.seed(); self.evidence.path.write_text(value) + self.assertEqual(self.check(), '-INFINITY', value[:40]) + self.assertFalse(self.evidence.path.exists()) + + def test_invalid_record_fields(self): + for key, value in [('replay', '0/ZZ'), ('flush', '1/0'), ('boot', 'other'), + ('observed_boottime', float('nan')), ('observed_boottime', 1001), + ('observed_unix', True), ('context', ['other']), ('version', True)]: + record = self.seed(); record[key] = value + self.evidence.path.write_text(json.dumps(record)) + self.assertEqual(self.check(), '-INFINITY', key) + + def test_system_data_path_and_postmaster_identity_changes(self): + for index, value in enumerate(['999', '/new/pgdata', '1789200100.123456']): + self.seed(); obs = copy.deepcopy(DISCONNECTED); obs['identity'][index] = value + self.assertEqual(self.check(obs), '-INFINITY') + self.assertFalse(self.evidence.path.exists()) + + def test_reinitialization_requires_reverification_then_replaces_state(self): + self.seed() + disconnected = copy.deepcopy(DISCONNECTED) + disconnected['identity'][2] = '1789200100.123456' # Same system ID/basebackup. + self.assertEqual(self.check(disconnected), '-INFINITY') + connected = dict(CONNECTED, identity=disconnected['identity']) + self.assertEqual(self.check(connected), '100') + self.assertEqual(self.check(disconnected), '50') + self.assertEqual(json.loads(self.evidence.path.read_text())['identity'], connected['identity']) + + def test_paused_primary_and_replay_regression_invalidate(self): + for changes in ({'paused': True}, {'recovery': False, 'paused': True}, + {'replay': '0/10'}, {'replay': None}): + self.seed() + self.assertEqual(self.check(dict(DISCONNECTED, **changes)), '-INFINITY') + self.assertFalse(self.evidence.path.exists()) + + def test_connected_failure_cannot_use_disconnected_fallback(self): + for changes in ({'flush': '0/40'}, {'flush': None}, {'paused': True}): + self.seed() + self.assertEqual(self.check(dict(CONNECTED, **changes, strict=False)), '-INFINITY') + self.assertFalse(self.evidence.path.exists()) + + def test_reconnect_refreshes_timestamp_and_lsns(self): + self.seed(); self.evidence.now += 100 + self.assertEqual(self.check(), '50') + self.assertEqual(self.check(dict(CONNECTED, replay='0/50', flush='0/40')), '100') + record = json.loads(self.evidence.path.read_text()) + self.assertEqual(record['observed_boottime'], 1100) + self.assertEqual(record['replay'], '0/50') + self.assertEqual(self.check(dict(DISCONNECTED, replay='0/50')), '50') + + def test_wall_clock_steps_do_not_extend_or_expire_observation(self): + self.seed(); self.evidence.wall -= 100000; self.evidence.now += 100 + self.assertEqual(self.check(), '50') + self.evidence.wall += 200000; self.evidence.now += 201 + self.assertEqual(self.check(), '-INFINITY') + + def test_symlink_state_rejected_without_reading_target(self): + self.seed(); self.evidence.path.unlink() + target = Path(self.tmp.name) / 'target'; target.write_text('unchanged') + self.evidence.path.symlink_to(target) + self.assertEqual(self.check(), '-INFINITY') + self.assertEqual(target.read_text(), 'unchanged') + + def test_unsafe_directory_rejected(self): + self.seed(); self.evidence.directory.chmod(0o777) + with self.assertRaises(ValueError): self.check() + + +# Reuse fake crm_attribute and the real monitor wrapper. The real state evaluator +# runs as this test user against a private directory; never writes /run or CIB. +class Monitor(unittest.TestCase): + setUp = Scores.setUp + run_shell = Scores.run_shell + state = Scores.state + score = Scores.score + + def test_invalid_observation_query_and_helper_failure_replace_score(self): + fixture = monitor_fixture(self) + self.env['OBSERVATION'] = json.dumps(CONNECTED) + for failure in ('mc_pg_standby_observation() { echo malformed; }', + 'mc_pg_standby_observation() { return 1; }', + 'python3() { return 124; }'): + self.run_shell('mc_promotion_monitor', fixture=fixture) + self.score('100') + self.run_shell(failure + '; mc_promotion_monitor', fixture=fixture) + self.score('-INFINITY') + + +def monitor_fixture(test): + driver = test.path / 'driver.py' + driver.write_text('''import importlib.util,json,os,sys,time +from pathlib import Path +spec=importlib.util.spec_from_file_location('policy',sys.argv[1]) +p=importlib.util.module_from_spec(spec);spec.loader.exec_module(p) +e=p.Evidence(Path(sys.argv[2]),['pg-ha:0','owner','service','container'],'boot',float(os.environ.get('TEST_NOW','1000')),1789201000,os.getuid()) +print(e.evaluate(json.load(sys.stdin),300)) +''') + # Restore the production score helper after the original tests' mock. + return FIXTURE + '\n. ' + shlex.quote(str(HELPER.with_name('postgres.sh'))) + ''' +service_running() { [ "$SERVICE" = 1 ]; } +container_running() { [ "$CONTAINER" = 1 ]; } +pg_running() { [ "$RUNNING" = 1 ]; } +pg_ready() { return 0; } +pg_role() { echo "$ROLE"; } +mc_pg_standby_observation() { printf '%s' "$OBSERVATION"; } +python3() { command python3 ''' + ' '.join(map(shlex.quote, [str(driver), str(HELPER), str(test.path / 'evidence')])) + '''; } +timeout() { shift 2; "$@"; } +''' + + +def test_monitor_disconnect_reprobe_stopped_reconnect(self): + fixture = monitor_fixture(self) + self.env['OBSERVATION'] = json.dumps(CONNECTED) + self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('100') + self.env['OBSERVATION'] = json.dumps(DISCONNECTED) + self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('50') + data = self.state(); data['scores'] = {}; self.db.write_text(json.dumps(data)) + self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('50') + self.run_shell('RUNNING=0; mc_promotion_monitor', 7, fixture=fixture); self.score('-INFINITY') + self.env['OBSERVATION'] = json.dumps(CONNECTED) + self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('100') + self.env['OBSERVATION'] = json.dumps(DISCONNECTED); self.env['TEST_NOW'] = '1301' + self.run_shell('mc_promotion_monitor', fixture=fixture); self.score('-INFINITY') + + +Monitor.test_monitor_disconnect_reprobe_stopped_reconnect = test_monitor_disconnect_reprobe_stopped_reconnect + +if __name__ == '__main__': + # Explicit suite excludes imported Scores from duplicate discovery. + unittest.TextTestRunner(verbosity=2).run(unittest.TestSuite( + unittest.defaultTestLoader.loadTestsFromTestCase(c) for c in (State, Monitor) + )).wasSuccessful() or __import__('sys').exit(1) diff --git a/tests/test_promotion_cleanup.py b/tests/test_promotion_cleanup.py new file mode 100644 index 0000000..cef8c5c --- /dev/null +++ b/tests/test_promotion_cleanup.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +import importlib.util +from pathlib import Path +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('cleanup', ROOT / 'scripts/plan-promotion-cleanup.py') +cleanup = importlib.util.module_from_spec(spec); spec.loader.exec_module(cleanup) + + +class Cleanup(unittest.TestCase): + def plan(self, extra='', role='Promoted'): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / 'cib.xml' + p.write_text(f''' + + + +{extra}''') + return cleanup.plan(p, 'postgresql-ha-clone', 'david-eve') + + def test_exact_commands_from_real_ids(self): + self.assertEqual(self.plan(), ['sudo pcs constraint remove actual-eve-preference']) + + def test_preserve_unrelated_rule(self): + self.assertEqual(self.plan('')[0], + 'sudo pcs constraint remove actual-rule') + + def test_ambiguous_match_refused(self): + with self.assertRaises(ValueError): + self.plan('') + + def test_unexpected_target_role_refused(self): + with self.assertRaises(ValueError): + self.plan(role='Started') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_promotion_scheduler.py b/tests/test_promotion_scheduler.py new file mode 100644 index 0000000..a644739 --- /dev/null +++ b/tests/test_promotion_scheduler.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Real Pacemaker scheduler tests using synthetic CIB files only.""" +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +import xml.etree.ElementTree as ET + + +@unittest.skipUnless(shutil.which('crm_simulate') and shutil.which('cibadmin'), 'Pacemaker CLI not installed') +class Scheduler(unittest.TestCase): + def simulate(self, scores, target=None, fail_primary=False): + if fail_primary and target is None: + target = 'Promoted' # Required production policy. + with tempfile.TemporaryDirectory(prefix='mc-scheduler-') as directory: + root = ET.fromstring(subprocess.check_output(['cibadmin', '--empty'], text=True)) + root.set('have-quorum', '1'); root.set('dc-uuid', '1') + config = root.find('configuration') + props = ET.SubElement(config.find('crm_config'), 'cluster_property_set', id='test-props') + # ONLY this synthetic file: no fencing device is required for an offline simulation. + ET.SubElement(props, 'nvpair', id='test-stonith', name='stonith-enabled', value='true' if fail_primary else 'false') + for i, score in enumerate(scores, 1): + node = 'node-' + str(i) + ET.SubElement(config.find('nodes'), 'node', id=str(i), uname=node) + state = ET.SubElement(root.find('status'), 'node_state', id=str(i), uname=node, + in_ccm='true', crmd='online', join='member', expected='member') + transient = ET.SubElement(state, 'transient_attributes', id=str(i)) + attrs = ET.SubElement(transient, 'instance_attributes', id='attrs-' + str(i)) + # Names generated by crm_attribute --promotion=pg-ha, not old pgsql state. + ET.SubElement(attrs, 'nvpair', id='score-' + str(i), name='master-pg-ha', value=score) + ET.SubElement(attrs, 'nvpair', id='legacy-' + str(i), name='master-postgresql', value='-INFINITY') + if fail_primary: + ET.SubElement(config.find('resources'), 'primitive', + {'id': 'test-fence', 'class': 'stonith', 'type': 'fence_dummy'}) + clone = ET.SubElement(config.find('resources'), 'clone', id='pg-ha-clone') + meta = ET.SubElement(clone, 'meta_attributes', id='clone-meta') + values = {'promotable': 'true', 'clone-max': str(len(scores)), 'clone-node-max': '1', + 'promoted-max': '1', 'promoted-node-max': '1', 'notify': 'true'} + if target is not None: + values['target-role'] = target + for name, value in values.items(): + ET.SubElement(meta, 'nvpair', id='clone-' + name, name=name, value=value) + ET.SubElement(clone, 'primitive', {'id': 'pg-ha', 'class': 'ocf', 'provider': 'loxthelion', 'type': 'managed-container'}) + cib = Path(directory) / 'cib.xml'; graph = Path(directory) / 'graph.xml' + ET.ElementTree(root).write(cib) + if fail_primary: + # Establish running primary/standbys in a synthetic simulation; + # then inject an unexpected failure. No agents/devices execute. + subprocess.run(['crm_simulate', '--xml-file', str(cib), '--in-place'], + check=True, capture_output=True, text=True) + subprocess.run(['crm_simulate', '--xml-file', str(cib), '--run', '--save-graph', str(graph)] + + (['--node-fail', 'node-1'] if fail_primary else []), + check=True, capture_output=True, text=True) + parsed = ET.parse(graph) + if fail_primary: + # Verify fencing is a transitive prerequisite of every promote. + actions = {op.get('id'): op for op in parsed.findall('./synapse/action_set/*')} + dependencies = {} + for synapse in parsed.findall('./synapse'): + for op in synapse.findall('./action_set/*'): + dependencies[op.get('id')] = {x.get('id') for x in synapse.findall('./inputs/trigger/*')} + fences = {key for key, op in actions.items() if op.get('operation') == 'stonith'} + self.assertTrue(fences, ET.tostring(parsed.getroot()).decode()) + def ancestors(key, seen=None): + seen = set() if seen is None else seen + for parent in dependencies.get(key, set()) - seen: + seen.add(parent); ancestors(parent, seen) + return seen + for key, op in actions.items(): + if op.get('operation') == 'promote': + self.assertTrue(ancestors(key) & fences, 'promotion must depend on fencing') + return [op.get('on_node') for op in ET.parse(graph).findall('./synapse/action_set/rsc_op') + if op.get('operation') == 'promote'] + + def test_default_started_allows_one_promotion(self): + self.assertEqual(len(self.simulate(['100', '100'])), 1) + + def test_explicit_started_allows_one_promotion(self): + self.assertEqual(len(self.simulate(['100', '100'], 'Started')), 1) + + def test_explicit_promoted_allows_one_promotion(self): + self.assertEqual(len(self.simulate(['100', '100'], 'Promoted')), 1) + + def test_unpromoted_target_prevents_promotion(self): + self.assertEqual(self.simulate(['100', '100'], 'Unpromoted'), []) + + def test_higher_score_wins_without_host_preference(self): + self.assertEqual(self.simulate(['100', '1000']), ['node-2']) + + def test_ineligible_node_not_selected(self): + self.assertEqual(self.simulate(['-INFINITY', '100']), ['node-2']) + + def test_all_ineligible_no_promotion(self): + self.assertEqual(self.simulate(['-INFINITY', '-INFINITY']), []) + + def test_failed_primary_one_disconnected_candidate(self): + self.assertEqual(self.simulate(['1000', '50'], fail_primary=True), ['node-2']) + + def test_failed_primary_two_disconnected_candidates(self): + first = self.simulate(['1000', '50', '50'], fail_primary=True) + self.assertEqual(len(first), 1) + self.assertIn(first[0], ['node-2', 'node-3']) + self.assertEqual(first, self.simulate(['1000', '50', '50'], fail_primary=True)) + + def test_failed_primary_all_disconnected_stale(self): + self.assertEqual(self.simulate(['1000', '-INFINITY', '-INFINITY'], fail_primary=True), []) + + def test_connected_evidence_preferred(self): + self.assertEqual(self.simulate(['50', '100']), ['node-2']) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/test_promotion_scores.py b/tests/test_promotion_scores.py new file mode 100644 index 0000000..4a07726 --- /dev/null +++ b/tests/test_promotion_scores.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Offline scoring/action tests. All cluster and PostgreSQL boundaries are fake.""" +import json +import os +from pathlib import Path +import shlex +import shutil +import xml.etree.ElementTree as ET +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +AGENT = ROOT / 'refactored/managed-container' +MODULES = ('common', 'container', 'postgres', 'auth', 'replication', 'notify', 'promote', 'demote', 'rejoin') +LOAD = 'MC_PROMOTION_RESOURCE_INSTANCE="${OCF_RESOURCE_INSTANCE:-}"\n: "${OCF_ROOT:=/usr/lib/ocf}"\n. "$OCF_ROOT/lib/heartbeat/ocf-shellfuncs"\n: "${OCF_RUNNING_PROMOTED:=${OCF_RUNNING_MASTER:-8}}"\n' +LOAD += '\n'.join('. ' + shlex.quote(str(AGENT.parent / 'managed-container.d' / (m + '.sh'))) for m in MODULES) + +CRM = '''#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +args=sys.argv[1:] +p=Path(os.environ['SCORE_DB']) +d=json.loads(p.read_text()) +assert '--lifetime=reboot' in args, args +resource=next(x.split('=',1)[1] for x in args if x.startswith('--promotion=')) +assert not any(x.startswith('--name') for x in args), args +node=os.environ.get('OCF_RESKEY_CRM_meta_on_node','node-a') +key=resource+'|'+node +d['calls'].append(args) +if '--query' in args: + p.write_text(json.dumps(d)) + if os.environ.get('QUERY_FAIL')=='1': sys.exit(1) + if key not in d['scores']: sys.exit(6) + print(d['scores'][key]); sys.exit(0) +value=next(x.split('=',1)[1] for x in args if x.startswith('--update=')) +if os.environ.get('UPDATE_FAIL')=='1': + p.write_text(json.dumps(d)); sys.exit(1) +d['scores'][key]=value +d['writes'].append([key,value]) +p.write_text(json.dumps(d)) +''' + +# Original action implementations execute against these fake external boundaries. +FIXTURE = r''' +ROLE=t +RUNNING=1 +SERVICE=1 +CONTAINER=1 +ELIGIBLE=1 +SIGNAL=1 +ocf_log() { :; } +ocf_exit_reason() { printf '%s\n' "$*" >&2; } +service_running() { [ "$SERVICE" = 1 ]; } +container_running() { [ "$CONTAINER" = 1 ]; } +pg_running() { [ "$RUNNING" = 1 ]; } +pg_ready() { return 0; } +pg_role() { echo "$ROLE"; } +pg_wal_receiver_status() { echo streaming; } +mc_pg_promotion_eligible() { [ "$ELIGIBLE" = 1 ]; } +mc_pg_standby_score() { mc_pg_promotion_eligible && echo 100; } +validate_all() { return 0; } +ensure_local_ha_slots() { return 0; } +ensure_rewind_privileges() { return 0; } +demote_target() { echo node-b; } +node_to_addr() { echo 192.0.2.2; } +candidate_is_standby() { return 0; } +candidate_caught_up() { return 0; } +ensure_remote_slot() { return 0; } +pgdata_path() { echo /test/pgdata; } +pg_remote_query() { echo f; } +pg_replication_exec() { return 0; } +pg_command() { + case "$*" in + *'test -f'*) [ "$SIGNAL" = 1 ] ;; + *'promote -w'*) ROLE=f ;; + *'stop -m fast -w'*) RUNNING=0 ;; + *'start -w'*) RUNNING=1 ;; + *) echo "Unexpected pg_command: $*" >&2; return 99 ;; + esac +} +as_service_user() { + case "$*" in + 'systemctl --user stop '*) SERVICE=0; CONTAINER=0 ;; + 'systemctl --user start '*) SERVICE=1; CONTAINER=1 ;; + *RA_PRIMARY_HOST=*) ROLE=t ;; + *) echo "Unexpected external access: $*" >&2; return 99 ;; + esac +} +sleep() { :; } +''' + + +class Scores(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix='mc-score-test-') + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) + self.db = self.path / 'scores.json' + self.db.write_text(json.dumps({'scores': {}, 'calls': [], 'writes': []})) + (self.path / 'crm_attribute').write_text(CRM) + (self.path / 'crm_attribute').chmod(0o755) + self.env = dict(os.environ, PATH=str(self.path) + ':' + os.environ['PATH'], + SCORE_DB=str(self.db), OCF_RESOURCE_INSTANCE='pg-ha:0', + OCF_RESKEY_CRM_meta_on_node='node-a') + + def run_shell(self, code, rc=0, fixture=FIXTURE, env=None): + result = subprocess.run(['bash', '-c', LOAD + fixture + '\n' + code], + env=env or self.env, text=True, capture_output=True, timeout=15) + self.assertEqual(result.returncode, rc, result.stderr) + return result + + def state(self): + return json.loads(self.db.read_text()) + + def score(self, value, node='node-a'): + self.assertEqual(self.state()['scores']['pg-ha:0|' + node], value) + + def test_primary_and_standby(self): + self.run_shell('ROLE=f; mc_promotion_monitor', 8) + self.score('1000') + self.env['OCF_RESKEY_CRM_meta_on_node'] = 'node-b' + self.run_shell('mc_promotion_monitor') + self.score('100', 'node-b') + + def test_two_standbys_equal(self): + for node in ('node-a', 'node-b'): + self.env['OCF_RESKEY_CRM_meta_on_node'] = node + self.run_shell('mc_promotion_monitor') + self.score('100', node) + + def test_ineligible_replaces_stale_positive(self): + cases = [('RUNNING=0', 7), ('CONTAINER=0', 7), ('SERVICE=0', 7), + ('ELIGIBLE=0', 0), ('pg_ready() { return 1; }', 1), + ('pg_role() { return 1; }', 1), ('ROLE=invalid', 1)] + for setup, rc in cases: + with self.subTest(setup=setup): + self.run_shell('mc_promotion_score_set 1000') + self.run_shell(setup + '; mc_promotion_monitor', rc) + self.score('-INFINITY') + + def test_repeat_monitors_write_once(self): + for _ in range(3): + self.run_shell('mc_promotion_monitor') + self.assertEqual(len(self.state()['writes']), 1) + self.assertEqual(len(self.state()['calls']), 4) # Three queries, one update. + + def test_restart_reprobe_recreates_attribute(self): + self.run_shell('mc_promotion_monitor') + data = self.state(); data['scores'] = {}; self.db.write_text(json.dumps(data)) + self.run_shell('mc_promotion_monitor') + self.score('100') + self.assertEqual(len(self.state()['writes']), 2) + + def test_stop_start_original_actions(self): + self.run_shell('mc_promotion_monitor; mc_promotion_action stop; ' + '[ "$RUNNING$SERVICE$CONTAINER" = 000 ]; ' + 'mc_promotion_action start') + self.score('100') + self.assertEqual([x[1] for x in self.state()['writes']], ['100', '-INFINITY', '100']) + + def test_primary_start_refusal(self): + self.run_shell('ROLE=f; mc_promotion_action start', 1) + self.score('-INFINITY') + + def test_missing_standby_signal(self): + self.run_shell('RUNNING=0; SIGNAL=0; mc_promotion_action start', 1) + self.score('-INFINITY') + + def test_promote_original_action(self): + self.run_shell('mc_promotion_monitor; mc_promotion_action promote') + self.score('1000') + self.assertEqual([x[1] for x in self.state()['writes']], ['100', '-INFINITY', '1000']) + + def test_demote_original_action(self): + self.run_shell('ROLE=f; mc_promotion_monitor; mc_promotion_action demote; ' + '[ "$ROLE$SERVICE$CONTAINER" = t11 ]') + self.score('100') + self.assertEqual([x[1] for x in self.state()['writes']], ['1000', '-INFINITY', '100']) + + def test_failed_demote_stays_negative_until_next_observation(self): + self.run_shell('ROLE=f; demote_target() { return 1; }; mc_promotion_action demote', 1) + self.score('-INFINITY') + self.run_shell('ROLE=f; mc_promotion_monitor', 8) + self.score('1000') + + def test_failed_promote_invariants_withdraw_score(self): + self.run_shell('ensure_local_ha_slots() { return 1; }; mc_promotion_action promote', 1) + self.score('-INFINITY') + + def test_validation_failure_withdraws_score(self): + self.run_shell('mc_promotion_monitor; validate_all() { return 6; }; mc_promotion_action start', 6) + self.score('-INFINITY') + + def test_stop_does_not_require_validation(self): + self.run_shell('validate_all() { return 6; }; mc_promotion_action stop') + self.score('-INFINITY') + + def test_rejoin_original_action(self): + self.run_shell('RUNNING=0; REJOIN_SOURCE_HOST=192.0.2.2; REPLICATION_SLOT=ha_node; ' + 'mc_promotion_action rejoin') + self.score('100') + + def test_update_failure_preserves_primary_role_rc(self): + self.env['UPDATE_FAIL'] = '1' + result = self.run_shell('ocf_log() { echo "$*" >&2; }; ROLE=f; mc_promotion_monitor', 8) + self.assertIn('prior score may remain', result.stderr) + self.assertFalse(self.state()['scores']) + + def test_update_failure_does_not_block_stop(self): + self.env['UPDATE_FAIL'] = '1' + self.run_shell('mc_promotion_action stop; [ "$RUNNING" = 0 ]') + + def test_query_failure_attempts_replace(self): + self.env['QUERY_FAIL'] = '1' + self.run_shell('ELIGIBLE=0; mc_promotion_monitor') + self.score('-INFINITY') + + def test_scoring_dependency_validation(self): + self.run_shell('command() { return 1; }; mc_promotion_validate', 5) + + def test_missing_instance_never_uses_default_attribute(self): + env = dict(self.env) + env.pop('OCF_RESOURCE_INSTANCE') + self.run_shell('mc_promotion_monitor', env=env) + self.assertFalse(self.state()['calls']) + + def test_target_role_does_not_affect_scoring(self): + for value in ('', 'Started', 'Promoted'): + self.env['OCF_RESKEY_CRM_meta_target_role'] = value + self.run_shell('mc_promotion_monitor') + self.score('100') + + def test_unrelated_legacy_attributes_untouched(self): + d = self.state() + d['scores']['master-postgresql'] = '-INFINITY' + d['scores']['postgresql-data-status'] = 'DISCONNECT' + d['scores']['postgresql-status'] = 'STOP' + self.db.write_text(json.dumps(d)) + self.run_shell('mc_promotion_monitor') + for key, value in d['scores'].items(): + self.assertEqual(self.state()['scores'][key], value) + + @unittest.skipUnless(shutil.which('crm_attribute') and shutil.which('cibadmin'), 'Pacemaker CLI not installed') + def test_real_crm_attribute_against_offline_cib(self): + # CIB_file forces Pacemaker's file backend: no live daemon is contacted. + cib = self.path / 'offline.xml' + tree = ET.fromstring(subprocess.check_output(['cibadmin', '--empty'], text=True)) + ET.SubElement(tree.find('./configuration/nodes'), 'node', id='1', uname='node-a') + ET.SubElement(tree.find('./status'), 'node_state', id='1', uname='node-a') + cib.write_text(ET.tostring(tree, encoding='unicode')) + env = dict(self.env, PATH=os.environ['PATH'], CIB_file=str(cib)) + self.run_shell('mc_promotion_score_set 100', env=env) + before = cib.read_bytes() + self.run_shell('mc_promotion_score_set 100', env=env) + self.assertEqual(cib.read_bytes(), before, 'Identical observations must not rewrite CIB') + self.run_shell('mc_promotion_score_set -INFINITY', env=env) + pairs = ET.parse(cib).findall('./status/node_state/transient_attributes/instance_attributes/nvpair') + self.assertEqual([(p.get('name'), p.get('value')) for p in pairs], [('master-pg-ha:0', '-INFINITY')]) + # Reprobe after loss of transient state must recreate the correct score. + tree = ET.parse(cib) + state = tree.find('./status/node_state') + state.remove(state.find('./transient_attributes')) + tree.write(cib) + self.run_shell('mc_promotion_score_set 1000', env=env) + self.assertEqual(ET.parse(cib).find('.//transient_attributes//nvpair').get('value'), '1000') + + def test_actual_eligibility_helper_rejects_non_true_or_failed_sql(self): + # Exercise the production shell helper's output and error handling. + for answer, sql_rc, expected in [('t', 0, 0), ('f', 0, 1), ('', 0, 1), + ('t\nf', 0, 1), ('t', 1, 1)]: + fixture = '\nas_service_user() { printf "%s" ' + shlex.quote(answer) + '; return ' + str(sql_rc) + '; }\n' + self.run_shell('mc_pg_promotion_eligible', expected, fixture=fixture) + + def test_dispatch_routes_only_operational_actions_through_scoring(self): + # Use the real dispatcher with fake handlers; no cluster calls occur. + dispatch = AGENT.read_text().split('case "$1" in', 1)[1] + for action in ('start', 'stop', 'promote', 'demote', 'rejoin'): + code = 'mc_promotion_action() { echo "scored:$1"; return 0; }; set -- ' + action + '\ncase "$1" in' + dispatch + result = self.run_shell(code) + self.assertEqual(result.stdout.strip(), 'scored:' + action) + for action in ('monitor', 'status'): + code = 'mc_promotion_monitor() { echo scored-monitor; return 8; }; set -- ' + action + '\ncase "$1" in' + dispatch + self.assertEqual(self.run_shell(code, 8).stdout.strip(), 'scored-monitor') + for action, handler in [('meta-data', 'mc_promotion_metadata'), ('validate-all', 'mc_promotion_validate'), ('notify', 'managed_notify')]: + code = handler + '() { echo plain; return 0; }; set -- ' + action + '\ncase "$1" in' + dispatch + self.assertEqual(self.run_shell(code).stdout.strip(), 'plain') + self.assertFalse(self.state()['calls']) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/verify_installed_rpm.py b/tests/verify_installed_rpm.py index 1acd1eb..7689feb 100644 --- a/tests/verify_installed_rpm.py +++ b/tests/verify_installed_rpm.py @@ -18,9 +18,11 @@ assert output('rpm', '-q', '--qf', '%{ARCH}', PACKAGE) == 'noarch' assert output('rpm', '-q', '--scripts', PACKAGE) == '' subprocess.run(['rpm', '-V', PACKAGE], check=True) expected = {AGENT: ROOT / 'refactored/managed-container'} -for path in (ROOT / 'refactored/managed-container.d').glob('*.sh'): +for path in (ROOT / 'refactored/managed-container.d').glob('*'): + if not path.is_file(): + continue expected[PROVIDER / 'managed-container.d' / path.name] = path -assert len(expected) == 10 +assert len(expected) == 11 for path, source in expected.items(): assert path.read_bytes() == source.read_bytes(), path info = path.stat() @@ -32,7 +34,7 @@ allowed = {str(path) for path in expected} | {str(PROVIDER), str(PROVIDER / 'man unexpected = {path for path in owned - allowed if not path.startswith('/usr/share/doc/' + PACKAGE)} assert not unexpected, unexpected -reference = output(str(ROOT / 'reference/managed-container'), 'meta-data') +reference = output(str(ROOT / 'refactored/managed-container'), 'meta-data') installed = subprocess.check_output([str(AGENT), 'meta-data'], cwd='/tmp', text=True) assert installed == reference print('PASS installed RPM: noarch, file integrity, root ownership, modes, payload, no scriptlets, metadata/module loading') diff --git a/tests/verify_promotion_sql.py b/tests/verify_promotion_sql.py new file mode 100644 index 0000000..9e8c234 --- /dev/null +++ b/tests/verify_promotion_sql.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Run the production eligibility SQL against controlled PostgreSQL 18 fixtures. + +Explicit opt-in integration test: creates/removes only its own disposable Podman +container. No host ports, volumes, cluster credentials or live databases used. +The postgres:18 image must already be present locally. +""" +from pathlib import Path +import os +import json +import shlex +import subprocess +import tempfile +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +with tempfile.TemporaryDirectory(prefix='mc-score-sql-') as directory: + query_file = Path(directory) / 'query.sql' + env = dict(os.environ, QUERY_FILE=str(query_file)) + extract = '. ' + shlex.quote(str(ROOT / 'refactored/managed-container.d/postgres.sh')) + ''' +as_service_user() { printf '%s' "${@: -1}" > "$QUERY_FILE"; echo t; } +mc_pg_promotion_eligible +''' + subprocess.run(['bash', '-c', extract], env=env, check=True) + query = query_file.read_text() + subprocess.run(['bash', '-c', extract.replace('mc_pg_promotion_eligible', 'mc_pg_standby_observation')], env=env, check=True) + observation_query = query_file.read_text() + name = 'mc-score-sql-' + uuid.uuid4().hex[:12] + subprocess.run(['podman', 'run', '-d', '--pull=never', '--name', name, + '--network=none', '--tmpfs', '/var/lib/postgresql', + '-e', 'POSTGRES_HOST_AUTH_METHOD=trust', + 'docker.io/library/postgres:18'], check=True, capture_output=True) + try: + for _ in range(40): + ready = subprocess.run(['podman', 'exec', name, 'pg_isready', '-U', 'postgres'], + capture_output=True) + if ready.returncode == 0: + break + time.sleep(0.5) + else: + raise RuntimeError('Disposable PostgreSQL did not become ready') + def sql(text): + return subprocess.check_output(['podman', 'exec', '-i', name, 'psql', + '-X', '-U', 'postgres', '-v', 'ON_ERROR_STOP=1', + '-Atq'], input=text, text=True).strip() + assert sql('SHOW server_version_num;').startswith('18') + sql(''' +CREATE SCHEMA score_test; +CREATE TABLE score_test.state (recovery bool, paused bool, replay pg_lsn); +INSERT INTO score_test.state VALUES (true,false,'0/20'); +CREATE TABLE score_test.pg_stat_wal_receiver (status text, flushed_lsn pg_lsn); +CREATE FUNCTION score_test.pg_is_in_recovery() RETURNS bool LANGUAGE sql +AS 'SELECT recovery FROM score_test.state'; +CREATE FUNCTION score_test.pg_is_wal_replay_paused() RETURNS bool LANGUAGE plpgsql AS $$ +BEGIN + IF NOT (SELECT recovery FROM score_test.state) THEN + RAISE EXCEPTION 'replay pause is undefined on a primary'; + END IF; + RETURN (SELECT paused FROM score_test.state); +END $$; +CREATE FUNCTION score_test.pg_last_wal_replay_lsn() RETURNS pg_lsn LANGUAGE sql +AS 'SELECT replay FROM score_test.state'; +''') + cases = [ + ('caught-up streaming standby', True, False, '0/20', 'streaming', '0/20', 't'), + ('replay ahead of receiver snapshot', True, False, '0/30', 'streaming', '0/20', 't'), + ('replay lag', True, False, '0/10', 'streaming', '0/20', 'f'), + ('paused replay', True, True, '0/20', 'streaming', '0/20', 'f'), + ('no replay position', True, False, None, 'streaming', '0/20', 'f'), + ('unknown flushed position', True, False, '0/20', 'streaming', None, 'f'), + ('receiver stopped', True, False, '0/20', 'stopping', '0/20', 'f'), + ('no receiver after primary failure', True, False, '0/20', None, None, 'f'), + ('primary skips recovery-only function', False, False, '0/20', None, None, 'f'), + ] + def literal(value): + return 'NULL' if value is None else "'" + value + "'" + for label, recovery, paused, replay, status, flushed, expected in cases: + setup = f'''SET search_path=score_test,pg_catalog; +UPDATE score_test.state SET recovery={str(recovery).lower()},paused={str(paused).lower()},replay={literal(replay)}; +TRUNCATE score_test.pg_stat_wal_receiver; +''' + if status is not None: + setup += f'INSERT INTO score_test.pg_stat_wal_receiver VALUES ({literal(status)},{literal(flushed)});\n' + result = sql(setup + query) + assert result == expected, (label, result, expected) + observation = json.loads(sql(setup + observation_query)) + assert (observation['strict'] is True) == (expected == 't'), (label, observation) + assert observation['identity'][0].isdigit() and observation['identity'][1].startswith('/') + assert observation['recovery'] == recovery + assert observation['receiver'] == status + print('PASS strict and observation SQL:', label) + finally: + subprocess.run(['podman', 'rm', '-f', name], check=True, capture_output=True) diff --git a/tests/verify_promotion_state_cli.py b/tests/verify_promotion_state_cli.py new file mode 100644 index 0000000..34fe357 --- /dev/null +++ b/tests/verify_promotion_state_cli.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Explicit disposable-container-only test of the root helper CLI and /run state.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import uuid + +from test_disconnected_state import CONNECTED, DISCONNECTED + +assert os.geteuid() == 0 and os.environ.get('MC_DISPOSABLE_STATE_TEST') == '1', ( + 'Run only in a disposable container with MC_DISPOSABLE_STATE_TEST=1') +helper = (Path(sys.argv[1]) if len(sys.argv) > 1 else + Path(__file__).resolve().parents[1] / 'refactored/managed-container.d/promotion_state.py') +resource = 'mc-disposable-' + uuid.uuid4().hex +args = ['python3', str(helper), resource, 'test-owner', 'test.service', 'test-container', '300'] + +def run(obs, expected): + result = subprocess.run(args, input=json.dumps(obs), text=True, capture_output=True, timeout=3) + assert result.returncode == 0 and result.stdout.strip() == expected, result + +run(DISCONNECTED, '-INFINITY') +run(CONNECTED, '100') +files = list(Path('/run/managed-container/promotion').glob('*.json')) +record_path = next(p for p in files if json.loads(p.read_text())['context'][0] == resource) +record = json.loads(record_path.read_text()) +assert record['replay'] == CONNECTED['replay'] and record['flush'] == CONNECTED['flush'] +for path, mode in [(record_path, 0o600), (record_path.parent, 0o700), (record_path.parent.parent, 0o700)]: + info = path.stat() + assert (info.st_uid, info.st_gid, info.st_mode & 0o777) == (0, 0, mode), path +run(DISCONNECTED, '50') +assert json.loads(record_path.read_text()) == record # Disconnection never renews evidence. +record_path.write_text('{malformed') +run(DISCONNECTED, '-INFINITY') +assert not record_path.exists() +run(CONNECTED, '100') +run(dict(DISCONNECTED, identity=['999', '/new', '1789201000.123456']), '-INFINITY') +assert not record_path.exists() +print('PASS root CLI: fixed /run location, ownership/modes, persistence, disconnect, malformed and replaced data') diff --git a/tests/verify_refactor.py b/tests/verify_refactor.py index c3fbc89..3611bbe 100644 --- a/tests/verify_refactor.py +++ b/tests/verify_refactor.py @@ -41,13 +41,19 @@ for path in (REFERENCE, ROOT / 'reference/managed-container.pre-refactor-5eae1f3 assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_hash print('PASS both reference hashes') -assert functions(SOURCE) == functions('\n'.join(p.read_text() for p in MODULES)) +current_functions = functions('\n'.join(p.read_text() for p in MODULES)) +for name, definition in functions(SOURCE).items(): + assert current_functions[name] == definition, name +assert set(current_functions) - set(functions(SOURCE)) == { + 'mc_promotion_score_set', 'mc_promotion_score_from_monitor', + 'mc_promotion_monitor', 'mc_promotion_action', 'mc_promotion_validate', 'mc_pg_promotion_eligible', + 'mc_promotion_metadata', 'mc_pg_standby_observation', 'mc_pg_standby_score', +} assert len(functions(SOURCE)) == 29 common = (AGENT.parent / 'managed-container.d/common.sh').read_text() defaults = SOURCE[SOURCE.index('OCF_RESKEY_service_user_default='):SOURCE.index('\n###############################################################################\n# Metadata')].strip() assert defaults in common -assert SOURCE[SOURCE.index('case "$1" in'):] == AGENT.read_text()[AGENT.read_text().index('case "$1" in'):] -print('PASS all 29 function bodies, defaults/globals and dispatcher identical') +print('PASS all 29 original function bodies and defaults/globals identical; nine scoring helpers added') for path in [AGENT, *MODULES]: subprocess.run(['bash', '-n', str(path)], check=True) @@ -58,11 +64,19 @@ for path in (REFERENCE, AGENT): result = subprocess.run([str(path), 'meta-data'], cwd='/tmp', capture_output=True, text=True) assert result.returncode == 0 and not result.stderr, result metadata.append(result.stdout) -assert metadata[0] == metadata[1] +extended = ET.fromstring(metadata[1]) +parameters = extended.find('parameters') +added = parameters.find("parameter[@name='disconnected_max_age']") +assert added is not None and added.find('content').get('default') == '300' +parameters.remove(added) +def normalize(element): + return (element.tag, element.attrib, (element.text or '').strip(), + [normalize(child) for child in element]) +assert normalize(extended) == normalize(ET.fromstring(metadata[0])) xml = ET.fromstring(metadata[0]) assert len(xml.findall('./parameters/parameter')) == 12 assert len(xml.findall('./actions/action')) == 8 -print('PASS metadata byte equality and XML parsing from unrelated cwd') +print('PASS original metadata preserved plus freshness parameter and XML parsing from unrelated cwd') # Sourcing production modules in a fresh shell tests their real loading order. # No dispatcher executes. All operational boundaries are overridden below.