Freeze HA scoring and document reported production failover

refactored/managed-container

@@ -13,6 +13,9 @@
 
 : "${OCF_ROOT:=/usr/lib/ocf}"
 OCF_FUNCTIONS_DIR="${OCF_ROOT}/lib/heartbeat"
+# Preserve explicit resource identity before ocf-shellfuncs supplies its
+# standalone-invocation default; never publish a score for that dummy resource.
+MC_PROMOTION_RESOURCE_INSTANCE="${OCF_RESOURCE_INSTANCE:-}"
 . "${OCF_FUNCTIONS_DIR}/ocf-shellfuncs"
 
 # Pacemaker uses "Promoted" terminology, while many OCF shell libraries
@@ -33,40 +36,37 @@ unset MC_MODULE
 
 case "$1" in
     meta-data)
-        meta_data
+        mc_promotion_metadata
         exit "${OCF_SUCCESS}"
         ;;
 
     validate-all)
-        validate_all
+        mc_promotion_validate
         exit $?
         ;;
 
     monitor|status)
-        managed_monitor
+        mc_promotion_monitor
         exit $?
         ;;
 
     start)
-        validate_all || exit $?
-        managed_start
+        mc_promotion_action start
         exit $?
         ;;
 
     stop)
-        managed_stop
+        mc_promotion_action stop
         exit $?
         ;;
 
     promote)
-        validate_all || exit $?
-        managed_promote
+        mc_promotion_action promote
         exit $?
         ;;
 
     demote)
-        validate_all || exit $?
-        managed_demote
+        mc_promotion_action demote
         exit $?
         ;;
 
@@ -76,8 +76,7 @@ case "$1" in
         ;;
 
     rejoin)
-        validate_all || exit $?
-        managed_rejoin
+        mc_promotion_action rejoin
         exit $?
         ;;
 

refactored/managed-container.d/common.sh

@@ -225,3 +225,126 @@ validate_all()
 
     return "${OCF_SUCCESS}"
 }
+
+# Promotion scores are scoped by Pacemaker to OCF_RESOURCE_INSTANCE and the
+# executing node. Do not construct a legacy attribute name or cache locally:
+# querying attrd on every observation also repairs a lost transient attribute.
+mc_promotion_score_set()
+{
+    local score="$1" current
+    local resource="${MC_PROMOTION_RESOURCE_INSTANCE:-}"
+
+    if [ -z "${resource}" ]; then
+        ocf_log warn "Cannot publish promotion score without OCF_RESOURCE_INSTANCE"
+        return 1
+    fi
+
+    case "${score}" in
+        1000|100|50|-INFINITY) ;;
+        *) return 1 ;;
+    esac
+
+    current="$(
+        timeout --kill-after=1s 2s crm_attribute \
+            --promotion="${resource}" --lifetime=reboot \
+            --query --quiet 2>/dev/null
+    )" || current=""
+
+    [ "${current}" = "${score}" ] && return 0
+
+    if ! timeout --kill-after=1s 2s crm_attribute \
+        --promotion="${resource}" --lifetime=reboot \
+        --update="${score}" >/dev/null 2>&1
+    then
+        ocf_log err "Unable to publish promotion score ${score} for ${resource}; prior score may remain"
+        return 1
+    fi
+    return 0
+}
+
+mc_promotion_score_from_monitor()
+{
+    local monitor_rc="$1" score=-INFINITY
+
+    case "${monitor_rc}" in
+        "${OCF_RUNNING_PROMOTED}") score=1000 ;;
+        "${OCF_SUCCESS}")
+            score="$(mc_pg_standby_score)" || score=-INFINITY
+            ;;
+    esac
+    case "${score}" in
+        1000|100|50|-INFINITY) ;;
+        *) score=-INFINITY ;;
+    esac
+    mc_promotion_score_set "${score}"
+}
+
+# Score publication must not change OCF role detection or its return code.
+mc_promotion_monitor()
+{
+    local monitor_rc
+
+    managed_monitor
+    monitor_rc=$?
+    mc_promotion_score_from_monitor "${monitor_rc}" || true
+    return "${monitor_rc}"
+}
+
+# Lifecycle functions retain their original checks, commands and return codes.
+# Withdraw the previous score before any transition (including validation
+# failure), then reassess only after success. Their internal monitor calls do
+# not publish scores, so they cannot restore a positive score mid-transition.
+mc_promotion_action()
+{
+    local action="$1" action_rc monitor_rc
+
+    case "${action}" in
+        start|stop|promote|demote|rejoin) ;;
+        *) return "${OCF_ERR_UNIMPLEMENTED}" ;;
+    esac
+
+    mc_promotion_score_set -INFINITY || true
+    if [ "${action}" != stop ]; then
+        mc_promotion_validate
+        action_rc=$?
+        [ "${action_rc}" -eq "${OCF_SUCCESS}" ] || return "${action_rc}"
+    fi
+
+    "managed_${action}"
+    action_rc=$?
+    if [ "${action_rc}" -eq "${OCF_SUCCESS}" ] && [ "${action}" != stop ]; then
+        managed_monitor
+        monitor_rc=$?
+        mc_promotion_score_from_monitor "${monitor_rc}" || true
+    fi
+    return "${action_rc}"
+}
+
+mc_promotion_validate()
+{
+    local utility
+
+    if ! [[ "${OCF_RESKEY_disconnected_max_age:-300}" =~ ^(0|[1-9][0-9]{0,4})$ ]]; then
+        ocf_exit_reason "disconnected_max_age must be an integer from 0 to 99999 seconds"
+        return "${OCF_ERR_CONFIGURED}"
+    fi
+
+    for utility in crm_attribute timeout python3; do
+        if ! command -v "${utility}" >/dev/null 2>&1; then
+            ocf_exit_reason "Required promotion scoring command '${utility}' is not installed"
+            return "${OCF_ERR_INSTALLED}"
+        fi
+    done
+    validate_all
+}
+
+# Keep the original metadata function intact; advertise the added policy here.
+mc_promotion_metadata()
+{
+    meta_data | sed '/<\/parameters>/i\
+    <parameter name="disconnected_max_age" unique="0" required="0">\
+      <longdesc lang="en">Maximum age in seconds of a verified streaming observation for disconnected promotion eligibility. Default 300; 0 disables disconnected eligibility. Requires fencing managed by Pacemaker. PostgreSQL restart requires fresh streaming verification.</longdesc>\
+      <shortdesc lang="en">Disconnected standby observation lifetime</shortdesc>\
+      <content type="integer" default="300"/>\
+    </parameter>'
+}

refactored/managed-container.d/postgres.sh

@@ -290,3 +290,76 @@ managed_stop()
 
     return "${OCF_SUCCESS}"
 }
+
+# Conservative scoring eligibility only; this does not replace role detection,
+# fencing, the promotion action, or the existing controlled-demote WAL checks.
+# This strict predicate remains unchanged; disconnected policy is separate below.
+mc_pg_promotion_eligible()
+{
+    local eligible
+
+    eligible="$(
+        as_service_user timeout --kill-after=1s 3s \
+            podman exec --user "${PG_OS_USER}" \
+            --env 'PGOPTIONS=-c statement_timeout=2000' \
+            "${CONTAINER_NAME}" \
+            psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc "
+                SELECT CASE WHEN pg_is_in_recovery() THEN
+                    NOT pg_is_wal_replay_paused()
+                    AND EXISTS (
+                        SELECT 1 FROM pg_stat_wal_receiver
+                        WHERE status = 'streaming'
+                          AND flushed_lsn IS NOT NULL
+                          AND pg_last_wal_replay_lsn() >= flushed_lsn
+                    )
+                ELSE false END;
+            " 2>/dev/null
+    )" || return 1
+    [ "${eligible}" = t ]
+}
+
+# One observation supplies both strict connected eligibility and disconnected
+# evidence. PostgreSQL restart time deliberately invalidates prior observations,
+# including a new basebackup sharing the same PostgreSQL system identifier.
+mc_pg_standby_observation()
+{
+    as_service_user timeout --kill-after=1s 3s \
+        podman exec --user "${PG_OS_USER}" \
+        --env 'PGOPTIONS=-c statement_timeout=2000' \
+        "${CONTAINER_NAME}" \
+        psql -X -v ON_ERROR_STOP=1 -U "${DB_USER}" -d "${DB_NAME}" -Atqc "
+            WITH s AS MATERIALIZED (
+                SELECT pg_is_in_recovery() AS recovery,
+                       pg_last_wal_replay_lsn() AS replay,
+                       r.status AS receiver, r.flushed_lsn AS flush
+                FROM (SELECT 1) AS singleton
+                LEFT JOIN pg_stat_wal_receiver r ON true
+            )
+            SELECT json_build_object(
+                'identity', json_build_array(
+                    (SELECT system_identifier::text FROM pg_control_system()),
+                    current_setting('data_directory'),
+                    extract(epoch FROM pg_postmaster_start_time())::text),
+                'recovery', recovery,
+                'paused', CASE WHEN recovery THEN pg_is_wal_replay_paused() ELSE true END,
+                'receiver', receiver, 'replay', replay::text, 'flush', flush::text,
+                'strict', CASE WHEN recovery THEN
+                    NOT pg_is_wal_replay_paused() AND receiver = 'streaming'
+                    AND flush IS NOT NULL AND replay >= flush
+                    ELSE false END)
+            FROM s;
+        " 2>/dev/null
+}
+
+mc_pg_standby_score()
+{
+    local observation
+    observation="$(mc_pg_standby_observation)" || return 1
+    # The helper is root-owned alongside the modules; state never enters the
+    # rootless container. A failed/slow helper always withdraws eligibility.
+    printf '%s' "${observation}" | timeout --kill-after=1s 1s python3 \
+        "${MC_MODULE_DIR}/promotion_state.py" \
+        "${MC_PROMOTION_RESOURCE_INSTANCE:-}" \
+        "${SERVICE_USER}" "${SERVICE_NAME}" "${CONTAINER_NAME}" \
+        "${OCF_RESKEY_disconnected_max_age:-300}"
+}

refactored/managed-container.d/promotion_state.py

@@ -0,0 +1,173 @@
+"""Local promotion evidence. Invoked by the root RA; never invokes cluster tools.
+
+State is a bounded JSON record, never executable input. The public CLI uses a
+fixed root-owned /run directory and CLOCK_BOOTTIME, including suspend duration.
+"""
+import fcntl
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import re
+import stat
+import sys
+import tempfile
+import time
+
+STATE_DIR = Path('/run/managed-container/promotion')
+
+
+def unique_object(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError('duplicate JSON key')
+        result[key] = value
+    return result
+
+
+def lsn(value):
+    if not isinstance(value, str) or not re.fullmatch(r'[0-9A-F]{1,8}/[0-9A-F]{1,8}', value):
+        raise ValueError('invalid LSN')
+    high, low = value.split('/')
+    return (int(high, 16) << 32) | int(low, 16)
+
+
+def number(value):
+    return type(value) in (int, float) and math.isfinite(value) and value >= 0
+
+
+def validate_observation(obs):
+    if set(obs) != {'identity', 'recovery', 'paused', 'receiver', 'replay', 'flush', 'strict'}:
+        raise ValueError('invalid observation fields')
+    identity = obs['identity']
+    if not isinstance(identity, list) or len(identity) != 3:
+        raise ValueError('invalid identity')
+    system, path, started = identity
+    if (not isinstance(system, str) or not re.fullmatch(r'[0-9]{1,20}', system)
+            or not isinstance(path, str) or not path.startswith('/') or len(path) > 2048
+            or not isinstance(started, str) or not re.fullmatch(r'[0-9]{1,12}\.[0-9]{1,6}', started)):
+        raise ValueError('invalid identity values')
+    if type(obs['recovery']) is not bool or type(obs['paused']) is not bool:
+        raise ValueError('invalid role')
+    if obs['strict'] is not None and type(obs['strict']) is not bool:
+        raise ValueError('invalid strict flag')
+    if obs['receiver'] is not None and obs['receiver'] not in (
+            'starting', 'streaming', 'waiting', 'restarting', 'stopping'):
+        raise ValueError('invalid receiver status')
+    replay = lsn(obs['replay']) if obs['replay'] is not None else None
+    flush = lsn(obs['flush']) if obs['flush'] is not None else None
+    strict = (obs['recovery'] and not obs['paused'] and obs['receiver'] == 'streaming'
+              and replay is not None and flush is not None and replay >= flush)
+    if (obs['strict'] is True) != strict:
+        raise ValueError('inconsistent strict observation')
+    return strict
+
+
+class Evidence:
+    def __init__(self, directory, context, boot, now, wall, owner=0):
+        self.directory, self.context = Path(directory), context
+        self.boot, self.now, self.wall, self.owner = boot, now, wall, owner
+        key = hashlib.sha256(json.dumps(context, separators=(',', ':')).encode()).hexdigest()
+        self.path = self.directory / (key + '.json')
+        self.lock = self.directory / (key + '.lock')
+
+    def secure(self, path, mode):
+        info = path.lstat()
+        if (info.st_uid != self.owner or stat.S_IMODE(info.st_mode) != mode
+                or (not stat.S_ISREG(info.st_mode) if mode == 0o600 else not stat.S_ISDIR(info.st_mode))):
+            raise ValueError('unsafe state ownership/type/mode')
+
+    def discard(self):
+        self.path.unlink(missing_ok=True)
+
+    def evaluate(self, obs, max_age):
+        if type(max_age) is not int or not 0 <= max_age <= 99999:
+            raise ValueError('invalid freshness threshold')
+        self.directory.mkdir(mode=0o700, exist_ok=True)
+        self.secure(self.directory, 0o700)
+        fd = os.open(self.lock, os.O_CREAT | os.O_NOFOLLOW | os.O_RDWR, 0o600)
+        with os.fdopen(fd, 'r+') as lock:
+            self.secure(self.lock, 0o600)
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            try:
+                return self._evaluate(obs, max_age)
+            except (ValueError, TypeError, KeyError, OverflowError):
+                self.discard()
+                return '-INFINITY'
+
+    def _evaluate(self, obs, max_age):
+        strict = validate_observation(obs)
+        if strict:
+            record = dict(version=1, context=self.context, boot=self.boot,
+                          identity=obs['identity'], observed_boottime=self.now,
+                          observed_unix=self.wall, replay=obs['replay'], flush=obs['flush'])
+            # Same-directory atomic replace; failed persistence cannot authorize
+            # subsequent disconnected promotion. No secrets are recorded.
+            fd, temporary = tempfile.mkstemp(prefix='.observation-', dir=self.directory)
+            try:
+                with os.fdopen(fd, 'w') as out:
+                    json.dump(record, out, separators=(',', ':'))
+                    out.flush()
+                    os.fsync(out.fileno())
+                os.replace(temporary, self.path)
+            finally:
+                Path(temporary).unlink(missing_ok=True)
+            return '100'
+        if (not obs['recovery'] or obs['paused'] or obs['receiver'] == 'streaming'
+                or obs['replay'] is None or max_age == 0):
+            self.discard()
+            return '-INFINITY'
+        if not self.path.exists():
+            return '-INFINITY'
+        self.secure(self.path, 0o600)
+        if self.path.stat().st_size > 4096:
+            raise ValueError('oversized state')
+        with self.path.open() as source:
+            record = json.load(source, object_pairs_hook=unique_object)
+        if set(record) != {'version', 'context', 'boot', 'identity', 'observed_boottime',
+                           'observed_unix', 'replay', 'flush'}:
+            raise ValueError('invalid record fields')
+        if (type(record['version']) is not int or record['version'] != 1
+                or record['context'] != self.context or record['boot'] != self.boot
+                or record['identity'] != obs['identity']
+                or not number(record['observed_boottime']) or not number(record['observed_unix'])
+                or not 0 <= self.now - record['observed_boottime'] <= max_age
+                or lsn(record['replay']) < lsn(record['flush'])
+                or lsn(obs['replay']) < lsn(record['replay'])):
+            raise ValueError('expired or incompatible observation')
+        return '50'
+
+
+def main():
+    if os.geteuid() != 0 or len(sys.argv) != 6 or not all(sys.argv[1:5]):
+        raise ValueError('root and explicit resource context required')
+    age = sys.argv[5]
+    if not re.fullmatch(r'(0|[1-9][0-9]{0,4})', age):
+        raise ValueError('invalid freshness threshold')
+    # /run is trusted root-owned; validate our parent before using a child.
+    parent = STATE_DIR.parent
+    parent.mkdir(mode=0o700, exist_ok=True)
+    info = parent.lstat()
+    if info.st_uid != 0 or not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o700:
+        raise ValueError('unsafe state parent')
+    raw = sys.stdin.read(8193)
+    if len(raw) > 8192:
+        raise ValueError('oversized observation')
+    evidence = Evidence(STATE_DIR, sys.argv[1:5],
+                        Path('/proc/sys/kernel/random/boot_id').read_text().strip(),
+                        time.clock_gettime(time.CLOCK_BOOTTIME), time.time())
+    try:
+        obs = json.loads(raw, object_pairs_hook=unique_object)
+    except ValueError:
+        obs = {}  # Invalid query output withdraws and removes prior evidence.
+    print(evidence.evaluate(obs, int(age)))
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except (OSError, ValueError, TypeError, KeyError, OverflowError):
+        print('-INFINITY')
+        sys.exit(1)