Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Lyncoln
|
3d1714d995 | ||
|
Lyncoln
|
cbb865729a |
@@ -54,12 +54,12 @@ mc_promotion_score_from_monitor()
|
||||
}
|
||||
|
||||
# managed-containerd is authoritative for recurring monitor/status results.
|
||||
# The legacy application monitor remains temporarily available as a shadow
|
||||
# comparison only. Promotion scoring is still supplied by the existing
|
||||
# application scoring hook during this migration stage.
|
||||
# The packaged application monitor is used only when managed-containerd
|
||||
# explicitly delegates monitoring to the container backend. Promotion scoring
|
||||
# remains supplied by the existing application scoring hook.
|
||||
mc_promotion_monitor()
|
||||
{
|
||||
local monitor_rc legacy_rc proxy_result proxy_rc cached_rc
|
||||
local monitor_rc proxy_result proxy_rc cached_rc
|
||||
|
||||
proxy_result="$(mc_proxy_monitor)"
|
||||
proxy_rc=$?
|
||||
@@ -91,13 +91,6 @@ mc_promotion_monitor()
|
||||
ocf_log warn \
|
||||
"managed-containerd unavailable; using recent cached authoritative rc=${monitor_rc} for bounded control-plane grace"
|
||||
|
||||
mc_application_monitor
|
||||
legacy_rc=$?
|
||||
|
||||
mc_proxy_legacy_shadow_monitor \
|
||||
"${monitor_rc}" \
|
||||
"${legacy_rc}" || true
|
||||
|
||||
return "${monitor_rc}"
|
||||
fi
|
||||
|
||||
@@ -119,6 +112,70 @@ mc_promotion_monitor()
|
||||
esac
|
||||
fi
|
||||
|
||||
# A successful managed-containerd response may explicitly delegate
|
||||
# monitoring to the RA's already-loaded packaged container/runtime path.
|
||||
#
|
||||
# The daemon remains authoritative for backend selection: this path is
|
||||
# reachable only after managed-containerd successfully requested it.
|
||||
case "${proxy_result}" in
|
||||
delegate\|*)
|
||||
local delegate_tag delegate_backend delegate_runtime delegate_extra
|
||||
local configured_runtime
|
||||
|
||||
IFS='|' read -r delegate_tag delegate_backend delegate_runtime delegate_extra <<EOF
|
||||
${proxy_result}
|
||||
EOF
|
||||
|
||||
if [ "${delegate_tag}" != delegate ] ||
|
||||
[ "${delegate_backend}" != container ] ||
|
||||
[ -z "${delegate_runtime}" ] ||
|
||||
[ -n "${delegate_extra}" ]; then
|
||||
|
||||
mc_proxy_monitor_cache_clear || true
|
||||
mc_promotion_score_set -INFINITY || true
|
||||
ocf_exit_reason "managed-containerd returned invalid monitor delegation: ${proxy_result}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
configured_runtime="${OCF_RESKEY_container_runtime:-podman}"
|
||||
|
||||
if [ "${delegate_runtime}" != "${configured_runtime}" ]; then
|
||||
mc_proxy_monitor_cache_clear || true
|
||||
mc_promotion_score_set -INFINITY || true
|
||||
ocf_exit_reason "managed-containerd delegated runtime ${delegate_runtime} does not match loaded RA runtime ${configured_runtime}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
fi
|
||||
|
||||
ocf_log info "managed-containerd delegated authoritative monitor: resource=${OCF_RESOURCE_INSTANCE:-unknown} backend=${delegate_backend} runtime=${delegate_runtime}"
|
||||
|
||||
mc_application_monitor
|
||||
monitor_rc=$?
|
||||
|
||||
case "${monitor_rc}" in
|
||||
"${OCF_SUCCESS}"| "${OCF_ERR_GENERIC}"| "${OCF_NOT_RUNNING}"| "${OCF_RUNNING_PROMOTED}")
|
||||
;;
|
||||
*)
|
||||
mc_proxy_monitor_cache_clear || true
|
||||
mc_promotion_score_set -INFINITY || true
|
||||
ocf_exit_reason "delegated application monitor returned invalid rc ${monitor_rc}"
|
||||
return "${OCF_ERR_GENERIC}"
|
||||
;;
|
||||
esac
|
||||
|
||||
# The delegated application result is now the authoritative
|
||||
# observation. Cache only healthy 0/8 results using the same
|
||||
# control-plane grace mechanism as daemon-native monitoring.
|
||||
mc_proxy_monitor_cache_write "${monitor_rc}" || {
|
||||
ocf_log warn "Unable to update delegated authoritative monitor cache"
|
||||
}
|
||||
|
||||
ocf_log info "managed-containerd delegated monitor result: resource=${OCF_RESOURCE_INSTANCE:-unknown} rc=${monitor_rc} backend=${delegate_backend} runtime=${delegate_runtime}"
|
||||
|
||||
mc_promotion_score_from_monitor "${monitor_rc}" || true
|
||||
return "${monitor_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
monitor_rc="${proxy_result%%|*}"
|
||||
|
||||
case "${monitor_rc}" in
|
||||
@@ -136,12 +193,9 @@ mc_promotion_monitor()
|
||||
|
||||
ocf_log info "managed-containerd authoritative monitor: resource=${OCF_RESOURCE_INSTANCE:-unknown} proxy=${proxy_result}"
|
||||
|
||||
# Legacy PostgreSQL monitoring remains observational only.
|
||||
mc_application_monitor
|
||||
legacy_rc=$?
|
||||
|
||||
mc_proxy_legacy_shadow_monitor "${monitor_rc}" "${legacy_rc}" || true
|
||||
|
||||
# The daemon-native monitor is authoritative. The packaged application
|
||||
# monitor is invoked only when managed-containerd explicitly delegates to
|
||||
# the container backend.
|
||||
mc_promotion_score_from_monitor "${monitor_rc}" || true
|
||||
return "${monitor_rc}"
|
||||
}
|
||||
|
||||
@@ -88,6 +88,16 @@ mc_proxy_monitor()
|
||||
return "${MC_PROXY_RC_HARD_FAILURE}"
|
||||
fi
|
||||
|
||||
# Delegated monitor execution is a valid control-plane response, but it is
|
||||
# not itself an OCF observation and must never be written to the healthy
|
||||
# role cache. The caller validates and executes the packaged runtime path.
|
||||
case "${response}" in
|
||||
delegate\|*)
|
||||
printf '%s\n' "${response}"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
rc="${response%%|*}"
|
||||
|
||||
# Cache only healthy daemon-authoritative role observations. An
|
||||
|
||||
@@ -117,6 +117,22 @@ def format_monitor_result(result):
|
||||
if not isinstance(result, dict):
|
||||
raise ProtocolFailure
|
||||
|
||||
if result.get("delegated") is True:
|
||||
backend = result.get("backend")
|
||||
runtime = result.get("container_runtime")
|
||||
|
||||
if backend != "container":
|
||||
raise ProtocolFailure
|
||||
|
||||
if runtime not in ("podman", "docker"):
|
||||
raise ProtocolFailure
|
||||
|
||||
return "|".join([
|
||||
"delegate",
|
||||
backend,
|
||||
runtime,
|
||||
])
|
||||
|
||||
ha = result.get("ha")
|
||||
|
||||
if not isinstance(ha, dict):
|
||||
|
||||
Executable
+174
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -u
|
||||
|
||||
OCF_SUCCESS=0
|
||||
OCF_ERR_GENERIC=1
|
||||
OCF_NOT_RUNNING=7
|
||||
OCF_RUNNING_PROMOTED=8
|
||||
|
||||
MC_PROXY_RC_TRANSPORT=10
|
||||
MC_PROXY_RC_HARD_FAILURE=11
|
||||
|
||||
OCF_RESOURCE_INSTANCE=postgresql-ha
|
||||
OCF_RESKEY_container_runtime=podman
|
||||
|
||||
logs=()
|
||||
exit_reason=""
|
||||
score=""
|
||||
cache_rc=""
|
||||
application_rc=0
|
||||
application_calls=0
|
||||
proxy_result_value="delegate|container|podman"
|
||||
|
||||
|
||||
# Load the real function under test first. Test doubles defined below then
|
||||
# override its external dependencies, including promotion-score helpers.
|
||||
source ocf/loxthelion/managed-container.d/common.sh
|
||||
|
||||
|
||||
ocf_log()
|
||||
{
|
||||
logs+=("$*")
|
||||
}
|
||||
|
||||
ocf_exit_reason()
|
||||
{
|
||||
exit_reason="$*"
|
||||
}
|
||||
|
||||
mc_proxy_monitor()
|
||||
{
|
||||
printf '%s\n' "${proxy_result_value}"
|
||||
return 0
|
||||
}
|
||||
|
||||
mc_proxy_cached_monitor()
|
||||
{
|
||||
return 1
|
||||
}
|
||||
|
||||
mc_proxy_monitor_cache_clear()
|
||||
{
|
||||
cache_rc="cleared"
|
||||
return 0
|
||||
}
|
||||
|
||||
mc_proxy_monitor_cache_write()
|
||||
{
|
||||
cache_rc="$1"
|
||||
return 0
|
||||
}
|
||||
|
||||
mc_proxy_legacy_shadow_monitor()
|
||||
{
|
||||
echo "ERROR: shadow monitor must not execute for delegation" >&2
|
||||
return 99
|
||||
}
|
||||
|
||||
mc_application_monitor()
|
||||
{
|
||||
application_calls=$((application_calls + 1))
|
||||
return "${application_rc}"
|
||||
}
|
||||
|
||||
mc_promotion_score_set()
|
||||
{
|
||||
score="$1"
|
||||
return 0
|
||||
}
|
||||
|
||||
mc_promotion_score_from_monitor()
|
||||
{
|
||||
case "$1" in
|
||||
0|8)
|
||||
score="healthy:$1"
|
||||
;;
|
||||
*)
|
||||
score="-INFINITY"
|
||||
;;
|
||||
esac
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
echo "===== PODMAN DELEGATION SUCCESS ====="
|
||||
|
||||
application_rc=0
|
||||
proxy_result_value="delegate|container|podman"
|
||||
cache_rc=""
|
||||
score=""
|
||||
application_calls=0
|
||||
|
||||
mc_promotion_monitor
|
||||
rc=$?
|
||||
|
||||
[ "$rc" -eq 0 ]
|
||||
[ "$application_calls" -eq 1 ]
|
||||
[ "$cache_rc" = 0 ]
|
||||
[ "$score" = "healthy:0" ]
|
||||
|
||||
echo "podman_success accepted"
|
||||
|
||||
|
||||
echo
|
||||
echo "===== PROMOTED DELEGATION ====="
|
||||
|
||||
application_rc=8
|
||||
proxy_result_value="delegate|container|podman"
|
||||
cache_rc=""
|
||||
score=""
|
||||
application_calls=0
|
||||
|
||||
mc_promotion_monitor
|
||||
rc=$?
|
||||
|
||||
[ "$rc" -eq 8 ]
|
||||
[ "$application_calls" -eq 1 ]
|
||||
[ "$cache_rc" = 8 ]
|
||||
[ "$score" = "healthy:8" ]
|
||||
|
||||
echo "podman_promoted accepted"
|
||||
|
||||
|
||||
echo
|
||||
echo "===== RUNTIME MISMATCH ====="
|
||||
|
||||
application_rc=0
|
||||
proxy_result_value="delegate|container|docker"
|
||||
cache_rc=""
|
||||
score=""
|
||||
application_calls=0
|
||||
|
||||
mc_promotion_monitor
|
||||
rc=$?
|
||||
|
||||
[ "$rc" -eq 1 ]
|
||||
[ "$application_calls" -eq 0 ]
|
||||
[ "$cache_rc" = "cleared" ]
|
||||
[ "$score" = "-INFINITY" ]
|
||||
|
||||
echo "runtime_mismatch rejected"
|
||||
|
||||
|
||||
echo
|
||||
echo "===== MALFORMED DELEGATION ====="
|
||||
|
||||
proxy_result_value="delegate|container|podman|extra"
|
||||
cache_rc=""
|
||||
score=""
|
||||
application_calls=0
|
||||
|
||||
mc_promotion_monitor
|
||||
rc=$?
|
||||
|
||||
[ "$rc" -eq 1 ]
|
||||
[ "$application_calls" -eq 0 ]
|
||||
[ "$cache_rc" = "cleared" ]
|
||||
[ "$score" = "-INFINITY" ]
|
||||
|
||||
echo "malformed_delegation rejected"
|
||||
|
||||
|
||||
echo
|
||||
echo "MONITOR_DELEGATION_SHELL_TEST_PASS=1"
|
||||
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"proxy_client",
|
||||
Path(
|
||||
"ocf/loxthelion/"
|
||||
"managed-container.d/"
|
||||
"proxy_client.py"
|
||||
),
|
||||
)
|
||||
|
||||
proxy = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(proxy)
|
||||
|
||||
|
||||
print("===== NORMAL API MONITOR RESULT =====")
|
||||
|
||||
normal = {
|
||||
"ha": {
|
||||
"ocf_rc": 0,
|
||||
"health": "healthy",
|
||||
"ha_role": "unpromoted",
|
||||
"state": "streaming",
|
||||
"promotion_eligible": True,
|
||||
},
|
||||
"role": "standby",
|
||||
"streaming": True,
|
||||
"wal_receiver": {
|
||||
"slot_name": "ha_pg_node1",
|
||||
"sender_host": "192.0.2.10",
|
||||
},
|
||||
}
|
||||
|
||||
formatted = proxy.format_monitor_result(normal)
|
||||
|
||||
expected = (
|
||||
"0|healthy|unpromoted|streaming|true|"
|
||||
"standby|true|ha_pg_node1|192.0.2.10"
|
||||
)
|
||||
|
||||
assert formatted == expected, formatted
|
||||
|
||||
print("api_monitor_format accepted")
|
||||
|
||||
|
||||
print()
|
||||
print("===== PODMAN DELEGATION =====")
|
||||
|
||||
delegated = {
|
||||
"delegated": True,
|
||||
"backend": "container",
|
||||
"container_runtime": "podman",
|
||||
}
|
||||
|
||||
formatted = proxy.format_monitor_result(delegated)
|
||||
|
||||
assert formatted == "delegate|container|podman", formatted
|
||||
|
||||
print("podman_delegation accepted")
|
||||
|
||||
|
||||
print()
|
||||
print("===== DOCKER DELEGATION =====")
|
||||
|
||||
delegated = {
|
||||
"delegated": True,
|
||||
"backend": "container",
|
||||
"container_runtime": "docker",
|
||||
}
|
||||
|
||||
formatted = proxy.format_monitor_result(delegated)
|
||||
|
||||
assert formatted == "delegate|container|docker", formatted
|
||||
|
||||
print("docker_delegation accepted")
|
||||
|
||||
|
||||
print()
|
||||
print("===== INVALID DELEGATED BACKEND =====")
|
||||
|
||||
bad = {
|
||||
"delegated": True,
|
||||
"backend": "api",
|
||||
"container_runtime": "podman",
|
||||
}
|
||||
|
||||
try:
|
||||
proxy.format_monitor_result(bad)
|
||||
except proxy.ProtocolFailure:
|
||||
print("invalid_backend rejected")
|
||||
else:
|
||||
raise AssertionError(
|
||||
"delegation with non-container backend was accepted"
|
||||
)
|
||||
|
||||
|
||||
print()
|
||||
print("===== INVALID DELEGATED RUNTIME =====")
|
||||
|
||||
bad = {
|
||||
"delegated": True,
|
||||
"backend": "container",
|
||||
"container_runtime": "lxc",
|
||||
}
|
||||
|
||||
try:
|
||||
proxy.format_monitor_result(bad)
|
||||
except proxy.ProtocolFailure:
|
||||
print("invalid_runtime rejected")
|
||||
else:
|
||||
raise AssertionError(
|
||||
"delegation with invalid runtime was accepted"
|
||||
)
|
||||
|
||||
|
||||
print()
|
||||
print("===== MALFORMED DELEGATION =====")
|
||||
|
||||
bad = {
|
||||
"delegated": True,
|
||||
"backend": "container",
|
||||
}
|
||||
|
||||
try:
|
||||
proxy.format_monitor_result(bad)
|
||||
except proxy.ProtocolFailure:
|
||||
print("missing_runtime rejected")
|
||||
else:
|
||||
raise AssertionError(
|
||||
"delegation without runtime was accepted"
|
||||
)
|
||||
|
||||
|
||||
print()
|
||||
print("PROXY_MONITOR_DELEGATION_TEST_PASS=1")
|
||||
@@ -150,6 +150,13 @@ with conn:
|
||||
elif mode == "bad_rc":
|
||||
response["result"]["ha"]["ocf_rc"] = 42
|
||||
|
||||
elif mode == "delegate":
|
||||
response["result"] = {
|
||||
"delegated": True,
|
||||
"backend": "container",
|
||||
"container_runtime": "podman",
|
||||
}
|
||||
|
||||
conn.sendall(
|
||||
json.dumps(response).encode("utf-8") + b"\n"
|
||||
)
|
||||
@@ -248,6 +255,7 @@ PY_REFUSED
|
||||
echo "===== PROXY PROTOCOL CLASSIFICATION ====="
|
||||
|
||||
run_case valid valid 0
|
||||
run_case delegation delegate 0
|
||||
run_case missing_protocol missing_protocol 11
|
||||
run_case wrong_protocol wrong_protocol 11
|
||||
run_case wrong_resource wrong_resource 11
|
||||
|
||||
Reference in New Issue
Block a user