465 lines
13 KiB
Bash
Executable File
465 lines
13 KiB
Bash
Executable File
# Generic managed-containerd client helpers.
|
|
# Shadow calls are observational only and must never affect OCF action results.
|
|
|
|
MC_PROXY_SOCKET_DEFAULT="/run/user/1000/managed-containerd/managed-containerd.sock"
|
|
|
|
mc_proxy_shadow_monitor()
|
|
{
|
|
local authoritative_rc="$1"
|
|
local socket_path="${MC_PROXY_SOCKET:-${MC_PROXY_SOCKET_DEFAULT}}"
|
|
local resource="${OCF_RESOURCE_INSTANCE:-}"
|
|
local response
|
|
|
|
resource="${resource%%:*}"
|
|
|
|
[ -n "${resource}" ] || return 0
|
|
[ -S "${socket_path}" ] || return 0
|
|
command -v python3 >/dev/null 2>&1 || return 0
|
|
|
|
response="$(
|
|
timeout --kill-after=1s 3s python3 - "${socket_path}" "${resource}" "${authoritative_rc}" <<'PY'
|
|
import json
|
|
import socket
|
|
import sys
|
|
|
|
path, resource, authoritative_raw = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
authoritative_rc = int(authoritative_raw)
|
|
request = {"action": "monitor", "resource": resource}
|
|
|
|
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
s.settimeout(2.0)
|
|
s.connect(path)
|
|
s.sendall(json.dumps(request).encode("utf-8") + b"\n")
|
|
|
|
raw = b""
|
|
while b"\n" not in raw:
|
|
chunk = s.recv(65536)
|
|
if not chunk:
|
|
break
|
|
raw += chunk
|
|
s.close()
|
|
|
|
obj = json.loads(raw.decode("utf-8").strip())
|
|
if not obj.get("ok"):
|
|
print("ERROR|" + str(obj.get("error", "unknown proxy error")))
|
|
raise SystemExit(0)
|
|
|
|
result = obj.get("result") or {}
|
|
ha = result.get("ha") or {}
|
|
proxy_rc = ha.get("ocf_rc")
|
|
|
|
if proxy_rc is None:
|
|
comparison = "UNCLASSIFIED"
|
|
else:
|
|
comparison = "MATCH" if int(proxy_rc) == authoritative_rc else "MISMATCH"
|
|
|
|
receiver = result.get("wal_receiver") or {}
|
|
print("OK|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}".format(
|
|
comparison,
|
|
"null" if proxy_rc is None else proxy_rc,
|
|
ha.get("health", "unknown"),
|
|
ha.get("ha_role", "unknown"),
|
|
ha.get("state", "unknown"),
|
|
str(ha.get("promotion_eligible", "unknown")).lower(),
|
|
result.get("role", "unknown"),
|
|
str(result.get("streaming", "unknown")).lower(),
|
|
receiver.get("slot_name") or "",
|
|
receiver.get("sender_host") or "",
|
|
))
|
|
PY
|
|
)" || response="TRANSPORT_ERROR"
|
|
|
|
case "${response}" in
|
|
OK\|MATCH\|*)
|
|
ocf_log info \
|
|
"managed-containerd shadow monitor: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}"
|
|
;;
|
|
OK\|UNCLASSIFIED\|*)
|
|
ocf_log warn \
|
|
"managed-containerd shadow monitor unclassified: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}"
|
|
;;
|
|
OK\|MISMATCH\|*)
|
|
ocf_log err \
|
|
"managed-containerd shadow monitor MISMATCH: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}"
|
|
;;
|
|
ERROR\|*)
|
|
ocf_log warn \
|
|
"managed-containerd shadow monitor failed: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}"
|
|
;;
|
|
*)
|
|
ocf_log warn \
|
|
"managed-containerd shadow monitor transport failure: resource=${resource} authoritative_rc=${authoritative_rc}"
|
|
;;
|
|
esac
|
|
|
|
return 0
|
|
}
|
|
|
|
|
|
# Query managed-containerd for the authoritative monitor result.
|
|
#
|
|
# stdout:
|
|
# <OCF rc>|<normalized diagnostic payload>
|
|
#
|
|
# return:
|
|
# 0 valid classified daemon result
|
|
# MC_PROXY_RC_TRANSPORT daemon transport/control-plane failure; bounded cache grace allowed
|
|
# MC_PROXY_RC_HARD_FAILURE configuration/protocol/result failure; fail closed, no cache grace
|
|
mc_proxy_monitor()
|
|
{
|
|
local socket_path="${MC_PROXY_SOCKET:-${MC_PROXY_SOCKET_DEFAULT}}"
|
|
local resource="${OCF_RESOURCE_INSTANCE:-}"
|
|
local response rc
|
|
|
|
resource="${resource%%:*}"
|
|
|
|
if [ -z "${resource}" ]; then
|
|
ocf_log err \
|
|
"managed-containerd monitor requires OCF_RESOURCE_INSTANCE"
|
|
return "${MC_PROXY_RC_HARD_FAILURE}"
|
|
fi
|
|
|
|
if [ ! -S "${socket_path}" ]; then
|
|
ocf_log warn \
|
|
"managed-containerd socket is unavailable: ${socket_path}"
|
|
return "${MC_PROXY_RC_TRANSPORT}"
|
|
fi
|
|
|
|
if ! command -v python3 >/dev/null 2>&1; then
|
|
ocf_log err \
|
|
"python3 is required for managed-containerd monitor transport"
|
|
return "${MC_PROXY_RC_HARD_FAILURE}"
|
|
fi
|
|
|
|
response="$(
|
|
timeout --kill-after=1s 3s \
|
|
python3 - \
|
|
"${socket_path}" \
|
|
"${resource}" \
|
|
"${MC_PROXY_RC_TRANSPORT}" \
|
|
"${MC_PROXY_RC_HARD_FAILURE}" <<'PY'
|
|
import json
|
|
import socket
|
|
import sys
|
|
|
|
path, resource = sys.argv[1], sys.argv[2]
|
|
TRANSPORT_FAILURE = int(sys.argv[3])
|
|
PROTOCOL_FAILURE = int(sys.argv[4])
|
|
|
|
request = {
|
|
"action": "monitor",
|
|
"resource": resource,
|
|
}
|
|
|
|
try:
|
|
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
s.settimeout(2.0)
|
|
|
|
try:
|
|
s.connect(path)
|
|
s.sendall(json.dumps(request).encode("utf-8") + b"\n")
|
|
|
|
raw = b""
|
|
|
|
while b"\n" not in raw:
|
|
chunk = s.recv(65536)
|
|
|
|
if not chunk:
|
|
break
|
|
|
|
raw += chunk
|
|
finally:
|
|
s.close()
|
|
|
|
except (socket.timeout, ConnectionError, OSError):
|
|
# Local daemon/control-plane reachability failure.
|
|
# Recent authoritative state may be used for bounded grace.
|
|
raise SystemExit(TRANSPORT_FAILURE)
|
|
|
|
# Transport succeeded. Invalid daemon output is therefore a
|
|
# protocol/result failure and must never be hidden by cached health.
|
|
if not raw:
|
|
raise SystemExit(PROTOCOL_FAILURE)
|
|
|
|
try:
|
|
obj = json.loads(raw.decode("utf-8").strip())
|
|
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
raise SystemExit(PROTOCOL_FAILURE)
|
|
|
|
if not isinstance(obj, dict):
|
|
raise SystemExit(PROTOCOL_FAILURE)
|
|
|
|
if not obj.get("ok"):
|
|
raise SystemExit(PROTOCOL_FAILURE)
|
|
|
|
result = obj.get("result")
|
|
if not isinstance(result, dict):
|
|
raise SystemExit(PROTOCOL_FAILURE)
|
|
|
|
ha = result.get("ha")
|
|
if not isinstance(ha, dict):
|
|
raise SystemExit(PROTOCOL_FAILURE)
|
|
|
|
rc = ha.get("ocf_rc")
|
|
|
|
try:
|
|
rc = int(rc)
|
|
except (TypeError, ValueError):
|
|
raise SystemExit(PROTOCOL_FAILURE)
|
|
|
|
if rc not in (0, 1, 7, 8):
|
|
raise SystemExit(PROTOCOL_FAILURE)
|
|
|
|
receiver = result.get("wal_receiver") or {}
|
|
|
|
if not isinstance(receiver, dict):
|
|
raise SystemExit(PROTOCOL_FAILURE)
|
|
|
|
payload = "|".join([
|
|
str(rc),
|
|
str(ha.get("health", "unknown")),
|
|
str(ha.get("ha_role", "unknown")),
|
|
str(ha.get("state", "unknown")),
|
|
str(ha.get("promotion_eligible", "unknown")).lower(),
|
|
str(result.get("role", "unknown")),
|
|
str(result.get("streaming", "unknown")).lower(),
|
|
str(receiver.get("slot_name") or ""),
|
|
str(receiver.get("sender_host") or ""),
|
|
])
|
|
|
|
print(payload)
|
|
PY
|
|
)"
|
|
rc=$?
|
|
|
|
case "${rc}" in
|
|
0)
|
|
;;
|
|
|
|
"${MC_PROXY_RC_TRANSPORT}"|124|137)
|
|
# MC_PROXY_RC_TRANSPORT = classified Python transport failure.
|
|
# 124/137 = timeout(1) expiry / forced termination.
|
|
ocf_log warn \
|
|
"managed-containerd authoritative monitor transport failure"
|
|
return "${MC_PROXY_RC_TRANSPORT}"
|
|
;;
|
|
|
|
"${MC_PROXY_RC_HARD_FAILURE}")
|
|
ocf_log err \
|
|
"managed-containerd authoritative monitor protocol or result failure"
|
|
return "${MC_PROXY_RC_HARD_FAILURE}"
|
|
;;
|
|
|
|
*)
|
|
ocf_log err \
|
|
"managed-containerd authoritative monitor client failure rc=${rc}"
|
|
return "${MC_PROXY_RC_HARD_FAILURE}"
|
|
;;
|
|
esac
|
|
|
|
if [ -z "${response}" ]; then
|
|
ocf_log err \
|
|
"managed-containerd authoritative monitor returned an empty result"
|
|
return "${MC_PROXY_RC_HARD_FAILURE}"
|
|
fi
|
|
|
|
rc="${response%%|*}"
|
|
|
|
# Cache only healthy daemon-authoritative role observations. An
|
|
# application-level failure automatically invalidates any previous cache.
|
|
mc_proxy_monitor_cache_write "${rc}" || {
|
|
ocf_log warn \
|
|
"Unable to update managed-containerd authoritative monitor cache"
|
|
}
|
|
|
|
printf '%s\n' "${response}"
|
|
return 0
|
|
}
|
|
|
|
|
|
# Compare the legacy application monitor against a daemon-authoritative rc.
|
|
# The legacy result is diagnostic only and can never change the returned
|
|
# Pacemaker monitor status.
|
|
mc_proxy_legacy_shadow_monitor()
|
|
{
|
|
local authoritative_rc="$1"
|
|
local legacy_rc="$2"
|
|
local resource="${OCF_RESOURCE_INSTANCE:-}"
|
|
|
|
resource="${resource%%:*}"
|
|
|
|
if [ "${legacy_rc}" -eq "${authoritative_rc}" ]; then
|
|
ocf_log info \
|
|
"legacy application shadow monitor: resource=${resource} authoritative_rc=${authoritative_rc} legacy_rc=${legacy_rc} comparison=MATCH"
|
|
else
|
|
ocf_log err \
|
|
"legacy application shadow monitor MISMATCH: resource=${resource} authoritative_rc=${authoritative_rc} legacy_rc=${legacy_rc}"
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
|
|
# Generic short-lived cache of the last healthy daemon-authoritative OCF role.
|
|
# This is control-plane state only; it contains no application-specific data.
|
|
#
|
|
# Use HA_RSCTMP when available because it is the standard resource-agent
|
|
# runtime location and is preferable to inventing a persistent filesystem path.
|
|
MC_PROXY_STATE_DIR_DEFAULT="${HA_RSCTMP:-/run/resource-agents}/managed-container"
|
|
|
|
|
|
mc_proxy_state_file()
|
|
{
|
|
local resource safe_resource
|
|
resource="${OCF_RESOURCE_INSTANCE:-}"
|
|
resource="${resource%%:*}"
|
|
|
|
[ -n "${resource}" ] || return 1
|
|
|
|
safe_resource="$(
|
|
printf '%s' "${resource}" |
|
|
tr -c 'A-Za-z0-9_.-' '_'
|
|
)"
|
|
|
|
printf '%s/%s.monitor\n' \
|
|
"${MC_PROXY_STATE_DIR:-${MC_PROXY_STATE_DIR_DEFAULT}}" \
|
|
"${safe_resource}"
|
|
}
|
|
|
|
|
|
mc_proxy_monitor_cache_clear()
|
|
{
|
|
local state_file
|
|
|
|
state_file="$(mc_proxy_state_file)" || return 0
|
|
rm -f "${state_file}" 2>/dev/null || true
|
|
}
|
|
|
|
|
|
mc_proxy_monitor_cache_write()
|
|
{
|
|
local rc="$1"
|
|
local state_file state_dir tmp now
|
|
|
|
# Only healthy authoritative role observations are eligible for
|
|
# temporary control-plane grace.
|
|
case "${rc}" in
|
|
"${OCF_SUCCESS}"|"${OCF_RUNNING_PROMOTED}")
|
|
;;
|
|
*)
|
|
mc_proxy_monitor_cache_clear
|
|
return 0
|
|
;;
|
|
esac
|
|
|
|
state_file="$(mc_proxy_state_file)" || return 1
|
|
state_dir="${state_file%/*}"
|
|
|
|
mkdir -p -m 700 "${state_dir}" || return 1
|
|
|
|
now="$(date +%s)"
|
|
tmp="${state_file}.$$"
|
|
|
|
umask 077
|
|
printf '%s|%s|0
|
|
' "${now}" "${rc}" > "${tmp}" || {
|
|
rm -f "${tmp}" 2>/dev/null || true
|
|
return 1
|
|
}
|
|
|
|
chmod 600 "${tmp}" 2>/dev/null || true
|
|
mv -f "${tmp}" "${state_file}"
|
|
}
|
|
|
|
|
|
# Print a cached healthy OCF rc when its observation is still inside the
|
|
# configured daemon control-plane failure grace.
|
|
mc_proxy_cached_monitor()
|
|
{
|
|
local state_file observed rc failure_started
|
|
local now failure_age observation_age grace max_observation_age
|
|
local tmp
|
|
|
|
grace="${OCF_RESKEY_proxy_failure_grace_seconds:-15}"
|
|
|
|
case "${grace}" in
|
|
""|*[!0-9]*)
|
|
return 1
|
|
;;
|
|
esac
|
|
|
|
[ "${grace}" -gt 0 ] || return 1
|
|
|
|
state_file="$(mc_proxy_state_file)" || return 1
|
|
[ -f "${state_file}" ] || return 1
|
|
|
|
IFS='|' read -r observed rc failure_started < "${state_file}" || {
|
|
mc_proxy_monitor_cache_clear
|
|
return 1
|
|
}
|
|
|
|
case "${observed}" in
|
|
""|*[!0-9]*)
|
|
mc_proxy_monitor_cache_clear
|
|
return 1
|
|
;;
|
|
esac
|
|
|
|
case "${failure_started}" in
|
|
""|*[!0-9]*)
|
|
mc_proxy_monitor_cache_clear
|
|
return 1
|
|
;;
|
|
esac
|
|
|
|
case "${rc}" in
|
|
"${OCF_SUCCESS}"|"${OCF_RUNNING_PROMOTED}")
|
|
;;
|
|
*)
|
|
mc_proxy_monitor_cache_clear
|
|
return 1
|
|
;;
|
|
esac
|
|
|
|
now="$(date +%s)"
|
|
observation_age=$((now - observed))
|
|
|
|
# A cached role must come from a reasonably recent successful daemon
|
|
# observation. This protects against resurrecting stale runtime state.
|
|
max_observation_age=$((grace + 60))
|
|
|
|
if [ "${observation_age}" -lt 0 ] ||
|
|
[ "${observation_age}" -gt "${max_observation_age}" ]; then
|
|
mc_proxy_monitor_cache_clear
|
|
return 1
|
|
fi
|
|
|
|
# First failed daemon observation starts the bounded grace period.
|
|
if [ "${failure_started}" -eq 0 ]; then
|
|
failure_started="${now}"
|
|
tmp="${state_file}.$$"
|
|
|
|
umask 077
|
|
printf '%s|%s|%s
|
|
' "${observed}" "${rc}" "${failure_started}" > "${tmp}" || {
|
|
rm -f "${tmp}" 2>/dev/null || true
|
|
return 1
|
|
}
|
|
|
|
chmod 600 "${tmp}" 2>/dev/null || true
|
|
mv -f "${tmp}" "${state_file}" || return 1
|
|
fi
|
|
|
|
failure_age=$((now - failure_started))
|
|
|
|
if [ "${failure_age}" -lt 0 ] ||
|
|
[ "${failure_age}" -ge "${grace}" ]; then
|
|
mc_proxy_monitor_cache_clear
|
|
return 1
|
|
fi
|
|
|
|
printf '%s
|
|
' "${rc}"
|
|
return 0
|
|
}
|