Initial managed-container HA resource agent baseline
This commit is contained in:
Lyncoln
Executable
+464
@@ -0,0 +1,464 @@
|
||||
# Generic managed-containerd client helpers.
|
||||
# Shadow calls are observational only and must never affect OCF action results.
|
||||
|
||||
MC_PROXY_SOCKET_DEFAULT="/run/user/1000/managed-containerd/managed-containerd.sock"
|
||||
|
||||
mc_proxy_shadow_monitor()
|
||||
{
|
||||
local authoritative_rc="$1"
|
||||
local socket_path="${MC_PROXY_SOCKET:-${MC_PROXY_SOCKET_DEFAULT}}"
|
||||
local resource="${OCF_RESOURCE_INSTANCE:-}"
|
||||
local response
|
||||
|
||||
resource="${resource%%:*}"
|
||||
|
||||
[ -n "${resource}" ] || return 0
|
||||
[ -S "${socket_path}" ] || return 0
|
||||
command -v python3 >/dev/null 2>&1 || return 0
|
||||
|
||||
response="$(
|
||||
timeout --kill-after=1s 3s python3 - "${socket_path}" "${resource}" "${authoritative_rc}" <<'PY'
|
||||
import json
|
||||
import socket
|
||||
import sys
|
||||
|
||||
path, resource, authoritative_raw = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
authoritative_rc = int(authoritative_raw)
|
||||
request = {"action": "monitor", "resource": resource}
|
||||
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.settimeout(2.0)
|
||||
s.connect(path)
|
||||
s.sendall(json.dumps(request).encode("utf-8") + b"\n")
|
||||
|
||||
raw = b""
|
||||
while b"\n" not in raw:
|
||||
chunk = s.recv(65536)
|
||||
if not chunk:
|
||||
break
|
||||
raw += chunk
|
||||
s.close()
|
||||
|
||||
obj = json.loads(raw.decode("utf-8").strip())
|
||||
if not obj.get("ok"):
|
||||
print("ERROR|" + str(obj.get("error", "unknown proxy error")))
|
||||
raise SystemExit(0)
|
||||
|
||||
result = obj.get("result") or {}
|
||||
ha = result.get("ha") or {}
|
||||
proxy_rc = ha.get("ocf_rc")
|
||||
|
||||
if proxy_rc is None:
|
||||
comparison = "UNCLASSIFIED"
|
||||
else:
|
||||
comparison = "MATCH" if int(proxy_rc) == authoritative_rc else "MISMATCH"
|
||||
|
||||
receiver = result.get("wal_receiver") or {}
|
||||
print("OK|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}".format(
|
||||
comparison,
|
||||
"null" if proxy_rc is None else proxy_rc,
|
||||
ha.get("health", "unknown"),
|
||||
ha.get("ha_role", "unknown"),
|
||||
ha.get("state", "unknown"),
|
||||
str(ha.get("promotion_eligible", "unknown")).lower(),
|
||||
result.get("role", "unknown"),
|
||||
str(result.get("streaming", "unknown")).lower(),
|
||||
receiver.get("slot_name") or "",
|
||||
receiver.get("sender_host") or "",
|
||||
))
|
||||
PY
|
||||
)" || response="TRANSPORT_ERROR"
|
||||
|
||||
case "${response}" in
|
||||
OK\|MATCH\|*)
|
||||
ocf_log info \
|
||||
"managed-containerd shadow monitor: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}"
|
||||
;;
|
||||
OK\|UNCLASSIFIED\|*)
|
||||
ocf_log warn \
|
||||
"managed-containerd shadow monitor unclassified: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}"
|
||||
;;
|
||||
OK\|MISMATCH\|*)
|
||||
ocf_log err \
|
||||
"managed-containerd shadow monitor MISMATCH: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}"
|
||||
;;
|
||||
ERROR\|*)
|
||||
ocf_log warn \
|
||||
"managed-containerd shadow monitor failed: resource=${resource} authoritative_rc=${authoritative_rc} proxy=${response}"
|
||||
;;
|
||||
*)
|
||||
ocf_log warn \
|
||||
"managed-containerd shadow monitor transport failure: resource=${resource} authoritative_rc=${authoritative_rc}"
|
||||
;;
|
||||
esac
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
# Query managed-containerd for the authoritative monitor result.
|
||||
#
|
||||
# stdout:
|
||||
# <OCF rc>|<normalized diagnostic payload>
|
||||
#
|
||||
# return:
|
||||
# 0 valid classified daemon result
|
||||
# MC_PROXY_RC_TRANSPORT daemon transport/control-plane failure; bounded cache grace allowed
|
||||
# MC_PROXY_RC_HARD_FAILURE configuration/protocol/result failure; fail closed, no cache grace
|
||||
mc_proxy_monitor()
|
||||
{
|
||||
local socket_path="${MC_PROXY_SOCKET:-${MC_PROXY_SOCKET_DEFAULT}}"
|
||||
local resource="${OCF_RESOURCE_INSTANCE:-}"
|
||||
local response rc
|
||||
|
||||
resource="${resource%%:*}"
|
||||
|
||||
if [ -z "${resource}" ]; then
|
||||
ocf_log err \
|
||||
"managed-containerd monitor requires OCF_RESOURCE_INSTANCE"
|
||||
return "${MC_PROXY_RC_HARD_FAILURE}"
|
||||
fi
|
||||
|
||||
if [ ! -S "${socket_path}" ]; then
|
||||
ocf_log warn \
|
||||
"managed-containerd socket is unavailable: ${socket_path}"
|
||||
return "${MC_PROXY_RC_TRANSPORT}"
|
||||
fi
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
ocf_log err \
|
||||
"python3 is required for managed-containerd monitor transport"
|
||||
return "${MC_PROXY_RC_HARD_FAILURE}"
|
||||
fi
|
||||
|
||||
response="$(
|
||||
timeout --kill-after=1s 3s \
|
||||
python3 - \
|
||||
"${socket_path}" \
|
||||
"${resource}" \
|
||||
"${MC_PROXY_RC_TRANSPORT}" \
|
||||
"${MC_PROXY_RC_HARD_FAILURE}" <<'PY'
|
||||
import json
|
||||
import socket
|
||||
import sys
|
||||
|
||||
path, resource = sys.argv[1], sys.argv[2]
|
||||
TRANSPORT_FAILURE = int(sys.argv[3])
|
||||
PROTOCOL_FAILURE = int(sys.argv[4])
|
||||
|
||||
request = {
|
||||
"action": "monitor",
|
||||
"resource": resource,
|
||||
}
|
||||
|
||||
try:
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.settimeout(2.0)
|
||||
|
||||
try:
|
||||
s.connect(path)
|
||||
s.sendall(json.dumps(request).encode("utf-8") + b"\n")
|
||||
|
||||
raw = b""
|
||||
|
||||
while b"\n" not in raw:
|
||||
chunk = s.recv(65536)
|
||||
|
||||
if not chunk:
|
||||
break
|
||||
|
||||
raw += chunk
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
except (socket.timeout, ConnectionError, OSError):
|
||||
# Local daemon/control-plane reachability failure.
|
||||
# Recent authoritative state may be used for bounded grace.
|
||||
raise SystemExit(TRANSPORT_FAILURE)
|
||||
|
||||
# Transport succeeded. Invalid daemon output is therefore a
|
||||
# protocol/result failure and must never be hidden by cached health.
|
||||
if not raw:
|
||||
raise SystemExit(PROTOCOL_FAILURE)
|
||||
|
||||
try:
|
||||
obj = json.loads(raw.decode("utf-8").strip())
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
raise SystemExit(PROTOCOL_FAILURE)
|
||||
|
||||
if not isinstance(obj, dict):
|
||||
raise SystemExit(PROTOCOL_FAILURE)
|
||||
|
||||
if not obj.get("ok"):
|
||||
raise SystemExit(PROTOCOL_FAILURE)
|
||||
|
||||
result = obj.get("result")
|
||||
if not isinstance(result, dict):
|
||||
raise SystemExit(PROTOCOL_FAILURE)
|
||||
|
||||
ha = result.get("ha")
|
||||
if not isinstance(ha, dict):
|
||||
raise SystemExit(PROTOCOL_FAILURE)
|
||||
|
||||
rc = ha.get("ocf_rc")
|
||||
|
||||
try:
|
||||
rc = int(rc)
|
||||
except (TypeError, ValueError):
|
||||
raise SystemExit(PROTOCOL_FAILURE)
|
||||
|
||||
if rc not in (0, 1, 7, 8):
|
||||
raise SystemExit(PROTOCOL_FAILURE)
|
||||
|
||||
receiver = result.get("wal_receiver") or {}
|
||||
|
||||
if not isinstance(receiver, dict):
|
||||
raise SystemExit(PROTOCOL_FAILURE)
|
||||
|
||||
payload = "|".join([
|
||||
str(rc),
|
||||
str(ha.get("health", "unknown")),
|
||||
str(ha.get("ha_role", "unknown")),
|
||||
str(ha.get("state", "unknown")),
|
||||
str(ha.get("promotion_eligible", "unknown")).lower(),
|
||||
str(result.get("role", "unknown")),
|
||||
str(result.get("streaming", "unknown")).lower(),
|
||||
str(receiver.get("slot_name") or ""),
|
||||
str(receiver.get("sender_host") or ""),
|
||||
])
|
||||
|
||||
print(payload)
|
||||
PY
|
||||
)"
|
||||
rc=$?
|
||||
|
||||
case "${rc}" in
|
||||
0)
|
||||
;;
|
||||
|
||||
"${MC_PROXY_RC_TRANSPORT}"|124|137)
|
||||
# MC_PROXY_RC_TRANSPORT = classified Python transport failure.
|
||||
# 124/137 = timeout(1) expiry / forced termination.
|
||||
ocf_log warn \
|
||||
"managed-containerd authoritative monitor transport failure"
|
||||
return "${MC_PROXY_RC_TRANSPORT}"
|
||||
;;
|
||||
|
||||
"${MC_PROXY_RC_HARD_FAILURE}")
|
||||
ocf_log err \
|
||||
"managed-containerd authoritative monitor protocol or result failure"
|
||||
return "${MC_PROXY_RC_HARD_FAILURE}"
|
||||
;;
|
||||
|
||||
*)
|
||||
ocf_log err \
|
||||
"managed-containerd authoritative monitor client failure rc=${rc}"
|
||||
return "${MC_PROXY_RC_HARD_FAILURE}"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "${response}" ]; then
|
||||
ocf_log err \
|
||||
"managed-containerd authoritative monitor returned an empty result"
|
||||
return "${MC_PROXY_RC_HARD_FAILURE}"
|
||||
fi
|
||||
|
||||
rc="${response%%|*}"
|
||||
|
||||
# Cache only healthy daemon-authoritative role observations. An
|
||||
# application-level failure automatically invalidates any previous cache.
|
||||
mc_proxy_monitor_cache_write "${rc}" || {
|
||||
ocf_log warn \
|
||||
"Unable to update managed-containerd authoritative monitor cache"
|
||||
}
|
||||
|
||||
printf '%s\n' "${response}"
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
# Compare the legacy application monitor against a daemon-authoritative rc.
|
||||
# The legacy result is diagnostic only and can never change the returned
|
||||
# Pacemaker monitor status.
|
||||
mc_proxy_legacy_shadow_monitor()
|
||||
{
|
||||
local authoritative_rc="$1"
|
||||
local legacy_rc="$2"
|
||||
local resource="${OCF_RESOURCE_INSTANCE:-}"
|
||||
|
||||
resource="${resource%%:*}"
|
||||
|
||||
if [ "${legacy_rc}" -eq "${authoritative_rc}" ]; then
|
||||
ocf_log info \
|
||||
"legacy application shadow monitor: resource=${resource} authoritative_rc=${authoritative_rc} legacy_rc=${legacy_rc} comparison=MATCH"
|
||||
else
|
||||
ocf_log err \
|
||||
"legacy application shadow monitor MISMATCH: resource=${resource} authoritative_rc=${authoritative_rc} legacy_rc=${legacy_rc}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
# Generic short-lived cache of the last healthy daemon-authoritative OCF role.
|
||||
# This is control-plane state only; it contains no application-specific data.
|
||||
#
|
||||
# Use HA_RSCTMP when available because it is the standard resource-agent
|
||||
# runtime location and is preferable to inventing a persistent filesystem path.
|
||||
MC_PROXY_STATE_DIR_DEFAULT="${HA_RSCTMP:-/run/resource-agents}/managed-container"
|
||||
|
||||
|
||||
mc_proxy_state_file()
|
||||
{
|
||||
local resource safe_resource
|
||||
resource="${OCF_RESOURCE_INSTANCE:-}"
|
||||
resource="${resource%%:*}"
|
||||
|
||||
[ -n "${resource}" ] || return 1
|
||||
|
||||
safe_resource="$(
|
||||
printf '%s' "${resource}" |
|
||||
tr -c 'A-Za-z0-9_.-' '_'
|
||||
)"
|
||||
|
||||
printf '%s/%s.monitor\n' \
|
||||
"${MC_PROXY_STATE_DIR:-${MC_PROXY_STATE_DIR_DEFAULT}}" \
|
||||
"${safe_resource}"
|
||||
}
|
||||
|
||||
|
||||
mc_proxy_monitor_cache_clear()
|
||||
{
|
||||
local state_file
|
||||
|
||||
state_file="$(mc_proxy_state_file)" || return 0
|
||||
rm -f "${state_file}" 2>/dev/null || true
|
||||
}
|
||||
|
||||
|
||||
mc_proxy_monitor_cache_write()
|
||||
{
|
||||
local rc="$1"
|
||||
local state_file state_dir tmp now
|
||||
|
||||
# Only healthy authoritative role observations are eligible for
|
||||
# temporary control-plane grace.
|
||||
case "${rc}" in
|
||||
"${OCF_SUCCESS}"|"${OCF_RUNNING_PROMOTED}")
|
||||
;;
|
||||
*)
|
||||
mc_proxy_monitor_cache_clear
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
state_file="$(mc_proxy_state_file)" || return 1
|
||||
state_dir="${state_file%/*}"
|
||||
|
||||
mkdir -p -m 700 "${state_dir}" || return 1
|
||||
|
||||
now="$(date +%s)"
|
||||
tmp="${state_file}.$$"
|
||||
|
||||
umask 077
|
||||
printf '%s|%s|0
|
||||
' "${now}" "${rc}" > "${tmp}" || {
|
||||
rm -f "${tmp}" 2>/dev/null || true
|
||||
return 1
|
||||
}
|
||||
|
||||
chmod 600 "${tmp}" 2>/dev/null || true
|
||||
mv -f "${tmp}" "${state_file}"
|
||||
}
|
||||
|
||||
|
||||
# Print a cached healthy OCF rc when its observation is still inside the
|
||||
# configured daemon control-plane failure grace.
|
||||
mc_proxy_cached_monitor()
|
||||
{
|
||||
local state_file observed rc failure_started
|
||||
local now failure_age observation_age grace max_observation_age
|
||||
local tmp
|
||||
|
||||
grace="${OCF_RESKEY_proxy_failure_grace_seconds:-15}"
|
||||
|
||||
case "${grace}" in
|
||||
""|*[!0-9]*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
[ "${grace}" -gt 0 ] || return 1
|
||||
|
||||
state_file="$(mc_proxy_state_file)" || return 1
|
||||
[ -f "${state_file}" ] || return 1
|
||||
|
||||
IFS='|' read -r observed rc failure_started < "${state_file}" || {
|
||||
mc_proxy_monitor_cache_clear
|
||||
return 1
|
||||
}
|
||||
|
||||
case "${observed}" in
|
||||
""|*[!0-9]*)
|
||||
mc_proxy_monitor_cache_clear
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "${failure_started}" in
|
||||
""|*[!0-9]*)
|
||||
mc_proxy_monitor_cache_clear
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "${rc}" in
|
||||
"${OCF_SUCCESS}"|"${OCF_RUNNING_PROMOTED}")
|
||||
;;
|
||||
*)
|
||||
mc_proxy_monitor_cache_clear
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
now="$(date +%s)"
|
||||
observation_age=$((now - observed))
|
||||
|
||||
# A cached role must come from a reasonably recent successful daemon
|
||||
# observation. This protects against resurrecting stale runtime state.
|
||||
max_observation_age=$((grace + 60))
|
||||
|
||||
if [ "${observation_age}" -lt 0 ] ||
|
||||
[ "${observation_age}" -gt "${max_observation_age}" ]; then
|
||||
mc_proxy_monitor_cache_clear
|
||||
return 1
|
||||
fi
|
||||
|
||||
# First failed daemon observation starts the bounded grace period.
|
||||
if [ "${failure_started}" -eq 0 ]; then
|
||||
failure_started="${now}"
|
||||
tmp="${state_file}.$$"
|
||||
|
||||
umask 077
|
||||
printf '%s|%s|%s
|
||||
' "${observed}" "${rc}" "${failure_started}" > "${tmp}" || {
|
||||
rm -f "${tmp}" 2>/dev/null || true
|
||||
return 1
|
||||
}
|
||||
|
||||
chmod 600 "${tmp}" 2>/dev/null || true
|
||||
mv -f "${tmp}" "${state_file}" || return 1
|
||||
fi
|
||||
|
||||
failure_age=$((now - failure_started))
|
||||
|
||||
if [ "${failure_age}" -lt 0 ] ||
|
||||
[ "${failure_age}" -ge "${grace}" ]; then
|
||||
mc_proxy_monitor_cache_clear
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf '%s
|
||||
' "${rc}"
|
||||
return 0
|
||||
}
|
||||
Reference in New Issue
Block a user