david/dns-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Recover interrupted certificate replacement

Browse Source
This commit is contained in:
Lyncoln
2026-09-20 01:54:30 -04:00
parent 2668651a98
commit 387332b5db
5 changed files with 658 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dns_sync/adapters.py
+32
View File
@@ -252,3 +252,35 @@ class PlanAction:
class EndpointPlan:
endpoint_id: str
actions: tuple[PlanAction, ...]
retirements: tuple[RetireRequest, ...] = ()
def __post_init__(self) -> None:
retirements = tuple(self.retirements)
if any(
not isinstance(retirement, RetireRequest)
for retirement in retirements
):
raise ValueError(
"EndpointPlan retirements must be "
"RetireRequest instances"
)
identities = [
(
retirement.object.object_type,
retirement.object.local_key,
)
for retirement in retirements
]
if len(set(identities)) != len(identities):
raise ValueError(
"EndpointPlan retirements cannot contain duplicates"
)
object.__setattr__(
self,
"retirements",
retirements,
)
dns_sync/executor.py
+54 -3
View File
@@ -321,6 +321,7 @@ def _preflight(
dict[str, ExecutionAdapter],
list[PlanAction],
list[PlanAction],
list[tuple[str, RetireRequest]],
]:
adapters_by_endpoint: dict[str, ExecutionAdapter] = {}
for adapter in adapters:
@@ -333,6 +334,9 @@ def _preflight(
executable: list[PlanAction] = []
skipped: list[PlanAction] = []
planned_retirements: list[
tuple[str, RetireRequest]
] = []
for plan in plans:
adapter = adapters_by_endpoint.get(plan.endpoint_id)
@@ -341,6 +345,14 @@ def _preflight(
f"No adapter for endpoint plan {plan.endpoint_id}"
)
for retirement in plan.retirements:
planned_retirements.append(
(
plan.endpoint_id,
retirement,
)
)
for action in plan.actions:
if action.endpoint_id != plan.endpoint_id:
raise RuntimeError(
@@ -382,7 +394,12 @@ def _preflight(
)
executable.append(action)
return adapters_by_endpoint, executable, skipped
return (
adapters_by_endpoint,
executable,
skipped,
planned_retirements,
)
def execute_endpoint_plans(
@@ -399,14 +416,20 @@ def execute_endpoint_plans(
)
timestamp = timestamp.astimezone(timezone.utc)
adapters_by_endpoint, executable, skipped = _preflight(
(
adapters_by_endpoint,
executable,
skipped,
planned_retirements,
) = _preflight(
adapters,
plans,
)
applied: list[ExecutedAction] = []
pending_retirements: list[
tuple[str, RetireRequest]
] = []
] = list(planned_retirements)
local_references: dict[
str,
@@ -421,6 +444,18 @@ def execute_endpoint_plans(
)
)
for action in skipped:
if (
action.operation == PlanOperation.NOOP
and action.current is not None
and action.current.local_key is not None
):
_remember_local_object(
local_references[action.endpoint_id],
action.current,
authoritative=True,
)
for action in order_plan_actions(executable):
adapter = adapters_by_endpoint[action.endpoint_id]
resolved_action = _resolve_action_references(
@@ -500,7 +535,23 @@ def execute_endpoint_plans(
)
)
retired_identities: set[
tuple[str, str, str]
] = set()
for endpoint_id, retirement in pending_retirements:
retired = retirement.object
identity = (
endpoint_id,
retired.object_type,
retired.local_key,
)
if identity in retired_identities:
continue
retired_identities.add(identity)
adapter = adapters_by_endpoint[endpoint_id]
retire = getattr(adapter, "retire", None)
dns_sync/reconciliation.py
+184 -18
View File
@@ -16,6 +16,7 @@ from .adapters import (
Observation,
PlanAction,
PlanOperation,
RetireRequest,
)
from .dependencies import order_plan_actions
@@ -342,6 +343,81 @@ class CertificateNewestPolicy:
== self._portable_state(desired)
)
@staticmethod
def _equivalent_selection_key(
object: NormalizedObject,
) -> tuple[int, str]:
# Preserve a native ACME-renewable certificate over an
# imported replica when both contain the same portable
# certificate state.
provider = object.payload.get("provider")
return (
1 if provider == "letsencrypt" else 0,
object.local_key or "",
)
def select_current(
self,
observations: Sequence[Observation],
desired: NormalizedObject,
) -> tuple[
NormalizedObject,
tuple[RetireRequest, ...],
]:
if not observations:
raise PlanningConflict(
"Certificate current-state selection requires "
"at least one observation"
)
candidates = [
observation.object
for observation in observations
]
if any(
candidate.local_key is None
for candidate in candidates
):
raise PlanningConflict(
"Observed certificate requires endpoint-local identity"
)
matching = [
candidate
for candidate in candidates
if self.equivalent(
candidate,
desired,
)
]
if matching:
selected = max(
matching,
key=self._equivalent_selection_key,
)
else:
# No local copy is already desired. Replace the newest
# local certificate and retire every other stale copy
# after the replacement and its dependents verify.
selected = max(
candidates,
key=lambda candidate: (
self._newest_key(candidate),
candidate.local_key or "",
),
)
retirements = tuple(
RetireRequest(object=candidate)
for candidate in candidates
if candidate.local_key != selected.local_key
)
return selected, retirements
class ProxyHostConsensusPolicy:
object_type = PROXY_HOSTS
@@ -481,13 +557,31 @@ def build_endpoint_plans(
else policies
)
raw_observations = tuple(observations)
current_by_identity: dict[
tuple[str, str, str],
list[Observation],
] = defaultdict(list)
raw_current_by_identity: dict[
tuple[str, str, str],
list[Observation],
] = defaultdict(list)
for observation in raw_observations:
raw_current_by_identity[
(
observation.endpoint_id,
observation.object.object_type,
observation.object.logical_key,
)
].append(observation)
logicalized_observations = (
logicalize_observation_references(observations)
logicalize_observation_references(
raw_observations
)
)
for observation in logicalized_observations:
@@ -521,6 +615,10 @@ def build_endpoint_plans(
continue
actions: list[PlanAction] = []
planned_retirements: list[
RetireRequest
] = []
for identity, desired in sorted(
desired_by_identity.items()
):
@@ -532,22 +630,42 @@ def build_endpoint_plans(
(endpoint_id, object_type, logical_key),
[],
)
policy = selected_policies.get(object_type)
if len(current_rows) > 1:
raise PlanningConflict(
f"Multiple endpoint objects for {endpoint_id} "
f"{object_type} {logical_key!r}"
selector = (
getattr(
policy,
"select_current",
None,
)
if policy is not None
else None
)
current = (
current_rows[0].object
if current_rows
else None
)
if selector is None:
raise PlanningConflict(
f"Multiple endpoint objects for {endpoint_id} "
f"{object_type} {logical_key!r}"
)
current, retirements = selector(
tuple(current_rows),
desired,
)
planned_retirements.extend(
retirements
)
else:
current = (
current_rows[0].object
if current_rows
else None
)
if current is None:
operation = PlanOperation.CREATE
else:
policy = selected_policies.get(object_type)
if policy is None:
equivalent = (
current.payload == desired.payload
@@ -590,15 +708,22 @@ def build_endpoint_plans(
)
}
if replaced:
retiring_local = {
(
retirement.object.object_type,
retirement.object.local_key,
)
for retirement in planned_retirements
}
if replaced or retiring_local:
promoted: list[PlanAction] = []
for action in actions:
desired = action.desired
if (
action.operation == PlanOperation.NOOP
and desired is not None
logical_dependency_replaced = (
desired is not None
and any(
(
reference.object_type,
@@ -608,13 +733,51 @@ def build_endpoint_plans(
for reference in desired.references
if reference.logical_key is not None
)
)
raw_rows = (
raw_current_by_identity.get(
(
endpoint_id,
desired.object_type,
desired.logical_key,
),
[],
)
if desired is not None
else []
)
local_dependency_retiring = any(
(
reference.object_type,
reference.local_key,
)
in retiring_local
for row in raw_rows
for reference in row.object.references
if reference.local_key is not None
)
if (
action.operation == PlanOperation.NOOP
and (
logical_dependency_replaced
or local_dependency_retiring
)
):
reason = (
"referenced object is being replaced"
if logical_dependency_replaced
else (
"referenced local object is being retired"
)
)
action = replace(
action,
operation=PlanOperation.UPDATE,
reason=(
"referenced object is being replaced"
),
reason=reason,
)
promoted.append(action)
@@ -625,6 +788,9 @@ def build_endpoint_plans(
EndpointPlan(
endpoint_id=endpoint_id,
actions=order_plan_actions(actions),
retirements=tuple(
planned_retirements
),
)
)