Recover interrupted certificate replacement

2026-09-20 01:54:30 -04:00
parent 2668651a98
commit 387332b5db
5 changed files with 658 additions and 21 deletions
@@ -414,6 +414,183 @@ class ExecutionTests(unittest.TestCase):
        )


+    def test_restart_recovery_rebinds_before_planned_retirement(self):
+        endpoint = Endpoint(
+            "npm",
+            "192.168.1.4:81",
+        )
+
+        adapter = FakeAdapter(
+            endpoint,
+            capabilities(
+                CERTIFICATES,
+                PROXY_HOSTS,
+            ),
+            apply_local_keys={
+                (
+                    PROXY_HOSTS,
+                    "app.example.com",
+                ): "12",
+            },
+        )
+
+        old_certificate = NormalizedObject(
+            object_type=CERTIFICATES,
+            logical_key="app.example.com",
+            local_key="67",
+            payload={
+                "domain_names": ["app.example.com"],
+                "valid_from": "2026-08-01T00:00:00+00:00",
+                "expires_on": "2026-11-01T00:00:00+00:00",
+            },
+        )
+
+        new_certificate = NormalizedObject(
+            object_type=CERTIFICATES,
+            logical_key="app.example.com",
+            local_key="84",
+            payload={
+                "domain_names": ["app.example.com"],
+                "valid_from": "2026-09-01T00:00:00+00:00",
+                "expires_on": "2026-12-01T00:00:00+00:00",
+            },
+        )
+
+        old_proxy = NormalizedObject(
+            object_type=PROXY_HOSTS,
+            logical_key="app.example.com",
+            local_key="12",
+            payload={
+                "domain_names": ["app.example.com"],
+                "forward_scheme": "https",
+                "forward_host": "backend.internal",
+                "forward_port": 443,
+                "enabled": True,
+            },
+            references=(
+                ObjectReference(
+                    relation="certificate",
+                    object_type=CERTIFICATES,
+                    local_key="67",
+                ),
+            ),
+        )
+
+        desired_certificate = NormalizedObject(
+            object_type=CERTIFICATES,
+            logical_key="app.example.com",
+            payload=dict(new_certificate.payload),
+        )
+
+        desired_proxy = NormalizedObject(
+            object_type=PROXY_HOSTS,
+            logical_key="app.example.com",
+            payload=dict(old_proxy.payload),
+            references=(
+                ObjectReference(
+                    relation="certificate",
+                    object_type=CERTIFICATES,
+                    logical_key="app.example.com",
+                ),
+            ),
+        )
+
+        plan = EndpointPlan(
+            endpoint.identity,
+            (
+                PlanAction(
+                    endpoint_id=endpoint.identity,
+                    operation=PlanOperation.NOOP,
+                    desired=desired_certificate,
+                    current=new_certificate,
+                ),
+                PlanAction(
+                    endpoint_id=endpoint.identity,
+                    operation=PlanOperation.UPDATE,
+                    desired=desired_proxy,
+                    current=old_proxy,
+                    reason=(
+                        "referenced local object is being retired"
+                    ),
+                ),
+            ),
+            retirements=(
+                RetireRequest(
+                    object=old_certificate,
+                ),
+            ),
+        )
+
+        with SQLiteStateStore(":memory:") as store:
+            store.replace_observations(
+                endpoint.identity,
+                CERTIFICATES,
+                [
+                    Observation(
+                        endpoint.identity,
+                        old_certificate,
+                        NOW,
+                    ),
+                    Observation(
+                        endpoint.identity,
+                        new_certificate,
+                        NOW,
+                    ),
+                ],
+                NOW,
+            )
+
+            store.replace_observations(
+                endpoint.identity,
+                PROXY_HOSTS,
+                [
+                    Observation(
+                        endpoint.identity,
+                        old_proxy,
+                        NOW,
+                    )
+                ],
+                NOW,
+            )
+
+            execute_endpoint_plans(
+                [adapter],
+                [plan],
+                store,
+                executed_at=NOW,
+            )
+
+        self.assertEqual(
+            tuple(event[0] for event in adapter.events),
+            (
+                "apply",
+                "verify",
+                "retire",
+            ),
+        )
+
+        proxy_action = adapter.applied_actions[0]
+
+        self.assertEqual(
+            proxy_action.desired.references,
+            (
+                ObjectReference(
+                    relation="certificate",
+                    object_type=CERTIFICATES,
+                    local_key="84",
+                ),
+            ),
+        )
+
+        self.assertEqual(
+            adapter.retired,
+            [
+                RetireRequest(
+                    object=old_certificate,
+                )
+            ],
+        )
+
    def test_failed_verification_is_persisted_and_raises(self):
        endpoint = Endpoint("verizon", "192.168.1.1:443")
        adapter = FakeAdapter(