359 lines
11 KiB
Python
359 lines
11 KiB
Python
import json
|
|
import sqlite3
|
|
from collections.abc import Iterable, Mapping
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from os import PathLike
|
|
from typing import Any
|
|
|
|
from .adapters import NormalizedObject, Observation
|
|
|
|
|
|
_FORBIDDEN_PERSISTED_KEYS = frozenset(
|
|
{
|
|
"private_key",
|
|
"certificate_key",
|
|
"certificate",
|
|
"dns_provider_credentials",
|
|
"secret",
|
|
}
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ObservationStatus:
|
|
endpoint_id: str
|
|
object_type: str
|
|
last_attempt_at: datetime
|
|
last_success_at: datetime | None
|
|
last_error: str | None
|
|
|
|
@property
|
|
def is_live(self) -> bool:
|
|
return self.last_error is None
|
|
|
|
|
|
def _utc(value: datetime, label: str) -> datetime:
|
|
if value.tzinfo is None:
|
|
raise RuntimeError(f"{label} must be timezone-aware")
|
|
return value.astimezone(timezone.utc)
|
|
|
|
|
|
def _timestamp(value: datetime, label: str) -> str:
|
|
return _utc(value, label).isoformat()
|
|
|
|
|
|
def _parse_timestamp(value: str | None) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
parsed = datetime.fromisoformat(value)
|
|
if parsed.tzinfo is None:
|
|
raise RuntimeError(
|
|
f"Persisted timestamp is not timezone-aware: {value!r}"
|
|
)
|
|
return parsed.astimezone(timezone.utc)
|
|
|
|
|
|
def _validate_safe_payload(value: Any, path: str = "payload") -> None:
|
|
if isinstance(value, Mapping):
|
|
for raw_key, child in value.items():
|
|
key = str(raw_key).strip().lower().replace("-", "_")
|
|
if key in _FORBIDDEN_PERSISTED_KEYS:
|
|
raise RuntimeError(
|
|
f"Refusing to persist sensitive field {path}.{key}"
|
|
)
|
|
_validate_safe_payload(child, f"{path}.{key}")
|
|
elif isinstance(value, (list, tuple)):
|
|
for index, child in enumerate(value):
|
|
_validate_safe_payload(child, f"{path}[{index}]")
|
|
|
|
|
|
def _serialize_payload(payload: Mapping[str, Any]) -> str:
|
|
_validate_safe_payload(payload)
|
|
try:
|
|
return json.dumps(
|
|
payload,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
except (TypeError, ValueError) as exc:
|
|
raise RuntimeError(
|
|
f"Object payload is not JSON serializable: {exc}"
|
|
) from exc
|
|
|
|
|
|
def _deserialize_payload(value: str) -> dict[str, Any]:
|
|
try:
|
|
payload = json.loads(value)
|
|
except json.JSONDecodeError as exc:
|
|
raise RuntimeError("Persisted object payload is invalid JSON") from exc
|
|
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError("Persisted object payload is not a mapping")
|
|
return payload
|
|
|
|
|
|
class SQLiteStateStore:
|
|
def __init__(self, path: str | PathLike[str]):
|
|
self.path = str(path)
|
|
self.connection = sqlite3.connect(self.path)
|
|
self.connection.row_factory = sqlite3.Row
|
|
self.connection.execute("PRAGMA foreign_keys = ON")
|
|
self.connection.execute("PRAGMA busy_timeout = 5000")
|
|
if self.path != ":memory:":
|
|
self.connection.execute("PRAGMA journal_mode = WAL")
|
|
self._initialize()
|
|
|
|
def _initialize(self) -> None:
|
|
with self.connection:
|
|
self.connection.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS schema_meta (
|
|
version INTEGER NOT NULL
|
|
);
|
|
|
|
INSERT INTO schema_meta(version)
|
|
SELECT 1
|
|
WHERE NOT EXISTS (
|
|
SELECT 1 FROM schema_meta
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS observations (
|
|
endpoint_id TEXT NOT NULL,
|
|
object_type TEXT NOT NULL,
|
|
logical_key TEXT NOT NULL,
|
|
local_key TEXT NOT NULL,
|
|
payload_json TEXT NOT NULL,
|
|
observed_at TEXT NOT NULL,
|
|
PRIMARY KEY (
|
|
endpoint_id,
|
|
object_type,
|
|
local_key
|
|
)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS
|
|
observations_by_logical_identity
|
|
ON observations (
|
|
object_type,
|
|
logical_key
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS observation_status (
|
|
endpoint_id TEXT NOT NULL,
|
|
object_type TEXT NOT NULL,
|
|
last_attempt_at TEXT NOT NULL,
|
|
last_success_at TEXT,
|
|
last_error TEXT,
|
|
PRIMARY KEY (
|
|
endpoint_id,
|
|
object_type
|
|
)
|
|
);
|
|
"""
|
|
)
|
|
|
|
def close(self) -> None:
|
|
self.connection.close()
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, traceback):
|
|
self.close()
|
|
|
|
def replace_observations(
|
|
self,
|
|
endpoint_id: str,
|
|
object_type: str,
|
|
observations: Iterable[Observation],
|
|
snapshot_at: datetime,
|
|
) -> None:
|
|
rows = list(observations)
|
|
snapshot_timestamp = _timestamp(
|
|
snapshot_at,
|
|
"Observation snapshot timestamp",
|
|
)
|
|
prepared: list[tuple[str, str, str, str, str, str]] = []
|
|
local_keys: set[str] = set()
|
|
|
|
for observation in rows:
|
|
normalized = observation.object
|
|
if observation.endpoint_id != endpoint_id:
|
|
raise RuntimeError(
|
|
"Observation endpoint does not match snapshot endpoint"
|
|
)
|
|
if normalized.object_type != object_type:
|
|
raise RuntimeError(
|
|
"Observation object type does not match snapshot type"
|
|
)
|
|
if normalized.local_key is None:
|
|
raise RuntimeError(
|
|
"Observed objects require an endpoint-local key"
|
|
)
|
|
if normalized.local_key in local_keys:
|
|
raise RuntimeError(
|
|
f"Duplicate endpoint-local key: {normalized.local_key}"
|
|
)
|
|
|
|
local_keys.add(normalized.local_key)
|
|
prepared.append(
|
|
(
|
|
endpoint_id,
|
|
object_type,
|
|
normalized.logical_key,
|
|
normalized.local_key,
|
|
_serialize_payload(normalized.payload),
|
|
_timestamp(
|
|
observation.observed_at,
|
|
"Observation timestamp",
|
|
),
|
|
)
|
|
)
|
|
|
|
with self.connection:
|
|
self.connection.execute(
|
|
"""
|
|
DELETE FROM observations
|
|
WHERE endpoint_id = ? AND object_type = ?
|
|
""",
|
|
(endpoint_id, object_type),
|
|
)
|
|
self.connection.executemany(
|
|
"""
|
|
INSERT INTO observations (
|
|
endpoint_id,
|
|
object_type,
|
|
logical_key,
|
|
local_key,
|
|
payload_json,
|
|
observed_at
|
|
) VALUES (?, ?, ?, ?, ?, ?)
|
|
""",
|
|
prepared,
|
|
)
|
|
self.connection.execute(
|
|
"""
|
|
INSERT INTO observation_status (
|
|
endpoint_id,
|
|
object_type,
|
|
last_attempt_at,
|
|
last_success_at,
|
|
last_error
|
|
) VALUES (?, ?, ?, ?, NULL)
|
|
ON CONFLICT(endpoint_id, object_type) DO UPDATE SET
|
|
last_attempt_at = excluded.last_attempt_at,
|
|
last_success_at = excluded.last_success_at,
|
|
last_error = NULL
|
|
""",
|
|
(
|
|
endpoint_id,
|
|
object_type,
|
|
snapshot_timestamp,
|
|
snapshot_timestamp,
|
|
),
|
|
)
|
|
|
|
def record_discovery_failure(
|
|
self,
|
|
endpoint_id: str,
|
|
object_type: str,
|
|
error: str,
|
|
attempted_at: datetime,
|
|
) -> None:
|
|
attempted_timestamp = _timestamp(
|
|
attempted_at,
|
|
"Discovery attempt timestamp",
|
|
)
|
|
with self.connection:
|
|
self.connection.execute(
|
|
"""
|
|
INSERT INTO observation_status (
|
|
endpoint_id,
|
|
object_type,
|
|
last_attempt_at,
|
|
last_success_at,
|
|
last_error
|
|
) VALUES (?, ?, ?, NULL, ?)
|
|
ON CONFLICT(endpoint_id, object_type) DO UPDATE SET
|
|
last_attempt_at = excluded.last_attempt_at,
|
|
last_error = excluded.last_error
|
|
""",
|
|
(
|
|
endpoint_id,
|
|
object_type,
|
|
attempted_timestamp,
|
|
str(error),
|
|
),
|
|
)
|
|
|
|
def load_observations(
|
|
self,
|
|
object_type: str | None = None,
|
|
) -> list[Observation]:
|
|
sql = """
|
|
SELECT
|
|
endpoint_id,
|
|
object_type,
|
|
logical_key,
|
|
local_key,
|
|
payload_json,
|
|
observed_at
|
|
FROM observations
|
|
"""
|
|
parameters: tuple[str, ...] = ()
|
|
if object_type is not None:
|
|
sql += " WHERE object_type = ?"
|
|
parameters = (object_type,)
|
|
sql += """
|
|
ORDER BY
|
|
endpoint_id,
|
|
object_type,
|
|
logical_key,
|
|
local_key
|
|
"""
|
|
|
|
rows = self.connection.execute(sql, parameters).fetchall()
|
|
return [
|
|
Observation(
|
|
endpoint_id=row["endpoint_id"],
|
|
object=NormalizedObject(
|
|
object_type=row["object_type"],
|
|
logical_key=row["logical_key"],
|
|
local_key=row["local_key"],
|
|
payload=_deserialize_payload(row["payload_json"]),
|
|
),
|
|
observed_at=_parse_timestamp(row["observed_at"]),
|
|
)
|
|
for row in rows
|
|
]
|
|
|
|
def observation_status(
|
|
self,
|
|
endpoint_id: str,
|
|
object_type: str,
|
|
) -> ObservationStatus | None:
|
|
row = self.connection.execute(
|
|
"""
|
|
SELECT
|
|
endpoint_id,
|
|
object_type,
|
|
last_attempt_at,
|
|
last_success_at,
|
|
last_error
|
|
FROM observation_status
|
|
WHERE endpoint_id = ? AND object_type = ?
|
|
""",
|
|
(endpoint_id, object_type),
|
|
).fetchone()
|
|
if row is None:
|
|
return None
|
|
|
|
return ObservationStatus(
|
|
endpoint_id=row["endpoint_id"],
|
|
object_type=row["object_type"],
|
|
last_attempt_at=_parse_timestamp(row["last_attempt_at"]),
|
|
last_success_at=_parse_timestamp(row["last_success_at"]),
|
|
last_error=row["last_error"],
|
|
)
|