keepalive.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477
  1. # -*- coding: utf-8 -*-
  2. """
  3. KeepAlive-Manager für Netzwerkverbindungen.
  4. Bietet automatisches Ping/Pong-Scheduling und
  5. Verbindungsüberwachung für langlebige Connections.
  6. """
  7. from __future__ import annotations
  8. import asyncio
  9. import logging
  10. from dataclasses import dataclass, field
  11. from datetime import datetime, timedelta
  12. from enum import Enum, auto
  13. from typing import Any, Awaitable, Callable, Protocol
  14. from weakref import WeakValueDictionary
  15. class ConnectionState(Enum):
  16. """Verbindungszustand."""
  17. CONNECTED = auto()
  18. DISCONNECTED = auto()
  19. STALE = auto() # Keine Antwort auf Ping
  20. RECONNECTING = auto()
  21. class Pingable(Protocol):
  22. """Protokoll für pingbare Verbindungen."""
  23. async def send_ping(self) -> None:
  24. """Sendet einen Ping."""
  25. ...
  26. async def handle_pong(self, data: Any = None) -> None:
  27. """Verarbeitet empfangenen Pong."""
  28. ...
  29. @dataclass
  30. class KeepAliveConfig:
  31. """Konfiguration für KeepAlive-Verhalten."""
  32. ping_interval: float = 30.0 # Sekunden zwischen Pings
  33. pong_timeout: float = 10.0 # Max. Wartezeit auf Pong
  34. max_missed_pongs: int = 3 # Verlorene Pongs vor Disconnect
  35. jitter: float = 0.1 # Variation im Intervall (0.0-1.0)
  36. initial_delay: float = 5.0 # Verzögerung vor erstem Ping
  37. adaptive: bool = True # Intervall anpassen basierend auf RTT
  38. min_interval: float = 10.0 # Minimales Intervall bei adaptiv
  39. max_interval: float = 120.0 # Maximales Intervall bei adaptiv
  40. @dataclass
  41. class ConnectionStats:
  42. """Statistiken für eine Verbindung."""
  43. pings_sent: int = 0
  44. pongs_received: int = 0
  45. missed_pongs: int = 0
  46. last_ping_at: datetime | None = None
  47. last_pong_at: datetime | None = None
  48. last_rtt_ms: float = 0.0
  49. avg_rtt_ms: float = 0.0
  50. min_rtt_ms: float = float("inf")
  51. max_rtt_ms: float = 0.0
  52. state: ConnectionState = ConnectionState.CONNECTED
  53. state_changed_at: datetime = field(default_factory=datetime.now)
  54. reconnect_count: int = 0
  55. class KeepAliveSession:
  56. """
  57. KeepAlive-Sitzung für eine einzelne Verbindung.
  58. Verwaltet Ping/Pong-Zyklen und überwacht den
  59. Verbindungszustand.
  60. """
  61. def __init__(
  62. self,
  63. connection_id: str,
  64. ping_func: Callable[[], Awaitable[None]],
  65. config: KeepAliveConfig | None = None,
  66. on_timeout: Callable[[str], Awaitable[None]] | None = None,
  67. on_disconnect: Callable[[str], Awaitable[None]] | None = None,
  68. logger: logging.Logger | None = None,
  69. ) -> None:
  70. """
  71. Initialisiert die KeepAlive-Sitzung.
  72. Args:
  73. connection_id: Eindeutige Verbindungs-ID
  74. ping_func: Async-Funktion zum Senden von Pings
  75. config: KeepAlive-Konfiguration
  76. on_timeout: Callback bei Timeout
  77. on_disconnect: Callback bei Disconnect
  78. logger: Logger-Instanz
  79. """
  80. self.connection_id = connection_id
  81. self._ping_func = ping_func
  82. self.config = config or KeepAliveConfig()
  83. self._on_timeout = on_timeout
  84. self._on_disconnect = on_disconnect
  85. self.logger = logger or logging.getLogger(__name__)
  86. self.stats = ConnectionStats()
  87. self._task: asyncio.Task | None = None
  88. self._waiting_for_pong = False
  89. self._ping_sent_time: datetime | None = None
  90. self._rtt_samples: list[float] = []
  91. self._current_interval = self.config.ping_interval
  92. @property
  93. def is_running(self) -> bool:
  94. """Prüft ob KeepAlive aktiv ist."""
  95. return self._task is not None and not self._task.done()
  96. @property
  97. def state(self) -> ConnectionState:
  98. """Aktueller Verbindungszustand."""
  99. return self.stats.state
  100. async def start(self) -> None:
  101. """Startet KeepAlive-Schleife."""
  102. if self.is_running:
  103. return
  104. self.stats.state = ConnectionState.CONNECTED
  105. self.stats.state_changed_at = datetime.now()
  106. self._task = asyncio.create_task(self._keepalive_loop())
  107. self.logger.debug(f"KeepAlive gestartet für {self.connection_id}")
  108. async def stop(self) -> None:
  109. """Stoppt KeepAlive-Schleife."""
  110. if self._task:
  111. self._task.cancel()
  112. try:
  113. await self._task
  114. except asyncio.CancelledError:
  115. pass
  116. self._task = None
  117. self.logger.debug(f"KeepAlive gestoppt für {self.connection_id}")
  118. def handle_pong(self, timestamp: datetime | None = None) -> None:
  119. """
  120. Verarbeitet empfangenen Pong.
  121. Args:
  122. timestamp: Zeitpunkt des Pong-Empfangs
  123. """
  124. if not self._waiting_for_pong:
  125. return
  126. self._waiting_for_pong = False
  127. self.stats.pongs_received += 1
  128. self.stats.last_pong_at = timestamp or datetime.now()
  129. # RTT berechnen
  130. if self._ping_sent_time:
  131. rtt_ms = (self.stats.last_pong_at - self._ping_sent_time).total_seconds() * 1000
  132. self.stats.last_rtt_ms = rtt_ms
  133. self.stats.min_rtt_ms = min(self.stats.min_rtt_ms, rtt_ms)
  134. self.stats.max_rtt_ms = max(self.stats.max_rtt_ms, rtt_ms)
  135. # Durchschnitt aktualisieren
  136. self._rtt_samples.append(rtt_ms)
  137. if len(self._rtt_samples) > 20:
  138. self._rtt_samples.pop(0)
  139. self.stats.avg_rtt_ms = sum(self._rtt_samples) / len(self._rtt_samples)
  140. # Adaptives Intervall
  141. if self.config.adaptive:
  142. self._update_interval()
  143. # Zustand aktualisieren
  144. if self.stats.state == ConnectionState.STALE:
  145. self.stats.state = ConnectionState.CONNECTED
  146. self.stats.state_changed_at = datetime.now()
  147. self.stats.missed_pongs = 0
  148. def _update_interval(self) -> None:
  149. """Passt Ping-Intervall basierend auf RTT an."""
  150. avg_rtt = self.stats.avg_rtt_ms
  151. if avg_rtt < 50:
  152. # Sehr schnelle Verbindung - längeres Intervall
  153. self._current_interval = min(
  154. self._current_interval * 1.2,
  155. self.config.max_interval
  156. )
  157. elif avg_rtt > 500:
  158. # Langsame Verbindung - kürzeres Intervall
  159. self._current_interval = max(
  160. self._current_interval * 0.8,
  161. self.config.min_interval
  162. )
  163. async def _keepalive_loop(self) -> None:
  164. """Haupt-KeepAlive-Schleife."""
  165. # Initiale Verzögerung
  166. await asyncio.sleep(self.config.initial_delay)
  167. while True:
  168. try:
  169. # Jitter berechnen
  170. jitter_range = self._current_interval * self.config.jitter
  171. import random
  172. actual_interval = self._current_interval + random.uniform(
  173. -jitter_range, jitter_range
  174. )
  175. await asyncio.sleep(actual_interval)
  176. # Ping senden
  177. self._waiting_for_pong = True
  178. self._ping_sent_time = datetime.now()
  179. self.stats.pings_sent += 1
  180. self.stats.last_ping_at = self._ping_sent_time
  181. try:
  182. await self._ping_func()
  183. except Exception as e:
  184. self.logger.warning(f"Ping fehlgeschlagen für {self.connection_id}: {e}")
  185. continue
  186. # Auf Pong warten
  187. try:
  188. await asyncio.wait_for(
  189. self._wait_for_pong(),
  190. timeout=self.config.pong_timeout
  191. )
  192. except asyncio.TimeoutError:
  193. await self._handle_timeout()
  194. except asyncio.CancelledError:
  195. break
  196. except Exception as e:
  197. self.logger.error(f"KeepAlive-Fehler für {self.connection_id}: {e}")
  198. await asyncio.sleep(5) # Kurze Pause bei Fehler
  199. async def _wait_for_pong(self) -> None:
  200. """Wartet auf Pong-Empfang."""
  201. while self._waiting_for_pong:
  202. await asyncio.sleep(0.1)
  203. async def _handle_timeout(self) -> None:
  204. """Behandelt Pong-Timeout."""
  205. self._waiting_for_pong = False
  206. self.stats.missed_pongs += 1
  207. self.logger.warning(
  208. f"Pong-Timeout für {self.connection_id} "
  209. f"({self.stats.missed_pongs}/{self.config.max_missed_pongs})"
  210. )
  211. # Timeout-Callback
  212. if self._on_timeout:
  213. try:
  214. await self._on_timeout(self.connection_id)
  215. except Exception as e:
  216. self.logger.error(f"Timeout-Callback Fehler: {e}")
  217. # Zustand aktualisieren
  218. if self.stats.state != ConnectionState.STALE:
  219. self.stats.state = ConnectionState.STALE
  220. self.stats.state_changed_at = datetime.now()
  221. # Zu viele verlorene Pongs?
  222. if self.stats.missed_pongs >= self.config.max_missed_pongs:
  223. self.stats.state = ConnectionState.DISCONNECTED
  224. self.stats.state_changed_at = datetime.now()
  225. if self._on_disconnect:
  226. try:
  227. await self._on_disconnect(self.connection_id)
  228. except Exception as e:
  229. self.logger.error(f"Disconnect-Callback Fehler: {e}")
  230. class KeepAliveManager:
  231. """
  232. Zentraler Manager für KeepAlive-Sitzungen.
  233. Verwaltet mehrere Verbindungen und bietet eine
  234. einheitliche Schnittstelle für KeepAlive-Operationen.
  235. """
  236. def __init__(
  237. self,
  238. default_config: KeepAliveConfig | None = None,
  239. logger: logging.Logger | None = None,
  240. ) -> None:
  241. """
  242. Initialisiert den KeepAlive-Manager.
  243. Args:
  244. default_config: Standard-Konfiguration für neue Sitzungen
  245. logger: Logger-Instanz
  246. """
  247. self.default_config = default_config or KeepAliveConfig()
  248. self.logger = logger or logging.getLogger(__name__)
  249. self._sessions: dict[str, KeepAliveSession] = {}
  250. self._on_timeout_handlers: list[Callable[[str], Awaitable[None]]] = []
  251. self._on_disconnect_handlers: list[Callable[[str], Awaitable[None]]] = []
  252. @property
  253. def session_count(self) -> int:
  254. """Anzahl aktiver Sitzungen."""
  255. return len(self._sessions)
  256. @property
  257. def active_count(self) -> int:
  258. """Anzahl laufender Sitzungen."""
  259. return sum(1 for s in self._sessions.values() if s.is_running)
  260. def on_timeout(self, handler: Callable[[str], Awaitable[None]]) -> None:
  261. """Registriert globalen Timeout-Handler."""
  262. self._on_timeout_handlers.append(handler)
  263. def on_disconnect(self, handler: Callable[[str], Awaitable[None]]) -> None:
  264. """Registriert globalen Disconnect-Handler."""
  265. self._on_disconnect_handlers.append(handler)
  266. async def _global_timeout_handler(self, connection_id: str) -> None:
  267. """Ruft alle Timeout-Handler auf."""
  268. for handler in self._on_timeout_handlers:
  269. try:
  270. await handler(connection_id)
  271. except Exception as e:
  272. self.logger.error(f"Timeout-Handler Fehler: {e}")
  273. async def _global_disconnect_handler(self, connection_id: str) -> None:
  274. """Ruft alle Disconnect-Handler auf."""
  275. for handler in self._on_disconnect_handlers:
  276. try:
  277. await handler(connection_id)
  278. except Exception as e:
  279. self.logger.error(f"Disconnect-Handler Fehler: {e}")
  280. async def register(
  281. self,
  282. connection_id: str,
  283. ping_func: Callable[[], Awaitable[None]],
  284. config: KeepAliveConfig | None = None,
  285. auto_start: bool = True,
  286. ) -> KeepAliveSession:
  287. """
  288. Registriert eine Verbindung für KeepAlive.
  289. Args:
  290. connection_id: Eindeutige Verbindungs-ID
  291. ping_func: Funktion zum Senden von Pings
  292. config: Optional: Benutzerdefinierte Konfiguration
  293. auto_start: Automatisch starten
  294. Returns:
  295. KeepAlive-Sitzung
  296. """
  297. if connection_id in self._sessions:
  298. await self._sessions[connection_id].stop()
  299. session = KeepAliveSession(
  300. connection_id=connection_id,
  301. ping_func=ping_func,
  302. config=config or self.default_config,
  303. on_timeout=self._global_timeout_handler,
  304. on_disconnect=self._global_disconnect_handler,
  305. logger=self.logger,
  306. )
  307. self._sessions[connection_id] = session
  308. if auto_start:
  309. await session.start()
  310. return session
  311. async def unregister(self, connection_id: str) -> bool:
  312. """
  313. Entfernt Verbindung aus KeepAlive.
  314. Args:
  315. connection_id: Verbindungs-ID
  316. Returns:
  317. True wenn entfernt
  318. """
  319. if connection_id not in self._sessions:
  320. return False
  321. session = self._sessions.pop(connection_id)
  322. await session.stop()
  323. return True
  324. def get_session(self, connection_id: str) -> KeepAliveSession | None:
  325. """Holt Sitzung nach ID."""
  326. return self._sessions.get(connection_id)
  327. def handle_pong(
  328. self,
  329. connection_id: str,
  330. timestamp: datetime | None = None,
  331. ) -> None:
  332. """
  333. Verarbeitet Pong für Verbindung.
  334. Args:
  335. connection_id: Verbindungs-ID
  336. timestamp: Pong-Zeitstempel
  337. """
  338. session = self._sessions.get(connection_id)
  339. if session:
  340. session.handle_pong(timestamp)
  341. def get_stats(self, connection_id: str) -> ConnectionStats | None:
  342. """Holt Statistiken für Verbindung."""
  343. session = self._sessions.get(connection_id)
  344. return session.stats if session else None
  345. def get_all_stats(self) -> dict[str, ConnectionStats]:
  346. """Holt Statistiken für alle Verbindungen."""
  347. return {cid: s.stats for cid, s in self._sessions.items()}
  348. def get_stale_connections(self) -> list[str]:
  349. """Findet Verbindungen im STALE-Zustand."""
  350. return [
  351. cid for cid, session in self._sessions.items()
  352. if session.state == ConnectionState.STALE
  353. ]
  354. def get_disconnected(self) -> list[str]:
  355. """Findet getrennte Verbindungen."""
  356. return [
  357. cid for cid, session in self._sessions.items()
  358. if session.state == ConnectionState.DISCONNECTED
  359. ]
  360. async def start_all(self) -> None:
  361. """Startet alle registrierten Sitzungen."""
  362. for session in self._sessions.values():
  363. if not session.is_running:
  364. await session.start()
  365. async def stop_all(self) -> None:
  366. """Stoppt alle Sitzungen."""
  367. for session in self._sessions.values():
  368. await session.stop()
  369. async def cleanup(self) -> int:
  370. """
  371. Entfernt getrennte Verbindungen.
  372. Returns:
  373. Anzahl entfernter Verbindungen
  374. """
  375. to_remove = self.get_disconnected()
  376. for cid in to_remove:
  377. await self.unregister(cid)
  378. return len(to_remove)
  379. def get_summary(self) -> dict[str, Any]:
  380. """Liefert Übersicht über alle Verbindungen."""
  381. states = {}
  382. for session in self._sessions.values():
  383. state_name = session.state.name
  384. states[state_name] = states.get(state_name, 0) + 1
  385. return {
  386. "total_sessions": self.session_count,
  387. "active_sessions": self.active_count,
  388. "states": states,
  389. "total_pings_sent": sum(
  390. s.stats.pings_sent for s in self._sessions.values()
  391. ),
  392. "total_pongs_received": sum(
  393. s.stats.pongs_received for s in self._sessions.values()
  394. ),
  395. }