Story 9

src/retry.py

@@ -0,0 +1,114 @@
+"""
+Retry decorator with exponential backoff for transient failures.
+
+Usage:
+    @retry(max_attempts=3, step="extract")
+    def extract_frames(...): ...
+
+Non-retryable error types are re-raised immediately without consuming attempts.
+"""
+
+import functools
+import logging
+import time
+from typing import Callable, Optional, Tuple, Type
+
+logger = logging.getLogger(__name__)
+
+# Error class names that should never be retried
+_NON_RETRYABLE_NAMES = frozenset({
+    "CodecUnsupportedError",
+    "FileCorruptError",
+    "InvalidPathError",
+    "FileNotFoundError",
+    "PermissionError",
+    "IsADirectoryError",
+})
+
+
+class RetryExhaustedError(Exception):
+    """Raised when all retry attempts have been exhausted."""
+
+
+def retry(
+    max_attempts: int = 3,
+    initial_delay: float = 1.0,
+    backoff_factor: float = 2.0,
+    exceptions: Tuple[Type[Exception], ...] = (Exception,),
+    step: str = "unknown",
+):
+    """
+    Decorator: retry ``func`` up to ``max_attempts`` times on retryable exceptions.
+
+    Non-retryable exceptions (see _NON_RETRYABLE_NAMES) propagate immediately.
+    Each retry waits ``initial_delay * backoff_factor ** attempt`` seconds.
+    """
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exc: Optional[Exception] = None
+
+            for attempt in range(max_attempts):
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as exc:
+                    if _is_non_retryable(exc):
+                        logger.error(
+                            "Non-retryable error in step=%s (attempt %d/%d): %s: %s",
+                            step, attempt + 1, max_attempts,
+                            type(exc).__name__, exc,
+                        )
+                        raise
+
+                    last_exc = exc
+                    _increment_retry_counter(step)
+
+                    if attempt < max_attempts - 1:
+                        delay = initial_delay * (backoff_factor ** attempt)
+                        logger.warning(
+                            "Transient error in step=%s (attempt %d/%d), "
+                            "retrying in %.1fs: %s: %s",
+                            step, attempt + 1, max_attempts, delay,
+                            type(exc).__name__, exc,
+                        )
+                        time.sleep(delay)
+                    else:
+                        logger.error(
+                            "All %d attempts exhausted in step=%s: %s: %s",
+                            max_attempts, step, type(exc).__name__, exc,
+                        )
+
+            raise RetryExhaustedError(
+                f"step={step} failed after {max_attempts} attempts"
+            ) from last_exc
+
+        return wrapper
+    return decorator
+
+
+def retry_from_config(config, step: str = "unknown"):
+    """Build a ``@retry`` decorator from config.yaml monitoring.retry settings."""
+    mon = config.get_section("monitoring")
+    cfg = mon.get("retry", {})
+    return retry(
+        max_attempts=int(cfg.get("max_attempts", 3)),
+        initial_delay=float(cfg.get("initial_delay", 1.0)),
+        backoff_factor=float(cfg.get("backoff_factor", 2.0)),
+        step=step,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _is_non_retryable(exc: Exception) -> bool:
+    return type(exc).__name__ in _NON_RETRYABLE_NAMES
+
+
+def _increment_retry_counter(step: str):
+    try:
+        from metrics import retry_attempts_total
+        retry_attempts_total.labels(step=step).inc()
+    except Exception:
+        pass  # metrics not available; don't break the retry logic