115 lines
3.7 KiB
Python
115 lines
3.7 KiB
Python
"""
|
|
Retry decorator with exponential backoff for transient failures.
|
|
|
|
Usage:
|
|
@retry(max_attempts=3, step="extract")
|
|
def extract_frames(...): ...
|
|
|
|
Non-retryable error types are re-raised immediately without consuming attempts.
|
|
"""
|
|
|
|
import functools
|
|
import logging
|
|
import time
|
|
from typing import Callable, Optional, Tuple, Type
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Error class names that should never be retried
|
|
_NON_RETRYABLE_NAMES = frozenset({
|
|
"CodecUnsupportedError",
|
|
"FileCorruptError",
|
|
"InvalidPathError",
|
|
"FileNotFoundError",
|
|
"PermissionError",
|
|
"IsADirectoryError",
|
|
})
|
|
|
|
|
|
class RetryExhaustedError(Exception):
|
|
"""Raised when all retry attempts have been exhausted."""
|
|
|
|
|
|
def retry(
|
|
max_attempts: int = 3,
|
|
initial_delay: float = 1.0,
|
|
backoff_factor: float = 2.0,
|
|
exceptions: Tuple[Type[Exception], ...] = (Exception,),
|
|
step: str = "unknown",
|
|
):
|
|
"""
|
|
Decorator: retry ``func`` up to ``max_attempts`` times on retryable exceptions.
|
|
|
|
Non-retryable exceptions (see _NON_RETRYABLE_NAMES) propagate immediately.
|
|
Each retry waits ``initial_delay * backoff_factor ** attempt`` seconds.
|
|
"""
|
|
def decorator(func: Callable) -> Callable:
|
|
@functools.wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
last_exc: Optional[Exception] = None
|
|
|
|
for attempt in range(max_attempts):
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except exceptions as exc:
|
|
if _is_non_retryable(exc):
|
|
logger.error(
|
|
"Non-retryable error in step=%s (attempt %d/%d): %s: %s",
|
|
step, attempt + 1, max_attempts,
|
|
type(exc).__name__, exc,
|
|
)
|
|
raise
|
|
|
|
last_exc = exc
|
|
_increment_retry_counter(step)
|
|
|
|
if attempt < max_attempts - 1:
|
|
delay = initial_delay * (backoff_factor ** attempt)
|
|
logger.warning(
|
|
"Transient error in step=%s (attempt %d/%d), "
|
|
"retrying in %.1fs: %s: %s",
|
|
step, attempt + 1, max_attempts, delay,
|
|
type(exc).__name__, exc,
|
|
)
|
|
time.sleep(delay)
|
|
else:
|
|
logger.error(
|
|
"All %d attempts exhausted in step=%s: %s: %s",
|
|
max_attempts, step, type(exc).__name__, exc,
|
|
)
|
|
|
|
raise RetryExhaustedError(
|
|
f"step={step} failed after {max_attempts} attempts"
|
|
) from last_exc
|
|
|
|
return wrapper
|
|
return decorator
|
|
|
|
|
|
def retry_from_config(config, step: str = "unknown"):
|
|
"""Build a ``@retry`` decorator from config.yaml monitoring.retry settings."""
|
|
mon = config.get_section("monitoring")
|
|
cfg = mon.get("retry", {})
|
|
return retry(
|
|
max_attempts=int(cfg.get("max_attempts", 3)),
|
|
initial_delay=float(cfg.get("initial_delay", 1.0)),
|
|
backoff_factor=float(cfg.get("backoff_factor", 2.0)),
|
|
step=step,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _is_non_retryable(exc: Exception) -> bool:
|
|
return type(exc).__name__ in _NON_RETRYABLE_NAMES
|
|
|
|
|
|
def _increment_retry_counter(step: str):
|
|
try:
|
|
from metrics import retry_attempts_total
|
|
retry_attempts_total.labels(step=step).inc()
|
|
except Exception:
|
|
pass # metrics not available; don't break the retry logic
|