This commit is contained in:
Ryan Shpeherd
2026-09-08 18:34:31 -04:00
parent 83f980f7f8
commit 78358e0d7b
4 changed files with 118 additions and 25 deletions
+6 -2
View File
@@ -88,13 +88,17 @@ scanner:
# ----------------------------------------------------- # -----------------------------------------------------
codec: codec:
whitelist: whitelist:
- avc1 # H.264 - avc1 # H.264 (MP4 container tag)
- hevc # H.265 - h264 # H.264 (ffprobe codec name)
- hevc # H.265 (MP4 container tag)
- h265 # H.265 (ffprobe codec name)
- vp8 - vp8
- vp9 - vp9
- av01 # AV1 - av01 # AV1
- mjpeg - mjpeg
- mp4v - mp4v
- wmv3 # Windows Media Video 7
- mpeg4 # MPEG-4 Part 2
default_status_on_error: UNSCANNABLE default_status_on_error: UNSCANNABLE
# ----------------------------------------------------- # -----------------------------------------------------
+3
View File
@@ -59,6 +59,9 @@ services:
memory: 24G memory: 24G
networks: networks:
- videodetect-network - videodetect-network
depends_on:
mariadb:
condition: service_healthy
healthcheck: healthcheck:
test: ["CMD", "python3", "-c", "import torch; print(torch.cuda.is_available())"] test: ["CMD", "python3", "-c", "import torch; print(torch.cuda.is_available())"]
interval: 30s interval: 30s
+108 -23
View File
@@ -3,7 +3,7 @@ Database connection layer with connection pooling for VideoDetect.
Provides: Provides:
- Connection pooling via DBUtils + PyMySQL - Connection pooling via DBUtils + PyMySQL
- Automatic reconnection on disconnect - Automatic reconnection on disconnect (ping + retry on transient errors)
- Context manager support - Context manager support
- Prepared statements for all queries - Prepared statements for all queries
- Transaction support for atomic state transitions - Transaction support for atomic state transitions
@@ -11,6 +11,7 @@ Provides:
import contextlib import contextlib
import logging import logging
import time
from typing import Optional from typing import Optional
from dbutils.pooled_db import PooledDB from dbutils.pooled_db import PooledDB
@@ -22,6 +23,12 @@ logger = logging.getLogger(__name__)
class DBConnector: class DBConnector:
"""Thread-safe database connection pool manager.""" """Thread-safe database connection pool manager."""
# Transient errors that indicate a dropped/stale connection and are safe
# to retry with a fresh connection from the pool.
_RETRYABLE_ERRORS = (pymysql.err.InterfaceError, pymysql.err.OperationalError)
_MAX_RETRIES = 3
_RETRY_BACKOFF_SECONDS = 0.5
def __init__( def __init__(
self, self,
host: str = "mariadb", host: str = "mariadb",
@@ -41,9 +48,13 @@ class DBConnector:
"maxcached": pool_size, "maxcached": pool_size,
"maxusage": 200, "maxusage": 200,
"blocking": True, "blocking": True,
# ping(0) = check if connection is alive BEFORE returning it from
# the pool. This is the correct DBUtils mechanism for detecting
# dead/stale connections (idle timeout, network blip, DB restart).
"ping": 0,
} }
# PyMySQL connection parameters # PyMySQL connection parameters
connection_params = { connection_params = {
"host": host, "host": host,
"port": port, "port": port,
@@ -55,10 +66,11 @@ class DBConnector:
"read_timeout": 30, "read_timeout": 30,
"write_timeout": 30, "write_timeout": 30,
} }
self._pool = PooledDB(**pool_config, **connection_params) self._pool = PooledDB(**pool_config, **connection_params)
self._pool_recycle = pool_recycle
logger.info( logger.info(
"DB pool initialized: host=%s db=%s pool_size=%d min=%d", "DB pool initialized: host=%s db=%s pool_size=%d min=%d ping=0",
host, database, pool_size, pool_min, host, database, pool_size, pool_min,
) )
@@ -66,10 +78,85 @@ class DBConnector:
"""Get a connection from the pool.""" """Get a connection from the pool."""
return self._pool.connection() return self._pool.connection()
def _run_query(self, query: str, params, transaction: bool, fetch: str):
"""Run a single query on a fresh connection, with retry on transient errors.
A pooled connection can be dropped by the server (idle timeout, network
blip, DB restart). When that happens the query fails with an
InterfaceError/OperationalError. We discard the dead connection and
retry with a fresh one from the pool.
Args:
query: SQL statement.
params: Bound parameters (or None).
transaction: If True, wrap the query in a transaction.
fetch: One of "none", "one", or "all" controlling the return value.
Returns:
- "none": the affected row count.
- "one": a single row (dict) or None.
- "all": a list of rows (dicts).
"""
last_exc = None
for attempt in range(1, self._MAX_RETRIES + 1):
conn = self.get_connection()
try:
cursor = conn.cursor()
if transaction:
conn.begin()
cursor.execute(query, params or ())
if fetch == "one":
result = cursor.fetchone()
elif fetch == "all":
result = cursor.fetchall()
else:
result = cursor.rowcount
if transaction:
conn.commit()
cursor.close()
conn.close()
return result
except self._RETRYABLE_ERRORS as e:
last_exc = e
logger.warning(
"Transient DB error (attempt %d/%d): %s",
attempt, self._MAX_RETRIES, e,
)
# Discard the dead connection; a fresh one is taken on retry.
self._safe_close(conn)
if attempt < self._MAX_RETRIES:
time.sleep(self._RETRY_BACKOFF_SECONDS * attempt)
except Exception:
# Non-transient error: roll back if in a transaction and re-raise.
if transaction:
self._safe_rollback(conn)
self._safe_close(conn)
raise
raise last_exc
@staticmethod
def _safe_close(conn):
try:
conn.close()
except Exception:
pass
@staticmethod
def _safe_rollback(conn):
try:
conn.rollback()
except Exception:
pass
@contextlib.contextmanager @contextlib.contextmanager
def get_cursor(self, transaction: bool = False): def get_cursor(self, transaction: bool = False):
"""Context manager for getting a cursor with optional transaction support.""" """Context manager for getting a cursor with optional transaction support.
Note: this does not retry on transient errors; use execute/fetchone/
fetchall for retry-safe single-statement queries.
"""
conn = self.get_connection() conn = self.get_connection()
cursor = None
try: try:
cursor = conn.cursor() cursor = conn.cursor()
if transaction: if transaction:
@@ -79,11 +166,15 @@ class DBConnector:
conn.commit() conn.commit()
except Exception: except Exception:
if transaction: if transaction:
conn.rollback() self._safe_rollback(conn)
raise raise
finally: finally:
cursor.close() if cursor is not None:
conn.close() try:
cursor.close()
except Exception:
pass
self._safe_close(conn)
@contextlib.contextmanager @contextlib.contextmanager
def transaction(self): def transaction(self):
@@ -94,28 +185,22 @@ class DBConnector:
yield conn yield conn
conn.commit() conn.commit()
except Exception: except Exception:
conn.rollback() self._safe_rollback(conn)
raise raise
finally: finally:
conn.close() self._safe_close(conn)
def execute(self, query: str, params=None, transaction: bool = False): def execute(self, query: str, params=None, transaction: bool = False):
"""Execute a query and return affected rows.""" """Execute a query and return affected rows. Retries on transient errors."""
with self.get_cursor(transaction=transaction) as cursor: return self._run_query(query, params, transaction, fetch="none")
cursor.execute(query, params or ())
return cursor.rowcount
def fetchone(self, query: str, params=None): def fetchone(self, query: str, params=None):
"""Execute a query and return one row.""" """Execute a query and return one row. Retries on transient errors."""
with self.get_cursor() as cursor: return self._run_query(query, params, transaction=False, fetch="one")
cursor.execute(query, params or ())
return cursor.fetchone()
def fetchall(self, query: str, params=None): def fetchall(self, query: str, params=None):
"""Execute a query and return all rows.""" """Execute a query and return all rows. Retries on transient errors."""
with self.get_cursor() as cursor: return self._run_query(query, params, transaction=False, fetch="all")
cursor.execute(query, params or ())
return cursor.fetchall()
def initialize_schema(self, schema_path: str = "db/schema.sql"): def initialize_schema(self, schema_path: str = "db/schema.sql"):
"""Initialize the database schema from SQL file.""" """Initialize the database schema from SQL file."""
+1
View File
@@ -7,6 +7,7 @@ Handles errors gracefully for corrupt or unsupported files.
import json import json
import logging import logging
import os
import subprocess import subprocess
import time import time
from dataclasses import dataclass, field from dataclasses import dataclass, field