Fixes
This commit is contained in:
+6
-2
@@ -88,13 +88,17 @@ scanner:
|
|||||||
# -----------------------------------------------------
|
# -----------------------------------------------------
|
||||||
codec:
|
codec:
|
||||||
whitelist:
|
whitelist:
|
||||||
- avc1 # H.264
|
- avc1 # H.264 (MP4 container tag)
|
||||||
- hevc # H.265
|
- h264 # H.264 (ffprobe codec name)
|
||||||
|
- hevc # H.265 (MP4 container tag)
|
||||||
|
- h265 # H.265 (ffprobe codec name)
|
||||||
- vp8
|
- vp8
|
||||||
- vp9
|
- vp9
|
||||||
- av01 # AV1
|
- av01 # AV1
|
||||||
- mjpeg
|
- mjpeg
|
||||||
- mp4v
|
- mp4v
|
||||||
|
- wmv3 # Windows Media Video 7
|
||||||
|
- mpeg4 # MPEG-4 Part 2
|
||||||
default_status_on_error: UNSCANNABLE
|
default_status_on_error: UNSCANNABLE
|
||||||
|
|
||||||
# -----------------------------------------------------
|
# -----------------------------------------------------
|
||||||
|
|||||||
@@ -59,6 +59,9 @@ services:
|
|||||||
memory: 24G
|
memory: 24G
|
||||||
networks:
|
networks:
|
||||||
- videodetect-network
|
- videodetect-network
|
||||||
|
depends_on:
|
||||||
|
mariadb:
|
||||||
|
condition: service_healthy
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "python3", "-c", "import torch; print(torch.cuda.is_available())"]
|
test: ["CMD", "python3", "-c", "import torch; print(torch.cuda.is_available())"]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
|
|||||||
+104
-19
@@ -3,7 +3,7 @@ Database connection layer with connection pooling for VideoDetect.
|
|||||||
|
|
||||||
Provides:
|
Provides:
|
||||||
- Connection pooling via DBUtils + PyMySQL
|
- Connection pooling via DBUtils + PyMySQL
|
||||||
- Automatic reconnection on disconnect
|
- Automatic reconnection on disconnect (ping + retry on transient errors)
|
||||||
- Context manager support
|
- Context manager support
|
||||||
- Prepared statements for all queries
|
- Prepared statements for all queries
|
||||||
- Transaction support for atomic state transitions
|
- Transaction support for atomic state transitions
|
||||||
@@ -11,6 +11,7 @@ Provides:
|
|||||||
|
|
||||||
import contextlib
|
import contextlib
|
||||||
import logging
|
import logging
|
||||||
|
import time
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from dbutils.pooled_db import PooledDB
|
from dbutils.pooled_db import PooledDB
|
||||||
@@ -22,6 +23,12 @@ logger = logging.getLogger(__name__)
|
|||||||
class DBConnector:
|
class DBConnector:
|
||||||
"""Thread-safe database connection pool manager."""
|
"""Thread-safe database connection pool manager."""
|
||||||
|
|
||||||
|
# Transient errors that indicate a dropped/stale connection and are safe
|
||||||
|
# to retry with a fresh connection from the pool.
|
||||||
|
_RETRYABLE_ERRORS = (pymysql.err.InterfaceError, pymysql.err.OperationalError)
|
||||||
|
_MAX_RETRIES = 3
|
||||||
|
_RETRY_BACKOFF_SECONDS = 0.5
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
host: str = "mariadb",
|
host: str = "mariadb",
|
||||||
@@ -41,6 +48,10 @@ class DBConnector:
|
|||||||
"maxcached": pool_size,
|
"maxcached": pool_size,
|
||||||
"maxusage": 200,
|
"maxusage": 200,
|
||||||
"blocking": True,
|
"blocking": True,
|
||||||
|
# ping(0) = check if connection is alive BEFORE returning it from
|
||||||
|
# the pool. This is the correct DBUtils mechanism for detecting
|
||||||
|
# dead/stale connections (idle timeout, network blip, DB restart).
|
||||||
|
"ping": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
# PyMySQL connection parameters
|
# PyMySQL connection parameters
|
||||||
@@ -57,8 +68,9 @@ class DBConnector:
|
|||||||
}
|
}
|
||||||
|
|
||||||
self._pool = PooledDB(**pool_config, **connection_params)
|
self._pool = PooledDB(**pool_config, **connection_params)
|
||||||
|
self._pool_recycle = pool_recycle
|
||||||
logger.info(
|
logger.info(
|
||||||
"DB pool initialized: host=%s db=%s pool_size=%d min=%d",
|
"DB pool initialized: host=%s db=%s pool_size=%d min=%d ping=0",
|
||||||
host, database, pool_size, pool_min,
|
host, database, pool_size, pool_min,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -66,10 +78,85 @@ class DBConnector:
|
|||||||
"""Get a connection from the pool."""
|
"""Get a connection from the pool."""
|
||||||
return self._pool.connection()
|
return self._pool.connection()
|
||||||
|
|
||||||
|
def _run_query(self, query: str, params, transaction: bool, fetch: str):
|
||||||
|
"""Run a single query on a fresh connection, with retry on transient errors.
|
||||||
|
|
||||||
|
A pooled connection can be dropped by the server (idle timeout, network
|
||||||
|
blip, DB restart). When that happens the query fails with an
|
||||||
|
InterfaceError/OperationalError. We discard the dead connection and
|
||||||
|
retry with a fresh one from the pool.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: SQL statement.
|
||||||
|
params: Bound parameters (or None).
|
||||||
|
transaction: If True, wrap the query in a transaction.
|
||||||
|
fetch: One of "none", "one", or "all" controlling the return value.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
- "none": the affected row count.
|
||||||
|
- "one": a single row (dict) or None.
|
||||||
|
- "all": a list of rows (dicts).
|
||||||
|
"""
|
||||||
|
last_exc = None
|
||||||
|
for attempt in range(1, self._MAX_RETRIES + 1):
|
||||||
|
conn = self.get_connection()
|
||||||
|
try:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
if transaction:
|
||||||
|
conn.begin()
|
||||||
|
cursor.execute(query, params or ())
|
||||||
|
if fetch == "one":
|
||||||
|
result = cursor.fetchone()
|
||||||
|
elif fetch == "all":
|
||||||
|
result = cursor.fetchall()
|
||||||
|
else:
|
||||||
|
result = cursor.rowcount
|
||||||
|
if transaction:
|
||||||
|
conn.commit()
|
||||||
|
cursor.close()
|
||||||
|
conn.close()
|
||||||
|
return result
|
||||||
|
except self._RETRYABLE_ERRORS as e:
|
||||||
|
last_exc = e
|
||||||
|
logger.warning(
|
||||||
|
"Transient DB error (attempt %d/%d): %s",
|
||||||
|
attempt, self._MAX_RETRIES, e,
|
||||||
|
)
|
||||||
|
# Discard the dead connection; a fresh one is taken on retry.
|
||||||
|
self._safe_close(conn)
|
||||||
|
if attempt < self._MAX_RETRIES:
|
||||||
|
time.sleep(self._RETRY_BACKOFF_SECONDS * attempt)
|
||||||
|
except Exception:
|
||||||
|
# Non-transient error: roll back if in a transaction and re-raise.
|
||||||
|
if transaction:
|
||||||
|
self._safe_rollback(conn)
|
||||||
|
self._safe_close(conn)
|
||||||
|
raise
|
||||||
|
raise last_exc
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _safe_close(conn):
|
||||||
|
try:
|
||||||
|
conn.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _safe_rollback(conn):
|
||||||
|
try:
|
||||||
|
conn.rollback()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def get_cursor(self, transaction: bool = False):
|
def get_cursor(self, transaction: bool = False):
|
||||||
"""Context manager for getting a cursor with optional transaction support."""
|
"""Context manager for getting a cursor with optional transaction support.
|
||||||
|
|
||||||
|
Note: this does not retry on transient errors; use execute/fetchone/
|
||||||
|
fetchall for retry-safe single-statement queries.
|
||||||
|
"""
|
||||||
conn = self.get_connection()
|
conn = self.get_connection()
|
||||||
|
cursor = None
|
||||||
try:
|
try:
|
||||||
cursor = conn.cursor()
|
cursor = conn.cursor()
|
||||||
if transaction:
|
if transaction:
|
||||||
@@ -79,11 +166,15 @@ class DBConnector:
|
|||||||
conn.commit()
|
conn.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
if transaction:
|
if transaction:
|
||||||
conn.rollback()
|
self._safe_rollback(conn)
|
||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
|
if cursor is not None:
|
||||||
|
try:
|
||||||
cursor.close()
|
cursor.close()
|
||||||
conn.close()
|
except Exception:
|
||||||
|
pass
|
||||||
|
self._safe_close(conn)
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def transaction(self):
|
def transaction(self):
|
||||||
@@ -94,28 +185,22 @@ class DBConnector:
|
|||||||
yield conn
|
yield conn
|
||||||
conn.commit()
|
conn.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
conn.rollback()
|
self._safe_rollback(conn)
|
||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
self._safe_close(conn)
|
||||||
|
|
||||||
def execute(self, query: str, params=None, transaction: bool = False):
|
def execute(self, query: str, params=None, transaction: bool = False):
|
||||||
"""Execute a query and return affected rows."""
|
"""Execute a query and return affected rows. Retries on transient errors."""
|
||||||
with self.get_cursor(transaction=transaction) as cursor:
|
return self._run_query(query, params, transaction, fetch="none")
|
||||||
cursor.execute(query, params or ())
|
|
||||||
return cursor.rowcount
|
|
||||||
|
|
||||||
def fetchone(self, query: str, params=None):
|
def fetchone(self, query: str, params=None):
|
||||||
"""Execute a query and return one row."""
|
"""Execute a query and return one row. Retries on transient errors."""
|
||||||
with self.get_cursor() as cursor:
|
return self._run_query(query, params, transaction=False, fetch="one")
|
||||||
cursor.execute(query, params or ())
|
|
||||||
return cursor.fetchone()
|
|
||||||
|
|
||||||
def fetchall(self, query: str, params=None):
|
def fetchall(self, query: str, params=None):
|
||||||
"""Execute a query and return all rows."""
|
"""Execute a query and return all rows. Retries on transient errors."""
|
||||||
with self.get_cursor() as cursor:
|
return self._run_query(query, params, transaction=False, fetch="all")
|
||||||
cursor.execute(query, params or ())
|
|
||||||
return cursor.fetchall()
|
|
||||||
|
|
||||||
def initialize_schema(self, schema_path: str = "db/schema.sql"):
|
def initialize_schema(self, schema_path: str = "db/schema.sql"):
|
||||||
"""Initialize the database schema from SQL file."""
|
"""Initialize the database schema from SQL file."""
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ Handles errors gracefully for corrupt or unsupported files.
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|||||||
Reference in New Issue
Block a user