After updates
This commit is contained in:
Ryan Shpeherd
+24
-1
@@ -8,6 +8,7 @@ Initializes all components and starts the processing pipeline.
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
@@ -18,6 +19,7 @@ from config_loader import get_config
|
||||
from db_connector import DBConnector
|
||||
from logging_config import setup_logging
|
||||
from orchestrator import WorkerPool
|
||||
from scanner import DirectoryScanner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -84,7 +86,27 @@ def main():
|
||||
except ImportError:
|
||||
logger.warning("PyTorch not installed. GPU features disabled.")
|
||||
|
||||
logger.info("Worker initialization complete. Starting processing loop...")
|
||||
# Initialize the directory scanner.
|
||||
# It scans the permanent storage location in place (no staging/copy step),
|
||||
# periodically discovering new/removed video files and queueing any that
|
||||
# have not been scanned yet as PENDING for the worker pool to pick up.
|
||||
storage_config = config.get_section("storage")
|
||||
scanner_config = config.get_section("scanner")
|
||||
scanner = DirectoryScanner(
|
||||
db_connector=db,
|
||||
config=config.data,
|
||||
input_path=storage_config.get("input_path", "/data/input"),
|
||||
scan_interval=int(scanner_config.get("scan_interval_seconds", 60)),
|
||||
walker_threads=int(scanner_config.get("walker_threads", 8)),
|
||||
)
|
||||
|
||||
logger.info("Worker initialization complete. Starting scanner and processing loop...")
|
||||
|
||||
# Run the scanner in a background thread (its start() is a blocking loop).
|
||||
scanner_thread = threading.Thread(
|
||||
target=scanner.start, name="directory-scanner", daemon=True
|
||||
)
|
||||
scanner_thread.start()
|
||||
|
||||
pool = WorkerPool(db, config.data, max_workers=1)
|
||||
|
||||
@@ -93,6 +115,7 @@ def main():
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Worker shutting down.")
|
||||
pool.stop()
|
||||
scanner.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+157
-9
@@ -10,9 +10,12 @@ import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
@@ -54,29 +57,164 @@ class DirectoryScanner:
|
||||
self._total_files_modified = 0
|
||||
self._total_files_unscannable = 0
|
||||
|
||||
# Single-instance guard: prevents overlapping scans both within this
|
||||
# process (threading lock) and across replicas (DB lock with a lease).
|
||||
self._instance_id = f"{socket.gethostname()}-{os.getpid()}-{uuid.uuid4().hex[:8]}"
|
||||
self._lock_name = "directory_scanner"
|
||||
self._lock_lease_seconds = int(
|
||||
config.get("scanner", {}).get("lock_lease_seconds", 21600)
|
||||
)
|
||||
self._heartbeat_every = int(
|
||||
config.get("scanner", {}).get("heartbeat_interval_files", 500)
|
||||
)
|
||||
self._scan_lock = threading.Lock()
|
||||
self._lock_table_ensured = False
|
||||
|
||||
def start(self):
|
||||
"""Start the scanner loop."""
|
||||
"""Start the scanner loop.
|
||||
|
||||
Each cycle is guarded so that at most one scan runs at a time:
|
||||
- an in-process threading lock prevents re-entrant scans, and
|
||||
- a DB lock (with a lease) prevents overlapping scans across replicas.
|
||||
A long-running scan keeps its lease alive via heartbeats, so the next
|
||||
scheduled tick (or another replica) waits instead of starting a second
|
||||
parallel scan.
|
||||
"""
|
||||
self._running = True
|
||||
logger.info("Scanner starting: input_path=%s interval=%ds threads=%d",
|
||||
self.input_path, self.scan_interval, self.walker_threads)
|
||||
logger.info(
|
||||
"Scanner starting: input_path=%s interval=%ds threads=%d instance=%s",
|
||||
self.input_path, self.scan_interval, self.walker_threads, self._instance_id,
|
||||
)
|
||||
|
||||
while self._running:
|
||||
# In-process re-entrancy guard: never run two scans at once.
|
||||
if not self._scan_lock.acquire(blocking=False):
|
||||
logger.warning("A scan is already in progress; skipping this cycle.")
|
||||
self._sleep_interval()
|
||||
continue
|
||||
|
||||
try:
|
||||
self._run_scan()
|
||||
if self.acquire_lock():
|
||||
try:
|
||||
self._run_scan()
|
||||
finally:
|
||||
self.release_lock()
|
||||
else:
|
||||
logger.info(
|
||||
"Scanner lock held by another instance; skipping this cycle."
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Scanner error: %s", e, exc_info=True)
|
||||
finally:
|
||||
self._scan_lock.release()
|
||||
|
||||
# Sleep until next scan
|
||||
for _ in range(self.scan_interval):
|
||||
if not self._running:
|
||||
break
|
||||
time.sleep(1)
|
||||
self._sleep_interval()
|
||||
|
||||
def _sleep_interval(self):
|
||||
"""Sleep for the scan interval, waking early if stopped."""
|
||||
for _ in range(self.scan_interval):
|
||||
if not self._running:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
def stop(self):
|
||||
"""Stop the scanner."""
|
||||
self._running = False
|
||||
logger.info("Scanner stopping. Total scans: %d", self._scan_count)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Single-instance lock (cross-process / cross-replica guard)
|
||||
# ------------------------------------------------------------------
|
||||
def _ensure_lock_table(self):
|
||||
"""Create the scanner_lock table if it does not already exist."""
|
||||
if self._lock_table_ensured:
|
||||
return
|
||||
self.db.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS scanner_lock (
|
||||
lock_name VARCHAR(64) PRIMARY KEY,
|
||||
owner VARCHAR(128) NOT NULL,
|
||||
locked_at DATETIME NOT NULL,
|
||||
lease_seconds INT NOT NULL DEFAULT 21600
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci"""
|
||||
)
|
||||
self._lock_table_ensured = True
|
||||
|
||||
def acquire_lock(self) -> bool:
|
||||
"""Attempt to acquire the cross-process scanner lock.
|
||||
|
||||
Returns True if this instance now owns the lock, False otherwise.
|
||||
A stale lock (held longer than the lease) is taken over so a crashed
|
||||
scanner does not block scanning forever.
|
||||
"""
|
||||
try:
|
||||
self._ensure_lock_table()
|
||||
except Exception as e:
|
||||
logger.warning("Could not ensure scanner_lock table: %s", e)
|
||||
return True # fail-open: keep scanning rather than block entirely
|
||||
|
||||
now = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
# 1) Try to insert a fresh lock row.
|
||||
try:
|
||||
self.db.execute(
|
||||
"""INSERT INTO scanner_lock (lock_name, owner, locked_at, lease_seconds)
|
||||
VALUES (%s, %s, %s, %s)""",
|
||||
(self._lock_name, self._instance_id, now, self._lock_lease_seconds),
|
||||
transaction=True,
|
||||
)
|
||||
logger.info("Acquired scanner lock (fresh). owner=%s", self._instance_id)
|
||||
return True
|
||||
except Exception:
|
||||
# Row already exists -> fall through to steal-if-stale.
|
||||
pass
|
||||
|
||||
# 2) Take over the lock if it is stale or already ours.
|
||||
stale_before = now - timedelta(seconds=self._lock_lease_seconds)
|
||||
try:
|
||||
affected = self.db.execute(
|
||||
"""UPDATE scanner_lock
|
||||
SET owner = %s, locked_at = %s
|
||||
WHERE lock_name = %s
|
||||
AND (owner = %s OR locked_at < %s)""",
|
||||
(self._instance_id, now, self._lock_name, self._instance_id, stale_before),
|
||||
transaction=True,
|
||||
)
|
||||
if affected and affected > 0:
|
||||
logger.info("Acquired scanner lock (stale takeover). owner=%s",
|
||||
self._instance_id)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("Failed to check scanner lock: %s", e)
|
||||
return True # fail-open
|
||||
|
||||
logger.info("Scanner lock held by another instance; not acquiring.")
|
||||
return False
|
||||
|
||||
def release_lock(self):
|
||||
"""Release the scanner lock if we own it."""
|
||||
try:
|
||||
self.db.execute(
|
||||
"""DELETE FROM scanner_lock WHERE lock_name = %s AND owner = %s""",
|
||||
(self._lock_name, self._instance_id),
|
||||
transaction=True,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to release scanner lock: %s", e)
|
||||
|
||||
def _heartbeat(self):
|
||||
"""Refresh the lock lease so a long-running scan is not stolen."""
|
||||
try:
|
||||
now = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
self.db.execute(
|
||||
"""UPDATE scanner_lock SET locked_at = %s
|
||||
WHERE lock_name = %s AND owner = %s""",
|
||||
(now, self._lock_name, self._instance_id),
|
||||
transaction=True,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Scanner lock heartbeat failed: %s", e)
|
||||
|
||||
def _run_scan(self):
|
||||
"""Execute a single scan cycle."""
|
||||
scan_start = time.time()
|
||||
@@ -181,6 +319,7 @@ class DirectoryScanner:
|
||||
def _process_files_batch(self, files: List[Path]) -> List[dict]:
|
||||
"""Process a batch of files in parallel."""
|
||||
results = []
|
||||
processed = 0
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.walker_threads) as executor:
|
||||
future_to_file = {
|
||||
@@ -201,6 +340,11 @@ class DirectoryScanner:
|
||||
"error_message": str(e),
|
||||
})
|
||||
|
||||
processed += 1
|
||||
# Keep the single-instance lock alive during long scans.
|
||||
if self._heartbeat_every and processed % self._heartbeat_every == 0:
|
||||
self._heartbeat()
|
||||
|
||||
return results
|
||||
|
||||
def _process_single_file(self, file_path: Path) -> dict:
|
||||
@@ -384,6 +528,10 @@ class DirectoryScanner:
|
||||
"total_files_new": self._total_files_new,
|
||||
"total_files_modified": self._total_files_modified,
|
||||
"total_files_unscannable": self._total_files_unscannable,
|
||||
"instance_id": self._instance_id,
|
||||
"scan_interval_seconds": self.scan_interval,
|
||||
"lock_lease_seconds": self._lock_lease_seconds,
|
||||
"scan_in_progress": self._scan_lock.locked(),
|
||||
"input_path": str(self.input_path),
|
||||
"is_running": self._running,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user