262 lines
7.3 KiB
YAML
262 lines
7.3 KiB
YAML
# VideoDetect Configuration
|
|
# All values can be overridden by environment variables (e.g., VD_SAMPLING_INTERVAL_SECONDS)
|
|
|
|
# -----------------------------------------------------
|
|
# Frame Sampling
|
|
# -----------------------------------------------------
|
|
sampling:
|
|
interval_seconds: 30
|
|
override_per_job: true
|
|
quality: 2 # JPEG quality (1-31, lower=better quality)
|
|
format: jpeg
|
|
|
|
# -----------------------------------------------------
|
|
# Confidence Thresholds
|
|
# -----------------------------------------------------
|
|
thresholds:
|
|
T_high: 0.75 # C >= T_high → MATCH
|
|
T_low: 0.45 # T_low <= C < T_high → REVIEW; C < T_low → SKIP
|
|
|
|
# -----------------------------------------------------
|
|
# GPU Configuration
|
|
# -----------------------------------------------------
|
|
gpu:
|
|
max_memory_gb: 18
|
|
batch_size: auto # auto-tune based on available VRAM
|
|
device: cuda
|
|
|
|
# -----------------------------------------------------
|
|
# Storage Paths
|
|
# -----------------------------------------------------
|
|
storage:
|
|
scratch_path: /scratch
|
|
input_path: /data/input
|
|
output_path: /data/output
|
|
models_path: /models
|
|
training_path: /data/training
|
|
|
|
# -----------------------------------------------------
|
|
# Database
|
|
# -----------------------------------------------------
|
|
database:
|
|
host: mariadb
|
|
port: 3306
|
|
name: videodetect
|
|
user: videodetect
|
|
password: videodetect123
|
|
pool_size: 20
|
|
pool_min: 5
|
|
pool_recycle: 3600
|
|
|
|
# -----------------------------------------------------
|
|
# Logging
|
|
# -----------------------------------------------------
|
|
logging:
|
|
format: json
|
|
level: INFO
|
|
rotation_max_bytes: 104857600 # 100MB
|
|
rotation_backup_count: 10
|
|
|
|
# -----------------------------------------------------
|
|
# Model Configuration
|
|
# -----------------------------------------------------
|
|
model:
|
|
face_detector: yolo8n
|
|
face_detector_path: /models/face_detector/face_detector.trt
|
|
classifier: mobilenetv3-small
|
|
classifier_path: /models/classifier/classifier.trt
|
|
input_size: 224
|
|
|
|
# -----------------------------------------------------
|
|
# Directory Scanner
|
|
# -----------------------------------------------------
|
|
# Scans the permanent storage location in place (no staging/copy step).
|
|
# The corpus is large (~163k files / ~41TB), so the default interval is 2 hours.
|
|
# A single-instance guard (in-process lock + DB lock with a lease) ensures a
|
|
# long-running scan never overlaps another scan, even across replicas.
|
|
scanner:
|
|
scan_interval_seconds: 7200 # 2 hours (adaptive: raise for very large corpora)
|
|
walker_threads: 8
|
|
ffprobe_timeout_seconds: 10
|
|
hash_algorithm: sha256
|
|
hash_chunk_size_mb: 1
|
|
lock_lease_seconds: 21600 # 6 hours: max time a scan may hold the lock before it is considered stale
|
|
heartbeat_interval_files: 500 # refresh the lock lease every N files processed
|
|
|
|
# -----------------------------------------------------
|
|
# Codec Validation
|
|
# -----------------------------------------------------
|
|
codec:
|
|
whitelist:
|
|
- avc1 # H.264 (MP4 container tag)
|
|
- h264 # H.264 (ffprobe codec name)
|
|
- hevc # H.265 (MP4 container tag)
|
|
- h265 # H.265 (ffprobe codec name)
|
|
- vp8
|
|
- vp9
|
|
- av01 # AV1
|
|
- mjpeg
|
|
- mp4v
|
|
- wmv3 # Windows Media Video 7
|
|
- mpeg4 # MPEG-4 Part 2
|
|
default_status_on_error: UNSCANNABLE
|
|
|
|
# -----------------------------------------------------
|
|
# Processing Queue
|
|
# -----------------------------------------------------
|
|
queue:
|
|
priority: modification_time # newest first
|
|
max_concurrent_per_gpu: 1
|
|
lock_timeout_seconds: 300
|
|
|
|
# -----------------------------------------------------
|
|
# Face Detection
|
|
# -----------------------------------------------------
|
|
face_detection:
|
|
model: yolo8n
|
|
model_path: /models/face_detector/face_detector.trt
|
|
input_size: 640
|
|
confidence_threshold: 0.25
|
|
iou_threshold: 0.45
|
|
max_faces_per_frame: 10
|
|
max_faces_per_video: 100
|
|
|
|
# -----------------------------------------------------
|
|
# Batching
|
|
# -----------------------------------------------------
|
|
batching:
|
|
max_batch_size: 16
|
|
batch_timeout_ms: 100
|
|
vram_target_gb: 16
|
|
vram_reduce_threshold_gb: 16
|
|
vram_increase_threshold_gb: 10
|
|
|
|
# -----------------------------------------------------
|
|
# Classification
|
|
# -----------------------------------------------------
|
|
classifier:
|
|
model: mobilenetv3-small
|
|
model_path: /models/classifier/classifier.trt
|
|
input_size: 224
|
|
temperature: 1.0 # calibration temperature
|
|
default_strategy: max # max, weighted_mean, top_k_mean
|
|
|
|
# -----------------------------------------------------
|
|
# Confidence Aggregation
|
|
# -----------------------------------------------------
|
|
aggregation:
|
|
strategy: max
|
|
alpha: 1.0 # for weighted_mean
|
|
beta: 0.1 # for weighted_mean
|
|
top_k: 3 # for top_k_mean
|
|
|
|
# -----------------------------------------------------
|
|
# Routing
|
|
# -----------------------------------------------------
|
|
routing:
|
|
T_high: 0.75
|
|
T_low: 0.45
|
|
no_faces_decision: SKIP
|
|
|
|
# -----------------------------------------------------
|
|
# Results Persistence
|
|
# -----------------------------------------------------
|
|
results:
|
|
persist_before_cleanup: true
|
|
transaction_safe: true
|
|
state_guard: true # only update PROCESSING → COMPLETED
|
|
|
|
# -----------------------------------------------------
|
|
# Data Export
|
|
# -----------------------------------------------------
|
|
export:
|
|
format: parquet # parquet, jsonl, or both
|
|
output_path: /data/output
|
|
compression: snappy
|
|
batch_size: 100 # export after N videos
|
|
include_frame_confidences: true
|
|
|
|
# -----------------------------------------------------
|
|
# Review UI
|
|
# -----------------------------------------------------
|
|
review_ui:
|
|
host: "0.0.0.0"
|
|
port: 5000
|
|
per_page: 20
|
|
top_k_frames: 5
|
|
export_path: /data/output/reviews
|
|
auth_enabled: false
|
|
ssl_enabled: false
|
|
|
|
# -----------------------------------------------------
|
|
# Active Learning
|
|
# -----------------------------------------------------
|
|
active_learning:
|
|
enabled: true
|
|
min_annotated_samples: 100
|
|
seed: 42 # RNG seed for deterministic train/val splits and training
|
|
training:
|
|
epochs: 20
|
|
batch_size: 32
|
|
learning_rate: 0.001
|
|
weight_decay: 0.01
|
|
early_stopping_patience: 5
|
|
lr_scheduler: ReduceLROnPlateau
|
|
lr_factor: 0.5
|
|
lr_patience: 3
|
|
validation:
|
|
val_split: 0.2
|
|
min_f1_improvement: 0.02
|
|
max_ece: 0.08
|
|
deployment:
|
|
auto_deploy: true
|
|
hot_reload: true
|
|
rollback_enabled: true
|
|
augmentation:
|
|
horizontal_flip: true
|
|
color_jitter: true
|
|
affine: true
|
|
affine_degrees: 10
|
|
affine_scale: 0.1
|
|
|
|
# -----------------------------------------------------
|
|
# Monitoring
|
|
# -----------------------------------------------------
|
|
monitoring:
|
|
prometheus:
|
|
enabled: true
|
|
port: 9090
|
|
metrics_path: /metrics
|
|
grafana:
|
|
enabled: true
|
|
port: 3000
|
|
alerts:
|
|
confidence_drift_threshold: 0.10
|
|
review_queue_max_size: 1000
|
|
review_queue_max_age_hours: 24
|
|
throughput_min_videos_per_hour: 20
|
|
throughput_min_duration_hours: 1
|
|
error_rate_threshold: 0.05
|
|
error_rate_window_hours: 1
|
|
drift_detection:
|
|
enabled: true
|
|
schedule: "0 2 * * 0" # cron: Sundays at 2 AM
|
|
baseline_source: db
|
|
health_check:
|
|
port: 8080
|
|
crash_recovery:
|
|
lock_timeout_minutes: 5
|
|
auto_requeue: true
|
|
retry:
|
|
max_attempts: 3
|
|
initial_delay: 1.0
|
|
backoff_factor: 2.0
|
|
retryable_errors:
|
|
- timeout
|
|
- gpu_oom
|
|
- file_lock
|
|
non_retryable_errors:
|
|
- codec_unsupported
|
|
- file_corrupt
|
|
- invalid_path
|