After story 1

2026-08-03 11:30:49 -04:00
commit 82590c392f
32 changed files with 4375 additions and 0 deletions
@@ -0,0 +1,248 @@
+# VideoDetect Configuration
+# All values can be overridden by environment variables (e.g., VD_SAMPLING_INTERVAL_SECONDS)
+
+# -----------------------------------------------------
+# Frame Sampling
+# -----------------------------------------------------
+sampling:
+  interval_seconds: 30
+  override_per_job: true
+  quality: 2  # JPEG quality (1-31, lower=better quality)
+  format: jpeg
+
+# -----------------------------------------------------
+# Confidence Thresholds
+# -----------------------------------------------------
+thresholds:
+  T_high: 0.75  # C >= T_high → MATCH
+  T_low: 0.45   # T_low <= C < T_high → REVIEW; C < T_low → SKIP
+
+# -----------------------------------------------------
+# GPU Configuration
+# -----------------------------------------------------
+gpu:
+  max_memory_gb: 18
+  batch_size: auto  # auto-tune based on available VRAM
+  device: cuda
+
+# -----------------------------------------------------
+# Storage Paths
+# -----------------------------------------------------
+storage:
+  scratch_path: /scratch
+  input_path: /data/input
+  output_path: /data/output
+  models_path: /models
+  training_path: /data/training
+
+# -----------------------------------------------------
+# Database
+# -----------------------------------------------------
+database:
+  host: mariadb
+  port: 3306
+  name: videodetect
+  user: videodetect
+  password: videodetect123
+  pool_size: 20
+  pool_min: 5
+  pool_recycle: 3600
+
+# -----------------------------------------------------
+# Logging
+# -----------------------------------------------------
+logging:
+  format: json
+  level: INFO
+  rotation_max_bytes: 104857600  # 100MB
+  rotation_backup_count: 10
+
+# -----------------------------------------------------
+# Model Configuration
+# -----------------------------------------------------
+model:
+  face_detector: yolo8n
+  face_detector_path: /models/face_detector/face_detector.trt
+  classifier: mobilenetv3-small
+  classifier_path: /models/classifier/classifier.trt
+  input_size: 224
+
+# -----------------------------------------------------
+# Directory Scanner
+# -----------------------------------------------------
+scanner:
+  scan_interval_seconds: 60
+  walker_threads: 8
+  ffprobe_timeout_seconds: 10
+  hash_algorithm: sha256
+  hash_chunk_size_mb: 1
+
+# -----------------------------------------------------
+# Codec Validation
+# -----------------------------------------------------
+codec:
+  whitelist:
+    - avc1      # H.264
+    - hevc      # H.265
+    - vp8
+    - vp9
+    - av01      # AV1
+    - mjpeg
+    - mp4v
+  default_status_on_error: UNSCANNABLE
+
+# -----------------------------------------------------
+# Processing Queue
+# -----------------------------------------------------
+queue:
+  priority: modification_time  # newest first
+  max_concurrent_per_gpu: 1
+  lock_timeout_seconds: 300
+
+# -----------------------------------------------------
+# Face Detection
+# -----------------------------------------------------
+face_detection:
+  model: yolo8n
+  model_path: /models/face_detector/face_detector.trt
+  input_size: 640
+  confidence_threshold: 0.25
+  iou_threshold: 0.45
+  max_faces_per_frame: 10
+  max_faces_per_video: 100
+
+# -----------------------------------------------------
+# Batching
+# -----------------------------------------------------
+batching:
+  max_batch_size: 16
+  batch_timeout_ms: 100
+  vram_target_gb: 16
+  vram_reduce_threshold_gb: 16
+  vram_increase_threshold_gb: 10
+
+# -----------------------------------------------------
+# Classification
+# -----------------------------------------------------
+classifier:
+  model: mobilenetv3-small
+  model_path: /models/classifier/classifier.trt
+  input_size: 224
+  temperature: 1.0  # calibration temperature
+  default_strategy: max  # max, weighted_mean, top_k_mean
+
+# -----------------------------------------------------
+# Confidence Aggregation
+# -----------------------------------------------------
+aggregation:
+  strategy: max
+  alpha: 1.0   # for weighted_mean
+  beta: 0.1    # for weighted_mean
+  top_k: 3     # for top_k_mean
+
+# -----------------------------------------------------
+# Routing
+# -----------------------------------------------------
+routing:
+  T_high: 0.75
+  T_low: 0.45
+  no_faces_decision: SKIP
+
+# -----------------------------------------------------
+# Results Persistence
+# -----------------------------------------------------
+results:
+  persist_before_cleanup: true
+  transaction_safe: true
+  state_guard: true  # only update PROCESSING → COMPLETED
+
+# -----------------------------------------------------
+# Data Export
+# -----------------------------------------------------
+export:
+  format: parquet  # parquet, jsonl, or both
+  output_path: /data/output
+  compression: snappy
+  batch_size: 100  # export after N videos
+  include_frame_confidences: true
+
+# -----------------------------------------------------
+# Review UI
+# -----------------------------------------------------
+review_ui:
+  host: "0.0.0.0"
+  port: 5000
+  per_page: 20
+  top_k_frames: 5
+  export_path: /data/output/reviews
+  auth_enabled: false
+  ssl_enabled: false
+
+# -----------------------------------------------------
+# Active Learning
+# -----------------------------------------------------
+active_learning:
+  enabled: true
+  min_annotated_samples: 100
+  training:
+    epochs: 20
+    batch_size: 32
+    learning_rate: 0.001
+    weight_decay: 0.01
+    early_stopping_patience: 5
+    lr_scheduler: ReduceLROnPlateau
+    lr_factor: 0.5
+    lr_patience: 3
+  validation:
+    val_split: 0.2
+    min_f1_improvement: 0.02
+    max_ece: 0.08
+  deployment:
+    auto_deploy: true
+    hot_reload: true
+    rollback_enabled: true
+  augmentation:
+    horizontal_flip: true
+    color_jitter: true
+    affine: true
+    affine_degrees: 10
+    affine_scale: 0.1
+
+# -----------------------------------------------------
+# Monitoring
+# -----------------------------------------------------
+monitoring:
+  prometheus:
+    enabled: true
+    port: 9090
+    metrics_path: /metrics
+  grafana:
+    enabled: true
+    port: 3000
+  alerts:
+    confidence_drift_threshold: 0.10
+    review_queue_max_size: 1000
+    review_queue_max_age_hours: 24
+    throughput_min_videos_per_hour: 20
+    throughput_min_duration_hours: 1
+    error_rate_threshold: 0.05
+    error_rate_window_hours: 1
+  drift_detection:
+    enabled: true
+    schedule: "0 2 * * 0"  # cron: Sundays at 2 AM
+    baseline_source: db
+  crash_recovery:
+    lock_timeout_minutes: 5
+    auto_requeue: true
+  retry:
+    max_attempts: 3
+    initial_delay: 1.0
+    backoff_factor: 2.0
+    retryable_errors:
+      - timeout
+      - gpu_oom
+      - file_lock
+    non_retryable_errors:
+      - codec_unsupported
+      - file_corrupt
+      - invalid_path