From 554f20a1e2c7c6dfa133c61c32e762699167a1c5 Mon Sep 17 00:00:00 2001 From: Ryan Shpeherd Date: Wed, 28 Oct 2026 09:32:41 -0400 Subject: [PATCH] Small fixes --- aiworker/docker-compose.yml | 3 + aiworker/run.pl | 4 +- scanner/fix_hashes.pl | 105 ++++++++++++++++++++++++++ scanner/scanner.pl | 25 ++++++- transcoder/transcoder.pl | 144 ++++++++++++++++++++++++++++++++++++ 5 files changed, 276 insertions(+), 5 deletions(-) create mode 100755 scanner/fix_hashes.pl create mode 100755 transcoder/transcoder.pl diff --git a/aiworker/docker-compose.yml b/aiworker/docker-compose.yml index c7c6288..629cd14 100644 --- a/aiworker/docker-compose.yml +++ b/aiworker/docker-compose.yml @@ -13,6 +13,9 @@ services: - OLLAMA_URL=${OLLAMA_URL:-http://10.0.1.7:11434} - SECONDS_PER_FRAME=${SECONDS_PER_FRAME:-30} - TASK_COUNT=${TASK_COUNT:-50} + - SENSITIVITY=${SENSITIVITY:-30} + - IMAGE_COUNT=${IMAGE_COUNT:-10} + - MIN_TIME=${MIN_TIME:-5} volumes: - nas_input:/data:ro - ./run.pl:/app/run.pl:ro diff --git a/aiworker/run.pl b/aiworker/run.pl index c105fa9..66df673 100755 --- a/aiworker/run.pl +++ b/aiworker/run.pl @@ -11,7 +11,7 @@ my $api = $ENV{'API_BASE_URL'} || 'http://10.0.1.7:3000'; my $task_count = $ENV{'TASK_COUNT'} || 1; my $image_count = $ENV{'IMAGE_COUNT'} || 10; # Number of images to extract from the video my $min_time = $ENV{'MIN_TIME'} || 5; # Minimum time between images -my $sensitivity = 20; +my $sensitivity = $ENV{'SENSITIVITY'} || 30; my $resolution = "672:672"; #my $model = "hf.co/Jommarn/UNSEEN_Gemma_4_26B_NSFW-GGUF:Q8_0"; my $model = "hf.co/Jommarn/UNSEEN_Gemma_4_26B_NSFW-GGUF:Q5_K_M"; @@ -19,7 +19,7 @@ my $model = "hf.co/Jommarn/UNSEEN_Gemma_4_26B_NSFW-GGUF:Q5_K_M"; chdir("/tmp") || die "Unable to change directory to /tmp: $!\n"; my $ua = LWP::UserAgent->new; -$ua->timeout(60); +$ua->timeout(300); if(@ARGV) { CheckPath($ARGV[0], {'duration' => 0}); diff --git a/scanner/fix_hashes.pl b/scanner/fix_hashes.pl new file mode 100755 index 0000000..fa61893 --- /dev/null +++ b/scanner/fix_hashes.pl @@ -0,0 +1,105 @@ +#!/usr/bin/perl +use strict; +use warnings; +use DBI; +use Digest::SHA qw(sha256_hex); +use JSON; +use String::ShellQuote qw/shell_quote/; + +my $db_host = $ENV{'DB_HOST'} || 'mariadb'; +my $db_name = $ENV{'DB_NAME'} || 'videodetect'; +my $db_user = $ENV{'DB_USER'} || 'videodetect'; +my $db_pass = $ENV{'DB_PASSWORD'} || 'videodetect'; + +$SIG{INT} = sub { die "Interrupted\n"; }; + +my $dbh = DBI->connect( + "DBI:mysql:database=$db_name;host=$db_host;mysql_enable_utf8mb4=1", + $db_user, + $db_pass, + { mysql_auto_reconnect => 1 } +); + + +my $sth = $dbh->prepare("SELECT id, file_path FROM videos ORDER BY id"); +my $upd = $dbh->prepare("UPDATE videos SET file_size=?, file_hash=?, resolution_w=?, resolution_h=?, codec=?, duration=? WHERE id=?"); +$sth->execute(); +while (my($id,$path)=$sth->fetchrow_array()) { + print "$path\n"; + my $info = get_video_info($path); + if ($info) { + $upd->execute( + -s $path, + $info->{file_hash}, + $info->{resolution_w}, + $info->{resolution_h}, + $info->{codec}, + $info->{duration}, + $id + ); + } +} +$sth->finish(); +$upd->finish(); +$dbh->disconnect(); + +sub get_video_info { + my ($file_path) = @_; + + # --- Compute SHA-256 hash of the file --- + return undef unless -f $file_path && -r $file_path; + open(my $fh, '<:raw', $file_path) or do { warn "Cannot open $file_path: $!"; return undef; }; + my $hash = Digest::SHA->new(256)->addfile($fh)->hexdigest; + close($fh); + + + + # --- Run ffprobe to extract metadata --- + my $probe_cmd = 'ffprobe -v quiet -print_format json -show_format -show_streams ' . shell_quote($file_path); + my $output = `$probe_cmd`; + return undef unless defined $output && length($output); + + my $json = JSON->new->utf8->canonical(1); + my $data = $json->decode($output); + + # --- Extract codec from video stream (prefer first video stream found) --- + my $codec = undef; + if (exists $data->{streams} && ref($data->{streams}) eq 'ARRAY') { + for my $stream (@{$data->{streams}}) { + if ($stream->{codec_type} eq 'video') { + $codec = $stream->{codec_name}; + last; + } + } + } + + # --- Extract resolution and duration from format/streams --- + my ($resolution_w, $resolution_h, $duration); + + # Duration from format first, then stream + if (exists $data->{format} && exists $data->{format}{duration}) { + $duration = $data->{format}{duration} + 0; # force numeric + } elsif (exists $data->{streams}[0] && exists $data->{streams}[0]{duration}) { + $duration = $data->{streams}[0]{duration} + 0; + } + + # Resolution from video stream first, then format side data + if (exists $data->{streams} && ref($data->{streams}) eq 'ARRAY') { + for my $stream (@{$data->{streams}}) { + next unless $stream->{codec_type} eq 'video'; + if ($stream->{width} && $stream->{height}) { + $resolution_w = $stream->{width}; + $resolution_h = $stream->{height}; + last; + } + } + } + + return { + file_hash => $hash, + resolution_w => $resolution_w // 0, + resolution_h => $resolution_h // 0, + codec => $codec // 'unknown', + duration => $duration, + }; +} diff --git a/scanner/scanner.pl b/scanner/scanner.pl index 7a59c14..26279a0 100755 --- a/scanner/scanner.pl +++ b/scanner/scanner.pl @@ -82,8 +82,9 @@ sub process_file { my $file_size = -s $file_path; if ($file_size != $row->{file_size}) { warn "File size mismatch for $file_path. Updating record."; - my $update_sth = $dbh->prepare("UPDATE videos SET file_size=?, last_scan_time=NOW() WHERE id=?"); - $update_sth->execute($file_size, $row->{id}); + my $info = get_video_info($file_path); + my $update_sth = $dbh->prepare("UPDATE videos SET file_size=?, last_scan_time=NOW(), resolution_w=?, resolution_h=?, codec=?, duration=?, updated_at=NOW() WHERE id=?"); + $update_sth->execute($file_size, $info->{resolution_w}, $info->{resolution_h}, $info->{codec}, $info->{duration}, $row->{id}); $update_sth->finish(); create_aiscan_task($row->{id}); } else { @@ -101,6 +102,24 @@ sub process_file { warn "Failed to get video info for $file_path. Skipping."; return; } + + $sth = $dbh->prepare("SELECT id,file_path FROM videos WHERE file_size=? AND file_hash=?"); + $sth->execute(-s $file_path, $info->{file_hash}); + while(my $row = $sth->fetchrow_hashref()) { + if(-e $row->{file_path}) { + print "Duplicate found: $file_path matches existing file $row->{file_path}\n"; + } else { + print "MOVED FILE: $file_path previously existed as $row->{file_path}\n"; + my $upd = $dbh->prepare("UPDATE videos SET file_path=?,last_scan_time=NOW(),updated_at=NOW() WHERE id=?"); + $upd->execute($file_path, $row->{id}); + $upd->finish(); + $sth->finish(); + return; + } + } + $sth->finish(); + + $sth = $dbh->prepare("INSERT INTO videos (file_path, file_size, file_hash, resolution_w, resolution_h, codec, duration, last_scan_time) VALUES (?, ?, ?, ?, ?, ?, ?, NOW())"); $sth->execute( $file_path, @@ -136,7 +155,7 @@ sub get_video_info { # --- Compute SHA-256 hash of the file --- return undef unless -f $file_path && -r $file_path; open(my $fh, '<:raw', $file_path) or do { warn "Cannot open $file_path: $!"; return undef; }; - my $hash = sha256_hex($fh); + my $hash = Digest::SHA->new(256)->addfile($fh)->hexdigest; close($fh); diff --git a/transcoder/transcoder.pl b/transcoder/transcoder.pl new file mode 100755 index 0000000..25c91de --- /dev/null +++ b/transcoder/transcoder.pl @@ -0,0 +1,144 @@ +#!/usr/bin/perl +use strict; +use warnings; +use DBI; +use Digest::SHA qw(sha256_hex); +use JSON; +use File::Basename qw(dirname basename); +use File::Copy qw(move); +use String::ShellQuote qw/shell_quote/; + +my $db_host = $ENV{'DB_HOST'} || 'mariadb'; +my $db_name = $ENV{'DB_NAME'} || 'videodetect'; +my $db_user = $ENV{'DB_USER'} || 'videodetect'; +my $db_pass = $ENV{'DB_PASSWORD'} || 'videodetect'; +my $tasks = $ENV{'TASK_COUNT'} || 1; + +$SIG{INT} = sub { die "Interrupted\n"; }; + +my $dbh = DBI->connect( + "DBI:mysql:database=$db_name;host=$db_host;mysql_enable_utf8mb4=1", + $db_user, + $db_pass, + { mysql_auto_reconnect => 1 } +); + +my $sth_claimtask = $dbh->prepare(" +UPDATE tasks +SET status='IN_PROGRESS', updated_at=NOW(), assign_key=?, assigned_at=NOW() +WHERE status='PENDING' AND task_type='TRANSCODE' AND assign_key IS NULL +LIMIT 1 +"); +my $sth_gettask = $dbh->prepare("SELECT * FROM tasks WHERE assign_key=? AND status='IN_PROGRESS' LIMIT 1"); +my $sth_getvideo = $dbh->prepare("SELECT * from videos WHERE id=?"); +my $sth_updatevideo = $dbh->prepare("UPDATE videos SET file_path=?, file_size=?, file_hash=?, updated_at=NOW() WHERE id=?"); +my $sth_finishtask = $dbh->prepare("UPDATE tasks SET status='COMPLETED', updated_at=NOW(), assign_key=NULL, assigned_at=NULL, results=? WHERE id=?"); + +for(my $i=0; $i<$tasks; $i++) { + my $assign_key = "worker_" . int(rand(10000000)); + print "Assigning task with key: $assign_key\n"; + my $count = $sth_claimtask->execute($assign_key); + if($count != 1) { + print "Failed to claim task\n"; + last; + } + + $sth_gettask->execute($assign_key); + my $task = $sth_gettask->fetchrow_hashref(); + if(!ref $task) { + print "Failed to fetch task\n"; + last; + } + + my $video_id = $task->{video_id}; + $sth_getvideo->execute($video_id); + my $video = $sth_getvideo->fetchrow_hashref(); + if ($video) { + print "Processing video: $video->{id}\n"; + } + + my $results = transcode_video($video); + if ($results->{success}) { + my $file_path = $results->{output}; + my $file_size = -s $file_path; + my $file_hash = $results->{newhash}; + delete $results->{newhash}; + $sth_updatevideo->execute($file_path, $file_size, $file_hash, $video_id); + $sth_finishtask->execute(JSON->new->encode($results), $task->{id}); + } +} + +sub get_hash { + my ($file_path) = @_; + # --- Compute SHA-256 hash of the file --- + return undef unless -f $file_path && -r $file_path; + open(my $fh, '<:raw', $file_path) or do { warn "Cannot open $file_path: $!"; return undef; }; + my $hash = sha256_hex($fh); + close($fh); + return $hash; +} + +sub transcode_video { + my ($video, $retry) = @_; + + my $file = $video ? $video->{file_path} : undef; + + # --- validate --- + unless (defined $file && -f $file) { + warn "SKIP '" . ($file // 'undef') . "' not found\n"; + return { success => 0, reason => 'not_found' }; + } + + # --- derive paths --- + my $dir = dirname($file); + my $base = basename($file); + my $name = $base; + $name =~ s/\.[^.]*$//; # strip last extension ("foo.mkv" -> "foo") + $name = $base if $name eq ''; # guard against dot-only names + + my $output = "$dir/$name.mp4"; # equals $file when it's already .mp4 + my $tmp = "/tmp/${name}~.mp4"; + + # --- transcode --- + print "WORK $file → $output\n"; + + my $cmd = 'ffmpeg -y -nostdin ' + . '-hwaccel cuda ' + . ($retry ? '' : '-hwaccel_output_format cuda ') + . '-i ' . shell_quote($file) . ' ' + . '-c:v h264_nvenc -preset p6 -rc vbr -cq 23 -b:v 0 ' + . ($retry ? '-pix_fmt yuv420p ' : '') + . '-c:a aac -b:a 128k ' + . '-movflags +faststart ' + . shell_quote($tmp); + + my $log = `$cmd`; + my $exit_code = $? >> 8; + + if ($exit_code == 0) { + my $newhash = get_hash($tmp); + # success — delete original, move transcoded file into place + unless (unlink($file)) { + warn " WARN could not delete original '$file': $!\n"; + } + unless (move($tmp, $output)) { + warn " FAIL could not move '$tmp' -> '$output': $!\n"; + return { success => 0, reason => "move_failed: $!" }; + } + print " OK original removed, output: $output\n"; + return { success => 1, output => $output, original => $file, newhash => $newhash }; + } + + # failure — clean up temp file, keep original + unlink($tmp) if -e $tmp; + warn " FAIL keeping original: $file\n"; + warn $log if defined $log && length $log; + + if($retry) { + return { success => 0, reason => 'transcode_failed', exit_code => $exit_code }; + } else { + return transcode_video($video, 1); # retry with $retry set to 1 + } +} + +