Small fixes
This commit is contained in:
Ryan Shpeherd
@@ -13,6 +13,9 @@ services:
|
||||
- OLLAMA_URL=${OLLAMA_URL:-http://10.0.1.7:11434}
|
||||
- SECONDS_PER_FRAME=${SECONDS_PER_FRAME:-30}
|
||||
- TASK_COUNT=${TASK_COUNT:-50}
|
||||
- SENSITIVITY=${SENSITIVITY:-30}
|
||||
- IMAGE_COUNT=${IMAGE_COUNT:-10}
|
||||
- MIN_TIME=${MIN_TIME:-5}
|
||||
volumes:
|
||||
- nas_input:/data:ro
|
||||
- ./run.pl:/app/run.pl:ro
|
||||
|
||||
+2
-2
@@ -11,7 +11,7 @@ my $api = $ENV{'API_BASE_URL'} || 'http://10.0.1.7:3000';
|
||||
my $task_count = $ENV{'TASK_COUNT'} || 1;
|
||||
my $image_count = $ENV{'IMAGE_COUNT'} || 10; # Number of images to extract from the video
|
||||
my $min_time = $ENV{'MIN_TIME'} || 5; # Minimum time between images
|
||||
my $sensitivity = 20;
|
||||
my $sensitivity = $ENV{'SENSITIVITY'} || 30;
|
||||
my $resolution = "672:672";
|
||||
#my $model = "hf.co/Jommarn/UNSEEN_Gemma_4_26B_NSFW-GGUF:Q8_0";
|
||||
my $model = "hf.co/Jommarn/UNSEEN_Gemma_4_26B_NSFW-GGUF:Q5_K_M";
|
||||
@@ -19,7 +19,7 @@ my $model = "hf.co/Jommarn/UNSEEN_Gemma_4_26B_NSFW-GGUF:Q5_K_M";
|
||||
chdir("/tmp") || die "Unable to change directory to /tmp: $!\n";
|
||||
|
||||
my $ua = LWP::UserAgent->new;
|
||||
$ua->timeout(60);
|
||||
$ua->timeout(300);
|
||||
|
||||
if(@ARGV) {
|
||||
CheckPath($ARGV[0], {'duration' => 0});
|
||||
|
||||
Executable
+105
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/perl
|
||||
use strict;
|
||||
use warnings;
|
||||
use DBI;
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use JSON;
|
||||
use String::ShellQuote qw/shell_quote/;
|
||||
|
||||
my $db_host = $ENV{'DB_HOST'} || 'mariadb';
|
||||
my $db_name = $ENV{'DB_NAME'} || 'videodetect';
|
||||
my $db_user = $ENV{'DB_USER'} || 'videodetect';
|
||||
my $db_pass = $ENV{'DB_PASSWORD'} || 'videodetect';
|
||||
|
||||
$SIG{INT} = sub { die "Interrupted\n"; };
|
||||
|
||||
my $dbh = DBI->connect(
|
||||
"DBI:mysql:database=$db_name;host=$db_host;mysql_enable_utf8mb4=1",
|
||||
$db_user,
|
||||
$db_pass,
|
||||
{ mysql_auto_reconnect => 1 }
|
||||
);
|
||||
|
||||
|
||||
my $sth = $dbh->prepare("SELECT id, file_path FROM videos ORDER BY id");
|
||||
my $upd = $dbh->prepare("UPDATE videos SET file_size=?, file_hash=?, resolution_w=?, resolution_h=?, codec=?, duration=? WHERE id=?");
|
||||
$sth->execute();
|
||||
while (my($id,$path)=$sth->fetchrow_array()) {
|
||||
print "$path\n";
|
||||
my $info = get_video_info($path);
|
||||
if ($info) {
|
||||
$upd->execute(
|
||||
-s $path,
|
||||
$info->{file_hash},
|
||||
$info->{resolution_w},
|
||||
$info->{resolution_h},
|
||||
$info->{codec},
|
||||
$info->{duration},
|
||||
$id
|
||||
);
|
||||
}
|
||||
}
|
||||
$sth->finish();
|
||||
$upd->finish();
|
||||
$dbh->disconnect();
|
||||
|
||||
sub get_video_info {
|
||||
my ($file_path) = @_;
|
||||
|
||||
# --- Compute SHA-256 hash of the file ---
|
||||
return undef unless -f $file_path && -r $file_path;
|
||||
open(my $fh, '<:raw', $file_path) or do { warn "Cannot open $file_path: $!"; return undef; };
|
||||
my $hash = Digest::SHA->new(256)->addfile($fh)->hexdigest;
|
||||
close($fh);
|
||||
|
||||
|
||||
|
||||
# --- Run ffprobe to extract metadata ---
|
||||
my $probe_cmd = 'ffprobe -v quiet -print_format json -show_format -show_streams ' . shell_quote($file_path);
|
||||
my $output = `$probe_cmd`;
|
||||
return undef unless defined $output && length($output);
|
||||
|
||||
my $json = JSON->new->utf8->canonical(1);
|
||||
my $data = $json->decode($output);
|
||||
|
||||
# --- Extract codec from video stream (prefer first video stream found) ---
|
||||
my $codec = undef;
|
||||
if (exists $data->{streams} && ref($data->{streams}) eq 'ARRAY') {
|
||||
for my $stream (@{$data->{streams}}) {
|
||||
if ($stream->{codec_type} eq 'video') {
|
||||
$codec = $stream->{codec_name};
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# --- Extract resolution and duration from format/streams ---
|
||||
my ($resolution_w, $resolution_h, $duration);
|
||||
|
||||
# Duration from format first, then stream
|
||||
if (exists $data->{format} && exists $data->{format}{duration}) {
|
||||
$duration = $data->{format}{duration} + 0; # force numeric
|
||||
} elsif (exists $data->{streams}[0] && exists $data->{streams}[0]{duration}) {
|
||||
$duration = $data->{streams}[0]{duration} + 0;
|
||||
}
|
||||
|
||||
# Resolution from video stream first, then format side data
|
||||
if (exists $data->{streams} && ref($data->{streams}) eq 'ARRAY') {
|
||||
for my $stream (@{$data->{streams}}) {
|
||||
next unless $stream->{codec_type} eq 'video';
|
||||
if ($stream->{width} && $stream->{height}) {
|
||||
$resolution_w = $stream->{width};
|
||||
$resolution_h = $stream->{height};
|
||||
last;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
file_hash => $hash,
|
||||
resolution_w => $resolution_w // 0,
|
||||
resolution_h => $resolution_h // 0,
|
||||
codec => $codec // 'unknown',
|
||||
duration => $duration,
|
||||
};
|
||||
}
|
||||
+22
-3
@@ -82,8 +82,9 @@ sub process_file {
|
||||
my $file_size = -s $file_path;
|
||||
if ($file_size != $row->{file_size}) {
|
||||
warn "File size mismatch for $file_path. Updating record.";
|
||||
my $update_sth = $dbh->prepare("UPDATE videos SET file_size=?, last_scan_time=NOW() WHERE id=?");
|
||||
$update_sth->execute($file_size, $row->{id});
|
||||
my $info = get_video_info($file_path);
|
||||
my $update_sth = $dbh->prepare("UPDATE videos SET file_size=?, last_scan_time=NOW(), resolution_w=?, resolution_h=?, codec=?, duration=?, updated_at=NOW() WHERE id=?");
|
||||
$update_sth->execute($file_size, $info->{resolution_w}, $info->{resolution_h}, $info->{codec}, $info->{duration}, $row->{id});
|
||||
$update_sth->finish();
|
||||
create_aiscan_task($row->{id});
|
||||
} else {
|
||||
@@ -101,6 +102,24 @@ sub process_file {
|
||||
warn "Failed to get video info for $file_path. Skipping.";
|
||||
return;
|
||||
}
|
||||
|
||||
$sth = $dbh->prepare("SELECT id,file_path FROM videos WHERE file_size=? AND file_hash=?");
|
||||
$sth->execute(-s $file_path, $info->{file_hash});
|
||||
while(my $row = $sth->fetchrow_hashref()) {
|
||||
if(-e $row->{file_path}) {
|
||||
print "Duplicate found: $file_path matches existing file $row->{file_path}\n";
|
||||
} else {
|
||||
print "MOVED FILE: $file_path previously existed as $row->{file_path}\n";
|
||||
my $upd = $dbh->prepare("UPDATE videos SET file_path=?,last_scan_time=NOW(),updated_at=NOW() WHERE id=?");
|
||||
$upd->execute($file_path, $row->{id});
|
||||
$upd->finish();
|
||||
$sth->finish();
|
||||
return;
|
||||
}
|
||||
}
|
||||
$sth->finish();
|
||||
|
||||
|
||||
$sth = $dbh->prepare("INSERT INTO videos (file_path, file_size, file_hash, resolution_w, resolution_h, codec, duration, last_scan_time) VALUES (?, ?, ?, ?, ?, ?, ?, NOW())");
|
||||
$sth->execute(
|
||||
$file_path,
|
||||
@@ -136,7 +155,7 @@ sub get_video_info {
|
||||
# --- Compute SHA-256 hash of the file ---
|
||||
return undef unless -f $file_path && -r $file_path;
|
||||
open(my $fh, '<:raw', $file_path) or do { warn "Cannot open $file_path: $!"; return undef; };
|
||||
my $hash = sha256_hex($fh);
|
||||
my $hash = Digest::SHA->new(256)->addfile($fh)->hexdigest;
|
||||
close($fh);
|
||||
|
||||
|
||||
|
||||
Executable
+144
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/perl
|
||||
use strict;
|
||||
use warnings;
|
||||
use DBI;
|
||||
use Digest::SHA qw(sha256_hex);
|
||||
use JSON;
|
||||
use File::Basename qw(dirname basename);
|
||||
use File::Copy qw(move);
|
||||
use String::ShellQuote qw/shell_quote/;
|
||||
|
||||
my $db_host = $ENV{'DB_HOST'} || 'mariadb';
|
||||
my $db_name = $ENV{'DB_NAME'} || 'videodetect';
|
||||
my $db_user = $ENV{'DB_USER'} || 'videodetect';
|
||||
my $db_pass = $ENV{'DB_PASSWORD'} || 'videodetect';
|
||||
my $tasks = $ENV{'TASK_COUNT'} || 1;
|
||||
|
||||
$SIG{INT} = sub { die "Interrupted\n"; };
|
||||
|
||||
my $dbh = DBI->connect(
|
||||
"DBI:mysql:database=$db_name;host=$db_host;mysql_enable_utf8mb4=1",
|
||||
$db_user,
|
||||
$db_pass,
|
||||
{ mysql_auto_reconnect => 1 }
|
||||
);
|
||||
|
||||
my $sth_claimtask = $dbh->prepare("
|
||||
UPDATE tasks
|
||||
SET status='IN_PROGRESS', updated_at=NOW(), assign_key=?, assigned_at=NOW()
|
||||
WHERE status='PENDING' AND task_type='TRANSCODE' AND assign_key IS NULL
|
||||
LIMIT 1
|
||||
");
|
||||
my $sth_gettask = $dbh->prepare("SELECT * FROM tasks WHERE assign_key=? AND status='IN_PROGRESS' LIMIT 1");
|
||||
my $sth_getvideo = $dbh->prepare("SELECT * from videos WHERE id=?");
|
||||
my $sth_updatevideo = $dbh->prepare("UPDATE videos SET file_path=?, file_size=?, file_hash=?, updated_at=NOW() WHERE id=?");
|
||||
my $sth_finishtask = $dbh->prepare("UPDATE tasks SET status='COMPLETED', updated_at=NOW(), assign_key=NULL, assigned_at=NULL, results=? WHERE id=?");
|
||||
|
||||
for(my $i=0; $i<$tasks; $i++) {
|
||||
my $assign_key = "worker_" . int(rand(10000000));
|
||||
print "Assigning task with key: $assign_key\n";
|
||||
my $count = $sth_claimtask->execute($assign_key);
|
||||
if($count != 1) {
|
||||
print "Failed to claim task\n";
|
||||
last;
|
||||
}
|
||||
|
||||
$sth_gettask->execute($assign_key);
|
||||
my $task = $sth_gettask->fetchrow_hashref();
|
||||
if(!ref $task) {
|
||||
print "Failed to fetch task\n";
|
||||
last;
|
||||
}
|
||||
|
||||
my $video_id = $task->{video_id};
|
||||
$sth_getvideo->execute($video_id);
|
||||
my $video = $sth_getvideo->fetchrow_hashref();
|
||||
if ($video) {
|
||||
print "Processing video: $video->{id}\n";
|
||||
}
|
||||
|
||||
my $results = transcode_video($video);
|
||||
if ($results->{success}) {
|
||||
my $file_path = $results->{output};
|
||||
my $file_size = -s $file_path;
|
||||
my $file_hash = $results->{newhash};
|
||||
delete $results->{newhash};
|
||||
$sth_updatevideo->execute($file_path, $file_size, $file_hash, $video_id);
|
||||
$sth_finishtask->execute(JSON->new->encode($results), $task->{id});
|
||||
}
|
||||
}
|
||||
|
||||
sub get_hash {
|
||||
my ($file_path) = @_;
|
||||
# --- Compute SHA-256 hash of the file ---
|
||||
return undef unless -f $file_path && -r $file_path;
|
||||
open(my $fh, '<:raw', $file_path) or do { warn "Cannot open $file_path: $!"; return undef; };
|
||||
my $hash = sha256_hex($fh);
|
||||
close($fh);
|
||||
return $hash;
|
||||
}
|
||||
|
||||
sub transcode_video {
|
||||
my ($video, $retry) = @_;
|
||||
|
||||
my $file = $video ? $video->{file_path} : undef;
|
||||
|
||||
# --- validate ---
|
||||
unless (defined $file && -f $file) {
|
||||
warn "SKIP '" . ($file // 'undef') . "' not found\n";
|
||||
return { success => 0, reason => 'not_found' };
|
||||
}
|
||||
|
||||
# --- derive paths ---
|
||||
my $dir = dirname($file);
|
||||
my $base = basename($file);
|
||||
my $name = $base;
|
||||
$name =~ s/\.[^.]*$//; # strip last extension ("foo.mkv" -> "foo")
|
||||
$name = $base if $name eq ''; # guard against dot-only names
|
||||
|
||||
my $output = "$dir/$name.mp4"; # equals $file when it's already .mp4
|
||||
my $tmp = "/tmp/${name}~.mp4";
|
||||
|
||||
# --- transcode ---
|
||||
print "WORK $file → $output\n";
|
||||
|
||||
my $cmd = 'ffmpeg -y -nostdin '
|
||||
. '-hwaccel cuda '
|
||||
. ($retry ? '' : '-hwaccel_output_format cuda ')
|
||||
. '-i ' . shell_quote($file) . ' '
|
||||
. '-c:v h264_nvenc -preset p6 -rc vbr -cq 23 -b:v 0 '
|
||||
. ($retry ? '-pix_fmt yuv420p ' : '')
|
||||
. '-c:a aac -b:a 128k '
|
||||
. '-movflags +faststart '
|
||||
. shell_quote($tmp);
|
||||
|
||||
my $log = `$cmd`;
|
||||
my $exit_code = $? >> 8;
|
||||
|
||||
if ($exit_code == 0) {
|
||||
my $newhash = get_hash($tmp);
|
||||
# success — delete original, move transcoded file into place
|
||||
unless (unlink($file)) {
|
||||
warn " WARN could not delete original '$file': $!\n";
|
||||
}
|
||||
unless (move($tmp, $output)) {
|
||||
warn " FAIL could not move '$tmp' -> '$output': $!\n";
|
||||
return { success => 0, reason => "move_failed: $!" };
|
||||
}
|
||||
print " OK original removed, output: $output\n";
|
||||
return { success => 1, output => $output, original => $file, newhash => $newhash };
|
||||
}
|
||||
|
||||
# failure — clean up temp file, keep original
|
||||
unlink($tmp) if -e $tmp;
|
||||
warn " FAIL keeping original: $file\n";
|
||||
warn $log if defined $log && length $log;
|
||||
|
||||
if($retry) {
|
||||
return { success => 0, reason => 'transcode_failed', exit_code => $exit_code };
|
||||
} else {
|
||||
return transcode_video($video, 1); # retry with $retry set to 1
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user