4 Commits
Author SHA1 Message Date
Ryan Shpeherd 5e5972c31a Fix scanner task creator 2026-09-26 10:59:52 -04:00
Ryan Shpeherd a04e6e241b Updates to ai worker 2026-09-26 10:23:55 -04:00
Ryan Shpeherd e443191833 Add model and path to results 2026-09-25 22:43:54 -04:00
Ryan Shpeherd 589b775cda Fixes and new ai model 2026-09-25 22:40:24 -04:00
3 changed files with 54 additions and 30 deletions
+34 -8
View File
@@ -6,16 +6,27 @@ use JSON;
use Data::Dumper; use Data::Dumper;
my $seconds = $ENV{'SECONDS_PER_FRAME'} || 30; my $seconds = $ENV{'SECONDS_PER_FRAME'} || 30;
my $ollama = $ENV{'OLLAMA_URL'} || die "OLLAMA_URL environment variable not set\n"; my $ollama = $ENV{'OLLAMA_URL'} || 'http://10.0.1.7:11434';
my $api = $ENV{'API_BASE_URL'} || die "API_BASE_URL environment variable not set\n"; my $api = $ENV{'API_BASE_URL'} || 'http://10.0.1.7:3000';
my $task_count = $ENV{'TASK_COUNT'} || 1; my $task_count = $ENV{'TASK_COUNT'} || 1;
my $image_count = $ENV{'IMAGE_COUNT'} || 10; # Number of images to extract from the video
my $min_time = $ENV{'MIN_TIME'} || 5; # Minimum time between images
my $sensitivity = 20; my $sensitivity = 20;
my $resolution = "672:672"; my $resolution = "672:672";
#my $model = "hf.co/Jommarn/UNSEEN_Gemma_4_26B_NSFW-GGUF:Q8_0";
my $model = "hf.co/Jommarn/UNSEEN_Gemma_4_26B_NSFW-GGUF:Q5_K_M";
chdir("/tmp") || die "Unable to change directory to /tmp: $!\n"; chdir("/tmp") || die "Unable to change directory to /tmp: $!\n";
my $ua = LWP::UserAgent->new; my $ua = LWP::UserAgent->new;
$ua->timeout(60); $ua->timeout(60);
if(@ARGV) {
CheckPath($ARGV[0], {'duration' => 0});
exit;
}
for(my $i=0; $i<$task_count; $i++) { for(my $i=0; $i<$task_count; $i++) {
my $ref = GetNext(); my $ref = GetNext();
my $task = $ref->{'task'}; my $task = $ref->{'task'};
@@ -24,7 +35,7 @@ for(my $i=0; $i<$task_count; $i++) {
my $task_id = $task->{'id'}; my $task_id = $task->{'id'};
my $video_id = $video->{'id'}; my $video_id = $video->{'id'};
my $path = $video->{'file_path'}; my $path = $video->{'file_path'};
my $result = CheckPath($path); my $result = CheckPath($path, $video);
next unless $result; next unless $result;
@@ -35,12 +46,21 @@ print "Done\n";
exit(0); exit(0);
sub CheckPath { sub CheckPath {
my ($path) = @_; my ($path, $video) = @_;
die "File path not set\n" unless $path; die "File path not set\n" unless $path;
die "File does not exist: $path\n" unless -e $path; die "File does not exist: $path\n" unless -e $path;
unlink(<*.png>); unlink(<*.png>);
system("ffmpeg -i \"$path\" -vf fps=1/$seconds,scale=$resolution output_%04d.png");
# Pick a frame interval that spreads $image_count frames across the video,
# but never more frequently than 1 frame per $min_time seconds.
my $interval = $seconds; # fallback if duration is unknown
if (my $duration = $video->{'duration'}) {
my $desired = $duration / $image_count;
$interval = ($desired > $min_time) ? $desired : $min_time;
}
system("ffmpeg -i \"$path\" -vf fps=1/$interval,scale=$resolution output_%04d.png 2>&1");
my $total=0; my $total=0;
@@ -56,17 +76,21 @@ sub CheckPath {
$data=undef; $data=undef;
my $ref = { my $ref = {
'model' => 'srizon/pixie:latest', 'model' => $model,
'prompt' => "Are there any african american people in this photo, whether they are shown completely or not? Please answer 'yes' or 'no' only.", 'prompt' => "Are there any african american men in this photo, whether they are shown completely or not? Judge based on skin tone, accounting for shadows. Answer yes for very dark skin.Please answer 'yes' or 'no' only.",
'images' => [$base64], 'images' => [$base64],
}; };
my $url = "$ollama/api/generate"; my $url = "$ollama/api/generate";
my $res = GetRest($url, $ref); my $res = GetRest($url, $ref);
print "\t$res\n"; print "$file\t$res\n";
#Remove thought information from the response
$res =~ s/<\|channel>.*?<channel\|>//s;
$res =~ s/^\s+|\s+$//g;
$total++; $total++;
if($res=~/yes/i) { if($res=~/yes/i) {
$yes++; $yes++;
print "\tYES\n";
} }
unlink($file); unlink($file);
} }
@@ -91,6 +115,8 @@ sub CheckPath {
'yes' => $yes, 'yes' => $yes,
'percentage'=> $percentage, 'percentage'=> $percentage,
'verdict' => $verdict, 'verdict' => $verdict,
'model' => $model,
'path' => $path,
}; };
} }
+5 -17
View File
@@ -196,30 +196,18 @@ sub CreateReviewTasks {
# This avoids MariaDB's inconsistent JSON functions in SQL WHERE clauses, # This avoids MariaDB's inconsistent JSON functions in SQL WHERE clauses,
# which behave differently between CLI client and DBI connections. # which behave differently between CLI client and DBI connections.
my $sth = $dbh->prepare(" my $sth = $dbh->prepare("
SELECT id, video_id, results SELECT id, video_id
FROM tasks FROM tasks
WHERE task_type='AISCAN' AND status='COMPLETED' WHERE task_type='AISCAN'
AND status='COMPLETED'
AND JSON_EXTRACT(results, '\$.verdict') = 1
AND (SELECT 1 FROM tasks t2 WHERE t2.video_id = tasks.video_id AND t2.task_type = 'REVIEW' LIMIT 1) IS NULL
"); ");
$sth->execute(); $sth->execute();
while (my $row = $sth->fetchrow_hashref()) { while (my $row = $sth->fetchrow_hashref()) {
my $results = decode_json($row->{results});
next unless $results && ref($results) eq 'HASH';
next unless $results->{verdict} == 1;
next if exists $results->{reviewed};
print "Creating REVIEW task for video_id: $row->{video_id}\n"; print "Creating REVIEW task for video_id: $row->{video_id}\n";
create_task($row->{video_id}, 'REVIEW'); create_task($row->{video_id}, 'REVIEW');
# Re-encode the whole JSON blob in Perl and update it.
# This avoids MariaDB JSON_SET path parsing issues through DBI connections.
$results->{reviewed} = 1;
my $updated = encode_json($results);
my $sth_update = $dbh->prepare(
"UPDATE tasks SET results = ? WHERE id = ?"
);
$sth_update->execute($updated, $row->{id});
$sth_update->finish();
} }
$sth->finish(); $sth->finish();
} }
+15 -5
View File
@@ -214,6 +214,10 @@ def index():
@app.route("/review/<int:task_id>") @app.route("/review/<int:task_id>")
def review_task(task_id): def review_task(task_id):
"""Detail view: show task info + embedded video player.""" """Detail view: show task info + embedded video player."""
return_page = request.args.get("page", 1, type=int) or 1
return_page = max(return_page, 1)
return_queue_url = "/" if return_page == 1 else f"/?page={return_page}"
try: try:
data = api_get(f"task/{task_id}") data = api_get(f"task/{task_id}")
if not data or not data.get("id"): if not data or not data.get("id"):
@@ -240,7 +244,11 @@ def review_task(task_id):
except (json.JSONDecodeError, TypeError): except (json.JSONDecodeError, TypeError):
video_obj["prev_results"] = {} video_obj["prev_results"] = {}
return render_template_string(_REVIEW_TEMPLATE, video=video_obj) return render_template_string(
_REVIEW_TEMPLATE,
video=video_obj,
return_queue_url=return_queue_url,
)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -411,7 +419,7 @@ _INDEX_TEMPLATE = """
<td><span class="badge">{{ t.task_type }}</span></td> <td><span class="badge">{{ t.task_type }}</span></td>
<td>{{ t.status }}</td> <td>{{ t.status }}</td>
<td>{{ t.created_at|string|truncate(19, True, '') if t.created_at else '&mdash;' }}</td> <td>{{ t.created_at|string|truncate(19, True, '') if t.created_at else '&mdash;' }}</td>
<td><a href="/review/{{ t.id }}" class="btn">Review</a></td> <td><a href="{{ url_for('review_task', task_id=t.id, page=page) }}" class="btn">Review</a></td>
</tr> </tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
@@ -464,7 +472,7 @@ _REVIEW_TEMPLATE = """
<div id="flash"></div> <div id="flash"></div>
<div class="header"> <div class="header">
<h1>VideoDetect</h1> <h1>VideoDetect</h1>
<a href="/">&laquo; Back to queue</a> <a href="{{ return_queue_url }}">&laquo; Back to queue</a>
</div> </div>
<div class="container"> <div class="container">
{% if video.prev_results and video.prev_results.review_verdict %} {% if video.prev_results and video.prev_results.review_verdict %}
@@ -505,6 +513,8 @@ _REVIEW_TEMPLATE = """
</div> </div>
<script> <script>
const returnQueueUrl = {{ return_queue_url|tojson }};
async function submitVerdict(verdict) { async function submitVerdict(verdict) {
setActionButtonsDisabled(true); setActionButtonsDisabled(true);
showFlash('Submitting verdict...', '#007bff'); showFlash('Submitting verdict...', '#007bff');
@@ -518,7 +528,7 @@ async function submitVerdict(verdict) {
const body = await resp.json().catch(() => ({})); const body = await resp.json().catch(() => ({}));
if (resp.ok) { if (resp.ok) {
showFlash('Verdict submitted', '#28a745'); showFlash('Verdict submitted', '#28a745');
setTimeout(() => location.href = '/', 1000); setTimeout(() => location.href = returnQueueUrl, 1000);
} else { } else {
alert(body.error || 'Submission failed'); alert(body.error || 'Submission failed');
setActionButtonsDisabled(false); setActionButtonsDisabled(false);
@@ -538,7 +548,7 @@ async function skipTask() {
const resp = await fetch(`/api/review/skip/{{ video.id }}`, {method: 'POST'}); const resp = await fetch(`/api/review/skip/{{ video.id }}`, {method: 'POST'});
if (resp.ok) { if (resp.ok) {
showFlash('Task skipped, returned to queue', '#6c757d'); showFlash('Task skipped, returned to queue', '#6c757d');
setTimeout(() => location.href = '/', 1000); setTimeout(() => location.href = returnQueueUrl, 1000);
} else { } else {
alert('Skip failed'); alert('Skip failed');
setActionButtonsDisabled(false); setActionButtonsDisabled(false);