vndb/manned
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cache-html.pl: Now with transaction batching support

Browse source 
Because I'm trying to seed the cache from another machine and this
significantly reduces the effects of network latency.
This commit is contained in:
Yorhel 2025-05-25 15:13:42 +02:00
parent d3bebc8888
commit bed455039e
1 changed files with 31 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

50
util/cache-html.pl
View file

@ -8,10 +8,13 @@
# Be more verbose. # Be more verbose.
# #
# --delay=$SEC # --delay=$SEC
# Seconds (fraction supported) to wait between rendering subsequent pages. # Seconds (fraction supported) to wait between batches.
# #
# --maxpages=$NUM # --batch=$NUM
# Maximum number of pages to render before exiting. # Number of pages to render per batch.
#
# --maxbatches=$NUM
# Maximum number of batches to run before exiting.
# #
# Multiple instances of this script can run in parallel in order to speed up cache generation. # Multiple instances of this script can run in parallel in order to speed up cache generation.
@ -28,30 +31,39 @@ use ManUtils;
my $verbose = 0; my $verbose = 0;
my $delay = 0; my $delay = 0;
my $maxpages = 0; my $batch = 1;
GetOptions('verbose' => \$verbose, 'delay=i' => \$delay, 'maxpages=i' => \$maxpages); my $maxbatches = 0;
GetOptions('verbose' => \$verbose, 'delay=i' => \$delay, 'batch=i' => \$batch, 'maxbatches=i' => \$maxbatches);
my $conn = FU::Pg->connect($ENV{MANNED_PG}//''); my $conn = FU::Pg->connect($ENV{MANNED_PG}//'');
while (1) { while (1) {
my $txn = $conn->txn; my $txn = $conn->txn;
my($id, $content) = $txn->q('SELECT id, content FROM contents WHERE html IS NULL FOR UPDATE SKIP LOCKED LIMIT 1')->rowl;
last if !$id;
my $start = time; my $lst = $txn->q('SELECT id, content FROM contents WHERE html IS NULL FOR UPDATE SKIP LOCKED LIMIT $1', $batch)->flat;
my $html = eval { ManUtils::html ManUtils::fmt $content }; last if !@$lst;
my $end = time;
# Should be rare. Do save *something* in the database, so we won't get my @save;
# stuck trying this again and we can easily query for broken pages. for my($id, $content) (@$lst) {
if (!defined $html) { my $start = time;
$html = '(Error rendering man page)'; my $html = eval { ManUtils::html ManUtils::fmt $content };
warn "$id: Error rendering page: $@\n"; my $end = time;
# Should be rare. Do save *something* in the database, so we won't get
# stuck trying this again and we can easily query for broken pages.
if (!defined $html) {
$html = '(Error rendering man page)';
warn "$id: Error rendering page: $@\n";
}
printf "%10d: %5.1f ms, %d raw, %d html\n", $id, ($end-$start)*1000, length($content), length($html) if $verbose;
push @save, $id, $html;
} }
$txn->q('UPDATE contents SET html = nhtml FROM (VALUES '.
$txn->q('UPDATE contents SET html = $1 WHERE id = $2', $html, $id)->exec; join(', ', map sprintf('($%d::int, $%d)', $_*2-1, $_*2), 1..@save/2).
') d(nid, nhtml) WHERE id = nid', @save
)->exec;
$txn->commit; $txn->commit;
printf "%10d: %5.1f ms, %d raw, %d html\n", $id, ($end-$start)*1000, length($content), length($html) if $verbose; last if !--$maxbatches;
last if !--$maxpages;
sleep $delay if $delay; sleep $delay if $delay;
} }