d3bebc8888
Downside is that this consumes significant disk space, requires recreating the entire cache when changing something to the way that pages are rendered and removes flexibility to add dynamic render-influencing settings in the future. Alas, crawlers are getting more aggressive and I don't like the idea of adding more invasive anti-bot tech. This might not be enough in the long term, we also have a few slow SQL queries that I'm not yet sure how to optimize. But this ought to give us more time, at least.
57 lines
1.7 KiB
Perl
Executable file
57 lines
1.7 KiB
Perl
Executable file
#!/usr/bin/env perl
|
|
|
|
# This script populates the HTML-rendered man page cache in the database.
|
|
#
|
|
# Usage: cache-html.pl
|
|
#
|
|
# --verbose
|
|
# Be more verbose.
|
|
#
|
|
# --delay=$SEC
|
|
# Seconds (fraction supported) to wait between rendering subsequent pages.
|
|
#
|
|
# --maxpages=$NUM
|
|
# Maximum number of pages to render before exiting.
|
|
#
|
|
# Multiple instances of this script can run in parallel in order to speed up cache generation.
|
|
|
|
use v5.36;
|
|
use FU::Pg;
|
|
use Getopt::Long;
|
|
use Time::HiRes 'time';
|
|
use Cwd 'abs_path';
|
|
our $ROOT;
|
|
BEGIN { ($ROOT = abs_path $0) =~ s{/util/cache-html\.pl$}{}; }
|
|
|
|
use lib "$ROOT/ManUtils/blib/lib", "$ROOT/ManUtils/blib/arch";
|
|
use ManUtils;
|
|
|
|
my $verbose = 0;
|
|
my $delay = 0;
|
|
my $maxpages = 0;
|
|
GetOptions('verbose' => \$verbose, 'delay=i' => \$delay, 'maxpages=i' => \$maxpages);
|
|
|
|
my $conn = FU::Pg->connect($ENV{MANNED_PG}//'');
|
|
|
|
while (1) {
|
|
my $txn = $conn->txn;
|
|
my($id, $content) = $txn->q('SELECT id, content FROM contents WHERE html IS NULL FOR UPDATE SKIP LOCKED LIMIT 1')->rowl;
|
|
last if !$id;
|
|
|
|
my $start = time;
|
|
my $html = eval { ManUtils::html ManUtils::fmt $content };
|
|
my $end = time;
|
|
|
|
# Should be rare. Do save *something* in the database, so we won't get
|
|
# stuck trying this again and we can easily query for broken pages.
|
|
if (!defined $html) {
|
|
$html = '(Error rendering man page)';
|
|
warn "$id: Error rendering page: $@\n";
|
|
}
|
|
|
|
$txn->q('UPDATE contents SET html = $1 WHERE id = $2', $html, $id)->exec;
|
|
$txn->commit;
|
|
printf "%10d: %5.1f ms, %d raw, %d html\n", $id, ($end-$start)*1000, length($content), length($html) if $verbose;
|
|
last if !--$maxpages;
|
|
sleep $delay if $delay;
|
|
}
|