vndb/manned
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Normalize package info tables + change browsing URLs

Browse source 
This splits the 'package' table into 'packages' and 'package_versions',
which should improve performance in some cases and simplify some future
queries.

Previously it wasn't very well defined whether packages were uniquely
identified by (system, name) or by (system, category, name). This is now
normalized to the latter form. This required changes to the package URLs
to include the category.
This commit is contained in:
Yorhel 2016-10-02 20:07:57 +02:00
parent 26aefaebcd
commit 03d278e4ff
7 changed files with 180 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

59
schema.sql → sql/schema.sql
View file

@ -1,9 +1,3 @@
-- TODO: "system" -> "repository"?
-- TODO: index of (reverse) man page references?
-- TODO: Use some consistent naming of tables and columns
CREATE TABLE systems ( CREATE TABLE systems (
id integer PRIMARY KEY, -- hardcoded ID. id integer PRIMARY KEY, -- hardcoded ID.
name varchar NOT NULL, name varchar NOT NULL,
@ -12,52 +6,49 @@ CREATE TABLE systems (
short varchar NOT NULL short varchar NOT NULL
); );
CREATE TABLE contents ( CREATE TABLE contents (
hash bytea PRIMARY KEY, hash bytea PRIMARY KEY,
content varchar NOT NULL content varchar NOT NULL
); );
CREATE TABLE packages (
-- Note: If there are multiple arches available for the same package, then
-- generally only a single one is chosen (not stored here which one).
-- Also, a package may be listed here even if it has no man pages indexed, in
-- order for the fetcher to determine whether it has already processed the
-- package or not. This doesn't mean all packages of a repository are listed
-- here. For example, the Arch fetcher checks the file list of a package before
-- considering to handle it.
CREATE TABLE package (
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
system integer NOT NULL REFERENCES systems(id), system integer NOT NULL REFERENCES systems(id),
category varchar, -- depends on system (e.g. "community" on Arch, "x11" on Debian) category varchar,
name varchar NOT NULL, name varchar NOT NULL,
version varchar NOT NULL, UNIQUE(system, name, category) -- Note the order, lookups on (system,name) are common
released date NOT NULL,
UNIQUE(system, name, version)
); );
CREATE TABLE package_versions (
id SERIAL PRIMARY KEY,
package integer NOT NULL REFERENCES packages(id),
version varchar NOT NULL,
released date NOT NULL,
UNIQUE(package, version)
);
CREATE TABLE man ( CREATE TABLE man (
package integer NOT NULL REFERENCES package(id), package integer NOT NULL REFERENCES package_versions(id),
name varchar NOT NULL, -- 'fopen', 'du', etc (TODO: An index on name_from_filename(filename) may also work) name varchar NOT NULL,
section varchar NOT NULL, -- extracted from filename (TODO: Is this column really necessary?) section varchar NOT NULL,
filename varchar NOT NULL, -- full path + file name filename varchar NOT NULL,
locale varchar, -- parsed from the file name, NULL for the "main" man page (in the C or en_US locale) locale varchar,
hash bytea NOT NULL REFERENCES contents(hash), hash bytea NOT NULL REFERENCES contents(hash),
UNIQUE(package, filename) UNIQUE(package, filename)
); );
CREATE INDEX ON man (hash);
CREATE INDEX ON man USING hash (hash);
CREATE INDEX ON man (name); CREATE INDEX ON man (name);
CREATE TABLE man_index AS SELECT DISTINCT name, section FROM man; CREATE TABLE man_index AS SELECT DISTINCT name, section FROM man;
CREATE INDEX ON man_index USING btree(lower(name) text_pattern_ops); CREATE INDEX ON man_index USING btree(lower(name) text_pattern_ops);
CREATE TABLE stats_cache AS SELECT count(distinct hash) AS hashes, count(distinct name) AS mans, count(*) AS files, count(distinct package) AS packages FROM man; CREATE TABLE stats_cache AS SELECT count(distinct hash) AS hashes, count(distinct name) AS mans, count(*) AS files, count(distinct package) AS packages FROM man;
INSERT INTO systems (id, name, release, short, relorder) VALUES INSERT INTO systems (id, name, release, short, relorder) VALUES
(1, 'Arch Linux', NULL, 'arch', 0), (1, 'Arch Linux', NULL, 'arch', 0),
(2, 'Ubuntu', '4.10', 'ubuntu-warty', 0), (2, 'Ubuntu', '4.10', 'ubuntu-warty', 0),
@ -153,6 +144,7 @@ INSERT INTO systems (id, name, release, short, relorder) VALUES
(92, 'Ubuntu', '15.10', 'ubuntu-wily', 22), (92, 'Ubuntu', '15.10', 'ubuntu-wily', 22),
(93, 'Ubuntu', '16.04', 'ubuntu-xenial', 23); (93, 'Ubuntu', '16.04', 'ubuntu-xenial', 23);
-- Removes any path components and compression extensions from the filename. -- Removes any path components and compression extensions from the filename.
CREATE OR REPLACE FUNCTION basename_from_filename(fn text) RETURNS text AS $$ CREATE OR REPLACE FUNCTION basename_from_filename(fn text) RETURNS text AS $$
DECLARE DECLARE
@ -178,16 +170,3 @@ $$ LANGUAGE SQL;
CREATE OR REPLACE FUNCTION name_from_filename(text) RETURNS text AS $$ CREATE OR REPLACE FUNCTION name_from_filename(text) RETURNS text AS $$
SELECT regexp_replace(basename_from_filename($1), E'^(.+)\\.[^.]+$', E'\\1'); SELECT regexp_replace(basename_from_filename($1), E'^(.+)\\.[^.]+$', E'\\1');
$$ LANGUAGE SQL; $$ LANGUAGE SQL;
-- Some handy admin queries
--BEGIN;
--DELETE FROM man WHERE package IN(SELECT id FROM package WHERE name = '');
--DELETE FROM package WHERE name = '';
--DELETE FROM contents c WHERE NOT EXISTS(SELECT 1 FROM man m WHERE m.hash = c.hash);
--COMMIT;
--DELETE FROM package WHERE system = 18 AND NOT EXISTS(SELECT 1 FROM man WHERE id = package);

30
sql/update-2016-10-02.sql Normal file
View file

@ -0,0 +1,30 @@
CREATE TABLE packages (
id SERIAL PRIMARY KEY,
system integer NOT NULL REFERENCES systems(id),
category varchar,
name varchar NOT NULL,
UNIQUE(system, name, category) -- Note the order, lookups on (system,name) are common
);
CREATE TABLE package_versions (
id SERIAL PRIMARY KEY,
package integer NOT NULL REFERENCES packages(id),
version varchar NOT NULL,
released date NOT NULL,
UNIQUE(package, version)
);
INSERT INTO packages (system, category, name) SELECT system, category, name FROM package GROUP BY system, category, name;
INSERT INTO package_versions (id, package, version, released)
SELECT p.id, pn.id, p.version, p.released FROM package p JOIN packages pn ON pn.system = p.system AND pn.category = p.category AND pn.name = p.name;
SELECT setval('package_versions_id_seq', nextval('package_id_seq'));
ALTER TABLE man DROP CONSTRAINT man_package_fkey;
ALTER TABLE man ADD FOREIGN KEY (package) REFERENCES package_versions(id);
-- Use a proper b-tree index
DROP INDEX man_hash_idx;
CREATE INDEX ON man (hash);
-- DROP TABLE package;

13
util/common.sh
View file

@ -11,13 +11,18 @@ trap "rm -rf $TMP" EXIT
# Usage: add_pkginfo sysid category name version date # Usage: add_pkginfo sysid category name version date
# Returns 0 if the package is already in the database or if an error occured. # Returns 0 if the package is already in the database or if an error occured.
# Otherwise adds the package, sets PKGID to the new ID, and returns 1. # Otherwise adds the package, sets PKGID to the new package_versions.id, and returns 1.
PKGID= PKGID=
add_pkginfo() { add_pkginfo() {
RES=`echo "SELECT id FROM package WHERE system = :'sysid' AND name = :'name' AND version = :'ver'"\ RES=`echo "SELECT pv.id FROM packages p JOIN package_versions pv ON pv.package = p.id
| $PSQL -v "sysid=$1" -v "name=$3" -v "ver=$4"` WHERE p.system = :'sysid' AND p.category = :'cat' AND p.name = :'name' AND pv.version = :'ver'"\
| $PSQL -v "sysid=$1" -v "cat=$2" -v "name=$3" -v "ver=$4"`
[ "$?" -ne 0 -o -n "$RES" ] && return 0 [ "$?" -ne 0 -o -n "$RES" ] && return 0
RES=`echo "INSERT INTO package (system, category, name, version, released) VALUES(:'sysid',:'cat',:'name',:'ver',:'rel') RETURNING id"\ RES=`echo "
INSERT INTO packages (system, category, name) VALUES(:'sysid', :'cat', :'name') ON CONFLICT DO NOTHING;
INSERT INTO package_versions (version, released, package) VALUES(:'ver', :'rel',
(SELECT packages.id FROM packages WHERE system = :'sysid' AND category = :'cat' AND name = :'name'))
RETURNING id"\
| $PSQL -v "sysid=$1" -v "cat=$2" -v "name=$3" -v "ver=$4" -v "rel=$5"` | $PSQL -v "sysid=$1" -v "cat=$2" -v "name=$3" -v "ver=$4" -v "rel=$5"`
[ "$?" -ne 0 ] && return 0 [ "$?" -ne 0 ] && return 0
PKGID=$RES PKGID=$RES

11
util/deb.sh
View file

@ -31,11 +31,10 @@ checkpkg() {
DATE=`date -d "$DATE" +%F` DATE=`date -d "$DATE" +%F`
# Insert package in the database # Insert package in the database
PKGID=`echo "INSERT INTO package (system, category, name, version, released) VALUES(:'sysid',:'cat',:'name',:'ver',:'rel') RETURNING id"\ add_pkginfo $SYSID $SECTION $NAME $VERSION $DATE
| $PSQL -v "sysid=$SYSID" -v "cat=$SECTION" -v "name=$NAME" -v "ver=$VERSION" -v "rel=$DATE"`
# Extract and handle the man pages # Extract and handle the man pages
if [ "$?" -eq 0 -a -n "$PKGID" ]; then if [ "$?" -eq 1 -a -n "$PKGID" ]; then
# Old format # Old format
if [ "`head -c8 \"$FN\"`" = "0.939000" ]; then if [ "`head -c8 \"$FN\"`" = "0.939000" ]; then
tail -n+3 "$FN" | tail -c+"`head -n2 \"$FN\" | tail -n1`" | tail -c+2 | add_tar - $PKGID -z tail -n+3 "$FN" | tail -c+"`head -n2 \"$FN\" | tail -n1`" | tail -c+2 | add_tar - $PKGID -z
@ -116,11 +115,13 @@ syncrepo() {
if($p && $v && $s && $f) { if($p && $v && $s && $f) {
$f =~ s{^(Debian-1.[12])/}{dists/$1/main/}; $f =~ s{^(Debian-1.[12])/}{dists/$1/main/};
print "$p $v $s $f" if $pkg{$p} && $pkg{$p} == 1 print "$p $v $s $f" if $pkg{$p} && $pkg{$p} == 1
&& !$db->selectrow_arrayref(q{SELECT 1 FROM package WHERE system = ? AND name = ? AND version = ?}, {}, $sysid, $p, $v); && !$db->selectrow_arrayref(q{
SELECT 1 FROM packages p JOIN package_versions pv ON pv.package = p.id
WHERE p.system = ? AND p.category = ? AND p.name = ? AND pv.version = ?}, {}, $sysid, $s, $p, $v);
#warn "Duplicate package? $p\n" if $pkg{$p} && $pkg{$p} == 2; #warn "Duplicate package? $p\n" if $pkg{$p} && $pkg{$p} == 2;
$pkg{$p} = 2; $pkg{$p} = 2;
} }
$p=$v=$f=undef $p=$v=$f=$s=undef
} }
} }
close F; close F;

21
util/freebsd.sh
View file

@ -52,8 +52,7 @@ check_pkg() { # <sysid> <base-url> <category> <filename> <name> <version>
return return
fi fi
PKGID=`echo "INSERT INTO package (system, category, name, version, released) VALUES(:'sysid',:'cat',:'name',:'ver',:'rel') RETURNING id"\ add_pkginfo $SYSID $CAT $NAME $VER $DATE
| $PSQL -v "sysid=$SYSID" -v "cat=$CAT" -v "name=$NAME" -v "ver=$VER" -v "rel=$DATE"`
add_tar "$TMP/$FN" $PKGID add_tar "$TMP/$FN" $PKGID
rm -f "$TMP/$FN" rm -f "$TMP/$FN"
} }
@ -84,8 +83,8 @@ check_pkgdir() { # <sysid> <url>
echo "== Error fetching package index for /$CAT/" echo "== Error fetching package index for /$CAT/"
continue continue
fi fi
perl -l - "$TMP/pkgnames" "$TMP/pkglist" $SYSID <<'EOP' >"$TMP/newpkgs" perl -l - "$TMP/pkgnames" "$TMP/pkglist" $SYSID $CAT <<'EOP' >"$TMP/newpkgs"
($names, $list, $sysid) = @ARGV; ($names, $list, $sysid, $cat) = @ARGV;
use DBI; use DBI;
$db = DBI->connect('dbi:Pg:dbname=manned', 'manned', '', {RaiseError => 1}); $db = DBI->connect('dbi:Pg:dbname=manned', 'manned', '', {RaiseError => 1});
@ -103,7 +102,9 @@ check_pkgdir() { # <sysid> <url>
if(!$n || !$names{$n} || !$v) { if(!$n || !$names{$n} || !$v) {
warn "== Unknown package: $c\n"; warn "== Unknown package: $c\n";
} else { } else {
print "$c $n $v" if !$db->selectrow_arrayref(q{SELECT 1 FROM package WHERE system = ? AND name = ? AND version = ?}, {}, $sysid, $n, $v); print "$c $n $v" if !$db->selectrow_arrayref(q{
SELECT 1 FROM packages p JOIN package_versions pv ON pv.package = p.id
WHERE p.system = ? AND p.category = ? AND p.name = ? AND pv.version = ?}, {}, $sysid, $cat, $n, $v);
} }
} }
close F; close F;
@ -720,15 +721,7 @@ f9_1() {
} }
f9_2() { f9_2() {
MIR="http://ftp.dk.freebsd.org/pub/FreeBSD/releases/i386/9.2-RELEASE/" MIR="http://ftp-archive.freebsd.org/mirror/FreeBSD-Archive/old-releases/i386/9.2-RELEASE/"
echo "============ $MIR"
check_dist 86 "$MIR/base.txz" "core-base" "2013-09-27"
check_dist 86 "$MIR/games.txz" "core-games" "2013-09-27"
check_pkgdir 86 "$MIR/packages"
}
f9_3() {
MIR="http://ftp.dk.freebsd.org/pub/FreeBSD/releases/i386/9.2-RELEASE/"
echo "============ $MIR" echo "============ $MIR"
check_dist 86 "$MIR/base.txz" "core-base" "2013-09-27" check_dist 86 "$MIR/base.txz" "core-base" "2013-09-27"
check_dist 86 "$MIR/games.txz" "core-games" "2013-09-27" check_dist 86 "$MIR/games.txz" "core-games" "2013-09-27"

144
www/index.pl
View file

@ -46,8 +46,23 @@ TUWF::register(
qr// => \&home, qr// => \&home,
qr{info/about} => \&about, qr{info/about} => \&about,
qr{browse/search} => \&browsesearch, qr{browse/search} => \&browsesearch,
qr{browse/([^/]+)} => \&browsesys,
qr{browse/([^/]+)/([^/]+)(?:/([^/]+))?} => \&browsepkg, qr{pkg/([^/]+)} => \&pkg_list,
# pkg/$system/$category/$name (/$version); $category may contain a slash, too.
qr{pkg/([^/]+)/(.+)?} => \&pkg_info,
# Redirects for old URLs.
# /browse/<pkg> has been moved to /pkg/ with the package category added to the path
qr{browse/([^/]+)} => sub { $_[0]->resRedirect("/pkg/$_[1]", 'perm'); },
qr{browse/([^/]+)/([^/]+)(?:/([^/]+))?} => sub {
my($self, $sys, $name, $ver) = @_;
$sys = $self->{sysbyshort}{$sys};
return $self->resNotFound if !$sys;
my $pkgs = $self->dbPackageGet(sysid => $sys->{id}, name => $name, results => 1);
return $self->resNotFound if !@$pkgs;
$self->resRedirect("/pkg/$sys->{short}/$pkgs->[0]{category}/$name".($ver ? "/$ver" :''), 'perm');
},
qr{xml/search\.xml} => \&xmlsearch, qr{xml/search\.xml} => \&xmlsearch,
qr{([^/]+)/([0-9a-f]{8})} => \&man, qr{([^/]+)/([0-9a-f]{8})} => \&man,
qr{([^/]+)/([0-9a-f]{8})/src} => \&src, qr{([^/]+)/([0-9a-f]{8})/src} => \&src,
@ -82,13 +97,13 @@ sub home {
$sys = $sys{$sys}; $sys = $sys{$sys};
(my $img = $sys->[0]{short}) =~ s/^(.+)-.+$/$1/; (my $img = $sys->[0]{short}) =~ s/^(.+)-.+$/$1/;
li; li;
a href => "/browse/$sys->[0]{short}" if @$sys == 1; a href => "/pkg/$sys->[0]{short}" if @$sys == 1;
span style => "background-image: url('images/$img.png')", ''; span style => "background-image: url('images/$img.png')", '';
b $sys->[0]{name}; b $sys->[0]{name};
if(@$sys > 1) { if(@$sys > 1) {
my $i = 0; my $i = 0;
for(reverse @$sys) { for(reverse @$sys) {
a href => "/browse/$_->{short}", ++$i > 3 ? (class => 'hidden') : (), $_->{release}; a href => "/pkg/$_->{short}", ++$i > 3 ? (class => 'hidden') : (), $_->{release};
lit ' '; lit ' ';
} }
a href => "#", class => 'more', 'more...' if $i > 3; a href => "#", class => 'more', 'more...' if $i > 3;
@ -272,7 +287,7 @@ sub browsesearch {
} }
sub browsesys { sub pkg_list {
my($self, $short) = @_; my($self, $short) = @_;
my $sys = $self->{sysbyshort}{$short}; my $sys = $self->{sysbyshort}{$short};
@ -284,7 +299,7 @@ sub browsesys {
); );
return $self->resNotFound if $f->{_err}; return $self->resNotFound if $f->{_err};
my $pkg = $self->dbPackageList( my $pkg = $self->dbPackageGet(
hasman => 1, hasman => 1,
sysid => $sys->{id}, sysid => $sys->{id},
char => $f->{c} eq 'all' ? undef : $f->{c}, char => $f->{c} eq 'all' ? undef : $f->{c},
@ -299,7 +314,7 @@ sub browsesys {
use utf8; use utf8;
if($more) { if($more) {
p class => 'pagination'; p class => 'pagination';
a href => "/browse/$short?c=$f->{c};s=$pkg->[199]{name}", 'next »'; a href => "/pkg/$short?c=$f->{c};s=$pkg->[199]{name}", 'next »';
end; end;
} }
}; };
@ -310,7 +325,7 @@ sub browsesys {
p id => 'charselect'; p id => 'charselect';
for('all', 0, 'a'..'z') { for('all', 0, 'a'..'z') {
a href => "/browse/$short?c=$_", $_?uc$_:'#' if $_ ne $f->{c}; a href => "/pkg/$short?c=$_", $_?uc$_:'#' if $_ ne $f->{c};
b $_?uc$_:'#' if $_ eq $f->{c}; b $_?uc$_:'#' if $_ eq $f->{c};
} }
end; end;
@ -320,7 +335,7 @@ sub browsesys {
ul id => 'packages'; ul id => 'packages';
for(@$pkg) { for(@$pkg) {
li; li;
a href => "/browse/$short/$_->{name}", $_->{name}; a href => "/pkg/$short/$_->{category}/$_->{name}", $_->{name};
i ' '.$_->{category}; i ' '.$_->{category};
end; end;
} }
@ -330,14 +345,44 @@ sub browsesys {
} }
sub browsepkg { sub pkg_frompath {
my($self, $short, $name, $ver) = @_; my($self, $sys, $path) = @_;
# $path should be "$category/$name" or "$category/$name/$version", since
# $category may contain a slash, let's try both options.
# $category/$name
# e.g. contrib/games/alien
if($path =~ m{^(.+)/([^/]+)$}) {
my($category, $name) = ($1, $2);
my $pkg = $self->dbPackageGet(sysid => $sys, category => $category, name => $name, hasman => 1)->[0];
return ($pkg, '') if $pkg;
}
# $category/$name/$version
# e.g. contrib/games/alien/10.2
if($path =~ m{^(.+)/([^/]+)/([^/]+)$}) {
my($category, $name, $version) = ($1, $2, $3);
my $pkg = $self->dbPackageGet(sysid => $sys, category => $category, name => $name, hasman => 1)->[0];
return ($pkg, $version) if $pkg;
}
(undef, '');
}
sub pkg_info {
my($self, $short, $path) = @_;
my $sys = $self->{sysbyshort}{$short}; my $sys = $self->{sysbyshort}{$short};
return $self->resNotFound if !$sys; return $self->resNotFound if !$sys;
my $pkgs = $self->dbPackageGet($sys->{id}, $name); my($pkg, $ver) = pkg_frompath($self, $sys->{id}, $path);
my $sel = $ver ? (grep $_->{version} eq $ver, @$pkgs)[0] : $pkgs->[0]; return $self->resNotFound if !$pkg;
my $vers = $self->dbPackageVersions($pkg->{id});
my $sel = $ver ? (grep $_->{version} eq $ver, @$vers)[0] : $vers->[0];
return $self->resNotFound if !$sel; return $self->resNotFound if !$sel;
my $f = $self->formValidate({ get => 's', required => 0}); my $f = $self->formValidate({ get => 's', required => 0});
@ -346,33 +391,30 @@ sub browsepkg {
my $mans = $self->dbManInfo(package => $sel->{id}, results => 201, start => $f->{s}, sort => 'name'); my $mans = $self->dbManInfo(package => $sel->{id}, results => 201, start => $f->{s}, sort => 'name');
my $more = @$mans > 200 && pop @$mans; my $more = @$mans > 200 && pop @$mans;
# Latest version of this package determines last modification date of the page.
$self->setLastMod($vers->[0]{released});
# TODO: A "previous" link would be nice... # TODO: A "previous" link would be nice...
my $next = sub { my $next = sub {
use utf8; use utf8;
if($more) { if($more) {
p class => 'pagination'; p class => 'pagination';
a href => "/browse/$sys->{short}/$name/$sel->{version}?s=$mans->[199]{name}", 'next »'; a href => "/pkg/$sys->{short}/$pkg->{category}/$pkg->{name}/$sel->{version}?s=$mans->[199]{name}", 'next »';
end; end;
} }
}; };
# Latest version of this package determines last modification date of the page. my $title = "$sys->{name}".($sys->{release}?" $sys->{release}":"")." / $pkg->{category} / $pkg->{name} $sel->{version}";
$self->setLastMod($pkgs->[0]{released});
my $title = "$sys->{name}".($sys->{release}?" $sys->{release}":"")." / $name $sel->{version}";
$self->htmlHeader(title => $title); $self->htmlHeader(title => $title);
h1 $title; h1 $title;
# TODO: Link back to the system browsing page
h2 'Versions'; h2 'Versions';
ul id => 'packages'; ul id => 'packages';
for(@$pkgs) { for(@$vers) {
li; li;
txt "$_->{released} "; txt "$_->{released} ";
a href => "/browse/$sys->{short}/$name/$_->{version}", $_->{version} if $_ != $sel; a href => "/pkg/$sys->{short}/$pkg->{category}/$pkg->{name}/$_->{version}", $_->{version} if $_ != $sel;
b " $_->{version}" if $_ == $sel; b " $_->{version}" if $_ == $sel;
i " $_->{category}";
end; end;
} }
end; end;
@ -405,13 +447,13 @@ sub manjslist {
do { do {
my %pkgs; my %pkgs;
for(@{$sys{$_}}) { for(@{$sys{$_}}) {
my $pn = "$_->{package}-$_->{version}"; my $pn = "$_->{category}-$_->{package}-$_->{version}";
$pkgs{$pn} = [ $_->{package}, $_->{version}, [], $_->{released} ] if !$pkgs{$pn}; $pkgs{$pn} = [ $_->{category}, $_->{package}, $_->{version}, [], $_->{released} ] if !$pkgs{$pn};
push @{$pkgs{$pn}[2]}, [ $_->{section}, $_->{locale}, substr $_->{hash}, 0, 8 ]; push @{$pkgs{$pn}[3]}, [ $_->{section}, $_->{locale}, substr $_->{hash}, 0, 8 ];
} }
[ grep [ grep
delete($_->[3]) && ($_->[2] = [sort { $a->[0] cmp $b->[0] || ($a->[1]||'') cmp ($b->[1]||'') } @{$_->[2]}]), delete($_->[4]) && ($_->[3] = [sort { $a->[0] cmp $b->[0] || ($a->[1]||'') cmp ($b->[1]||'') } @{$_->[3]}]),
sort { $a->[0] cmp $b->[0] || $b->[3] cmp $a->[3] } values %pkgs ]; sort { $a->[0] cmp $b->[0] || $a->[1] cmp $b->[1] || $b->[4] cmp $a->[4] } values %pkgs ];
} }
], ],
sort { my $x=$self->{sysbyid}{$a}; my $y=$self->{sysbyid}{$b}; $x->{name} cmp $y->{name} or $y->{relorder} <=> $x->{relorder} } keys %sys sort { my $x=$self->{sysbyid}{$a}; my $y=$self->{sysbyid}{$b}; $x->{name} cmp $y->{name} or $y->{relorder} <=> $x->{relorder} } keys %sys
@ -650,7 +692,7 @@ sub dbManContent {
} }
# Options: name, section, shorthash, locale, package, start, results, sort # Options: name, section, shorthash, package, start, results, sort
sub dbManInfo { sub dbManInfo {
my $s = shift; my $s = shift;
my %o = @_; my %o = @_;
@ -662,21 +704,21 @@ sub dbManInfo {
$o{section} ? ('m.section = ?' => $o{section}) : (), $o{section} ? ('m.section = ?' => $o{section}) : (),
$o{shorthash} ? (q{substring(m.hash from 1 for 4) = decode(?, 'hex')} => $o{shorthash}) : (), $o{shorthash} ? (q{substring(m.hash from 1 for 4) = decode(?, 'hex')} => $o{shorthash}) : (),
$o{hash} ? (q{m.hash = decode(?, 'hex')} => $o{hash}) : (), $o{hash} ? (q{m.hash = decode(?, 'hex')} => $o{hash}) : (),
$o{locale} ? ('m.locale = ?', $o{locale}) : exists $o{locale} ? ('m.locale IS NULL' => 1) : (),
$o{start} ? ('m.name > ?' => $o{start}) : (), $o{start} ? ('m.name > ?' => $o{start}) : (),
); );
# TODO: Flags to indicate what to information to fetch # TODO: Flags to indicate what to information to fetch
return $s->dbAll(q{ return $s->dbAll(q{
SELECT p.system, p.category, p.name AS package, p.version, p.released, m.name, m.section, m.filename, m.locale, encode(m.hash, 'hex') AS hash SELECT p.system, p.category, p.name AS package, pv.version, pv.released, m.name, m.section, m.filename, m.locale, encode(m.hash, 'hex') AS hash
FROM package p FROM packages p
JOIN man m ON m.package = p.id JOIN package_versions pv ON p.id = pv.package
JOIN man m ON m.package = pv.id
!W !W
!s !s
LIMIT ? LIMIT ?
}, },
\%where, \%where,
$o{sort} ? 'ORDER BY name, locale NULLS FIRST' : '', $o{sort} ? 'ORDER BY name' : '',
$o{results}||10000 $o{results}||10000
); );
} }
@ -705,29 +747,25 @@ sub dbSystemGet {
} }
# TODO: Optimize
# Options: sysid char hasman start results # Options: sysid char hasman start results
sub dbPackageList { sub dbPackageGet {
my $s = shift; my $s = shift;
my %o = (results => 10, @_); my %o = (results => 10, @_);
my @where = ( my @where = (
$o{sysid} ? ('system = ?' => $o{sysid}) : (), $o{sysid} ? ('system = ?' => $o{sysid}) : (),
$o{category} ? ('category = ?' => $o{category}) : (),
$o{name} ? ('name = ?' => $o{name}) : (),
$o{start} ? ('name > ?' => $o{start}) : (), $o{start} ? ('name > ?' => $o{start}) : (),
defined($o{hasman}) ? ('!s EXISTS(SELECT 1 FROM man m WHERE m.package = p.id)' => $o{hasman}?'':'NOT') : (), # This seems slow, perhaps cache?
defined($o{hasman}) ? ('!s EXISTS(SELECT 1 FROM package_versions pv WHERE pv.package = p.id AND EXISTS(SELECT 1 FROM man m WHERE m.package = pv.id))' => $o{hasman}?'':'NOT') : (),
$o{char} ? ( 'LOWER(SUBSTR(name, 1, 1)) = ?' => $o{char} ) : (), $o{char} ? ( 'LOWER(SUBSTR(name, 1, 1)) = ?' => $o{char} ) : (),
defined($o{char}) && !$o{char} ? ( '(ASCII(name) < 97 OR ASCII(name) > 122) AND (ASCII(name) < 65 OR ASCII(name) > 90)' => 1 ) : (), defined($o{char}) && !$o{char} ? ( '(ASCII(name) < 97 OR ASCII(name) > 122) AND (ASCII(name) < 65 OR ASCII(name) > 90)' => 1 ) : (),
# Only get the latest version
# TODO: This is more efficient than using, e.g. SELECT DISTINCT to achieve
# the same effect. The downside of this solution is that packages of which
# the latest release does not include man pages are not listed, even if an
# earlier version does have mans.
'NOT EXISTS(SELECT 1 FROM package p2 WHERE p2.system = p.system AND p2.name = p.name AND p2.released > p.released)'
); );
return $s->dbAll(q{ return $s->dbAll(q{
SELECT name, category SELECT id, system, name, category
FROM package p FROM packages p
!W !W
ORDER BY name ORDER BY name
LIMIT ?}, LIMIT ?},
@ -735,18 +773,16 @@ sub dbPackageList {
} }
# TODO: Optimize? sub dbPackageVersions {
sub dbPackageGet { my($s, $id) = @_;
my($s, $sysid, $name) = @_;
return $s->dbAll(q{ return $s->dbAll(q{
SELECT id, category, name, version, released SELECT id, version, released
FROM package p FROM package_versions v
WHERE system = ? WHERE package = ?
AND name = ? AND EXISTS(SELECT 1 FROM man m WHERE m.package = v.id)
AND EXISTS(SELECT 1 FROM man m WHERE m.package = p.id)
ORDER BY released DESC}, ORDER BY released DESC},
$sysid, $name) $id)
} }

36
www/man.js
View file

@ -346,7 +346,7 @@ function bsDecode(s) {
/* Structure of VARS.mans: /* Structure of VARS.mans:
[ [
["System", "Full name", "short", [ ["System", "Full name", "short", [
[ "package", "version", [ [ "category", "package", "version", [
[ "section", "locale"||null ], [ "section", "locale"||null ],
... ...
], ],
@ -358,6 +358,8 @@ function bsDecode(s) {
], ],
... ...
] ]
The godawful navigation code desperately needs a rewrite.
*/ */
navShowLocales = false; navShowLocales = false;
@ -410,14 +412,14 @@ function navCreate(nav) {
function navCreatePkg(nav, view, dd, sys, n) { function navCreatePkg(nav, view, dd, sys, n) {
var pkg = sys[3][n]; var pkg = sys[3][n];
var isold = n > 0 && sys[3][n-1][0] == pkg[0]; var isold = n > 0 && sys[3][n-1][0] == pkg[0] && sys[3][n-1][1] == pkg[1];;
if(isold && !pkg[3]) if(isold && !pkg[4])
return false; return false;
var mannum = 0; var mannum = 0;
var pdd = tag('dd', null); var pdd = tag('dd', null);
for(var i=0; i<pkg[2].length; i++) { for(var i=0; i<pkg[3].length; i++) {
var man = pkg[2][i]; var man = pkg[3][i];
var txt = man[0] + (man[1] ? '.'+man[1] : ''); var txt = man[0] + (man[1] ? '.'+man[1] : '');
if(man[2] != VARS.hash && man[1]) if(man[2] != VARS.hash && man[1])
navHasLocale = true; navHasLocale = true;
@ -430,17 +432,17 @@ function navCreatePkg(nav, view, dd, sys, n) {
} }
if(mannum > 0) { if(mannum > 0) {
dd.appendChild(tag('dt', tag('a', {href:'/browse/'+sys[2]+'/'+pkg[0]+'/'+pkg[1]}, pkg[0]), dd.appendChild(tag('dt', tag('a', {href:'/pkg/'+sys[2]+'/'+pkg[0]+'/'+pkg[1]+'/'+pkg[2]}, pkg[1]),
isold || !sys[3][n+1] || sys[3][n+1][0] != pkg[0] ? null : tag('a', isold || !sys[3][n+1] || sys[3][n+1][0] != pkg[0] || sys[3][n+1][1] != pkg[1] ? null : tag('a',
{href:'#', _pkgn: pkg[0], _pkgi:n, 'class':'expand', {href:'#', _pkgn: pkg[0]+'-'+pkg[1], _pkgi:n, 'class':'expand',
title: 'Show/hide historical versions of this package', title: 'Show/hide historical versions of this package',
onclick: function() { onclick: function() {
for(var j=this._pkgi+1; j<sys[3].length && sys[3][j][0] == this._pkgn; j++) for(var j=this._pkgi+1; j<sys[3].length && sys[3][j][0]+'-'+sys[3][j][1] == this._pkgn; j++)
sys[3][j][3] = !sys[3][j][3]; sys[3][j][4] = !sys[3][j][4];
navCreate(nav); navCreate(nav);
return false return false
}}, sys[3][n+1][3] ? expanded_icon : collapsed_icon), }}, sys[3][n+1][4] ? expanded_icon : collapsed_icon),
tag('i', pkg[1]))); tag('i', pkg[0] + ' / ' + pkg[2])));
dd.appendChild(pdd); dd.appendChild(pdd);
return true; return true;
} }
@ -461,7 +463,7 @@ function navCreateLinks(nav) {
// Serializes the current navigation view into a short string. The string is a // Serializes the current navigation view into a short string. The string is a
// bsEncode()ed bit array, creates as follows: // bsEncode()ed bit array, created as follows:
// array.push(navShowLocales); // array.push(navShowLocales);
// for(each system that has an expand button) // for(each system that has an expand button)
// array.push(is the butten expanded or not); // array.push(is the butten expanded or not);
@ -478,8 +480,8 @@ function navSerialize() {
a.push(!!VARS.mans[i+1][4]); a.push(!!VARS.mans[i+1][4]);
for(var i=0; i<VARS.mans.length; i++) for(var i=0; i<VARS.mans.length; i++)
for(var j=0; j<VARS.mans[i][3].length; j++) for(var j=0; j<VARS.mans[i][3].length; j++)
if(j+1 < VARS.mans[i][3].length && VARS.mans[i][3][j+1][0] == VARS.mans[i][3][j][0] && (j == 0 || VARS.mans[i][3][j-1][0] != VARS.mans[i][3][j][0])) if(j+1 < VARS.mans[i][3].length && VARS.mans[i][3][j+1][0] == VARS.mans[i][3][j][0] && VARS.mans[i][3][j+1][1] == VARS.mans[i][3][j][1] && (j == 0 || VARS.mans[i][3][j-1][0] != VARS.mans[i][3][j][0] || VARS.mans[i][3][j-1][1] != VARS.mans[i][3][j][1]))
a.push(!!VARS.mans[i][3][j+1][3]); a.push(!!VARS.mans[i][3][j+1][4]);
return bsEncode(a).replace(/(.)a+$/, '$1'); return bsEncode(a).replace(/(.)a+$/, '$1');
} }
@ -492,8 +494,8 @@ function navLoad(s) {
VARS.mans[i][4] = i > 1 && VARS.mans[i-2][0] == VARS.mans[i-1][0] ? VARS.mans[i-1][4] : !!a.shift(); VARS.mans[i][4] = i > 1 && VARS.mans[i-2][0] == VARS.mans[i-1][0] ? VARS.mans[i-1][4] : !!a.shift();
for(var i=0; i<VARS.mans.length; i++) for(var i=0; i<VARS.mans.length; i++)
for(var j=0; j<VARS.mans[i][3].length; j++) for(var j=0; j<VARS.mans[i][3].length; j++)
if(j > 0 && VARS.mans[i][3][j-1][0] == VARS.mans[i][3][j][0]) if(j > 0 && VARS.mans[i][3][j-1][0] == VARS.mans[i][3][j][0] && VARS.mans[i][3][j-1][1] == VARS.mans[i][3][j][1])
VARS.mans[i][3][j][3] = j > 1 && VARS.mans[i][3][j-2][0] == VARS.mans[i][3][j-1][0] ? VARS.mans[i][3][j-1][3] : !!a.shift(); VARS.mans[i][3][j][4] = j > 1 && VARS.mans[i][3][j-2][0] == VARS.mans[i][3][j-1][0] && VARS.mans[i][3][j-2][1] == VARS.mans[i][3][j-1][1] ? VARS.mans[i][3][j-1][4] : !!a.shift();
} }