83ab6c3671
Whether or not the package name itself or the (category,name) tuple
uniquely identified a package within a system has been a source of
confusion for a long time. Back in
03d278e4ff I ended up playing playing it
"safe" by going for (category,name), but in practice this doesn't make a
whole lot of sense. While it's *possible* for the same package name to
refer to completely different packages in different "categories", in
reality distributions can't sanely support this anyway.
For distributions where the category referred to a repository, the only
cases where the same package name was used in different repos was when
the package has moved from one repo to another. Those should certainly
not be treated as different packages.
For distributions where the category really referred to a category,
there's the Debian approach where the category is purely a tag and
doesn't help identify the package in any way, and then there's FreeBSD
where the category technically ought to be part of the name. There were
a few cases where FreeBSD used categories to separate out different
versions of the same package (e.g. ipv6 vs non-ipv6), but none were
relevant for man pages so I ended up merging those as well.
Getting rid of the categories simplifies and shortens URLs, unclutters
the UI a little bit and merges the packages in listings that should've
been merged all along.
Migration script:
-- Merge packages that are in multiple categories.
-- All versions are moved to the package with the lowest ID.
-- If the same version already exists in a lower ID, the higher-ID version is deleted.
BEGIN;
WITH migrate(old, new, second) AS (
SELECT q.id, MIN(p.id), MAX(p.id)
FROM packages p
JOIN packages q ON q.id > p.id AND p.system = q.system AND p.name = q.name
GROUP BY q.id
), ded(n) AS (
UPDATE packages SET dead = false
FROM migrate m
JOIN packages q ON q.id = m.old
WHERE packages.id = m.new AND packages.dead AND NOT q.dead
RETURNING 1
), mov(n) AS (
UPDATE package_versions SET package = m.new
FROM migrate m
WHERE package_versions.package = m.old
AND NOT EXISTS(
SELECT 1
FROM package_versions v
WHERE v.package IN(m.new, m.second)
AND v.version = package_versions.version)
RETURNING 1
), del(n) AS (
DELETE FROM packages WHERE id IN(SELECT old FROM migrate)
RETURNING 1
) SELECT (SELECT count(*) FROM migrate) AS migrate,
(SELECT count(*) FROM ded) AS ded,
(SELECT count(*) FROM mov) AS mov,
(SELECT count(*) FROM del) AS del;
ALTER TABLE packages DROP CONSTRAINT packages_system_name_category_key;
CREATE UNIQUE INDEX packages_system_name_key ON packages (system, name);
ALTER TABLE packages DROP COLUMN category;
COMMIT;
120 lines
4.4 KiB
PL/PgSQL
120 lines
4.4 KiB
PL/PgSQL
CREATE TABLE systems (
|
|
-- Manually assigned number. The id is also used for ordering different
|
|
-- releases of the same system, as identified by 'name'.
|
|
id integer PRIMARY KEY,
|
|
name varchar NOT NULL,
|
|
release varchar,
|
|
short varchar NOT NULL
|
|
);
|
|
|
|
|
|
CREATE TABLE contents (
|
|
id SERIAL PRIMARY KEY,
|
|
-- 'hash' is the SHA1 of the man page file after decompression but *before*
|
|
-- encoding conversion and removing 0-bytes. This means taking sha1(content)
|
|
-- may not necessary match the hash, and it's possible for the same content
|
|
-- to be in the database under multiple hashes (but I suspect that's rare).
|
|
hash bytea NOT NULL UNIQUE,
|
|
content text NOT NULL
|
|
);
|
|
|
|
|
|
-- Unique man page, as identified by name & section
|
|
CREATE TABLE mans (
|
|
id SERIAL PRIMARY KEY,
|
|
name text NOT NULL,
|
|
section text NOT NULL,
|
|
UNIQUE(name, section)
|
|
);
|
|
CREATE INDEX mans_name ON mans USING btree(lower(name) text_pattern_ops);
|
|
|
|
|
|
-- List of man page locales for efficient referencing. Some locales include
|
|
-- the encoding in their name, which isn't really correct or even necessary
|
|
-- since we convert everything to UTF-8 anyway, but w/e, Can fix later.
|
|
CREATE TABLE locales (
|
|
id SMALLSERIAL PRIMARY KEY,
|
|
locale text NOT NULL UNIQUE
|
|
);
|
|
|
|
|
|
-- List of encodings for efficient referencing.
|
|
CREATE TABLE encodings (
|
|
id SMALLSERIAL PRIMARY KEY,
|
|
encoding text NOT NULL UNIQUE
|
|
);
|
|
|
|
|
|
CREATE TABLE packages (
|
|
id SERIAL PRIMARY KEY,
|
|
system integer NOT NULL REFERENCES systems(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
|
name varchar NOT NULL,
|
|
-- Whether this package has been seen in the last repository update. This
|
|
-- field is only updated for a few systems that are likely to delete packages
|
|
-- over time; non-rolling-release distros tend to not delete packages after
|
|
-- all.
|
|
-- Packages where the latest version does not have any man pages may also be
|
|
-- marked as dead even if the package is still available in the repos.
|
|
dead boolean NOT NULL DEFAULT FALSE,
|
|
UNIQUE(system, name)
|
|
);
|
|
|
|
|
|
CREATE TABLE package_versions (
|
|
id SERIAL PRIMARY KEY,
|
|
package integer NOT NULL REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
|
version varchar NOT NULL,
|
|
released date NOT NULL,
|
|
arch varchar,
|
|
UNIQUE(package, version)
|
|
);
|
|
|
|
|
|
CREATE TABLE files (
|
|
pkgver integer NOT NULL REFERENCES package_versions(id) ON DELETE CASCADE ON UPDATE CASCADE,
|
|
man integer NOT NULL REFERENCES mans(id),
|
|
content integer NOT NULL REFERENCES content(id),
|
|
shorthash integer NOT NULL, -- cache: hash_to_shorthash(content.hash)
|
|
locale smallint NOT NULL REFERENCES locales(id)
|
|
-- The original encoding the man page was found in. This column isn't really
|
|
-- used at the moment, but is potentially useful when investigating encoding
|
|
-- issues.
|
|
encoding smallint NOT NULL REFERENCES encodings(id),
|
|
filename text NOT NULL,
|
|
PRIMARY KEY(pkgver, filename)
|
|
);
|
|
CREATE INDEX ON files (man, shorthash);
|
|
CREATE INDEX ON files (content);
|
|
|
|
|
|
-- For stats_cache
|
|
\i util/update_indices.sql
|
|
|
|
|
|
|
|
-- Interpret first 4 bytes of hash as a signed 32-bit integer.
|
|
CREATE OR REPLACE FUNCTION hash_to_shorthash(hash bytea) RETURNS integer AS $$
|
|
SELECT CASE WHEN get_byte(hash, 3) < 128
|
|
THEN (get_byte(hash, 3)::int<<24) + (get_byte(hash, 2)::int<<16) + (get_byte(hash, 1)::int<<8) + get_byte(hash, 0)
|
|
ELSE -2147483648 + ((get_byte(hash, 3)::int - 128)<<24) + (get_byte(hash, 2)::int<<16) + (get_byte(hash, 1)::int<<8) + get_byte(hash, 0)
|
|
END;
|
|
$$ LANGUAGE SQL IMMUTABLE;
|
|
|
|
|
|
CREATE OR REPLACE FUNCTION is_english_locale(locale text) RETURNS bool AS $$
|
|
SELECT locale IS NULL OR locale = '' OR locale LIKE 'en%';
|
|
$$ IMMUTABLE LANGUAGE SQL;
|
|
|
|
|
|
CREATE OR REPLACE FUNCTION is_standard_man_location(path text) RETURNS bool AS $$
|
|
SELECT path LIKE '/usr/share/man/man%' OR path LIKE '/usr/local/man/man%';
|
|
$$ IMMUTABLE LANGUAGE sql;
|
|
|
|
-- Convenient function to match the first character of a string. Second argument must be lowercase 'a'-'z' or '0'.
|
|
-- Postgres can inline and partially evaluate this function into the query plan, so it's fairly efficient.
|
|
CREATE OR REPLACE FUNCTION match_firstchar(str text, chr text) RETURNS boolean AS $$
|
|
SELECT CASE WHEN chr = '0'
|
|
THEN (ascii(str) < 97 OR ascii(str) > 122) AND (ascii(str) < 65 OR ascii(str) > 90)
|
|
ELSE ascii(str) IN(ascii(chr),ascii(upper(chr)))
|
|
END;
|
|
$$ LANGUAGE SQL IMMUTABLE;
|