Large-ish SQL schema revamp/optimizations

Primarily aimed at reducing the size of the old 'man' (now: files)
table, using smaller integers to refer to man contents and text fields,
and storing a shorthash as an integer for quick lookups. This better
normalization also removes the need to keep a separate 'man_index' cache
for the search function.

The old schema wasn't necessarily bad, but I was in the mood for some
optimizations. And a little cleanup.

Prolly introduces a bunch of new bugs, I haven't tested this too well.

sql/schema.sql

@@ -7,15 +7,44 @@ CREATE TABLE systems (
   short    varchar NOT NULL
 );
 
+
 CREATE TABLE contents (
+  id      SERIAL PRIMARY KEY,
   -- 'hash' is the SHA1 of the man page file after decompression but *before*
   -- encoding conversion and removing 0-bytes. This means taking sha1(content)
   -- may not necessary match the hash, and it's possible for the same content
   -- to be in the database under multiple hashes (but I suspect that's rare).
-  hash    bytea      PRIMARY KEY,
-  content varchar    NOT NULL
+  hash    bytea      NOT NULL UNIQUE,
+  content text       NOT NULL
 );
 
+
+-- Unique man page, as identified by name & section
+CREATE TABLE mans (
+  id      SERIAL PRIMARY KEY,
+  name    text NOT NULL,
+  section text NOT NULL,
+  UNIQUE(name, section)
+);
+CREATE INDEX mans_name ON mans USING btree(lower(name) text_pattern_ops);
+
+
+-- List of man page locales for efficient referencing.  Some locales include
+-- the encoding in their name, which isn't really correct or even necessary
+-- since we convert everything to UTF-8 anyway, but w/e, Can fix later.
+CREATE TABLE locales (
+  id      SMALLSERIAL PRIMARY KEY,
+  locale  text NOT NULL UNIQUE
+);
+
+
+-- List of encodings for efficient referencing.
+CREATE TABLE encodings (
+  id       SMALLSERIAL PRIMARY KEY,
+  encoding text NOT NULL UNIQUE
+);
+
+
 CREATE TABLE packages (
   id       SERIAL    PRIMARY KEY,
   system   integer   NOT NULL REFERENCES systems(id) ON DELETE CASCADE ON UPDATE CASCADE,
@@ -31,6 +60,7 @@ CREATE TABLE packages (
   UNIQUE(system, name, category) -- Note the order, lookups on (system,name) are common
 );
 
+
 CREATE TABLE package_versions (
   id       SERIAL    PRIMARY KEY,
   package  integer   NOT NULL REFERENCES packages(id) ON DELETE CASCADE,
@@ -40,43 +70,36 @@ CREATE TABLE package_versions (
   UNIQUE(package, version)
 );
 
-CREATE TABLE man (
-  package  integer   NOT NULL REFERENCES package_versions(id) ON DELETE CASCADE,
-  name     varchar   NOT NULL,
-  filename varchar   NOT NULL,
-  locale   varchar,
-  hash     bytea     NOT NULL,
-  section  varchar   NOT NULL,
-  encoding varchar,
-  UNIQUE(package, filename)
+
+CREATE TABLE files (
+  pkgver    integer   NOT NULL REFERENCES package_versions(id) ON DELETE CASCADE,
+  man       integer   NOT NULL REFERENCES mans(id),
+  content   integer   NOT NULL REFERENCES content(id),
+  shorthash integer   NOT NULL, -- cache: hash_to_shorthash(content.hash)
+  locale    smallint  NOT NULL REFERENCES locales(id)
+  -- The original encoding the man page was found in. This column isn't really
+  -- used at the moment, but is potentially useful when investigating encoding
+  -- issues.
+  encoding  smallint  NOT NULL REFERENCES encodings(id),
+  filename  text      NOT NULL,
+  PRIMARY KEY(pkgver, filename)
 );
+CREATE INDEX ON files (man, shorthash);
+CREATE INDEX ON files (content);
 
-CREATE INDEX ON man (hash);
-CREATE INDEX ON man (name);
+
+-- For stats_cache
+\i util/update_indices.sql
 
 
 
-CREATE TABLE man_index AS SELECT DISTINCT name, section FROM man;
-CREATE INDEX ON man_index USING btree(lower(name) text_pattern_ops);
-
-CREATE TABLE stats_cache AS SELECT count(distinct hash) AS hashes, count(distinct name) AS mans, count(*) AS files, count(distinct package) AS packages FROM man;
-
-
-
--- Removes any path components and compression extensions from the filename.
-CREATE OR REPLACE FUNCTION basename_from_filename(fn text) RETURNS text AS $$
-  SELECT regexp_replace(fn, '^.+/([^/][^/]*?)(?:\.gz|\.lzma|\.xz|\.bz2|\.zst)*$', '\1');
-$$ LANGUAGE SQL;
-
-
-CREATE OR REPLACE FUNCTION section_from_filename(text) RETURNS text AS $$
-  SELECT regexp_replace(basename_from_filename($1), '^.+\.([^.]+)$', '\1');
-$$ LANGUAGE SQL;
-
-
-CREATE OR REPLACE FUNCTION name_from_filename(text) RETURNS text AS $$
-  SELECT regexp_replace(basename_from_filename($1), '^(.+)\.[^.]+$', '\1');
-$$ LANGUAGE SQL;
+-- Interpret first 4 bytes of hash as a signed 32-bit integer.
+CREATE OR REPLACE FUNCTION hash_to_shorthash(hash bytea) RETURNS integer AS $$
+  SELECT CASE WHEN get_byte(hash, 3) < 128
+         THEN (get_byte(hash, 3)::int<<24) + (get_byte(hash, 2)::int<<16) + (get_byte(hash, 1)::int<<8) + get_byte(hash, 0)
+         ELSE -2147483648 + ((get_byte(hash, 3)::int - 128)<<24) + (get_byte(hash, 2)::int<<16) + (get_byte(hash, 1)::int<<8) + get_byte(hash, 0)
+         END;
+$$ LANGUAGE SQL IMMUTABLE;
 
 
 CREATE OR REPLACE FUNCTION is_english_locale(locale text) RETURNS bool AS $$