diff --git a/sql/schema.sql b/schema.sql similarity index 100% rename from sql/schema.sql rename to schema.sql diff --git a/sql/update-2021-12-13.sql b/sql/update-2021-12-13.sql deleted file mode 100644 index d49baa0..0000000 --- a/sql/update-2021-12-13.sql +++ /dev/null @@ -1,44 +0,0 @@ -CREATE OR REPLACE FUNCTION match_firstchar(str text, chr text) RETURNS boolean AS $$ - SELECT CASE WHEN chr = '0' - THEN (ascii(str) < 97 OR ascii(str) > 122) AND (ascii(str) < 65 OR ascii(str) > 90) - ELSE ascii(str) IN(ascii(chr),ascii(upper(chr))) - END; -$$ LANGUAGE SQL IMMUTABLE; - - -ALTER TABLE packages DROP CONSTRAINT packages_system_fkey; -ALTER TABLE packages ADD CONSTRAINT packages_system_fkey FOREIGN KEY (system) REFERENCES systems(id) ON DELETE CASCADE ON UPDATE CASCADE; - --- Remapping system IDs so we can sort on that rather than 'relorder' --- (Yes, this means adding a new system somewhere in-between requires changing --- system IDs again, but that's fine, these IDs are only used internally) -BEGIN; -WITH map(old,new) AS ( - -- FreeBSD - SELECT 80, 84 - UNION SELECT 84, 80 - UNION SELECT 132, 99 - UNION SELECT 99, 129 - UNION SELECT 129, 132 - UNION SELECT 188, 198 - UNION SELECT 192, 188 - UNION SELECT 198, 203 - UNION SELECT 203, 192 - -- CentOS - UNION SELECT 175, 176 - UNION SELECT 176, 177 - UNION SELECT 177, 178 - UNION SELECT 178, 179 - UNION SELECT 179, 182 - UNION SELECT 182, 183 - UNION SELECT 183, 175 - UNION SELECT 195, 200 - UNION SELECT 196, 201 - UNION SELECT 200, 195 - UNION SELECT 201, 206 - UNION SELECT 206, 196 -) UPDATE systems SET id = new+10000 FROM map WHERE id = old; -UPDATE systems SET id = id-10000 WHERE id >= 10000; -COMMIT; - -ALTER TABLE systems DROP COLUMN relorder; diff --git a/sql/update-2021-12-13b.sql b/sql/update-2021-12-13b.sql deleted file mode 100644 index 9dad7ae..0000000 --- a/sql/update-2021-12-13b.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE packages - ALTER COLUMN category SET NOT NULL, - ADD COLUMN dead boolean NOT NULL DEFAULT FALSE; diff --git a/sql/update-2021-12-14.sql b/sql/update-2021-12-14.sql deleted file mode 100644 index 2a86a92..0000000 --- a/sql/update-2021-12-14.sql +++ /dev/null @@ -1,106 +0,0 @@ --- WARNING: This script does a complete rewrite of the two largest tables. --- This requires ~25G of temporary space and takes anywhere between 10 and 20 minutes. --- The site needs to be taken down during that time. - - -DROP FUNCTION section_from_filename(text); -DROP FUNCTION name_from_filename(text); -DROP FUNCTION basename_from_filename(text); - --- Interpret first 4 bytes of hash as a signed 32-bit integer. -CREATE OR REPLACE FUNCTION hash_to_shorthash(hash bytea) RETURNS integer AS $$ - SELECT CASE WHEN get_byte(hash, 3) < 128 - THEN (get_byte(hash, 3)::int<<24) + (get_byte(hash, 2)::int<<16) + (get_byte(hash, 1)::int<<8) + get_byte(hash, 0) - ELSE -2147483648 + ((get_byte(hash, 3)::int - 128)<<24) + (get_byte(hash, 2)::int<<16) + (get_byte(hash, 1)::int<<8) + get_byte(hash, 0) - END; -$$ LANGUAGE SQL IMMUTABLE; - - -ALTER TABLE contents DROP CONSTRAINT contents_pkey; -ALTER TABLE contents RENAME TO contents_old; - -CREATE SEQUENCE contents_id_seq AS integer; -CREATE TABLE contents ( - id integer NOT NULL DEFAULT nextval('contents_id_seq'::regclass), - hash bytea NOT NULL, - content text NOT NULL -); -ALTER SEQUENCE contents_id_seq OWNED BY contents.id; --- 4m15s; start 29.3G end 53.3G; +24G -INSERT INTO contents (hash, content) SELECT hash, content FROM contents_old; -DROP TABLE contents_old; -- back to 29.3G -ALTER TABLE contents ADD PRIMARY KEY (id); -ALTER TABLE contents ADD UNIQUE (hash); --- 29.7G at this point - - - -CREATE TABLE mans ( - id SERIAL PRIMARY KEY, - name text NOT NULL, - section text NOT NULL, - UNIQUE(name, section) -); -INSERT INTO mans (name, section) SELECT DISTINCT name, section FROM man; -CREATE INDEX mans_name ON mans USING btree(lower(name) text_pattern_ops); - -CREATE TABLE locales ( - id SMALLSERIAL PRIMARY KEY, - locale text NOT NULL UNIQUE -); -INSERT INTO locales (id, locale) VALUES (0,''); -- 0 for default locale is handy, I guess -INSERT INTO locales (locale) SELECT locale FROM man WHERE locale IS NOT NULL GROUP BY locale ORDER BY locale; - --- Encodings are stored for reference but are never actually used anywhere. -CREATE TABLE encodings ( - id SMALLSERIAL PRIMARY KEY, - encoding text NOT NULL UNIQUE -); -INSERT INTO encodings (id, encoding) VALUES (0,''); -INSERT INTO encodings (encoding) SELECT encoding FROM man WHERE encoding IS NOT NULL GROUP BY encoding ORDER BY encoding; - - --- Replaces the 'man' table; It's the largest table in terms of number of rows --- and it's pretty frequently accessed, so keeping the rows small helps. -CREATE TABLE files ( - pkgver integer NOT NULL, -- package_versions.id - man integer NOT NULL, -- mans.id - content integer NOT NULL, -- content.id - shorthash integer NOT NULL, -- cache: hash_to_shorthash(content.hash) - locale smallint NOT NULL, -- locales.id - encoding smallint NOT NULL, -- encodings.id - filename text NOT NULL -); - --- 1min; 29.7G -> 31.8G -INSERT INTO files - SELECT o.package, m.id, c.id, hash_to_shorthash(o.hash), l.id, e.id, o.filename - FROM man o - JOIN mans m ON m.name = o.name AND m.section = o.section - JOIN contents c ON c.hash = o.hash - JOIN locales l ON l.locale = coalesce(o.locale, '') - JOIN encodings e ON e.encoding = coalesce(o.encoding, ''); - --- 1min; 31.8G -> 33.7G for both indices -ALTER TABLE files ADD PRIMARY KEY (pkgver, filename); -CREATE INDEX ON files (man, shorthash); -CREATE INDEX ON files (content); - --- 20sec to verify -ALTER TABLE files - ADD CONSTRAINT files_pkgver_fkey FOREIGN KEY (pkgver) REFERENCES package_versions (id) ON DELETE CASCADE, - ADD CONSTRAINT files_man_fkey FOREIGN KEY (man) REFERENCES mans (id), - ADD CONSTRAINT files_content_fkey FOREIGN KEY (content) REFERENCES contents (id), - ADD CONSTRAINT files_locale_fkey FOREIGN KEY (locale) REFERENCES locales (id), - ADD CONSTRAINT files_encoding_fkey FOREIGN KEY (encoding) REFERENCES encodings (id); - -DROP TABLE man; -DROP TABLE man_index; --- final: 29.1G; we saved a whole 300M! \o/ - - --- There's only about a 100 unreferenced rows, leftovers from removals of --- incorrectly indexed packages. Let's remove them while we're at it. -DELETE FROM contents WHERE NOT EXISTS(SELECT 1 FROM files WHERE content = id); - -VACUUM ANALYZE mans, files, contents, locales, encodings; diff --git a/sql/update-2021-12-16.sql b/sql/update-2021-12-16.sql deleted file mode 100644 index 9a93131..0000000 --- a/sql/update-2021-12-16.sql +++ /dev/null @@ -1,8 +0,0 @@ --- 'overrides' is not a valid locale. Rather, that was a directory that CentOS --- used for updated man pages without overwriting the pages already installed --- on the system. --- 'man' is just a badly nested directory. --- The others are openmpi using non-standard directories for some reason. -WITH b(id) AS (SELECT id FROM locales WHERE locale IN('man', 'overrides') OR locale ~ '(openmpi|mpich|mvapich)'), - f AS (UPDATE files SET locale = 0 WHERE locale IN(SELECT id FROM b)) -DELETE FROM locales WHERE id IN(SELECT id FROM b);