diff --git a/sql/schema.sql b/sql/schema.sql index a9e2538..c81d284 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -1,19 +1,24 @@ CREATE TABLE systems ( - id integer PRIMARY KEY, -- hardcoded ID. + -- Manually assigned number. The id is also used for ordering different + -- releases of the same system, as identified by 'name'. + id integer PRIMARY KEY, name varchar NOT NULL, release varchar, - relorder integer NOT NULL DEFAULT 0, -- simple way of ordering different releases for the same system short varchar NOT NULL ); CREATE TABLE contents ( + -- 'hash' is the SHA1 of the man page file after decompression but *before* + -- encoding conversion and removing 0-bytes. This means taking sha1(content) + -- may not necessary match the hash, and it's possible for the same content + -- to be in the database under multiple hashes (but I suspect that's rare). hash bytea PRIMARY KEY, content varchar NOT NULL ); CREATE TABLE packages ( id SERIAL PRIMARY KEY, - system integer NOT NULL REFERENCES systems(id) ON DELETE CASCADE, + system integer NOT NULL REFERENCES systems(id) ON DELETE CASCADE ON UPDATE CASCADE, category varchar, name varchar NOT NULL, UNIQUE(system, name, category) -- Note the order, lookups on (system,name) are common @@ -51,240 +56,6 @@ CREATE TABLE stats_cache AS SELECT count(distinct hash) AS hashes, count(distinc -INSERT INTO systems (id, name, release, short, relorder) VALUES - (1, 'Arch Linux', NULL, 'arch', 0), - (2, 'Ubuntu', '4.10', 'ubuntu-warty', 0), - (3, 'Ubuntu', '5.04', 'ubuntu-hoary', 1), - (4, 'Ubuntu', '5.10', 'ubuntu-breezy', 2), - (5, 'Ubuntu', '6.06', 'ubuntu-dapper', 3), - (6, 'Ubuntu', '6.10', 'ubuntu-edgy', 4), - (7, 'Ubuntu', '7.04', 'ubuntu-feisty', 5), - (8, 'Ubuntu', '7.10', 'ubuntu-gutsy', 6), - (9, 'Ubuntu', '8.04', 'ubuntu-hardy', 7), - (10, 'Ubuntu', '8.10', 'ubuntu-intrepid', 8), - (11, 'Ubuntu', '9.04', 'ubuntu-jaunty', 9), - (12, 'Ubuntu', '9.10', 'ubuntu-karmic', 10), - (13, 'Ubuntu', '10.04', 'ubuntu-lucid', 11), - (14, 'Ubuntu', '10.10', 'ubuntu-maverick', 12), - (15, 'Ubuntu', '11.04', 'ubuntu-natty', 13), - (16, 'Ubuntu', '11.10', 'ubuntu-oneiric', 14), - (17, 'Ubuntu', '12.04', 'ubuntu-precise', 15), - (18, 'Debian', '1.1', 'debian-buzz', 0), - (19, 'Debian', '1.2', 'debian-rex', 1), - (20, 'Debian', '1.3', 'debian-bo', 2), - (21, 'Debian', '2.0', 'debian-hamm', 3), - (22, 'Debian', '2.1', 'debian-slink', 4), - (23, 'Debian', '2.2', 'debian-potato', 5), - (24, 'Debian', '3.0', 'debian-woody', 6), - (25, 'Debian', '3.1', 'debian-sarge', 7), - (26, 'Debian', '4.0', 'debian-etch', 8), - (27, 'Debian', '5.0', 'debian-lenny', 9), - (28, 'Debian', '6.0', 'debian-squeeze', 10), - (29, 'FreeBSD', '1.0', 'freebsd-1.0', 0), - (30, 'FreeBSD', '2.0.5', 'freebsd-2.0.5', 1), - (31, 'FreeBSD', '2.1.5', 'freebsd-2.1.5', 2), - (32, 'FreeBSD', '2.1.7', 'freebsd-2.1.7', 3), - (33, 'FreeBSD', '2.2.2', 'freebsd-2.2.2', 4), - (34, 'FreeBSD', '2.2.5', 'freebsd-2.2.5', 5), - (35, 'FreeBSD', '2.2.6', 'freebsd-2.2.6', 6), - (36, 'FreeBSD', '2.2.7', 'freebsd-2.2.7', 7), - (37, 'FreeBSD', '2.2.8', 'freebsd-2.2.8', 8), - (38, 'FreeBSD', '3.0', 'freebsd-3.0', 9), - (39, 'FreeBSD', '3.1', 'freebsd-3.1', 10), - (40, 'FreeBSD', '3.2', 'freebsd-3.2', 11), - (41, 'FreeBSD', '3.3', 'freebsd-3.3', 12), - (42, 'FreeBSD', '3.4', 'freebsd-3.4', 13), - (43, 'FreeBSD', '3.5', 'freebsd-3.5', 14), - (44, 'FreeBSD', '3.5.1', 'freebsd-3.5.1', 15), - (45, 'FreeBSD', '4.0', 'freebsd-4.0', 16), - (46, 'FreeBSD', '4.1', 'freebsd-4.1', 17), - (47, 'FreeBSD', '4.1.1', 'freebsd-4.1.1', 18), - (48, 'FreeBSD', '4.2', 'freebsd-4.2', 19), - (49, 'FreeBSD', '4.3', 'freebsd-4.3', 20), - (50, 'FreeBSD', '4.4', 'freebsd-4.4', 21), - (51, 'FreeBSD', '4.5', 'freebsd-4.5', 22), - (52, 'FreeBSD', '4.6', 'freebsd-4.6', 23), - (53, 'FreeBSD', '4.6.2', 'freebsd-4.6.2', 24), - (54, 'FreeBSD', '4.7', 'freebsd-4.7', 25), - (55, 'FreeBSD', '4.8', 'freebsd-4.8', 26), - (56, 'FreeBSD', '4.9', 'freebsd-4.9', 27), - (57, 'FreeBSD', '4.10', 'freebsd-4.10', 28), - (58, 'FreeBSD', '4.11', 'freebsd-4.11', 29), - (59, 'FreeBSD', '5.0', 'freebsd-5.0', 30), - (60, 'FreeBSD', '5.1', 'freebsd-5.1', 31), - (61, 'FreeBSD', '5.2', 'freebsd-5.2', 32), - (62, 'FreeBSD', '5.2.1', 'freebsd-5.2.1', 33), - (63, 'FreeBSD', '5.3', 'freebsd-5.3', 34), - (64, 'FreeBSD', '5.4', 'freebsd-5.4', 35), - (65, 'FreeBSD', '5.5', 'freebsd-5.5', 36), - (66, 'FreeBSD', '6.0', 'freebsd-6.0', 37), - (67, 'FreeBSD', '6.1', 'freebsd-6.1', 38), - (68, 'FreeBSD', '6.2', 'freebsd-6.2', 39), - (69, 'FreeBSD', '6.3', 'freebsd-6.3', 40), - (70, 'FreeBSD', '6.4', 'freebsd-6.4', 41), - (71, 'FreeBSD', '7.0', 'freebsd-7.0', 42), - (72, 'FreeBSD', '7.1', 'freebsd-7.1', 43), - (73, 'FreeBSD', '7.2', 'freebsd-7.2', 44), - (74, 'FreeBSD', '7.3', 'freebsd-7.3', 45), - (75, 'FreeBSD', '7.4', 'freebsd-7.4', 46), - (76, 'FreeBSD', '8.0', 'freebsd-8.0', 47), - (77, 'FreeBSD', '8.1', 'freebsd-8.1', 48), - (78, 'FreeBSD', '8.2', 'freebsd-8.2', 49), - (79, 'FreeBSD', '8.3', 'freebsd-8.3', 50), - (80, 'FreeBSD', '9.0', 'freebsd-9.0', 52), - (81, 'Ubuntu', '12.10', 'ubuntu-quantal', 16), - (82, 'Ubuntu', '13.04', 'ubuntu-raring', 17), - (83, 'Debian', '7', 'debian-wheezy', 11), - (84, 'FreeBSD', '8.4', 'freebsd-8.4', 51), - (85, 'FreeBSD', '9.1', 'freebsd-9.1', 53), - (86, 'FreeBSD', '9.2', 'freebsd-9.2', 54), - (87, 'Ubuntu', '13.10', 'ubuntu-saucy', 18), - (88, 'Ubuntu', '14.04', 'ubuntu-trusty', 19), - (89, 'Ubuntu', '14.10', 'ubuntu-utopic', 20), - (90, 'Ubuntu', '15.04', 'ubuntu-vivid', 21), - (91, 'Debian', '8', 'debian-jessie', 12), - (92, 'Ubuntu', '15.10', 'ubuntu-wily', 22), - (93, 'Ubuntu', '16.04', 'ubuntu-xenial', 23), - (94, 'FreeBSD', '9.3', 'freebsd-9.3', 55), - (95, 'FreeBSD', '10.0', 'freebsd-10.0', 56), - (96, 'FreeBSD', '10.1', 'freebsd-10.1', 57), - (97, 'FreeBSD', '10.2', 'freebsd-10.2', 58), - (98, 'FreeBSD', '10.3', 'freebsd-10.3', 59), - (99, 'FreeBSD', '11.0', 'freebsd-11.0', 61), - (100,'Ubuntu', '16.10', 'ubuntu-yakkety', 24), - (101,'Fedora', '1', 'fedora-1', 0), - (102,'Fedora', '2', 'fedora-2', 1), - (103,'Fedora', '3', 'fedora-3', 2), - (104,'Fedora', '4', 'fedora-4', 3), - (105,'Fedora', '5', 'fedora-5', 4), - (106,'Fedora', '6', 'fedora-6', 5), - (107,'Fedora', '7', 'fedora-7', 6), - (108,'Fedora', '8', 'fedora-8', 7), - (109,'Fedora', '9', 'fedora-9', 8), - (110,'Fedora', '10', 'fedora-10', 9), - (111,'Fedora', '11', 'fedora-11', 10), - (112,'Fedora', '12', 'fedora-12', 11), - (113,'Fedora', '13', 'fedora-13', 12), - (114,'Fedora', '14', 'fedora-14', 13), - (115,'Fedora', '15', 'fedora-15', 14), - (116,'Fedora', '16', 'fedora-16', 15), - (117,'Fedora', '17', 'fedora-17', 16), - (118,'Fedora', '18', 'fedora-18', 17), - (119,'Fedora', '19', 'fedora-19', 18), - (120,'Fedora', '20', 'fedora-20', 19), - (121,'Fedora', '21', 'fedora-21', 20), - (122,'Fedora', '22', 'fedora-22', 21), - (123,'Fedora', '23', 'fedora-23', 22), - (124,'Fedora', '24', 'fedora-24', 23), - (125,'Fedora', '25', 'fedora-25', 24), - (126,'Ubuntu', '17.04', 'ubuntu-zesty', 25), - (127,'Debian', '9', 'debian-stretch', 13), - (128,'Fedora', '26', 'fedora-26', 25), - (129,'FreeBSD', '11.1', 'freebsd-11.1', 62), - (130,'Ubuntu', '17.10', 'ubuntu-artful', 26), - (131,'Fedora', '27', 'fedora-27', 26), - (132,'FreeBSD', '10.4', 'freebsd-10.4', 60), - (133,'CentOS', '2.1', 'centos-2.1', 1), - (134,'CentOS', '3.1', 'centos-3.1', 2), - (136,'CentOS', '3.3', 'centos-3.3', 4), - (137,'CentOS', '3.4', 'centos-3.4', 5), - (138,'CentOS', '3.5', 'centos-3.5', 6), - (139,'CentOS', '3.6', 'centos-3.6', 7), - (140,'CentOS', '3.7', 'centos-3.7', 8), - (141,'CentOS', '3.8', 'centos-3.8', 9), - (142,'CentOS', '3.9', 'centos-3.9', 10), - (143,'CentOS', '4.0', 'centos-4.0', 11), - (144,'CentOS', '4.1', 'centos-4.1', 12), - (145,'CentOS', '4.2', 'centos-4.2', 13), - (146,'CentOS', '4.3', 'centos-4.3', 14), - (147,'CentOS', '4.4', 'centos-4.4', 15), - (148,'CentOS', '4.5', 'centos-4.5', 16), - (149,'CentOS', '4.6', 'centos-4.6', 17), - (150,'CentOS', '4.7', 'centos-4.7', 18), - (151,'CentOS', '4.8', 'centos-4.8', 19), - (152,'CentOS', '4.9', 'centos-4.9', 20), - (153,'CentOS', '5.0', 'centos-5.0', 21), - (154,'CentOS', '5.1', 'centos-5.1', 22), - (155,'CentOS', '5.2', 'centos-5.2', 23), - (156,'CentOS', '5.3', 'centos-5.3', 24), - (157,'CentOS', '5.4', 'centos-5.4', 25), - (158,'CentOS', '5.5', 'centos-5.5', 26), - (159,'CentOS', '5.6', 'centos-5.6', 27), - (160,'CentOS', '5.7', 'centos-5.7', 28), - (161,'CentOS', '5.8', 'centos-5.8', 29), - (162,'CentOS', '5.9', 'centos-5.9', 30), - (163,'CentOS', '5.10', 'centos-5.10', 31), - (164,'CentOS', '5.11', 'centos-5.11', 32), - (165,'CentOS', '6.0', 'centos-6.0', 33), - (166,'CentOS', '6.1', 'centos-6.1', 34), - (167,'CentOS', '6.2', 'centos-6.2', 35), - (168,'CentOS', '6.3', 'centos-6.3', 36), - (169,'CentOS', '6.4', 'centos-6.4', 37), - (170,'CentOS', '6.5', 'centos-6.5', 38), - (171,'CentOS', '6.6', 'centos-6.6', 39), - (172,'CentOS', '6.7', 'centos-6.7', 40), - (173,'CentOS', '6.8', 'centos-6.8', 41), - (174,'CentOS', '6.9', 'centos-6.9', 42), - (175,'CentOS', '7.0', 'centos-7.0', 44), - (176,'CentOS', '7.1', 'centos-7.1', 45), - (177,'CentOS', '7.2', 'centos-7.2', 46), - (178,'CentOS', '7.3', 'centos-7.3', 47), - (179,'CentOS', '7.4', 'centos-7.4', 48), - (180,'Ubuntu', '18.04', 'ubuntu-bionic', 27), - (181,'Fedora', '28', 'fedora-28', 27), - (182,'CentOS', '7.5', 'centos-7.5', 49), - (183,'CentOS', '6.10', 'centos-6.10', 43), - (184,'FreeBSD', '11.2', 'freebsd-11.2', 63), - (185,'Ubuntu', '18.10', 'ubuntu-cosmic', 28), - (186,'Fedora', '29', 'fedora-29', 28), - (187,'CentOS', '7.6', 'centos-7.6', 50), - (188,'FreeBSD', '12.0', 'freebsd-12.0', 70), - (189,'Ubuntu', '19.04', 'ubuntu-disco', 29), - (190,'Debian', '10', 'debian-buster', 14), - (191,'Fedora', '30', 'fedora-30', 29), - (192,'FreeBSD', '11.3', 'freebsd-11.3', 64), - (193,'Ubuntu', '19.10', 'ubuntu-eoan', 30), - (194,'CentOS', '7.7', 'centos-7.7', 51), - (195,'CentOS', '8.0', 'centos-8.0', 60), - (196,'CentOS', '8.1', 'centos-8.1', 61), - (197,'Fedora', '31', 'fedora-31', 30), - (198,'FreeBSD', '12.1', 'freebsd-12.1', 71), - (199,'Ubuntu', '20.04', 'ubuntu-focal', 31), - (200,'CentOS', '7.8', 'centos-7.8', 52), - (201,'CentOS', '8.2', 'centos-8.2', 62), - (202,'Fedora', '32', 'fedora-32', 31), - (203,'FreeBSD', '11.4', 'freebsd-11.4', 65), - (204,'Ubuntu', '20.10', 'ubuntu-groovy', 32), - (205,'Fedora', '33', 'fedora-33', 32), - (206,'CentOS', '7.9', 'centos-7.9', 53), - (207,'CentOS', '8.3', 'centos-8.3', 63), - (208,'FreeBSD', '12.2', 'freebsd-12.2', 72), - (209,'FreeBSD', '13.0', 'freebsd-13.0', 80), - (210,'Ubuntu', '21.04', 'ubuntu-hirsute', 33), - (211,'Fedora', '34', 'fedora-34', 33), - (212,'Debian', '11 (stable)','debian-bullseye',15), - (213,'Debian', '12 (testing)','debian-bookworm',16), - (214,'CentOS', '8.4', 'centos-8.4', 64), - (215,'Ubuntu', '21.10', 'ubuntu-impish', 34), - (216,'CentOS', '8.5', 'centos-8.5', 65), - (217,'Alpine Linux','3.0', 'alpine-3.0', 300), - (218,'Alpine Linux','3.1', 'alpine-3.1', 301), - (219,'Alpine Linux','3.2', 'alpine-3.2', 302), - (220,'Alpine Linux','3.3', 'alpine-3.3', 303), - (221,'Alpine Linux','3.4', 'alpine-3.4', 304), - (222,'Alpine Linux','3.5', 'alpine-3.5', 305), - (223,'Alpine Linux','3.6', 'alpine-3.6', 306), - (224,'Alpine Linux','3.7', 'alpine-3.7', 307), - (225,'Alpine Linux','3.8', 'alpine-3.8', 308), - (226,'Alpine Linux','3.9', 'alpine-3.9', 309), - (227,'Alpine Linux','3.10', 'alpine-3.10', 310), - (228,'Alpine Linux','3.11', 'alpine-3.11', 311), - (229,'Alpine Linux','3.12', 'alpine-3.12', 312), - (230,'Alpine Linux','3.13', 'alpine-3.13', 313), - (231,'Alpine Linux','3.14', 'alpine-3.14', 314), - (232,'Alpine Linux','3.15', 'alpine-3.15', 315); - - -- Removes any path components and compression extensions from the filename. CREATE OR REPLACE FUNCTION basename_from_filename(fn text) RETURNS text AS $$ DECLARE @@ -320,3 +91,12 @@ $$ IMMUTABLE LANGUAGE SQL; CREATE OR REPLACE FUNCTION is_standard_man_location(path text) RETURNS bool AS $$ SELECT path LIKE '/usr/share/man/man%' OR path LIKE '/usr/local/man/man%'; $$ IMMUTABLE LANGUAGE sql; + +-- Convenient function to match the first character of a string. Second argument must be lowercase 'a'-'z' or '0'. +-- Postgres can inline and partially evaluate this function into the query plan, so it's fairly efficient. +CREATE OR REPLACE FUNCTION match_firstchar(str text, chr text) RETURNS boolean AS $$ + SELECT CASE WHEN chr = '0' + THEN (ascii(str) < 97 OR ascii(str) > 122) AND (ascii(str) < 65 OR ascii(str) > 90) + ELSE ascii(str) IN(ascii(chr),ascii(upper(chr))) + END; +$$ LANGUAGE SQL IMMUTABLE; diff --git a/sql/update-2016-10-02.sql b/sql/update-2016-10-02.sql deleted file mode 100644 index a669314..0000000 --- a/sql/update-2016-10-02.sql +++ /dev/null @@ -1,30 +0,0 @@ -CREATE TABLE packages ( - id SERIAL PRIMARY KEY, - system integer NOT NULL REFERENCES systems(id), - category varchar, - name varchar NOT NULL, - UNIQUE(system, name, category) -- Note the order, lookups on (system,name) are common -); - -CREATE TABLE package_versions ( - id SERIAL PRIMARY KEY, - package integer NOT NULL REFERENCES packages(id), - version varchar NOT NULL, - released date NOT NULL, - UNIQUE(package, version) -); - -INSERT INTO packages (system, category, name) SELECT system, category, name FROM package GROUP BY system, category, name; -INSERT INTO package_versions (id, package, version, released) - SELECT p.id, pn.id, p.version, p.released FROM package p JOIN packages pn ON pn.system = p.system AND pn.category = p.category AND pn.name = p.name; - -SELECT setval('package_versions_id_seq', nextval('package_id_seq')); - -ALTER TABLE man DROP CONSTRAINT man_package_fkey; -ALTER TABLE man ADD FOREIGN KEY (package) REFERENCES package_versions(id); - --- Use a proper b-tree index -DROP INDEX man_hash_idx; -CREATE INDEX ON man (hash); - --- DROP TABLE package; diff --git a/sql/update-2016-10-03.sql b/sql/update-2016-10-03.sql deleted file mode 100644 index 68605d2..0000000 --- a/sql/update-2016-10-03.sql +++ /dev/null @@ -1,4 +0,0 @@ --- This check is not consistent with the HTML-check in util/add_dir.pl, but it --- happens to match exactly the same man pages currently. -DELETE FROM man WHERE section = 'html'; -DELETE FROM contents c WHERE NOT EXISTS(SELECT 1 FROM man m WHERE m.hash = c.hash); diff --git a/sql/update-2016-10-06.sql b/sql/update-2016-10-06.sql deleted file mode 100644 index ff4b394..0000000 --- a/sql/update-2016-10-06.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE packages DROP CONSTRAINT packages_system_fkey, ADD CONSTRAINT packages_system_fkey FOREIGN KEY(system) REFERENCES systems(id) ON DELETE CASCADE; -ALTER TABLE package_versions DROP CONSTRAINT package_versions_package_fkey, ADD CONSTRAINT package_versions_package_fkey FOREIGN KEY(package) REFERENCES packages(id) ON DELETE CASCADE; -ALTER TABLE man DROP CONSTRAINT man_package_fkey, ADD CONSTRAINT man_package_fkey FOREIGN KEY(package) REFERENCES package_versions(id) ON DELETE CASCADE; diff --git a/sql/update-2016-10-09.sql b/sql/update-2016-10-09.sql deleted file mode 100644 index 3c3ed7c..0000000 --- a/sql/update-2016-10-09.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE OR REPLACE FUNCTION is_english_locale(locale text) RETURNS bool AS $$ - SELECT locale IS NULL OR locale LIKE 'en%'; -$$ IMMUTABLE LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION is_standard_man_location(path text) RETURNS bool AS $$ - SELECT path LIKE '/usr/share/man/man%' OR path LIKE '/usr/local/man/man%'; -$$ IMMUTABLE LANGUAGE sql; diff --git a/sql/update-2016-10-16.sql b/sql/update-2016-10-16.sql deleted file mode 100644 index 2a630a6..0000000 --- a/sql/update-2016-10-16.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Various non-manpages -DELETE FROM man - WHERE filename ~ '/Makefile\.(in|am)$' - OR filename ~ '/\.cvsignore(\.gz)?$' - OR filename !~ '/[^/]*\.[^/]*$' - OR filename ~ '/man\.tmp$'; - --- Wrong locales, found with: --- SELECT DISTINCT Locale FROM man ORDER BY locale; -UPDATE man SET locale = NULL - WHERE locale = '5man' - OR locale = 'c' - OR locale ~ '^man.?$' - OR locale ~ '^Man-Part[12]$'; - --- Man page containing only a '$1'. Likely a build failure in earlier FreeBSD releases. -DELETE FROM man WHERE hash = '\x5ea7b8101325c704551852f70b652e0a2b0d7c12'; - -DELETE FROM contents c WHERE NOT EXISTS(SELECT 1 FROM man m WHERE m.hash = c.hash); diff --git a/sql/update-2016-11-06.sql b/sql/update-2016-11-06.sql deleted file mode 100644 index 7f4e820..0000000 --- a/sql/update-2016-11-06.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE package_versions ADD COLUMN arch varchar; -ALTER TABLE man ADD COLUMN encoding varchar; diff --git a/sql/update-2016-12-18.sql b/sql/update-2016-12-18.sql deleted file mode 100644 index 479e114..0000000 --- a/sql/update-2016-12-18.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Delete some non-man-pages, these are already blacklisted in indexer/man.rs:validate(). --- Found with: --- select hash from contents where content like e'\x7fELF%' or length(content) < 9 or content = e'timestamp\n' or content = e'.so man3/\n'; -DELETE FROM man WHERE hash IN( - '\x0c8b9d6f753e8d8ec9276bfe98e993a133847642', - '\x2c0f4624792234e2c289eec5b6bad1c699f84128', - '\x2d8b07a585e3f072aaa2c2b0ebac91d9039ccd54', - '\x4853d24dbb7d81e4782ef1bb1162c143f808dda1', - '\x6dfe263ccab32880795dff4479d990ade1daa839', - '\x72d08cf649f7a5cb0a9434d5b591878b2f7a0df9', - '\x816aff0cb37d6c9ccbfa48fffec83e39b37193fa', - '\x8af6f3e189ca29abb3a0ba51d7ef5e7e70451639', - '\x8e5113f6f47ce34e0437c2105441dbb70f01491a', - '\x92d754c27d4a6f851505be63aad6366857060f42', - '\x9312a9f378cc7750c4f473e3fdbd1d9b4aaf1efa', - '\x9f83db09859f909ce36b5aa97ec412c09ea27a76', - '\xadc83b19e793491b1c6ea0fd8b46cd9f32e592fc', - '\xb226262bf9f49e1098612ffdfc01680f7f305f70', - '\xc1a1a4edf60cfd417d52fc7bf1698bf0b6c814e6', - '\xda39a3ee5e6b4b0d3255bfef95601890afd80709', - '\xe04aededffd5ff6a3bc1a5294796ce1efa4dc68d', - '\xfe847f886e2d883a946282a552123dbba00f9596', - '\xffecacd94bd2bc4488db35a6b761ed430a65ac8f' -); - -DELETE FROM contents c WHERE NOT EXISTS(SELECT 1 FROM man m WHERE m.hash = c.hash); diff --git a/sql/update-2021-12-13.sql b/sql/update-2021-12-13.sql new file mode 100644 index 0000000..d49baa0 --- /dev/null +++ b/sql/update-2021-12-13.sql @@ -0,0 +1,44 @@ +CREATE OR REPLACE FUNCTION match_firstchar(str text, chr text) RETURNS boolean AS $$ + SELECT CASE WHEN chr = '0' + THEN (ascii(str) < 97 OR ascii(str) > 122) AND (ascii(str) < 65 OR ascii(str) > 90) + ELSE ascii(str) IN(ascii(chr),ascii(upper(chr))) + END; +$$ LANGUAGE SQL IMMUTABLE; + + +ALTER TABLE packages DROP CONSTRAINT packages_system_fkey; +ALTER TABLE packages ADD CONSTRAINT packages_system_fkey FOREIGN KEY (system) REFERENCES systems(id) ON DELETE CASCADE ON UPDATE CASCADE; + +-- Remapping system IDs so we can sort on that rather than 'relorder' +-- (Yes, this means adding a new system somewhere in-between requires changing +-- system IDs again, but that's fine, these IDs are only used internally) +BEGIN; +WITH map(old,new) AS ( + -- FreeBSD + SELECT 80, 84 + UNION SELECT 84, 80 + UNION SELECT 132, 99 + UNION SELECT 99, 129 + UNION SELECT 129, 132 + UNION SELECT 188, 198 + UNION SELECT 192, 188 + UNION SELECT 198, 203 + UNION SELECT 203, 192 + -- CentOS + UNION SELECT 175, 176 + UNION SELECT 176, 177 + UNION SELECT 177, 178 + UNION SELECT 178, 179 + UNION SELECT 179, 182 + UNION SELECT 182, 183 + UNION SELECT 183, 175 + UNION SELECT 195, 200 + UNION SELECT 196, 201 + UNION SELECT 200, 195 + UNION SELECT 201, 206 + UNION SELECT 206, 196 +) UPDATE systems SET id = new+10000 FROM map WHERE id = old; +UPDATE systems SET id = id-10000 WHERE id >= 10000; +COMMIT; + +ALTER TABLE systems DROP COLUMN relorder; diff --git a/www/index.pl b/www/index.pl index 0b7f31a..acd7ec7 100755 --- a/www/index.pl +++ b/www/index.pl @@ -63,7 +63,7 @@ sub systems { state $s ||= [ map { $_->{full} = $_->{name}.($_->{release}?' '.$_->{release}:''); $_ - } tuwf->dbAll('SELECT id, name, release, short, relorder FROM systems ORDER BY name, relorder')->@* ]; + } tuwf->dbAll('SELECT id, name, release, short FROM systems ORDER BY name, id')->@* ]; } sub sysbyid { state $s ||= { map +($_->{id}, $_), systems->@* } } @@ -134,13 +134,16 @@ sub man_pref { # 3. stdloc: Prefer man pages in standard locations # 4. secmatch: Prefer an exact section match # 5. arch: Prefer Arch over other systems (because it tends to be the most up-to-date, and closest to upstreams) - # 6. debian: If there's no Arch, prefer Debian over other systems (again, tends to be more up-to-date) + # 6. debian: If there's no Arch, prefer latest Debian over other systems (again, tends to be more up-to-date) # (also resolves distro-specific tooling disputes such as https://code.blicky.net/yorhel/manned/issues/1 ) - # 7. sysrel: Prefer a later system release over an older release + # 7. sysrel: Prefer a more recent system release over an older release # 8. secorder: Lower sections before higher sections (because man does it this way, for some reason) # 9. pkgdate: Prefer more recent packages (cross-distro) # 10. Fall back on hash comparison, to ensure the result is stable + state $archid = sysbyshort->{arch}{id}; + state $debid = (sort { $b->{id} <=> $a->{id} } grep $_->{short} =~ /^debian-/, systems->@*)[0]{id}; + tuwf->dbRowi(q{ WITH unfiltered AS ( SELECT s AS sys, p AS pkg, v AS ver, m AS man @@ -158,11 +161,11 @@ sub man_pref { ), f_secmatch AS( SELECT * FROM f_stdloc WHERE NOT EXISTS(SELECT 1 FROM f_stdloc WHERE (man).section =}, \$section, q{) OR (man).section =}, \$section, q{ ), f_arch AS( - SELECT * FROM f_secmatch WHERE NOT EXISTS(SELECT 1 FROM}, length $section ? 'f_secmatch' : 'f_stdloc', q{WHERE (sys).id = 1) OR (sys).id = 1 + SELECT * FROM f_secmatch WHERE NOT EXISTS(SELECT 1 FROM}, length $section ? 'f_secmatch' : 'f_stdloc', qq{WHERE (sys).id = $archid) OR (sys).id = $archid ), f_debian AS( - SELECT * FROM f_arch WHERE NOT EXISTS(SELECT 1 FROM f_arch WHERE (sys).name = 'Debian') OR (sys).name = 'Debian' + SELECT * FROM f_arch WHERE NOT EXISTS(SELECT 1 FROM f_arch WHERE (sys).id = $debid) OR (sys).id = $debid ), f_sysrel AS( - SELECT * FROM f_debian a WHERE NOT EXISTS(SELECT 1 FROM f_debian b WHERE (a.sys).name = (b.sys).name AND (a.sys).relorder < (b.sys).relorder) + SELECT * FROM f_debian a WHERE NOT EXISTS(SELECT 1 FROM f_debian b WHERE (a.sys).name = (b.sys).name AND (a.sys).id < (b.sys).id) ), f_secorder AS( SELECT * FROM f_sysrel a WHERE NOT EXISTS(SELECT 1 FROM f_sysrel b WHERE (a.man).section > (b.man).section) ), f_pkgdate AS( @@ -778,9 +781,7 @@ TUWF::get qr{/pkg/([^/]+)} => sub { p => { onerror => 1, uint => 1, range => [1,200] }, )->data; - my $where = sql 'system =', \$sys->{id}, - $f->{c} eq '0' ? ('AND (ASCII(name) < 97 OR ASCII(name) > 122) AND (ASCII(name) < 65 OR ASCII(name) > 90)') : - $f->{c} ne 'all' ? ('AND LOWER(SUBSTR(name, 1, 1)) =', \$f->{c}) : (); + my $where = sql 'system =', \$sys->{id}, $f->{c} ne 'all' ? ('AND match_firstchar(name,', \$f->{c}, ')') : (); my $count = tuwf->dbVali('SELECT count(*) FROM', $packages_with_man, 'p WHERE', $where); my $pkg = tuwf->dbPagei({ results => 200, page => $f->{p} }, 'SELECT id, system, name, category FROM', $packages_with_man, 'p WHERE', $where, 'ORDER BY name, category' @@ -960,7 +961,7 @@ TUWF::get '/json/tree.json' => sub { length $f->{locale} ? sql 'm.locale =', \$f->{locale} : (), defined $f->{locale} && $f->{locale} eq '' ? 'm.locale IS NULL' : (), ), ' - ORDER BY s.name, s.relorder DESC, p.name, v.released DESC, m.name, m.locale NULLS FIRST, m.filename + ORDER BY s.name, s.id DESC, p.name, v.released DESC, m.name, m.locale NULLS FIRST, m.filename '); # Convert the list into a tree