vndb/manned
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Get rid of package categories

Browse source 
Whether or not the package name itself or the (category,name) tuple
uniquely identified a package within a system has been a source of
confusion for a long time. Back in
03d278e4ff I ended up playing playing it
"safe" by going for (category,name), but in practice this doesn't make a
whole lot of sense. While it's *possible* for the same package name to
refer to completely different packages in different "categories", in
reality distributions can't sanely support this anyway.

For distributions where the category referred to a repository, the only
cases where the same package name was used in different repos was when
the package has moved from one repo to another. Those should certainly
not be treated as different packages.

For distributions where the category really referred to a category,
there's the Debian approach where the category is purely a tag and
doesn't help identify the package in any way, and then there's FreeBSD
where the category technically ought to be part of the name.  There were
a few cases where FreeBSD used categories to separate out different
versions of the same package (e.g. ipv6 vs non-ipv6), but none were
relevant for man pages so I ended up merging those as well.

Getting rid of the categories simplifies and shortens URLs, unclutters
the UI a little bit and merges the packages in listings that should've
been merged all along.

Migration script:

  -- Merge packages that are in multiple categories.
  -- All versions are moved to the package with the lowest ID.
  -- If the same version already exists in a lower ID, the higher-ID version is deleted.
  BEGIN;
  WITH migrate(old, new, second) AS (
    SELECT q.id, MIN(p.id), MAX(p.id)
      FROM packages p
      JOIN packages q ON q.id > p.id AND p.system = q.system AND p.name = q.name
     GROUP BY q.id
  ), ded(n) AS (
    UPDATE packages SET dead = false
      FROM migrate m
      JOIN packages q ON q.id = m.old
     WHERE packages.id = m.new AND packages.dead AND NOT q.dead
    RETURNING 1
  ), mov(n) AS (
    UPDATE package_versions SET package = m.new
      FROM migrate m
     WHERE package_versions.package = m.old
       AND NOT EXISTS(
          SELECT 1
            FROM package_versions v
           WHERE v.package IN(m.new, m.second)
             AND v.version = package_versions.version)
    RETURNING 1
  ), del(n) AS (
    DELETE FROM packages WHERE id IN(SELECT old FROM migrate)
    RETURNING 1
  ) SELECT (SELECT count(*) FROM migrate) AS migrate,
           (SELECT count(*) FROM ded) AS ded,
           (SELECT count(*) FROM mov) AS mov,
           (SELECT count(*) FROM del) AS del;

  ALTER TABLE packages DROP CONSTRAINT packages_system_name_category_key;
  CREATE UNIQUE INDEX packages_system_name_key ON packages (system, name);
  ALTER TABLE packages DROP COLUMN category;
  COMMIT;
This commit is contained in:
Yorhel 2024-04-28 10:37:02 +02:00
parent bc26633fc7
commit 83ab6c3671
16 changed files with 152 additions and 182 deletions
Download patch file Download diff file Expand all files Collapse all files

6
indexer/src/main.rs
View file

@ -35,7 +35,6 @@ fn main() {
(about: "Index a single package") (about: "Index a single package")
(@arg force: --force "Overwrite existing indexed package") (@arg force: --force "Overwrite existing indexed package")
(@arg sys: --sys +required +takes_value "System short-name") (@arg sys: --sys +required +takes_value "System short-name")
(@arg cat: --cat +required +takes_value "Package category")
(@arg pkg: --pkg +required +takes_value "Package name") (@arg pkg: --pkg +required +takes_value "Package name")
(@arg ver: --ver +required +takes_value "Package version") (@arg ver: --ver +required +takes_value "Package version")
(@arg date: --date +required +takes_value "Package release date") (@arg date: --date +required +takes_value "Package release date")
@ -75,13 +74,11 @@ fn main() {
(@subcommand rpmdir => (@subcommand rpmdir =>
(about: "Index a bare RPM directory") (about: "Index a bare RPM directory")
(@arg sys: --sys +required +takes_value "System short-name") (@arg sys: --sys +required +takes_value "System short-name")
(@arg cat: --cat +required +takes_value "Category to set for all packages")
(@arg mirror: --mirror +required +takes_value "Mirror URL") (@arg mirror: --mirror +required +takes_value "Mirror URL")
) )
(@subcommand rpm => (@subcommand rpm =>
(about: "Index an RPM repository") (about: "Index an RPM repository")
(@arg sys: --sys +required +takes_value "System short-name") (@arg sys: --sys +required +takes_value "System short-name")
(@arg cat: --cat +required +takes_value "Category to set for all packages")
(@arg mirror: --mirror +required +takes_value "Mirror URL") (@arg mirror: --mirror +required +takes_value "Mirror URL")
) )
).get_matches(); ).get_matches();
@ -125,7 +122,6 @@ fn main() {
pkg::pkg(&mut db, pkg::PkgOpt { pkg::pkg(&mut db, pkg::PkgOpt {
force: matches.is_present("force"), force: matches.is_present("force"),
sys: sys, sys: sys,
cat: matches.value_of("cat").unwrap(),
pkg: matches.value_of("pkg").unwrap(), pkg: matches.value_of("pkg").unwrap(),
ver: matches.value_of("ver").unwrap(), ver: matches.value_of("ver").unwrap(),
date: date, date: date,
@ -177,7 +173,6 @@ fn main() {
if let Some(matches) = arg.subcommand_matches("rpmdir") { if let Some(matches) = arg.subcommand_matches("rpmdir") {
let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap()); let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap());
sys_rpmdir::sync(&mut db, sys, sys_rpmdir::sync(&mut db, sys,
matches.value_of("cat").unwrap(),
matches.value_of("mirror").unwrap() matches.value_of("mirror").unwrap()
).unwrap_or_else(|e| error!("{}", e)); ).unwrap_or_else(|e| error!("{}", e));
} }
@ -185,7 +180,6 @@ fn main() {
if let Some(matches) = arg.subcommand_matches("rpm") { if let Some(matches) = arg.subcommand_matches("rpm") {
let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap()); let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap());
sys_rpm::sync(&mut db, sys, sys_rpm::sync(&mut db, sys,
matches.value_of("cat").unwrap(),
matches.value_of("mirror").unwrap() matches.value_of("mirror").unwrap()
).unwrap_or_else(|e| error!("{}", e)); ).unwrap_or_else(|e| error!("{}", e));
} }

9
indexer/src/pkg.rs
View file

@ -39,7 +39,6 @@ impl<'a> Date<'a> {
pub struct PkgOpt<'a> { pub struct PkgOpt<'a> {
pub force: bool, pub force: bool,
pub sys: i32, pub sys: i32,
pub cat: &'a str,
pub pkg: &'a str, pub pkg: &'a str,
pub ver: &'a str, pub ver: &'a str,
pub date: Date<'a>, pub date: Date<'a>,
@ -49,17 +48,17 @@ pub struct PkgOpt<'a> {
fn insert_pkg(tr: &mut postgres::Transaction, opt: &PkgOpt) -> Option<i32> { fn insert_pkg(tr: &mut postgres::Transaction, opt: &PkgOpt) -> Option<i32> {
let pkginfo = format!("sys {} / {} / {} - {} @ {:?} @ {}", opt.sys, opt.cat, opt.pkg, opt.ver, opt.date, opt.file.path); let pkginfo = format!("sys {} / {} - {} @ {:?} @ {}", opt.sys, opt.pkg, opt.ver, opt.date, opt.file.path);
// Use a custom CTE-based insert-or-update. Using an INSERT with an ON CONFLICT clause would be // Use a custom CTE-based insert-or-update. Using an INSERT with an ON CONFLICT clause would be
// easier, but has the downside of allocating a new package id even if one already exists. // easier, but has the downside of allocating a new package id even if one already exists.
// The separate UPDATE query makes sure to unflag the package as dead while not causing any // The separate UPDATE query makes sure to unflag the package as dead while not causing any
// database writes when the row's already fine. // database writes when the row's already fine.
let q = "WITH p(id) AS (SELECT id FROM packages WHERE system = $1 AND category = $2 AND name = $3), let q = "WITH p(id) AS (SELECT id FROM packages WHERE system = $1 AND name = $2),
u AS (UPDATE packages SET dead = FALSE FROM p WHERE packages.id = p.id AND dead), u AS (UPDATE packages SET dead = FALSE FROM p WHERE packages.id = p.id AND dead),
i(id) AS (INSERT INTO packages (system, category, name) SELECT $1, $2, $3 WHERE NOT EXISTS(SELECT 1 FROM p) RETURNING id) i(id) AS (INSERT INTO packages (system, name) SELECT $1, $2 WHERE NOT EXISTS(SELECT 1 FROM p) RETURNING id)
SELECT id FROM p UNION SELECT id FROM i"; SELECT id FROM p UNION SELECT id FROM i";
let pkgid: i32 = match tr.query_one(q, &[&opt.sys, &opt.cat, &opt.pkg]) { let pkgid: i32 = match tr.query_one(q, &[&opt.sys, &opt.pkg]) {
Err(e) => { Err(e) => {
error!("Can't insert package in database: {}", e); error!("Can't insert package in database: {}", e);
return None; return None;

1
indexer/src/sys_alpine.rs
View file

@ -50,7 +50,6 @@ pub fn read_index<T: postgres::GenericClient, R: Read>(pg: &mut T, sys: i32, mir
pkg::pkg(pg, pkg::PkgOpt{ pkg::pkg(pg, pkg::PkgOpt{
force: false, force: false,
sys: sys, sys: sys,
cat: repo,
pkg: pname, pkg: pname,
ver: pver, ver: pver,
date: builddate.map(pkg::Date::Found).unwrap_or(pkg::Date::Max), date: builddate.map(pkg::Date::Found).unwrap_or(pkg::Date::Max),

1
indexer/src/sys_arch.rs
View file

@ -111,7 +111,6 @@ pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str, repo
pkg::pkg(pg, pkg::PkgOpt{ pkg::pkg(pg, pkg::PkgOpt{
force: false, force: false,
sys: sys, sys: sys,
cat: repo,
pkg: &m.name, pkg: &m.name,
ver: &m.version, ver: &m.version,
date: pkg::Date::Known(&m.date), date: pkg::Date::Known(&m.date),

4
indexer/src/sys_deb.rs
View file

@ -49,7 +49,6 @@ fn get_contents(f: Option<open::Path>) -> Result<HashSet<String>> {
#[derive(Default)] #[derive(Default)]
struct Pkg { struct Pkg {
name: Option<String>, name: Option<String>,
section: Option<String>,
arch: Option<String>, arch: Option<String>,
version: Option<String>, version: Option<String>,
filename: Option<String>, filename: Option<String>,
@ -61,7 +60,6 @@ fn handlepkg<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str, man
if manpkgs.len() > 0 && !manpkgs.contains(name) { if manpkgs.len() > 0 && !manpkgs.contains(name) {
return return
} }
let section = match pkg.section { Some(ref x) => x, None => { error!("Package {} has no section", name); return } };
let version = match pkg.version { Some(ref x) => x, None => { error!("Package {} has no version", name); return } }; let version = match pkg.version { Some(ref x) => x, None => { error!("Package {} has no version", name); return } };
let filename = match pkg.filename { Some(ref x) => x, None => { error!("Package {} has no filename", name); return } }; let filename = match pkg.filename { Some(ref x) => x, None => { error!("Package {} has no filename", name); return } };
@ -80,7 +78,6 @@ fn handlepkg<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str, man
pkg::pkg(pg, pkg::PkgOpt{ pkg::pkg(pg, pkg::PkgOpt{
force: false, force: false,
sys: sys, sys: sys,
cat: &section,
pkg: &name, pkg: &name,
ver: &version, ver: &version,
date: pkg::Date::Deb, date: pkg::Date::Deb,
@ -128,7 +125,6 @@ pub fn sync<T: postgres::GenericClient >(pg: &mut T, sys: i32, mirror: &str, con
// Use case-insensitive matching, older package archives used lowercase keys // Use case-insensitive matching, older package archives used lowercase keys
match str::from_utf8(&cap[1]).unwrap().to_lowercase().as_ref() { match str::from_utf8(&cap[1]).unwrap().to_lowercase().as_ref() {
"package" => pkg.name = Some(val.to_string()), "package" => pkg.name = Some(val.to_string()),
"section" => pkg.section = Some(val.to_string()),
"version" => pkg.version = Some(val.to_string()), "version" => pkg.version = Some(val.to_string()),
"architecture" => pkg.arch = Some(val.to_string()), "architecture" => pkg.arch = Some(val.to_string()),
"filename" => pkg.filename = Some(val.to_string()), "filename" => pkg.filename = Some(val.to_string()),

1
indexer/src/sys_freebsd1.rs
View file

@ -71,7 +71,6 @@ pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, arch: &str, mirror
pkg::pkg(pg, pkg::PkgOpt{ pkg::pkg(pg, pkg::PkgOpt{
force: false, force: false,
sys: sys, sys: sys,
cat: &cat,
pkg: pkg, pkg: pkg,
ver: ver, ver: ver,
date: pkg::Date::Desc, date: pkg::Date::Desc,

3
indexer/src/sys_freebsd2.rs
View file

@ -31,7 +31,6 @@ pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str) -> R
lazy_static!( lazy_static!(
static ref RE_NAME : Regex = Regex::new(r#""name"\s*:\s*"(?u:([^ "]+))""#).unwrap(); static ref RE_NAME : Regex = Regex::new(r#""name"\s*:\s*"(?u:([^ "]+))""#).unwrap();
static ref RE_VER : Regex = Regex::new(r#""version"\s*:\s*"(?u:([^ "]+))""#).unwrap(); static ref RE_VER : Regex = Regex::new(r#""version"\s*:\s*"(?u:([^ "]+))""#).unwrap();
static ref RE_CAT : Regex = Regex::new(r#""origin"\s*:\s*"(?u:([^ "/]+))"#).unwrap();
static ref RE_PATH : Regex = Regex::new(r#""path"\s*:\s*"(?u:([^ "]+))""#).unwrap(); static ref RE_PATH : Regex = Regex::new(r#""path"\s*:\s*"(?u:([^ "]+))""#).unwrap();
static ref RE_ARCH : Regex = Regex::new(r#""arch"\s*:\s*"(?u:([^ "]+))""#).unwrap(); static ref RE_ARCH : Regex = Regex::new(r#""arch"\s*:\s*"(?u:([^ "]+))""#).unwrap();
); );
@ -40,14 +39,12 @@ pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str) -> R
let line = line?; let line = line?;
let name = match RE_NAME.captures(&line) { None => continue, Some(c) => str::from_utf8(c.get(1).unwrap().as_bytes()).unwrap() }; let name = match RE_NAME.captures(&line) { None => continue, Some(c) => str::from_utf8(c.get(1).unwrap().as_bytes()).unwrap() };
let ver = match RE_VER .captures(&line) { None => continue, Some(c) => str::from_utf8(c.get(1).unwrap().as_bytes()).unwrap() }; let ver = match RE_VER .captures(&line) { None => continue, Some(c) => str::from_utf8(c.get(1).unwrap().as_bytes()).unwrap() };
let cat = match RE_CAT .captures(&line) { None => continue, Some(c) => str::from_utf8(c.get(1).unwrap().as_bytes()).unwrap() };
let path = match RE_PATH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.get(1).unwrap().as_bytes()).unwrap() }; let path = match RE_PATH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.get(1).unwrap().as_bytes()).unwrap() };
let arch = match RE_ARCH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.get(1).unwrap().as_bytes()).unwrap() }; let arch = match RE_ARCH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.get(1).unwrap().as_bytes()).unwrap() };
let uri = format!("{}{}", mirror, path); let uri = format!("{}{}", mirror, path);
pkg::pkg(pg, pkg::PkgOpt{ pkg::pkg(pg, pkg::PkgOpt{
force: false, force: false,
sys: sys, sys: sys,
cat: cat,
pkg: name, pkg: name,
ver: ver, ver: ver,
date: pkg::Date::Max, date: pkg::Date::Max,

3
indexer/src/sys_rpm.rs
View file

@ -156,7 +156,7 @@ fn repomd(url: String) -> Result<(String,String),Box<dyn Error>> {
} }
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, cat: &str, mirror: &str) -> Result<(),Box<dyn Error>> { pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str) -> Result<(),Box<dyn Error>> {
let(primary, filelists) = repomd(format!("{}repodata/repomd.xml", mirror))?; let(primary, filelists) = repomd(format!("{}repodata/repomd.xml", mirror))?;
let mut pkgswithman = HashSet::new(); let mut pkgswithman = HashSet::new();
@ -172,7 +172,6 @@ pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, cat: &str, mirror:
pkg::pkg(pg, pkg::PkgOpt{ pkg::pkg(pg, pkg::PkgOpt{
force: false, force: false,
sys: sys, sys: sys,
cat: cat,
pkg: &name, pkg: &name,
ver: &pkg.ver.unwrap(), ver: &pkg.ver.unwrap(),
date: pkg::Date::Known(&date), date: pkg::Date::Known(&date),

3
indexer/src/sys_rpmdir.rs
View file

@ -5,7 +5,7 @@ use postgres;
use crate::open; use crate::open;
use crate::pkg; use crate::pkg;
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, cat: &str, mirror: &str) -> Result<()> { pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str) -> Result<()> {
let pkgs : Vec<String> = open::Path{path: mirror, cache: true, canbelocal: false} let pkgs : Vec<String> = open::Path{path: mirror, cache: true, canbelocal: false}
.dirlist()?.into_iter() .dirlist()?.into_iter()
.filter_map(|(n,d)| if d { None } else { Some(n) }) .filter_map(|(n,d)| if d { None } else { Some(n) })
@ -29,7 +29,6 @@ pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, cat: &str, mirror:
pkg::pkg(pg, pkg::PkgOpt{ pkg::pkg(pg, pkg::PkgOpt{
force: false, force: false,
sys: sys, sys: sys,
cat: cat,
pkg: name, pkg: name,
ver: ver, ver: ver,
date: pkg::Date::Max, date: pkg::Date::Max,

7
schema.sql
View file

@ -48,7 +48,6 @@ CREATE TABLE encodings (
CREATE TABLE packages ( CREATE TABLE packages (
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
system integer NOT NULL REFERENCES systems(id) ON DELETE CASCADE ON UPDATE CASCADE, system integer NOT NULL REFERENCES systems(id) ON DELETE CASCADE ON UPDATE CASCADE,
category varchar NOT NULL,
name varchar NOT NULL, name varchar NOT NULL,
-- Whether this package has been seen in the last repository update. This -- Whether this package has been seen in the last repository update. This
-- field is only updated for a few systems that are likely to delete packages -- field is only updated for a few systems that are likely to delete packages
@ -57,13 +56,13 @@ CREATE TABLE packages (
-- Packages where the latest version does not have any man pages may also be -- Packages where the latest version does not have any man pages may also be
-- marked as dead even if the package is still available in the repos. -- marked as dead even if the package is still available in the repos.
dead boolean NOT NULL DEFAULT FALSE, dead boolean NOT NULL DEFAULT FALSE,
UNIQUE(system, name, category) -- Note the order, lookups on (system,name) are common UNIQUE(system, name)
); );
CREATE TABLE package_versions ( CREATE TABLE package_versions (
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
package integer NOT NULL REFERENCES packages(id) ON DELETE CASCADE, package integer NOT NULL REFERENCES packages(id) ON DELETE CASCADE ON UPDATE CASCADE,
version varchar NOT NULL, version varchar NOT NULL,
released date NOT NULL, released date NOT NULL,
arch varchar, arch varchar,
@ -72,7 +71,7 @@ CREATE TABLE package_versions (
CREATE TABLE files ( CREATE TABLE files (
pkgver integer NOT NULL REFERENCES package_versions(id) ON DELETE CASCADE, pkgver integer NOT NULL REFERENCES package_versions(id) ON DELETE CASCADE ON UPDATE CASCADE,
man integer NOT NULL REFERENCES mans(id), man integer NOT NULL REFERENCES mans(id),
content integer NOT NULL REFERENCES content(id), content integer NOT NULL REFERENCES content(id),
shorthash integer NOT NULL, -- cache: hash_to_shorthash(content.hash) shorthash integer NOT NULL, -- cache: hash_to_shorthash(content.hash)

48
util/centos.sh
View file

@ -9,23 +9,23 @@ SMIRROR=http://mirror.transip.net/centos-stream/
# Centos 3.1 - 3.6 (doesn't have useful repo metadata) # Centos 3.1 - 3.6 (doesn't have useful repo metadata)
centa() { centa() {
local VER=$1 local VER=$1
index rpmdir --sys centos-$VER --cat os --mirror "$VMIRROR$VER/os/i386/RedHat/RPMS/" index rpmdir --sys centos-$VER --mirror "$VMIRROR$VER/os/i386/RedHat/RPMS/"
index rpmdir --sys centos-$VER --cat os --mirror "$VMIRROR$VER/updates/i386/RPMS/" index rpmdir --sys centos-$VER --mirror "$VMIRROR$VER/updates/i386/RPMS/"
index rpmdir --sys centos-$VER --cat extras --mirror "$VMIRROR$VER/extras/i386/RPMS/" index rpmdir --sys centos-$VER --mirror "$VMIRROR$VER/extras/i386/RPMS/"
index rpmdir --sys centos-$VER --cat addons --mirror "$VMIRROR$VER/addons/i386/RPMS/" index rpmdir --sys centos-$VER --mirror "$VMIRROR$VER/addons/i386/RPMS/"
index rpmdir --sys centos-$VER --cat contrib --mirror "$VMIRROR$VER/contrib/i386/RPMS/" index rpmdir --sys centos-$VER --mirror "$VMIRROR$VER/contrib/i386/RPMS/"
} }
# Centos 3.7+ (same structure, but has more repos and metadata we can use) # Centos 3.7+ (same structure, but has more repos and metadata we can use)
centb() { centb() {
local VER=$1 local VER=$1
local MIR=${2:-$VMIRROR} local MIR=${2:-$VMIRROR}
index rpm --sys centos-$VER --cat os --mirror "$MIR$VER/os/i386/" index rpm --sys centos-$VER --mirror "$MIR$VER/os/i386/"
index rpm --sys centos-$VER --cat os --mirror "$MIR$VER/updates/i386/" index rpm --sys centos-$VER --mirror "$MIR$VER/updates/i386/"
index rpm --sys centos-$VER --cat extras --mirror "$MIR$VER/extras/i386/" index rpm --sys centos-$VER --mirror "$MIR$VER/extras/i386/"
index rpm --sys centos-$VER --cat addons --mirror "$MIR$VER/addons/i386/" # not present in 6.0+ index rpm --sys centos-$VER --mirror "$MIR$VER/addons/i386/" # not present in 6.0+
index rpm --sys centos-$VER --cat contrib --mirror "$MIR$VER/contrib/i386/" # not present in some 5.x releases index rpm --sys centos-$VER --mirror "$MIR$VER/contrib/i386/" # not present in some 5.x releases
index rpm --sys centos-$VER --cat centosplus --mirror "$MIR$VER/centosplus/i386/" index rpm --sys centos-$VER --mirror "$MIR$VER/centosplus/i386/"
} }
# CentOS 7.0+ (different versioning, using x86_64) # CentOS 7.0+ (different versioning, using x86_64)
@ -33,10 +33,10 @@ centc() {
local VER=$1 local VER=$1
local DIR=$2 local DIR=$2
local MIR=${3:-$VMIRROR} local MIR=${3:-$VMIRROR}
index rpm --sys centos-$VER --cat os --mirror "$MIR$DIR/os/x86_64/" index rpm --sys centos-$VER --mirror "$MIR$DIR/os/x86_64/"
index rpm --sys centos-$VER --cat os --mirror "$MIR$DIR/updates/x86_64/" index rpm --sys centos-$VER --mirror "$MIR$DIR/updates/x86_64/"
index rpm --sys centos-$VER --cat extras --mirror "$MIR$DIR/extras/x86_64/" index rpm --sys centos-$VER --mirror "$MIR$DIR/extras/x86_64/"
index rpm --sys centos-$VER --cat centosplus --mirror "$MIR$DIR/centosplus/x86_64/" index rpm --sys centos-$VER --mirror "$MIR$DIR/centosplus/x86_64/"
} }
# CentOS 8.0+ # CentOS 8.0+
@ -44,11 +44,11 @@ centd() {
local VER=$1 local VER=$1
local DIR=$2 local DIR=$2
local MIR=${3:-$VMIRROR} local MIR=${3:-$VMIRROR}
index rpm --sys centos-$VER --cat BaseOS --mirror "$MIR$DIR/BaseOS/x86_64/os/" index rpm --sys centos-$VER --mirror "$MIR$DIR/BaseOS/x86_64/os/"
index rpm --sys centos-$VER --cat AppStream --mirror "$MIR$DIR/AppStream/x86_64/os/" index rpm --sys centos-$VER --mirror "$MIR$DIR/AppStream/x86_64/os/"
index rpm --sys centos-$VER --cat PowerTools --mirror "$MIR$DIR/PowerTools/x86_64/os/" index rpm --sys centos-$VER --mirror "$MIR$DIR/PowerTools/x86_64/os/"
index rpm --sys centos-$VER --cat extras --mirror "$MIR$DIR/extras/x86_64/os/" index rpm --sys centos-$VER --mirror "$MIR$DIR/extras/x86_64/os/"
index rpm --sys centos-$VER --cat centosplus --mirror "$MIR$DIR/centosplus/x86_64/os/" index rpm --sys centos-$VER --mirror "$MIR$DIR/centosplus/x86_64/os/"
} }
# CentOS Stream 9+ # CentOS Stream 9+
@ -56,16 +56,16 @@ cente() {
local VER=$1 local VER=$1
local DIR=$2 local DIR=$2
local MIR=${3:-$VMIRROR} local MIR=${3:-$VMIRROR}
index rpm --sys centos-$VER --cat BaseOS --mirror "$MIR$DIR/BaseOS/x86_64/os/" index rpm --sys centos-$VER --mirror "$MIR$DIR/BaseOS/x86_64/os/"
index rpm --sys centos-$VER --cat AppStream --mirror "$MIR$DIR/AppStream/x86_64/os/" index rpm --sys centos-$VER --mirror "$MIR$DIR/AppStream/x86_64/os/"
index rpm --sys centos-$VER --cat CRB --mirror "$MIR$DIR/CRB/x86_64/os/" index rpm --sys centos-$VER --mirror "$MIR$DIR/CRB/x86_64/os/"
# There's also NFV and RT (with lots of overlap) and HighAvailability and ResilientStorage (with lots of overlap). # There's also NFV and RT (with lots of overlap) and HighAvailability and ResilientStorage (with lots of overlap).
# Not sure which of those should be included. # Not sure which of those should be included.
} }
case "$1" in case "$1" in
2.1) 2.1)
index rpmdir --sys centos-2.1 --cat core --mirror "${VMIRROR}2.1/final/i386/CentOS/RPMS/" index rpmdir --sys centos-2.1 --mirror "${VMIRROR}2.1/final/i386/CentOS/RPMS/"
;; ;;
3.1) 3.1)
centa 3.1 centa 3.1

76
util/fedora.sh
View file

@ -10,33 +10,33 @@ CMIRROR=http://mirror.nl.leaseweb.net/fedora/linux/
fedora() { # release arch mirror fedora() { # release arch mirror
MIR=$AMIRROR MIR=$AMIRROR
[ -n "$3" ] && MIR=$3 [ -n "$3" ] && MIR=$3
index rpm --sys fedora-$1 --cat everything --mirror "${MIR}releases/$1/Everything/$2/os/" index rpm --sys fedora-$1 --mirror "${MIR}releases/$1/Everything/$2/os/"
index rpm --sys fedora-$1 --cat everything --mirror "${MIR}updates/$1/$2/" index rpm --sys fedora-$1 --mirror "${MIR}updates/$1/$2/"
} }
case "$1" in case "$1" in
1) 1)
index rpmdir --sys fedora-1 --cat core --mirror "${AMIRROR}core/1/i386/os/Fedora/RPMS/" index rpmdir --sys fedora-1 --mirror "${AMIRROR}core/1/i386/os/Fedora/RPMS/"
;; ;;
2) 2)
index rpm --sys fedora-2 --cat core --mirror "${AMIRROR}core/2/i386/os/" index rpm --sys fedora-2 --mirror "${AMIRROR}core/2/i386/os/"
;; ;;
3) 3)
index rpm --sys fedora-3 --cat core --mirror "${AMIRROR}core/3/i386/os/" index rpm --sys fedora-3 --mirror "${AMIRROR}core/3/i386/os/"
index rpm --sys fedora-3 --cat extras --mirror "${AMIRROR}extras/3/i386/" index rpm --sys fedora-3 --mirror "${AMIRROR}extras/3/i386/"
;; ;;
4) 4)
index rpm --sys fedora-4 --cat core --mirror "${AMIRROR}core/4/i386/os/" index rpm --sys fedora-4 --mirror "${AMIRROR}core/4/i386/os/"
index rpm --sys fedora-4 --cat extras --mirror "${AMIRROR}extras/4/i386/" index rpm --sys fedora-4 --mirror "${AMIRROR}extras/4/i386/"
;; ;;
5) 5)
index rpm --sys fedora-5 --cat core --mirror "${AMIRROR}core/5/i386/os/" index rpm --sys fedora-5 --mirror "${AMIRROR}core/5/i386/os/"
index rpm --sys fedora-5 --cat extras --mirror "${AMIRROR}extras/5/i386/" index rpm --sys fedora-5 --mirror "${AMIRROR}extras/5/i386/"
;; ;;
6) 6)
index rpm --sys fedora-6 --cat core --mirror "${AMIRROR}core/6/i386/os/" index rpm --sys fedora-6 --mirror "${AMIRROR}core/6/i386/os/"
index rpm --sys fedora-6 --cat extras --mirror "${AMIRROR}extras/6/i386/" index rpm --sys fedora-6 --mirror "${AMIRROR}extras/6/i386/"
;; ;;
7) 7)
fedora 7 i386 fedora 7 i386
@ -102,56 +102,56 @@ case "$1" in
fedora 27 x86_64 $CMIRROR fedora 27 x86_64 $CMIRROR
;; ;;
28) 28)
index rpm --sys fedora-28 --cat everything --mirror "${CMIRROR}releases/28/Everything/x86_64/os/" index rpm --sys fedora-28 --mirror "${CMIRROR}releases/28/Everything/x86_64/os/"
index rpm --sys fedora-28 --cat everything --mirror "${CMIRROR}updates/28/Everything/x86_64/" index rpm --sys fedora-28 --mirror "${CMIRROR}updates/28/Everything/x86_64/"
;; ;;
29) 29)
index rpm --sys fedora-29 --cat everything --mirror "${CMIRROR}releases/29/Everything/x86_64/os/" index rpm --sys fedora-29 --mirror "${CMIRROR}releases/29/Everything/x86_64/os/"
index rpm --sys fedora-29 --cat everything --mirror "${CMIRROR}updates/29/Everything/x86_64/" index rpm --sys fedora-29 --mirror "${CMIRROR}updates/29/Everything/x86_64/"
;; ;;
30) 30)
index rpm --sys fedora-30 --cat everything --mirror "${CMIRROR}releases/30/Everything/x86_64/os/" index rpm --sys fedora-30 --mirror "${CMIRROR}releases/30/Everything/x86_64/os/"
index rpm --sys fedora-30 --cat everything --mirror "${CMIRROR}updates/30/Everything/x86_64/" index rpm --sys fedora-30 --mirror "${CMIRROR}updates/30/Everything/x86_64/"
;; ;;
31) 31)
index rpm --sys fedora-31 --cat everything --mirror "${CMIRROR}releases/31/Everything/x86_64/os/" index rpm --sys fedora-31 --mirror "${CMIRROR}releases/31/Everything/x86_64/os/"
index rpm --sys fedora-31 --cat everything --mirror "${CMIRROR}updates/31/Everything/x86_64/" index rpm --sys fedora-31 --mirror "${CMIRROR}updates/31/Everything/x86_64/"
;; ;;
32) 32)
index rpm --sys fedora-32 --cat everything --mirror "${CMIRROR}releases/32/Everything/x86_64/os/" index rpm --sys fedora-32 --mirror "${CMIRROR}releases/32/Everything/x86_64/os/"
index rpm --sys fedora-32 --cat everything --mirror "${CMIRROR}updates/32/Everything/x86_64/" index rpm --sys fedora-32 --mirror "${CMIRROR}updates/32/Everything/x86_64/"
;; ;;
33) 33)
index rpm --sys fedora-33 --cat everything --mirror "${CMIRROR}releases/33/Everything/x86_64/os/" index rpm --sys fedora-33 --mirror "${CMIRROR}releases/33/Everything/x86_64/os/"
index rpm --sys fedora-33 --cat everything --mirror "${CMIRROR}updates/33/Everything/x86_64/" index rpm --sys fedora-33 --mirror "${CMIRROR}updates/33/Everything/x86_64/"
;; ;;
34) 34)
index rpm --sys fedora-34 --cat everything --mirror "${CMIRROR}releases/34/Everything/x86_64/os/" index rpm --sys fedora-34 --mirror "${CMIRROR}releases/34/Everything/x86_64/os/"
index rpm --sys fedora-34 --cat everything --mirror "${CMIRROR}updates/34/Everything/x86_64/" index rpm --sys fedora-34 --mirror "${CMIRROR}updates/34/Everything/x86_64/"
;; ;;
35) 35)
index rpm --sys fedora-35 --cat everything --mirror "${CMIRROR}releases/35/Everything/x86_64/os/" index rpm --sys fedora-35 --mirror "${CMIRROR}releases/35/Everything/x86_64/os/"
index rpm --sys fedora-35 --cat everything --mirror "${CMIRROR}updates/35/Everything/x86_64/" index rpm --sys fedora-35 --mirror "${CMIRROR}updates/35/Everything/x86_64/"
;; ;;
36) 36)
index rpm --sys fedora-36 --cat everything --mirror "${CMIRROR}releases/36/Everything/x86_64/os/" index rpm --sys fedora-36 --mirror "${CMIRROR}releases/36/Everything/x86_64/os/"
index rpm --sys fedora-36 --cat everything --mirror "${CMIRROR}updates/36/Everything/x86_64/" index rpm --sys fedora-36 --mirror "${CMIRROR}updates/36/Everything/x86_64/"
;; ;;
37) 37)
index rpm --sys fedora-37 --cat everything --mirror "${CMIRROR}releases/37/Everything/x86_64/os/" index rpm --sys fedora-37 --mirror "${CMIRROR}releases/37/Everything/x86_64/os/"
index rpm --sys fedora-37 --cat everything --mirror "${CMIRROR}updates/37/Everything/x86_64/" index rpm --sys fedora-37 --mirror "${CMIRROR}updates/37/Everything/x86_64/"
;; ;;
38) 38)
index rpm --sys fedora-38 --cat everything --mirror "${CMIRROR}releases/38/Everything/x86_64/os/" index rpm --sys fedora-38 --mirror "${CMIRROR}releases/38/Everything/x86_64/os/"
index rpm --sys fedora-38 --cat everything --mirror "${CMIRROR}updates/38/Everything/x86_64/" index rpm --sys fedora-38 --mirror "${CMIRROR}updates/38/Everything/x86_64/"
;; ;;
39) 39)
index rpm --sys fedora-39 --cat everything --mirror "${CMIRROR}releases/39/Everything/x86_64/os/" index rpm --sys fedora-39 --mirror "${CMIRROR}releases/39/Everything/x86_64/os/"
index rpm --sys fedora-39 --cat everything --mirror "${CMIRROR}updates/39/Everything/x86_64/" index rpm --sys fedora-39 --mirror "${CMIRROR}updates/39/Everything/x86_64/"
;; ;;
40) 40)
index rpm --sys fedora-40 --cat everything --mirror "${CMIRROR}releases/40/Everything/x86_64/os/" index rpm --sys fedora-40 --mirror "${CMIRROR}releases/40/Everything/x86_64/os/"
index rpm --sys fedora-40 --cat everything --mirror "${CMIRROR}updates/40/Everything/x86_64/" index rpm --sys fedora-40 --mirror "${CMIRROR}updates/40/Everything/x86_64/"
;; ;;
old) old)
$0 1 $0 1

51
util/freebsd.sh
View file

@ -10,8 +10,7 @@ CURL="curl -fSs -A manual-page-crawler,info@manned.org"
SPLITTAR="$TMPDIR/freebsd-merged-tar" SPLITTAR="$TMPDIR/freebsd-merged-tar"
# Index a "core" file. Simple wrapper around 'index pkg', with --ver = date, # Index a "core" file. Simple wrapper around 'index pkg', with --ver = date and support for split tar files.
# --cat="core", and support for split tar files.
index_core() { # <sys> <url-prefix> <pkgname> <date> <last-sequence> index_core() { # <sys> <url-prefix> <pkgname> <date> <last-sequence>
local FN=$2 local FN=$2
if [ -n "$5" ]; then if [ -n "$5" ]; then
@ -22,7 +21,7 @@ index_core() { # <sys> <url-prefix> <pkgname> <date> <last-sequence>
$CURL "$FN{"`perl -le "print join ',', 'aa'..'$5'"`'}' >$SPLITTAR || return 1 $CURL "$FN{"`perl -le "print join ',', 'aa'..'$5'"`'}' >$SPLITTAR || return 1
FN=$SPLITTAR FN=$SPLITTAR
fi fi
index pkg --force --sys $1 --cat core --pkg $3 --ver $4 --date $4 $FN index pkg --force --sys $1 --pkg $3 --ver $4 --date $4 $FN
} }
@ -35,29 +34,29 @@ case $1 in
index_core freebsd-1.0 "${MIR}tarballs/xfree86/man.tgz" core-xfree86-man 1993-10-20 index_core freebsd-1.0 "${MIR}tarballs/xfree86/man.tgz" core-xfree86-man 1993-10-20
index_core freebsd-1.0 "${MIR}tarballs/xfree86/pex.tgz" core-xfree86-pex 1993-10-21 index_core freebsd-1.0 "${MIR}tarballs/xfree86/pex.tgz" core-xfree86-pex 1993-10-21
# A few packages # A few packages
index pkg --sys freebsd-1.0 --cat packages --pkg emacs-19-19_bin --ver 1993-09-13 --date 1993-09-13 "${MIR}packages/emacs-19-19_bin.tgz" index pkg --sys freebsd-1.0 --pkg emacs-19-19_bin --ver 1993-09-13 --date 1993-09-13 "${MIR}packages/emacs-19-19_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg f2c_bin --ver 1993-10-01 --date 1993-10-01 "${MIR}packages/f2c_bin.tgz" index pkg --sys freebsd-1.0 --pkg f2c_bin --ver 1993-10-01 --date 1993-10-01 "${MIR}packages/f2c_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg fileutils_bin --ver 1993-10-06 --date 1993-10-06 "${MIR}packages/fileutils_bin.tgz" index pkg --sys freebsd-1.0 --pkg fileutils_bin --ver 1993-10-06 --date 1993-10-06 "${MIR}packages/fileutils_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg ghostscript_bin --ver 1993-10-02 --date 1993-10-02 "${MIR}packages/ghostscript_bin.tgz" index pkg --sys freebsd-1.0 --pkg ghostscript_bin --ver 1993-10-02 --date 1993-10-02 "${MIR}packages/ghostscript_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg gopher_bin --ver 1993-10-15 --date 1993-10-15 "${MIR}packages/gopher_bin.tgz" index pkg --sys freebsd-1.0 --pkg gopher_bin --ver 1993-10-15 --date 1993-10-15 "${MIR}packages/gopher_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg info-zip_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/info-zip_bin.tgz" index pkg --sys freebsd-1.0 --pkg info-zip_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/info-zip_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg jpeg_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/jpeg_bin.tgz" index pkg --sys freebsd-1.0 --pkg jpeg_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/jpeg_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg kermit_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/kermit_bin.tgz" index pkg --sys freebsd-1.0 --pkg kermit_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/kermit_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg ksh_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/ksh_bin.tgz" index pkg --sys freebsd-1.0 --pkg ksh_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/ksh_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg miscutils_bin --ver 1993-09-06 --date 1993-09-06 "${MIR}packages/miscutils_bin.tgz" index pkg --sys freebsd-1.0 --pkg miscutils_bin --ver 1993-09-06 --date 1993-09-06 "${MIR}packages/miscutils_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg mtools_bin --ver 1993-08-30 --date 1993-08-30 "${MIR}packages/mtools_bin.tgz" index pkg --sys freebsd-1.0 --pkg mtools_bin --ver 1993-08-30 --date 1993-08-30 "${MIR}packages/mtools_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg pbmplus_bin --ver 1993-10-05 --date 1993-10-05 "${MIR}packages/pbmplus_bin.tgz" index pkg --sys freebsd-1.0 --pkg pbmplus_bin --ver 1993-10-05 --date 1993-10-05 "${MIR}packages/pbmplus_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg pkg_install --ver 1993-10-10 --date 1993-10-10 "${MIR}packages/pkg_install.tar.gz" index pkg --sys freebsd-1.0 --pkg pkg_install --ver 1993-10-10 --date 1993-10-10 "${MIR}packages/pkg_install.tar.gz"
index pkg --sys freebsd-1.0 --cat packages --pkg shellutils_bin --ver 1993-10-06 --date 1993-10-06 "${MIR}packages/shellutils_bin.tgz" index pkg --sys freebsd-1.0 --pkg shellutils_bin --ver 1993-10-06 --date 1993-10-06 "${MIR}packages/shellutils_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg tcl_bin --ver 1993-09-18 --date 1993-09-18 "${MIR}packages/tcl_bin.tgz" index pkg --sys freebsd-1.0 --pkg tcl_bin --ver 1993-09-18 --date 1993-09-18 "${MIR}packages/tcl_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg tcsh_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/tcsh_bin.tgz" index pkg --sys freebsd-1.0 --pkg tcsh_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/tcsh_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg textutils_bin --ver 1993-09-05 --date 1993-09-05 "${MIR}packages/textutils_bin.tgz" index pkg --sys freebsd-1.0 --pkg textutils_bin --ver 1993-09-05 --date 1993-09-05 "${MIR}packages/textutils_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg tk_bin --ver 1993-09-18 --date 1993-09-18 "${MIR}packages/tk_bin.tgz" index pkg --sys freebsd-1.0 --pkg tk_bin --ver 1993-09-18 --date 1993-09-18 "${MIR}packages/tk_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg urt_bin --ver 1993-10-05 --date 1993-10-05 "${MIR}packages/urt_bin.tgz" index pkg --sys freebsd-1.0 --pkg urt_bin --ver 1993-10-05 --date 1993-10-05 "${MIR}packages/urt_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg xlock_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/xlock_bin.tgz" index pkg --sys freebsd-1.0 --pkg xlock_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/xlock_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg xv_bin --ver 1993-09-06 --date 1993-09-06 "${MIR}packages/xv_bin.tgz" index pkg --sys freebsd-1.0 --pkg xv_bin --ver 1993-09-06 --date 1993-09-06 "${MIR}packages/xv_bin.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg xview32b --ver 1993-09-16 --date 1993-09-16 "${MIR}packages/xview32b.tgz" index pkg --sys freebsd-1.0 --pkg xview32b --ver 1993-09-16 --date 1993-09-16 "${MIR}packages/xview32b.tgz"
index pkg --sys freebsd-1.0 --cat packages --pkg zsh_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/zsh_bin.tgz" index pkg --sys freebsd-1.0 --pkg zsh_bin --ver 1993-09-04 --date 1993-09-04 "${MIR}packages/zsh_bin.tgz"
;; ;;
2.0.5) 2.0.5)
MIR="${AMIRROR}i386/2.0.5-RELEASE/" MIR="${AMIRROR}i386/2.0.5-RELEASE/"

2
util/netbsd.sh
View file

@ -15,7 +15,7 @@ index_base() {
local mirror=${4:-$CMIRROR} local mirror=${4:-$CMIRROR}
local arch=${5:-amd64} local arch=${5:-amd64}
for pkg in comp games man text xbase xcomp xserver; do for pkg in comp games man text xbase xcomp xserver; do
index pkg --sys netbsd-$ver --cat base --pkg $pkg --ver $ver --date $date "${mirror}NetBSD-$ver/$arch/binary/sets/$pkg.$ext" index pkg --sys netbsd-$ver --pkg $pkg --ver $ver --date $date "${mirror}NetBSD-$ver/$arch/binary/sets/$pkg.$ext"
done done
} }

2
util/openbsd.sh
View file

@ -10,7 +10,7 @@ index_base() {
local date=$2 local date=$2
local mirror=${3:-$CMIRROR} local mirror=${3:-$CMIRROR}
for pkg in base comp game man xbase xserv xshare; do for pkg in base comp game man xbase xserv xshare; do
index pkg --sys openbsd-$ver --cat base --pkg $pkg --ver $ver --date $date "${mirror}$ver/amd64/$pkg${ver/./}.tgz" index pkg --sys openbsd-$ver --pkg $pkg --ver $ver --date $date "${mirror}$ver/amd64/$pkg${ver/./}.tgz"
done done
} }

117
www/index.pl
View file

@ -91,32 +91,36 @@ sub sql_and { @_ ? sql_join 'AND', map sql('(', $_, ')'), @_ : sql '1=1' }
sub sql_or { @_ ? sql_join 'OR', map sql('(', $_, ')'), @_ : sql '1=0' } sub sql_or { @_ ? sql_join 'OR', map sql('(', $_, ')'), @_ : sql '1=0' }
# Returns ($pkg_obj, $ver_str, $should_redir)
sub pkg_frompath { sub pkg_frompath {
my($sys_where, $path) = @_; my($sys_where, $path) = @_;
# $path should be "$category/$name" or "$category/$name/$version", since # $path could either be:
# $category may contain a slash, let's try both options. # $name
# $name/$version
# $category/$name (deprecated)
# $category/$name/$version (deprecated)
# $category may contain a slash. We don't have the categories in the
# database anymore, so we'll just provide a redirect for anything that
# looks like it might have been a category.
# $name currently never contains a slash but may do so in the future, so
# let's also handle that.
my sub lookup { my @comp = split '/', $path;
my($cat, $name) = @_; my @names = map join('/', @$_), map +([@comp[$_..$#comp]], [@comp[$_..$#comp-1]]), 0..$#comp;
tuwf->dbRowi('SELECT id, system, name, category FROM', $packages_with_man, 'p WHERE', $sys_where, 'AND category =', \$cat, 'AND name =', \$name);
}
# $category/$name my $pkg = tuwf->dbRowi('
# e.g. contrib/games/alien SELECT id, system, name
if($path =~ m{^(.+)/([^/]+)$}) { FROM', $packages_with_man, 'p
my $pkg = lookup $1, $2; WHERE', $sys_where, 'AND name IN', \@names, '
return ($pkg, '') if $pkg->{id}; ORDER BY system DESC, length(name) DESC
} LIMIT 1
');
# $category/$name/$version return (undef, '', 0) if !$pkg->{id};
# e.g. contrib/games/alien/10.2
if($path =~ m{^(.+)/([^/]+)/([^/]+)$}) {
my $pkg = lookup $1, $2;
return ($pkg, $3) if $pkg->{id};
}
(undef, ''); my $ver = $path =~ m{\Q$pkg->{name}\E/([^/]+)$} ? $1 : '';
($pkg, $ver, $path !~ /^\Q$pkg->{name}/);
} }
@ -170,7 +174,7 @@ sub man_pref {
), f_pkgdate AS( ), f_pkgdate AS(
SELECT * FROM f_secorder a WHERE NOT EXISTS(SELECT 1 FROM f_secorder b WHERE (a.ver).released < (b.ver).released) SELECT * FROM f_secorder a WHERE NOT EXISTS(SELECT 1 FROM f_secorder b WHERE (a.ver).released < (b.ver).released)
) )
SELECT (pkg).system, (pkg).category, (pkg).name AS package, (ver).version, (ver).released, (ver).id AS verid, SELECT (pkg).system, (pkg).name AS package, (ver).version, (ver).released, (ver).id AS verid,
name, section, filename, locale, shorthash, content name, section, filename, locale, shorthash, content
FROM f_pkgdate ORDER BY shorthash LIMIT 1 FROM f_pkgdate ORDER BY shorthash LIMIT 1
}); });
@ -396,12 +400,12 @@ TUWF::get '/info/about' => sub {
Will get the latest version of a man page from the given system, e.g.:<br> Will get the latest version of a man page from the given system, e.g.:<br>
<a href="/man/ubuntu/rsync">/man/ubuntu/rsync</a><br> <a href="/man/ubuntu/rsync">/man/ubuntu/rsync</a><br>
<a href="/man/ubuntu-xenial/rsync">/man/ubuntu-xenial/rsync</a></dd> <a href="/man/ubuntu-xenial/rsync">/man/ubuntu-xenial/rsync</a></dd>
<dt><code>/man/&lt;system>/&lt;category>/&lt;package>/&lt;name>[.&lt;section>]</code></dt><dd> <dt><code>/man/&lt;system>/&lt;package>/&lt;name>[.&lt;section>]</code></dt><dd>
Will get the latest version of a man page from the given package, e.g.:<br> Will get the latest version of a man page from the given package, e.g.:<br>
<a href="/man/ubuntu-xenial/net/rsync/rsync">/man/ubuntu-xenial/net/rsync/rsync</a></dd> <a href="/man/ubuntu-xenial/rsync/rsync">/man/ubuntu-xenial/rsync/rsync</a></dd>
<dt><code>/man/&lt;system>/&lt;category>/&lt;package>/&lt;version>/&lt;name>[.&lt;section>]</code></dt><dd> <dt><code>/man/&lt;system>/&lt;package>/&lt;version>/&lt;name>[.&lt;section>]</code></dt><dd>
Will get the man page from a specific package version, e.g.:<br> Will get the man page from a specific package version, e.g.:<br>
<a href="/man/ubuntu-xenial/net/rsync/3.1.1-3ubuntu1/rsync">/man/ubuntu-xenial/net/rsync/3.1.1-3ubuntu1/rsync</a></dd> <a href="/man/ubuntu-xenial/rsync/3.1.1-3ubuntu1/rsync">/man/ubuntu-xenial/rsync/3.1.1-3ubuntu1/rsync</a></dd>
<dt><code>/man.&lt;language>/...</code></dt><dd> <dt><code>/man.&lt;language>/...</code></dt><dd>
Adding a language code to the <code>/man/</code> component will select Adding a language code to the <code>/man/</code> component will select
the man page in the requested language. The man page has to be available the man page in the requested language. The man page has to be available
@ -427,7 +431,7 @@ TUWF::get '/info/about' => sub {
In all of the above URL formats, you can change <code>/man</code> with In all of the above URL formats, you can change <code>/man</code> with
<code>/raw</code> to get the raw UTF-8 encoded man page source, e.g.:<br> <code>/raw</code> to get the raw UTF-8 encoded man page source, e.g.:<br>
<a href="/raw/socket.7">/raw/socket.7</a><br> <a href="/raw/socket.7">/raw/socket.7</a><br>
<a href="/raw/ubuntu-xenial/net/rsync/3.1.1-3ubuntu1/rsync">/raw/ubuntu-xenial/net/rsync/3.1.1-3ubuntu1/rsync</a><br> <a href="/raw/ubuntu-xenial/rsync/3.1.1-3ubuntu1/rsync">/raw/ubuntu-xenial/rsync/3.1.1-3ubuntu1/rsync</a><br>
<a href="/raw.de/faked-tcp">/raw.de/faked-tcp</a><br> <a href="/raw.de/faked-tcp">/raw.de/faked-tcp</a><br>
<a href="/raw.910be0ed/fedora/ls">/raw.910be0ed/fedora/ls</a></dd> <a href="/raw.910be0ed/fedora/ls">/raw.910be0ed/fedora/ls</a></dd>
<dt><code>/&lt;name>/&lt;8-hex-digits></code></dt><dd> <dt><code>/&lt;name>/&lt;8-hex-digits></code></dt><dd>
@ -444,12 +448,12 @@ TUWF::get '/info/about' => sub {
<p>Linking to individual packages is also possible. These pages will show a <p>Linking to individual packages is also possible. These pages will show a
listing of all manual pages available in the given package.</p> listing of all manual pages available in the given package.</p>
<dl> <dl>
<dt><code>/pkg/&lt;system>/&lt;category>/&lt;package></code></dt><dd> <dt><code>/pkg/&lt;system>/&lt;package></code></dt><dd>
For the latest version of a package (e.g. <a For the latest version of a package (e.g. <a
href="/pkg/arch/core/coreutils">/pkg/arch/core/coreutils</a>).</dd> href="/pkg/arch/coreutils">/pkg/arch/coreutils</a>).</dd>
<dt><code>/pkg/&lt;system>/&lt;category>/&lt;package>/&lt;version></code></dt><dd> <dt><code>/pkg/&lt;system>/&lt;package>/&lt;version></code></dt><dd>
For a particular version of a package (e.g. <a For a particular version of a package (e.g. <a
href="/pkg/arch/core/coreutils/8.25-2">/pkg/arch/core/coreutils/8.25-2</a>).</dd> href="/pkg/arch/coreutils/8.25-2">/pkg/arch/coreutils/8.25-2</a>).</dd>
</dl> </dl>
<p>This site only indexes packages that actually have manual pages, <p>This site only indexes packages that actually have manual pages,
linking to a package that doesn't have any will result in a 404 page.</p> linking to a package that doesn't have any will result in a 404 page.</p>
@ -645,17 +649,17 @@ TUWF::get '/xml/search.xml' => sub {
# shorthash => 8-char hex # shorthash => 8-char hex
# lang => language code # lang => language code
# system => system shortname # system => system shortname
# category => package category
# package => name of the package # package => name of the package
# version => package version # version => package version
# man => name of the man page # man => name of the man page
# section => man page section # section => man page section
# #
# URL format: # URL format:
# /$fmt[.$shorthash][.$lang][/$system[/$category/$package[/$version]]]/$man[.$section] # /$fmt[.$shorthash][.$lang][/$system[[/$category]/$package[/$version]]]/$man[.$section]
# #
# Note that the URL format has some ambiguity: # Note that the URL format has some ambiguity:
# - $category may contain a slash, so a database lookup is required to # - $category (deprecated, only used for compatibility with old URLs) and
# $package may contain a slash, so a database lookup is required to
# disambiguate between URLs with [/$version] and those without. # disambiguate between URLs with [/$version] and those without.
# - $man may contain a dot, so a database lookup is required to disambiguate # - $man may contain a dot, so a database lookup is required to disambiguate
# between URLs with [.$section] and those without # between URLs with [.$section] and those without
@ -671,7 +675,7 @@ package ManUrl {
my($o)=@_; my($o)=@_;
"/$o->{fmt}".(defined $o->{shorthash} ? ".$o->{shorthash}" : '').(defined $o->{lang} ? ".$o->{lang}" : '') "/$o->{fmt}".(defined $o->{shorthash} ? ".$o->{shorthash}" : '').(defined $o->{lang} ? ".$o->{lang}" : '')
.(defined $o->{system} ? ("/$o->{system}" .(defined $o->{system} ? ("/$o->{system}"
.(defined $o->{category} ? ("/$o->{category}/$o->{package}" .(defined $o->{package} ? ("/$o->{package}"
.(defined $o->{version} ? "/$o->{version}" : '')) : '')) : '') .(defined $o->{version} ? "/$o->{version}" : '')) : '')) : '')
.'/'.$o->mansect .'/'.$o->mansect
}; };
@ -852,7 +856,7 @@ sub man_page {
'data-hasversions' => $hasversions?1:0, 'data-hasversions' => $hasversions?1:0,
sub { sub {
li_ sub { a_ href => $url->set(fmt => 'raw'), 'source' }; li_ sub { a_ href => $url->set(fmt => 'raw'), 'source' };
li_ sub { a_ href => $url->set(system => sysbyid->{$man->{system}}{short}, category => undef, shorthash => shorthash_to_hex $man->{shorthash}), 'permalink' }; li_ sub { a_ href => $url->set(system => sysbyid->{$man->{system}}{short}, package => undef, shorthash => shorthash_to_hex $man->{shorthash}), 'permalink' };
li_ sub { a_ href => "/loc/$content->{hash}", 'locations' }; li_ sub { a_ href => "/loc/$content->{hash}", 'locations' };
} }
}; };
@ -907,10 +911,7 @@ TUWF::get qr{/(?<fmt>man|txt|raw)(?:\.(?<shorthash>[a-fA-F0-9]{8}))?(?:\.(?<lang
push @where, sql 'system IN', $sysid; push @where, sql 'system IN', $sysid;
} }
# $path is now either: my($pkg, $ver, $redir) = length $path ? pkg_frompath sql_and(@where), $path : (undef,undef);
# 1. $category/$package
# 2. $cagegory/$package/$version
my($pkg, $ver) = length $path ? pkg_frompath sql_and(@where), $path : (undef,undef);
return tuwf->resNotFound if length $path && !$pkg; return tuwf->resNotFound if length $path && !$pkg;
push @where, sql 'p.id =', \$pkg->{id} if $pkg; push @where, sql 'p.id =', \$pkg->{id} if $pkg;
push @where, sql 'v.version =', \$ver if length $ver; push @where, sql 'v.version =', \$ver if length $ver;
@ -926,12 +927,12 @@ TUWF::get qr{/(?<fmt>man|txt|raw)(?:\.(?<shorthash>[a-fA-F0-9]{8}))?(?:\.(?<lang
shorthash => $shorthash, shorthash => $shorthash,
lang => $lang, lang => $lang,
system => length $system ? $system : undef, system => length $system ? $system : undef,
category => $pkg ? $pkg->{category} : undef,
package => $pkg ? $pkg->{name} : undef, package => $pkg ? $pkg->{name} : undef,
version => length $ver ? $ver : undef, version => length $ver ? $ver : undef,
man => length $section ? $man->{name} : $name, man => length $section ? $man->{name} : $name,
section => length $section ? $section : undef, section => length $section ? $section : undef,
); );
return tuwf->resRedirect($url, 'perm') if $redir;
man_page $man, $url; man_page $man, $url;
}; };
@ -950,7 +951,7 @@ TUWF::get qr{/pkg/([^/]+)} => sub {
my $where = sql 'NOT dead AND system =', \$sys->{id}, $f->{c} ne 'all' ? ('AND match_firstchar(name,', \$f->{c}, ')') : (); my $where = sql 'NOT dead AND system =', \$sys->{id}, $f->{c} ne 'all' ? ('AND match_firstchar(name,', \$f->{c}, ')') : ();
my $count = tuwf->dbVali('SELECT count(*) FROM', $packages_with_man, 'p WHERE', $where); my $count = tuwf->dbVali('SELECT count(*) FROM', $packages_with_man, 'p WHERE', $where);
my $pkg = tuwf->dbPagei({ results => 200, page => $f->{p} }, my $pkg = tuwf->dbPagei({ results => 200, page => $f->{p} },
'SELECT id, system, name, category, dead FROM', $packages_with_man, 'p WHERE', $where, 'ORDER BY name, category' 'SELECT id, system, name FROM', $packages_with_man, 'p WHERE', $where, 'ORDER BY name'
); );
framework_ title => $sys->{full}, mainclass => 'pkglist', sub { framework_ title => $sys->{full}, mainclass => 'pkglist', sub {
@ -970,8 +971,7 @@ TUWF::get qr{/pkg/([^/]+)} => sub {
paginate_ "/pkg/$short?c=$f->{c};p=", $count, 200, $f->{p}; paginate_ "/pkg/$short?c=$f->{c};p=", $count, 200, $f->{p};
ul_ sub { ul_ sub {
li_ sub { li_ sub {
a_ href => "/pkg/$short/$_->{category}/$_->{name}", $_->{name}; a_ href => "/pkg/$short/$_->{name}", $_->{name};
small_ ' '.$_->{category};
} for @$pkg; } for @$pkg;
}; };
paginate_ "/pkg/$short?c=$f->{c};p=", $count, 200, $f->{p}; paginate_ "/pkg/$short?c=$f->{c};p=", $count, 200, $f->{p};
@ -979,15 +979,16 @@ TUWF::get qr{/pkg/([^/]+)} => sub {
}; };
# Package info: /pkg/$system/$category/$name (/$version); $category may contain a slash, too. # Package info: /pkg/$system[/$category]/$name[/$version]; $category and $name may contain slashes, too.
TUWF::get qr{/pkg/([^/]+)/(.+)} => sub { TUWF::get qr{/pkg/([^/]+)/(.+)} => sub {
my ($short, $path) = tuwf->captures(1,2); my ($short, $path) = tuwf->captures(1,2);
my $sys = sysbyshort->{$short}; my $sys = sysbyshort->{$short};
return tuwf->resNotFound if !$sys; return tuwf->resNotFound if !$sys;
my($pkg, $ver) = pkg_frompath(sql('system =', \$sys->{id}), $path); my($pkg, $ver, $redir) = pkg_frompath(sql('system =', \$sys->{id}), $path);
return tuwf->resNotFound if !$pkg; return tuwf->resNotFound if !$pkg;
return tuwf->resRedirect("/pkg/$short/$pkg->{name}".($ver?"/$ver":''), 'perm') if $redir;
my $vers = tuwf->dbAlli(' my $vers = tuwf->dbAlli('
SELECT id, version, released SELECT id, version, released
@ -1023,8 +1024,8 @@ TUWF::get qr{/pkg/([^/]+)/(.+)} => sub {
# Latest version of this package determines last modification date of the page. # Latest version of this package determines last modification date of the page.
tuwf->resLastMod($vers->[0]{released}); tuwf->resLastMod($vers->[0]{released});
my $subtitle = " / $pkg->{category} / $pkg->{name}"; my $subtitle = " / $pkg->{name}";
my $pkgpath = "$sys->{short}/$pkg->{category}/$pkg->{name}"; my $pkgpath = "$sys->{short}/$pkg->{name}";
framework_ title => "$sys->{full}$subtitle $sel->{version}", mainclass => 'pkgpage', sub { framework_ title => "$sys->{full}$subtitle $sel->{version}", mainclass => 'pkgpage', sub {
h1_ sub { h1_ sub {
a_ href => "/pkg/$sys->{short}", $sys->{full}; a_ href => "/pkg/$sys->{short}", $sys->{full};
@ -1061,17 +1062,8 @@ TUWF::get qr{/pkg/([^/]+)/(.+)} => sub {
} }
}; };
# /browse/<pkg> has been moved to /pkg/ with the package category added to the path # /browse/<pkg> has been moved to /pkg/.
TUWF::get qr{/browse/([^/]+)} => sub { tuwf->resRedirect('/pkg/'.tuwf->capture(1), 'perm') }; TUWF::get qr{/browse/(.+)} => sub { tuwf->resRedirect('/pkg/'.tuwf->capture(1), 'perm') };
TUWF::get qr{/browse/([^/]+)/([^/]+)(?:/([^/]+))?} => sub {
my($sys, $name, $ver) = tuwf->captures(1,2,3);
$sys = sysbyshort->{$sys};
return tuwf->resNotFound if !$sys;
my $pkgs = tuwf->dbRowi('SELECT category FROM packages WHERE system =', \$sys->{id}, 'AND name =', \$name, 'LIMIT 1');
return tuwf->resNotFound if !defined $pkgs->{category};
tuwf->resRedirect("/pkg/$sys->{short}/$pkgs->{category}/$name".($ver ? "/$ver" :''), 'perm');
};
# Redirect for the system selection box, for visitors who have disabled JS. # Redirect for the system selection box, for visitors who have disabled JS.
TUWF::get qr{/sysredir/([^/]+)} => sub { tuwf->resRedirect('/man/'.(tuwf->reqGet('system')//'arch').'/'.tuwf->capture(1), 'temp') }; TUWF::get qr{/sysredir/([^/]+)} => sub { tuwf->resRedirect('/man/'.(tuwf->reqGet('system')//'arch').'/'.tuwf->capture(1), 'temp') };
@ -1092,7 +1084,7 @@ TUWF::get qr{/loc/([a-fA-F0-9]{40})}, sub {
my $maxpersys = 500; my $maxpersys = 500;
my $l = tuwf->dbAlli(' my $l = tuwf->dbAlli('
SELECT p.system, p.category, p.name AS package, v.version, f.filename, f.shorthash, m.name, m.section SELECT p.system, p.name AS package, v.version, f.filename, f.shorthash, m.name, m.section
FROM files f FROM files f
JOIN mans m ON m.id = f.man JOIN mans m ON m.id = f.man
JOIN package_versions v ON v.id = f.pkgver JOIN package_versions v ON v.id = f.pkgver
@ -1153,8 +1145,7 @@ TUWF::get qr{/loc/([a-fA-F0-9]{40})}, sub {
txt_ $sys->{release}; txt_ $sys->{release};
} if $sys->{release}; } if $sys->{release};
td_ sub { td_ sub {
a_ href => "/pkg/$sys->{short}/$_->{category}/$_->{package}/$_->{version}", $_->{package}.'-'.$_->{version}; a_ href => "/pkg/$sys->{short}/$_->{package}/$_->{version}", $_->{package}.'-'.$_->{version};
small_ ' '.$_->{category};
}; };
td_ $_->{filename}; td_ $_->{filename};
} for @{$sys{$sysname}}[0..min $maxpersys, $#{$sys{$sysname}}]; } for @{$sys{$sysname}}[0..min $maxpersys, $#{$sys{$sysname}}];
@ -1176,7 +1167,7 @@ TUWF::get '/json/tree.json' => sub {
return tuwf->resNotFound() if !$f->{hash} && !($f->{section} && $f->{name}); return tuwf->resNotFound() if !$f->{hash} && !($f->{section} && $f->{name});
my $l = tuwf->dbAlli(" my $l = tuwf->dbAlli("
SELECT p.system, p.category, p.name AS package, v.version, v.released, v.id AS verid, m.name, m.section, f.filename, f.shorthash, l.locale SELECT p.system, p.name AS package, v.version, v.released, v.id AS verid, m.name, m.section, f.filename, f.shorthash, l.locale
FROM files f FROM files f
JOIN locales l ON l.id = f.locale JOIN locales l ON l.id = f.locale
JOIN mans m ON m.id = f.man JOIN mans m ON m.id = f.man
@ -1212,14 +1203,14 @@ TUWF::get '/json/tree.json' => sub {
} }
if(!$pkg || $m->{package} ne $pkg->{name}) { if(!$pkg || $m->{package} ne $pkg->{name}) {
$pkg = { name => $m->{package}, i => $m->{category}, table => [] }; $pkg = { name => $m->{package}, table => [] };
$pkgver = undef; $pkgver = undef;
push @{$sysver->{childs}}, $pkg; push @{$sysver->{childs}}, $pkg;
} }
push @{$pkg->{table}}, [ push @{$pkg->{table}}, [
$pkgver && $pkgver eq $m->{version} ? {name=>''} : $pkgver && $pkgver eq $m->{version} ? {name=>''} :
{name => $m->{version}, href => "/pkg/".sysbyid->{$m->{system}}{short}."/$m->{category}/$m->{package}/$m->{version}"}, {name => $m->{version}, href => "/pkg/".sysbyid->{$m->{system}}{short}."/$m->{package}/$m->{version}"},
{ name => "$m->{name}($m->{section})", { name => "$m->{name}($m->{section})",
$f->{hash} || $cur == $m->{shorthash} ? () $f->{hash} || $cur == $m->{shorthash} ? ()
: (href => sprintf('/%s/%s', $m->{name}, shorthash_to_hex $m->{shorthash})) : (href => sprintf('/%s/%s', $m->{name}, shorthash_to_hex $m->{shorthash}))