vndb/manned
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make the Rust garbage compile again

Browse source
This commit is contained in:
Yorhel 2021-12-05 18:58:58 +01:00
parent b8dca570a0
commit 4588e67b64
12 changed files with 1348 additions and 1078 deletions
Download patch file Download diff file Expand all files Collapse all files

2239
indexer/Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

2
indexer/Cargo.toml
View file

@ -12,7 +12,7 @@ libc = "0.2.39"
libarchive3-sys = "0.1.2"
encoding = { git = "https://github.com/lifthrasiir/rust-encoding", features = ["no-optimized-legacy-encoding"] }
ring = "0.14.6"
postgres = "0.15.2"
postgres = "0.17.5"
clap = "2.31.2"
reqwest = "0.9.17"
url = "1.7.0"

39
indexer/src/main.rs
View file

@ -27,12 +27,12 @@ mod sys_rpm;
// Convenience function to get a system id by short-name. Panics if the system doesn't exist.
fn sysbyshort(conn: &postgres::GenericConnection, short: &str) -> i32 {
let r = conn.query("SELECT id FROM systems WHERE short = $1", &[&short]).unwrap();
if r.is_empty() {
fn sysbyshort<T>(conn: &mut T, short: &str) -> i32 where T: postgres::GenericClient {
if let Some(r) = conn.query_opt("SELECT id FROM systems WHERE short = $1", &[&short]).unwrap() {
return r.get(0);
} else {
panic!("Invalid system: {}", short);
}
r.get(0).get(0)
}
@ -112,7 +112,7 @@ fn main() {
Ok(x) => x,
Err(_) => { error!("MANNED_PG not set."); return }
};
let db = match postgres::Connection::connect(&dbhost[..], postgres::TlsMode::None) {
let mut db = match postgres::Client::connect(&dbhost[..], postgres::tls::NoTls) {
Ok(x) => x,
Err(x) => { error!("Can't connect to postgres: {}", x); return },
};
@ -125,9 +125,10 @@ fn main() {
"max" => pkg::Date::Max,
s => pkg::Date::Known(s),
};
pkg::pkg(&db, pkg::PkgOpt {
let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap());
pkg::pkg(&mut db, pkg::PkgOpt {
force: matches.is_present("force"),
sys: sysbyshort(&db, matches.value_of("sys").unwrap()),
sys: sys,
cat: matches.value_of("cat").unwrap(),
pkg: matches.value_of("pkg").unwrap(),
ver: matches.value_of("ver").unwrap(),
@ -138,16 +139,16 @@ fn main() {
}
if let Some(matches) = arg.subcommand_matches("arch") {
sys_arch::sync(&db,
sysbyshort(&db, matches.value_of("sys").unwrap()),
let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap());
sys_arch::sync(&mut db, sys,
matches.value_of("mirror").unwrap(),
matches.value_of("repo").unwrap()
);
}
if let Some(matches) = arg.subcommand_matches("deb") {
sys_deb::sync(&db,
sysbyshort(&db, matches.value_of("sys").unwrap()),
let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap());
sys_deb::sync(&mut db, sys,
matches.value_of("mirror").unwrap(),
matches.value_of("contents").map(|e| { open::Path{ path: e, cache: true, canbelocal: true} }),
open::Path{ path: matches.value_of("packages").unwrap(), cache: true, canbelocal: true},
@ -155,31 +156,31 @@ fn main() {
}
if let Some(matches) = arg.subcommand_matches("freebsd1") {
sys_freebsd1::sync(&db,
sysbyshort(&db, matches.value_of("sys").unwrap()),
let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap());
sys_freebsd1::sync(&mut db, sys,
matches.value_of("arch").unwrap(),
matches.value_of("mirror").unwrap()
).unwrap_or_else(|e| error!("{}", e));
}
if let Some(matches) = arg.subcommand_matches("freebsd2") {
sys_freebsd2::sync(&db,
sysbyshort(&db, matches.value_of("sys").unwrap()),
let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap());
sys_freebsd2::sync(&mut db, sys,
matches.value_of("mirror").unwrap()
).unwrap_or_else(|e| error!("{}", e));
}
if let Some(matches) = arg.subcommand_matches("rpmdir") {
sys_rpmdir::sync(&db,
sysbyshort(&db, matches.value_of("sys").unwrap()),
let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap());
sys_rpmdir::sync(&mut db, sys,
matches.value_of("cat").unwrap(),
matches.value_of("mirror").unwrap()
).unwrap_or_else(|e| error!("{}", e));
}
if let Some(matches) = arg.subcommand_matches("rpm") {
sys_rpm::sync(&db,
sysbyshort(&db, matches.value_of("sys").unwrap()),
let sys = sysbyshort(&mut db, matches.value_of("sys").unwrap());
sys_rpm::sync(&mut db, sys,
matches.value_of("cat").unwrap(),
matches.value_of("mirror").unwrap()
).unwrap_or_else(|e| error!("{}", e));

70
indexer/src/pkg.rs
View file

@ -48,23 +48,23 @@ pub struct PkgOpt<'a> {
}
fn insert_pkg(tr: &postgres::transaction::Transaction, opt: &PkgOpt) -> Option<i32> {
fn insert_pkg(tr: &mut postgres::Transaction, opt: &PkgOpt) -> Option<i32> {
let pkginfo = format!("sys {} / {} / {} - {} @ {:?} @ {}", opt.sys, opt.cat, opt.pkg, opt.ver, opt.date, opt.file.path);
// The ON CONFLICT .. DO UPDATE is used instead of DO NOTHING because in that case the
// RETURNING clause wouldn't give us a package id.
let q = "INSERT INTO packages (system, category, name) VALUES($1, $2, $3)
ON CONFLICT ON CONSTRAINT packages_system_name_category_key DO UPDATE SET name=$3 RETURNING id";
let pkgid: i32 = match tr.query(q, &[&opt.sys, &opt.cat, &opt.pkg]) {
let pkgid: i32 = match tr.query_one(q, &[&opt.sys, &opt.cat, &opt.pkg]) {
Err(e) => {
error!("Can't insert package in database: {}", e);
return None;
},
Ok(r) => r.get(0).get(0),
Ok(r) => r.get(0),
};
let q = "SELECT id FROM package_versions WHERE package = $1 AND version = $2";
let res = tr.query(q, &[&pkgid, &opt.ver]).unwrap();
let res = tr.query_opt(q, &[&pkgid, &opt.ver]).unwrap();
let verid : i32;
@ -73,27 +73,27 @@ fn insert_pkg(tr: &postgres::transaction::Transaction, opt: &PkgOpt) -> Option<i
_ => "1980-01-01", // Placeholder
};
if res.is_empty() {
if res.is_none() {
let q = "INSERT INTO package_versions (package, version, released, arch) VALUES($1, $2, $3::text::date, $4) RETURNING id";
verid = tr.query(q, &[&pkgid, &opt.ver, &date, &opt.arch]).unwrap().get(0).get(0);
verid = tr.query_one(q, &[&pkgid, &opt.ver, &date, &opt.arch]).unwrap().get(0);
info!("New package pkgid {} verid {}, {}", pkgid, verid, pkginfo);
Some(verid)
} else if opt.force {
// XXX: Should we update released & arch here?
verid = res.get(0).get(0);
verid = res?.get(0);
info!("Overwriting package pkgid {} verid {}, {}", pkgid, verid, pkginfo);
tr.query("DELETE FROM man WHERE package = $1", &[&verid]).unwrap();
Some(verid)
} else {
debug!("Package already in database, pkgid {} verid {}, {}", pkgid, res.get(0).get::<usize,i32>(0), pkginfo);
debug!("Package already in database, pkgid {} verid {}, {}", pkgid, res?.get::<usize,i32>(0), pkginfo);
None
}
}
fn insert_man_row(tr: &postgres::GenericConnection, verid: i32, path: &str, enc: &str, hash: &[u8]) {
fn insert_man_row<T: postgres::GenericClient>(tr: &mut T, verid: i32, path: &str, enc: &str, hash: &[u8]) {
let (name, sect, locale) = man::parse_path(path).unwrap();
let locale = if locale == "" { None } else { Some(locale) };
if let Err(e) = tr.execute(
@ -107,7 +107,7 @@ fn insert_man_row(tr: &postgres::GenericConnection, verid: i32, path: &str, enc:
}
fn insert_man(tr: &postgres::GenericConnection, verid: i32, paths: &[&str], ent: &mut Read) {
fn insert_man<T: postgres::GenericClient>(tr: &mut T, verid: i32, paths: &[&str], ent: &mut Read) {
let (dig, enc, mut cont) = match man::decode(paths, ent) {
Err(e) => { error!("Error decoding {}: {}", paths[0], e); return },
Ok(x) => x,
@ -131,14 +131,16 @@ fn insert_man(tr: &postgres::GenericConnection, verid: i32, paths: &[&str], ent:
}
fn insert_link(tr: &postgres::GenericConnection, verid: i32, src: &str, dest: &str) {
let res = tr.query("SELECT hash, encoding FROM man WHERE package = $1 AND filename = '/'||$2", &[&verid, &dest]).unwrap();
if res.is_empty() { /* Can happen if man::decode() failed previously. */
error!("Link to unindexed man page: {} -> {}", src, dest);
return;
}
let hash: Vec<u8> = res.get(0).get(0);
let enc: String = res.get(0).get(1);
fn insert_link<T>(tr: &mut T, verid: i32, src: &str, dest: &str) where T: postgres::GenericClient {
let res = match tr.query_opt("SELECT hash, encoding FROM man WHERE package = $1 AND filename = '/'||$2", &[&verid, &dest]).unwrap() {
None => { /* Can happen if man::decode() failed previously. */
error!("Link to unindexed man page: {} -> {}", src, dest);
return;
},
Some(x) => x
};
let hash: Vec<u8> = res.get(0);
let enc: String = res.get(1);
insert_man_row(tr, verid, src, &enc, &hash);
info!("Inserted man link: {} -> {}", src, dest);
}
@ -173,19 +175,20 @@ fn with_pkg<F,T>(opt: &mut PkgOpt, cb: F) -> std::io::Result<T>
}
fn index_pkg(tr: &postgres::GenericConnection, mut opt: PkgOpt, verid: i32) -> std::io::Result<()> {
let indexfunc = |paths: &[&str], ent: &mut ArchiveEntry| {
insert_man(tr, verid, paths, ent);
Ok(()) /* Don't propagate errors, continue handling other man pages */
};
fn index_pkg<T: postgres::GenericClient>(tr: &mut T, mut opt: PkgOpt, verid: i32) -> std::io::Result<()> {
let missed = with_pkg(&mut opt, |e, opt| {
archread::FileList::read(e, man::ismanpath, |ent| opt.date.update(ent), &indexfunc)
archread::FileList::read(e, man::ismanpath, |ent| opt.date.update(ent), |paths, ent| {
insert_man(tr, verid, paths, ent);
Ok(())
})
})?.links(|src, dest| { insert_link(tr, verid, src, dest) });
if let Some(missed) = missed {
warn!("Some links were missed, reading package again");
with_pkg(&mut opt, |e, _| { missed.read(e, indexfunc) })?
with_pkg(&mut opt, |e, _| { missed.read(e, |paths, ent| {
insert_man(tr, verid, paths, ent);
Ok(())
}) })?
}
match opt.date {
@ -201,21 +204,20 @@ fn index_pkg(tr: &postgres::GenericConnection, mut opt: PkgOpt, verid: i32) -> s
}
pub fn pkg(conn: &postgres::GenericConnection, opt: PkgOpt) {
let tr = conn.transaction().unwrap();
tr.set_rollback();
pub fn pkg<T>(conn: &mut T, opt: PkgOpt) where T: postgres::GenericClient {
let mut tr = conn.transaction().unwrap();
let verid = match insert_pkg(&tr, &opt) { Some(x) => x, None => return };
let verid = match insert_pkg(&mut tr, &opt) { Some(x) => x, None => return };
if unsafe { DRY_RUN } {
return;
}
match index_pkg(&tr, opt, verid) {
Err(e) => error!("Error reading package: {}", e),
Ok(_) => tr.set_commit()
if let Err(e) = index_pkg(&mut tr, opt, verid) {
error!("Error reading package: {}", e);
return;
}
if let Err(e) = tr.finish() {
if let Err(e) = tr.commit() {
error!("Error finishing transaction: {}", e);
}
}

2
indexer/src/sys_arch.rs
View file

@ -75,7 +75,7 @@ fn read_desc(rd: &mut archive::ArchiveEntry) -> Result<Option<Meta>> {
}
pub fn sync(pg: &postgres::GenericConnection, sys: i32, mirror: &str, repo: &str) {
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str, repo: &str) {
info!("Reading packages from {} {}", mirror, repo);
let path = format!("{}/{}/os/x86_64/{1:}.files.tar.gz", mirror, repo);

4
indexer/src/sys_deb.rs
View file

@ -56,7 +56,7 @@ struct Pkg {
}
fn handlepkg(pg: &postgres::GenericConnection, sys: i32, mirror: &str, manpkgs: &HashSet<String>, pkg: &Pkg) {
fn handlepkg<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str, manpkgs: &HashSet<String>, pkg: &Pkg) {
let name = match pkg.name { Some(ref x) => x, None => return };
if manpkgs.len() > 0 && !manpkgs.contains(name) {
return
@ -94,7 +94,7 @@ fn handlepkg(pg: &postgres::GenericConnection, sys: i32, mirror: &str, manpkgs:
}
pub fn sync(pg: &postgres::GenericConnection, sys: i32, mirror: &str, contents: Option<open::Path>, packages: open::Path) {
pub fn sync<T: postgres::GenericClient >(pg: &mut T, sys: i32, mirror: &str, contents: Option<open::Path>, packages: open::Path) {
let manpkgs = match get_contents(contents) {
Err(e) => { error!("Can't read {}: {}", contents.unwrap().path, e); return },
Ok(x) => x,

2
indexer/src/sys_freebsd1.rs
View file

@ -26,7 +26,7 @@ use pkg;
// 'pear-PHPUnit version 1.3.3', because there is a 'pear' package in 'Latest' but no
// 'pear-PHPUnit'. This is handled with a static list of package names to add to the 'pkgs' list,
// see EXTRA_PKGS below.
pub fn sync(pg: &postgres::GenericConnection, sys: i32, arch: &str, mirror: &str) -> Result<()> {
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, arch: &str, mirror: &str) -> Result<()> {
let path = format!("{}Latest/", mirror);
let mut pkgs : Vec<String> = open::Path{path: &path, cache: true, canbelocal: false}
.dirlist()?.into_iter()

2
indexer/src/sys_freebsd2.rs
View file

@ -19,7 +19,7 @@ fn getpkgsite(mut ent: Option<ArchiveEntry>) -> Result<ArchiveEntry> {
}
pub fn sync(pg: &postgres::GenericConnection, sys: i32, mirror: &str) -> Result<()> {
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str) -> Result<()> {
let path = format!("{}packagesite.txz", mirror);
let mut rd = open::Path{path: &path, cache: true, canbelocal: false}.open()?;

2
indexer/src/sys_rpm.rs
View file

@ -173,7 +173,7 @@ fn repomd(url: String) -> Result<(String,String),Box<Error>> {
}
pub fn sync(pg: &postgres::GenericConnection, sys: i32, cat: &str, mirror: &str) -> Result<(),Box<Error>> {
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, cat: &str, mirror: &str) -> Result<(),Box<Error>> {
let(primary, filelists) = repomd(format!("{}repodata/repomd.xml", mirror))?;
let mut pkgswithman = HashSet::new();

2
indexer/src/sys_rpmdir.rs
View file

@ -5,7 +5,7 @@ use postgres;
use open;
use pkg;
pub fn sync(pg: &postgres::GenericConnection, sys: i32, cat: &str, mirror: &str) -> Result<()> {
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, cat: &str, mirror: &str) -> Result<()> {
let pkgs : Vec<String> = open::Path{path: mirror, cache: true, canbelocal: false}
.dirlist()?.into_iter()
.filter_map(|(n,d)| if d { None } else { Some(n) })