vndb/manned
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

indexer: More crate updates + warning fixes + 2018 edition

Browse source
This commit is contained in:
Yorhel 2021-12-11 14:52:04 +01:00
parent c48feedc85
commit c9e81a8922
14 changed files with 193 additions and 499 deletions
Download patch file Download diff file Expand all files Collapse all files

19
indexer/src/archive.rs
View file

@ -1,6 +1,5 @@
use std::str;
use std::ptr;
use std::error::Error as ErrorTrait;
use std::io::{Result,Error,Read};
use std::ffi::{CStr,CString};
@ -23,7 +22,7 @@ use libarchive3_sys::ffi;
pub struct Archive<'a> {
a: *mut ffi::Struct_archive,
rd: &'a mut Read,
rd: &'a mut dyn Read,
buf: Vec<u8>,
err: Option<Error>,
eof: bool,
@ -61,7 +60,7 @@ unsafe extern "C" fn archive_read_cb(_: *mut ffi::Struct_archive, data: *mut c_v
match arch.rd.read(&mut arch.buf[..]) {
Ok(s) => s as ssize_t,
Err(e) => {
let desc = CString::new(e.description()).unwrap();
let desc = CString::new(e.to_string()).unwrap();
let fmt = CString::new("%s").unwrap();
ffi::archive_set_error(arch.a, e.raw_os_error().unwrap_or(0), fmt.as_ptr(), desc.as_ptr());
arch.err = Some(e);
@ -72,7 +71,7 @@ unsafe extern "C" fn archive_read_cb(_: *mut ffi::Struct_archive, data: *mut c_v
impl<'a> Archive<'a> {
fn new(rd: &mut Read, a: *mut ffi::Struct_archive) -> Result<Box<Archive>> {
fn new(rd: &mut dyn Read, a: *mut ffi::Struct_archive) -> Result<Box<Archive>> {
let bufsize = 64*1024;
let mut buf = Vec::with_capacity(bufsize);
unsafe { buf.set_len(bufsize) };
@ -131,17 +130,17 @@ impl<'a> Archive<'a> {
}
}
pub fn open_archive(rd: &mut Read) -> Result<Option<ArchiveEntry>> {
pub fn open_archive(rd: &mut dyn Read) -> Result<Option<ArchiveEntry>> {
let a = unsafe {
let a = ffi::archive_read_new();
ffi::archive_read_support_filter_all(a);
ffi::archive_read_support_format_all(a);
a
};
try!(Self::new(rd, a)).entry()
Self::new(rd, a)?.entry()
}
pub fn open_raw(rd: &mut Read) -> Result<RawEntry> {
pub fn open_raw(rd: &mut dyn Read) -> Result<RawEntry> {
let a = unsafe {
let a = ffi::archive_read_new();
ffi::archive_read_support_filter_all(a);
@ -149,7 +148,7 @@ impl<'a> Archive<'a> {
ffi::archive_read_support_format_empty(a);
a
};
let mut a = try!(Self::new(rd, a));
let mut a = Self::new(rd, a)?;
let mut e: *mut ffi::Struct_archive_entry = ptr::null_mut();
let res = unsafe { ffi::archive_read_next_header(a.a, &mut e) };
match res {
@ -282,10 +281,10 @@ pub fn walk<F>(ent: Option<ArchiveEntry>, mut cb: F) -> Result<()>
{
let mut ent = ent;
while let Some(mut e) = ent {
if !try!(cb(&mut e)) {
if !cb(&mut e)? {
break;
}
ent = try!(e.next());
ent = e.next()?;
}
Ok(())
}

12
indexer/src/archread.rs
View file

@ -1,7 +1,7 @@
use std::io::Result;
use std::collections::HashMap;
use archive::{walk,ArchiveEntry,FileType};
use crate::archive::{walk,ArchiveEntry,FileType};
/* I had hoped that reading man pages from an archive would just be a simple:
*
@ -121,7 +121,7 @@ impl FileList {
links: Vec::new(),
};
try!(walk(ent, |mut e| {
walk(ent, |mut e| {
let path = match e.path() {
Some(x) => x.to_string(),
None => { warn!("Invalid UTF-8 filename in archive"); return Ok(true) }
@ -144,7 +144,7 @@ impl FileList {
EntryType::Hardlink
} else if interest_cb(&path) {
let pathv = [&path as &str];
try!(file_cb(&pathv[..], &mut e));
file_cb(&pathv[..], &mut e)?;
EntryType::Handled
} else {
EntryType::Regular
@ -157,7 +157,7 @@ impl FileList {
let opath = ocomp.join("/");
if interest_cb(&opath) {
let pathv = [&opath as &str];
try!(file_cb(&pathv[..], &mut e));
file_cb(&pathv[..], &mut e)?;
*fl.seen.get_mut(&opath).unwrap() = EntryType::Handled;
}
}
@ -173,7 +173,7 @@ impl FileList {
fl.seen.insert(path, et);
Ok(true)
}));
})?;
Ok(fl)
}
@ -301,7 +301,7 @@ impl MissedFiles {
walk(ent, |mut e| {
if let Some(f) = e.path().and_then(|p| self.0.remove(p)) {
let v: Vec<&str> = f.iter().map(|x| x as &str).collect();
try!(file_cb(&v, &mut e))
file_cb(&v, &mut e)?
}
Ok(self.0.len() > 0)
})

11
indexer/src/main.rs
View file

@ -1,17 +1,6 @@
#[macro_use] extern crate log;
#[macro_use] extern crate lazy_static;
#[macro_use] extern crate clap;
extern crate env_logger;
extern crate regex;
extern crate libarchive3_sys;
extern crate libc;
extern crate ring;
extern crate encoding;
extern crate postgres;
extern crate ureq;
extern crate chrono;
extern crate quick_xml;
extern crate percent_encoding;
mod archive;
mod archread;

10
indexer/src/man.rs
View file

@ -8,7 +8,7 @@ use encoding::{all,EncodingRef};
use encoding::label::encoding_from_whatwg_label;
use ring::digest;
use archive::Archive;
use crate::archive::Archive;
// Anything larger than this just isn't a man page. I hope.
const MAX_MAN_SIZE: u64 = 20*1024*1024;
@ -195,10 +195,10 @@ fn codec_from_path(path: &str) -> Option<EncodingRef> {
// Decompresses / decodes a man page and returns its SHA-1 hash, encoding name, and UTF-8 contents.
pub fn decode(paths: &[&str], ent: &mut Read) -> io::Result<(digest::Digest,&'static str,String)> {
let mut decomp = try!(Archive::open_raw(ent)).take(MAX_MAN_SIZE+1);
pub fn decode(paths: &[&str], ent: &mut dyn Read) -> io::Result<(digest::Digest,&'static str,String)> {
let mut decomp = Archive::open_raw(ent)?.take(MAX_MAN_SIZE+1);
let mut data = Vec::new();
try!(decomp.read_to_end(&mut data));
decomp.read_to_end(&mut data)?;
if let Some(e) = validate(&data) {
return Err(io::Error::new(io::ErrorKind::InvalidData, e));
@ -301,6 +301,6 @@ fn test_decode_zh() {
assert_eq!(dig.as_ref(), &filehash[..]);
assert_eq!(enc, "gbk");
let utf8dig = digest::digest(&digest::SHA1, s.as_bytes());
let utf8dig = digest::digest(&digest::SHA1_FOR_LEGACY_USE_ONLY, s.as_bytes());
assert_eq!(utf8dig.as_ref(), &utf8hash[..]);
}

21
indexer/src/open.rs
View file

@ -19,21 +19,20 @@ pub struct Path<'a> {
}
fn fetch(url: &str) -> Result<Box<Read>> {
let res = try!(ureq::get(url)
fn fetch(url: &str) -> Result<Box<dyn Read>> {
let res = ureq::get(url)
.set("User-Agent", "Man page crawler (info@manned.org; https://manned.org/)")
.call()
.map_err(|e| Error::new(ErrorKind::Other, format!("Ureq: {}", e)))
);
.map_err(|e| Error::new(ErrorKind::Other, format!("Ureq: {}", e)))?;
if res.status() != 200 {
return Err(Error::new(ErrorKind::Other, format!("HTTP: {}", res.status()) ));
}
Ok(Box::new(res.into_reader()) as Box<Read>)
Ok(Box::new(res.into_reader()) as Box<dyn Read>)
}
fn file(path: &str) -> Result<Box<Read>> {
Ok(Box::new(try!(File::open(path))) as Box<Read>)
fn file(path: &str) -> Result<Box<dyn Read>> {
Ok(Box::new(File::open(path)?) as Box<dyn Read>)
}
@ -51,7 +50,7 @@ pub fn clear_cache() -> Result<()> {
impl<'a> Path<'a> {
pub fn open(&self) -> Result<Box<Read>> {
pub fn open(&self) -> Result<Box<dyn Read>> {
if self.path.starts_with("http://") || self.path.starts_with("https://") {
if self.cache {
let hash = digest::digest(&digest::SHA256, self.path.as_bytes())
@ -63,9 +62,9 @@ impl<'a> Path<'a> {
return Ok(f);
}
{
let mut rd = try!(fetch(self.path));
let mut wr = try!(File::create(&cfn));
try!(copy(&mut rd, &mut wr));
let mut rd = fetch(self.path)?;
let mut wr = File::create(&cfn)?;
copy(&mut rd, &mut wr)?;
}
file(&cfn)

10
indexer/src/pkg.rs
View file

@ -3,10 +3,10 @@ use std::io::{Error,ErrorKind,Read};
use postgres;
use chrono::NaiveDateTime;
use open;
use archread;
use man;
use archive::{Format,Archive,ArchiveEntry};
use crate::open;
use crate::archread;
use crate::man;
use crate::archive::{Format,Archive,ArchiveEntry};
pub static mut DRY_RUN: bool = false;
@ -107,7 +107,7 @@ fn insert_man_row<T: postgres::GenericClient>(tr: &mut T, verid: i32, path: &str
}
fn insert_man<T: postgres::GenericClient>(tr: &mut T, verid: i32, paths: &[&str], ent: &mut Read) {
fn insert_man<T: postgres::GenericClient>(tr: &mut T, verid: i32, paths: &[&str], ent: &mut dyn Read) {
let (dig, enc, mut cont) = match man::decode(paths, ent) {
Err(e) => { error!("Error decoding {}: {}", paths[0], e); return },
Ok(x) => x,

17
indexer/src/sys_arch.rs
View file

@ -4,10 +4,10 @@ use regex::Regex;
use chrono::NaiveDateTime;
use postgres;
use archive;
use open;
use man;
use pkg;
use crate::archive;
use crate::open;
use crate::man;
use crate::pkg;
struct Meta {
@ -22,8 +22,7 @@ struct Meta {
fn read_files<T: Read>(lst: T) -> Result<bool> {
let rd = BufReader::new(lst);
for line in rd.lines() {
let line = try!(line);
if man::ismanpath(&line) {
if man::ismanpath(&line?) {
return Ok(true);
}
}
@ -33,7 +32,7 @@ fn read_files<T: Read>(lst: T) -> Result<bool> {
fn read_desc(rd: &mut archive::ArchiveEntry) -> Result<Option<Meta>> {
let mut data = String::new();
try!(rd.take(64*1024).read_to_string(&mut data));
rd.take(64*1024).read_to_string(&mut data)?;
let path = rd.path().unwrap();
lazy_static! {
@ -97,9 +96,9 @@ pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, mirror: &str, repo
hasman = false;
meta = None;
} else if x.path().unwrap().ends_with("/files") {
hasman = try!(read_files(x));
hasman = read_files(x)?;
} else if x.path().unwrap().ends_with("/desc") {
meta = try!(read_desc(x));
meta = read_desc(x)?;
}
if hasman && meta.is_some() {

8
indexer/src/sys_deb.rs
View file

@ -5,10 +5,10 @@ use postgres;
use regex;
use regex::bytes::Regex;
use man;
use pkg;
use open;
use archive;
use crate::man;
use crate::pkg;
use crate::open;
use crate::archive;
// Reference: https://wiki.debian.org/RepositoryFormat

4
indexer/src/sys_freebsd1.rs
View file

@ -3,8 +3,8 @@ use std::io::Result;
use regex::Regex;
use postgres;
use open;
use pkg;
use crate::open;
use crate::pkg;
// Sync a FreeBSD <= 9.2 package respository.

6
indexer/src/sys_freebsd2.rs
View file

@ -3,9 +3,9 @@ use regex::bytes::Regex;
use std::str;
use postgres;
use open;
use pkg;
use archive::{Archive,ArchiveEntry};
use crate::open;
use crate::pkg;
use crate::archive::{Archive,ArchiveEntry};
fn getpkgsite(mut ent: Option<ArchiveEntry>) -> Result<ArchiveEntry> {

43
indexer/src/sys_rpm.rs
View file

@ -2,41 +2,25 @@ use std::collections::HashSet;
use std::io::BufReader;
use std::str::FromStr;
use std::error::Error;
use std::fmt;
use chrono::NaiveDateTime;
use postgres;
use quick_xml as xml;
use quick_xml::events::Event;
use archive;
use open;
use pkg;
use man;
use crate::archive;
use crate::open;
use crate::pkg;
use crate::man;
// Ugh, quick-xml's Error type does not implement Error.
#[derive(Debug)]
struct XmlError(String);
impl fmt::Display for XmlError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.0) }
}
impl Error for XmlError {
fn description(&self) -> &str { self.0.as_ref() }
}
fn to_err(e: xml::Error) -> XmlError {
XmlError(format!("{}", e))
}
fn xml_getattr(e: &xml::events::BytesStart, attr: &str) -> Result<String,Box<Error>> {
fn xml_getattr(e: &xml::events::BytesStart, attr: &str) -> Result<String,Box<dyn Error>> {
for kv in e.attributes().with_checks(false) {
let kv = kv.map_err(to_err)?;
let kv = kv?;
if kv.key == attr.as_bytes() {
return Ok(String::from_utf8(kv.value.into_owned())?);
}
}
Err(Box::new(XmlError(format!("Attribute '{}' not found", attr))))
Err(Box::new(xml::Error::UnexpectedToken(format!("Attribute '{}' not found", attr))))
}
@ -53,7 +37,7 @@ struct PkgInfo {
// Shared function to read primary.xml.gz and filelists.xml.gz. Runs the callback for each package
// with the info that was found.
fn readpkgs<F>(url: String, mut cb: F) -> Result<(),Box<Error>>
fn readpkgs<F>(url: String, mut cb: F) -> Result<(),Box<dyn Error>>
where F: FnMut(PkgInfo)
{
debug!("Reading {}", url);
@ -74,8 +58,7 @@ fn readpkgs<F>(url: String, mut cb: F) -> Result<(),Box<Error>>
loop {
buf.clear();
let event = xml.read_event(&mut buf);
let event = event.map_err(to_err)?;
let event = xml.read_event(&mut buf)?;
match event {
@ -97,7 +80,7 @@ fn readpkgs<F>(url: String, mut cb: F) -> Result<(),Box<Error>>
Event::Text(e) =>
if savestr {
saved = Some(e.unescape_and_decode(&xml).map_err(to_err)?);
saved = Some(e.unescape_and_decode(&xml)?);
savestr = false
},
@ -126,7 +109,7 @@ fn readpkgs<F>(url: String, mut cb: F) -> Result<(),Box<Error>>
// Reads repomd.xml and returns the path to the primary.xml.gz and filelists.xml.gz
fn repomd(url: String) -> Result<(String,String),Box<Error>> {
fn repomd(url: String) -> Result<(String,String),Box<dyn Error>> {
debug!("Reading {}", url);
let mut fd = open::Path{path: &url, cache: true, canbelocal: false}.open()?;
let mut xml = xml::Reader::from_reader(
@ -143,7 +126,7 @@ fn repomd(url: String) -> Result<(String,String),Box<Error>> {
loop {
buf.clear();
let event = xml.read_event(&mut buf).map_err(to_err)?;
let event = xml.read_event(&mut buf)?;
match event {
Event::Start(ref e) |
Event::Empty(ref e) => {
@ -173,7 +156,7 @@ fn repomd(url: String) -> Result<(String,String),Box<Error>> {
}
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, cat: &str, mirror: &str) -> Result<(),Box<Error>> {
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, cat: &str, mirror: &str) -> Result<(),Box<dyn Error>> {
let(primary, filelists) = repomd(format!("{}repodata/repomd.xml", mirror))?;
let mut pkgswithman = HashSet::new();

4
indexer/src/sys_rpmdir.rs
View file

@ -2,8 +2,8 @@ use std::io::Result;
use regex::Regex;
use postgres;
use open;
use pkg;
use crate::open;
use crate::pkg;
pub fn sync<T: postgres::GenericClient>(pg: &mut T, sys: i32, cat: &str, mirror: &str) -> Result<()> {
let pkgs : Vec<String> = open::Path{path: mirror, cache: true, canbelocal: false}