vndb/manned
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

indexer: Improve Debian Contents file parsing performance by 5.2x

Browse source 
Further improvements can be gained by caching the results of
get_contents(), since the same Contents file is often parsed multiple
times in a single cron run. But this is already a significant
achievement.
This commit is contained in:
Yorhel 2016-11-26 16:55:47 +01:00
parent de28175cd3
commit eb15b6e2c7
1 changed files with 19 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

34
indexer/src/sys_deb.rs
View file

@ -18,24 +18,28 @@ fn get_contents(f: Option<open::Path>) -> Result<HashSet<String>> {
let rd = archive::Archive::open_raw(&mut fd)?;
let brd = BufReader::new(rd);
let mut pkgs = HashSet::new();
let mut filecnt = 0;
let mut filecnt = -1;
let mut mancnt = 0;
// Run the regex on bytes instead of strings, as paths aren't always UTF-8. This regex will
// not match non-UTF-8 paths.
let re = Regex::new(r"^(?u:([^\s].*?))\s+(?u:([^\s]+))\s*$").unwrap();
for line in brd.split(b'\n') {
re.captures(&line?).map(|cap| {
filecnt += 1;
let path = str::from_utf8(cap.at(1).unwrap()).unwrap();
if man::ismanpath(path) {
mancnt += 1;
pkgs.extend( str::from_utf8(cap.at(2).unwrap()).unwrap().split(',').map(|e| {
e.split('/').last().unwrap().to_string()
}) );
}
});
let line = line?;
let line = match str::from_utf8(&line) { Ok(x) => x, _ => continue };
if line.starts_with("FILE ") {
filecnt = 0;
continue;
} else if filecnt < 0 {
continue;
}
filecnt += 1;
let mut it = line.split(' ');
let pkg = it.next_back().unwrap();
let path = it.fold(String::new(), |acc, x| acc + " " + x);
if man::ismanpath(&path.trim()) {
mancnt += 1;
pkgs.extend( pkg.split(',').map(|e| {
e.split('/').last().unwrap().to_string()
}) );
}
}
debug!("Found {}/{} man files in {} relevant packages from {}", mancnt, filecnt, pkgs.len(), f.path);