WIP: Rewritten man page indexer in Rust
Currently just figuring out how to read archives. Turns out to not be as simple as I had expected.
This commit is contained in:
Yorhel
parent
965aa9a2f6
commit
022e9acc4f
7 changed files with 669 additions and 0 deletions
143
indexer/Cargo.lock
generated
Normal file
143
indexer/Cargo.lock
generated
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
[root]
|
||||
name = "indexer"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libarchive 0.1.1 (git+https://github.com/17dec/libarchive-rust)",
|
||||
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 0.1.77 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 0.1.77 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kernel32-sys"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libarchive"
|
||||
version = "0.1.1"
|
||||
source = "git+https://github.com/17dec/libarchive-rust#3f723cf0064561f21f0cebbd534a75076e6dbcaa"
|
||||
dependencies = [
|
||||
"libarchive3-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libarchive3-sys"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "0.1.77"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"aho-corasick 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"thread_local 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "thread-id"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8-ranges"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-build"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[metadata]
|
||||
"checksum aho-corasick 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ca972c2ea5f742bfce5687b9aef75506a764f61d37f8f649047846a9686ddb66"
|
||||
"checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f"
|
||||
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
|
||||
"checksum libarchive 0.1.1 (git+https://github.com/17dec/libarchive-rust)" = "<none>"
|
||||
"checksum libarchive3-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3cd3beae8f59a4c7a806523269b5392037577c150446e88d684dfa6de6031ca7"
|
||||
"checksum libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "044d1360593a78f5c8e5e710beccdc24ab71d1f01bc19a29bcacdba22e8475d8"
|
||||
"checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054"
|
||||
"checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20"
|
||||
"checksum pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8cee804ecc7eaf201a4a207241472cc870e825206f6c031e3ee2a72fa425f2fa"
|
||||
"checksum regex 0.1.77 (registry+https://github.com/rust-lang/crates.io-index)" = "64b03446c466d35b42f2a8b203c8e03ed8b91c0f17b56e1f84f7210a257aa665"
|
||||
"checksum regex-syntax 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "48f0573bcee95a48da786f8823465b5f2a1fae288a55407aca991e5b3e0eae11"
|
||||
"checksum thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9539db560102d1cef46b8b78ce737ff0bb64e7e18d35b2a5688f7d097d0ff03"
|
||||
"checksum thread_local 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8576dbbfcaef9641452d5cf0df9b0e7eeab7694956dd33bb61515fb8f18cfdd5"
|
||||
"checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f"
|
||||
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
|
||||
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
|
||||
10
indexer/Cargo.toml
Normal file
10
indexer/Cargo.toml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
[package]
|
||||
name = "indexer"
|
||||
version = "0.1.0"
|
||||
authors = ["Yorhel <git@yorhel.nl>"]
|
||||
|
||||
[dependencies]
|
||||
regex = "0.1.77"
|
||||
log = "0.3.6"
|
||||
env_logger = "0.3.5"
|
||||
libarchive = { git = "https://github.com/17dec/libarchive-rust" }
|
||||
368
indexer/src/archive.rs
Normal file
368
indexer/src/archive.rs
Normal file
|
|
@ -0,0 +1,368 @@
|
|||
use std::path::Path;
|
||||
use std::collections::HashMap;
|
||||
use libarchive::reader::Reader as ArchiveReader;
|
||||
use libarchive::reader::{FileReader,Builder};
|
||||
use libarchive::archive::{Entry,FileType,ReadFormat,ReadFilter};
|
||||
use libarchive::error::ArchiveResult;
|
||||
|
||||
|
||||
pub fn open_file<T: AsRef<Path>>(path: T) -> ArchiveResult<FileReader> {
|
||||
let mut builder = Builder::new();
|
||||
try!(builder.support_format(ReadFormat::All));
|
||||
try!(builder.support_filter(ReadFilter::All));
|
||||
builder.open_file(path)
|
||||
}
|
||||
|
||||
|
||||
#[derive(Clone,Debug,PartialEq,Eq)]
|
||||
pub enum EntryType {
|
||||
// Regular file that has been handled/indexed
|
||||
Handled,
|
||||
// Regular file that hasn't been handled because the caller wasn't interested in it. Could
|
||||
// still be an interesting file if it is referenced from an interesting path.
|
||||
Regular,
|
||||
// Link to another file (interesting or not is irrelevant)
|
||||
Link(String),
|
||||
// Directory; need this information when resolving links
|
||||
Directory,
|
||||
// Something that couldn't be a an interesting file (chardev/socket/etc); If any link resolves
|
||||
// to this we know we're done.
|
||||
Other,
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* I had hoped that reading man pages from an archive would just be a simple:
|
||||
*
|
||||
* 1. Walk through all files in the archive in a streaming fashion
|
||||
* 2. Parse/index man pages
|
||||
*
|
||||
* But alas, it was not to be. Symlinks and hardlinks have ruined it. Now we have to...
|
||||
*
|
||||
* 1. Walk through all entries in the archive in a streaming fashion
|
||||
* 2. Parse/index regular file man pages
|
||||
* 3. Keep track of all paths in the archive
|
||||
* 4. Use the result of step (3) to resolve symlinks/hardlinks to their actual file
|
||||
* 5. Read the entire damn archive again if one of the links resolved to a file that was not
|
||||
* recognized as a man page in step (2). Luckily, this isn't very common.
|
||||
*
|
||||
* And this doesn't even cover the problem of duplicate entries in a tar, which is also quite
|
||||
* annoying to handle.
|
||||
*
|
||||
* What annoys me the most about all of this is that it's not possible to stream an archive from
|
||||
* the network and read/index the entire thing in a single step. Now we have to buffer packages to
|
||||
* disk in order to be able to read the archive a second time.
|
||||
*
|
||||
* (Note that it is possible to resolve links while walking through the entries, which will allow
|
||||
* us to match files found later in the archive against links found earlier, thus potentially
|
||||
* saving the need to read the archive a second time. This is merely a performance improvement for
|
||||
* an uncommon case, and it certainly won't simplify the code)
|
||||
*
|
||||
* (Note that it's also possible to just flush all files <10MB* to disk to completely avoid the
|
||||
* need for a second archive read, but that's going to significantly slow down the common case in
|
||||
* order to handle a rare case. It's possible to further optimize this using some heuristics to
|
||||
* determine whether a file is potentially a man page, but that's both complex and may not even
|
||||
* save much)
|
||||
*
|
||||
* (* So apparently some man pages are close to 10MB...)
|
||||
*/
|
||||
pub struct Reader {
|
||||
// List of seen files. This is used to resolve links
|
||||
seen: HashMap<String, EntryType>,
|
||||
// List of interesting links
|
||||
links: Vec<String>,
|
||||
// List of files we have to read in a second walk through the archive
|
||||
missedfiles: HashMap<String, Vec<String>>,
|
||||
}
|
||||
|
||||
|
||||
// Generalized API:
|
||||
// 1. Read once
|
||||
// reader.read(file, interest_cb, file_cb) -> Error
|
||||
// file: A libarchive::Reader
|
||||
// interest_cb(path) -> bool
|
||||
// Called on every file/link name, should return whether it's a file the caller is interested
|
||||
// in. (e.g. parse_path(), but also +DESC and other metadata).
|
||||
// file_cb(path, reader, entry) -> Error
|
||||
// Called on every interesting (actual) file, given the (normalized?) path, the
|
||||
// libarchive::Reader and a ReaderEntry
|
||||
//
|
||||
// 2. Read links
|
||||
// reader.links(link_cb) -> Error
|
||||
// link_cb(path, dest) -> Error
|
||||
// Called on every link which has as 'dest' a file path that has already been given to
|
||||
// file_cb() before.
|
||||
//
|
||||
// 3. (Optionally) read a second time
|
||||
// if reader.need_reread() {
|
||||
// reader.reread(file, file_cb)
|
||||
// }
|
||||
impl Reader {
|
||||
pub fn new() -> Reader {
|
||||
Reader {
|
||||
seen: HashMap::new(),
|
||||
links: Vec::new(),
|
||||
missedfiles: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
// Convenience function to read the path/type/link from the next header.
|
||||
fn read_header(rd: &mut ArchiveReader) -> Option<(String, EntryType)> {
|
||||
let ent = match rd.next_header() {
|
||||
Some(x) => x,
|
||||
None => return None,
|
||||
};
|
||||
let path = ent.pathname().trim_left_matches('/').trim_left_matches("./").trim_right_matches('/').to_string();
|
||||
|
||||
// Hard links are apparently relative to the root of the archive.
|
||||
let link = ent.hardlink().map(|x| format!("/{}", x))
|
||||
.or(ent.symlink().map(str::to_string));
|
||||
|
||||
let(fts, ret) = match ent.filetype() {
|
||||
FileType::BlockDevice => ("blk", EntryType::Other),
|
||||
FileType::SymbolicLink => ("sym", match link { Some(l) => EntryType::Link(l), _ => EntryType::Other }),
|
||||
FileType::Socket => ("sck", EntryType::Other),
|
||||
FileType::CharacterDevice => ("chr", EntryType::Other),
|
||||
FileType::Directory => ("dir", EntryType::Directory),
|
||||
FileType::NamedPipe => ("fif", EntryType::Other),
|
||||
FileType::Mount => ("mnt", EntryType::Other),
|
||||
FileType::RegularFile => ("reg", EntryType::Regular),
|
||||
FileType::Unknown => ("unk", match link { Some(l) => EntryType::Link(l), _ => EntryType::Other }),
|
||||
};
|
||||
|
||||
trace!("Archive entry: {}{:10} bytes, path={:?} type={:?}", fts, ent.size(), path, ret);
|
||||
Some((path, ret))
|
||||
}
|
||||
|
||||
pub fn read<F,G>(&mut self, rd: &mut ArchiveReader, interest_cb: F, mut file_cb: G) -> ArchiveResult<()>
|
||||
where F: Fn(&str) -> bool, G: FnMut(&[&str], &mut ArchiveReader) -> ArchiveResult<()>
|
||||
{
|
||||
while let Some((path, t)) = Self::read_header(rd) {
|
||||
// We ought to throw away the result of the previous entry with the same name and use
|
||||
// this new entry instead, but fuck it. This case is too rare, so let's just warn! it.
|
||||
if let Some(_) = self.seen.get(&path) {
|
||||
warn!("Duplicate file entry: {}", path);
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut newt = t;
|
||||
match newt {
|
||||
EntryType::Regular if interest_cb(&path) => {
|
||||
let pathv = [&path as &str];
|
||||
try!(file_cb(&pathv[..], rd));
|
||||
newt = EntryType::Handled
|
||||
},
|
||||
EntryType::Link(_) if interest_cb(&path) => {
|
||||
self.links.push(path.clone());
|
||||
},
|
||||
_ => ()
|
||||
};
|
||||
self.seen.insert(path, newt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// This is basically realpath(), using the virtual filesystem in self.seen.
|
||||
// This method is not particularly efficient, it allocates like crazy.
|
||||
fn resolve_link(&self, base: &str, path: &str, depth: usize) -> Option<(EntryType, Vec<String>)> {
|
||||
if depth < 1 {
|
||||
warn!("Unresolved link: {} -> {}; Recursion depth exceeded", base, path);
|
||||
return None
|
||||
}
|
||||
|
||||
// Remove filename from the base
|
||||
let basedir = if let Some(i) = base.rfind('/') { base.split_at(i).0 } else { return None };
|
||||
|
||||
let comp : Vec<&str> =
|
||||
if path.starts_with('/') { path.split('/').collect() }
|
||||
else { basedir.split('/').chain(path.split('/')).collect() };
|
||||
|
||||
let mut dest = Vec::new();
|
||||
|
||||
for (i, &c) in comp.iter().enumerate() {
|
||||
if c == "" || c == "." {
|
||||
continue;
|
||||
}
|
||||
if c == ".." {
|
||||
if dest.len() > 1 {
|
||||
dest.pop();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
dest.push(c.to_string());
|
||||
let curpath = dest.join("/");
|
||||
match self.seen.get(&curpath) {
|
||||
|
||||
// If it's a directory, we're good
|
||||
Some(&EntryType::Directory) => (),
|
||||
|
||||
// If it's a file or man page, it must be the last item.
|
||||
Some(& ref x@ EntryType::Regular) |
|
||||
Some(& ref x@ EntryType::Handled) => return
|
||||
if i == comp.len()-1 {
|
||||
Some((x.clone(), dest))
|
||||
} else {
|
||||
warn!("Unresolved link: {} -> {}; Non-directory component", base, path);
|
||||
None
|
||||
},
|
||||
|
||||
// Links... Ugh
|
||||
Some(&EntryType::Link(ref d)) => {
|
||||
match self.resolve_link(&curpath, &d, depth-1) {
|
||||
// Same as above, with dirs we can continue, files have to be last
|
||||
Some((EntryType::Directory, d)) => dest = d,
|
||||
x@Some((EntryType::Regular, _)) |
|
||||
x@Some((EntryType::Handled, _)) => return
|
||||
if i == comp.len()-1 { x }
|
||||
else {
|
||||
warn!("Unresolved link: {} -> {}; Non-directory link component", base, path);
|
||||
None
|
||||
},
|
||||
_ => return None,
|
||||
}
|
||||
},
|
||||
|
||||
// Don't care about anything else, just stop.
|
||||
_ => {
|
||||
warn!("Unresolved link: {} -> {}; Special or non-existing file", base, path);
|
||||
return None
|
||||
}
|
||||
}
|
||||
}
|
||||
Some((EntryType::Directory, dest))
|
||||
}
|
||||
|
||||
pub fn links<F>(&mut self, mut cb: F) where F: FnMut(&str, &str) {
|
||||
for p in self.links.iter() {
|
||||
let dest = match self.seen.get(p) { Some(&EntryType::Link(ref x)) => x, _ => unreachable!() };
|
||||
|
||||
match self.resolve_link(&p, dest, 32) {
|
||||
Some((EntryType::Handled, d)) => {
|
||||
let dstr = d.join("/");
|
||||
cb(&p, &dstr)
|
||||
},
|
||||
Some((EntryType::Regular, d)) => {
|
||||
let dstr = d.join("/");
|
||||
self.missedfiles.entry(dstr).or_insert_with(Vec::new).push(p.to_string());
|
||||
}
|
||||
_ => {},
|
||||
}
|
||||
}
|
||||
// We can reclaim this memory early.
|
||||
self.links = Vec::new();
|
||||
self.seen = HashMap::new();
|
||||
}
|
||||
|
||||
pub fn need_reread(&self) -> bool {
|
||||
self.missedfiles.len() > 0
|
||||
}
|
||||
|
||||
pub fn reread<G>(&mut self, rd: &mut ArchiveReader, mut file_cb: G) -> ArchiveResult<()>
|
||||
where G: FnMut(&[&str], &mut ArchiveReader) -> ArchiveResult<()>
|
||||
{
|
||||
while let Some((path, _)) = Self::read_header(rd) {
|
||||
if let Some(f) = self.missedfiles.remove(&path) {
|
||||
let v: Vec<&str> = f.iter().map(|x| x as &str).collect();
|
||||
try!(file_cb(&v, rd))
|
||||
}
|
||||
if self.missedfiles.len() < 1 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use env_logger;
|
||||
|
||||
fn test_read(r: &mut Reader) {
|
||||
let mut f = open_file("tests/testarchive.tar.xz").unwrap();
|
||||
let mut files = Vec::new();
|
||||
r.read(&mut f,
|
||||
|p| p.starts_with("man/man"),
|
||||
|p,_| { files.extend(p.iter().map(|x| x.to_string())); Ok(()) }
|
||||
).unwrap();
|
||||
assert_eq!(files, vec!["man/man3/helloworld.3".to_string()]);
|
||||
}
|
||||
|
||||
fn test_resolve_links(r: &mut Reader) {
|
||||
let res = |p| {
|
||||
if let Some(&EntryType::Link(ref l)) = r.seen.get(p) {
|
||||
r.resolve_link(p, &l, 5)
|
||||
} else {
|
||||
panic!("Not found or not a link: {}", p);
|
||||
}
|
||||
};
|
||||
let helloworld = Some((EntryType::Handled, vec!["man".to_string(), "man3".to_string(), "helloworld.3".to_string()]));
|
||||
|
||||
assert_eq!(res("man/mans"), Some((EntryType::Directory, vec!["man".to_string(), "man3".to_string()])));
|
||||
assert_eq!(res("man/man6/hardlink.6"), helloworld);
|
||||
assert_eq!(res("man/man1/symlinkbefore.1"), helloworld);
|
||||
assert_eq!(res("man/man6/symlinkafter.6"), helloworld);
|
||||
|
||||
assert_eq!(res("man/man1/badsymlink1.1"), None);
|
||||
assert_eq!(res("man/man1/badsymlink2.1"), None);
|
||||
assert_eq!(res("man/man1/badsymlink3.1"), None);
|
||||
assert_eq!(res("man/man1/badsymlink4.1"), None);
|
||||
assert_eq!(res("man/man1/badsymlink5.1"), None);
|
||||
|
||||
assert_eq!(res("man/man1/doublesymlink1.1"), helloworld);
|
||||
assert_eq!(res("man/man1/doublesymlink2.1"), helloworld);
|
||||
assert_eq!(res("man/man1/triplesymlink.1"), helloworld);
|
||||
assert_eq!(res("man/man1/infinitesymlink.1"), None);
|
||||
}
|
||||
|
||||
fn test_links(r: &mut Reader) {
|
||||
let mut links = Vec::new();
|
||||
r.links(|p,d| links.push((p.to_string(), d.to_string())));
|
||||
links.sort();
|
||||
|
||||
{
|
||||
let mut res = |p:&str| {
|
||||
let r = links.remove(0);
|
||||
assert_eq!(r.0, p.to_string());
|
||||
assert_eq!(r.1, "man/man3/helloworld.3".to_string());
|
||||
};
|
||||
res("man/man1/doublesymlink1.1");
|
||||
res("man/man1/doublesymlink2.1");
|
||||
res("man/man1/symlinkbefore.1");
|
||||
res("man/man1/triplesymlink.1");
|
||||
res("man/man6/hardlink.6");
|
||||
res("man/man6/symlinkafter.6");
|
||||
}
|
||||
assert_eq!(links.len(), 0);
|
||||
}
|
||||
|
||||
fn test_reread(r: &mut Reader) {
|
||||
assert!(r.need_reread());
|
||||
|
||||
let mut f = open_file("tests/testarchive.tar.xz").unwrap();
|
||||
let mut files = Vec::new();
|
||||
r.reread(&mut f,
|
||||
|p,_| { files.extend(p.iter().map(|x| x.to_string())); Ok(()) }
|
||||
).unwrap();
|
||||
|
||||
files.sort();
|
||||
assert_eq!(files, vec![
|
||||
"man/man3/needreread.3".to_string(),
|
||||
"man/man6/needreread.6".to_string()
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reader() {
|
||||
env_logger::init().unwrap();
|
||||
|
||||
let mut r = Reader::new();
|
||||
test_read(&mut r);
|
||||
test_resolve_links(&mut r);
|
||||
test_links(&mut r);
|
||||
test_reread(&mut r);
|
||||
}
|
||||
}
|
||||
97
indexer/src/main.rs
Normal file
97
indexer/src/main.rs
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
#[macro_use] extern crate log;
|
||||
extern crate env_logger;
|
||||
extern crate libarchive;
|
||||
extern crate regex;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
mod archive;
|
||||
|
||||
|
||||
// Checks a path for a man page candidate. Returns None if it doesn't seem like a man page
|
||||
// location, otherwise Some((manPageName, Section, Locale)).
|
||||
fn parse_path(path: &str) -> Option<(&str, &str, &str)> {
|
||||
// Roughly: man[/locale]/man1/manpage.section[.compression]+
|
||||
// TODO: lazy_static
|
||||
let re = Regex::new(r"(?x)
|
||||
man
|
||||
(?: / ([^/]+) )? # Optional locale
|
||||
/man[a-z0-9]/ # Subdir
|
||||
([^/]+?) # Man page name (non-greedy)
|
||||
\. ([^/\.]+) # Section
|
||||
(?: \. (?: gz|lzma|bz2|xz ))* $ # Any number of compression extensions
|
||||
").unwrap();
|
||||
|
||||
let cap = match re.captures(path) { Some(x) => x, None => return None };
|
||||
let locale = cap.at(1).unwrap_or("");
|
||||
let name = cap.at(2).unwrap();
|
||||
let section = cap.at(3).unwrap();
|
||||
|
||||
// Not everything matching the regex is necessarily a man page, exclude some special cases.
|
||||
match (name, section, locale) {
|
||||
// Files that totally aren't man pages
|
||||
("Makefile", "in", _) |
|
||||
("Makefile", "am", _) |
|
||||
(".cvsignore", _, _) |
|
||||
(_, "gz", _) |
|
||||
(_, "lzma", _) |
|
||||
(_, "bz2", _) |
|
||||
(_, "xz", _) |
|
||||
(_, "html", _) => None,
|
||||
// Some weird directories that happen to match the locale
|
||||
(n, s, "5man") |
|
||||
(n, s, "c") |
|
||||
(n, s, "man1") |
|
||||
(n, s, "man2") |
|
||||
(n, s, "man3") |
|
||||
(n, s, "man4") |
|
||||
(n, s, "man5") |
|
||||
(n, s, "man6") |
|
||||
(n, s, "man7") |
|
||||
(n, s, "man8") |
|
||||
(n, s, "Man-Part1") |
|
||||
(n, s, "Man-Part2") => Some((n, s, "")),
|
||||
// Nothing special!
|
||||
x => Some(x)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fn main() {
|
||||
env_logger::init().unwrap();
|
||||
info!("Hello, world!");
|
||||
}
|
||||
|
||||
|
||||
#[test]
|
||||
fn test_parse_path() {
|
||||
// Generic tests
|
||||
assert_eq!(parse_path("/"), None);
|
||||
assert_eq!(parse_path("/man1/ncdu.1"), None);
|
||||
assert_eq!(parse_path("/man/man?/ncdu.1"), None);
|
||||
assert_eq!(parse_path("/man/man1/ncdu.1"), Some(("ncdu", "1", "")));
|
||||
assert_eq!(parse_path("/man/man1/ncdu.1.gz.lzma.xz.bz2.gz"), Some(("ncdu", "1", ""))); // This stuff happens
|
||||
assert_eq!(parse_path("/man/en_US.UTF-8/man1/ncdu.1"), Some(("ncdu", "1", "en_US.UTF-8")));
|
||||
|
||||
// Special cases
|
||||
assert_eq!(parse_path("/usr/share/man/man1/INDEX"), None);
|
||||
assert_eq!(parse_path("/usr/share/man/man1/Makefile"), None);
|
||||
assert_eq!(parse_path("/usr/share/man/man1/Makefile.am"), None);
|
||||
assert_eq!(parse_path("/usr/share/man/man1/Makefile.in"), None);
|
||||
assert_eq!(parse_path("/usr/share/man/man1/.cvsignore"), None);
|
||||
assert_eq!(parse_path("/usr/share/man/man1/.cvsignore.gz"), None);
|
||||
|
||||
// Some actual locations
|
||||
assert_eq!(parse_path("/usr/local/man/man1/list_audio_tracks.1.gz"), Some(("list_audio_tracks", "1", "")));
|
||||
assert_eq!(parse_path("/usr/local/lib/perl5/site_perl/man/man3/DBIx::Class::Helper::ResultSet::DateMethods1::Announcement.3.gz"), Some(("DBIx::Class::Helper::ResultSet::DateMethods1::Announcement", "3", "")));
|
||||
assert_eq!(parse_path("/usr/man/man3/exit.3tk"), Some(("exit", "3tk", "")));
|
||||
assert_eq!(parse_path("/usr/local/brlcad/share/man/mann/exit.nged.gz"), Some(("exit", "nged", "")));
|
||||
assert_eq!(parse_path("/usr/X11R6/man/man3/intro.3xglut.gz"), Some(("intro", "3xglut", "")));
|
||||
assert_eq!(parse_path("/usr/local/share/man/ko_KR.eucKR/man3/intro.3.gz"), Some(("intro", "3", "ko_KR.eucKR")));
|
||||
|
||||
assert_eq!(parse_path("/usr/lib/scilab/man/Man-Part1/man1/ans.1"), Some(("ans", "1", "")));
|
||||
assert_eq!(parse_path("/heirloom/usr/share/man/5man/man1/chgrp.1.gz"), Some(("chgrp", "1", "")));
|
||||
|
||||
assert_eq!(parse_path("/usr/local/plan9/man/man8/index.html"), None);
|
||||
assert_eq!(parse_path("/usr/local/share/doc/gmt/html/man/grdpaste.html"), None);
|
||||
}
|
||||
49
indexer/tests/mktar.sh
Executable file
49
indexer/tests/mktar.sh
Executable file
|
|
@ -0,0 +1,49 @@
|
|||
#!/bin/sh
|
||||
|
||||
# The order of inserting the files into the tar is not fully deterministic this
|
||||
# way. The tests will fail quite badly if hardlink.6 is considered the
|
||||
# "original" version.
|
||||
|
||||
mkdir man
|
||||
cd man
|
||||
|
||||
mkdir man1
|
||||
mkdir man3
|
||||
mkdir man6
|
||||
ln -s man3 mans
|
||||
|
||||
echo 'Hello World' >man3/helloworld.3
|
||||
echo 'Not a very interesting file' >notinteresting
|
||||
echo 'Potentially interesting file' >possiblyinteresting
|
||||
|
||||
ln man3/helloworld.3 man6/hardlink.6
|
||||
|
||||
ln -s ../man3/helloworld.3 man1/symlinkbefore.1
|
||||
ln -s ../man3/helloworld.3 man6/symlinkafter.6
|
||||
|
||||
ln -s notadir/../../man3/helloworld.3 man1/badsymlink1.1
|
||||
ln -s man3/helloworld.3 man1/badsymlink2.1
|
||||
ln -s ../man3/helloworld.3/. man1/badsymlink3.1
|
||||
ln -s ../man3/helloworld.3/../helloworld.3 man1/badsymlink4.1
|
||||
ln -s ../man1/symlinkbefore.1/../../man1/helloworld.3 man1/badsymlink5.1
|
||||
|
||||
ln -s symlinkbefore.1 man1/doublesymlink1.1
|
||||
ln -s ../mans/helloworld.3 man1/doublesymlink2.1
|
||||
ln -s ../mans/../man1/symlinkbefore.1 man1/triplesymlink.1
|
||||
ln -s infinitesymlink.1 man1/infinitesymlink.1
|
||||
|
||||
ln -s ../possiblyinteresting man3/needreread.3
|
||||
ln -s ../possiblyinteresting man6/needreread.6
|
||||
|
||||
cd ..
|
||||
rm -f testarchive.tar
|
||||
tar -cf testarchive.tar man/
|
||||
rm -r man/
|
||||
|
||||
mkdir man
|
||||
echo 'Overwritten file' >man/possiblyinteresting
|
||||
tar -rf testarchive.tar man/
|
||||
rm -r man/
|
||||
|
||||
rm -f testarchive.tar.xz
|
||||
xz testarchive.tar
|
||||
BIN
indexer/tests/testarchive.tar.xz
Normal file
BIN
indexer/tests/testarchive.tar.xz
Normal file
Binary file not shown.
Loading…
Add table
Add a link
Reference in a new issue