vndb/manned
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rust: hyper -> reqwest for the indexer

Browse source 
Since Hyper doesn't provide a synchronous API anymore.
This commit is contained in:
Yorhel 2019-05-25 08:44:23 +02:00
parent f0df5092c3
commit 2974ee929e
4 changed files with 988 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

1012
indexer/Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

2
indexer/Cargo.toml
View file

@ -14,7 +14,7 @@ encoding = { git = "https://github.com/lifthrasiir/rust-encoding", features = ["
ring = "0.14.6" ring = "0.14.6"
postgres = "0.15.2" postgres = "0.15.2"
clap = "2.31.2" clap = "2.31.2"
hyper = { version = "0.10.13", default-features = false } # TODO: Update? reqwest = "0.9.17"
url = "1.7.0" url = "1.7.0"
chrono = "0.4.0" chrono = "0.4.0"
quick-xml = "0.14.0" quick-xml = "0.14.0"

2
indexer/src/main.rs
View file

@ -8,7 +8,7 @@ extern crate libc;
extern crate ring; extern crate ring;
extern crate encoding; extern crate encoding;
extern crate postgres; extern crate postgres;
extern crate hyper; extern crate reqwest;
extern crate url; extern crate url;
extern crate chrono; extern crate chrono;
extern crate quick_xml; extern crate quick_xml;

12
indexer/src/open.rs
View file

@ -5,7 +5,7 @@ use regex::bytes::Regex;
use ring::digest; use ring::digest;
use url::Url; use url::Url;
use url::percent_encoding::percent_decode; use url::percent_encoding::percent_decode;
use hyper; use reqwest;
const CACHE_PATH: &'static str = "/var/tmp/manned-indexer"; const CACHE_PATH: &'static str = "/var/tmp/manned-indexer";
@ -33,14 +33,14 @@ fn cache_fn(url: &Url) -> String {
fn fetch(url: &str) -> Result<Box<Read>> { fn fetch(url: &str) -> Result<Box<Read>> {
let res = try!(hyper::Client::new() let res = try!(reqwest::Client::new()
.get(url) .get(url)
.header(hyper::header::UserAgent("Man page crawler (info@manned.org; https://manned.org/)".to_owned())) .header("User-Agent", "Man page crawler (info@manned.org; https://manned.org/)")
.send() .send()
.map_err(|e| Error::new(ErrorKind::Other, format!("Hyper: {}", e))) .map_err(|e| Error::new(ErrorKind::Other, format!("Reqwest: {}", e)))
); );
if !res.status.is_success() { if !res.status().is_success() {
return Err(Error::new(ErrorKind::Other, format!("HTTP: {}", res.status) )); return Err(Error::new(ErrorKind::Other, format!("HTTP: {}", res.status()) ));
} }
Ok(Box::new(res) as Box<Read>) Ok(Box::new(res) as Box<Read>)
} }