diff --git a/indexer/Cargo.lock b/indexer/Cargo.lock index 905815c..62ea8f8 100644 --- a/indexer/Cargo.lock +++ b/indexer/Cargo.lock @@ -74,8 +74,6 @@ name = "cookie" version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "openssl 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -157,20 +155,6 @@ name = "fallible-iterator" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "gcc" -version = "0.3.39" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "gdi32-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "hex" version = "0.2.0" @@ -200,8 +184,6 @@ dependencies = [ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-verify 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "solicit 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", @@ -254,14 +236,6 @@ name = "libc" version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "libressl-pnacl-sys" -version = "2.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "pnacl-build-helper 1.4.10 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "log" version = "0.3.6" @@ -333,49 +307,6 @@ dependencies = [ "libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "openssl" -version = "0.7.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gcc 0.3.39 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys-extras 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl-sys" -version = "0.7.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "libressl-pnacl-sys 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "user32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl-sys-extras" -version = "0.7.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "gcc 0.3.39 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl-verify" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "openssl 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "phf" version = "0.7.20" @@ -394,14 +325,6 @@ name = "pkg-config" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "pnacl-build-helper" -version = "1.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "postgres" version = "0.13.3" @@ -426,14 +349,6 @@ dependencies = [ "md5 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "rand" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "regex" version = "0.1.80" @@ -492,14 +407,6 @@ name = "strsim" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "tempdir" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "term_size" version = "0.2.1" @@ -592,15 +499,6 @@ dependencies = [ "matches 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "user32-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "utf8-ranges" version = "0.1.3" @@ -640,8 +538,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum encoding_index_tests 0.1.5 (git+https://github.com/lifthrasiir/rust-encoding)" = "" "checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f" "checksum fallible-iterator 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5d48ab1bc11a086628e8cc0cc2c2dc200b884ac05c4b48fb71d6036b6999ff1d" -"checksum gcc 0.3.39 (registry+https://github.com/rust-lang/crates.io-index)" = "771e4a97ff6f237cf0f7d5f5102f6e28bb9743814b6198d684da5c58b76c11e0" -"checksum gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0912515a8ff24ba900422ecda800b52f4016a56251922d397c576bf92c690518" "checksum hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa" "checksum hpack 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d2da7d3a34cf6406d9d700111b8eafafe9a251de41ae71d8052748259343b58" "checksum httparse 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e7a63e511f9edffbab707141fbb8707d1a3098615fb2adbd5769cdfcc9b17d" @@ -652,7 +548,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6abe0ee2e758cd6bc8a2cd56726359007748fbf4128da998b65d0b70f881e19b" "checksum libarchive3-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3cd3beae8f59a4c7a806523269b5392037577c150446e88d684dfa6de6031ca7" "checksum libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "a51822fc847e7a8101514d1d44e354ba2ffa7d4c194dcab48870740e327cac70" -"checksum libressl-pnacl-sys 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "cbc058951ab6a3ef35ca16462d7642c4867e6403520811f28537a4e2f2db3e71" "checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054" "checksum matches 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efd7622e3022e1a6eaa602c4cea8912254e5582c9c692e9167714182244801b1" "checksum md5 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7df230903ccdffd6b3b4ec21624498ea64c912ce50297846907f0b8e1bb249dd" @@ -663,17 +558,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum num-iter 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "287a1c9969a847055e1122ec0ea7a5c5d6f72aad97934e131c83d5c08ab4e45c" "checksum num-traits 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "a16a42856a256b39c6d3484f097f6713e14feacd9bfb02290917904fae46c81c" "checksum num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "55aabf4e2d6271a2e4e4c0f2ea1f5b07cc589cc1a9e9213013b54a76678ca4f3" -"checksum openssl 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "c4117b6244aac42ed0150a6019b4d953d28247c5dd6ae6f46ae469b5f2318733" -"checksum openssl-sys 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)" = "89c47ee94c352eea9ddaf8e364be7f978a3bb6d66d73176572484238dd5a5c3f" -"checksum openssl-sys-extras 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "11c5e1dba7d3d03d80f045bf0d60111dc69213b67651e7c889527a3badabb9fa" -"checksum openssl-verify 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ed86cce894f6b0ed4572e21eb34026f1dc8869cb9ee3869029131bc8c3feb2d" "checksum phf 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)" = "0c6afb2057bb5f846a7b75703f90bc1cef4970c35209f712925db7768e999202" "checksum phf_shared 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)" = "286385a0e50d4147bce15b2c19f0cf84c395b0e061aaf840898a7bf664c2cfb7" "checksum pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8cee804ecc7eaf201a4a207241472cc870e825206f6c031e3ee2a72fa425f2fa" -"checksum pnacl-build-helper 1.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "61c9231d31aea845007443d62fcbb58bb6949ab9c18081ee1e09920e0cf1118b" "checksum postgres 0.13.3 (git+https://github.com/sfackler/rust-postgres)" = "" "checksum postgres-protocol 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8ba21a6309a3d518b151c6c3154a926f055b969898541ae9df3ac68309c07e9e" -"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" "checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f" "checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957" "checksum ring 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0743ef007bcff4909b107907a410418eb7e5c6ad55b843d70b39f62bfb7112e" @@ -682,7 +571,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" "checksum solicit 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "172382bac9424588d7840732b250faeeef88942e37b6e35317dce98cafdd75b2" "checksum strsim 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "67f84c44fbb2f91db7fef94554e6b2ac05909c9c0b0bc23bb98d3a1aebfe7f7c" -"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" "checksum term_size 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f7f5f3f71b0040cecc71af239414c23fd3c73570f5ff54cf50e03cef637f2a0" "checksum thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9539db560102d1cef46b8b78ce737ff0bb64e7e18d35b2a5688f7d097d0ff03" "checksum thread_local 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8576dbbfcaef9641452d5cf0df9b0e7eeab7694956dd33bb61515fb8f18cfdd5" @@ -696,7 +584,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum unicode-width 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d6722facc10989f63ee0e20a83cd4e1714a9ae11529403ac7e0afd069abc39e" "checksum untrusted 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "193df64312e3515fd983ded55ad5bcaa7647a035804828ed757e832ce6029ef3" "checksum url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "48ccf7bd87a81b769cf84ad556e034541fb90e1cd6d4bc375c822ed9500cd9d7" -"checksum user32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ef4711d107b21b410a3a974b1204d9accc8b10dad75d8324b5d755de1617d47" "checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f" "checksum vec_map 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cac5efe5cb0fa14ec2f84f83c701c562ee63f6dcc680861b21d65c682adfb05f" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index b33767b..f8ede69 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -14,6 +14,6 @@ encoding = { git = "https://github.com/lifthrasiir/rust-encoding", features = [" ring = "0.5.3" postgres = { git = "https://github.com/sfackler/rust-postgres" } clap = "2.16.3" -hyper = "0.9.11" +hyper = { version = "0.9.11", default-features = false } url = "1.2.3" chrono = "0.2.25" diff --git a/indexer/src/open.rs b/indexer/src/open.rs index 8fa2864..bf1ef01 100644 --- a/indexer/src/open.rs +++ b/indexer/src/open.rs @@ -1,8 +1,8 @@ use std::io::{BufRead,BufReader,Read,Result,Error,ErrorKind,copy}; use std::fs::{File,create_dir_all,metadata,read_dir,remove_file}; -use std::hash::{Hash,Hasher,SipHasher}; use std::time::{Duration,SystemTime}; use regex::bytes::Regex; +use ring::digest; use url::Url; use url::percent_encoding::percent_decode; use hyper; @@ -24,9 +24,11 @@ fn cache_fn(url: &Url) -> String { let name = url.path_segments().unwrap().last().unwrap(); let name = if name == "" { "index" } else { name }; - let mut hash = SipHasher::new(); - url.hash(&mut hash); - format!("{}/{}-{}-{:x}", CACHE_PATH, url.host_str().unwrap(), name, hash.finish()) + let hash = digest::digest(&digest::SHA1, url.as_str().as_bytes()) + .as_ref()[0..8].into_iter() + .fold(0u64, |a, &e| (a<<8) + e as u64); + + format!("{}/{}-{}-{:x}", CACHE_PATH, url.host_str().unwrap(), name, hash) } @@ -65,7 +67,7 @@ pub fn clear_cache() -> Result<()> { impl<'a> Path<'a> { pub fn open(&self) -> Result> { if let Ok(url) = Url::parse(self.path) { - if url.scheme() != "http" && url.scheme() != "https" { + if url.scheme() != "http" { return Err(Error::new(ErrorKind::Other, "Invalid scheme")); }