indexer: Support for indexing FreeBSD <9.3 repositories

This commit is contained in:
Yorhel 2016-12-11 10:59:54 +01:00
parent 1ca0cd4325
commit defaa032f8
9 changed files with 892 additions and 826 deletions

128
indexer/Cargo.lock generated
View file

@ -3,15 +3,15 @@ name = "indexer"
version = "0.1.0"
dependencies = [
"chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.17.1 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.19.2 (registry+https://github.com/rust-lang/crates.io-index)",
"encoding 0.3.0-dev (git+https://github.com/lifthrasiir/rust-encoding)",
"env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.9.13 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libarchive3-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"postgres 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"postgres 0.13.3 (git+https://github.com/sfackler/rust-postgres)",
"regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)",
"ring 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -56,15 +56,15 @@ dependencies = [
[[package]]
name = "clap"
version = "2.17.1"
version = "2.19.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"strsim 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"strsim 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"term_size 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-segmentation 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-segmentation 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-width 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"vec_map 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -75,7 +75,7 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"openssl 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -159,7 +159,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "gcc"
version = "0.3.38"
version = "0.3.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -186,23 +186,23 @@ dependencies = [
[[package]]
name = "httparse"
version = "1.1.2"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "hyper"
version = "0.9.11"
version = "0.9.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cookie 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"httparse 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"mime 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl-verify 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
"solicit 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
"traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -237,7 +237,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "lazy_static"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -245,13 +245,13 @@ name = "libarchive3-sys"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libc"
version = "0.2.17"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -282,7 +282,7 @@ name = "memchr"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -327,10 +327,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "num_cpus"
version = "1.1.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -339,9 +339,9 @@ version = "0.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"gcc 0.3.39 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl-sys 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl-sys-extras 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -352,7 +352,7 @@ version = "0.7.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"libressl-pnacl-sys 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
"user32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -363,8 +363,8 @@ name = "openssl-sys-extras"
version = "0.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"gcc 0.3.39 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl-sys 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -378,15 +378,15 @@ dependencies = [
[[package]]
name = "phf"
version = "0.7.15"
version = "0.7.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"phf_shared 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)",
"phf_shared 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "phf_shared"
version = "0.7.15"
version = "0.7.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -404,20 +404,20 @@ dependencies = [
[[package]]
name = "postgres"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
version = "0.13.3"
source = "git+https://github.com/sfackler/rust-postgres#8e5f28eb70d330335a8d697812be64e8f4c189ba"
dependencies = [
"bufstream 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"fallible-iterator 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"phf 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)",
"postgres-protocol 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"phf 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)",
"postgres-protocol 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "postgres-protocol"
version = "0.1.0"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -428,10 +428,10 @@ dependencies = [
[[package]]
name = "rand"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -456,13 +456,13 @@ name = "ring"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"untrusted 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rustc-serialize"
version = "0.3.19"
version = "0.3.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -489,7 +489,7 @@ dependencies = [
[[package]]
name = "strsim"
version = "0.5.1"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -497,7 +497,7 @@ name = "tempdir"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -506,7 +506,7 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -516,7 +516,7 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -533,7 +533,7 @@ version = "0.1.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -570,7 +570,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "unicode-segmentation"
version = "0.1.2"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -580,7 +580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "untrusted"
version = "0.3.1"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -628,7 +628,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum bufstream 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7b48dbe2ff0e98fa2f03377d204a9637d3c9816cd431bfe05a8abbd0ea11d074"
"checksum byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855"
"checksum chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9213f7cd7c27e95c2b57c49f0e69b1ea65b27138da84a170133fd21b07659c00"
"checksum clap 2.17.1 (registry+https://github.com/rust-lang/crates.io-index)" = "27dac76762fb56019b04aed3ccb43a770a18f80f9c2eb62ee1a18d9fb4ea2430"
"checksum clap 2.19.2 (registry+https://github.com/rust-lang/crates.io-index)" = "305ad043f009db535a110200541d4567b63e172b1fe030313fbb92565da7ed24"
"checksum cookie 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0e3d6405328b6edb412158b3b7710e2634e23f3614b9bb1c412df7952489a626"
"checksum encoding 0.3.0-dev (git+https://github.com/lifthrasiir/rust-encoding)" = "<none>"
"checksum encoding-index-japanese 1.20141219.6 (git+https://github.com/lifthrasiir/rust-encoding)" = "<none>"
@ -640,18 +640,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum encoding_index_tests 0.1.5 (git+https://github.com/lifthrasiir/rust-encoding)" = "<none>"
"checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f"
"checksum fallible-iterator 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5d48ab1bc11a086628e8cc0cc2c2dc200b884ac05c4b48fb71d6036b6999ff1d"
"checksum gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)" = "553f11439bdefe755bf366b264820f1da70f3aaf3924e594b886beb9c831bcf5"
"checksum gcc 0.3.39 (registry+https://github.com/rust-lang/crates.io-index)" = "771e4a97ff6f237cf0f7d5f5102f6e28bb9743814b6198d684da5c58b76c11e0"
"checksum gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0912515a8ff24ba900422ecda800b52f4016a56251922d397c576bf92c690518"
"checksum hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa"
"checksum hpack 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d2da7d3a34cf6406d9d700111b8eafafe9a251de41ae71d8052748259343b58"
"checksum httparse 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "46534074dbb80b070d60a5cb8ecadd8963a00a438ae1a95268850a7ef73b67ae"
"checksum hyper 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)" = "edd47c66782933e546a32ae89ca3c49263b2ba9bc29f3a0d5c52fff48e0ac67c"
"checksum httparse 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e7a63e511f9edffbab707141fbb8707d1a3098615fb2adbd5769cdfcc9b17d"
"checksum hyper 0.9.13 (registry+https://github.com/rust-lang/crates.io-index)" = "86ea0c0ff7e6ef09eff72234800ddb48b6263277936e7ecd6ecd3250345d705f"
"checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a"
"checksum lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "49247ec2a285bb3dcb23cbd9c35193c025e7251bfce77c1d5da97e6362dffe7f"
"checksum lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6abe0ee2e758cd6bc8a2cd56726359007748fbf4128da998b65d0b70f881e19b"
"checksum libarchive3-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3cd3beae8f59a4c7a806523269b5392037577c150446e88d684dfa6de6031ca7"
"checksum libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "044d1360593a78f5c8e5e710beccdc24ab71d1f01bc19a29bcacdba22e8475d8"
"checksum libc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "a51822fc847e7a8101514d1d44e354ba2ffa7d4c194dcab48870740e327cac70"
"checksum libressl-pnacl-sys 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "cbc058951ab6a3ef35ca16462d7642c4867e6403520811f28537a4e2f2db3e71"
"checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054"
"checksum matches 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efd7622e3022e1a6eaa602c4cea8912254e5582c9c692e9167714182244801b1"
@ -662,26 +662,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum num-integer 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "fb24d9bfb3f222010df27995441ded1e954f8f69cd35021f6bef02ca9552fb92"
"checksum num-iter 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "287a1c9969a847055e1122ec0ea7a5c5d6f72aad97934e131c83d5c08ab4e45c"
"checksum num-traits 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "a16a42856a256b39c6d3484f097f6713e14feacd9bfb02290917904fae46c81c"
"checksum num_cpus 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8890e6084723d57d0df8d2720b0d60c6ee67d6c93e7169630e4371e88765dcad"
"checksum num_cpus 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "55aabf4e2d6271a2e4e4c0f2ea1f5b07cc589cc1a9e9213013b54a76678ca4f3"
"checksum openssl 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "c4117b6244aac42ed0150a6019b4d953d28247c5dd6ae6f46ae469b5f2318733"
"checksum openssl-sys 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)" = "89c47ee94c352eea9ddaf8e364be7f978a3bb6d66d73176572484238dd5a5c3f"
"checksum openssl-sys-extras 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "11c5e1dba7d3d03d80f045bf0d60111dc69213b67651e7c889527a3badabb9fa"
"checksum openssl-verify 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ed86cce894f6b0ed4572e21eb34026f1dc8869cb9ee3869029131bc8c3feb2d"
"checksum phf 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)" = "17896951e179a6cbed7d3519b3078ac6c03a347d3e9cf8f303c8a1a73c5a3e44"
"checksum phf_shared 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6c14aac1140c2b06b41477096f249416b17c893d56386a892ac657edfdffba"
"checksum phf 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)" = "0c6afb2057bb5f846a7b75703f90bc1cef4970c35209f712925db7768e999202"
"checksum phf_shared 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)" = "286385a0e50d4147bce15b2c19f0cf84c395b0e061aaf840898a7bf664c2cfb7"
"checksum pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8cee804ecc7eaf201a4a207241472cc870e825206f6c031e3ee2a72fa425f2fa"
"checksum pnacl-build-helper 1.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "61c9231d31aea845007443d62fcbb58bb6949ab9c18081ee1e09920e0cf1118b"
"checksum postgres 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a7ef92468927003a037e175b54320319e358886865899b37f7318837a646a9fd"
"checksum postgres-protocol 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7e2fc3d800dacc2dd749b690ad15b9b78bc04c26c3f0525cbe163436559bc3fc"
"checksum rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "2791d88c6defac799c3f20d74f094ca33b9332612d9aef9078519c82e4fe04a5"
"checksum postgres 0.13.3 (git+https://github.com/sfackler/rust-postgres)" = "<none>"
"checksum postgres-protocol 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8ba21a6309a3d518b151c6c3154a926f055b969898541ae9df3ac68309c07e9e"
"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
"checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f"
"checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957"
"checksum ring 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0743ef007bcff4909b107907a410418eb7e5c6ad55b843d70b39f62bfb7112e"
"checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b"
"checksum rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "237546c689f20bb44980270c73c3b9edd0891c1be49cc1274406134a66d3957b"
"checksum rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084"
"checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac"
"checksum solicit 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "172382bac9424588d7840732b250faeeef88942e37b6e35317dce98cafdd75b2"
"checksum strsim 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "50c069df92e4b01425a8bf3576d5d417943a6a7272fbabaf5bd80b1aaa76442e"
"checksum strsim 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "67f84c44fbb2f91db7fef94554e6b2ac05909c9c0b0bc23bb98d3a1aebfe7f7c"
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"
"checksum term_size 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f7f5f3f71b0040cecc71af239414c23fd3c73570f5ff54cf50e03cef637f2a0"
"checksum thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9539db560102d1cef46b8b78ce737ff0bb64e7e18d35b2a5688f7d097d0ff03"
@ -692,9 +692,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a5906ca2b98c799f4b1ab4557b76367ebd6ae5ef14930ec841c74aed5f3764"
"checksum unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c1f7ceb96afdfeedee42bade65a0d585a6a0106f681b6749c8ff4daa8df30b3f"
"checksum unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "26643a2f83bac55f1976fb716c10234485f9202dcd65cfbdf9da49867b271172"
"checksum unicode-segmentation 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b905d0fc2a1f0befd86b0e72e31d1787944efef9d38b9358a9e92a69757f7e3b"
"checksum unicode-segmentation 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c3bc443ded17b11305ffffe6b37e2076f328a5a8cb6aa877b1b98f77699e98b5"
"checksum unicode-width 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d6722facc10989f63ee0e20a83cd4e1714a9ae11529403ac7e0afd069abc39e"
"checksum untrusted 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5d9bc0e6e73a10975d1fbff8ac3541e221181b0d8998351600fb5523de634c0d"
"checksum untrusted 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "193df64312e3515fd983ded55ad5bcaa7647a035804828ed757e832ce6029ef3"
"checksum url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "48ccf7bd87a81b769cf84ad556e034541fb90e1cd6d4bc375c822ed9500cd9d7"
"checksum user32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ef4711d107b21b410a3a974b1204d9accc8b10dad75d8324b5d755de1617d47"
"checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f"

View file

@ -12,7 +12,7 @@ libc = "0.2.17"
libarchive3-sys = "0.1.2"
encoding = { git = "https://github.com/lifthrasiir/rust-encoding", features = ["no-optimized-legacy-encoding"] }
ring = "0.5.3"
postgres = "0.12.0"
postgres = { git = "https://github.com/sfackler/rust-postgres" }
clap = "2.16.3"
hyper = "0.9.11"
url = "1.2.3"

View file

@ -19,6 +19,7 @@ mod open;
mod pkg;
mod sys_arch;
mod sys_deb;
mod sys_freebsd1;
// Convenience function to get a system id by short-name. Panics if the system doesn't exist.
@ -60,6 +61,12 @@ fn main() {
(@arg contents: --contents +takes_value "Contents file")
(@arg packages: --packages +required +takes_value "Packages file")
)
(@subcommand freebsd1 =>
(about: "Index packages from a FreeBSD <= 9.2 package repo")
(@arg sys: --sys +required +takes_value "System short-name")
(@arg mirror: --mirror +required +takes_value "Mirror URL (should point to the packages/ dir)")
(@arg arch: --arch +required +takes_value "Arch")
)
).get_matches();
unsafe { pkg::DRY_RUN = arg.is_present("dry") };
@ -93,6 +100,7 @@ fn main() {
if let Some(matches) = arg.subcommand_matches("pkg") {
let date = match matches.value_of("date").unwrap() {
"deb" => pkg::Date::Deb,
"desc" => pkg::Date::Desc,
s => pkg::Date::Known(s),
};
pkg::pkg(&db, pkg::PkgOpt {
@ -124,5 +132,13 @@ fn main() {
);
}
if let Some(matches) = arg.subcommand_matches("freebsd1") {
sys_freebsd1::sync(&db,
sysbyshort(&db, matches.value_of("sys").unwrap()),
matches.value_of("arch").unwrap(),
matches.value_of("mirror").unwrap()
).unwrap_or_else(|e| error!("{}", e));
}
trace!("Exiting");
}

View file

@ -1,8 +1,10 @@
use std::io::{Read,Result,Error,ErrorKind,copy};
use std::io::{BufRead,BufReader,Read,Result,Error,ErrorKind,copy};
use std::fs::{File,create_dir_all,metadata,read_dir,remove_file};
use std::hash::{Hash,Hasher,SipHasher};
use std::time::{Duration,SystemTime};
use regex::bytes::Regex;
use url::Url;
use url::percent_encoding::percent_decode;
use hyper;
@ -90,4 +92,41 @@ impl<'a> Path<'a> {
Err(Error::new(ErrorKind::Other, "Invalid URL"))
}
}
// Attempt to parse a HTTP directory listing. Returns the name and whether it's a directory for
// each item.
// Only tested with a lighttpd/1.4 and apache 2.4 server.
// (I tried using FTP before, but that didn't work out well; While FTP does return a more easily
// parsable file list, some servers have issues with generating a list of a large directory)
pub fn dirlist(&self) -> Result<Vec<(String,bool)>> {
lazy_static!(
static ref RE: Regex = Regex::new("(?i:<a +href *= *\"([^?/\"]+)(/?)\">)").unwrap();
);
let rd = self.open()?;
let brd = BufReader::new(rd);
let mut res = Vec::new();
for line in brd.split(b'\n') {
let line = line?;
let mut matches = RE.captures_iter(&line);
let first = matches.next();
// There's only a single link per line.
if first.is_some() && matches.next().is_some() {
continue;
}
if let Some(cap) = first {
let name = cap.at(1).unwrap();
if name == b".." || name.starts_with(b"/") {
continue;
}
if let Ok(name) = percent_decode(name).decode_utf8() {
let isdir = cap.at(2) == Some(b"/");
res.push((name.to_string(), isdir));
}
}
}
Ok(res)
}
}

View file

@ -16,6 +16,7 @@ pub enum Date<'a> {
Known(&'a str), // Given in PkgOpt
Found(i64), // Found in package
Deb, // Should be read from the timestamp of the 'debian-binary' file
Desc, // Should be read from the '+DESC' file (FreeBSD <= 9.2)
}
@ -24,6 +25,7 @@ impl<'a> Date<'a> {
// TODO: Validate that the mtime() date is sensible (e.g. 1990 < date < now)
*self = match *self {
Date::Deb if ent.format() == Format::Ar && ent.path() == Some("debian-binary") => Date::Found(ent.mtime()),
Date::Desc if ent.path() == Some("+DESC") => Date::Found(ent.mtime()),
x => x,
}
}

186
indexer/src/sys_freebsd1.rs Normal file
View file

@ -0,0 +1,186 @@
use std::collections::HashSet;
use std::ascii::AsciiExt;
use std::io::Result;
use regex::Regex;
use postgres;
use open;
use pkg;
// Sync a FreeBSD <= 9.2 package respository.
//
// Reads "." to get a list of categories, "Latest" to get a list of all packages, and all category
// directories to figure out which package belongs in which category.
//
// Splitting a package filename into a package name and version is a hard problem. There are two
// strategies:
// 1. Use the listing from 'Latest' to get the list of package names, and use that to find the
// longest matching substring in the package filename to split off the version.
// 2. Guessing, like splitver() below.
//
// Both strategies lead to errors. (1) doesn't always work because the 'Latest' directory tends to
// miss a few packages. (2) doesn't always work because version strings are too damn irregular.
// This function tries (1) first, then falls back to (2) if it couldn't find a matching package.
// This combined solution also isn't perfect, as sometimes a package prefix does exist, but is
// incomplete. E.g. 'pear-PHPUnit-1.3.3.tbz' is parsed as 'pear version PHPUnit-1.3.3' rather than
// 'pear-PHPUnit version 1.3.3', because there is a 'pear' package in 'Latest' but no
// 'pear-PHPUnit'. This is handled with a static list of package names to add to the 'pkgs' list,
// see EXTRA_PKGS below.
pub fn sync(pg: &postgres::GenericConnection, sys: i32, arch: &str, mirror: &str) -> Result<()> {
let path = format!("{}Latest/", mirror);
let mut pkgs : Vec<String> = open::Path{path: &path, cache: true, canbelocal: false}
.dirlist()?.into_iter()
.map(|(n,_)| trimext(&n).to_string())
.collect();
pkgs.extend(EXTRA_PKGS.into_iter().map(|e| e.to_string()));
pkgs.sort_by(|a, b| b.len().cmp(&a.len())); // Longest first
// List of packages (name+version) we've already seen; Some packages are present in multiple
// categories, we only index the first found.
let mut seenpkgs = HashSet::new();
let cats = open::Path{path: mirror, cache: true, canbelocal: false}
.dirlist()?.into_iter()
.filter(|&(ref n,i)| i && n != "All" && n != "Latest")
.map(|(n,_)| n);
for cat in cats {
trace!("Category: {}", cat);
let path = format!("{}{}/", mirror, cat);
let lst = open::Path{path: &path, cache: true, canbelocal: false}.dirlist()?.into_iter().map(|(n,_)| n);
for f in lst {
let name = trimext(&f);
if !name.is_ascii() {
warn!("Non-ASCII package name: {}", f);
continue;
}
// The take() mystifies me; why is it necessary?
let pkg = pkgs.iter()
.find(|p| name.len() > p.len()+1 && name.starts_with(&p as &str) && &name[p.len() .. p.len()+1] == "-")
.take().map(|p| (p as &str, &name[p.len()+1 .. ]))
.or_else(|| splitver(name));
if let Some((pkg, ver)) = pkg {
if !seenpkgs.insert((pkg.to_string(), ver.to_string())) {
continue;
}
let path = format!("{}{}/{}", mirror, cat, f);
pkg::pkg(pg, pkg::PkgOpt{
force: false,
sys: sys,
cat: &cat,
pkg: pkg,
ver: ver,
date: pkg::Date::Desc,
arch: Some(arch),
file: open::Path{
path: &path,
cache: false,
canbelocal: false,
},
});
} else {
warn!("Unknown package: {}/{}", cat, f);
}
}
}
Ok(())
}
fn trimext(n: &str) -> &str {
n.trim_right_matches(".tgz").trim_right_matches(".tbz")
}
fn splitver(n: &str) -> Option<(&str, &str)> {
lazy_static!(
static ref RE1: Regex = Regex::new("^(.+?)-([0-9].*)$").unwrap();
static ref RE2: Regex = Regex::new("^(.+)-([^-]+)$").unwrap();
);
if let Some(cap) = RE1.captures(n) {
Some((cap.at(1).unwrap(), cap.at(2).unwrap()))
} else if let Some(cap) = RE2.captures(n) {
Some((cap.at(1).unwrap(), cap.at(2).unwrap()))
} else {
None
}
}
// This list may not be complete, and these packages may not necessarily have man pages.
const EXTRA_PKGS : &'static [&'static str] = &[
"amanda-client",
"amanda-server",
"apache-event",
"apache-itk",
"apache-peruser",
"apache-tomcat",
"apache-worker",
"bison-devel",
"boxbackup-devel",
"boxbackup-devel",
"ffmpeg-devel",
"flex-sdk",
"fpc-gdb",
"freeradius-mysql",
"gdb-insight",
"glib-reference",
"gmime-24",
"gmime-24-sharp",
"gtk-reference",
"gtk-sharp",
"gtkmm-reference",
"horde-content",
"horde-groupware",
"horde-timeobjects",
"horde-webmail",
"hping-devel",
"ja-jvim-direct_canna",
"ja-mutt-devel",
"kdelibs-experimental",
"kdepim-runtime",
"lame-devel",
"libdivxdecore-devel",
"libquicktime-lame",
"libtorrent-rasterbar",
"linux-netscape-communicator",
"mkisofs-devel",
"mldonkey-core-devel",
"mldonkey-gui-devel",
"mod_log_sql-dtc",
"nethack-qt",
"nfdump-devel",
"openssl-beta",
"pear-PHPUnit",
"pear-XML_Query2XML",
"pear-phpunit-PHPUnit",
"pgadmin3-unicode",
"proftpd-mod_ldap",
"proftpd-mod_sql_mysql",
"proftpd-mod_sql_odbc",
"proftpd-mod_sql_postgres",
"proftpd-mod_sql_sqlite",
"proftpd-mod_sql_tds",
"qt-static",
"rsyslog-gnutls",
"rsyslog-gssapi",
"rsyslog-libdbi",
"rsyslog-mysql",
"rsyslog-pgsql",
"rsyslog-relp",
"rsyslog-rfc3195",
"rsyslog-snmp",
"samba-libsmbclient",
"samba-nmblookup",
"squirrelmail-shared_calendars-plugin",
"tcl-thread",
"wxgtk2-common-devel",
"wxgtk2-contrib-common-devel",
"wxgtk2-utils-devel",
];