indexer: Support FreeBSD 9.3+; remove now obsolete add_index.pl

indexer/src/main.rs

@@ -20,6 +20,7 @@ mod pkg;
 mod sys_arch;
 mod sys_deb;
 mod sys_freebsd1;
+mod sys_freebsd2;
 
 
 // Convenience function to get a system id by short-name. Panics if the system doesn't exist.
@@ -67,6 +68,11 @@ fn main() {
             (@arg mirror: --mirror +required +takes_value "Mirror URL (should point to the packages/ dir)")
             (@arg arch: --arch +required +takes_value "Arch")
         )
+        (@subcommand freebsd2 =>
+            (about: "Index packages from a FreeBSD >= 9.3 package repo")
+            (@arg sys: --sys +required +takes_value "System short-name")
+            (@arg mirror: --mirror +required +takes_value "Mirror URL")
+        )
     ).get_matches();
 
     unsafe { pkg::DRY_RUN = arg.is_present("dry") };
@@ -101,6 +107,7 @@ fn main() {
         let date = match matches.value_of("date").unwrap() {
             "deb" => pkg::Date::Deb,
             "desc" => pkg::Date::Desc,
+            "max" => pkg::Date::Max,
             s => pkg::Date::Known(s),
         };
         pkg::pkg(&db, pkg::PkgOpt {
@@ -140,5 +147,12 @@ fn main() {
         ).unwrap_or_else(|e| error!("{}", e));
     }
 
+    if let Some(matches) = arg.subcommand_matches("freebsd2") {
+        sys_freebsd2::sync(&db,
+            sysbyshort(&db, matches.value_of("sys").unwrap()),
+            matches.value_of("mirror").unwrap()
+        ).unwrap_or_else(|e| error!("{}", e));
+    }
+
     trace!("Exiting");
 }

indexer/src/pkg.rs

@@ -17,6 +17,8 @@ pub enum Date<'a> {
     Found(i64),     // Found in package
     Deb,            // Should be read from the timestamp of the 'debian-binary' file
     Desc,           // Should be read from the '+DESC' file (FreeBSD <= 9.2)
+    Max,            // Use the latest timestamp in the archive
+    MaxVal(i64),
 }
 
 
@@ -26,6 +28,8 @@ impl<'a> Date<'a> {
         *self = match *self {
             Date::Deb if ent.format() == Format::Ar && ent.path() == Some("debian-binary") => Date::Found(ent.mtime()),
             Date::Desc if ent.path() == Some("+DESC") => Date::Found(ent.mtime()),
+            Date::Max => Date::MaxVal(ent.mtime()),
+            Date::MaxVal(t) if ent.mtime() > t => Date::MaxVal(ent.mtime()),
             x => x,
         }
     }
@@ -193,7 +197,7 @@ fn index_pkg(tr: &postgres::GenericConnection, mut opt: PkgOpt, verid: i32) -> s
 
     match opt.date {
         Date::Known(_) => Ok(()),
-        Date::Found(t) => {
+        Date::Found(t) | Date::MaxVal(t) => {
             let date = NaiveDateTime::from_timestamp(t, 0).format("%Y-%m-%d").to_string();
             debug!("Date from package: {}", date);
             tr.execute("UPDATE package_versions SET released = $1::text::date WHERE id = $2", &[&date, &verid]).unwrap();

indexer/src/sys_freebsd2.rs

@@ -0,0 +1,63 @@
+use std::io::{BufReader,BufRead,Result,Error,ErrorKind};
+use regex::bytes::Regex;
+use std::str;
+use postgres;
+
+use open;
+use pkg;
+use archive::{Archive,ArchiveEntry};
+
+
+fn getpkgsite(mut ent: Option<ArchiveEntry>) -> Result<ArchiveEntry> {
+    while let Some(e) = ent {
+        if e.path() == Some("packagesite.yaml") {
+            return Ok(e)
+        }
+        ent = e.next()?
+    }
+    Err(Error::new(ErrorKind::Other, "No packagesite.yaml found"))
+}
+
+
+pub fn sync(pg: &postgres::GenericConnection, sys: i32, mirror: &str) -> Result<()> {
+    let path = format!("{}packagesite.txz", mirror);
+    let mut rd = open::Path{path: &path, cache: true, canbelocal: false}.open()?;
+
+    let ent = Archive::open_archive(&mut rd)?;
+    let brd = BufReader::new(getpkgsite(ent)?);
+
+    // It's technically a JSON/YAML file, but rather than bothering with a proper JSON parser,
+    // these regexes will do fine.
+    lazy_static!(
+        static ref RE_NAME : Regex = Regex::new(r#""name"\s*:\s*"(?u:([^ "]+))""#).unwrap();
+        static ref RE_VER  : Regex = Regex::new(r#""version"\s*:\s*"(?u:([^ "]+))""#).unwrap();
+        static ref RE_CAT  : Regex = Regex::new(r#""origin"\s*:\s*"(?u:([^ "/]+))"#).unwrap();
+        static ref RE_PATH : Regex = Regex::new(r#""path"\s*:\s*"(?u:([^ "]+))""#).unwrap();
+        static ref RE_ARCH : Regex = Regex::new(r#""arch"\s*:\s*"(?u:([^ "]+))""#).unwrap();
+    );
+
+    for line in brd.split(b'\n') {
+        let line = line?;
+        let name = match RE_NAME.captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+        let ver  = match RE_VER .captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+        let cat  = match RE_CAT .captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+        let path = match RE_PATH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+        let arch = match RE_ARCH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+        let uri = format!("{}{}", mirror, path);
+        pkg::pkg(pg, pkg::PkgOpt{
+            force: false,
+            sys: sys,
+            cat: cat,
+            pkg: name,
+            ver: ver,
+            date: pkg::Date::Max,
+            arch: Some(arch),
+            file: open::Path{
+                path: &uri,
+                cache: false,
+                canbelocal: false,
+            },
+        });
+    }
+    Ok(())
+}