Indexer: Improve logging + cache management

indexer/src/main.rs

@@ -72,6 +72,11 @@ fn main() {
         .filter(Some("postgres"), if verbose >= 4 { log::LogLevelFilter::Trace } else { log::LogLevelFilter::Info })
         .init().unwrap();
 
+    if let Err(e) = open::clear_cache() {
+        error!("Error clearing cache: {}", e);
+        return;
+    }
+
     let dbhost = match std::env::var("MANNED_PG") {
         Ok(x) => x,
         Err(_) => { error!("MANNED_PG not set."); return }

indexer/src/open.rs

@@ -1,5 +1,5 @@
 use std::io::{Read,Result,Error,ErrorKind,copy};
-use std::fs::{File,create_dir_all,metadata};
+use std::fs::{File,create_dir_all,metadata,read_dir,remove_file};
 use std::hash::{Hash,Hasher,SipHasher};
 use std::time::{Duration,SystemTime};
 use url::Url;
@@ -7,7 +7,7 @@ use hyper;
 
 
 const CACHE_PATH: &'static str = "/var/tmp/manned-indexer";
-const CACHE_TIME: u64 = 23*3600;
+const CACHE_TIME: u64 = 20*3600;
 
 
 #[derive(Clone,Copy)]
@@ -47,6 +47,19 @@ fn file(path: &str) -> Result<Box<Read>> {
 }
 
 
+pub fn clear_cache() -> Result<()> {
+    create_dir_all(CACHE_PATH)?;
+    for f in read_dir(CACHE_PATH)? {
+        let f = f?.path();
+        let m = metadata(&f)?;
+        if m.modified().unwrap() < SystemTime::now() - Duration::from_secs(CACHE_TIME) {
+            remove_file(&f)?;
+        }
+    }
+    Ok(())
+}
+
+
 impl<'a> Path<'a> {
     pub fn open(&self) -> Result<Box<Read>> {
         if let Ok(url) = Url::parse(self.path) {
@@ -56,12 +69,9 @@ impl<'a> Path<'a> {
 
             if self.cache {
                 let cfn = cache_fn(&url);
-                if let Ok(m) = metadata(&cfn) {
+                if let Ok(f) = file(&cfn) {
-                    if m.modified().unwrap() > SystemTime::now() - Duration::from_secs(CACHE_TIME) {
+                    return Ok(f);
-                        return file(&cfn);
-                    }
                 }
-                try!(create_dir_all(CACHE_PATH));
                 {
                     let mut rd = try!(fetch(url.as_str()));
                     let mut wr = try!(File::create(&cfn));

indexer/src/pkg.rs

@@ -20,6 +20,8 @@ pub struct PkgOpt<'a> {
 
 
 fn insert_pkg(tr: &postgres::transaction::Transaction, opt: &PkgOpt) -> Option<i32> {
+    let pkginfo = format!("sys {} / {} / {} - {} @ {} @ {}", opt.sys, opt.cat, opt.pkg, opt.ver, opt.date, opt.file.path);
+
     // The ON CONFLICT .. DO UPDATE is used instead of DO NOTHING because in that case the
     // RETURNING clause wouldn't give us a package id.
     let q = "INSERT INTO packages (system, category, name) VALUES($1, $2, $3)
@@ -39,17 +41,17 @@ fn insert_pkg(tr: &postgres::transaction::Transaction, opt: &PkgOpt) -> Option<i
     if res.is_empty() {
         let q = "INSERT INTO package_versions (package, version, released, arch) VALUES($1, $2, $3::text::date, $4) RETURNING id";
         verid = tr.query(q, &[&pkgid, &opt.ver, &opt.date, &opt.arch]).unwrap().get(0).get(0);
-        info!("New package pkgid {} verid {}", pkgid, verid);
+        info!("New package pkgid {} verid {}, {}", pkgid, verid, pkginfo);
         Some(verid)
 
     } else if opt.force {
         verid = res.get(0).get(0);
-        info!("Overwriting package pkgid {} verid {}", pkgid, verid);
+        info!("Overwriting package pkgid {} verid {}, {}", pkgid, verid, pkginfo);
         tr.query("DELETE FROM man WHERE package = $1", &[&verid]).unwrap();
         Some(verid)
 
     } else {
-        info!("Package already in database, pkgid {} verid {}", pkgid, res.get(0).get::<usize,i32>(0));
+        debug!("Package already in database, pkgid {} verid {}, {}", pkgid, res.get(0).get::<usize,i32>(0), pkginfo);
         None
     }
 }
@@ -86,7 +88,7 @@ fn insert_man(tr: &postgres::GenericConnection, verid: i32, paths: &[&str], ent:
 
     for path in paths {
         insert_man_row(tr, verid, path, enc, dig.as_ref());
-        debug!("Inserted man page: {} ({})", path, enc);
+        info!("Inserted man page: {} ({})", path, enc);
     }
 }
 
@@ -100,7 +102,7 @@ fn insert_link(tr: &postgres::GenericConnection, verid: i32, src: &str, dest: &s
     let hash: Vec<u8> = res.get(0).get(0);
     let enc: String = res.get(0).get(1);
     insert_man_row(tr, verid, src, &enc, &hash);
-    debug!("Inserted man link: {} -> {}", src, dest);
+    info!("Inserted man link: {} -> {}", src, dest);
 }
 
 
@@ -149,8 +151,6 @@ fn index_pkg(tr: &postgres::GenericConnection, opt: &PkgOpt, verid: i32) -> std:
 
 
 pub fn pkg(conn: &postgres::GenericConnection, opt: PkgOpt) {
-    info!("Handling pkg: {} / {} / {} - {} @ {} @ {}", opt.sys, opt.cat, opt.pkg, opt.ver, opt.date, opt.file.path);
-
     let tr = conn.transaction().unwrap();
     tr.set_rollback();