Connect keyring to modules

modules/dev/twilio-lookup.lua

@@ -1,6 +1,7 @@
 -- Description: Retrieve additional information about a phone number
 -- Version: 0.1.0
 -- Source: phonenumbers
+-- Keyring-Access: twilio
 -- License: GPL-3.0
 
 function run(arg)
@@ -10,9 +11,14 @@ function run(arg)
 
     --debug(url)
 
+    key = keyring('twilio')[1]
+    if not key then
+        return 'Missing required twilio access key'
+    end
+
     session = http_mksession()
     req = http_request(session, 'GET', url, {
-        basic_auth={'redacted', 'redacted'},
+        basic_auth={key['access_key'], key['secret_key']},
     })
     reply = http_send(req)
     if last_err() then return end

modules/harness/keyring.lua

@@ -0,0 +1,9 @@
+-- Description: Request access to keyring
+-- Version: 0.1.0
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    keys = keyring('twilio')
+    debug(keys)
+end

sn0int-registry/sn0int-common/src/metadata.rs

@@ -8,6 +8,7 @@ pub enum EntryType {
     Description,
     Version,
     Source,
+    KeyringAccess,
     License,
 }
 
@@ -19,6 +20,7 @@ impl FromStr for EntryType {
             "Description" => Ok(EntryType::Description),
             "Version" => Ok(EntryType::Version),
             "Source" => Ok(EntryType::Source),
+            "Keyring-Access" => Ok(EntryType::KeyringAccess),
             "License" => Ok(EntryType::License),
             x => bail!("Unknown EntryType: {:?}", x),
         }
@@ -82,6 +84,7 @@ pub struct Metadata {
     pub description: String,
     pub version: String,
     pub source: Option<Source>,
+    pub keyring_access: Vec<String>,
     pub license: License,
 }
 
@@ -99,6 +102,7 @@ impl FromStr for Metadata {
                 EntryType::Description => data.description = Some(v),
                 EntryType::Version => data.version = Some(v),
                 EntryType::Source => data.source = Some(v),
+                EntryType::KeyringAccess => data.keyring_access.push(v),
                 EntryType::License => data.license = Some(v),
             }
         }
@@ -112,6 +116,7 @@ pub struct NewMetadata<'a> {
     pub description: Option<&'a str>,
     pub version: Option<&'a str>,
     pub source: Option<&'a str>,
+    pub keyring_access: Vec<&'a str>,
     pub license: Option<&'a str>,
 }
 
@@ -123,6 +128,9 @@ impl<'a> NewMetadata<'a> {
             Some(x) => Some(x.parse()?),
             _ => None,
         };
+        let keyring_access = self.keyring_access.into_iter()
+            .map(String::from)
+            .collect();
         let license = self.license.ok_or_else(|| format_err!("License is required"))?;
         let license = license.parse()?;
 
@@ -130,6 +138,7 @@ impl<'a> NewMetadata<'a> {
             description: description.to_string(),
             version: version.to_string(),
             source,
+            keyring_access,
             license,
         })
     }

src/cmd/keyring_cmd.rs

@@ -61,9 +61,10 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
 }
 
 fn keyring_add(keyring: &mut KeyRing, add: KeyRingAdd) -> Result<()> {
+    // TODO: there's no non-interactive way to add a key without a secret key
     let secret = match add.secret {
-        Some(secret) => secret,
-        None => utils::question("Secretkey")?,
+        Some(secret) => Some(secret),
+        None => utils::question_opt("Secretkey")?,
     };
 
     keyring.insert(add.key, secret)
@@ -76,11 +77,15 @@ fn keyring_delete(keyring: &mut KeyRing, delete: KeyRingDelete) -> Result<()> {
 fn keyring_get(keyring: &KeyRing, get: KeyRingGet) -> Result<()> {
     if let Some(key) = keyring.get(&get.key) {
         if get.quiet {
-            println!("{}", key);
+            if let Some(secret_key) = key.secret_key {
+                println!("{}", secret_key);
+            }
         } else {
             println!("Namespace:    {:?}", get.key.namespace);
             println!("Access Key:   {:?}", get.key.name);
-            println!("Secret:       {:?}", key);
+            if let Some(secret_key) = key.secret_key {
+                println!("Secret:       {:?}", secret_key);
+            }
         }
     }
     Ok(())

src/engine/ctx.rs

@@ -4,6 +4,7 @@ use crate::db::Family;
 use crate::engine::{Environment, Reporter};
 use crate::geoip::{GeoIP, AsnDB};
 use crate::hlua::{self, AnyLuaValue};
+use crate::keyring::KeyRingEntry;
 use crate::models::{Insert, Update};
 use crate::psl::Psl;
 use crate::runtime;
@@ -80,6 +81,8 @@ pub trait State {
         reply.map_err(|err| format_err!("Failed to read stdin: {:?}", err))
     }
 
+    fn keyring(&self, namespace: &str) -> Vec<&KeyRingEntry>;
+
     fn dns_config(&self) -> Arc<Resolver>;
 
     fn psl(&self) -> Arc<Psl>;
@@ -101,6 +104,7 @@ pub struct LuaState {
     logger: Arc<Mutex<Option<Arc<Mutex<Box<Reporter>>>>>>,
     http_sessions: Arc<Mutex<HashMap<String, HttpSession>>>,
     verbose: u64,
+    keyring: Arc<Vec<KeyRingEntry>>, // TODO: maybe hashmap
     dns_config: Arc<Resolver>,
     psl: Arc<Psl>,
     geoip: Arc<GeoIP>,
@@ -152,6 +156,12 @@ impl State for LuaState {
         self.verbose
     }
 
+    fn keyring(&self, namespace: &str) -> Vec<&KeyRingEntry> {
+        self.keyring.iter()
+            .filter(|x| x.namespace == namespace)
+            .collect()
+    }
+
     fn dns_config(&self) -> Arc<Resolver> {
         self.dns_config.clone()
     }
@@ -205,6 +215,7 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
         http_sessions: Arc::new(Mutex::new(HashMap::new())),
 
         verbose: env.verbose,
+        keyring: Arc::new(env.keyring),
         dns_config: Arc::new(env.dns_config),
         psl: Arc::new(env.psl),
         geoip: Arc::new(env.geoip),
@@ -229,6 +240,7 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
     runtime::json_decode(&mut lua, state.clone());
     runtime::json_decode_stream(&mut lua, state.clone());
     runtime::json_encode(&mut lua, state.clone());
+    runtime::keyring(&mut lua, state.clone());
     runtime::last_err(&mut lua, state.clone());
     runtime::pgp_pubkey(&mut lua, state.clone());
     runtime::pgp_pubkey_armored(&mut lua, state.clone());
@@ -309,6 +321,7 @@ impl Script {
     pub fn test(&self) -> Result<()> {
         use crate::engine::tests::DummyReporter;
         use crate::geoip::Maxmind;
+        let keyring = Vec::new();
         let dns_config = Resolver::from_system()?;
         let psl = Psl::from_str(r#"
 // ===BEGIN ICANN DOMAINS===
@@ -320,6 +333,7 @@ com
 
         let env = Environment {
             verbose: 0,
+            keyring,
             dns_config,
             psl,
             geoip,

src/engine/isolation.rs

@@ -2,6 +2,7 @@ use crate::errors::*;
 use chrootable_https::dns::Resolver;
 use crate::engine::{Environment, Module, Reporter};
 use crate::geoip::{GeoIP, AsnDB, Maxmind};
+use crate::keyring::KeyRingEntry;
 use crate::psl::Psl;
 use serde_json;
 use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
@@ -16,15 +17,17 @@ use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
 #[derive(Debug, Serialize, Deserialize)]
 pub struct StartCommand {
     verbose: u64,
+    keyring: Vec<KeyRingEntry>,
     dns_config: Resolver,
     module: Module,
     arg: serde_json::Value,
 }
 
 impl StartCommand {
-    pub fn new(verbose: u64, dns_config: Resolver, module: Module, arg: serde_json::Value) -> StartCommand {
+    pub fn new(verbose: u64, keyring: Vec<KeyRingEntry>, dns_config: Resolver, module: Module, arg: serde_json::Value) -> StartCommand {
         StartCommand {
             verbose,
+            keyring,
             dns_config,
             module,
             arg
@@ -157,7 +160,7 @@ impl Reporter for StdioReporter {
     }
 }
 
-pub fn spawn_module(module: Module, tx: &EventSender, arg: serde_json::Value, verbose: u64, has_stdin: bool) -> Result<()> {
+pub fn spawn_module(module: Module, tx: &EventSender, arg: serde_json::Value, keyring: Vec<KeyRingEntry>, verbose: u64, has_stdin: bool) -> Result<()> {
     let dns_config = Resolver::from_system()?;
 
     let mut reader = if has_stdin {
@@ -167,7 +170,7 @@ pub fn spawn_module(module: Module, tx: &EventSender, arg: serde_json::Value, ve
     };
 
     let mut supervisor = Supervisor::setup(&module)?;
-    supervisor.send_start(&StartCommand::new(verbose, dns_config, module, arg))?;
+    supervisor.send_start(&StartCommand::new(verbose, keyring, dns_config, module, arg))?;
 
     loop {
         match supervisor.recv()? {
@@ -199,6 +202,7 @@ pub fn run_worker(geoip: Vec<u8>, asn: Vec<u8>, psl: String) -> Result<()> {
 
     let environment = Environment {
         verbose: start.verbose,
+        keyring: start.keyring,
         dns_config: start.dns_config,
         psl,
         geoip,

src/engine/mod.rs

@@ -2,6 +2,7 @@ use crate::errors::*;
 
 use crate::geoip::{GeoIP, AsnDB};
 use crate::json::LuaJsonValue;
+use crate::keyring::KeyRingEntry;
 use serde_json;
 use std::fs;
 use std::fmt::Debug;
@@ -27,6 +28,7 @@ pub mod structs;
 #[derive(Debug)]
 pub struct Environment {
   pub verbose: u64,
+  pub keyring: Vec<KeyRingEntry>,
   pub dns_config: Resolver,
   pub psl: Psl,
   pub geoip: GeoIP,
@@ -151,6 +153,7 @@ pub struct Module {
     description: String,
     version: String,
     source: Option<Source>,
+    keyring_access: Vec<String>,
     script: Script,
 }
 
@@ -171,6 +174,7 @@ impl Module {
             description: metadata.description,
             version: metadata.version,
             source: metadata.source,
+            keyring_access: metadata.keyring_access,
             script,
         })
     }
@@ -202,6 +206,10 @@ impl Module {
         &self.source
     }
 
+    pub fn keyring_access(&self) -> &[String] {
+        &self.keyring_access
+    }
+
     pub fn run(&self, env: Environment, reporter: Arc<Mutex<Box<Reporter>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
         self.script.run(env, reporter, arg.into())

src/keyring.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 
+use crate::engine::Module;
 use crate::paths;
 use std::collections::HashMap;
 use std::fs;
@@ -21,7 +22,7 @@ impl KeyName {
         }
     }
 
-    pub fn for_each(k: &str, v: &HashMap<String, String>) -> Vec<KeyName> {
+    pub fn for_each(k: &str, v: &HashMap<String, Option<String>>) -> Vec<KeyName> {
         v.iter()
             .map(move |(x, _)| KeyName::new(k, x.as_str()))
             .collect()
@@ -57,7 +58,7 @@ impl FromStr for KeyName {
 
 pub struct KeyRing {
     path: PathBuf,
-    keys: HashMap<String, HashMap<String, String>>,
+    keys: HashMap<String, HashMap<String, Option<String>>>,
 }
 
 impl KeyRing {
@@ -95,7 +96,7 @@ impl KeyRing {
         Ok(())
     }
 
-    pub fn insert(&mut self, key: KeyName, secret: String) -> Result<()> {
+    pub fn insert(&mut self, key: KeyName, secret: Option<String>) -> Result<()> {
         // get the namespace or create a new one
         let mut x = self.keys.remove(&key.namespace)
             .unwrap_or_else(|| HashMap::new());
@@ -137,11 +138,31 @@ impl KeyRing {
             .collect()
     }
 
-    pub fn get(&self, key: &KeyName) -> Option<String> {
+    pub fn get(&self, key: &KeyName) -> Option<KeyRingEntry> {
         let x = self.keys.get(&key.namespace)?;
-        let x = x.get(&key.name)?;
-        Some(x.to_string())
+        let secret_key = x.get(&key.name)?;
+
+        Some(KeyRingEntry {
+            namespace: key.namespace.to_owned(),
+            access_key: key.name.to_owned(),
+            secret_key: secret_key.to_owned(),
+        })
     }
+
+    pub fn request_keys(&self, module: &Module) -> Vec<KeyRingEntry> {
+        // TODO: we probably want to randomize the order
+        module.keyring_access().iter()
+            .flat_map(|namespace| self.list_for(namespace))
+            .flat_map(|x| self.get(&x))
+            .collect()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct KeyRingEntry {
+    pub namespace: String,
+    pub access_key: String,
+    pub secret_key: Option<String>,
 }
 
 #[cfg(test)]

src/runtime/keyring.rs

@@ -0,0 +1,16 @@
+use crate::engine::ctx::State;
+use crate::hlua::{self, AnyLuaValue};
+use crate::json::LuaJsonValue;
+use std::sync::Arc;
+
+
+pub fn keyring(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("keyring", hlua::function1(move |namespace: String| -> Vec<AnyLuaValue> {
+        state.keyring(&namespace).into_iter()
+            .map(|x| {
+                let v = serde_json::to_value(&x).unwrap();
+                LuaJsonValue::from(v).into()
+            })
+            .collect()
+    }))
+}

src/runtime/mod.rs

@@ -12,6 +12,7 @@ import_fns!(geoip);
 import_fns!(http);
 import_fns!(html);
 import_fns!(json);
+import_fns!(keyring);
 import_fns!(logger);
 import_fns!(pgp);
 import_fns!(psl);

src/worker.rs

@@ -215,6 +215,8 @@ pub fn spawn(rl: &mut Readline, module: &Module, args: Vec<(serde_json::Value, O
         return;
     }
 
+    let keyring = rl.keyring().request_keys(&module);
+
     let mut stack = StackedSpinners::new();
 
     let (tx, rx) = channel::bounded(1);
@@ -229,6 +231,7 @@ pub fn spawn(rl: &mut Readline, module: &Module, args: Vec<(serde_json::Value, O
 
         let tx = tx.clone();
         let module = module.clone();
+        let keyring = keyring.clone();
         let signal_register = rl.signal_register().clone();
         pool.execute(move || {
             let tx = EventSender::new(name, tx);
@@ -239,7 +242,7 @@ pub fn spawn(rl: &mut Readline, module: &Module, args: Vec<(serde_json::Value, O
             }
 
             tx.send(Event2::Start);
-            let event = match engine::isolation::spawn_module(module, &tx, arg, verbose, has_stdin) {
+            let event = match engine::isolation::spawn_module(module, &tx, arg, keyring, verbose, has_stdin) {
                 Ok(_) => ExitEvent::Ok,
                 Err(err) => ExitEvent::Err(err.to_string()),
             };