Add basic pgp keyserver script

modules/dev/54267.lua

@@ -1,13 +0,0 @@
--- Description: Reproduce a rust stdlib panic
--- Version: 0.1.0
--- Source: domains
--- License: GPL-3.0
-
-function run()
-    --[[
-    See for details:
-    https://github.com/rust-lang/rust/issues/54267
-    https://github.com/rust-lang/rust/issues/39364
-    ]]--
-    sleep(10)
-end

modules/dev/otx-passive-dns.lua

@@ -1,4 +1,4 @@
--- Description: Query alienvault otx passive dns to discover subdomains
+-- Description: Query alienvault otx passive dns for subdomains of a domain
 -- Version: 0.1.0
 -- Source: domains
 -- License: GPL-3.0

modules/dev/pgp-keyserver.lua

@@ -0,0 +1,53 @@
+-- Description: Query pgp keyserver for email addresses
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    session = http_mksession()
+
+    --lookup_url = 'https://pgp.mit.edu/pks/lookup'
+    lookup_url = 'https://sks-keyservers.net/pks/lookup'
+
+    req = http_request(session, 'GET', lookup_url, {
+        query={
+            search=arg['value'],
+        }
+    })
+
+    resp = http_send(req)
+    if last_err() then return end
+    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+    links = html_select_list(resp['text'], 'a')
+    i = 1
+    while i <= #links do
+        href = links[i]['attrs']['href']
+
+        if href:find('/pks/lookup%?op=get&search=') == 1 then
+            url = url_join(lookup_url, href)
+
+            req = http_request(session, 'GET', url, {})
+
+            resp = http_send(req)
+            -- TODO: do not abort script if one attempt fails
+            if last_err() then return end
+            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+            pubkey = pgp_pubkey_armored(resp['text'])
+
+            print(pubkey)
+
+            -- TODO: ensure at least one email matches our target domain
+            if pubkey['uids'] then
+                j = 1
+                while j <= #pubkey['uids'] do
+                    print(pubkey['uids'][j])
+                    j = j+1
+                end
+            end
+        end
+
+        i = i+1
+    end
+end

modules/dev/threatminer-ip-addr.lua

@@ -1,4 +1,4 @@
--- Description: Query ThreatMiner passive dns to discover domains for ip
+-- Description: Query ThreatMiner passive dns for subdomains of an ip address
 -- Version: 0.1.0
 -- Source: ipaddrs
 -- License: GPL-3.0
@@ -6,6 +6,8 @@
 function run(arg)
     session = http_mksession()
 
+    -- TODO: add option to filter old entries based on last_seen
+
     req = http_request(session, 'GET', 'https://api.threatminer.org/v2/host.php', {
         query={
             rt='2',

modules/dev/threatminer-passive-dns.lua

@@ -1,4 +1,4 @@
--- Description: Query ThreatMiner passive dns to discover subdomains
+-- Description: Query ThreatMiner passive dns for subdomains of a domain
 -- Version: 0.1.0
 -- Source: domains
 -- License: GPL-3.0

modules/dev/url-scan.lua

@@ -1,4 +1,4 @@
--- Description: Check subdomains for websites
+-- Description: Scan subdomains for websites
 -- Version: 0.1.0
 -- Source: subdomains
 -- License: GPL-3.0

modules/harness/sleep.lua

@@ -0,0 +1,8 @@
+-- Description: Sleep for 10 seconds
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run()
+    sleep(10)
+end

src/engine/mod.rs

@@ -12,6 +12,7 @@ use sn0int_common::metadata::{Metadata, Source};
 use chrootable_https::dns::DnsConfig;
 use psl::Psl;
 use paths;
+use std::cmp::Ordering;
 use term;
 use worker::{self, Event};
 
@@ -100,12 +101,13 @@ impl Engine {
         }
     }
 
-    // TODO: this should return an iter
     pub fn list(&self) -> Vec<&Module> {
-        self.modules.iter()
+        let mut modules: Vec<_> = self.modules.iter()
             .filter(|(key, _)| key.contains('/'))
             .flat_map(|(_, v)| v.iter())
-            .collect()
+            .collect();
+        modules.sort_by(|a, b| a.cmp_canonical(b));
+        modules
     }
 
     pub fn variants(&self) -> Vec<String> {
@@ -173,6 +175,14 @@ impl Module {
         debug!("Executing lua script {}", self.canonical());
         self.script.run(dns_config, psl, reporter, arg.into())
     }
+
+    fn cmp_canonical(&self, other: &Module) -> Ordering {
+        if self.author == other.author {
+            self.name.cmp(&other.name)
+        } else {
+            self.author.cmp(&other.author)
+        }
+    }
 }
 
 pub trait Reporter: Debug {

src/engine/structs.rs

@@ -93,6 +93,35 @@ impl Into<AnyLuaValue> for LuaMap {
     }
 }
 
+#[derive(Debug, Default)]
+pub struct LuaList(Vec<(AnyLuaValue, AnyLuaValue)>);
+
+impl LuaList {
+    #[inline]
+    pub fn new() -> LuaList {
+        LuaList::default()
+    }
+
+    pub fn push<V: Into<AnyLuaValue>>(&mut self, v: V) {
+        let idx = self.0.len() + 1;
+        self.0.push((AnyLuaValue::LuaNumber(idx as f64), v.into()));
+    }
+
+    pub fn push_str<I: Into<String>>(&mut self, v: I) {
+        self.push(AnyLuaValue::LuaString(v.into()))
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+}
+
+impl Into<AnyLuaValue> for LuaList {
+    fn into(self: LuaList) -> AnyLuaValue {
+        AnyLuaValue::LuaArray(self.0)
+    }
+}
+
 pub fn byte_array(bytes: AnyLuaValue) -> Result<Vec<u8>> {
     match bytes {
         AnyLuaValue::LuaAnyString(bytes) => Ok(bytes.0),

src/json.rs

@@ -64,6 +64,7 @@ impl Into<AnyLuaValue> for LuaJsonValue {
             // TODO: not sure if this might fail
             LuaJsonValue::Number(v) => AnyLuaValue::LuaNumber(v.as_f64().unwrap()),
             LuaJsonValue::String(v) => AnyLuaValue::LuaString(v),
+            // TODO: ensure lua tables always start at 1
             LuaJsonValue::Array(v) => AnyLuaValue::LuaArray(v.into_iter().enumerate()
                 .map(|(i, x)| (AnyLuaValue::LuaNumber(i as f64), x.into()))
                 .collect()

src/runtime/pgp.rs

@@ -2,25 +2,29 @@ use errors::*;
 
 use sloppy_rfc4880::{self, Tag};
 use engine::ctx::State;
-use engine::structs::{LuaMap, byte_array};
+use engine::structs::{LuaMap, LuaList, byte_array};
 use hlua::{self, AnyLuaValue};
 use std::sync::Arc;
 use std::io::BufReader;
 
 
 fn pgp_pubkey_lua(pubkey: Vec<u8>) -> Result<AnyLuaValue> {
-    let mut map = LuaMap::new();
+    let mut uids = LuaList::new();
 
     for (tag, body) in sloppy_rfc4880::Parser::new(pubkey.as_slice()) {
         match tag {
             Tag::UserID => {
                 let body = String::from_utf8(body)?;
-                map.insert_str("uid", body);
+                uids.push_str(body);
             },
             _ => (),
         }
     }
 
+    let mut map = LuaMap::new();
+    if !uids.is_empty() {
+        map.insert("uids", uids);
+    }
     Ok(map.into())
 }