Release v0.23.0

Merge pull request #215 from kpcyrd/user-agent
Allow setting a different default user agent
2021-10-23 20:02:24 +02:00 · 2021-10-23 12:23:24 +00:00 · 2021-10-23 14:03:12 +02:00 · 2021-10-23 13:50:32 +02:00 · 2021-10-23 13:15:50 +02:00 · 2021-10-02 15:03:25 +02:00
62 changed files with 1066 additions and 929 deletions
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -1 +1,2 @@
 github: [kpcyrd]
+patreon: kpcyrd
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.21.2"
+version = "0.23.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -24,7 +24,6 @@ the results in a unified format for followup investigations."""
 section = "utils"
 priority = "optional"
 depends = "$auto, publicsuffix"
-build-depends = "pkg-config, libsqlite3-dev, libseccomp-dev, libsodium-dev, publicsuffix"
 assets = [
    ["target/release/sn0int", "usr/bin/", "755"],
 ]
@@ -33,11 +32,11 @@ assets = [
 sqlite-bundled = ["libsqlite3-sys/bundled"]

 [dependencies]
-sn0int-common = { version="0.12.0", path="sn0int-common" }
-sn0int-std = { version="=0.21.0", path="sn0int-std" }
-rustyline = "8.0"
+sn0int-common = { version="0.13.0", path="sn0int-common" }
+sn0int-std = { version="=0.23.0", path="sn0int-std" }
+rustyline = "9.0"
 log = "0.4"
-env_logger = "0.8"
+env_logger = "0.9"
 hlua-badtouch = "0.4"
 structopt = "0.3"
 failure = "0.1"
@@ -47,7 +46,7 @@ lazy_static = "1.0"
 shellwords = "1.0"
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-libsqlite3-sys = { version = "0.18.0", features = ["bundled-windows"] }
+libsqlite3-sys = { version = "0.22.0", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
 dirs-next = "2.0"
 url = "2.0"
@@ -61,7 +60,7 @@ serde_urlencoded = "0.7"
 serde_json = "1.0"
 crossbeam-channel = "0.5"
 ctrlc = "3.1"
-opener = "0.4"
+opener = "0.5"
 separator = "0.4"
 maplit = "1.0.1"
 sloppy-rfc4880 = "0.2"
@@ -69,12 +68,12 @@ regex = "1.0"
 toml = "0.5"
 threadpool = "1.7"
 atty = "0.2"
-semver = "0.11"
+semver = "1"
 bytes = "0.4"
 bytesize = "1.0"
 ipnetwork = "0.18"
-strum = "0.20"
-strum_macros = "0.20"
+strum = "0.22"
+strum_macros = "0.22"
 embedded-triple = "0.1.0"
 humansize = "1.1.0"

@@ -88,13 +87,13 @@ hmac = "0.11"
 walkdir = "2.2"
 nude = "0.3"
 glob = "0.3.0"
-os-version = "0.1.1"
+os-version = "0.2"

 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.15"
-nix = "0.20"
+syscallz = "0.16"
+nix = "0.23"

 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.4"
@@ -102,5 +101,5 @@ unveil = "0.3"

 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.11"
+boxxy = "0.12"
 tempfile = "3.0"
--- a/README.md
+++ b/README.md
@@ -14,33 +14,34 @@
 [registry]:     https://sn0int.com/

 sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
-package manager. It was built for IT security professionals and bug hunters to
-gather intelligence about a given target or about yourself. sn0int is
-enumerating attack surface by semi-automatically processing public information
-and mapping the results in a unified format for followup investigations.
+package manager. It's used by IT security professionals, bug bounty hunters,
+law enforcement agencies and in security awareness trainings to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.

 [ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt

 Among other things, sn0int is currently able to:

 - Harvest subdomains from certificate transparency logs and passive dns
+- Mass resolve collected subdomains and scan for http or https services
 - Enrich ip addresses with asn and geoip info
 - Harvest emails from pgp keyservers and whois
 - Discover compromised logins in breaches
 - Find somebody's profiles across the internet
 - Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
- Attempt to bypass cloudflare with shodan
- Harvest data and images from instagram profiles
- Scan images for nudity
+- Harvest activity and images from social media profiles
+- Basic image processing

 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource. None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
+in the source, instead they are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+you to ship updates for your modules on your own instead of pull-requesting
+them into the sn0int codebase.

 For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)

@@ -226,6 +227,9 @@ For everything else please have a look at the [detailed list][1].
  - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
  - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
  - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_ahash](https://sn0int.readthedocs.io/en/latest/reference.html#img-ahash)
+  - [img_dhash](https://sn0int.readthedocs.io/en/latest/reference.html#img-dhash)
+  - [img_phash](https://sn0int.readthedocs.io/en/latest/reference.html#img-phash)
  - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
  - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
--- a/docs/reference.rst
+++ b/docs/reference.rst
@@ -504,6 +504,39 @@ Extract exif metadata from an image.
    if last_err() then return end
    debug(exif)

+img_ahash
+---------
+
+Calculate the Mean (aHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_ahash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_dhash
+---------
+
+Calculate the Gradient (dHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_dhash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_phash
+---------
+
+Calculate the DCT (pHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_phash(blob)
+    if last_err() then return end
+    debug(hash)
+
 img_nudity
 ----------

--- a/modules/harness/img-hash.lua
+++ b/modules/harness/img-hash.lua
@@ -0,0 +1,11 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    debug(arg)
+    info(img_ahash(arg['value']))
+    info(img_dhash(arg['value']))
+    info(img_phash(arg['value']))
+end
--- a/sn0int-common/Cargo.toml
+++ b/sn0int-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.12.0"
+version = "0.13.0"
 description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -12,4 +12,4 @@ serde = { version = "1.0", features=["derive"] }
 #rocket_failure_errors = { path = "../../../rocket_failure/rocket_failure_errors" }
 rocket_failure_errors = "0.2"
 anyhow = "1.0"
-nom = "6.0"
+nom = "7.0"
--- a/sn0int-common/src/lib.rs
+++ b/sn0int-common/src/lib.rs
@@ -1,5 +1,3 @@
-#[macro_use] extern crate nom;
-
 pub mod api;
 pub mod errors;
 pub use crate::errors::*;
--- a/sn0int-common/src/metadata/mod.rs
+++ b/sn0int-common/src/metadata/mod.rs
@@ -1,4 +1,8 @@
 use crate::errors::*;
+use nom::IResult;
+use nom::bytes::complete::{tag, take_until};
+use nom::combinator::map_res;
+use nom::multi::fold_many0;
 use serde::{Serialize, Deserialize};
 use std::str::FromStr;

@@ -223,23 +227,32 @@ impl<'a> NewMetadata<'a> {
    }
 }

-named!(metaline<&str, (EntryType, &str)>, do_parse!(
-    tag!("-- ") >>
-    name: map_res!(take_until!(": "), EntryType::from_str) >>
-    tag!(": ") >>
-    value: take_until!("\n") >>
-    tag!("\n") >>
-    (name, value)
-));
+fn metaline(input: &str) -> IResult<&str, (EntryType, &str)> {
+    let (input, _) = tag("-- ")(input)?;
+    let (input, name) = map_res(
+        take_until(": "),
+        EntryType::from_str
+    )(input)?;
+    let (input, _) = tag(": ")(input)?;
+    let (input, value) = take_until("\n")(input)?;
+    let (input, _) = tag("\n")(input)?;

-named!(metalines<&str, Vec<(EntryType, &str)>>, do_parse!(
-    lines: fold_many0!(metaline, Vec::new(), |mut acc: Vec<_>, item| {
-        acc.push(item);
-        acc
-    }) >>
-    tag!("\n") >>
-    (lines)
-));
+    Ok((input, (name, value)))
+}
+
+fn metalines(input: &str) -> IResult<&str, Vec<(EntryType, &str)>> {
+    let (input, lines) = fold_many0(
+        metaline,
+        Vec::new,
+        |mut acc: Vec<_>, item| {
+            acc.push(item);
+            acc
+        }
+    )(input)?;
+    let (input, _) = tag("\n")(input)?;
+
+    Ok((input, lines))
+}

 #[cfg(test)]
 mod tests {
--- a/sn0int-registry/Cargo.toml
+++ b/sn0int-registry/Cargo.toml
@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"

 [dependencies]
-sn0int-common = { version="0.12.0", path="../sn0int-common" }
+sn0int-common = { version="0.13.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }
@@ -23,7 +23,7 @@ oauth2 = "2.0.0"
 failure = "0.1"
 url = "1.0"
 log = "0.4"
-semver = "0.9.0"
+semver = "1"
 lazy_static = "1"
 blake2 = "0.8.0"
 hex = "0.4"
@@ -33,5 +33,5 @@ serde_json = "1.0"
 syntect = "3.3"

 dotenv = "0.15"
-env_logger = "0.7"
+env_logger = "0.9"
 structopt = "0.3.21"
--- a/sn0int-std/Cargo.toml
+++ b/sn0int-std/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-std"
-version = "0.21.0"
+version = "0.23.0"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
@@ -27,13 +27,13 @@ ct-logs = "0.7"
 chrootable-https = "0.16"
 http = "0.2"
 bufstream = "0.1.4"
-pem = "0.8"
+pem = "1"
 url = "2.0"
 tungstenite = { version = "0.13", default-features = false }
 kuchiki = "0.8.0"
-maxminddb = "0.17"
-x509-parser = "0.9"
-der-parser = "5"
+maxminddb = "0.21"
+x509-parser = "0.12"
+der-parser = "6"
 publicsuffix = { version="2", default-features=false }
 xml-rs = "0.8"
 geo = "0.18"
@@ -45,11 +45,12 @@ sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }

 image = "0.23.0"
 kamadak-exif = "0.5.1"
+img_hash_median = "4.0.0"

 bs58 = "0.4"
 digest = "0.9"
 blake2 = "0.9"

 [dev-dependencies]
-env_logger = "0.8"
+env_logger = "0.9"
 maplit = "1.0.1"
--- a/sn0int-std/src/crt.rs
+++ b/sn0int-std/src/crt.rs
@@ -5,6 +5,7 @@ use std::net::IpAddr;
 use x509_parser::x509::X509Version;
 use x509_parser::certificate::X509Certificate;
 use x509_parser::extensions::{GeneralName, ParsedExtension};
+use x509_parser::prelude::*;

 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
@@ -31,7 +32,7 @@ impl Certificate {
    }

    pub fn from_bytes(crt: &[u8]) -> Result<Certificate> {
-        let crt = match X509Certificate::from_der(&crt) {
+        let crt = match X509Certificate::from_der(crt) {
            Ok((remaining, der)) => {
                if !remaining.is_empty() {
                    bail!("input cert has trailing garbage");
@@ -55,7 +56,7 @@ impl Certificate {
            }
        }

-        for (_oid, ext) in crt.tbs_certificate.extensions {
+        for ext in crt.tbs_certificate.extensions() {
            if let ParsedExtension::SubjectAlternativeName(san) = ext.parsed_extension() {
                for name in &san.general_names {
                    debug!("Certificate is valid for {:?}", name);
--- a/sn0int-std/src/crypto.rs
+++ b/sn0int-std/src/crypto.rs
@@ -22,7 +22,7 @@ pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
    let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
        .ok_or_else(|| format_err!("Nonce has wrong length"))?;
    let ciphertext = &encrypted[secretbox::NONCEBYTES..];
-    let plain = secretbox::open(&ciphertext, &nonce, &key)
+    let plain = secretbox::open(ciphertext, &nonce, &key)
        .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
    Ok(plain)
 }
--- a/sn0int-std/src/engine/structs.rs
+++ b/sn0int-std/src/engine/structs.rs
@@ -86,16 +86,16 @@ impl From<Vec<(AnyLuaValue, AnyLuaValue)>> for LuaMap {
    }
 }

-impl Into<HashMap<AnyHashableLuaValue, AnyLuaValue>> for LuaMap {
-    fn into(self: LuaMap) -> HashMap<AnyHashableLuaValue, AnyLuaValue> {
-        self.0
+impl From<LuaMap> for HashMap<AnyHashableLuaValue, AnyLuaValue> {
+    fn from(map: LuaMap) -> Self {
+        map.0
    }
 }

-impl Into<AnyLuaValue> for LuaMap {
-    fn into(self: LuaMap) -> AnyLuaValue {
+impl From<LuaMap> for AnyLuaValue {
+    fn from(map: LuaMap) -> AnyLuaValue {
        AnyLuaValue::LuaArray(
-            self.into_iter()
+            map.into_iter()
                .filter_map(|(k, v)| {
                    match k {
                        AnyHashableLuaValue::LuaString(x) => Some((AnyLuaValue::LuaString(x), v)),
@@ -136,9 +136,9 @@ impl LuaList {
    }
 }

-impl Into<AnyLuaValue> for LuaList {
-    fn into(self: LuaList) -> AnyLuaValue {
-        AnyLuaValue::LuaArray(self.0)
+impl From<LuaList> for AnyLuaValue {
+    fn from(list: LuaList) -> AnyLuaValue {
+        AnyLuaValue::LuaArray(list.0)
    }
 }

@@ -149,7 +149,7 @@ pub fn byte_array(bytes: AnyLuaValue) -> Result<Vec<u8>> {
        AnyLuaValue::LuaArray(bytes) => {
            Ok(bytes.into_iter()
                .map(|num| match num.1 {
-                    AnyLuaValue::LuaNumber(num) if num <= 255.0 && num >= 0.0 && (num % 1.0 == 0.0) =>
+                    AnyLuaValue::LuaNumber(num) if (0.0..=255.0).contains(&num) && (num % 1.0 == 0.0) =>
                            Ok(num as u8),
                    AnyLuaValue::LuaNumber(num) =>
                            Err(format_err!("number is out of range: {:?}", num)),
--- a/sn0int-std/src/gfx/mod.rs
+++ b/sn0int-std/src/gfx/mod.rs
@@ -1,5 +1,6 @@
 use crate::errors::*;
 use image::{self, DynamicImage, GenericImageView};
+pub use img_hash_median::HashAlg;

 pub mod exif;

@@ -45,9 +46,9 @@ impl ImageFormat {
    }
 }

-impl Into<image::ImageFormat> for ImageFormat {
-    fn into(self) -> image::ImageFormat {
-        match self {
+impl From<ImageFormat> for image::ImageFormat {
+    fn from(format: ImageFormat) -> image::ImageFormat {
+        match format {
            ImageFormat::Png => image::ImageFormat::Png,
            ImageFormat::Jpeg => image::ImageFormat::Jpeg,
            ImageFormat::Gif => image::ImageFormat::Gif,
@@ -79,6 +80,14 @@ impl Image {
    pub fn height(&self) -> u32 {
        self.image.height()
    }
+
+    pub fn perception_hash(&self, hash_alg: HashAlg) -> String {
+        let hasher = img_hash_median::HasherConfig::new()
+            .hash_alg(hash_alg)
+            .to_hasher();
+        let hash = hasher.hash_image(&self.image);
+        hash.to_base64()
+    }
 }

 impl AsRef<DynamicImage> for Image {
@@ -94,10 +103,10 @@ pub fn guess_format(buf: &[u8]) -> Result<ImageFormat> {
 }

 pub fn load(buf: &[u8]) -> Result<Image> {
-    let img_format = image::guess_format(&buf)?;
+    let img_format = image::guess_format(buf)?;
    let format = ImageFormat::try_from(&img_format)?;

-    let image = image::load_from_memory_with_format(&buf, img_format)?;
+    let image = image::load_from_memory_with_format(buf, img_format)?;

    Ok(Image {
        image,
@@ -105,7 +114,6 @@ pub fn load(buf: &[u8]) -> Result<Image> {
    })
 }

-
 #[cfg(test)]
 mod tests {
    use super::*;
--- a/sn0int-std/src/html.rs
+++ b/sn0int-std/src/html.rs
@@ -11,13 +11,13 @@ pub struct Element {
    pub html: String,
 }

-impl Into<AnyLuaValue> for Element {
-    fn into(self) -> AnyLuaValue {
+impl From<Element> for AnyLuaValue {
+    fn from(elem: Element) -> AnyLuaValue {
        let mut map = LuaMap::new();

-        map.insert_str("text", self.text);
-        map.insert("attrs", LuaMap::from(self.attrs));
-        map.insert_str("html", self.html);
+        map.insert_str("text", elem.text);
+        map.insert("attrs", LuaMap::from(elem.attrs));
+        map.insert_str("html", elem.html);

        map.into()
    }
--- a/sn0int-std/src/json.rs
+++ b/sn0int-std/src/json.rs
@@ -1,6 +1,5 @@
 use crate::errors::*;

-use std::iter::FromIterator;
 use std::collections::HashMap;
 use crate::hlua::AnyLuaValue;
 use serde_json::{self, Deserializer, Value, Number, Map};
@@ -22,7 +21,7 @@ pub fn encode(v: AnyLuaValue) -> Result<String> {
 }

 pub fn decode_stream(x: &str) -> Result<Vec<AnyLuaValue>> {
-    let stream = Deserializer::from_str(&x).into_iter::<Value>();
+    let stream = Deserializer::from_str(x).into_iter::<Value>();

    let list = stream
        .filter_map(|x| x.ok())
@@ -52,9 +51,9 @@ pub enum LuaJsonValue {
    Object(HashMap<String, LuaJsonValue>),
 }

-impl Into<AnyLuaValue> for LuaJsonValue {
-    fn into(self) -> AnyLuaValue {
-        match self {
+impl From<LuaJsonValue> for AnyLuaValue {
+    fn from(value: LuaJsonValue) -> AnyLuaValue {
+        match value {
            LuaJsonValue::Null => AnyLuaValue::LuaNil,
            LuaJsonValue::Bool(v) => AnyLuaValue::LuaBoolean(v),
            // TODO: not sure if this might fail
@@ -111,9 +110,9 @@ impl From<AnyLuaValue> for LuaJsonValue {
    }
 }

-impl Into<serde_json::Value> for LuaJsonValue {
-    fn into(self) -> serde_json::Value {
-        match self {
+impl From<LuaJsonValue> for serde_json::Value {
+    fn from(value: LuaJsonValue) -> serde_json::Value {
+        match value {
            LuaJsonValue::Null => Value::Null,
            LuaJsonValue::Bool(v) => Value::Bool(v),
            LuaJsonValue::Number(v) => Value::Number(v),
@@ -122,9 +121,10 @@ impl Into<serde_json::Value> for LuaJsonValue {
                .map(|x| x.into())
                .collect()
            ),
-            LuaJsonValue::Object(v) => Value::Object(Map::from_iter(v.into_iter()
+            LuaJsonValue::Object(v) => Value::Object(v.into_iter()
                .map(|(k, v)| (k, v.into()))
-            )),
+                .collect::<Map<_, _>>()
+            ),
        }
    }
 }
@@ -140,9 +140,10 @@ impl From<serde_json::Value> for LuaJsonValue {
                .map(|x| x.into())
                .collect()
            ),
-            Value::Object(v) => LuaJsonValue::Object(HashMap::from_iter(v.into_iter()
+            Value::Object(v) => LuaJsonValue::Object(v.into_iter()
                .map(|(k, v)| (k, v.into()))
-            )),
+                .collect::<HashMap<_, _>>()
+            ),
        }
    }
 }
--- a/sn0int-std/src/sockets/mod.rs
+++ b/sn0int-std/src/sockets/mod.rs
@@ -107,7 +107,7 @@ impl Stream {
        let mut errors = Vec::new();

        for addr in addrs {
-            match Stream::connect_addr(host, (addr, port).into(), &options) {
+            match Stream::connect_addr(host, (addr, port).into(), options) {
                Ok(socket) => {
                    return Ok(socket);
                },
@@ -233,7 +233,7 @@ impl Socket {
    }

    pub fn send(&mut self, data: &[u8]) -> Result<()> {
-        match str::from_utf8(&data) {
+        match str::from_utf8(data) {
            Ok(data) => debug!("send: {:?}", data),
            Err(_) => debug!("send: {:?}", data),
        };
--- a/sn0int-std/src/sockets/tls.rs
+++ b/sn0int-std/src/sockets/tls.rs
@@ -47,9 +47,9 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
    }

    let dns_name = if let Some(v) = &options.sni_value {
-        get_dns_name(&mut config, &v)
+        get_dns_name(&mut config, v)
    } else {
-        get_dns_name(&mut config, &host)
+        get_dns_name(&mut config, host)
    };

    let config = Arc::new(config);
@@ -58,7 +58,7 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
 }

 fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
-    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(&host) {
+    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(host) {
        debug!("setting sni value to: {:?}", host);
        name.to_owned()
    } else {
--- a/sn0int-std/src/web.rs
+++ b/sn0int-std/src/web.rs
@@ -65,21 +65,21 @@ impl HttpSession {

 #[derive(Debug, Default, Deserialize)]
 pub struct RequestOptions {
-    query: Option<HashMap<String, String>>,
-    headers: Option<HashMap<String, String>>,
-    basic_auth: Option<(String, String)>,
-    user_agent: Option<String>,
-    json: Option<serde_json::Value>,
-    form: Option<serde_json::Value>,
+    pub query: Option<HashMap<String, String>>,
+    pub headers: Option<HashMap<String, String>>,
+    pub basic_auth: Option<(String, String)>,
+    pub user_agent: Option<String>,
+    pub json: Option<serde_json::Value>,
+    pub form: Option<serde_json::Value>,
    #[serde(default)]
-    follow_redirects: usize,
-    body: Option<String>,
-    timeout: Option<u64>,
+    pub follow_redirects: usize,
+    pub body: Option<String>,
+    pub timeout: Option<u64>,
    #[serde(default)]
-    into_blob: bool,
-    proxy: Option<SocketAddr>,
+    pub into_blob: bool,
+    pub proxy: Option<SocketAddr>,
    #[serde(default)]
-    binary: bool,
+    pub binary: bool,
 }

 impl RequestOptions {
@@ -110,11 +110,8 @@ pub struct HttpRequest {
 }

 impl HttpRequest {
-    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions, default_agent: fn() -> String) -> HttpRequest {
+    pub fn new(session: &HttpSession, method: String, url: String, user_agent: String, options: RequestOptions) -> HttpRequest {
        let cookies = session.cookies.clone();
-
-        let user_agent = options.user_agent.unwrap_or_else(default_agent);
-
        let timeout = options.timeout.map(Duration::from_millis);

        let mut request = HttpRequest {
@@ -216,7 +213,7 @@ impl HttpRequest {
            if self.follow_redirects > 0 && res.status >= 300 && res.status < 400 {
                if let Some(location) = res.headers.get("location") {
                    let base = Url::parse(&url.to_string())?;
-                    let joined = base.join(&location)?;
+                    let joined = base.join(location)?;
                    url = joined.to_string().parse()?;

                    req = self.mkrequest("GET", &url).body(Body::empty())?;
@@ -318,9 +315,9 @@ impl HttpRequest {
    }
 }

-impl Into<AnyLuaValue> for HttpRequest {
-    fn into(self) -> AnyLuaValue {
-        let v = serde_json::to_value(&self).unwrap();
+impl From<HttpRequest> for AnyLuaValue {
+    fn from(req: HttpRequest) -> AnyLuaValue {
+        let v = serde_json::to_value(&req).unwrap();
        LuaJsonValue::from(v).into()
    }
 }
--- a/sn0int-std/src/websockets.rs
+++ b/sn0int-std/src/websockets.rs
@@ -91,9 +91,11 @@ impl WebSocket {
    }

    pub fn options(&self, options: &WebSocketOptions) -> Result<()> {
-        let mut o = SocketOptions::default();
-        o.read_timeout = options.read_timeout;
-        o.write_timeout = options.write_timeout;
+        let o = SocketOptions {
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+            ..Default::default()
+        };
        o.apply(self.sock.get_ref())
    }

--- a/src/args.rs
+++ b/src/args.rs
@@ -9,7 +9,7 @@ use sn0int_common::ModuleID;
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
    /// Select a different workspace instead of the default
-    #[structopt(short="w", long="workspace")]
+    #[structopt(short="w", long="workspace", env="SN0INT_WORKSPACE")]
    pub workspace: Option<Workspace>,

    #[structopt(subcommand)]
@@ -97,18 +97,11 @@ pub enum SubCommand {

 #[derive(Debug, StructOpt)]
 pub struct Run {
-    /// Execute a module that has been installed
-    pub module: String,
+    #[structopt(flatten)]
+    pub run: cmd::run_cmd::Args,
    /// Run a module from a path
    #[structopt(short="f", long="file")]
    pub file: bool,
-    /// Run modules concurrently
-    #[structopt(short="j", long="threads", default_value="1")]
-    pub threads: usize,
-    /// Verbose logging, once to print inserts even if they don't add new
-    /// data, twice to activate the debug() function
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    pub verbose: u64,
    /// Expose stdin to modules
    #[structopt(long="stdin")]
    pub stdin: bool,
--- a/src/autonoscope/domain.rs
+++ b/src/autonoscope/domain.rs
@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;

@@ -94,8 +94,8 @@ impl RulePrecision for DomainRule {
    }
 }

-impl IntoRule for DomainRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for DomainRule {
+    fn to_rule(&self) -> (&'static str, String) {
        ("domain", self.to_string())
    }
 }
--- a/src/autonoscope/ip.rs
+++ b/src/autonoscope/ip.rs
@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 use std::net;
@@ -80,8 +80,8 @@ impl RulePrecision for IpRule {
    }
 }

-impl IntoRule for IpRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for IpRule {
+    fn to_rule(&self) -> (&'static str, String) {
        ("ip", self.to_string())
    }
 }
--- a/src/autonoscope/mod.rs
+++ b/src/autonoscope/mod.rs
@@ -112,15 +112,15 @@ impl RuleSet {
        match obj {
            RuleType::Domain => {
                self.domains.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
            },
            RuleType::Ip => {
                self.ips.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
            },
            RuleType::Url => {
                self.urls.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
            },
        }
        Ok(())
@@ -144,9 +144,9 @@ impl RuleSet {
    }

    #[inline]
-    fn push_rules_display<T: IntoRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
+    fn push_rules_display<T: ToRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
        for rule in rules {
-            let (object, value) = rule.into_rule();
+            let (object, value) = rule.to_rule();
            output.push((object, value, rule.scoped));
        }
    }
@@ -159,10 +159,8 @@ impl RuleSet {
            Insert::Url(url) => {
                if let Some(result) = Self::matches_any(&self.domains, url)? {
                    Some(result)
-                } else if let Some(result) = Self::matches_any(&self.urls, url)? {
-                    Some(result)
                } else {
-                    None
+                    Self::matches_any(&self.urls, url)?
                }
            },
            // Insert::Email(email) => unimplemented!(),
@@ -176,7 +174,7 @@ impl RuleSet {

    fn matches_any<T1, T2>(rules: &[Rule<T1>], object: &T2) -> Result<Option<bool>>
        where T1: AutoRule<T2>,
-            T1: IntoRule,
+            T1: ToRule,
    {
        for rule in rules {
            if rule.matches(object)? {
@@ -238,12 +236,12 @@ pub trait RulePrecision {
 }

 #[derive(Debug, PartialEq)]
-pub struct Rule<T: IntoRule> {
+pub struct Rule<T: ToRule> {
    rule: T,
    scoped: bool,
 }

-impl<T: IntoRule> Rule<T> {
+impl<T: ToRule> Rule<T> {
    pub fn new(rule: T, scoped: bool) -> Rule<T> {
        Rule {
            rule,
@@ -254,7 +252,7 @@ impl<T: IntoRule> Rule<T> {

 // TODO: maybe drop this
 use std::ops::Deref;
-impl<T: IntoRule> Deref for Rule<T> {
+impl<T: ToRule> Deref for Rule<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
@@ -262,23 +260,23 @@ impl<T: IntoRule> Deref for Rule<T> {
    }
 }

-impl<T: IntoRule + RulePrecision> RulePrecision for Rule<T> {
+impl<T: ToRule + RulePrecision> RulePrecision for Rule<T> {
    fn precision(&self) -> usize {
        self.rule.precision()
    }
 }

-pub trait IntoRule {
-    fn into_rule(&self) -> (&'static str, String);
+pub trait ToRule {
+    fn to_rule(&self) -> (&'static str, String);
 }

-impl<T: IntoRule> Into<NewAutonoscope> for &Rule<T> {
-    fn into(self) -> NewAutonoscope {
-        let (object, value) = self.rule.into_rule();
+impl<T: ToRule> From<&Rule<T>> for NewAutonoscope {
+    fn from(rule: &Rule<T>) -> NewAutonoscope {
+        let (object, value) = rule.rule.to_rule();
        NewAutonoscope {
            object: object.to_string(),
            value,
-            scoped: self.scoped,
+            scoped: rule.scoped,
        }
    }
 }
--- a/src/autonoscope/url.rs
+++ b/src/autonoscope/url.rs
@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;

@@ -86,8 +86,8 @@ impl RulePrecision for UrlRule {
    }
 }

-impl IntoRule for UrlRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for UrlRule {
+    fn to_rule(&self) -> (&'static str, String) {
        ("url", self.to_string())
    }
 }
--- a/src/cal/date.rs
+++ b/src/cal/date.rs
@@ -112,7 +112,7 @@ pub struct DateContext {

 impl DateContext {
    pub fn new(events: &[Activity], today: NaiveDate) -> DateContext {
-        let (events, max) = setup_graph_map(&events);
+        let (events, max) = setup_graph_map(events);
        DateContext {
            events,
            max,
@@ -296,25 +296,25 @@ mod tests {
        DateContext {
            events: HashMap::new(),
            max: 0,
-            today: NaiveDate::from_ymd(2020, 05, 30),
+            today: NaiveDate::from_ymd(2020, 5, 30),
        }
    }

    #[test]
    fn test_days_in_month_2020_05() {
-        let days = days_in_month(2020, 05);
+        let days = days_in_month(2020, 5);
        assert_eq!(days, 31);
    }

    #[test]
    fn test_days_in_month_2020_04() {
-        let days = days_in_month(2020, 04);
+        let days = days_in_month(2020, 4);
        assert_eq!(days, 30);
    }

    #[test]
    fn test_days_in_month_2020_02() {
-        let days = days_in_month(2020, 02);
+        let days = days_in_month(2020, 2);
        assert_eq!(days, 29);
    }

@@ -324,15 +324,15 @@ mod tests {
        let ctx = DateContext {
            events,
            max: 0,
-            today: NaiveDate::from_ymd(2020, 06, 06),
+            today: NaiveDate::from_ymd(2020, 6, 6),
        };
-        let grade = ctx.activity_for_day(&NaiveDate::from_ymd(2020, 06, 06));
+        let grade = ctx.activity_for_day(&NaiveDate::from_ymd(2020, 6, 6));
        assert_eq!(grade, ActivityGrade::None);
    }

    #[test]
    fn test_datespec_year_month() {
-        let ds = DateSpec::YearMonth((2020, 05));
+        let ds = DateSpec::YearMonth((2020, 5));
        let out = ds.to_term_string(&context());
        assert_eq!(out, "      May 2020       
 Su Mo Tu We Th Fr Sa
--- a/src/cal/time.rs
+++ b/src/cal/time.rs
@@ -68,7 +68,7 @@ pub struct DateTimeContext {

 impl DateTimeContext {
    pub fn new(events: &[Activity], now: NaiveDateTime, slice_width: u32, slice_duration: u32) -> DateTimeContext {
-        let (events, max) = setup_graph_map(&events, slice_duration);
+        let (events, max) = setup_graph_map(events, slice_duration);
        DateTimeContext {
            events,
            max,
--- a/src/cmd/add_cmd.rs
+++ b/src/cmd/add_cmd.rs
@@ -290,7 +290,7 @@ impl InsertFromString for AddUrl {
        let subdomain = parts.domain()
            .ok_or_else(|| format_err!("url doesn't have a domain host"))?;

-        let dns_name = rl.psl()?.parse_dns_name(&subdomain)
+        let dns_name = rl.psl()?.parse_dns_name(subdomain)
            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;

        let domain_id = match rl.db().insert_struct(NewDomain {
--- a/src/cmd/delete_cmd.rs
+++ b/src/cmd/delete_cmd.rs
@@ -19,20 +19,20 @@ pub struct Args {
 impl Cmd for Args {
    fn run(self, rl: &mut Shell) -> Result<()> {
        let rows = match &self.subcommand {
-            Target::Domains(filter) => delete::<Domain>(rl, &filter),
-            Target::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
-            Target::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
-            Target::Urls(filter) => delete::<Url>(rl, &filter),
-            Target::Emails(filter) => delete::<Email>(rl, &filter),
-            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
-            Target::Devices(filter) => delete::<Device>(rl, &filter),
-            Target::Networks(filter) => delete::<Network>(rl, &filter),
-            Target::Accounts(filter) => delete::<Account>(rl, &filter),
-            Target::Breaches(filter) => delete::<Breach>(rl, &filter),
-            Target::Images(filter) => delete::<Image>(rl, &filter),
-            Target::Ports(filter) => delete::<Port>(rl, &filter),
-            Target::Netblocks(filter) => delete::<Netblock>(rl, &filter),
-            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, &filter),
+            Target::Domains(filter) => delete::<Domain>(rl, filter),
+            Target::Subdomains(filter) => delete::<Subdomain>(rl, filter),
+            Target::IpAddrs(filter) => delete::<IpAddr>(rl, filter),
+            Target::Urls(filter) => delete::<Url>(rl, filter),
+            Target::Emails(filter) => delete::<Email>(rl, filter),
+            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, filter),
+            Target::Devices(filter) => delete::<Device>(rl, filter),
+            Target::Networks(filter) => delete::<Network>(rl, filter),
+            Target::Accounts(filter) => delete::<Account>(rl, filter),
+            Target::Breaches(filter) => delete::<Breach>(rl, filter),
+            Target::Images(filter) => delete::<Image>(rl, filter),
+            Target::Ports(filter) => delete::<Port>(rl, filter),
+            Target::Netblocks(filter) => delete::<Netblock>(rl, filter),
+            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, filter),
        }?;
        term::info(&format!("Deleted {} rows", rows));
        Ok(())
--- a/src/cmd/pkg_cmd.rs
+++ b/src/cmd/pkg_cmd.rs
@@ -137,7 +137,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->

            for module in library.list() {
                if let Some(source) = &list.source {
-                    if !module.source_equals(&source) {
+                    if !module.source_equals(source) {
                        continue;
                    }
                }
@@ -148,7 +148,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
                }

                if let Some(stealth) = &list.stealth {
-                    if !module.stealth().equal_or_better(&stealth) {
+                    if !module.stealth().equal_or_better(stealth) {
                        continue;
                    }
                }
@@ -163,17 +163,17 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
            Ok(ModuleReload::No)
        },
        SubCommand::Install(install) => {
-            registry::run_install(install, &config)?;
+            registry::run_install(install, config)?;
            // trigger reload
            Ok(ModuleReload::Yes)
        },
        SubCommand::Search(search) => {
-            registry::run_search(library, &search, &config)?;
+            registry::run_search(library, &search, config)?;
            Ok(ModuleReload::No)
        },
        SubCommand::Update(_) => {
            let mut autoupdate = AutoUpdater::load()?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let updater = Arc::new(Updater::new(config)?);

            let modules = library.list()
                .into_iter()
@@ -199,14 +199,14 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
            Ok(ModuleReload::Yes)
        },
        SubCommand::Uninstall(uninstall) => {
-            let updater = Updater::new(&config)?;
+            let updater = Updater::new(config)?;
            updater.uninstall(&uninstall.module)?;
            // trigger reload
            Ok(ModuleReload::Yes)
        },
        SubCommand::Quickstart => {
-            let client = Client::new(&config)?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let client = Client::new(config)?;
+            let updater = Arc::new(Updater::new(config)?);
            let mut autoupdate = AutoUpdater::load()?;

            let installed = library.list()
@@ -250,7 +250,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->

 impl LiteCmd for Args {
    fn run(self, config: &Config) -> Result<()> {
-        let library = Library::new(false, &config)?;
+        let library = Library::new(false, config)?;
        run_subcommand(self.subcommand, &library, config)?;
        Ok(())
    }
--- a/src/cmd/run_cmd.rs
+++ b/src/cmd/run_cmd.rs
@@ -17,6 +17,7 @@ use serde::Serialize;
 use sn0int_common::metadata::Source;
 use sn0int_std::ratelimits::Ratelimiter;
 use std::collections::HashMap;
+use std::net::SocketAddr;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;

@@ -24,14 +25,26 @@ use structopt::clap::AppSettings;
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
-    #[structopt(short="j", long="threads", default_value="1")]
-    threads: usize,
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    verbose: u64,
+    /// Execute a module that has been installed
+    pub module: Option<String>,
+    /// Run investigations concurrently
+    #[structopt(short="j", default_value="1")]
+    pub threads: usize,
+    /// Verbose logging, once to print inserts even if they don't add new
+    /// data, twice to activate the debug() function
+    #[structopt(short="v", long, parse(from_occurrences))]
+    pub verbose: u64,
+    /// Set a specific socks5 proxy to use
+    #[structopt(short="X", long)]
+    pub proxy: Option<SocketAddr>,
+    /// Set a different default user agent
+    #[structopt(long)]
+    pub user_agent: Option<String>,
 }

 #[derive(Debug, Clone)]
 pub struct Params<'a> {
+    pub module: Option<&'a String>,
    pub threads: usize,
    pub verbose: u64,
    pub stdin: bool,
@@ -39,25 +52,61 @@ pub struct Params<'a> {
    pub grant_full_keyring: bool,
    pub deny_keyring: bool,
    pub exit_on_error: bool,
+    pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<&'a String>,
+}
+
+impl<'a> Params<'a> {
+    pub fn get_module(&self, rl: &Shell) -> Result<Module> {
+        let module = if let Some(module) = self.module {
+            rl.library().get(module)?
+                .clone()
+        } else {
+            rl.module()
+                .map(|m| m.to_owned())
+                .ok_or_else(|| format_err!("No module selected"))?
+        };
+        Ok(module)
+    }
+
+    pub fn get_proxy(&self, rl: &Shell) -> Option<SocketAddr> {
+        if self.proxy.is_some() {
+            self.proxy.clone()
+        } else {
+            rl.config().network.proxy
+        }
+    }
+
+    pub fn get_user_agent(&self, rl: &Shell) -> Option<String> {
+        if let Some(user_agent) = self.user_agent {
+            Some(user_agent.to_string())
+        } else {
+            rl.config().network.user_agent.clone()
+        }
+    }
 }

 impl<'a> From<&'a args::Run> for Params<'a> {
    fn from(args: &args::Run) -> Params {
        Params {
-            threads: args.threads,
-            verbose: args.verbose,
+            module: args.run.module.as_ref(),
+            threads: args.run.threads,
+            verbose: args.run.verbose,
            stdin: args.stdin,
            grants: &args.grants,
            grant_full_keyring: args.grant_full_keyring,
            deny_keyring: args.deny_keyring,
            exit_on_error: args.exit_on_error,
+            proxy: args.run.proxy,
+            user_agent: args.run.user_agent.as_ref(),
        }
    }
 }

-impl From<Args> for Params<'static> {
-    fn from(args: Args) -> Params<'static> {
+impl<'a> From<&'a Args> for Params<'a> {
+    fn from(args: &Args) -> Params {
        Params {
+            module: args.module.as_ref(),
            threads: args.threads,
            verbose: args.verbose,
            stdin: false,
@@ -65,6 +114,8 @@ impl From<Args> for Params<'static> {
            grant_full_keyring: false,
            deny_keyring: false,
            exit_on_error: false,
+            proxy: args.proxy,
+            user_agent: args.user_agent.as_ref(),
        }
    }
 }
@@ -93,7 +144,7 @@ fn prepare_args<T: Scopable + Serialize + Model>(rl: &Shell, filter: &Filter, pa
 }

 pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
-    for namespace in keyring.unauthorized_namespaces(&module) {
+    for namespace in keyring.unauthorized_namespaces(module) {
        let grant_access = if params.deny_keyring {
            false
        } else if params.grant_full_keyring || params.grants.contains(namespace) {
@@ -104,7 +155,7 @@ pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params)
        };

        if grant_access {
-            keyring.grant_access(&module, namespace.to_string());
+            keyring.grant_access(module, namespace.to_string());
            term::info(&format!("Granted access to {:?}", namespace));
        }
    }
@@ -136,8 +187,8 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
        Some(Source::Notifications) => bail!("Notification modules can't be executed like this"),
        Some(Source::KeyRing(namespace)) => {
            let keyring = rl.keyring();
-            if keyring.is_access_granted(&module, &namespace) {
-                keyring.get_all_for(&namespace).into_iter()
+            if keyring.is_access_granted(module, namespace) {
+                keyring.get_all_for(namespace).into_iter()
                    .map(|key| {
                        let pretty = format!("{}:{}", key.namespace, key.access_key);
                        let arg = serde_json::to_value(key)?;
@@ -153,15 +204,14 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
 }

 pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);

    prepare_keyring(rl.keyring_mut(), &module, &params)?;
    let keyring = rl.keyring().request_keys(&module);

    let dns_config = Resolver::from_system_v4()?;
-    let proxy = rl.config().network.proxy;

    let args = get_args(rl, &module)?;
    for (arg, _pretty_arg, blobs) in args {
@@ -169,6 +219,7 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
                                          keyring.clone(),
                                          dns_config.clone(),
                                          proxy,
+                                          user_agent.clone(),
                                          options.clone(),
                                          module.clone(),
                                          arg,
@@ -181,15 +232,15 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
 }

 pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);

    prepare_keyring(rl.keyring_mut(), &module, &params)?;
    let args = get_args(rl, &module)?;

    rl.signal_register().catch_ctrl();
-    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, rl.config().network.proxy, options);
+    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, proxy, user_agent, options);
    rl.signal_register().reset_ctrlc();

    if errors > 0 {
@@ -212,6 +263,6 @@ impl Cmd for Args {
            Some(options) => options.clone(),
            _ => HashMap::new(),
        };
-        execute(rl, self.into(), options)
+        execute(rl, Params::from(&self), options)
    }
 }
--- a/src/cmd/select_cmd.rs
+++ b/src/cmd/select_cmd.rs
@@ -106,20 +106,20 @@ impl Cmd for Args {
        let printer = Printer::new(rl, &self);

        match &self.subcommand {
-            Target::Domains(filter) => printer.select::<Domain>(&filter),
-            Target::Subdomains(filter) => printer.select::<Subdomain>(&filter),
-            Target::IpAddrs(filter) => printer.select::<IpAddr>(&filter),
-            Target::Urls(filter) => printer.select::<Url>(&filter),
-            Target::Emails(filter) => printer.select::<Email>(&filter),
-            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(&filter),
-            Target::Devices(filter) => printer.select::<Device>(&filter),
-            Target::Networks(filter) => printer.select::<Network>(&filter),
-            Target::Accounts(filter) => printer.select::<Account>(&filter),
-            Target::Breaches(filter) => printer.select::<Breach>(&filter),
-            Target::Images(filter) => printer.select::<Image>(&filter),
-            Target::Ports(filter) => printer.select::<Port>(&filter),
-            Target::Netblocks(filter) => printer.select::<Netblock>(&filter),
-            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(&filter),
+            Target::Domains(filter) => printer.select::<Domain>(filter),
+            Target::Subdomains(filter) => printer.select::<Subdomain>(filter),
+            Target::IpAddrs(filter) => printer.select::<IpAddr>(filter),
+            Target::Urls(filter) => printer.select::<Url>(filter),
+            Target::Emails(filter) => printer.select::<Email>(filter),
+            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(filter),
+            Target::Devices(filter) => printer.select::<Device>(filter),
+            Target::Networks(filter) => printer.select::<Network>(filter),
+            Target::Accounts(filter) => printer.select::<Account>(filter),
+            Target::Breaches(filter) => printer.select::<Breach>(filter),
+            Target::Images(filter) => printer.select::<Image>(filter),
+            Target::Ports(filter) => printer.select::<Port>(filter),
+            Target::Netblocks(filter) => printer.select::<Netblock>(filter),
+            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(filter),
        }
    }
 }
--- a/src/cmd/stats_cmd.rs
+++ b/src/cmd/stats_cmd.rs
@@ -89,21 +89,21 @@ impl Stats {
    fn count(workspace: String, db: &Database) -> Result<Stats> {
        Ok(Stats {
            workspace,
-            domains: count_models::<Domain>(&db)?,
-            subdomains: count_models::<Subdomain>(&db)?,
-            ipaddrs: count_models::<IpAddr>(&db)?,
-            urls: count_models::<Url>(&db)?,
-            emails: count_models::<Email>(&db)?,
-            phonenumbers: count_models::<PhoneNumber>(&db)?,
-            devices: count_models::<Device>(&db)?,
-            networks: count_models::<Network>(&db)?,
-            accounts: count_models::<Account>(&db)?,
-            breaches: count_models::<Breach>(&db)?,
-            images: count_models::<Image>(&db)?,
-            ports: count_models::<Port>(&db)?,
-            netblocks: count_models::<Netblock>(&db)?,
-            cryptoaddrs: count_models::<CryptoAddr>(&db)?,
-            activity: Activity::count(&db)?,
+            domains: count_models::<Domain>(db)?,
+            subdomains: count_models::<Subdomain>(db)?,
+            ipaddrs: count_models::<IpAddr>(db)?,
+            urls: count_models::<Url>(db)?,
+            emails: count_models::<Email>(db)?,
+            phonenumbers: count_models::<PhoneNumber>(db)?,
+            devices: count_models::<Device>(db)?,
+            networks: count_models::<Network>(db)?,
+            accounts: count_models::<Account>(db)?,
+            breaches: count_models::<Breach>(db)?,
+            images: count_models::<Image>(db)?,
+            ports: count_models::<Port>(db)?,
+            netblocks: count_models::<Netblock>(db)?,
+            cryptoaddrs: count_models::<CryptoAddr>(db)?,
+            activity: Activity::count(db)?,
            blobs: None,
        })
    }
@@ -162,7 +162,7 @@ impl Cmd for Args {
                println!("{:>41}", workspace.bold());
            }

-            let mut stats = Stats::count(workspace.into(), &db)?;
+            let mut stats = Stats::count(workspace.into(), db)?;
            if !self.short {
                stats.add_blob_usage(rl.blobs())?;
            }
--- a/src/cmd/target_cmd.rs
+++ b/src/cmd/target_cmd.rs
@@ -81,7 +81,7 @@ fn count_selected(rl: &mut Shell, source: &Source) -> Result<usize> {
        Source::Netblocks => db.filter::<Netblock>(&filter)?.len(),
        Source::CryptoAddrs(currency) => db.filter_with_param::<CryptoAddr>(&filter, currency.as_ref())?.len(),
        Source::Notifications => bail!("Notifications can't be set as target"),
-        Source::KeyRing(namespace) => rl.keyring().list_for(&namespace).len(),
+        Source::KeyRing(namespace) => rl.keyring().list_for(namespace).len(),
    };
    Ok(num)
 }
--- a/src/config.rs
+++ b/src/config.rs
@@ -75,4 +75,5 @@ fn default_registry() -> String {
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct NetworkConfig {
    pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
 }
--- a/src/db/mod.rs
+++ b/src/db/mod.rs
@@ -198,14 +198,14 @@ impl Database {

            let update = obj.upsert(&existing);
            if update.is_dirty() {
-                update.apply(&self)?;
+                update.apply(self)?;
                Ok(Some((DbChange::Update(update.generic()), existing.id())))
            } else {
                Ok(Some((DbChange::None, existing.id())))
            }
        } else {
            obj.set_scoped(scoped);
-            obj.insert(&self)?;
+            obj.insert(self)?;
            let id = M::get_id(self, obj.value())?;
            Ok(Some((DbChange::Insert, id)))
        }
@@ -218,7 +218,7 @@ impl Database {
                return Ok(false);
            }
        }
-        obj.insert(&self)?;
+        obj.insert(self)?;
        Ok(true)
    }

@@ -254,7 +254,7 @@ impl Database {

            let update = obj.upsert(&existing);
            if update.is_dirty() {
-                update.apply(&self)?;
+                update.apply(self)?;
                Ok(Some((DbChange::Update(update.generic()), id)))
            } else {
                Ok(Some((DbChange::None, id)))
@@ -402,7 +402,7 @@ impl Database {
    }

    fn get_opt_typed<T: Model + Scopable>(&self, value: &T::ID) -> Result<Option<i32>> {
-        match T::get_opt(self, &value)? {
+        match T::get_opt(self, value)? {
            Some(ref obj) if obj.scoped() => Ok(Some(obj.id())),
            _ => Ok(None),
        }
@@ -410,23 +410,23 @@ impl Database {

    pub fn get_opt(&self, family: &Family, value: &str) -> Result<Option<i32>> {
        match family {
-            Family::Domain => self.get_opt_typed::<Domain>(&value),
-            Family::Subdomain => self.get_opt_typed::<Subdomain>(&value),
-            Family::Ipaddr => self.get_opt_typed::<IpAddr>(&value),
+            Family::Domain => self.get_opt_typed::<Domain>(value),
+            Family::Subdomain => self.get_opt_typed::<Subdomain>(value),
+            Family::Ipaddr => self.get_opt_typed::<IpAddr>(value),
            Family::SubdomainIpaddr => bail!("Unsupported operation"),
-            Family::Url => self.get_opt_typed::<Url>(&value),
-            Family::Email => self.get_opt_typed::<Email>(&value),
-            Family::Phonenumber => self.get_opt_typed::<PhoneNumber>(&value),
-            Family::Device => self.get_opt_typed::<Device>(&value),
-            Family::Network => self.get_opt_typed::<Network>(&value),
+            Family::Url => self.get_opt_typed::<Url>(value),
+            Family::Email => self.get_opt_typed::<Email>(value),
+            Family::Phonenumber => self.get_opt_typed::<PhoneNumber>(value),
+            Family::Device => self.get_opt_typed::<Device>(value),
+            Family::Network => self.get_opt_typed::<Network>(value),
            Family::NetworkDevice => bail!("Unsupported operation"),
-            Family::Account => self.get_opt_typed::<Account>(&value),
-            Family::Breach => self.get_opt_typed::<Breach>(&value),
+            Family::Account => self.get_opt_typed::<Account>(value),
+            Family::Breach => self.get_opt_typed::<Breach>(value),
            Family::BreachEmail => bail!("Unsupported operation"),
-            Family::Image => self.get_opt_typed::<Image>(&value),
-            Family::Port => self.get_opt_typed::<Port>(&value),
-            Family::Netblock => self.get_opt_typed::<Netblock>(&value),
-            Family::Cryptoaddr => self.get_opt_typed::<CryptoAddr>(&value),
+            Family::Image => self.get_opt_typed::<Image>(value),
+            Family::Port => self.get_opt_typed::<Port>(value),
+            Family::Netblock => self.get_opt_typed::<Netblock>(value),
+            Family::Cryptoaddr => self.get_opt_typed::<CryptoAddr>(value),
        }
    }

--- a/src/db/ttl.rs
+++ b/src/db/ttl.rs
@@ -135,7 +135,7 @@ pub fn reap_expired(rl: &mut Shell) -> Result<()> {

        let subject = format!("Deleted {} {:?}", &expired.family, &expired.value);
        let topic = &format!("db:{}:{}:delete", &expired.family, &expired.value);
-        if let Err(err) = notify::trigger_notify_event(rl, &mut Term, &mut ratelimit, &topic, &Notification {
+        if let Err(err) = notify::trigger_notify_event(rl, &mut Term, &mut ratelimit, topic, &Notification {
            subject,
            body: None,
        }) {
--- a/src/engine/ctx.rs
+++ b/src/engine/ctx.rs
@@ -215,6 +215,7 @@ pub struct LuaState {
    geoip: Option<Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>>,
    asn: Option<Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>>,
    proxy: Option<SocketAddr>,
+    user_agent: Option<String>,
    options: HashMap<String, String>,
 }

@@ -372,7 +373,15 @@ impl State for LuaState {
        let mtx = self.http_sessions.lock().unwrap();
        let session = mtx.get(session_id).expect("Invalid session reference"); // TODO

-        HttpRequest::new(&session, method, url, options, || format!("sn0int/{}", env!("CARGO_PKG_VERSION")))
+        let user_agent = if let Some(user_agent) = &options.user_agent {
+            user_agent.to_string()
+        } else if let Some(user_agent) = &self.user_agent {
+            user_agent.to_string()
+        } else {
+            format!("sn0int/{}", env!("CARGO_PKG_VERSION"))
+        };
+
+        HttpRequest::new(session, method, url, user_agent, options)
    }

    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
@@ -472,6 +481,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
        geoip,
        asn,
        proxy: env.proxy,
+        user_agent: env.user_agent,
        options: env.options,
    });

@@ -517,6 +527,9 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
    runtime::http_fetch_json(&mut lua, state.clone());
    runtime::img_exif(&mut lua, state.clone());
    runtime::img_load(&mut lua, state.clone());
+    runtime::img_ahash(&mut lua, state.clone());
+    runtime::img_dhash(&mut lua, state.clone());
+    runtime::img_phash(&mut lua, state.clone());
    runtime::img_nudity(&mut lua, state.clone());
    runtime::info(&mut lua, state.clone());
    runtime::intval(&mut lua, state.clone());
@@ -662,6 +675,7 @@ impl Script {
        let keyring = Vec::new();
        let dns_config = Resolver::from_system_v4()?;
        let proxy = None;
+        let user_agent = None;
        let psl = PslReader::String(r#"
 // ===BEGIN ICANN DOMAINS===
 com
@@ -679,6 +693,7 @@ a.prod.fastly.net
            keyring,
            dns_config,
            proxy,
+            user_agent,
            options: HashMap::new(),
            blobs: Vec::new(),
            psl,
--- a/src/engine/mod.rs
+++ b/src/engine/mod.rs
@@ -34,6 +34,7 @@ pub struct Environment {
    pub keyring: Vec<KeyRingEntry>,
    pub dns_config: Resolver,
    pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
    pub options: HashMap<String, String>,
    pub blobs: Vec<Blob>,
    pub psl: PslReader,
@@ -120,14 +121,14 @@ impl<'a> Library<'a> {
                folder.to_owned()
            } else {
                let folder = folder.strip_prefix("~/")
-                    .unwrap_or(&folder);
+                    .unwrap_or(folder);

                dirs_next::home_dir()
                    .ok_or_else(|| format_err!("Failed to find home folder"))?
                    .join(folder)
            };

-            self.load_module_folder(&folder, &author, true)?;
+            self.load_module_folder(&folder, author, true)?;
        }

        Ok(())
@@ -156,7 +157,7 @@ impl<'a> Library<'a> {
                continue;
            }

-            if let Err(err) = self.load_single_module(&module.path(), &author_name, &module_name, private_modules) {
+            if let Err(err) = self.load_single_module(&module.path(), author_name, module_name, private_modules) {
                let root = err.find_root_cause();
                term::warn(&format!("Failed to load {}/{}: {}", author_name, module_name, root));
            }
@@ -167,7 +168,7 @@ impl<'a> Library<'a> {

    pub fn load_single_module(&mut self, path: &Path, author_name: &str, module_name: &str, private_module: bool) -> Result<()> {
        let module_name = module_name.to_string();
-        let module = Module::load(path, &author_name, &module_name, private_module)
+        let module = Module::load(path, author_name, &module_name, private_module)
            .context(format!("Failed to parse {}/{}", author_name, module_name))?;

        for key in &[&module_name, &format!("{}/{}", author_name, module_name)] {
@@ -332,7 +333,7 @@ impl Module {
    pub fn source_equals(&self, other: &str) -> bool {
        match self.source() {
            Some(source) => source.group_as_str() == other,
-            None => other == "",
+            None => other.is_empty(),
        }
    }
 }
--- a/src/ipc/child.rs
+++ b/src/ipc/child.rs
@@ -86,6 +86,7 @@ pub fn run(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslRea
        keyring: start.keyring,
        dns_config: start.dns_config,
        proxy: start.proxy,
+        user_agent: start.user_agent,
        options: start.options,
        blobs: start.blobs,
        psl,
--- a/src/ipc/common.rs
+++ b/src/ipc/common.rs
@@ -12,6 +12,7 @@ pub struct StartCommand {
    pub keyring: Vec<KeyRingEntry>,
    pub dns_config: Resolver,
    pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
    pub options: HashMap<String, String>,
    pub module: Module,
    pub arg: serde_json::Value,
@@ -23,6 +24,7 @@ impl StartCommand {
               keyring: Vec<KeyRingEntry>,
               dns_config: Resolver,
               proxy: Option<SocketAddr>,
+               user_agent: Option<String>,
               options: HashMap<String, String>,
               module: Module,
               arg: serde_json::Value,
@@ -33,6 +35,7 @@ impl StartCommand {
            keyring,
            dns_config,
            proxy,
+            user_agent,
            options,
            module,
            arg,
--- a/src/ipc/parent.rs
+++ b/src/ipc/parent.rs
@@ -105,6 +105,7 @@ pub fn run(module: Module,
           verbose: u64,
           has_stdin: bool,
           proxy: Option<SocketAddr>,
+           user_agent: Option<String>,
           options: HashMap<String, String>,
           blobs: Vec<Blob>,
 ) -> Result<ExitEvent> {
@@ -117,15 +118,15 @@ pub fn run(module: Module,
    };

    let mut ipc_parent = IpcParent::setup(&module)?;
-    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
+    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, user_agent, options, module, arg, blobs))?;

    let exit = loop {
        match ipc_parent.recv()? {
            Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => ipc_parent.send_event_callback(*object, &tx),
+            Event::Database(object) => ipc_parent.send_event_callback(*object, tx),
            Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
-            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
-            Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),
+            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, tx),
+            Event::Blob(blob) => ipc_parent.send_event_callback(blob, tx),
            Event::Exit(event) => {
                if let ExitEvent::Err(err) = &event {
                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
--- a/src/keyring.rs
+++ b/src/keyring.rs
@@ -170,7 +170,7 @@ impl KeyRing {

    pub fn unauthorized_namespaces<'a>(&self, module: &'a Module) -> Vec<&'a String> {
        module.keyring_access().iter()
-            .filter(|namespace| !self.is_access_granted(&module, &namespace))
+            .filter(|namespace| !self.is_access_granted(module, namespace))
            .collect()
    }

@@ -192,7 +192,7 @@ impl KeyRing {
    pub fn request_keys(&self, module: &Module) -> Vec<KeyRingEntry> {
        // TODO: we probably want to randomize the order
        module.keyring_access().iter()
-            .filter(|namespace| self.is_access_granted(&module, &namespace))
+            .filter(|namespace| self.is_access_granted(module, namespace))
            .flat_map(|namespace| self.list_for(namespace))
            .flat_map(|x| self.get(&x))
            .collect()
--- a/src/main.rs
+++ b/src/main.rs
@@ -2,6 +2,7 @@ use env_logger::Env;
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
 use sn0int::cmd::{self, LiteCmd};
+use sn0int::cmd::run_cmd::Params;
 use sn0int::config::Config;
 use sn0int::db;
 use sn0int::errors::*;
@@ -23,34 +24,41 @@ use std::path::Path;
 fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
    let mut rl = shell::init(gargs, config, false)?;

+    let module = args.run.module.as_ref()
+        .ok_or_else(|| format_err!("Module is required"))?;
+
    let module = if args.file {
-        let path = Path::new(&args.module);
+        let path = Path::new(&module);

        let filename = path.file_stem()
            .ok_or_else(|| format_err!("Failed to decode filename"))?
            .to_str()
            .ok_or_else(|| format_err!("Failed to decode filename"))?;

-        Module::load(&path.to_path_buf(), "anonymous", &filename, true)
+        Module::load(&path.to_path_buf(), "anonymous", filename, true)
            .context(format!("Failed to parse {:?}", path))?
    } else {
-        rl.library().get(&args.module)?
+        rl.library().get(module)?
            .clone()
    };

    rl.set_module(module);

    if let Some(target) = &args.target {
-        let target = shellwords::split(&target)
+        let target = shellwords::split(target)
            .map_err(|_| format_err!("Failed to parse target quotes"))?;
        let target = db::Filter::parse(&target)?;
        rl.set_target(Some(target));
    }

+    let mut params = Params::from(args);
+    // The module was already set and loaded
+    params.module = None;
+
    if args.dump_sandbox_init_msg {
-        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, args.into(), Opt::collect(&args.options))
+        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, params, Opt::collect(&args.options))
    } else {
-        cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+        cmd::run_cmd::execute(&mut rl, params, Opt::collect(&args.options))
    }
 }

--- a/src/models/account.rs
+++ b/src/models/account.rs
@@ -342,7 +342,7 @@ impl Upsert for AccountUpdate {
    }

    fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_account(&self)
+        db.update_account(self)
    }
 }

--- a/src/models/cryptoaddr.rs
+++ b/src/models/cryptoaddr.rs
@@ -326,7 +326,7 @@ impl Upsert for CryptoAddrUpdate {
    }

    fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_cryptoaddr(&self)
+        db.update_cryptoaddr(self)
    }
 }

--- a/src/models/image.rs
+++ b/src/models/image.rs
@@ -442,7 +442,7 @@ impl Upsert for ImageUpdate {
    }

    fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_image(&self)
+        db.update_image(self)
    }
 }

--- a/src/models/port.rs
+++ b/src/models/port.rs
@@ -324,7 +324,7 @@ impl Upsert for PortUpdate {
    }

    fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_port(&self)
+        db.update_port(self)
    }
 }

--- a/src/models/url.rs
+++ b/src/models/url.rs
@@ -339,7 +339,7 @@ impl Upsert for UrlChangeset {
    }

    fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_url(&self)
+        db.update_url(self)
    }
 }

--- a/src/notify/mod.rs
+++ b/src/notify/mod.rs
@@ -88,6 +88,7 @@ pub fn exec(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, option
    }

    let params = Params {
+        module: None,
        threads: 1,
        verbose,
        stdin: false,
@@ -95,13 +96,15 @@ pub fn exec(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, option
        grant_full_keyring: false,
        deny_keyring: false,
        exit_on_error: false,
+        proxy: None,
+        user_agent: None,
    };

-    prepare_keyring(rl.keyring_mut(), &module, &params)?;
-    let args = vec![prepare_arg(&notification)?];
+    prepare_keyring(rl.keyring_mut(), module, &params)?;
+    let args = vec![prepare_arg(notification)?];

    debug!("Executing notification module {:?}", module_name);
-    let errors = worker::spawn(rl, &module, ratelimit, args, &params, rl.config().network.proxy, options);
+    let errors = worker::spawn(rl, module, ratelimit, args, &params, rl.config().network.proxy, None, options);
    debug!("Notification module {:?} exited with {:?} errors", module_name, errors);

    Ok(errors)
--- a/src/notify/rules.rs
+++ b/src/notify/rules.rs
@@ -18,7 +18,7 @@ impl Glob {

        loop {
            match (filter.next(), topic.next()) {
-                (Some(filter), Some(topic)) => if !filter.matches(&topic) {
+                (Some(filter), Some(topic)) => if !filter.matches(topic) {
                    return None;
                },
                (None, None) => return Some(!self.inverse),
--- a/src/registry.rs
+++ b/src/registry.rs
@@ -23,7 +23,7 @@ pub struct Updater {

 impl Updater {
    pub fn new(config: &Config) -> Result<Updater> {
-        let client = Client::new(&config)?;
+        let client = Client::new(config)?;
        Ok(Updater {
            client,
        })
@@ -97,7 +97,7 @@ pub fn run_publish(_args: &Args, publish: &Publish, config: &Config) -> Result<(
    let session = auth::load_token()
        .context("Failed to load auth token, login first")?;

-    let mut client = Client::new(&config)?;
+    let mut client = Client::new(config)?;
    client.authenticate(session);

    for path in &publish.paths {
@@ -247,7 +247,7 @@ fn write_tag(out: &mut String, color: Color, txt: &str) -> Result<()> {
 }

 pub fn run_search(library: &Library, search: &Search, config: &Config) -> Result<()> {
-    let client = Client::new(&config)?;
+    let client = Client::new(config)?;

    let label = format!("Searching {:?}", search.query);
    let modules = worker::spawn_fn(&label, || {
--- a/src/repl/mod.rs
+++ b/src/repl/mod.rs
@@ -35,10 +35,8 @@ impl<'a> Repl<'a> {

    fn update_globals(&mut self) {
        let mut globals = Vec::new();
-        for item in self.lua.globals_table().iter::<String, AnyLuaValue>() {
-            if let Some((k, _)) = item {
-                globals.push(k);
-            }
+        for (k, _) in self.lua.globals_table().iter::<String, AnyLuaValue>().flatten() {
+            globals.push(k);
        }
        if let Some(helper) = self.rl.helper_mut() {
            debug!("updating globals: {:?}", globals);
@@ -84,6 +82,7 @@ pub fn run(config: &Config) -> Result<()> {
    let keyring = Vec::new();
    let dns_config = Resolver::from_system_v4()?;
    let proxy = config.network.proxy;
+    let user_agent = config.network.user_agent.clone();

    let cache_dir = paths::cache_dir()?;
    let psl = PslReader::open(&cache_dir)?;
@@ -95,6 +94,7 @@ pub fn run(config: &Config) -> Result<()> {
        keyring,
        dns_config,
        proxy,
+        user_agent,
        options: HashMap::new(),
        blobs: Vec::new(),
        psl,
--- a/src/runtime/db.rs
+++ b/src/runtime/db.rs
@@ -199,7 +199,7 @@ fn gen_changeset2<T: Model, U: UpdateToChangeset<C>, C: Updateable<T>>(object: L
 }

 fn run_update(state: Arc<dyn State>, family: &str, object: AnyLuaValue, update: AnyLuaValue) -> Result<Option<i32>> {
-    let family = Family::from_str(&family)?;
+    let family = Family::from_str(family)?;
    let object = LuaJsonValue::from(object);
    let update = LuaJsonValue::from(update);

--- a/src/runtime/gfx.rs
+++ b/src/runtime/gfx.rs
@@ -36,6 +36,48 @@ pub fn img_load(lua: &mut hlua::Lua, state: Arc<dyn State>) {
    }))
 }

+pub fn img_ahash(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("img_ahash", hlua::function1(move |blob: String| -> Result<String> {
+        let img = state.get_blob(&blob)
+            .map_err(|err| state.set_error(err))?;
+
+        let img = gfx::load(&img.bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        let hash = img.perception_hash(gfx::HashAlg::Mean);
+
+        Ok(hash)
+    }))
+}
+
+pub fn img_dhash(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("img_dhash", hlua::function1(move |blob: String| -> Result<String> {
+        let img = state.get_blob(&blob)
+            .map_err(|err| state.set_error(err))?;
+
+        let img = gfx::load(&img.bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        let hash = img.perception_hash(gfx::HashAlg::Gradient);
+
+        Ok(hash)
+    }))
+}
+
+pub fn img_phash(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("img_phash", hlua::function1(move |blob: String| -> Result<String> {
+        let img = state.get_blob(&blob)
+            .map_err(|err| state.set_error(err))?;
+
+        let img = gfx::load(&img.bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        let hash = img.perception_hash(gfx::HashAlg::Median);
+
+        Ok(hash)
+    }))
+}
+
 #[derive(Debug, Serialize)]
 pub struct Nudity {
    nude: bool,
--- a/src/runtime/logger.rs
+++ b/src/runtime/logger.rs
@@ -20,10 +20,10 @@ pub fn format_lua(out: &mut String, x: &AnyLuaValue) {
                }

                let mut key = String::new();
-                format_lua(&mut key, &k);
+                format_lua(&mut key, k);

                let mut value = String::new();
-                format_lua(&mut value, &v);
+                format_lua(&mut value, v);

                out.push_str(&format!("{}: {}", key, value));

--- a/src/runtime/xml.rs
+++ b/src/runtime/xml.rs
@@ -59,12 +59,10 @@ fn match_element_name(xml: &AnyLuaValue, name: &str) -> bool {

 pub fn xml_named(lua: &mut hlua::Lua, _state: Arc<dyn State>) {
    lua.set("xml_named", hlua::function2(move |xml: AnyLuaValue, name: String| -> AnyLuaValue {
-        if let Some(value) = get_children(xml) {
-            if let AnyLuaValue::LuaArray(arr) = value {
-                for (_, value) in arr {
-                    if match_element_name(&value, &name) {
-                        return value;
-                    }
+        if let Some(AnyLuaValue::LuaArray(arr)) = get_children(xml) {
+            for (_, value) in arr {
+                if match_element_name(&value, &name) {
+                    return value;
                }
            }
        }
--- a/src/shell/complete.rs
+++ b/src/shell/complete.rs
@@ -61,7 +61,7 @@ impl Completer for CmdCompleter {
            return Ok((0, vec![]));
        }

-        let mut cmd = match shellwords::split(&line) {
+        let mut cmd = match shellwords::split(line) {
            Ok(cmd) => cmd,
            Err(_) => return Ok((0, vec![])),
        };
@@ -223,7 +223,7 @@ impl Completer for CmdCompleter {
                                    "passive",
                                    "offline",
                                ], current)),
-                                _ => if current.starts_with("-") {
+                                _ => if current.starts_with('-') {
                                    Ok(filter_options(line, &[
                                        "--source",
                                        "--stealth",
@@ -242,7 +242,7 @@ impl Completer for CmdCompleter {
                                "update",
                                "uninstall",
                                "quickstart",
-                            ], &subcommand))
+                            ], subcommand))
                        },
                        (_, _) => Ok((0, vec![])),
                    }
--- a/src/shell/mod.rs
+++ b/src/shell/mod.rs
@@ -56,6 +56,7 @@ pub enum Command {
    Quit,

    Interrupt,
+    Exec(String),
 }

 impl Command {
@@ -85,6 +86,7 @@ impl Command {
            Command::Workspace => "workspace",
            Command::Cal => "cal",
            Command::Interrupt => unreachable!(),
+            Command::Exec(_) => unreachable!(),
        }
    }

@@ -273,7 +275,7 @@ impl<'a> Shell<'a> {

    #[inline(always)]
    pub fn config(&self) -> &Config {
-        &self.config
+        self.config
    }

    #[inline(always)]
@@ -317,6 +319,10 @@ impl<'a> Shell<'a> {
                        return None;
                    }

+                    if let Some(cmd) = line.strip_prefix('!') {
+                        return Some((Command::Exec(cmd.to_string()), vec![]));
+                    }
+
                    let cmd = match shellwords::split(&line) {
                        Ok(cmd) => cmd,
                        Err(err) => {
@@ -518,6 +524,9 @@ pub fn run_once(rl: &mut Shell) -> Result<bool> {
        Some((Command::Exit, _)) => return Ok(true),
        Some((Command::Quit, _)) => return Ok(true),
        Some((Command::Interrupt, _)) => return Ok(true),
+        Some((Command::Exec(cmd), _)) => {
+            shell_exec(&cmd, rl.workspace())?;
+        },
        None => (),
    }

@@ -543,7 +552,7 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
    let psl = PslReader::open_or_download(&cache_dir,
            |cb| worker::spawn_fn("Downloading public suffix list", cb, false))
        .context("Failed to download public suffix list")?;
-    let library = Library::new(verbose_init, &config)?;
+    let library = Library::new(verbose_init, config)?;
    let keyring = KeyRing::init()?;

    if verbose_init && library.list().is_empty() {
@@ -555,15 +564,35 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
    if autoupdate.outdated() > 0 {
        term::warn(&format!("{} modules are outdated, run: \x1b[1mpkg update\x1b[0m", autoupdate.outdated()));
    }
-    autoupdate.check_background(&config, library.list());
+    autoupdate.check_background(config, library.list());

-    let mut rl = Shell::new(&config, db, blobs, psl, library, keyring);
+    let mut rl = Shell::new(config, db, blobs, psl, library, keyring);

    ttl::reap_expired(&mut rl)?;

    Ok(rl)
 }

+pub fn shell_exec(cmd: &str, workspace: &str) -> Result<()> {
+    use std::process::Command;
+
+    let sh = if cfg!(target_os = "windows") {
+        ("cmd", "/C")
+    } else {
+        ("sh", "-c")
+    };
+
+    Command::new(sh.0)
+            .arg(sh.1)
+            .arg(cmd)
+            .env("SN0INT_WORKSPACE", workspace)
+            .spawn()
+            .context("Failed to execute process")?
+            .wait()?;
+
+    Ok(())
+}
+
 pub fn run(args: &Args, config: &Config) -> Result<()> {
    print_banner();

--- a/src/worker.rs
+++ b/src/worker.rs
@@ -199,7 +199,7 @@ impl DatabaseEvent {
    }

    fn on_activity<T: SpinLogger>(rl: &mut Shell, spinner: &mut T, ratelimit: &mut Ratelimiter, object: &NewActivity, verbose: u64) {
-        Self::spinner_log_new_activity(spinner, &object, verbose);
+        Self::spinner_log_new_activity(spinner, object, verbose);

        // TODO: we don't want to copy the match arms everywhere
        let mut subject = format!("New activity: {:?}", object.topic);
@@ -330,7 +330,7 @@ impl DatabaseEvent {

        let result = match result {
            Ok(id) => {
-                Self::on_update(rl, spinner, ratelimit, family, &value, &update);
+                Self::on_update(rl, spinner, ratelimit, family, value, update);
                Ok(DatabaseResponse::Updated(id))
            },
            Err(err) => {
@@ -437,7 +437,15 @@ impl RatelimitEvent {
    }
 }

-pub fn spawn(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, args: Vec<(serde_json::Value, Option<String>, Vec<Blob>)>, params: &Params, proxy: Option<SocketAddr>, options: HashMap<String, String>) -> usize {
+pub fn spawn(rl: &mut Shell,
+             module: &Module,
+             ratelimit: &mut Ratelimiter,
+             args: Vec<(serde_json::Value, Option<String>, Vec<Blob>)>,
+             params: &Params,
+             proxy: Option<SocketAddr>,
+             user_agent: Option<String>,
+             options: HashMap<String, String>,
+) -> usize {
    // This function hangs if args is empty, so return early if that's the case
    if args.is_empty() {
        return 0;
@@ -445,7 +453,7 @@ pub fn spawn(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, args:

    let verbose = params.verbose;
    let has_stdin = params.stdin;
-    let keyring = rl.keyring().request_keys(&module);
+    let keyring = rl.keyring().request_keys(module);

    let mut stack = StackedSpinners::new();

@@ -463,6 +471,7 @@ pub fn spawn(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, args:
        let tx = tx.clone();
        let module = module.clone();
        let keyring = keyring.clone();
+        let user_agent = user_agent.clone();
        let options = options.clone();
        let signal_register = rl.signal_register().clone();
        pool.execute(move || {
@@ -476,7 +485,7 @@ pub fn spawn(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, args:
            }

            tx.send(Event2::Start);
-            let event = match ipc::parent::run(module, &tx, arg, keyring, verbose, has_stdin, proxy, options, blobs) {
+            let event = match ipc::parent::run(module, &tx, arg, keyring, verbose, has_stdin, proxy, user_agent, options, blobs) {
                Ok(exit) => exit,
                Err(err) => ExitEvent::SetupFailed(err.to_string()),
            };
--- a/src/workspaces.rs
+++ b/src/workspaces.rs
@@ -19,7 +19,7 @@ pub struct Workspace {
 impl Workspace {
    #[inline]
    pub fn db_path(&self) -> Result<PathBuf> {
-        Ok(paths::workspace_dir(&self)?
+        Ok(paths::workspace_dir(self)?
            .join("db.sqlite"))
    }

@@ -43,7 +43,7 @@ impl Workspace {
    pub fn delete(&self) -> Result<()> {
        self.migrate()?;

-        let path = paths::workspace_dir(&self)?;
+        let path = paths::workspace_dir(self)?;
        fs::remove_dir_all(path)?;

        Ok(())
Author	SHA1	Message	Date
kpcyrd	850d628a93	Release v0.23.0	2021-10-23 20:02:24 +02:00
kpcyrd	5bb03d39bc	Merge pull request #215 from kpcyrd/user-agent Allow setting a different default user agent	2021-10-23 12:23:24 +00:00
kpcyrd	f5660570d2	Update dependencies	2021-10-23 14:03:12 +02:00
kpcyrd	f1b0608daa	Allow setting a different default user agent	2021-10-23 13:50:32 +02:00
kpcyrd	c775ae1dee	Add patreon link	2021-10-23 13:15:50 +02:00
kpcyrd	36b5219008	Update readme	2021-10-02 15:03:25 +02:00
kpcyrd	79982fd5f0	Merge pull request #214 from kpcyrd/img-hash Add perceptual image hashing	2021-09-22 01:58:21 +00:00
kpcyrd	5fe71feec3	Fix regression for `sn0int run -f ./foo.lua`	2021-09-22 01:21:03 +02:00
kpcyrd	b8d4eb0f51	Add functions for perceptual image hashing	2021-09-22 01:19:59 +02:00
kpcyrd	e6444db009	Merge pull request #213 from kpcyrd/cli-proxy Allow setting a proxy with `run -X <proxy>`	2021-09-13 10:19:38 +00:00
kpcyrd	7dcb6b81dc	Allow setting a proxy with `run -X <proxy>`	2021-09-13 11:51:32 +02:00
kpcyrd	4fb56d91c2	Allow using `run <module>` interactively	2021-09-13 11:29:13 +02:00
kpcyrd	8c486b7e6e	Release v0.22.0	2021-09-08 18:44:54 +02:00
kpcyrd	e939a28469	Merge pull request #210 from kpcyrd/bump Update dependencies	2021-09-08 12:12:28 +00:00
kpcyrd	2dec9dd28c	Update dependencies	2021-09-08 13:50:57 +02:00
kpcyrd	06459098b1	Merge pull request #209 from kpcyrd/update Update dependencies, fix clippy warnings	2021-09-06 15:07:53 +00:00
kpcyrd	fc0447725c	Fix clippy warnings	2021-09-06 16:26:45 +02:00
kpcyrd	8312695e46	Update dependencies	2021-09-06 15:44:45 +02:00
kpcyrd	6c8e2df632	Update to nom7	2021-08-21 13:31:09 +02:00
kpcyrd	b1ee2d4ce7	Merge pull request #208 from kpcyrd/update Update dependencies	2021-08-17 20:48:13 +00:00
kpcyrd	4cbb72e4c8	Update dependencies	2021-08-17 21:22:48 +02:00
kpcyrd	9e76935f55	Merge pull request #207 from kpcyrd/readline-exec Support invoking shell commands with `!` from readline	2021-08-01 17:00:10 +00:00
kpcyrd	8b5ad635fa	Support invoking shell commands with `!` from readline	2021-07-29 14:15:00 +02:00