Release v0.20.0

Bump dependencies

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.19.1"
+version = "0.20.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -36,57 +36,57 @@ assets = [
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.11.0", path="sn0int-common" }
-sn0int-std = { version="=0.19.0", path="sn0int-std" }
-rustyline = "6.0"
+sn0int-common = { version="0.12.0", path="sn0int-common" }
+sn0int-std = { version="=0.20.0", path="sn0int-std" }
+rustyline = "7.0"
 log = "0.4"
-env_logger = "0.7"
+env_logger = "0.8"
 hlua-badtouch = "0.4"
 structopt = "0.3"
 failure = "0.1"
-rand = "0.7"
-colored = "1.6"
+rand = "0.8"
+colored = "2"
 lazy_static = "1.0"
 shellwords = "1.0"
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-libsqlite3-sys = { version = "0.17.0", features = ["bundled-windows"] }
+libsqlite3-sys = { version = "0.18.0", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
-dirs = "2.0"
+dirs-next = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.15.2"
-base64 = "0.12"
+chrootable-https = "0.16"
+base64 = "0.13"
 data-encoding = "2.1.2"
-serde_urlencoded = "0.6"
+serde_urlencoded = "0.7"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-crossbeam-channel = "0.4"
+crossbeam-channel = "0.5"
 ctrlc = "3.1"
 opener = "0.4"
 separator = "0.4"
 maplit = "1.0.1"
-sloppy-rfc4880 = "0.1.5"
+sloppy-rfc4880 = "0.2"
 regex = "1.0"
 toml = "0.5"
 threadpool = "1.7"
 atty = "0.2"
-semver = "0.9"
+semver = "0.11"
 bytes = "0.4"
 bytesize = "1.0"
-ipnetwork = "0.16"
-strum = "0.18"
-strum_macros = "0.18"
+ipnetwork = "0.17"
+strum = "0.20"
+strum_macros = "0.20"
 embedded-triple = "0.1.0"
 
-digest = "0.8.0"
-md-5 = "0.8.0"
-sha-1 = "0.8.1"
-sha2 = "0.8.0"
-sha3 = "0.8.0"
-hmac = "0.7"
+digest = "0.9"
+md-5 = "0.9"
+sha-1 = "0.9"
+sha2 = "0.9"
+sha3 = "0.9"
+hmac = "0.10"
 
 walkdir = "2.2"
 nude = "0.3"
@@ -94,10 +94,10 @@ glob = "0.3.0"
 os-version = "0.1.1"
 
 [target.'cfg(target_os="linux")'.dependencies]
-caps = "0.3"
+caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.14.0"
-nix = "0.17"
+syscallz = "0.15"
+nix = "0.19"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.4"

README.md

@@ -100,7 +100,8 @@ For everything else please have a look at the [detailed list][1].
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
-  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Debian >= bullseye, Ubuntu >= 20.04, Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-bullseye-ubuntu-20-04-kali)
+  - [Debian <= buster, Ubuntu <= 19.10](https://sn0int.readthedocs.io/en/latest/install.html#debian-buster-ubuntu-19-10)
   - [Fedora/CentOS/Redhat](https://sn0int.readthedocs.io/en/latest/install.html#fedora-centos-redhat)
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)

docs/install.rst

@@ -19,20 +19,34 @@ Mac OSX
 
     $ brew install sn0int
 
-Debian/Ubuntu/Kali
-------------------
+Debian >= bullseye, Ubuntu >= 20.04, Kali
+-----------------------------------------
 
 There are prebuilt packages signed by a debian maintainer. We can import the
 key for this repository out of the debian keyring.
 
 .. code-block:: bash
 
-    $ apt install debian-keyring
-    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
-    $ apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
-    $ echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
-    $ apt update
-    $ apt install sn0int
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg kpcyrd@archlinux.org | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int
+
+Debian <= buster, Ubuntu <= 19.10
+---------------------------------
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+.. code-block:: bash
+
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | sudo apt-key add -
+    $ sudo apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int
 
 Fedora/CentOS/Redhat
 --------------------

docs/reference.rst

@@ -383,6 +383,10 @@ options are set. The following options are available:
   The request body that should be json encoded.
 ``form``
   The request body that should be form encoded.
+``follow_redirects``
+  Automatically follow redirects, up to the specified number. If set to 1, only
+  one redirect is going to be followed. Defaults to 0 so redirects aren't
+  followed.
 ``body``
   The raw request body as string.
 ``into_blob``

modules/harness/onion-http.lua

@@ -1,5 +1,6 @@
 -- Description: Send a request to a hidden service
 -- Version: 0.1.0
+-- Stealth: passive
 -- License: GPL-3.0
 
 function run()

modules/harness/selftest.lua

@@ -1,5 +1,6 @@
 -- Description: basic selftest
 -- Version: 0.1.0
+-- Stealth: offline
 -- License: GPL-3.0
 
 function run()

modules/harness/stdin.lua

@@ -1,5 +1,6 @@
 -- Description: Read from stdin
 -- Version: 0.1.0
+-- Stealth: offline
 -- License: GPL-3.0
 
 function run()

modules/harness/xss.lua

@@ -1,6 +1,7 @@
 -- Description: jaVasCript:/*-/*`/*\`/*'/*"/**/(/* */oNcliCk=alert() )//%0D%0A%0d%0a//</stYle/</titLe/</teXtarEa/</scRipt/--!>\x3csVg/<sVg/oNloAd=alert()//>\x3e
 -- Version: 0.1.0
 -- License: GPL-3.0
+-- Stealth: loud
 -- Source: domains
 
 function run()

sn0int-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.11.0"
+version = "0.12.0"
 description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -8,9 +8,8 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-serde = "1.0"
-serde_derive = "1.0"
+serde = { version = "1.0", features=["derive"] }
 #rocket_failure_errors = { path = "../../../rocket_failure/rocket_failure_errors" }
 rocket_failure_errors = "0.2"
-failure = "0.1"
-nom = "5.0"
+anyhow = "1.0"
+nom = "6.0"

sn0int-common/src/api.rs

@@ -1,4 +1,5 @@
 use crate::id::ModuleID;
+use serde::{Serialize, Deserialize};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct WhoamiResponse {

sn0int-common/src/errors.rs

@@ -1,2 +1,2 @@
-pub use failure::{Error, ResultExt};
+pub use anyhow::{Error, Context, anyhow, format_err, bail};
 pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-common/src/id.rs

@@ -1,5 +1,4 @@
 use crate::errors::*;
-use nom;
 use serde::{de, Serialize, Serializer, Deserialize, Deserializer};
 use std::fmt;
 use std::result;
@@ -53,7 +52,7 @@ impl FromStr for ModuleID {
 
     fn from_str(s: &str) -> Result<ModuleID> {
         let (trailing, module) = module(s)
-            .map_err(|err| format_err!("Failed to parse module id: {:?}", err))?;
+            .map_err(|err| anyhow!("Failed to parse module id: {:?}", err))?;
         if !trailing.is_empty() {
             bail!("Trailing data in module id");
         }

sn0int-common/src/lib.rs

@@ -1,5 +1,3 @@
-#[macro_use] extern crate serde_derive;
-#[macro_use] extern crate failure;
 #[macro_use] extern crate nom;
 
 pub mod api;

sn0int-common/src/metadata/mod.rs

@@ -1,7 +1,9 @@
 use crate::errors::*;
-
+use serde::{Serialize, Deserialize};
 use std::str::FromStr;
 
+mod stealth;
+pub use self::stealth::Stealth;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum EntryType {
@@ -9,6 +11,9 @@ pub enum EntryType {
     Version,
     Source,
     KeyringAccess,
+    Stealth,
+    Author,
+    Repository,
     License,
 }
 
@@ -21,6 +26,9 @@ impl FromStr for EntryType {
             "Version" => Ok(EntryType::Version),
             "Source" => Ok(EntryType::Source),
             "Keyring-Access" => Ok(EntryType::KeyringAccess),
+            "Stealth" => Ok(EntryType::Stealth),
+            "Author" => Ok(EntryType::Author),
+            "Repository" => Ok(EntryType::Repository),
             "License" => Ok(EntryType::License),
             x => bail!("Unknown EntryType: {:?}", x),
         }
@@ -136,6 +144,9 @@ pub struct Metadata {
     pub version: String,
     pub source: Option<Source>,
     pub keyring_access: Vec<String>,
+    pub stealth: Stealth,
+    pub authors: Vec<String>,
+    pub repository: Option<String>,
     pub license: License,
 }
 
@@ -154,6 +165,9 @@ impl FromStr for Metadata {
                 EntryType::Version => data.version = Some(v),
                 EntryType::Source => data.source = Some(v),
                 EntryType::KeyringAccess => data.keyring_access.push(v),
+                EntryType::Stealth => data.stealth = Some(v),
+                EntryType::Author => data.authors.push(v),
+                EntryType::Repository => data.repository = Some(v),
                 EntryType::License => data.license = Some(v),
             }
         }
@@ -168,6 +182,9 @@ pub struct NewMetadata<'a> {
     pub version: Option<&'a str>,
     pub source: Option<&'a str>,
     pub keyring_access: Vec<&'a str>,
+    pub stealth: Option<&'a str>,
+    pub authors: Vec<&'a str>,
+    pub repository: Option<&'a str>,
     pub license: Option<&'a str>,
 }
 
@@ -182,6 +199,14 @@ impl<'a> NewMetadata<'a> {
         let keyring_access = self.keyring_access.into_iter()
             .map(String::from)
             .collect();
+        let stealth = match self.stealth {
+            Some(x) => x.parse()?,
+            _ => Stealth::Normal,
+        };
+        let authors = self.authors.into_iter()
+            .map(String::from)
+            .collect();
+        let repository = self.repository.map(String::from);
         let license = self.license.ok_or_else(|| format_err!("License is required"))?;
         let license = license.parse()?;
 
@@ -190,6 +215,9 @@ impl<'a> NewMetadata<'a> {
             version: version.to_string(),
             source,
             keyring_access,
+            stealth,
+            authors,
+            repository,
             license,
         })
     }
@@ -230,6 +258,36 @@ mod tests {
             version: "1.0.0".to_string(),
             license: License::WTFPL,
             source: Some(Source::Domains),
+            stealth: Stealth::Normal,
+            authors: vec![],
+            repository: None,
+            keyring_access: Vec::new(),
+        });
+    }
+
+    #[test]
+    fn verify_much_metadata() {
+        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+-- Version: 1.0.0
+-- Source: domains
+-- Stealth: passive
+-- Author: kpcyrd <git at rxv dot cc>
+-- Author: kpcyrd's cat
+-- Repository: https://github.com/kpcyrd/sn0int
+-- License: WTFPL
+
+"#).expect("parse");
+        assert_eq!(metadata, Metadata {
+            description: "Hello world, this is my description".to_string(),
+            version: "1.0.0".to_string(),
+            license: License::WTFPL,
+            source: Some(Source::Domains),
+            stealth: Stealth::Passive,
+            authors: vec![
+                "kpcyrd <git at rxv dot cc>".to_string(),
+                "kpcyrd's cat".to_string(),
+            ],
+            repository: Some("https://github.com/kpcyrd/sn0int".to_string()),
             keyring_access: Vec::new(),
         });
     }
@@ -246,6 +304,9 @@ mod tests {
             version: "1.0.0".to_string(),
             license: License::WTFPL,
             source: None,
+            stealth: Stealth::Normal,
+            authors: vec![],
+            repository: None,
             keyring_access: Vec::new(),
         });
     }

sn0int-common/src/metadata/stealth.rs

@@ -0,0 +1,53 @@
+use crate::errors::*;
+use serde::{Serialize, Deserialize};
+use std::str::FromStr;
+
+#[derive(Debug, Eq, PartialEq, PartialOrd, Clone, Serialize, Deserialize)]
+pub enum Stealth {
+    Loud,
+    Normal,
+    Passive,
+    Offline,
+}
+
+impl Stealth {
+    #[inline]
+    pub fn variants() -> &'static [&'static str] {
+        &[
+            "loud",
+            "normal",
+            "passive",
+            "offline",
+        ]
+    }
+
+    #[inline(always)]
+    fn as_num(&self) -> u8 {
+        match self {
+            Stealth::Loud => 3,
+            Stealth::Normal => 2,
+            Stealth::Passive => 1,
+            Stealth::Offline => 0,
+        }
+    }
+
+    #[inline]
+    pub fn equal_or_better(&self, other: &Self) -> bool {
+        self.as_num() <= other.as_num()
+    }
+}
+
+impl FromStr for Stealth {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Stealth> {
+        match s {
+            "loud" => Ok(Stealth::Loud),
+            // This is also the default level if none is provided
+            "normal" => Ok(Stealth::Normal),
+            "passive" => Ok(Stealth::Passive),
+            "offline" => Ok(Stealth::Offline),
+            x => bail!("Unknown stealth: {:?}", x),
+        }
+    }
+}

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.11.0", path="../sn0int-common" }
+sn0int-common = { version="0.12.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }

sn0int-std/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-std"
-version = "0.19.0"
+version = "0.20.0"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
@@ -14,43 +14,43 @@ log = "0.4"
 failure = "0.1"
 hlua-badtouch = "0.4"
 tokio = "0.1.14"
-rand = "0.7"
+rand = "0.8"
 regex = "1.0"
-rustls = { version="0.16", features=["dangerous_configuration"] }
+rustls = { version="0.18", features=["dangerous_configuration"] }
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-serde_urlencoded = "0.6"
+serde_urlencoded = "0.7"
 webpki = "0.21"
-webpki-roots = "0.19.0"
-ct-logs = "0.6"
+webpki-roots = "0.20"
+ct-logs = "0.7"
 #chrootable-https = { path = "../../chrootable-https" }
-chrootable-https = "0.15.2"
+chrootable-https = "0.16"
 http = "0.2"
 bufstream = "0.1.4"
-pem = "0.7"
+pem = "0.8"
 url = "2.0"
-tungstenite = { version = "0.10.1", default-features = false }
+tungstenite = { version = "0.11", default-features = false }
 kuchiki = "0.8.0"
-maxminddb = "0.14"
-x509-parser = "0.7"
-der-parser = "3.0"
+maxminddb = "0.17"
+x509-parser = "0.9"
+der-parser = "5"
 publicsuffix = { version="1.5", default-features=false }
 xml-rs = "0.8"
-geo = "0.13"
+geo = "0.16"
 bytes = "0.4"
-base64 = "0.12"
+base64 = "0.13"
 chrono = { version = "0.4", features = ["serde"] }
-mqtt-protocol = "0.8.1"
+mqtt-protocol = "0.10"
 sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
 
 image = "0.23.0"
 kamadak-exif = "0.5.1"
 
-bs58 = "0.3"
-digest = "0.8.0"
-blake2 = "0.8.0"
+bs58 = "0.4"
+digest = "0.9"
+blake2 = "0.9"
 
 [dev-dependencies]
-env_logger = "0.7"
+env_logger = "0.8"
 maplit = "1.0.1"

sn0int-std/src/blobs.rs

@@ -1,6 +1,6 @@
 use bytes::Bytes;
 use blake2::VarBlake2b;
-use digest::{Input, VariableOutput};
+use digest::{Update, VariableOutput};
 use serde::ser::{Serialize, Serializer};
 use serde::de::{self, Deserialize, Deserializer};
 use std::result;
@@ -22,8 +22,9 @@ impl Blob {
 
     pub fn hash(bytes: &[u8]) -> String {
         let mut h = VarBlake2b::new(32).unwrap();
-        h.input(bytes);
-        Self::encode_hash(&h.vec_result())
+        h.update(bytes);
+        let output = h.finalize_boxed();
+        Self::encode_hash(&output)
     }
 
     #[inline]

sn0int-std/src/crt.rs

@@ -1,78 +1,9 @@
 use crate::errors::*;
-
-use x509_parser;
-use der_parser::der::DerObject;
-use der_parser::ber::{BerObjectContent, BerTag};
-use der_parser::oid::Oid;
 use std::collections::HashSet;
-use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
-
-
-#[derive(Debug, PartialEq)]
-pub enum AlternativeName {
-    DnsName(String),
-    Email(String),
-    IpAddr(IpAddr),
-}
-
-pub fn san_extension(i: &[u8]) -> Result<Vec<AlternativeName>> {
-    let (rem, seq) = der_parser::der::parse_der_sequence(i)
-        .map_err(|_| format_err!("Failed to parse san extension"))?;
-
-    if !rem.is_empty() {
-        bail!("san extension has trailing garbage");
-    }
-
-    debug!("Decoded sequence: {:?}", seq);
-    if let BerObjectContent::Sequence(seq) = seq.content {
-        seq.into_iter()
-            .map(san_value)
-            .collect()
-    } else {
-        bail!("Expected der sequence");
-    }
-}
-
-pub fn san_value(o: DerObject) -> Result<AlternativeName> {
-    debug!("DER object in SAN extension: {:?}", o);
-
-    match (o.class, o.tag, &o.content) {
-        (2, BerTag::Integer, BerObjectContent::Unknown(BerTag::Integer, value)) => san_value_dns(value),
-        (2, BerTag::Boolean, BerObjectContent::Unknown(BerTag::Boolean, value)) => san_value_email(value),
-        (2, BerTag::ObjDescriptor, BerObjectContent::Unknown(BerTag::ObjDescriptor, value)) => san_value_ipaddr(value),
-        _ => bail!("Unexpected object: {:?}", o),
-    }
-}
-
-pub fn san_value_dns(v: &[u8]) -> Result<AlternativeName> {
-    debug!("Reading as dns name: {:?}", v);
-    String::from_utf8(v.to_vec())
-        .map(AlternativeName::DnsName)
-        .map_err(Error::from)
-}
-
-pub fn san_value_email(v: &[u8]) -> Result<AlternativeName> {
-    debug!("Reading as email: {:?}", v);
-    String::from_utf8(v.to_vec())
-        .map(AlternativeName::Email)
-        .map_err(Error::from)
-}
-
-pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
-    debug!("Reading as ipaddr: {:?}", v);
-    match v.len() {
-        4  => Ok(AlternativeName::IpAddr(Ipv4Addr::from([
-            v[0], v[1], v[2], v[3],
-        ]).into())),
-        16 => Ok(AlternativeName::IpAddr(Ipv6Addr::from([
-            v[0],  v[1],  v[2],  v[3],
-            v[4],  v[5],  v[6],  v[7],
-            v[8],  v[9],  v[10], v[11],
-            v[12], v[13], v[14], v[15],
-        ]).into())),
-        _ => Err(format_err!("Invalid ipaddr")),
-    }
-}
+use std::net::IpAddr;
+use x509_parser::x509::X509Version;
+use x509_parser::certificate::X509Certificate;
+use x509_parser::extensions::{GeneralName, ParsedExtension};
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
@@ -83,7 +14,7 @@ pub struct Certificate {
 
 impl Certificate {
     pub fn parse_pem(crt: &str) -> Result<Certificate> {
-        let pem = match x509_parser::pem::pem_to_der(crt.as_bytes()) {
+        let pem = match x509_parser::pem::parse_x509_pem(crt.as_bytes()) {
             Ok((remaining, pem)) => {
                 if !remaining.is_empty() {
                     bail!("input cert has trailing garbage");
@@ -99,12 +30,12 @@ impl Certificate {
     }
 
     pub fn from_bytes(crt: &[u8]) -> Result<Certificate> {
-        let crt = match x509_parser::parse_x509_der(&crt) {
+        let crt = match X509Certificate::from_der(&crt) {
             Ok((remaining, der)) => {
                 if !remaining.is_empty() {
                     bail!("input cert has trailing garbage");
                 }
-                if der.tbs_certificate.version != 2 {
+                if der.tbs_certificate.version != X509Version::V3 {
                     bail!("unexpected certificate version");
                 }
                 der
@@ -116,35 +47,45 @@ impl Certificate {
         let mut valid_emails = HashSet::new();
         let mut valid_ipaddrs = HashSet::new();
 
-        for x in crt.tbs_certificate.subject.rdn_seq {
-            for y in x.set {
-                // CommonName
-                if y.attr_type != Oid::from(&[2, 5, 4, 3]) {
-                    continue;
-                }
-
-                if let Ok(x) = y.attr_value.content.as_slice() {
-                    let value = String::from_utf8(x.to_vec())?;
-                    info!("Found CN in Subject: {:?}", value);
-                    valid_names.insert(value);
-                }
+        for attr in crt.subject().iter_common_name() {
+            if let Ok(cn) = attr.as_str() {
+                info!("Found CN in Subject: {:?}", cn);
+                valid_names.insert(cn.to_string());
             }
         }
 
-        for x in crt.tbs_certificate.extensions {
-            if x.oid != Oid::from(&[2, 5, 29, 17]) {
-                continue;
-            }
-
-            debug!("Found san extension: {:?}", x.value);
-            let values = san_extension(x.value)?;
-
-            for v in values {
-                match v {
-                    AlternativeName::DnsName(v) => valid_names.insert(v),
-                    AlternativeName::Email(v) => valid_emails.insert(v),
-                    AlternativeName::IpAddr(v) => valid_ipaddrs.insert(v),
-                };
+        for (_oid, ext) in crt.tbs_certificate.extensions {
+            if let ParsedExtension::SubjectAlternativeName(san) = ext.parsed_extension() {
+                for name in &san.general_names {
+                    debug!("Certificate is valid for {:?}", name);
+                    match name {
+                        GeneralName::DNSName(v) => {
+                            valid_names.insert(v.to_string());
+                        },
+                        GeneralName::RFC822Name(v) => {
+                            valid_emails.insert(v.to_string());
+                        },
+                        GeneralName::IPAddress(v) => {
+                            let ip = match v.len() {
+                                4 => Some(IpAddr::from([v[0], v[1], v[2], v[3]])),
+                                16 => Some(IpAddr::from([
+                                    v[0], v[1], v[2], v[3],
+                                    v[4], v[5], v[6], v[7],
+                                    v[8], v[9], v[10], v[11],
+                                    v[12], v[13], v[14], v[15],
+                                ])),
+                                _ => {
+                                    info!("Certificate is valid for invalid ip address: {:?}", v);
+                                    None
+                                },
+                            };
+                            if let Some(ip) = ip {
+                                valid_ipaddrs.insert(ip);
+                            }
+                        },
+                        _ => (),
+                    }
+                }
             }
         }
 
@@ -162,7 +103,6 @@ impl Certificate {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use der_parser::parse_der;
 
     #[test]
     fn test_parse_pem_github() {
@@ -260,19 +200,6 @@ ZkZZmqNn2Q8=
         });
     }
 
-    #[test]
-    fn test_san_extension() {
-        let ext = san_extension(&[48, 28,
-                130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109,
-                130, 14, 119, 119, 119, 46, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109
-            ])
-            .expect("Failed to parse extension");
-        assert_eq!(ext, vec![
-            AlternativeName::DnsName(String::from("github.com")),
-            AlternativeName::DnsName(String::from("www.github.com")),
-        ]);
-    }
-
     #[test]
     fn test_long_san_extension() {
         let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
@@ -372,48 +299,6 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
         });
     }
 
-    #[test]
-    fn test_san_value_dns() {
-        let (rem, v) = parse_der(&[130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
-            .expect("Failed to parse san value");
-        assert!(rem.is_empty());
-        println!("{:?}", v);
-        assert_eq!(v, DerObject {
-            class: 2,
-            structured: 0,
-            tag: BerTag::Integer,
-            content: BerObjectContent::Unknown(BerTag::Integer, &[103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
-        });
-        let content = match v.content {
-            BerObjectContent::Unknown(BerTag::Integer, v) => v,
-            _ => panic!("Wrong BerObjectContent"),
-        };
-        let v = san_value_dns(content)
-            .expect("Failed to process san value");
-        assert_eq!(v, AlternativeName::DnsName(String::from("github.com")));
-    }
-
-    #[test]
-    fn test_san_value_ipaddr() {
-        let (rem, v) = parse_der(&[135, 4, 1, 1, 1, 1])
-            .expect("Failed to parse san value");
-        assert!(rem.is_empty());
-        println!("{:?}", v);
-        assert_eq!(v, DerObject {
-            class: 2,
-            structured: 0,
-            tag: BerTag::ObjDescriptor,
-            content: BerObjectContent::Unknown(BerTag::ObjDescriptor, &[1, 1, 1, 1])
-        });
-        let content = match v.content {
-            BerObjectContent::Unknown(BerTag::ObjDescriptor, v) => v,
-            _ => panic!("Wrong BerObjectContent"),
-        };
-        let v = san_value_ipaddr(content)
-            .expect("Failed to process san value");
-        assert_eq!(v, AlternativeName::IpAddr("1.1.1.1".parse().unwrap()));
-    }
-
     #[test]
     fn test_san_email() {
         let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----

sn0int-std/src/engine/structs.rs

@@ -2,9 +2,6 @@ use crate::errors::*;
 use crate::hlua::{AnyHashableLuaValue, AnyLuaValue, AnyLuaString};
 use std::collections::{self, HashMap};
 use crate::json::LuaJsonValue;
-use serde;
-use serde_json;
-
 
 pub fn from_lua<T>(x: LuaJsonValue) -> Result<T>
     where for<'de> T: serde::Deserialize<'de>

sn0int-std/src/gfx/exif.rs

@@ -1,9 +1,6 @@
 use crate::errors::*;
-
-use exif;
 use std::io;
 
-
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Location {
     latitude: f64,

sn0int-std/src/html.rs

@@ -1,11 +1,8 @@
+use crate::engine::structs::LuaMap;
 use crate::errors::*;
-
-use kuchiki;
+use crate::hlua::AnyLuaValue;
 use kuchiki::traits::TendrilSink;
 use std::collections::HashMap;
-use crate::hlua::AnyLuaValue;
-use crate::engine::structs::LuaMap;
-
 
 #[derive(Debug, PartialEq)]
 pub struct Element {

sn0int-std/src/json.rs

@@ -35,11 +35,7 @@ pub fn decode_stream(x: &str) -> Result<Vec<AnyLuaValue>> {
 pub fn lua_array_is_list(array: &[(AnyLuaValue, AnyLuaValue)]) -> bool {
     if !array.is_empty() {
         let first = &array[0];
-        if let AnyLuaValue::LuaNumber(_) = first.0 {
-            true
-        } else {
-            false
-        }
+        matches!(first.0, AnyLuaValue::LuaNumber(_))
     } else {
         // true // TODO: this breaks unserialize
         false

sn0int-std/src/mqtt.rs

@@ -46,7 +46,7 @@ impl MqttClient {
             stream,
         };
 
-        let mut pkt = ConnectPacket::new("MQTT", "sn0int");
+        let mut pkt = ConnectPacket::new("sn0int");
         pkt.set_user_name(options.username.clone());
         pkt.set_password(options.password.clone());
 

sn0int-std/src/ratelimits.rs

@@ -10,6 +10,12 @@ pub struct Ratelimiter {
     buckets: HashMap<String, Bucket>,
 }
 
+impl Default for Ratelimiter {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl Ratelimiter {
     pub fn new() -> Ratelimiter {
         Ratelimiter {

sn0int-std/src/sockets/mod.rs

@@ -91,7 +91,7 @@ impl SocketOptions {
 
 pub enum Stream {
     Tcp(TcpStream),
-    Tls(rustls::StreamOwned<rustls::ClientSession, TcpStream>),
+    Tls(Box<rustls::StreamOwned<rustls::ClientSession, TcpStream>>),
 }
 
 impl Stream {

sn0int-std/src/sockets/tls.rs

@@ -107,7 +107,7 @@ fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, T
 
     info!("successfully established tls connection");
     let stream = rustls::StreamOwned::new(session, stream);
-    let stream = Stream::Tls(stream);
+    let stream = Stream::Tls(Box::new(stream));
     Ok((stream, tls))
 }
 

sn0int-std/src/web.rs

@@ -1,25 +1,24 @@
-pub use chrootable_https::{Client, HttpClient, Resolver, Response};
-
 use crate::blobs::{Blob, BlobState};
-use std::collections::{HashMap, HashSet};
-use std::time::Duration;
-use std::ops::Deref;
+use crate::engine::structs::LuaMap;
 use crate::errors::*;
+use crate::json::LuaJsonValue;
 use crate::hlua::AnyLuaValue;
-use serde_json;
+use chrootable_https::{Request, Body, Uri};
+pub use chrootable_https::{Client, HttpClient, Resolver, Response};
+use chrootable_https::http::HttpTryFrom;
+use chrootable_https::http::uri::Parts;
+use chrootable_https::http::request::Builder;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
-use std::fmt;
-use std::net::SocketAddr;
-use std::sync::Arc;
 use serde::Serialize;
-use crate::engine::structs::LuaMap;
-use crate::json::LuaJsonValue;
-use chrootable_https::http::uri::Parts;
-use chrootable_https::{Request, Body, Uri};
-use serde_urlencoded;
-use base64;
-
+use std::collections::{HashMap, HashSet};
+use std::fmt;
+use std::iter;
+use std::net::SocketAddr;
+use std::ops::Deref;
+use std::sync::Arc;
+use std::time::Duration;
+use url::Url;
 
 pub fn url_set_qs<S: Serialize + fmt::Debug>(url: Uri, query: &S) -> Result<Uri> {
     let mut parts = Parts::from(url);
@@ -51,7 +50,12 @@ pub struct HttpSession {
 
 impl HttpSession {
     pub fn new() -> (String, HttpSession) {
-        let id: String = thread_rng().sample_iter(&Alphanumeric).take(16).collect();
+        let mut rng = thread_rng();
+        let id: String = iter::repeat(())
+            .map(|()| rng.sample(Alphanumeric))
+            .map(char::from)
+            .take(16)
+            .collect();
         (id.clone(), HttpSession {
             id,
             cookies: CookieJar::default(),
@@ -67,6 +71,8 @@ pub struct RequestOptions {
     user_agent: Option<String>,
     json: Option<serde_json::Value>,
     form: Option<serde_json::Value>,
+    #[serde(default)]
+    follow_redirects: usize,
     body: Option<String>,
     timeout: Option<u64>,
     #[serde(default)]
@@ -84,10 +90,6 @@ impl RequestOptions {
     }
 }
 
-pub fn default_user_agent() -> String {
-    format!("sn0int/{}", env!("CARGO_PKG_VERSION"))
-}
-
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct HttpRequest {
     // reference to the HttpSession
@@ -98,7 +100,8 @@ pub struct HttpRequest {
     query: Option<HashMap<String, String>>,
     headers: Option<HashMap<String, String>>,
     basic_auth: Option<(String, String)>,
-    user_agent: Option<String>,
+    user_agent: String,
+    follow_redirects: usize,
     body: Option<ReqBody>,
     timeout: Option<Duration>,
     into_blob: bool,
@@ -107,10 +110,11 @@ pub struct HttpRequest {
 }
 
 impl HttpRequest {
-    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions) -> HttpRequest {
+    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions, default_agent: fn() -> String) -> HttpRequest {
         let cookies = session.cookies.clone();
 
-        let user_agent = options.user_agent.or_else(|| Some(default_user_agent()));
+        let user_agent = options.user_agent.unwrap_or_else(default_agent);
+
         let timeout = options.timeout.map(Duration::from_millis);
 
         let mut request = HttpRequest {
@@ -122,6 +126,7 @@ impl HttpRequest {
             headers: options.headers,
             basic_auth: options.basic_auth,
             user_agent,
+            follow_redirects: options.follow_redirects,
             body: None,
             timeout,
             into_blob: options.into_blob,
@@ -144,7 +149,7 @@ impl HttpRequest {
         request
     }
 
-    pub fn send(&self, state: &dyn WebState) -> Result<Response> {
+    pub fn send(&mut self, state: &dyn WebState) -> Result<Response> {
         let mut url = self.url.parse::<Uri>()?;
 
         // set query string
@@ -153,37 +158,9 @@ impl HttpRequest {
         }
 
         // start setting up request
-        let mut req = Request::builder();
-        req.method(self.method.as_str());
-        req.uri(url);
-
-        let mut observed_headers = HashSet::new();
-
-        // set cookies
-        {
-            use chrootable_https::header::COOKIE;
-            let mut cookies = String::new();
-
-            for (key, value) in self.cookies.iter() {
-                if !cookies.is_empty() {
-                    cookies += "; ";
-                }
-                cookies.push_str(&format!("{}={}", key, value));
-            }
-
-            if !cookies.is_empty() {
-                req.header(COOKIE, cookies.as_str());
-                observed_headers.insert(COOKIE.as_str().to_lowercase());
-            }
-        }
+        let mut req = self.mkrequest(self.method.as_str(), &url);
 
         // add headers
-        if let Some(ref agent) = self.user_agent {
-            use chrootable_https::header::USER_AGENT;
-            req.header(USER_AGENT, agent.as_str());
-            observed_headers.insert(USER_AGENT.as_str().to_lowercase());
-        }
-
         if let Some(ref auth) = self.basic_auth {
             use chrootable_https::header::AUTHORIZATION;
             let &(ref user, ref password) = auth;
@@ -191,9 +168,9 @@ impl HttpRequest {
             let auth = base64::encode(&format!("{}:{}", user, password));
             let auth = format!("Basic {}", auth);
             req.header(AUTHORIZATION, auth.as_str());
-            observed_headers.insert(AUTHORIZATION.as_str().to_lowercase());
         }
 
+        let mut observed_headers = HashSet::new();
         if let Some(ref headers) = self.headers {
             for (k, v) in headers {
                 req.header(k.as_str(), v.as_str());
@@ -220,24 +197,71 @@ impl HttpRequest {
             },
             None => Body::empty(),
         };
-        let req = req.body(body)?;
+        let mut req = req.body(body)?;
 
         debug!("Getting http client");
         let client = state.http(&self.proxy)?;
 
-        // send request
-        debug!("Sending http request: {:?}", req);
-        let res = client.request(req)
-            .with_timeout(self.timeout)
-            .wait_for_response()?;
+        let res = loop {
+            // send request
+            debug!("Sending http request: {:?}", req);
+            let res = client.request(req)
+                .with_timeout(self.timeout)
+                .wait_for_response()?;
 
-        for cookie in &res.cookies {
-            HttpRequest::register_cookies_on_state(&self.session, state, cookie);
-        }
+            for cookie in &res.cookies {
+                HttpRequest::register_cookies_on_state(&self.session, state, cookie);
+            }
+
+            if self.follow_redirects > 0 && res.status >= 300 && res.status < 400 {
+                if let Some(location) = res.headers.get("location") {
+                    let base = Url::parse(&url.to_string())?;
+                    let joined = base.join(&location)?;
+                    url = joined.to_string().parse()?;
+
+                    req = self.mkrequest("GET", &url).body(Body::empty())?;
+                    self.follow_redirects -= 1;
+                    continue;
+                }
+            }
+
+            break res;
+        };
 
         Ok(res)
     }
 
+    /// create a basic request, reusable when following redirects
+    fn mkrequest<T>(&self, method: &str, url: T) -> Builder
+        where Uri: HttpTryFrom<T>,
+    {
+        let mut req = Request::builder();
+        req.method(method);
+        req.uri(url);
+        self.attach_cookies(&mut req);
+
+        use chrootable_https::header::USER_AGENT;
+        req.header(USER_AGENT, self.user_agent.as_str());
+
+        req
+    }
+
+    fn attach_cookies(&self, req: &mut Builder) {
+        use chrootable_https::header::COOKIE;
+        let mut cookies = String::new();
+
+        for (key, value) in self.cookies.iter() {
+            if !cookies.is_empty() {
+                cookies += "; ";
+            }
+            cookies.push_str(&format!("{}={}", key, value));
+        }
+
+        if !cookies.is_empty() {
+            req.header(COOKIE, cookies.as_str());
+        }
+    }
+
     pub fn response_to_lua<S>(&self, state: &S, res: Response) -> Result<LuaMap>
         where S: WebState + BlobState
     {

sn0int-std/src/xml.rs

@@ -13,6 +13,12 @@ pub struct XmlDocument {
     pub children: Vec<XmlElement>,
 }
 
+impl Default for XmlDocument {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl XmlDocument {
     #[inline(always)]
     pub fn new() -> XmlDocument {

src/api.rs

@@ -1,18 +1,15 @@
 use crate::errors::*;
-use std::fmt;
+use crate::web;
 use crate::config::Config;
+use crate::utils;
 use chrootable_https::{self, HttpClient, Body, Request, Uri};
 use chrootable_https::http::request::Builder as RequestBuilder;
 use chrootable_https::header::CONTENT_TYPE;
-use rand::{Rng, thread_rng};
-use rand::distributions::Alphanumeric;
 use serde::de::DeserializeOwned;
 use serde::ser::Serialize;
-use serde_json;
 use sn0int_common::api::*;
 use sn0int_common::{ModuleID, ApiResponse};
-use crate::web;
-
+use std::fmt;
 
 pub struct Client {
     server: String,
@@ -46,14 +43,15 @@ impl Client {
         self.session = Some(session.into());
     }
 
+    #[inline]
     pub fn random_session() -> String {
-        thread_rng().sample_iter(&Alphanumeric).take(32).collect()
+        utils::random_string(32)
     }
 
     #[inline]
     fn user_agent(&self) -> String {
-        format!("{}, {}, {}",
-            web::default_user_agent(),
+        format!("sn0int/{}, {}, {}",
+            env!("CARGO_PKG_VERSION"),
             embedded_triple::get(),
             self.os_version,
         )

src/args.rs

@@ -18,10 +18,7 @@ pub struct Args {
 
 impl Args {
     pub fn is_sandbox(&self) -> bool {
-        match self.subcommand {
-            Some(SubCommand::Sandbox(_)) => true,
-            _ => false,
-        }
+        matches!(self.subcommand, Some(SubCommand::Sandbox(_)))
     }
 }
 

src/auth.rs

@@ -1,5 +1,4 @@
 use crate::errors::*;
-use opener;
 use std::fs;
 use std::path::PathBuf;
 use std::thread;

src/autonoscope/ip.rs

@@ -70,7 +70,7 @@ impl AutoRule<str> for IpRule {
 
 impl AutoRule<net::IpAddr> for IpRule {
     fn matches(&self, ipaddr: &net::IpAddr) -> Result<bool> {
-        Ok(self.network.contains(ipaddr.clone()))
+        Ok(self.network.contains(*ipaddr))
     }
 }
 

src/autonoscope/mod.rs

@@ -1,6 +1,5 @@
 use crate::db::DatabaseSock;
 use crate::errors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::schema::*;
 use crate::models::*;

src/cal/date.rs

@@ -80,7 +80,7 @@ fn setup_graph_map(events: &[Activity]) -> (HashMap<NaiveDate, u64>, u64) {
                 if ctr > max {
                     max = ctr;
                 }
-                map.insert(cur.clone(), ctr);
+                map.insert(*cur, ctr);
                 *cur = date;
                 ctr = 1;
             }

src/cal/time.rs

@@ -34,7 +34,7 @@ fn setup_graph_map(events: &[Activity], slice_duration: u32) -> (HashMap<NaiveDa
                 if ctr > max {
                     max = ctr;
                 }
-                map.insert(cur.clone(), ctr);
+                map.insert(*cur, ctr);
                 *cur = time;
                 ctr = 1;
             }
@@ -135,6 +135,21 @@ impl DateTimeSpec {
         &self.end
     }
 
+    fn push_date(&self, w: &mut String, date: &NaiveDate) {
+        let weekday = date.weekday();
+        let is_weekend = weekday == Weekday::Sat || weekday == Weekday::Sun;
+
+        if is_weekend {
+            w.push_str("\x1b[1m");
+        }
+
+        w.push_str(&date.format("%Y-%m-%d (%a) ").to_string());
+
+        if is_weekend {
+            w.push_str("\x1b[0m");
+        }
+    }
+
     pub fn to_term_string(&self, ctx: &DateTimeContext) -> String {
         let mut w = String::new();
 
@@ -154,7 +169,7 @@ impl DateTimeSpec {
         // add days
         let mut date = self.start;
         while date <= self.end {
-            w.push_str(&date.format("%Y-%m-%d ").to_string());
+            self.push_date(&mut w, &date);
 
             let mut hours = 0;
             let mut mins = 0;

src/cmd/add_cmd.rs

@@ -2,6 +2,7 @@ use crate::errors::*;
 
 use crate::blobs::Blob;
 use crate::cmd::Cmd;
+use crate::db::DbChange;
 use crate::gfx;
 use crate::models::*;
 use crate::shell::Shell;
@@ -9,10 +10,11 @@ use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::utils;
 use crate::term;
+use std::fmt::Debug;
 use std::fs;
+use std::io::{self, BufRead};
 use std::net;
 use std::net::SocketAddr;
-use ipnetwork;
 use std::path::Path;
 use walkdir::WalkDir;
 
@@ -24,6 +26,9 @@ pub struct Args {
     /// Do not actually insert into database
     #[structopt(short="n", long="dry-run")]
     dry_run: bool,
+    /// Stream structs from stdin line by line
+    #[structopt(long)]
+    stdin: bool,
 }
 
 #[derive(Debug, StructOpt)]
@@ -74,25 +79,68 @@ pub enum Target {
 
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
-        match self.subcommand {
-            Target::Domain(args) => args.insert(rl, self.dry_run),
-            Target::Subdomain(args) => args.insert(rl, self.dry_run),
-            Target::IpAddr(args) => args.insert(rl, self.dry_run),
-            Target::Url(args) => args.insert(rl, self.dry_run),
-            Target::Email(args) => args.insert(rl, self.dry_run),
-            Target::PhoneNumber(args) => args.insert(rl, self.dry_run),
-            Target::Device(args) => args.insert(rl, self.dry_run),
-            Target::Network(args) => args.insert(rl, self.dry_run),
-            Target::Account(args) => args.insert(rl, self.dry_run),
-            Target::Breach(args) => args.insert(rl, self.dry_run),
-            Target::Image(args) => args.insert(rl, self.dry_run),
-            Target::Netblock(args) => args.insert(rl, self.dry_run),
-            Target::Port(args) => args.insert(rl, self.dry_run),
-            Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
+        if self.stdin {
+            match self.subcommand {
+                Target::Domain(_) => stream_stdin::<AddDomain>(rl),
+                Target::Subdomain(_) => stream_stdin::<AddSubdomain>(rl),
+                Target::IpAddr(_) => stream_stdin::<AddIpAddr>(rl),
+                Target::Url(_) => stream_stdin::<AddUrl>(rl),
+                Target::Email(_) => stream_stdin::<AddEmail>(rl),
+                Target::PhoneNumber(_) => stream_stdin::<AddPhoneNumber>(rl),
+                Target::Port(_) => stream_stdin::<AddPort>(rl),
+                Target::CryptoAddr(_) => stream_stdin::<AddCryptoAddr>(rl),
+                Target::Device(_) | Target::Network(_) | Target::Account(_) | Target::Breach(_) | Target::Image(_) | Target::Netblock(_) => {
+                    bail!("This struct doesn't support streaming from stdin")
+                },
+            }
+        } else {
+            match self.subcommand {
+                Target::Domain(args) => args.insert(rl, self.dry_run),
+                Target::Subdomain(args) => args.insert(rl, self.dry_run),
+                Target::IpAddr(args) => args.insert(rl, self.dry_run),
+                Target::Url(args) => args.insert(rl, self.dry_run),
+                Target::Email(args) => args.insert(rl, self.dry_run),
+                Target::PhoneNumber(args) => args.insert(rl, self.dry_run),
+                Target::Device(args) => args.insert(rl, self.dry_run),
+                Target::Network(args) => args.insert(rl, self.dry_run),
+                Target::Account(args) => args.insert(rl, self.dry_run),
+                Target::Breach(args) => args.insert(rl, self.dry_run),
+                Target::Image(args) => args.insert(rl, self.dry_run),
+                Target::Netblock(args) => args.insert(rl, self.dry_run),
+                Target::Port(args) => args.insert(rl, self.dry_run),
+                Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
+            }
         }
     }
 }
 
+fn stream_stdin<T: InsertFromString>(rl: &mut Shell) -> Result<()> {
+    let stdin = io::stdin();
+    for line in stdin.lock().lines() {
+        let line = line?;
+        match T::from_string(rl, line) {
+            Ok(insert) => {
+                debug!("Received {:?}", insert);
+                let result = rl.db().insert_generic(insert.clone())?;
+
+                if let Some((DbChange::Insert, _)) = result {
+                    if let Ok(value) = insert.value(rl.db()) {
+                        let log = format!("Adding {} {:?}", insert.family(), value);
+                        term::success(&log);
+                    }
+                }
+            },
+            Err(err) => term::error(&format!("Error: {:#}", err)),
+        }
+    }
+    Ok(())
+}
+
+trait InsertFromString {
+    // TODO: some implementations run insert_struct instead of just transforming
+    fn from_string(rl: &mut Shell, x: String) -> Result<Insert>;
+}
+
 trait IntoInsert: Sized {
     fn into_insert(self, rl: &mut Shell) -> Result<Insert>;
 
@@ -116,7 +164,12 @@ impl IntoInsert for AddDomain {
             Some(domain) => domain,
             _ => utils::question("Domain")?,
         };
+        Self::from_string(rl, domain)
+    }
+}
 
+impl InsertFromString for AddDomain {
+    fn from_string(rl: &mut Shell, domain: String) -> Result<Insert> {
         // ensure input is a valid domain
         let dns_name = rl.psl()?.parse_dns_name(&domain)
             .map_err(|e| format_err!("Failed to parse domain: {}", e))?;
@@ -143,7 +196,12 @@ impl IntoInsert for AddSubdomain {
             Some(subdomain) => subdomain,
             _ => utils::question("Subdomain")?,
         };
+        Self::from_string(rl, subdomain)
+    }
+}
 
+impl InsertFromString for AddSubdomain {
+    fn from_string(rl: &mut Shell, subdomain: String) -> Result<Insert> {
         let dns_name = rl.psl()?.parse_dns_name(&subdomain)
             .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
 
@@ -170,7 +228,8 @@ pub struct AddIpAddr {
 }
 
 impl IntoInsert for AddIpAddr {
-    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
+        // TODO: change this to Option<String> so we don't need to parse back and forth
         let ipaddr = match self.ipaddr {
             Some(ipaddr) => ipaddr,
             _ => {
@@ -178,6 +237,13 @@ impl IntoInsert for AddIpAddr {
                 ipaddr.parse()?
             },
         };
+        Self::from_string(rl, ipaddr.to_string())
+    }
+}
+
+impl InsertFromString for AddIpAddr {
+    fn from_string(_rl: &mut Shell, ipaddr: String) -> Result<Insert> {
+        let ipaddr = ipaddr.parse()?;
 
         let family = match ipaddr {
             net::IpAddr::V4(_) => "4",
@@ -214,7 +280,12 @@ impl IntoInsert for AddUrl {
             Some(url) => url,
             _ => utils::question("URL")?,
         };
+        Self::from_string(rl, url)
+    }
+}
 
+impl InsertFromString for AddUrl {
+    fn from_string(rl: &mut Shell, url: String) -> Result<Insert> {
         let parts = url::Url::parse(&url)?;
         let subdomain = parts.domain()
             .ok_or_else(|| format_err!("url doesn't have a domain host"))?;
@@ -258,12 +329,18 @@ pub struct AddEmail {
 }
 
 impl IntoInsert for AddEmail {
-    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
         let email = match self.email {
             Some(email) => email,
             _ => utils::question("Email")?,
         };
+        Self::from_string(rl, email)
+    }
+}
 
+impl InsertFromString for AddEmail {
+    fn from_string(_rl: &mut Shell, email: String) -> Result<Insert> {
+        // TODO: consider doing basic validation
         Ok(Insert::Email(NewEmail {
             value: email,
             displayname: None,
@@ -290,9 +367,30 @@ impl IntoInsert for AddPhoneNumber {
             },
         };
 
+        // TODO: consider doing basic validation
         Ok(Insert::PhoneNumber(NewPhoneNumber {
             value: phonenumber,
-            name: name,
+            name,
+            valid: None,
+            last_online: None,
+            country: None,
+            carrier: None,
+            line: None,
+            is_ported: None,
+            last_ported: None,
+            caller_name: None,
+            caller_type: None,
+            unscoped: false,
+        }))
+    }
+}
+
+impl InsertFromString for AddPhoneNumber {
+    fn from_string(_rl: &mut Shell, phonenumber: String) -> Result<Insert> {
+        // TODO: consider doing basic validation
+        Ok(Insert::PhoneNumber(NewPhoneNumber {
+            value: phonenumber,
+            name: None,
             valid: None,
             last_online: None,
             country: None,
@@ -328,7 +426,7 @@ impl IntoInsert for AddDevice {
 
         Ok(Insert::Device(NewDevice {
             value: mac,
-            name: name,
+            name,
             hostname: None,
             vendor: None,
             last_seen: None,
@@ -393,9 +491,9 @@ impl IntoInsert for AddAccount {
         let value = format!("{}/{}", service, username);
 
         Ok(Insert::Account(NewAccount {
-            value: value,
-            service: service,
-            username: username,
+            value,
+            service,
+            username,
             displayname: None,
             email: None,
             url: None,
@@ -417,10 +515,7 @@ impl IntoInsert for AddBreach {
     fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
         let name = match self.name {
             Some(name) => name,
-            _ => {
-                let name = utils::question("Name")?;
-                name
-            }
+            _ => utils::question("Name")?,
         };
 
         Ok(Insert::Breach(NewBreach {
@@ -453,7 +548,7 @@ impl IntoInsert for AddImage {
                 let path = match path {
                     Ok(path) => path,
                     Err(err) => {
-                        let path = err.path().unwrap_or(Path::new("")).display();
+                        let path = err.path().unwrap_or_else(|| Path::new("")).display();
 
                         let err = err.io_error()
                             .map(|err| err.to_string())
@@ -575,12 +670,24 @@ impl IntoInsert for AddPort {
         };
 
         let addr = if let Some(addr) = self.addr {
+            // TODO: change this to a regular string so we don't parse->fmt->parse
             addr
         } else {
             let addr = utils::question("IP:Port")?;
             addr.parse()?
         };
 
+        Self::from_string(rl, format!("{}/{}", protocol, addr))
+    }
+}
+
+impl InsertFromString for AddPort {
+    fn from_string(rl: &mut Shell, value: String) -> Result<Insert> {
+        let idx = value.find('/')
+            .ok_or_else(|| format_err!("Could not locate `/` delimiter for protocol/ip:port"))?;
+        let protocol = value[..idx].to_string();
+        let addr = value[idx+1..].parse::<SocketAddr>()?;
+
         let family = match addr.ip() {
             net::IpAddr::V4(_) => "4",
             net::IpAddr::V6(_) => "6",
@@ -606,8 +713,6 @@ impl IntoInsert for AddPort {
             _ => bail!("IpAddr is out out of scope"),
         };
 
-        let value = format!("{}/{}", protocol, addr);
-
         Ok(Insert::Port(NewPort {
             ip_addr_id,
             value,
@@ -631,13 +736,18 @@ pub struct AddCryptoAddr {
 }
 
 impl IntoInsert for AddCryptoAddr {
-    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
         let address = if let Some(address) = self.address {
             address
         } else {
             utils::question("Address")?
         };
+        Self::from_string(rl, address)
+    }
+}
 
+impl InsertFromString for AddCryptoAddr {
+    fn from_string(_rl: &mut Shell, address: String) -> Result<Insert> {
         Ok(Insert::CryptoAddr(NewCryptoAddr {
             value: address,
             currency: None,

src/cmd/export_cmd.rs

@@ -5,7 +5,6 @@ use crate::cmd::Cmd;
 use crate::db::ttl;
 use crate::models::*;
 use crate::shell::Shell;
-use serde_json;
 use serde::Serialize;
 use std::io::{self, Write};
 use structopt::StructOpt;

src/cmd/pkg_cmd.rs

@@ -5,13 +5,14 @@ use crate::api::Client;
 use crate::args;
 use crate::config::Config;
 use crate::cmd::{Cmd, LiteCmd};
-use crate::engine::Library;
+use crate::engine::{Library, Module};
 use crate::registry::{self, InstallTask, UpdateTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
 use crate::worker;
-use colored::Colorize;
+use colored::{Color, Colorize};
 use sn0int_common::ModuleID;
+use sn0int_common::metadata::Stealth;
 use std::collections::HashSet;
 use std::fmt::Write;
 use std::sync::Arc;
@@ -71,7 +72,10 @@ pub struct List {
     pub source: Option<String>,
     /// List outdated modules
     #[structopt(long="outdated")]
-    pub outdated: bool,
+    pub outdated_only: bool,
+    /// Only show modules with equal or better stealth level
+    #[structopt(long="stealth", possible_values=Stealth::variants())]
+    pub stealth: Option<Stealth>,
     /// Filter by pattern
     #[structopt(default_value="*")]
     pub pattern: String,
@@ -96,6 +100,34 @@ enum ModuleReload {
     No,
 }
 
+#[inline]
+fn write_tag(out: &mut String, color: Color, txt: &str) -> Result<()> {
+    write!(out, " [{}]", txt.color(color))?;
+    Ok(())
+}
+
+fn print_module(module: &Module, is_outdated: bool) -> Result<()> {
+    let mut out = String::new();
+    write!(&mut out, "{}/{} {}", module.author().purple(),
+                                 module.name(),
+                                 module.version().blue())?;
+
+    match module.stealth() {
+        Stealth::Loud => write_tag(&mut out, Color::Yellow, "LOUD")?,
+        Stealth::Normal => (),
+        Stealth::Passive => write_tag(&mut out, Color::Green, "passive")?,
+        Stealth::Offline => write_tag(&mut out, Color::Green, "offline")?,
+    };
+
+    if is_outdated {
+        write_tag(&mut out, Color::Red, "outdated")?;
+    }
+
+    println!("{}", out.bold());
+    println!("    {}", module.description());
+    Ok(())
+}
+
 fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) -> Result<ModuleReload> {
     match subcommand {
         SubCommand::List(list) => {
@@ -115,16 +147,18 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
                     continue;
                 }
 
-                let mut out = String::new();
-                write!(&mut out, "{} ({})", canonical.green(),
-                                            module.version().yellow())?;
-                if autoupdate.is_outdated(&canonical) {
-                    write!(&mut out, " {}", "[outdated]".red())?;
-                } else if list.outdated {
+                if let Some(stealth) = &list.stealth {
+                    if !module.stealth().equal_or_better(&stealth) {
+                        continue;
+                    }
+                }
+
+                let is_outdated = autoupdate.is_outdated(&canonical);
+                if list.outdated_only && !is_outdated {
                     continue;
                 }
-                println!("{}", out);
-                println!("\t{}", module.description());
+
+                print_module(module, is_outdated)?;
             }
             Ok(ModuleReload::No)
         },

src/cmd/run_cmd.rs

@@ -14,7 +14,6 @@ use crate::term;
 use crate::utils;
 use crate::worker;
 use serde::Serialize;
-use serde_json;
 use sn0int_common::metadata::Source;
 use sn0int_std::ratelimits::Ratelimiter;
 use std::collections::HashMap;
@@ -162,14 +161,14 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
     let keyring = rl.keyring().request_keys(&module);
 
     let dns_config = Resolver::from_system_v4()?;
-    let proxy = rl.config().network.proxy.clone();
+    let proxy = rl.config().network.proxy;
 
     let args = get_args(rl, &module)?;
     for (arg, _pretty_arg, blobs) in args {
         let start_cmd = StartCommand::new(params.verbose,
                                           keyring.clone(),
                                           dns_config.clone(),
-                                          proxy.clone(),
+                                          proxy,
                                           options.clone(),
                                           module.clone(),
                                           arg,
@@ -190,7 +189,7 @@ pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>)
     let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
-    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, rl.config().network.proxy.clone(), options);
+    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, rl.config().network.proxy, options);
     rl.signal_register().reset_ctrlc();
 
     if errors > 0 {

src/cmd/select_cmd.rs

@@ -5,7 +5,6 @@ use crate::db::ttl;
 use crate::filters::{Target, Filter};
 use crate::shell::Shell;
 use serde::Serialize;
-use serde_json;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::models::*;

src/cmd/workspace_cmd.rs

@@ -28,10 +28,8 @@ pub struct Args {
 }
 
 fn delete(workspace: Workspace, force: bool) -> Result<()> {
-    if !force {
-        if !utils::no_else_yes(&format!("Do you really want to delete {:?}", workspace.as_str()))? {
-            return Ok(());
-        }
+    if !force && !utils::no_else_yes(&format!("Do you really want to delete {:?}", workspace.as_str()))? {
+        return Ok(());
     }
 
     term::info(&format!("Deleting workspace: {:?}", workspace.as_str()));

src/config.rs

@@ -1,12 +1,9 @@
-use dirs;
 use crate::errors::*;
 use crate::notify::NotificationConfig;
 use std::collections::HashMap;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::net::SocketAddr;
-use toml;
-
 
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct Config {
@@ -27,7 +24,7 @@ impl Config {
     }
 
     pub fn path() -> Result<PathBuf> {
-        let path = dirs::config_dir()
+        let path = dirs_next::config_dir()
             .ok_or_else(|| format_err!("Failed to find config directory"))?;
         let path = path.join("sn0int.toml");
         Ok(path)

src/db/mod.rs

@@ -1,6 +1,5 @@
 use crate::errors::*;
 
-use diesel;
 use diesel::expression::SqlLiteral;
 use diesel::expression::sql_literal::sql;
 use diesel::sql_types::Bool;
@@ -25,10 +24,7 @@ pub enum DbChange {
 
 impl DbChange {
     pub fn is_some(&self) -> bool {
-        match self {
-            DbChange::None => false,
-            _ => true,
-        }
+        !matches!(self, DbChange::None)
     }
 }
 

src/db/ttl.rs

@@ -6,7 +6,6 @@ use crate::schema::*;
 use crate::shell::Shell;
 use crate::term::{self, Term};
 use chrono::{NaiveDateTime, Duration, Utc};
-use diesel;
 use diesel::prelude::*;
 use sn0int_std::ratelimits::Ratelimiter;
 
@@ -21,25 +20,7 @@ pub struct Ttl {
     pub expire: NaiveDateTime,
 }
 
-#[derive(Insertable)]
-#[table_name="ttls"]
-pub struct NewTtl<'a> {
-    pub family: &'a str,
-    pub key: i32,
-    pub value: String,
-    pub expire: NaiveDateTime,
-}
-
 impl Ttl {
-    pub fn new(obj: &Insert, key: i32, value: String, expire: NaiveDateTime) -> NewTtl {
-        NewTtl {
-            family: obj.table(),
-            key,
-            value,
-            expire,
-        }
-    }
-
     pub fn find(obj: &Insert, my_key: i32, db: &Database) -> Result<Option<Ttl>> {
         use crate::schema::ttls::dsl::*;
 
@@ -135,6 +116,15 @@ impl Ttl {
     }
 }
 
+#[derive(Insertable)]
+#[table_name="ttls"]
+pub struct NewTtl<'a> {
+    pub family: &'a str,
+    pub key: i32,
+    pub value: String,
+    pub expire: NaiveDateTime,
+}
+
 pub fn reap_expired(rl: &mut Shell) -> Result<()> {
     debug!("Reaping expired entities");
 

src/engine/ctx.rs

@@ -5,17 +5,17 @@ use crate::engine::{Environment, IpcChild};
 use crate::geoip::{MaxmindReader, GeoIP, AsnDB};
 use crate::hlua::{self, AnyLuaValue};
 use crate::keyring::KeyRingEntry;
+use crate::lazy::Lazy;
 use crate::models::*;
 use crate::psl::{Psl, PslReader};
-use crate::lazy::Lazy;
+use crate::ratelimits::RatelimitResponse;
 use crate::runtime;
 use crate::sockets::{Socket, SocketOptions, TlsData};
+use crate::utils;
 use crate::web::{HttpSession, HttpRequest, RequestOptions};
 use crate::websockets::{WebSocket, WebSocketOptions};
 use crate::worker::{Event, LogEvent, DatabaseEvent, DatabaseResponse, StdioEvent, RatelimitEvent};
-use crate::ratelimits::RatelimitResponse;
 use chrootable_https::{self, Resolver};
-use serde_json;
 use sn0int_std::blobs::{Blob, BlobState};
 use sn0int_std::mqtt::{MqttClient, MqttOptions};
 use sn0int_std::web::WebState;
@@ -24,8 +24,6 @@ use std::result;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
 use std::thread;
-use rand::prelude::*;
-use rand::distributions::Alphanumeric;
 
 
 pub trait State {
@@ -80,14 +78,14 @@ pub trait State {
     }
 
     fn db_insert(&self, object: Insert) -> Result<DatabaseResponse> {
-        self.send(&Event::Database(DatabaseEvent::Insert(object)));
+        self.send(&Event::Database(Box::new(DatabaseEvent::Insert(object))));
         self.db_recv()
             .context("Failed to add to database")
             .map_err(Error::from)
     }
 
     fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<DatabaseResponse> {
-        self.send(&Event::Database(DatabaseEvent::InsertTtl((object, ttl))));
+        self.send(&Event::Database(Box::new(DatabaseEvent::InsertTtl((object, ttl)))));
         self.db_recv()
             .context("Failed to add to database")
             .map_err(Error::from)
@@ -96,7 +94,7 @@ pub trait State {
     fn db_activity(&self, activity: InsertActivity) -> Result<bool> {
         let activity = activity.try_into_new()?;
 
-        self.send(&Event::Database(DatabaseEvent::Activity(activity)));
+        self.send(&Event::Database(Box::new(DatabaseEvent::Activity(activity))));
         let r = self.db_recv()
             .context("Failed to log activity")?;
 
@@ -108,14 +106,14 @@ pub trait State {
     }
 
     fn db_select(&self, family: Family, value: String) -> Result<DatabaseResponse> {
-        self.send(&Event::Database(DatabaseEvent::Select((family, value))));
+        self.send(&Event::Database(Box::new(DatabaseEvent::Select((family, value)))));
         self.db_recv()
             .context("Failed to query database")
             .map_err(Error::from)
     }
 
     fn db_update(&self, family: Family, value: String, update: Update) -> Result<DatabaseResponse> {
-        self.send(&Event::Database(DatabaseEvent::Update((family, value, update))));
+        self.send(&Event::Database(Box::new(DatabaseEvent::Update((family, value, update)))));
         self.db_recv()
             .context("Failed to update database")
             .map_err(Error::from)
@@ -152,7 +150,7 @@ pub trait State {
 
     #[inline]
     fn random_id(&self) -> String {
-        thread_rng().sample_iter(&Alphanumeric).take(16).collect()
+        utils::random_string(16)
     }
 
     fn keyring(&self, namespace: &str) -> Vec<&KeyRingEntry>;
@@ -373,7 +371,7 @@ impl State for LuaState {
         let mtx = self.http_sessions.lock().unwrap();
         let session = mtx.get(session_id).expect("Invalid session reference"); // TODO
 
-        HttpRequest::new(&session, method, url, options)
+        HttpRequest::new(&session, method, url, options, || format!("sn0int/{}", env!("CARGO_PKG_VERSION")))
     }
 
     fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
@@ -620,6 +618,11 @@ impl Script {
         })
     }
 
+    #[inline]
+    pub fn code(&self) -> &str {
+        &self.code
+    }
+
     pub fn run(&self, env: Environment,
                       tx: Arc<Mutex<Box<dyn IpcChild>>>,
                       arg: AnyLuaValue,
@@ -681,6 +684,6 @@ a.prod.fastly.net
             geoip,
             asn,
         };
-        self.run(env, DummyIpcChild::new(), AnyLuaValue::LuaNil)
+        self.run(env, DummyIpcChild::create(), AnyLuaValue::LuaNil)
     }
 }

src/engine/mod.rs

@@ -14,14 +14,13 @@ use std::sync::{Arc, Mutex};
 use crate::engine::ctx::Script;
 use crate::ipc::child::IpcChild;
 use sn0int_common::ModuleID;
-use sn0int_common::metadata::{Metadata, Source};
+use sn0int_common::metadata::{Metadata, Source, Stealth};
 use chrootable_https::dns::Resolver;
 use crate::psl::PslReader;
 use crate::paths;
 use std::cmp::Ordering;
 use std::path::Path;
 use crate::term;
-use crate::worker;
 
 pub mod ctx;
 pub use sn0int_std::engine::structs;
@@ -68,11 +67,9 @@ impl<'a> Library<'a> {
     }
 
     pub fn reload_modules(&mut self) -> Result<usize> {
-        let modules = worker::spawn_fn("Loading modules", || {
-            self.reload_modules_quiet()
-                .context("Failed to load modules")?;
-            Ok(self.list().len())
-        }, true)?;
+        self.reload_modules_quiet()
+            .context("Failed to load modules")?;
+        let modules = self.list().len();
         term::info(&format!("Loaded {} modules", modules));
         Ok(modules)
     }
@@ -124,7 +121,7 @@ impl<'a> Library<'a> {
                 let folder = folder.strip_prefix("~/")
                     .unwrap_or(&folder);
 
-                dirs::home_dir()
+                dirs_next::home_dir()
                     .ok_or_else(|| format_err!("Failed to find home folder"))?
                     .join(folder)
             };
@@ -229,6 +226,8 @@ pub struct Module {
     version: String,
     source: Option<Source>,
     keyring_access: Vec<String>,
+    stealth: Stealth,
+
     private_module: bool,
     script: Script,
 }
@@ -240,7 +239,7 @@ impl Module {
             .context("Failed to read module")?;
 
         let metadata = code.parse::<Metadata>()
-            .context("Failed to parse module metadata")?;
+            .map_err(|e| format_err!("Failed to parse module metadata: {}", e))?;
 
         let script = Script::load_unchecked(code)?;
 
@@ -251,11 +250,18 @@ impl Module {
             version: metadata.version,
             source: metadata.source,
             keyring_access: metadata.keyring_access,
+            stealth: metadata.stealth,
+
             private_module,
             script,
         })
     }
 
+    #[inline]
+    pub fn author(&self) -> &str {
+        &self.author
+    }
+
     #[inline]
     pub fn name(&self) -> &str {
         &self.name
@@ -293,11 +299,21 @@ impl Module {
         &self.keyring_access
     }
 
+    #[inline]
+    pub fn stealth(&self) -> &Stealth {
+        &self.stealth
+    }
+
     #[inline]
     pub fn is_private(&self) -> bool {
         self.private_module
     }
 
+    #[inline]
+    pub fn code(&self) -> &str {
+        self.script.code()
+    }
+
     pub fn run(&self, env: Environment, ipc_child: Arc<Mutex<Box<dyn IpcChild>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
         self.script.run(env, ipc_child, arg.into())

src/ipc/child.rs

@@ -3,15 +3,12 @@ use crate::ipc::common::*;
 use crate::engine::Environment;
 use crate::geoip::MaxmindReader;
 use crate::psl::PslReader;
-use serde_json;
 use crate::worker::Event;
-
 use std::fmt::Debug;
 use std::io::prelude::*;
 use std::io::{self, Stdin, Stdout};
 use std::sync::{Arc, Mutex};
 
-
 pub trait IpcChild: Debug {
     fn send(&mut self, event: &Event) -> Result<()>;
 
@@ -65,7 +62,7 @@ impl IpcChild for StdioIpcChild {
 pub struct DummyIpcChild;
 
 impl DummyIpcChild {
-    pub fn new() -> Arc<Mutex<Box<dyn IpcChild>>> {
+    pub fn create() -> Arc<Mutex<Box<dyn IpcChild>>> {
         Arc::new(Mutex::new(Box::new(DummyIpcChild)))
     }
 }

src/ipc/common.rs

@@ -2,12 +2,9 @@ use chrootable_https::dns::Resolver;
 use crate::blobs::Blob;
 use crate::engine::Module;
 use crate::keyring::KeyRingEntry;
-use serde_json;
-
 use std::collections::HashMap;
 use std::net::SocketAddr;
 
-
 #[derive(Debug, Serialize, Deserialize)]
 pub struct StartCommand {
     pub verbose: u64,

src/ipc/parent.rs

@@ -4,9 +4,7 @@ use chrootable_https::dns::Resolver;
 use crate::blobs::Blob;
 use crate::engine::Module;
 use crate::keyring::KeyRingEntry;
-use serde_json;
 use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
-
 use std::collections::HashMap;
 use std::env;
 use std::ffi::OsString;
@@ -16,7 +14,6 @@ use std::net::SocketAddr;
 use std::sync::mpsc;
 use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
 
-
 pub struct IpcParent {
     child: Child,
     stdin: ChildStdin,
@@ -65,7 +62,7 @@ impl IpcParent {
 
     pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
         let value = serde_json::to_value(value).expect("Failed to serialize reply");
-        if let Err(_) = self.send(&value) {
+        if self.send(&value).is_err() {
             tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
         }
     }
@@ -125,7 +122,7 @@ pub fn run(module: Module,
     let exit = loop {
         match ipc_parent.recv()? {
             Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => ipc_parent.send_event_callback(object, &tx),
+            Event::Database(object) => ipc_parent.send_event_callback(*object, &tx),
             Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
             Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
             Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),

src/lib.rs

@@ -1,6 +1,9 @@
+#![allow(clippy::too_many_arguments)]
+#![allow(clippy::type_complexity)]
+#![allow(clippy::large_enum_variant)]
+
 #![warn(unused_extern_crates)]
 #[macro_use] extern crate maplit;
-use url;
 use hlua_badtouch as hlua;
 #[macro_use] extern crate serde_derive;
 #[macro_use] extern crate structopt;

src/main.rs

@@ -27,9 +27,9 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         let path = Path::new(&args.module);
 
         let filename = path.file_stem()
-            .ok_or(format_err!("Failed to decode filename"))?
+            .ok_or_else(|| format_err!("Failed to decode filename"))?
             .to_str()
-            .ok_or(format_err!("Failed to decode filename"))?;
+            .ok_or_else(|| format_err!("Failed to decode filename"))?;
 
         Module::load(&path.to_path_buf(), "anonymous", &filename, true)
             .context(format!("Failed to parse {:?}", path))?
@@ -71,6 +71,7 @@ fn run_cmd<T: cmd::Cmd>(gargs: &Args, args: T, config: &Config) -> Result<()> {
 }
 
 fn run_new(_gargs: &Args, args: &args::New) -> Result<()> {
+    // TODO: Add `-- Author: anonymous`
     let boilerplate = b"-- Description: TODO your description here
 -- Version: 0.1.0
 -- License: GPL-3.0

src/models/account.rs

@@ -1,11 +1,9 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use chrono::NaiveDateTime;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="accounts"]
 pub struct Account {
@@ -210,7 +208,7 @@ impl Detailed for Account {
             displayname: self.displayname.clone(),
             email: self.email.clone(),
             url: self.url.clone(),
-            last_seen: self.last_seen.clone(),
+            last_seen: self.last_seen,
             unscoped: self.unscoped,
             birthday: self.birthday.clone(),
             phonenumber: self.phonenumber.clone(),

src/models/activity.rs

@@ -1,6 +1,5 @@
 use crate::errors::*;
 use crate::schema::activity;
-use diesel;
 use diesel::prelude::*;
 use diesel::query_builder::BoxedSelectStatement;
 use crate::models::*;
@@ -8,7 +7,6 @@ use chrono::NaiveDateTime;
 use std::convert::TryFrom;
 use std::io::Write;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="activity"]
 pub struct Activity {

src/models/breach.rs

@@ -1,10 +1,8 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="breaches"]
 pub struct Breach {

src/models/breach_email.rs

@@ -1,9 +1,7 @@
 use crate::errors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 
-
 #[derive(Identifiable, Queryable, Associations, Serialize, Deserialize)]
 #[belongs_to(Breach)]
 #[belongs_to(Email)]
@@ -136,8 +134,8 @@ impl Printable<PrintableBreachEmail> for BreachEmail {
         let breach = Breach::by_id(db, self.breach_id)?;
         let email = Email::by_id(db, self.email_id)?;
         Ok(PrintableBreachEmail {
-            breach: breach.value.to_string(),
-            email: email.value.to_string(),
+            breach: breach.value,
+            email: email.value,
             password: self.password.clone(),
         })
     }
@@ -167,8 +165,8 @@ impl Printable<PrintableBreachEmail> for NewBreachEmail {
         let breach = Breach::by_id(db, self.breach_id)?;
         let email = Email::by_id(db, self.email_id)?;
         Ok(PrintableBreachEmail {
-            breach: breach.value.to_string(),
-            email: email.value.to_string(),
+            breach: breach.value,
+            email: email.value,
             password: self.password.clone(),
         })
     }

src/models/cryptoaddr.rs

@@ -1,11 +1,9 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use chrono::NaiveDateTime;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="cryptoaddrs"]
 pub struct CryptoAddr {
@@ -220,11 +218,11 @@ impl Detailed for CryptoAddr {
             id: self.id,
             value: self.value.to_string(),
             currency: self.currency.clone(),
-            denominator: self.denominator.clone(),
-            balance: self.balance.clone(),
-            received: self.received.clone(),
-            first_seen: self.first_seen.clone(),
-            last_withdrawal: self.last_withdrawal.clone(),
+            denominator: self.denominator,
+            balance: self.balance,
+            received: self.received,
+            first_seen: self.first_seen,
+            last_withdrawal: self.last_withdrawal,
             unscoped: self.unscoped,
             description: self.description.clone(),
         })

src/models/device.rs

@@ -1,11 +1,9 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use chrono::NaiveDateTime;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="devices"]
 pub struct Device {
@@ -216,7 +214,7 @@ impl Detailed for Device {
             ipaddr,
             network,
             unscoped: self.unscoped,
-            last_seen: self.last_seen.clone(),
+            last_seen: self.last_seen,
         })
     }
 }

src/models/domain.rs

@@ -1,10 +1,8 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="domains"]
 pub struct Domain {

src/models/email.rs

@@ -1,10 +1,8 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="emails"]
 pub struct Email {

src/models/image.rs

@@ -1,13 +1,11 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
 use chrono::NaiveDateTime;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use std::sync::Arc;
 use crate::engine::ctx::State;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="images"]
 pub struct Image {
@@ -174,8 +172,8 @@ impl Printable<PrintableImage> for Image {
         Ok(PrintableImage {
             value: self.value.to_string(),
             filename: self.filename.clone(),
-            width: self.width.clone(),
-            height: self.height.clone(),
+            width: self.width,
+            height: self.height,
         })
     }
 }
@@ -256,14 +254,14 @@ impl Detailed for Image {
 
             filename: self.filename.clone(),
             mime: self.mime.clone(),
-            width: self.width.clone(),
-            height: self.height.clone(),
-            created: self.created.clone(),
+            width: self.width,
+            height: self.height,
+            created: self.created,
 
-            latitude: self.latitude.clone(),
-            longitude: self.longitude.clone(),
+            latitude: self.latitude,
+            longitude: self.longitude,
 
-            nudity: self.nudity.clone(),
+            nudity: self.nudity,
             ahash: self.ahash.clone(),
             dhash: self.dhash.clone(),
             phash: self.phash.clone(),
@@ -341,8 +339,8 @@ impl Printable<PrintableImage> for NewImage {
         Ok(PrintableImage {
             value: self.value.to_string(),
             filename: self.filename.clone(),
-            width: self.width.clone(),
-            height: self.height.clone(),
+            width: self.width,
+            height: self.height,
         })
     }
 }

src/models/ipaddr.rs

@@ -2,12 +2,10 @@ use crate::errors::*;
 use crate::fmt::Write;
 use crate::fmt::colors::*;
 use crate::models::*;
-use diesel;
 use diesel::prelude::*;
 use std::net;
 use std::result;
 
-
 #[derive(Identifiable, Queryable, Associations, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="ipaddrs"]
 pub struct IpAddr {

src/models/netblock.rs

@@ -1,10 +1,7 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
-use ipnetwork;
-
 
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="netblocks"]

src/models/network.rs

@@ -1,10 +1,8 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="networks"]
 pub struct Network {
@@ -199,8 +197,8 @@ impl Detailed for Network {
             id: self.id,
             value: self.value.to_string(),
             unscoped: self.unscoped,
-            latitude: self.latitude.clone(),
-            longitude: self.longitude.clone(),
+            latitude: self.latitude,
+            longitude: self.longitude,
             description: self.description.clone(),
             devices,
         })

src/models/network_device.rs

@@ -1,10 +1,8 @@
 use crate::errors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use chrono::NaiveDateTime;
 
-
 #[derive(Identifiable, Queryable, Associations, Serialize, Deserialize)]
 #[belongs_to(Network)]
 #[belongs_to(Device)]
@@ -116,8 +114,8 @@ impl Printable<PrintableNetworkDevice> for NetworkDevice {
         let network = Network::by_id(db, self.network_id)?;
         let device = Device::by_id(db, self.device_id)?;
         Ok(PrintableNetworkDevice {
-            network: network.value.to_string(),
-            device: device.value.to_string(),
+            network: network.value,
+            device: device.value,
         })
     }
 }
@@ -148,8 +146,8 @@ impl Printable<PrintableNetworkDevice> for NewNetworkDevice {
         let network = Network::by_id(db, self.network_id)?;
         let device = Device::by_id(db, self.device_id)?;
         Ok(PrintableNetworkDevice {
-            network: network.value.to_string(),
-            device: device.value.to_string(),
+            network: network.value,
+            device: device.value,
         })
     }
 }

src/models/phonenumber.rs

@@ -2,10 +2,8 @@ use crate::errors::*;
 use crate::fmt::colors::*;
 use crate::models::*;
 use chrono::NaiveDateTime;
-use diesel;
 use diesel::prelude::*;
 
-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="phonenumbers"]
 pub struct PhoneNumber {

src/models/port.rs

@@ -2,11 +2,9 @@ use crate::errors::*;
 use crate::fmt::Write;
 use crate::fmt::colors::*;
 use crate::models::*;
-use diesel;
 use diesel::prelude::*;
 use std::net::{self, SocketAddr};
 
-
 #[derive(Identifiable, Queryable, Associations, Serialize, Deserialize, PartialEq, Debug)]
 #[belongs_to(IpAddr)]
 #[table_name="ports"]

src/models/subdomain.rs

@@ -1,11 +1,9 @@
 use crate::errors::*;
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use std::result;
 
-
 #[derive(Identifiable, Queryable, Associations, Serialize, Deserialize, PartialEq, Debug)]
 #[belongs_to(Domain)]
 #[table_name="subdomains"]

src/models/subdomain_ipaddr.rs

@@ -1,10 +1,8 @@
 use crate::errors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use std::net;
 
-
 #[derive(Identifiable, Queryable, Associations, Serialize, Deserialize)]
 #[belongs_to(Subdomain)]
 #[belongs_to(IpAddr)]
@@ -108,7 +106,7 @@ impl Printable<PrintableSubdomainIpAddr> for SubdomainIpAddr {
         let subdomain = Subdomain::by_id(db, self.subdomain_id)?;
         let ipaddr = IpAddr::by_id(db, self.ip_addr_id)?;
         Ok(PrintableSubdomainIpAddr {
-            subdomain: subdomain.value.to_string(),
+            subdomain: subdomain.value,
             ipaddr: ipaddr.value.parse()?,
         })
     }
@@ -126,7 +124,7 @@ impl Printable<PrintableSubdomainIpAddr> for NewSubdomainIpAddr {
         let subdomain = Subdomain::by_id(db, self.subdomain_id)?;
         let ipaddr = IpAddr::by_id(db, self.ip_addr_id)?;
         Ok(PrintableSubdomainIpAddr {
-            subdomain: subdomain.value.to_string(),
+            subdomain: subdomain.value,
             ipaddr: ipaddr.value.parse()?,
         })
     }

src/models/url.rs

@@ -3,10 +3,7 @@ use crate::errors::*;
 use crate::fmt::Write;
 use crate::fmt::colors::*;
 use crate::models::*;
-use diesel;
 use diesel::prelude::*;
-use crate::url;
-
 
 #[derive(Identifiable, Queryable, Associations, Serialize, Deserialize, PartialEq, Debug)]
 #[belongs_to(Subdomain)]
@@ -292,7 +289,7 @@ impl InsertToNew for InsertUrl {
     type Target = NewUrl;
 
     fn try_into_new(self) -> Result<NewUrl> {
-        let url = url::Url::parse(&self.value)?;
+        let url = ::url::Url::parse(&self.value)?;
         let path = url.path().to_string();
 
         let redirect = if let Some(redirect) = self.redirect {

src/notify/mod.rs

@@ -1,3 +1,5 @@
+mod rules;
+
 use crate::errors::*;
 use crate::cmd::run_cmd::prepare_keyring;
 use crate::cmd::run_cmd::Params;
@@ -6,14 +8,11 @@ use crate::options;
 use crate::shell::Shell;
 use crate::term::SpinLogger;
 use crate::worker;
-use serde::de::{self, Deserialize, Deserializer};
-use serde::ser::{Serialize, Serializer};
+use self::rules::Glob;
 use sn0int_common::metadata::Source;
 use sn0int_std::blobs::Blob;
 use sn0int_std::ratelimits::Ratelimiter;
 use std::collections::HashMap;
-use std::result;
-use std::str::FromStr;
 
 #[derive(Debug, StructOpt, Serialize)]
 pub struct Notification {
@@ -32,76 +31,33 @@ pub struct NotificationConfig {
     pub options: Vec<options::Opt>,
 }
 
-#[derive(Debug, Clone)]
-pub struct Glob {
-    patterns: Vec<glob::Pattern>,
-    src: String,
-}
-
-impl Glob {
-    fn matches(&self, topic: &str) -> bool {
-        let mut filter = self.patterns.iter();
-        let mut topic = topic.split(':');
-
-        loop {
-            match (filter.next(), topic.next()) {
-                (Some(filter), Some(topic)) => if !filter.matches(&topic) {
+fn apply_rule<T>(name: &str, filters: &[T], value: &str, cmp: fn(&T, &str) -> Option<bool>) -> bool {
+    if !filters.is_empty() {
+        debug!("{} filter is active", name);
+        for filter in filters {
+            match cmp(filter, value) {
+                Some(true) => {
+                    debug!("{} was allow-listed", name);
+                    return true;
+                }
+                Some(false) => {
+                    debug!("{} was excluded", name);
                     return false;
-                },
-                (None, None) => return true,
-                (_, _) => return false,
+                }
+                _ => (),
             }
         }
+        debug!("{} didn't match any rules, skipping", name);
+        false
+    } else {
+        true
     }
 }
 
-impl FromStr for Glob {
-    type Err = Error;
-
-    fn from_str(s: &str) -> Result<Glob> {
-        let patterns = s.split(':')
-            .map(|s| glob::Pattern::new(s).map_err(Error::from))
-            .collect::<Result<Vec<_>>>()?;
-        Ok(Glob {
-            patterns,
-            src: s.to_string(),
-        })
-    }
-}
-
-impl Serialize for Glob {
-    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
-        where S: Serializer
-    {
-        serializer.serialize_str(&self.src)
-    }
-}
-
-impl<'de> Deserialize<'de> for Glob {
-    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-        where D: Deserializer<'de>
-    {
-        let s = String::deserialize(deserializer)?;
-        FromStr::from_str(&s).map_err(de::Error::custom)
-    }
-}
-
-fn apply_rule<T>(name: &str, filter: &[T], value: &str, cmp: fn(&T, &str) -> bool) -> bool {
-    if !filter.is_empty() {
-        debug!("{} filter is active", name);
-        if !filter.iter().any(|filter| cmp(filter, value)) {
-            debug!("{} isn't allow-listed, aborting", name);
-            return false;
-        }
-        debug!("{} was allow-listed", name);
-    }
-    true
-}
-
 impl NotificationConfig {
     fn matches(&self, name: &str, workspace: &str, topic: &str) -> bool {
         debug!("Testing notification with rules: {:?}", name);
-        if !apply_rule("workspace", &self.workspaces, workspace, |filter, value| filter == value) {
+        if !apply_rule("workspace", &self.workspaces, workspace, |filter, value| if filter == value { Some(true) } else { None }) {
             return false;
         }
         if !apply_rule("topic", &self.topics, topic, |filter, value| filter.matches(value)) {
@@ -143,7 +99,7 @@ pub fn exec(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, option
     let args = vec![prepare_arg(&notification)?];
 
     debug!("Executing notification module {:?}", module_name);
-    let errors = worker::spawn(rl, &module, ratelimit, args, &params, rl.config().network.proxy.clone(), options);
+    let errors = worker::spawn(rl, &module, ratelimit, args, &params, rl.config().network.proxy, options);
     debug!("Notification module {:?} exited with {:?} errors", module_name, errors);
 
     Ok(errors)
@@ -190,58 +146,98 @@ pub fn run_router<T: SpinLogger>(rl: &mut Shell, spinner: &mut T, ratelimit: &mu
 mod tests {
     use super::*;
 
-    fn match_topic_str(filter: &str, value: &str) -> bool {
-        let filter: Glob = filter.parse().unwrap();
-        filter.matches(value)
+    fn mkconfig(topics: &[&str]) -> NotificationConfig {
+        let topics = topics.iter()
+            .map(|s| s.parse::<Glob>())
+            .collect::<Result<Vec<_>>>().unwrap();
+
+        NotificationConfig {
+            workspaces: Vec::new(),
+            topics,
+            script: "some/script".to_string(),
+            options: Vec::new(),
+        }
     }
 
     #[test]
-    fn test_match_topic_exact() {
-        assert!(match_topic_str("topic:hello-world", "topic:hello-world"));
+    fn test_empty_topic() {
+        let config = mkconfig(&[]);
+        assert!(config.matches("name", "workspace", "topic"));
     }
 
     #[test]
-    fn test_match_topic_starts_with() {
-        assert!(match_topic_str("topic:*", "topic:hello-world"));
+    fn test_match_topic() {
+        let config = mkconfig(&[
+            "db:subdomain:example.com:*",
+        ]);
+        assert!(config.matches("name", "workspace", "db:subdomain:example.com:update"));
     }
 
     #[test]
-    fn test_match_topic_ends_with() {
-        assert!(match_topic_str("*:hello-world", "topic:hello-world"));
+    fn test_not_match_topic() {
+        let config = mkconfig(&[
+            "db:subdomain:example.com:*",
+        ]);
+        assert!(!config.matches("name", "workspace", "db:subdomain:foobar.com:update"));
     }
 
     #[test]
-    fn test_match_topic_one_wildcard_one_section() {
-        assert!(match_topic_str("a:*:z", "a:b:z"));
+    fn test_exclude_topic() {
+        let config = mkconfig(&[
+            "!db:subdomain:example.com:*",
+        ]);
+        assert!(!config.matches("name", "workspace", "db:subdomain:example.com:update"));
     }
 
     #[test]
-    fn test_match_topic_one_wildcard_not_two_sections() {
-        assert!(!match_topic_str("a:*:z", "a:b:c:z"));
+    fn test_exclude_other_topic() {
+        let config = mkconfig(&[
+            "!db:subdomain:example.com:*",
+            "db:subdomain:foobar.com:*",
+        ]);
+        assert!(config.matches("name", "workspace", "db:subdomain:foobar.com:update"));
     }
 
     #[test]
-    fn test_match_topic_two_wildcards_two_sections() {
-        assert!(match_topic_str("a:*:*:z", "a:b:c:z"));
+    fn test_no_inverse_does_not_imply_match() {
+        let config = mkconfig(&[
+            "!db:subdomain:example.com:*",
+        ]);
+        assert!(!config.matches("name", "workspace", "db:subdomain:foobar.com:update"));
     }
 
     #[test]
-    fn test_match_topic_one_wildcard_not_two_sections_start() {
-        assert!(!match_topic_str("a:*", "a:b:c"));
+    fn test_everything_except() {
+        let config = mkconfig(&[
+            "!db:subdomain:example.com:*",
+            "*:*:*:*",
+        ]);
+        assert!(config.matches("name", "workspace", "db:subdomain:foobar.com:update"));
     }
 
     #[test]
-    fn test_match_topic_one_wildcard_not_two_sections_end() {
-        assert!(!match_topic_str("*:z", "b:c:z"));
+    fn test_exclude_everything() {
+        let config = mkconfig(&[
+            "!*:*:*:*",
+        ]);
+        assert!(!config.matches("name", "workspace", "db:subdomain:foobar.com:update"));
     }
 
     #[test]
-    fn test_match_topic_many_wildcards() {
-        assert!(match_topic_str("a:*:*:d:e:*:g:*:z", "a:b:c:d:e:f:g:h:z"));
+    fn test_execute_in_order_1() {
+        let config = mkconfig(&[
+            "!*:*:*:*",
+            "*:*:*:*",
+        ]);
+        assert!(!config.matches("name", "workspace", "db:subdomain:foobar.com:update"));
     }
 
     #[test]
-    fn test_match_topic_empty_filter() {
-        assert!(!match_topic_str("", "abc"));
+    fn test_execute_in_order_2() {
+        let config = mkconfig(&[
+            "*:*:*:*",
+            "!*:*:*:*",
+        ]);
+        assert!(config.matches("name", "workspace", "db:subdomain:foobar.com:update"));
     }
 }

src/notify/rules.rs

@@ -0,0 +1,138 @@
+use crate::errors::*;
+use serde::de::{self, Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
+use std::result;
+use std::str::FromStr;
+
+#[derive(Debug, Clone)]
+pub struct Glob {
+    patterns: Vec<glob::Pattern>,
+    src: String,
+    inverse: bool,
+}
+
+impl Glob {
+    pub fn matches(&self, topic: &str) -> Option<bool> {
+        let mut filter = self.patterns.iter();
+        let mut topic = topic.split(':');
+
+        loop {
+            match (filter.next(), topic.next()) {
+                (Some(filter), Some(topic)) => if !filter.matches(&topic) {
+                    return None;
+                },
+                (None, None) => return Some(!self.inverse),
+                (_, _) => return None,
+            }
+        }
+    }
+}
+
+impl FromStr for Glob {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Glob> {
+        let (s, inverse) = if let Some(stripped) = s.strip_prefix('!') {
+            (stripped, true)
+        } else {
+            (s, false)
+        };
+
+        let patterns = s.split(':')
+            .map(|s| glob::Pattern::new(s).map_err(Error::from))
+            .collect::<Result<Vec<_>>>()?;
+        Ok(Glob {
+            patterns,
+            src: s.to_string(),
+            inverse,
+        })
+    }
+}
+
+impl Serialize for Glob {
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+        where S: Serializer
+    {
+        serializer.serialize_str(&self.src)
+    }
+}
+
+impl<'de> Deserialize<'de> for Glob {
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+        where D: Deserializer<'de>
+    {
+        let s = String::deserialize(deserializer)?;
+        FromStr::from_str(&s).map_err(de::Error::custom)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn match_topic_str(filter: &str, value: &str) -> bool {
+        let filter: Glob = filter.parse().unwrap();
+        filter.matches(value).unwrap_or(false)
+    }
+
+    #[test]
+    fn test_match_topic_exact() {
+        assert!(match_topic_str("topic:hello-world", "topic:hello-world"));
+    }
+
+    #[test]
+    fn test_match_topic_starts_with() {
+        assert!(match_topic_str("topic:*", "topic:hello-world"));
+    }
+
+    #[test]
+    fn test_match_topic_ends_with() {
+        assert!(match_topic_str("*:hello-world", "topic:hello-world"));
+    }
+
+    #[test]
+    fn test_match_topic_one_wildcard_one_section() {
+        assert!(match_topic_str("a:*:z", "a:b:z"));
+    }
+
+    #[test]
+    fn test_match_topic_one_wildcard_not_two_sections() {
+        assert!(!match_topic_str("a:*:z", "a:b:c:z"));
+    }
+
+    #[test]
+    fn test_match_topic_two_wildcards_two_sections() {
+        assert!(match_topic_str("a:*:*:z", "a:b:c:z"));
+    }
+
+    #[test]
+    fn test_match_topic_one_wildcard_not_two_sections_start() {
+        assert!(!match_topic_str("a:*", "a:b:c"));
+    }
+
+    #[test]
+    fn test_match_topic_one_wildcard_not_two_sections_end() {
+        assert!(!match_topic_str("*:z", "b:c:z"));
+    }
+
+    #[test]
+    fn test_match_topic_many_wildcards() {
+        assert!(match_topic_str("a:*:*:d:e:*:g:*:z", "a:b:c:d:e:f:g:h:z"));
+    }
+
+    #[test]
+    fn test_match_topic_empty_filter() {
+        assert!(!match_topic_str("", "abc"));
+    }
+
+    #[test]
+    fn test_inverse_match() {
+        assert!(!match_topic_str("!topic:*", "topic:hello-world"));
+    }
+
+    #[test]
+    fn test_no_inverse_match() {
+        // Just because an inverse rule doesn't match this doesn't imply a match
+        assert!(!match_topic_str("!foo:*", "topic:hello-world"));
+    }
+}

src/paths.rs

@@ -1,14 +1,11 @@
 use crate::config::Config;
 use crate::errors::*;
 use crate::workspaces::Workspace;
-
-use dirs;
 use std::fs;
 use std::path::PathBuf;
 
-
 pub fn sn0int_dir() -> Result<PathBuf> {
-    let path = dirs::data_dir()
+    let path = dirs_next::data_dir()
         .ok_or_else(|| format_err!("Failed to find data directory"))?;
     let path = path.join("sn0int");
     fs::create_dir_all(&path)
@@ -56,7 +53,7 @@ pub fn blobs_dir(workspace: &Workspace) -> Result<PathBuf> {
 }
 
 pub fn cache_dir() -> Result<PathBuf> {
-    let path = dirs::cache_dir()
+    let path = dirs_next::cache_dir()
         .ok_or_else(|| format_err!("Failed to find cache directory"))?;
     let path = path.join("sn0int");
     fs::create_dir_all(&path)

src/registry.rs

@@ -4,11 +4,11 @@ use crate::api::Client;
 use crate::auth;
 use crate::config::Config;
 use crate::engine::{Library, Module};
-use colored::Colorize;
+use colored::{Color, Colorize};
 use separator::Separatable;
+use std::fmt::Write;
 use sn0int_common::ModuleID;
 use sn0int_common::api::ModuleInfoResponse;
-use sn0int_common::metadata::Metadata;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
@@ -54,7 +54,7 @@ impl Updater {
             fs::write(&path, module.code)
                 .context(format_err!("Failed to write to {:?}", path))?;
 
-            Ok(version.to_string())
+            Ok(version)
         } else {
             let infos = self.query_module(&install.module)
                         .context("Failed to query module infos")?;
@@ -112,13 +112,11 @@ pub fn run_publish(_args: &Args, publish: &Publish, config: &Config) -> Result<(
         let name = name.to_os_string().into_string()
             .map_err(|_| format_err!("Failed to decode file name"))?;
 
-        let code = fs::read_to_string(path)
-            .context("Failed to read module")?;
-        let metadata = code.parse::<Metadata>()?;
+        let module = Module::load(path, "anonymous", &name, false)?;
 
-        let label = format!("Uploading {} {} ({:?})", name, metadata.version, path);
+        let label = format!("Uploading {} {} ({:?})", name, module.version(), path);
         match worker::spawn_fn(&label, || {
-            client.publish_module(&name, code.to_string())
+            client.publish_module(&name, module.code().to_string())
         }, true) {
             Ok(result) => term::info(&format!("Published {}/{} {} ({:?})",
                                               result.author,
@@ -127,7 +125,7 @@ pub fn run_publish(_args: &Args, publish: &Publish, config: &Config) -> Result<(
                                               path)),
             Err(err) => term::error(&format!("Failed to publish {} {} ({:?}): {}",
                                              name,
-                                             metadata.version,
+                                             module.version(),
                                              path,
                                              err)),
         }
@@ -242,6 +240,12 @@ impl Task for UpdateTask {
     }
 }
 
+#[inline]
+fn write_tag(out: &mut String, color: Color, txt: &str) -> Result<()> {
+    write!(out, " [{}]", txt.color(color))?;
+    Ok(())
+}
+
 pub fn run_search(library: &Library, search: &Search, config: &Config) -> Result<()> {
     let client = Client::new(&config)?;
 
@@ -258,12 +262,23 @@ pub fn run_search(library: &Library, search: &Search, config: &Config) -> Result
             continue;
         }
 
-        println!("{} ({}) - {} downloads{}{}", canonical.green(),
-                            module.latest.yellow(),
-                            module.downloads.separated_string(),
-                            (if module.featured { " [featured]" } else { "" }).cyan(),
-                            (if installed.is_some() { " [installed]" } else { "" }).green());
-        println!("\t{}", module.description);
+        let mut out = format!("{}/{} {} - {} downloads",
+            module.author.purple(),
+            module.name,
+            module.latest.blue(),
+            module.downloads.separated_string(),
+        );
+
+        if module.featured {
+            write_tag(&mut out, Color::Cyan, "featured")?;
+        }
+
+        if installed.is_some() {
+            write_tag(&mut out, Color::Green, "installed")?;
+        }
+
+        println!("{}", out.bold());
+        println!("    {}", module.description);
     }
 
     Ok(())

src/repl/complete.rs

@@ -55,6 +55,8 @@ impl Highlighter for ReplCompleter {
 }
 
 impl Hinter for ReplCompleter {
+    type Hint = String;
+
     #[inline]
     fn hint(&self, _line: &str, _pos: usize, _ctx: &Context<'_>) -> Option<String> {
         None

src/repl/mod.rs

@@ -102,7 +102,7 @@ pub fn run(config: &Config) -> Result<()> {
         asn,
     };
 
-    let tx = DummyIpcChild::new();
+    let tx = DummyIpcChild::create();
     let (lua, state) = ctx::ctx(env, tx);
     let mut repl = Repl::new(lua, state);
 

src/runtime/db.rs

@@ -1,6 +1,5 @@
 use crate::errors::*;
 
-use serde;
 use crate::db::Family;
 use crate::engine::ctx::State;
 use crate::engine::structs;

src/runtime/dns.rs

@@ -5,7 +5,6 @@ use crate::engine;
 use crate::engine::ctx::State;
 use crate::json::LuaJsonValue;
 use crate::hlua::{self, AnyLuaValue};
-use serde_json;
 use std::sync::Arc;
 use std::net::SocketAddr;
 use std::time::Duration;

src/runtime/encoding.rs

@@ -1,6 +1,5 @@
 use crate::errors::*;
 
-use base64;
 use crate::engine::ctx::State;
 use crate::engine::structs::{byte_array, lua_bytes};
 use crate::hlua::{self, AnyLuaValue};
@@ -40,7 +39,7 @@ pub fn base64_custom_decode(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             bail!("alphabet isn't base64");
         }
         let spec = spec(&alphabet, &padding)
-            .map_err(|err| state.set_error(err.into()))?;
+            .map_err(|err| state.set_error(err))?;
         spec.decode(bytes.as_bytes())
             .map_err(|err| state.set_error(err.into()))
             .map(|bytes| lua_bytes(&bytes))
@@ -53,7 +52,7 @@ pub fn base64_custom_encode(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             bail!("alphabet isn't base64");
         }
         let spec = spec(&alphabet, &padding)
-            .map_err(|err| state.set_error(err.into()))?;
+            .map_err(|err| state.set_error(err))?;
         byte_array(bytes)
             .map_err(|err| state.set_error(err))
             .map(|bytes| spec.encode(&bytes))
@@ -66,7 +65,7 @@ pub fn base32_custom_decode(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             bail!("alphabet isn't base32");
         }
         let spec = spec(&alphabet, &padding)
-            .map_err(|err| state.set_error(err.into()))?;
+            .map_err(|err| state.set_error(err))?;
         spec.decode(bytes.as_bytes())
             .map_err(|err| state.set_error(err.into()))
             .map(|bytes| lua_bytes(&bytes))
@@ -79,7 +78,7 @@ pub fn base32_custom_encode(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             bail!("alphabet isn't base32");
         }
         let spec = spec(&alphabet, &padding)
-            .map_err(|err| state.set_error(err.into()))?;
+            .map_err(|err| state.set_error(err))?;
         byte_array(bytes)
             .map_err(|err| state.set_error(err))
             .map(|bytes| spec.encode(&bytes))

src/runtime/geo.rs

@@ -9,7 +9,7 @@ use std::sync::Arc;
 pub fn geo_polygon_contains(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("geo_polygon_contains", hlua::function2(move |polygon: Vec<AnyLuaValue>, point: AnyLuaValue| -> Result<bool> {
         let polygon = polygon.into_iter()
-            .map(|p| Point::try_from(p))
+            .map(Point::try_from)
             .collect::<Result<Vec<_>>>()
             .context("Invalid polygon")
             .map_err(|err| state.set_error(Error::from(err)))?;

src/runtime/geoip.rs

@@ -3,7 +3,6 @@ use crate::errors::*;
 use crate::engine::ctx::State;
 use crate::json::LuaJsonValue;
 use crate::hlua::{self, AnyLuaValue};
-use serde_json;
 use std::sync::Arc;
 use std::net::IpAddr;
 

src/runtime/gfx.rs

@@ -4,7 +4,6 @@ use crate::engine::ctx::State;
 use crate::gfx;
 use crate::json::LuaJsonValue;
 use crate::hlua::{self, AnyLuaValue};
-use nude;
 use std::sync::Arc;
 
 

src/runtime/hashes.rs

@@ -2,9 +2,9 @@ use crate::errors::*;
 use crate::engine::ctx::State;
 use crate::engine::structs::{byte_array, lua_bytes};
 use crate::hlua::{self, AnyLuaValue};
-use digest::{Digest, Input, BlockInput, FixedOutput, Reset};
+use digest::{Digest, Update, BlockInput, FixedOutput, Reset};
 use digest::generic_array::ArrayLength;
-use hmac::{Hmac, Mac};
+use hmac::{Hmac, Mac, NewMac};
 use std::sync::Arc;
 
 
@@ -58,7 +58,7 @@ pub fn sha3_512(lua: &mut hlua::Lua, state: Arc<dyn State>) {
 
 fn hmac<D>(secret: AnyLuaValue, msg: AnyLuaValue) -> Result<AnyLuaValue>
     where
-        D: Input + BlockInput + FixedOutput + Reset + Default + Clone,
+        D: Update + BlockInput + FixedOutput + Reset + Default + Clone,
         D::BlockSize: ArrayLength<u8> + Clone,
         D::OutputSize: ArrayLength<u8>,
 {
@@ -69,9 +69,9 @@ fn hmac<D>(secret: AnyLuaValue, msg: AnyLuaValue) -> Result<AnyLuaValue>
         Ok(mac) => mac,
         Err(_) => bail!("Invalid key length"),
     };
-    mac.input(&msg);
-    let result = mac.result();
-    Ok(lua_bytes(&result.code()))
+    mac.update(&msg);
+    let result = mac.finalize();
+    Ok(lua_bytes(&result.into_bytes()))
 }
 
 pub fn hmac_md5(lua: &mut hlua::Lua, state: Arc<dyn State>) {

src/runtime/http.rs

@@ -31,7 +31,7 @@ pub fn http_send<S>(lua: &mut hlua::Lua, state: Arc<S>)
     where S: State + WebState + BlobState + 'static
 {
     lua.set("http_send", hlua::function1(move |request: AnyLuaValue| -> Result<HashMap<AnyHashableLuaValue, AnyLuaValue>> {
-        let req = HttpRequest::try_from(request)
+        let mut req = HttpRequest::try_from(request)
             .context("invalid http request object")
             .map_err(|err| state.set_error(err.into()))?;
 
@@ -48,7 +48,7 @@ pub fn http_fetch<S>(lua: &mut hlua::Lua, state: Arc<S>)
     where S: State + WebState + BlobState + 'static
 {
     lua.set("http_fetch", hlua::function1(move |request: AnyLuaValue| -> Result<AnyLuaValue> {
-        let req = HttpRequest::try_from(request)
+        let mut req = HttpRequest::try_from(request)
             .context("invalid http request object")
             .map_err(|err| state.set_error(err.into()))?;
 
@@ -69,7 +69,7 @@ pub fn http_fetch_json<S>(lua: &mut hlua::Lua, state: Arc<S>)
     where S: State + WebState + 'static
 {
     lua.set("http_fetch_json", hlua::function1(move |request: AnyLuaValue| -> Result<AnyLuaValue> {
-        let req = HttpRequest::try_from(request)
+        let mut req = HttpRequest::try_from(request)
             .context("invalid http request object")
             .map_err(|err| state.set_error(err.into()))?;
 
@@ -271,4 +271,44 @@ mod tests {
         "#).expect("failed to load script");
         script.test().err().expect("Script should have failed");
     }
+
+    #[test]
+    #[ignore]
+    fn verify_fetch_redirects() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            session = http_mksession()
+            req = http_request(session, "GET", "http://github.com", {
+                follow_redirects=1,
+            })
+            x = http_send(req)
+            if last_err() then return end
+
+            if x['status'] ~= 200 then
+                return 'redirect wasn\'t followed'
+            end
+        end
+        "#).expect("failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    #[ignore]
+    fn verify_fetch_skip_redirects() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            session = http_mksession()
+            req = http_request(session, "GET", "http://github.com", {
+                follow_redirects=0,
+            })
+            x = http_send(req)
+            if last_err() then return end
+
+            if x['status'] ~= 301 then
+                return 'redirect was followed'
+            end
+        end
+        "#).expect("failed to load script");
+        script.test().expect("Script failed");
+    }
 }

src/runtime/logger.rs

@@ -11,7 +11,7 @@ pub fn format_lua(out: &mut String, x: &AnyLuaValue) {
         AnyLuaValue::LuaAnyString(ref x) => out.push_str(&format!("{:?}", x.0)),
         AnyLuaValue::LuaBoolean(ref x) => out.push_str(&format!("{:?}", x)),
         AnyLuaValue::LuaArray(ref x) => {
-            out.push_str("{");
+            out.push('{');
             let mut first = true;
 
             for &(ref k, ref v) in x {
@@ -29,7 +29,7 @@ pub fn format_lua(out: &mut String, x: &AnyLuaValue) {
 
                 first = false;
             }
-            out.push_str("}");
+            out.push('}');
         },
         AnyLuaValue::LuaOther => out.push_str("LuaOther"),
     }

src/runtime/pgp.rs

@@ -23,7 +23,7 @@ fn is_new_signing_key(seen: &mut HashSet<String>, sig: &Signature) -> bool {
     }
 
     seen.insert(key.to_string());
-    return true;
+    true
 }
 
 fn pgp_pubkey_lua(pubkey: &[u8]) -> Result<AnyLuaValue> {
@@ -81,7 +81,7 @@ pub fn pgp_pubkey_armored(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("pgp_pubkey_armored", hlua::function1(move |pubkey: String| -> Result<AnyLuaValue> {
         let mut r = BufReader::new(pubkey.as_bytes());
         let pubkey = sloppy_rfc4880::armor::read_armored(&mut r)
-            .map_err(|err| state.set_error(err))?;
+            .map_err(|err| state.set_error(format_err!("{:#}", err)))?;
 
         pgp_pubkey_lua(&pubkey)
             .map_err(|err| state.set_error(err))

src/runtime/sock.rs

@@ -180,7 +180,7 @@ pub fn sock_sendafter(lua: &mut hlua::Lua, state: Arc<dyn State>) {
 }
 
 pub fn sock_newline(lua: &mut hlua::Lua, state: Arc<dyn State>) {
-    lua.set("sock_newline", hlua::function2(move |sock: String, newline: String| -> () {
+    lua.set("sock_newline", hlua::function2(move |sock: String, newline: String| {
         let sock = state.get_sock(&sock);
         let mut sock = sock.lock().unwrap();
 

src/runtime/url.rs

@@ -6,7 +6,6 @@ use crate::json::LuaJsonValue;
 use url::Url;
 use percent_encoding::{percent_encode, percent_decode, NON_ALPHANUMERIC};
 use serde_json::Value;
-use serde_urlencoded;
 use std::sync::Arc;
 
 pub fn url_join(lua: &mut hlua::Lua, state: Arc<dyn State>) {

src/runtime/x509.rs

@@ -4,10 +4,8 @@ use crate::crt::Certificate;
 use crate::engine::ctx::State;
 use crate::json::LuaJsonValue;
 use crate::hlua::{self, AnyLuaValue};
-use serde_json;
 use std::sync::Arc;
 
-
 pub fn x509_parse_pem(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("x509_parse_pem", hlua::function1(move |cert: String| -> Result<AnyLuaValue> {
         let cert = Certificate::parse_pem(&cert)

src/runtime/xml.rs

@@ -60,15 +60,12 @@ fn match_element_name(xml: &AnyLuaValue, name: &str) -> bool {
 pub fn xml_named(lua: &mut hlua::Lua, _state: Arc<dyn State>) {
     lua.set("xml_named", hlua::function2(move |xml: AnyLuaValue, name: String| -> AnyLuaValue {
         if let Some(value) = get_children(xml) {
-            match value {
-                AnyLuaValue::LuaArray(arr) => {
-                    for (_, value) in arr {
-                        if match_element_name(&value, &name) {
-                            return value;
-                        }
+            if let AnyLuaValue::LuaArray(arr) = value {
+                for (_, value) in arr {
+                    if match_element_name(&value, &name) {
+                        return value;
                     }
-                },
-                _ => (),
+                }
             }
         }
 

src/sandbox/mod.rs

@@ -1,8 +1,6 @@
 use crate::errors::*;
 #[cfg(target_os = "linux")]
 use caps::{self, CapSet};
-#[cfg(target_os = "linux")]
-use nix;
 
 #[cfg(target_os = "openbsd")]
 use pledge::pledge;

src/shell/complete.rs

@@ -5,7 +5,6 @@ use rustyline::{self, Context};
 use rustyline::completion::Completer;
 use rustyline::highlight::Highlighter;
 use rustyline::hint::Hinter;
-use shellwords;
 use std::borrow::Cow::{self, Owned};
 use std::str::FromStr;
 use std::io::stdout;
@@ -249,6 +248,8 @@ impl Completer for CmdCompleter {
 
 // TODO: suggest rest of the line if only one possible completion
 impl Hinter for CmdCompleter {
+    type Hint = String;
+
     #[inline]
     fn hint(&self, _line: &str, _pos: usize, _ctx: &Context<'_>) -> Option<String> {
         // None

src/shell/mod.rs

@@ -395,7 +395,7 @@ impl<'a> Shell<'a> {
         self.rl.load_history(&paths::history_path()?)
     }
 
-    pub fn save_history(&self) -> Result<()> {
+    pub fn save_history(&mut self) -> Result<()> {
         self.rl.save_history(&paths::history_path()?)
     }
 
@@ -431,6 +431,12 @@ impl<'a> Shell<'a> {
 
 pub struct SignalRegister(AtomicUsize);
 
+impl Default for SignalRegister {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl SignalRegister {
     pub fn new() -> SignalRegister {
         SignalRegister(AtomicUsize::new(1))

src/shell/readline.rs

@@ -1,6 +1,6 @@
 use crate::errors::*;
 use std::path::Path;
-use rustyline::{self, CompletionType, EditMode, Editor, KeyPress, Movement, Word, At};
+use rustyline::{self, CompletionType, EditMode, Editor};
 pub use rustyline::error::ReadlineError;
 
 
@@ -15,6 +15,12 @@ impl Readline<()> {
     }
 }
 
+impl Default for Readline<()> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl<T: rustyline::Helper> Readline<T> {
     #[inline]
     pub fn with(helper: T) -> Readline<T> {
@@ -26,10 +32,8 @@ impl<T: rustyline::Helper> Readline<T> {
             .completion_type(CompletionType::List)
             .edit_mode(EditMode::Emacs)
             .build();
-        let mut rl: Editor<T> = Editor::with_config(rl_config);
-        rl.bind_sequence(KeyPress::ControlLeft, rustyline::Cmd::Move(Movement::BackwardWord(1, Word::Big)));
-        rl.bind_sequence(KeyPress::ControlRight, rustyline::Cmd::Move(Movement::ForwardWord(1, At::Start, Word::Big)));
 
+        let mut rl: Editor<T> = Editor::with_config(rl_config);
         rl.set_helper(helper);
 
         Readline {
@@ -38,7 +42,7 @@ impl<T: rustyline::Helper> Readline<T> {
     }
 
     #[inline]
-    pub fn save_history<P: AsRef<Path>>(&self, path: &P) -> Result<()> {
+    pub fn save_history<P: AsRef<Path>>(&mut self, path: &P) -> Result<()> {
         self.rl.save_history(path)
             .map_err(Error::from)
     }

src/term.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::print_literal)]
+
 use atty::{self, Stream};
 use crate::db;
 use crate::engine::Module;
@@ -88,7 +90,7 @@ pub trait SpinLogger {
 
     fn status(&mut self, status: String);
 
-    fn stacked_status(&mut self, name: &String, status: String);
+    fn stacked_status(&mut self, name: &str, status: String);
 }
 
 pub struct Spinner {
@@ -196,7 +198,7 @@ impl SpinLogger for Spinner {
     }
 
     #[inline]
-    fn stacked_status(&mut self, _name: &String, status: String) {
+    fn stacked_status(&mut self, _name: &str, status: String) {
         self.status = status;
     }
 }
@@ -254,7 +256,7 @@ impl SpinLogger for Term {
     }
 
     #[inline]
-    fn stacked_status(&mut self, _name: &String, _status: String) {
+    fn stacked_status(&mut self, _name: &str, _status: String) {
         unimplemented!()
     }
 }
@@ -294,6 +296,12 @@ pub struct StackedSpinners {
     warnings: HashSet<String>,
 }
 
+impl Default for StackedSpinners {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl StackedSpinners {
     #[inline]
     pub fn new() -> StackedSpinners {
@@ -337,7 +345,7 @@ impl StackedSpinners {
         for (i, (_, s)) in self.spinners.iter_mut().enumerate() {
             print!("{}", s.tick_bytes());
             if i < n {
-                print!("\n");
+                println!();
                 self.drawn += 1;
             }
         }
@@ -400,7 +408,7 @@ impl SpinLogger for StackedSpinners {
         self.error(&format!("TODO: set status: {:?}", status));
     }
 
-    fn stacked_status(&mut self, name: &String, status: String) {
+    fn stacked_status(&mut self, name: &str, status: String) {
         if let Some(spinner) = self.spinners.get_mut(name) {
             spinner.status(status);
         }
@@ -459,7 +467,7 @@ impl<'a, T: SpinLogger> SpinLogger for PrefixedLogger<'a, T> {
     }
 
     #[inline]
-    fn stacked_status(&mut self, prefix: &String, status: String) {
+    fn stacked_status(&mut self, prefix: &str, status: String) {
         self.s.stacked_status(prefix, status)
     }
 }

src/update.rs

@@ -81,9 +81,7 @@ impl AutoUpdater {
             return;
         }
 
-        let modules = modules.into_iter()
-            .map(|x| x.clone())
-            .collect();
+        let modules = modules.into_iter().cloned().collect();
 
         debug!("Checking for outdated modules in the background");
         match Client::new(config) {

src/utils.rs

@@ -1,7 +1,18 @@
 use crate::errors::*;
+use rand::{Rng, thread_rng};
+use rand::distributions::Alphanumeric;
 use rustyline::error::ReadlineError;
+use std::iter;
 use std::str::FromStr;
 
+pub fn random_string(len: usize) -> String {
+    let mut rng = thread_rng();
+    iter::repeat(())
+        .map(|()| rng.sample(Alphanumeric))
+        .map(char::from)
+        .take(len)
+        .collect()
+}
 
 pub fn read_line(prompt: &str) -> Result<String> {
     let mut rl = rustyline::Editor::<()>::new();

src/worker.rs

@@ -10,7 +10,6 @@ use crate::ipc;
 use crate::ipc::parent::IpcParent;
 use crate::models::*;
 use crate::notify::{self, Notification};
-use serde_json;
 use crate::ratelimits::{Ratelimiter, RatelimitResponse};
 use crate::shell::Shell;
 use sn0int_std::ratelimits::RatelimitSender;
@@ -40,7 +39,7 @@ pub enum DatabaseResponse {
 #[derive(Debug, Serialize, Deserialize)]
 pub enum Event {
     Log(LogEvent),
-    Database(DatabaseEvent),
+    Database(Box<DatabaseEvent>),
     Stdio(StdioEvent),
     Ratelimit(RatelimitEvent),
     Blob(Blob),
@@ -51,7 +50,7 @@ pub enum Event {
 pub enum Event2 {
     Start,
     Log(LogEvent),
-    Database((DatabaseEvent, DbSender)),
+    Database(Box<(DatabaseEvent, DbSender)>),
     Ratelimit((RatelimitEvent, RatelimitSender)),
     Blob((Blob, VoidSender)),
     Exit(ExitEvent),
@@ -120,7 +119,7 @@ impl From<Result<()>> for ExitEvent {
                     .map(|e| e.to_string())
                     .collect::<Vec<_>>()
                     .join(": ");
-                ExitEvent::Err(err.to_string())
+                ExitEvent::Err(err)
             },
         }
     }
@@ -164,7 +163,7 @@ impl EventWithCallback for DatabaseEvent {
     type Payload = DatabaseResponse;
 
     fn with_callback(self, tx: mpsc::Sender<result::Result<Self::Payload, String>>) -> Event2 {
-        Event2::Database((self, tx))
+        Event2::Database(Box::new((self, tx)))
     }
 }
 
@@ -222,7 +221,7 @@ impl DatabaseEvent {
             if let Some(radius) = &object.radius {
                 log.push_str(&format!(" | {}m", radius));
             }
-            log.push_str(")");
+            log.push(')');
         }
 
         if verbose > 0 {
@@ -500,7 +499,10 @@ pub fn spawn(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, args:
                             stack.add(name, label);
                         },
                         Event2::Log(log) => log.apply(&mut stack.prefixed(name)),
-                        Event2::Database((db, tx)) => db.apply(rl, &mut stack.prefixed(name), ratelimit, tx, verbose),
+                        Event2::Database(tuple) => {
+                            let (db, tx) = *tuple;
+                            db.apply(rl, &mut stack.prefixed(name), ratelimit, tx, verbose)
+                        },
                         Event2::Ratelimit((req, tx)) => ratelimit.pass(tx, &req.key, req.passes, req.time),
                         Event2::Blob((blob, tx)) => rl.store_blob(tx, &blob),
                         Event2::Exit(event) => {
@@ -670,5 +672,7 @@ pub fn spawn_multi<T: Task, F>(tasks: Vec<T>, mut done_fn: F, threads: usize) ->
         stack.tick();
     }
 
+    stack.clear();
+
     Ok(())
 }