Bump libsqlite3-sys from 0.22.2 to 0.25.1

Bumps [libsqlite3-sys](https://github.com/rusqlite/rusqlite) from 0.22.2 to 0.25.1.
- [Release notes](https://github.com/rusqlite/rusqlite/releases)
- [Changelog](https://github.com/rusqlite/rusqlite/blob/master/Changelog.md)
- [Commits](https://github.com/rusqlite/rusqlite/commits/v0.25.1)

---
updated-dependencies:
- dependency-name: libsqlite3-sys
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/FUNDING.yml

@@ -1 +1,2 @@
 github: [kpcyrd]
+patreon: kpcyrd

.github/workflows/docker-release.yml

@@ -0,0 +1,58 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to GitHub Registry
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/kpcyrd/sn0int
+          tags: |
+            type=semver,pattern={{raw}}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: Login to Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push Docker images
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          file: Dockerfile
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+      -
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/docker.yml

@@ -22,7 +22,7 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Build the Docker image
-      run: docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
+      run: DOCKER_BUILDKIT=1 docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
     - name: Test the Docker image
       run: docker run --rm ${{ matrix.build.name }} --help
 

Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "sn0int"
-version = "0.21.1"
+version = "0.25.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
-edition = "2018"
+edition = "2021"
 
 [workspace]
 members = ["sn0int-common",
@@ -24,7 +24,6 @@ the results in a unified format for followup investigations."""
 section = "utils"
 priority = "optional"
 depends = "$auto, publicsuffix"
-build-depends = "pkg-config, libsqlite3-dev, libseccomp-dev, libsodium-dev, publicsuffix"
 assets = [
     ["target/release/sn0int", "usr/bin/", "755"],
 ]
@@ -33,11 +32,11 @@ assets = [
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.12.0", path="sn0int-common" }
-sn0int-std = { version="=0.21.0", path="sn0int-std" }
-rustyline = "8.0"
+sn0int-common = { version="0.13.0", path="sn0int-common" }
+sn0int-std = { version="=0.25.0", path="sn0int-std" }
+rustyline = "10.0"
 log = "0.4"
-env_logger = "0.8"
+env_logger = "0.9"
 hlua-badtouch = "0.4"
 structopt = "0.3"
 failure = "0.1"
@@ -47,21 +46,20 @@ lazy_static = "1.0"
 shellwords = "1.0"
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-libsqlite3-sys = { version = "0.18.0", features = ["bundled-windows"] }
+libsqlite3-sys = { version = "0.25.1", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
 dirs-next = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
 chrootable-https = "0.16"
-base64 = "0.13"
-data-encoding = "2.1.2"
+data-encoding = "2.3.3"
 serde = { version = "1.0", features = ["derive"] }
 serde_urlencoded = "0.7"
 serde_json = "1.0"
 crossbeam-channel = "0.5"
 ctrlc = "3.1"
-opener = "0.4"
+opener = "0.5"
 separator = "0.4"
 maplit = "1.0.1"
 sloppy-rfc4880 = "0.2"
@@ -69,32 +67,32 @@ regex = "1.0"
 toml = "0.5"
 threadpool = "1.7"
 atty = "0.2"
-semver = "0.11"
+semver = "1"
 bytes = "0.4"
 bytesize = "1.0"
 ipnetwork = "0.18"
-strum = "0.20"
-strum_macros = "0.20"
+strum = "0.24"
+strum_macros = "0.24"
 embedded-triple = "0.1.0"
 humansize = "1.1.0"
 
-digest = "0.9"
-md-5 = "0.9"
-sha-1 = "0.9"
-sha2 = "0.9"
-sha3 = "0.9"
-hmac = "0.11"
+digest = "0.10"
+md-5 = "0.10"
+sha-1 = "0.10"
+sha2 = "0.10"
+sha3 = "0.10"
+hmac = "0.12"
 
 walkdir = "2.2"
 nude = "0.3"
 glob = "0.3.0"
-os-version = "0.1.1"
+os-version = "0.2"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.15"
-nix = "0.20"
+syscallz = "0.16"
+nix = "0.24"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.4"
@@ -102,5 +100,5 @@ unveil = "0.3"
 
 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.11"
+boxxy = "0.13"
 tempfile = "3.0"

Dockerfile

@@ -1,14 +1,18 @@
-FROM rust:alpine3.11
+FROM rust:alpine3.15
 ENV RUSTFLAGS="-C target-feature=-crt-static"
 RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo build --release --verbose
-RUN strip target/release/sn0int
+RUN --mount=type=cache,target=/var/cache/buildkit \
+    CARGO_HOME=/var/cache/buildkit/cargo \
+    CARGO_TARGET_DIR=/var/cache/buildkit/target \
+    cargo build --release --locked --verbose && \
+    cp -v /var/cache/buildkit/target/release/sn0int /
+RUN strip /sn0int
 
-FROM alpine:3.11
+FROM alpine:3.15
 RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
-COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+COPY --from=0 /sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \
     XDG_CACHE_HOME=/cache

README.md

@@ -14,33 +14,34 @@
 [registry]:     https://sn0int.com/
 
 sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
-package manager. It was built for IT security professionals and bug hunters to
-gather intelligence about a given target or about yourself. sn0int is
-enumerating attack surface by semi-automatically processing public information
-and mapping the results in a unified format for followup investigations.
+package manager. It's used by IT security professionals, bug bounty hunters,
+law enforcement agencies and in security awareness trainings to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.
 
 [ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
 - Harvest subdomains from certificate transparency logs and passive dns
+- Mass resolve collected subdomains and scan for http or https services
 - Enrich ip addresses with asn and geoip info
 - Harvest emails from pgp keyservers and whois
 - Discover compromised logins in breaches
 - Find somebody's profiles across the internet
 - Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Attempt to bypass cloudflare with shodan
-- Harvest data and images from instagram profiles
-- Scan images for nudity
+- Harvest activity and images from social media profiles
+- Basic image processing
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource. None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
+in the source, instead they are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+you to ship updates for your modules on your own instead of pull-requesting
+them into the sn0int codebase.
 
 For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
@@ -226,6 +227,9 @@ For everything else please have a look at the [detailed list][1].
   - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
   - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_ahash](https://sn0int.readthedocs.io/en/latest/reference.html#img-ahash)
+  - [img_dhash](https://sn0int.readthedocs.io/en/latest/reference.html#img-dhash)
+  - [img_phash](https://sn0int.readthedocs.io/en/latest/reference.html#img-phash)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
   - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)

ci/integration.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import subprocess
 from subprocess import DEVNULL, PIPE
+import shutil
+from pathlib import Path
 import tempfile
 import json
 import sys
@@ -98,6 +100,11 @@ def main(tempdir, binary):
         ('https://www.example.com/', 200),
     }
 
+    cache = Path.home() / '.cache' / 'sn0int'
+    if cache.exists():
+        print('[*] copying geoip files')
+        shutil.copytree(cache, tempdir + '/.cache/sn0int', dirs_exist_ok=True)
+
     print('[*] running geoip')
     sn0int(tempdir, binary, [
         'use geoip',

docs/build.rst

@@ -44,6 +44,13 @@ Alpine
 
     $ apk add sqlite-dev libseccomp-dev libsodium-dev
 
+Docker
+~~~~~~
+
+.. code-block:: bash
+
+    $ DOCKER_BUILDKIT=1 docker build -t kpcyrd/sn0int .
+
 OpenBSD
 ~~~~~~~
 

docs/reference.rst

@@ -504,6 +504,39 @@ Extract exif metadata from an image.
     if last_err() then return end
     debug(exif)
 
+img_ahash
+---------
+
+Calculate the Mean (aHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_ahash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_dhash
+---------
+
+Calculate the Gradient (dHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_dhash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_phash
+---------
+
+Calculate the DCT (pHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_phash(blob)
+    if last_err() then return end
+    debug(hash)
+
 img_nudity
 ----------
 

docs/sandbox.rst

@@ -119,7 +119,7 @@ You might experience a sandbox failure, especially on architectures that are
 less popular. This usually looks like this::
 
     [sn0int][example][kpcyrd/ctlogs] > run
-    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [-] Failed "example.com": Sandbox child has crashed
     [+] Finished kpcyrd/ctlogs (1 errors)
 
 A module that never finishes could also mean an IO thread inside the worker got

modules/harness/img-hash.lua

@@ -0,0 +1,11 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    debug(arg)
+    info(img_ahash(arg['value']))
+    info(img_dhash(arg['value']))
+    info(img_phash(arg['value']))
+end

modules/harness/import-subdomains.lua

@@ -3,7 +3,7 @@
 -- License: GPL-3.0
 
 function run()
-    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lu
+    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lua
 
     while true do
         local line = stdin_readline()

modules/harness/winkekatze-sub.lua

@@ -10,8 +10,17 @@ function run()
     mqtt_subscribe(sock, '#', 0)
 
     while true do
-        local pkt = mqtt_recv_text(sock)
+        -- read the next mqtt packet
+        local pkt = mqtt_recv(sock)
         if last_err() then return end
-        info(pkt)
+
+        local text
+        if pkt then
+            -- attempt to utf8 decode the body if there was a pkt
+            text = utf8_decode(pkt['body'])
+            if last_err() then clear_err() end
+        end
+
+        info({pkt=pkt, text=text})
     end
 end

sn0int-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.12.0"
+version = "0.13.0"
 description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -12,4 +12,4 @@ serde = { version = "1.0", features=["derive"] }
 #rocket_failure_errors = { path = "../../../rocket_failure/rocket_failure_errors" }
 rocket_failure_errors = "0.2"
 anyhow = "1.0"
-nom = "6.0"
+nom = "7.0"

sn0int-common/src/api.rs

@@ -1,5 +1,5 @@
 use crate::id::ModuleID;
-use serde::{Serialize, Deserialize};
+use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct WhoamiResponse {

sn0int-common/src/errors.rs

@@ -1,2 +1,2 @@
-pub use anyhow::{Error, Context, anyhow, format_err, bail};
+pub use anyhow::{anyhow, bail, format_err, Context, Error};
 pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-common/src/id.rs

@@ -1,10 +1,9 @@
 use crate::errors::*;
-use serde::{de, Serialize, Serializer, Deserialize, Deserializer};
+use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
 use std::fmt;
 use std::result;
 use std::str::FromStr;
 
-
 #[inline(always)]
 fn valid_char(c: char) -> bool {
     nom::character::is_alphanumeric(c as u8) || c == '-'
@@ -71,7 +70,8 @@ impl Serialize for ModuleID {
 
 impl<'de> Deserialize<'de> for ModuleID {
     fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-        where D: Deserializer<'de>
+    where
+        D: Deserializer<'de>,
     {
         let s = String::deserialize(deserializer)?;
         FromStr::from_str(&s).map_err(de::Error::custom)

sn0int-common/src/lib.rs

@@ -1,10 +1,8 @@
-#[macro_use] extern crate nom;
-
 pub mod api;
 pub mod errors;
 pub use crate::errors::*;
-pub mod metadata;
 pub mod id;
+pub mod metadata;
 pub use crate::id::*;
 
 pub use rocket_failure_errors::StrictApiResponse as ApiResponse;

sn0int-common/src/metadata/mod.rs

@@ -1,5 +1,9 @@
 use crate::errors::*;
-use serde::{Serialize, Deserialize};
+use nom::bytes::complete::{tag, take_until};
+use nom::combinator::map_res;
+use nom::multi::fold_many0;
+use nom::IResult;
+use serde::{Deserialize, Serialize};
 use std::str::FromStr;
 
 mod stealth;
@@ -223,23 +227,32 @@ impl<'a> NewMetadata<'a> {
     }
 }
 
-named!(metaline<&str, (EntryType, &str)>, do_parse!(
-    tag!("-- ") >>
-    name: map_res!(take_until!(": "), EntryType::from_str) >>
-    tag!(": ") >>
-    value: take_until!("\n") >>
-    tag!("\n") >>
-    (name, value)
-));
+fn metaline(input: &str) -> IResult<&str, (EntryType, &str)> {
+    let (input, _) = tag("-- ")(input)?;
+    let (input, name) = map_res(
+        take_until(": "),
+        EntryType::from_str
+    )(input)?;
+    let (input, _) = tag(": ")(input)?;
+    let (input, value) = take_until("\n")(input)?;
+    let (input, _) = tag("\n")(input)?;
 
-named!(metalines<&str, Vec<(EntryType, &str)>>, do_parse!(
-    lines: fold_many0!(metaline, Vec::new(), |mut acc: Vec<_>, item| {
-        acc.push(item);
-        acc
-    }) >>
-    tag!("\n") >>
-    (lines)
-));
+    Ok((input, (name, value)))
+}
+
+fn metalines(input: &str) -> IResult<&str, Vec<(EntryType, &str)>> {
+    let (input, lines) = fold_many0(
+        metaline,
+        Vec::new,
+        |mut acc: Vec<_>, item| {
+            acc.push(item);
+            acc
+        }
+    )(input)?;
+    let (input, _) = tag("\n")(input)?;
+
+    Ok((input, lines))
+}
 
 #[cfg(test)]
 mod tests {

sn0int-common/src/metadata/stealth.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use serde::{Serialize, Deserialize};
+use serde::{Deserialize, Serialize};
 use std::str::FromStr;
 
 #[derive(Debug, Eq, PartialEq, PartialOrd, Clone, Serialize, Deserialize)]

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.12.0", path="../sn0int-common" }
+sn0int-common = { version="0.13.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }
@@ -23,7 +23,7 @@ oauth2 = "2.0.0"
 failure = "0.1"
 url = "1.0"
 log = "0.4"
-semver = "0.9.0"
+semver = "1"
 lazy_static = "1"
 blake2 = "0.8.0"
 hex = "0.4"
@@ -33,5 +33,5 @@ serde_json = "1.0"
 syntect = "3.3"
 
 dotenv = "0.15"
-env_logger = "0.7"
+env_logger = "0.9"
 structopt = "0.3.21"

sn0int-registry/src/github.rs

@@ -5,6 +5,7 @@ pub fn get_username(oauth_token: &str) -> Result<String> {
     let client = reqwest::Client::new();
     let mut resp = client.get("https://api.github.com/user")
         .header("Authorization", format!("token {}", oauth_token))
+        .header("User-Agent", "sn0int-registry")
         .send()
         .context("Failed to check access_token")?
         .error_for_status()

sn0int-std/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "sn0int-std"
-version = "0.21.0"
+version = "0.25.0"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
 license = "GPL-3.0"
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -27,29 +27,31 @@ ct-logs = "0.7"
 chrootable-https = "0.16"
 http = "0.2"
 bufstream = "0.1.4"
-pem = "0.8"
+pem = "1"
 url = "2.0"
 tungstenite = { version = "0.13", default-features = false }
 kuchiki = "0.8.0"
-maxminddb = "0.17"
-x509-parser = "0.9"
-der-parser = "5"
+maxminddb = "0.23"
+x509-parser = "0.13"
+der-parser = "8"
 publicsuffix = { version="2", default-features=false }
 xml-rs = "0.8"
-geo = "0.18"
+geo = "0.23"
 bytes = "0.4"
-base64 = "0.13"
 chrono = { version = "0.4", features = ["serde"] }
 mqtt-protocol = "0.11"
 sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
 
-image = "0.23.0"
+image = "0.23"
 kamadak-exif = "0.5.1"
+img_hash_median = "4.0.0"
 
 bs58 = "0.4"
-digest = "0.9"
-blake2 = "0.9"
+digest = "0.10"
+blake2 = "0.10"
+data-encoding = "2.3.3"
+thiserror = "1.0.38"
 
 [dev-dependencies]
-env_logger = "0.8"
+env_logger = "0.9"
 maplit = "1.0.1"

sn0int-std/src/blobs.rs

@@ -1,8 +1,9 @@
+use blake2::Blake2bVar;
 use bytes::Bytes;
-use blake2::VarBlake2b;
+use data_encoding::BASE64;
 use digest::{Update, VariableOutput};
-use serde::ser::{Serialize, Serializer};
 use serde::de::{self, Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
 use std::result;
 
 #[derive(Debug, Clone, PartialEq)]
@@ -21,7 +22,7 @@ impl Blob {
     }
 
     pub fn hash(bytes: &[u8]) -> String {
-        let mut h = VarBlake2b::new(32).unwrap();
+        let mut h = Blake2bVar::new(32).unwrap();
         h.update(bytes);
         let output = h.finalize_boxed();
         Self::encode_hash(&output)
@@ -40,7 +41,7 @@ impl Serialize for Blob {
     where
         S: Serializer,
     {
-        let s = base64::encode(&self.bytes);
+        let s = BASE64.encode(&self.bytes);
         serializer.serialize_str(&s)
     }
 }
@@ -52,7 +53,7 @@ impl<'de> Deserialize<'de> for Blob {
         D: Deserializer<'de>,
     {
         let s = String::deserialize(deserializer)?;
-        let bytes = base64::decode(&s)
+        let bytes = BASE64.decode(s.as_bytes())
             .map_err(de::Error::custom)?;
         Ok(Blob::create(Bytes::from(bytes)))
     }

sn0int-std/src/crt.rs

@@ -5,6 +5,7 @@ use std::net::IpAddr;
 use x509_parser::x509::X509Version;
 use x509_parser::certificate::X509Certificate;
 use x509_parser::extensions::{GeneralName, ParsedExtension};
+use x509_parser::prelude::*;
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
@@ -31,7 +32,7 @@ impl Certificate {
     }
 
     pub fn from_bytes(crt: &[u8]) -> Result<Certificate> {
-        let crt = match X509Certificate::from_der(&crt) {
+        let crt = match X509Certificate::from_der(crt) {
             Ok((remaining, der)) => {
                 if !remaining.is_empty() {
                     bail!("input cert has trailing garbage");
@@ -55,7 +56,7 @@ impl Certificate {
             }
         }
 
-        for (_oid, ext) in crt.tbs_certificate.extensions {
+        for ext in crt.tbs_certificate.extensions() {
             if let ParsedExtension::SubjectAlternativeName(san) = ext.parsed_extension() {
                 for name in &san.general_names {
                     debug!("Certificate is valid for {:?}", name);

sn0int-std/src/crypto.rs

@@ -22,7 +22,7 @@ pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
     let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
         .ok_or_else(|| format_err!("Nonce has wrong length"))?;
     let ciphertext = &encrypted[secretbox::NONCEBYTES..];
-    let plain = secretbox::open(&ciphertext, &nonce, &key)
+    let plain = secretbox::open(ciphertext, &nonce, &key)
         .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
     Ok(plain)
 }

sn0int-std/src/engine/structs.rs

@@ -86,16 +86,16 @@ impl From<Vec<(AnyLuaValue, AnyLuaValue)>> for LuaMap {
     }
 }
 
-impl Into<HashMap<AnyHashableLuaValue, AnyLuaValue>> for LuaMap {
-    fn into(self: LuaMap) -> HashMap<AnyHashableLuaValue, AnyLuaValue> {
-        self.0
+impl From<LuaMap> for HashMap<AnyHashableLuaValue, AnyLuaValue> {
+    fn from(map: LuaMap) -> Self {
+        map.0
     }
 }
 
-impl Into<AnyLuaValue> for LuaMap {
-    fn into(self: LuaMap) -> AnyLuaValue {
+impl From<LuaMap> for AnyLuaValue {
+    fn from(map: LuaMap) -> AnyLuaValue {
         AnyLuaValue::LuaArray(
-            self.into_iter()
+            map.into_iter()
                 .filter_map(|(k, v)| {
                     match k {
                         AnyHashableLuaValue::LuaString(x) => Some((AnyLuaValue::LuaString(x), v)),
@@ -136,9 +136,9 @@ impl LuaList {
     }
 }
 
-impl Into<AnyLuaValue> for LuaList {
-    fn into(self: LuaList) -> AnyLuaValue {
-        AnyLuaValue::LuaArray(self.0)
+impl From<LuaList> for AnyLuaValue {
+    fn from(list: LuaList) -> AnyLuaValue {
+        AnyLuaValue::LuaArray(list.0)
     }
 }
 
@@ -149,7 +149,7 @@ pub fn byte_array(bytes: AnyLuaValue) -> Result<Vec<u8>> {
         AnyLuaValue::LuaArray(bytes) => {
             Ok(bytes.into_iter()
                 .map(|num| match num.1 {
-                    AnyLuaValue::LuaNumber(num) if num <= 255.0 && num >= 0.0 && (num % 1.0 == 0.0) =>
+                    AnyLuaValue::LuaNumber(num) if (0.0..=255.0).contains(&num) && (num % 1.0 == 0.0) =>
                             Ok(num as u8),
                     AnyLuaValue::LuaNumber(num) =>
                             Err(format_err!("number is out of range: {:?}", num)),

sn0int-std/src/errors.rs

@@ -1,3 +1,3 @@
-pub use log::{trace, debug, info, warn, error};
-pub use failure::{Error, ResultExt, format_err, bail};
+pub use failure::{bail, format_err, Error, ResultExt};
+pub use log::{debug, error, info, trace, warn};
 pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-std/src/geo.rs

@@ -1,7 +1,7 @@
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
-use geo::{LineString, Polygon, Coordinate};
+use geo::{LineString, Polygon, Coord};
 use geo::prelude::*;
 use serde::Deserialize;
 
@@ -21,7 +21,7 @@ impl Point {
 
 pub fn polygon_contains(ring: &[Point], p: &Point) -> bool {
     let ring = ring.iter()
-        .map(|p| Coordinate { x: p.lon, y: p.lat })
+        .map(|p| Coord { x: p.lon, y: p.lat })
         .collect::<Vec<_>>();
 
     let polygon = Polygon::new(LineString::from(ring), vec![]);

sn0int-std/src/geoip/models.rs

@@ -72,7 +72,7 @@ pub struct Continent {
 }
 
 impl Continent {
-    fn from_maxmind(continent: geoip2::model::Continent) -> Option<Self> {
+    fn from_maxmind(continent: geoip2::city::Continent) -> Option<Self> {
         let code = continent.code?;
         let name = from_geoip_model_names(continent.names)?;
 
@@ -90,7 +90,7 @@ pub struct Country {
 }
 
 impl Country {
-    fn from_maxmind(country: geoip2::model::Country) -> Option<Self> {
+    fn from_maxmind(country: geoip2::city::Country) -> Option<Self> {
         let code = country.iso_code?;
         let name = from_geoip_model_names(country.names)?;
 
@@ -108,7 +108,7 @@ pub struct Location {
 }
 
 impl Location {
-    fn from_maxmind(location: &geoip2::model::Location) -> Option<Self> {
+    fn from_maxmind(location: &geoip2::city::Location) -> Option<Self> {
         let latitude = match location.latitude {
             Some(latitude) => latitude,
             _ => return None,

sn0int-std/src/gfx/exif.rs

@@ -93,7 +93,7 @@ pub fn cardinal_direction_modifier(value: &exif::Value) -> Result<f64> {
             let s = s.get(0)
                 .ok_or_else(|| format_err!("Cardinal direction value is empty"))?;
 
-            match s.get(0) {
+            match s.first() {
                 Some(b'N') => Ok(1.0),
                 Some(b'S') => Ok(-1.0),
                 Some(b'E') => Ok(1.0),

sn0int-std/src/gfx/mod.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 use image::{self, DynamicImage, GenericImageView};
+pub use img_hash_median::HashAlg;
 
 pub mod exif;
 
@@ -45,9 +46,9 @@ impl ImageFormat {
     }
 }
 
-impl Into<image::ImageFormat> for ImageFormat {
-    fn into(self) -> image::ImageFormat {
-        match self {
+impl From<ImageFormat> for image::ImageFormat {
+    fn from(format: ImageFormat) -> image::ImageFormat {
+        match format {
             ImageFormat::Png => image::ImageFormat::Png,
             ImageFormat::Jpeg => image::ImageFormat::Jpeg,
             ImageFormat::Gif => image::ImageFormat::Gif,
@@ -79,6 +80,14 @@ impl Image {
     pub fn height(&self) -> u32 {
         self.image.height()
     }
+
+    pub fn perception_hash(&self, hash_alg: HashAlg) -> String {
+        let hasher = img_hash_median::HasherConfig::new()
+            .hash_alg(hash_alg)
+            .to_hasher();
+        let hash = hasher.hash_image(&self.image);
+        hash.to_base64()
+    }
 }
 
 impl AsRef<DynamicImage> for Image {
@@ -94,10 +103,10 @@ pub fn guess_format(buf: &[u8]) -> Result<ImageFormat> {
 }
 
 pub fn load(buf: &[u8]) -> Result<Image> {
-    let img_format = image::guess_format(&buf)?;
+    let img_format = image::guess_format(buf)?;
     let format = ImageFormat::try_from(&img_format)?;
 
-    let image = image::load_from_memory_with_format(&buf, img_format)?;
+    let image = image::load_from_memory_with_format(buf, img_format)?;
 
     Ok(Image {
         image,
@@ -105,7 +114,6 @@ pub fn load(buf: &[u8]) -> Result<Image> {
     })
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;

sn0int-std/src/html.rs

@@ -11,13 +11,13 @@ pub struct Element {
     pub html: String,
 }
 
-impl Into<AnyLuaValue> for Element {
-    fn into(self) -> AnyLuaValue {
+impl From<Element> for AnyLuaValue {
+    fn from(elem: Element) -> AnyLuaValue {
         let mut map = LuaMap::new();
 
-        map.insert_str("text", self.text);
-        map.insert("attrs", LuaMap::from(self.attrs));
-        map.insert_str("html", self.html);
+        map.insert_str("text", elem.text);
+        map.insert("attrs", LuaMap::from(elem.attrs));
+        map.insert_str("html", elem.html);
 
         map.into()
     }

sn0int-std/src/json.rs

@@ -1,6 +1,5 @@
 use crate::errors::*;
 
-use std::iter::FromIterator;
 use std::collections::HashMap;
 use crate::hlua::AnyLuaValue;
 use serde_json::{self, Deserializer, Value, Number, Map};
@@ -22,7 +21,7 @@ pub fn encode(v: AnyLuaValue) -> Result<String> {
 }
 
 pub fn decode_stream(x: &str) -> Result<Vec<AnyLuaValue>> {
-    let stream = Deserializer::from_str(&x).into_iter::<Value>();
+    let stream = Deserializer::from_str(x).into_iter::<Value>();
 
     let list = stream
         .filter_map(|x| x.ok())
@@ -52,9 +51,9 @@ pub enum LuaJsonValue {
     Object(HashMap<String, LuaJsonValue>),
 }
 
-impl Into<AnyLuaValue> for LuaJsonValue {
-    fn into(self) -> AnyLuaValue {
-        match self {
+impl From<LuaJsonValue> for AnyLuaValue {
+    fn from(value: LuaJsonValue) -> AnyLuaValue {
+        match value {
             LuaJsonValue::Null => AnyLuaValue::LuaNil,
             LuaJsonValue::Bool(v) => AnyLuaValue::LuaBoolean(v),
             // TODO: not sure if this might fail
@@ -111,9 +110,9 @@ impl From<AnyLuaValue> for LuaJsonValue {
     }
 }
 
-impl Into<serde_json::Value> for LuaJsonValue {
-    fn into(self) -> serde_json::Value {
-        match self {
+impl From<LuaJsonValue> for serde_json::Value {
+    fn from(value: LuaJsonValue) -> serde_json::Value {
+        match value {
             LuaJsonValue::Null => Value::Null,
             LuaJsonValue::Bool(v) => Value::Bool(v),
             LuaJsonValue::Number(v) => Value::Number(v),
@@ -122,9 +121,10 @@ impl Into<serde_json::Value> for LuaJsonValue {
                 .map(|x| x.into())
                 .collect()
             ),
-            LuaJsonValue::Object(v) => Value::Object(Map::from_iter(v.into_iter()
+            LuaJsonValue::Object(v) => Value::Object(v.into_iter()
                 .map(|(k, v)| (k, v.into()))
-            )),
+                .collect::<Map<_, _>>()
+            ),
         }
     }
 }
@@ -140,9 +140,10 @@ impl From<serde_json::Value> for LuaJsonValue {
                 .map(|x| x.into())
                 .collect()
             ),
-            Value::Object(v) => LuaJsonValue::Object(HashMap::from_iter(v.into_iter()
+            Value::Object(v) => LuaJsonValue::Object(v.into_iter()
                 .map(|(k, v)| (k, v.into()))
-            )),
+                .collect::<HashMap<_, _>>()
+            ),
         }
     }
 }

sn0int-std/src/lib.rs

@@ -3,8 +3,8 @@ use hlua_badtouch as hlua;
 pub mod blobs;
 pub mod crt;
 pub mod crypto;
-mod errors;
 pub mod engine;
+mod errors;
 pub mod geo;
 pub mod geoip;
 pub mod gfx;

sn0int-std/src/mqtt.rs

@@ -1,20 +1,26 @@
-use chrootable_https::DnsResolver;
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
-use mqtt::packet::VariablePacketError;
 use crate::json::LuaJsonValue;
-use crate::sockets::{Stream, SocketOptions};
-use mqtt::{TopicFilter, QualityOfService};
-use mqtt::control::ConnectReturnCode;
+use crate::sockets::{SocketOptions, Stream};
+use chrootable_https::DnsResolver;
 use mqtt::control::fixed_header::FixedHeaderError;
-use mqtt::encodable::{Encodable, Decodable};
-use mqtt::packet::{VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
-use serde::{Serialize, Deserialize};
+use mqtt::control::ConnectReturnCode;
+use mqtt::encodable::{Decodable, Encodable};
+use mqtt::packet::VariablePacketError;
+use mqtt::packet::{ConnectPacket, PingreqPacket, SubscribePacket, VariablePacket};
+use mqtt::{QualityOfService, TopicFilter};
+use serde::{Deserialize, Serialize};
 use std::convert::TryFrom;
 use std::io;
 use std::net::SocketAddr;
+use std::time::{Duration, Instant};
 use url::Url;
 
+// a reasonable default for keep-alive
+// some servers reject 0 as invalid with a very confusing error message
+const DEFAULT_PING_INTERVAL: u64 = 90;
+const DEFAULT_KEEP_ALIVE: u16 = 120;
+
 #[derive(Debug, Default, Deserialize)]
 pub struct MqttOptions {
     pub username: Option<String>,
@@ -23,10 +29,12 @@ pub struct MqttOptions {
     pub proxy: Option<SocketAddr>,
     #[serde(default)]
     pub connect_timeout: u64,
-    #[serde(default)]
-    pub read_timeout: u64,
+    pub read_timeout: Option<u64>,
     #[serde(default)]
     pub write_timeout: u64,
+
+    pub ping_interval: Option<u64>,
+    pub keep_alive: Option<u16>,
 }
 
 impl MqttOptions {
@@ -37,25 +45,44 @@ impl MqttOptions {
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum MqttRecvError {
+    #[error("Failed to read mqtt packet: {0:#}")]
+    Recv(#[from] VariablePacketError),
+    #[error("Failed to read mqtt packet: connection disconnected")]
+    RecvDisconnect,
+    #[error("Failed to interact with mqtt: {0:#}")]
+    Error(Error),
+}
+
+impl From<Error> for MqttRecvError {
+    fn from(err: Error) -> Self {
+        MqttRecvError::Error(err)
+    }
+}
+
 pub struct MqttClient {
     stream: Stream,
+    last_ping: Instant,
+    ping_interval: Option<u64>,
 }
 
 impl MqttClient {
     pub fn negotiate(stream: Stream, options: &MqttOptions) -> Result<MqttClient> {
+        // default to DEFAULT_PING_INTERVAL, if an explicit value of 0 was set, disable auto-ping
+        let ping_interval = Some(options.ping_interval.unwrap_or(DEFAULT_PING_INTERVAL));
+        ping_interval.filter(|s| *s != 0);
+
         let mut client = MqttClient {
             stream,
+            last_ping: Instant::now(),
+            ping_interval,
         };
 
         let mut pkt = ConnectPacket::new("sn0int");
         pkt.set_user_name(options.username.clone());
         pkt.set_password(options.password.clone());
-
-        /*
-        if let Some(keep_alive) = msg.keep_alive {
-            packet.set_keep_alive(keep_alive);
-        }
-        */
+        pkt.set_keep_alive(options.keep_alive.unwrap_or(DEFAULT_KEEP_ALIVE));
 
         client.send(pkt.into())?;
         let pkt = client.recv()?;
@@ -72,14 +99,19 @@ impl MqttClient {
         }
     }
 
-    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &MqttOptions) -> Result<MqttClient> {
+    pub fn connect<R: DnsResolver>(
+        resolver: &R,
+        url: Url,
+        options: &MqttOptions,
+    ) -> Result<MqttClient> {
         let tls = match url.scheme() {
             "mqtt" => false,
             "mqtts" => true,
             _ => bail!("Invalid mqtt protocol"),
         };
 
-        let host = url.host_str()
+        let host = url
+            .host_str()
             .ok_or_else(|| format_err!("Missing host in url"))?;
 
         let port = match (url.port(), tls) {
@@ -88,29 +120,57 @@ impl MqttClient {
             (None, false) => 1883,
         };
 
+        // if no read timeout is configured then keep alive won't work
+        let read_timeout = options.read_timeout.unwrap_or(DEFAULT_PING_INTERVAL);
 
-        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
-            tls,
-            sni_value: None,
-            disable_tls_verify: false,
-            proxy: options.proxy,
+        let stream = Stream::connect_stream(
+            resolver,
+            host,
+            port,
+            &SocketOptions {
+                tls,
+                sni_value: None,
+                disable_tls_verify: false,
+                proxy: options.proxy,
 
-            connect_timeout: options.connect_timeout,
-            read_timeout: options.read_timeout,
-            write_timeout: options.write_timeout,
-        })?;
+                connect_timeout: options.connect_timeout,
+                read_timeout,
+                write_timeout: options.write_timeout,
+            },
+        )?;
 
         Self::negotiate(stream, options)
     }
 
+    fn maintain_ping(&mut self) -> Result<()> {
+        if let Some(ping_interval) = self.ping_interval {
+            if self.last_ping.elapsed() >= Duration::from_secs(ping_interval) {
+                self.ping().context("Failed to ping")?;
+                self.last_ping = Instant::now();
+            }
+        }
+        Ok(())
+    }
+
     fn send(&mut self, pkt: VariablePacket) -> Result<()> {
+        self.maintain_ping()?;
         debug!("Sending mqtt packet: {:?}", pkt);
         pkt.encode(&mut self.stream)?;
         Ok(())
     }
 
-    fn recv(&mut self) -> std::result::Result<VariablePacket, VariablePacketError> {
-        let pkt = VariablePacket::decode(&mut self.stream)?;
+    fn recv(&mut self) -> std::result::Result<VariablePacket, MqttRecvError> {
+        self.maintain_ping()?;
+        let pkt = VariablePacket::decode(&mut self.stream).map_err(|err| match err {
+            // search for any io error and check if it's ErrorKind::UnexpectedEof
+            VariablePacketError::IoError(err)
+            | VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))
+                if err.kind() == io::ErrorKind::UnexpectedEof =>
+            {
+                MqttRecvError::RecvDisconnect
+            }
+            _ => MqttRecvError::Recv(err),
+        })?;
         debug!("Received mqtt packet: {:?}", pkt);
         Ok(pkt)
     }
@@ -139,30 +199,35 @@ impl MqttClient {
     pub fn recv_pkt(&mut self) -> Result<Option<Pkt>> {
         match self.recv() {
             Ok(pkt) => Ok(Some(Pkt::try_from(pkt)?)),
-            Err(VariablePacketError::IoError(err)) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
-            Err(VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
-            Err(err) => Err(Error::from(err))
+            // search for any io error and check if it's ErrorKind::WouldBlock
+            Err(MqttRecvError::Recv(
+                VariablePacketError::IoError(err)
+                | VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err)),
+            )) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(err) => Err(err.into()),
         }
     }
 
     pub fn ping(&mut self) -> Result<()> {
         let pkt = PingreqPacket::new();
-        self.send(pkt.into())
+        let pkt = VariablePacket::PingreqPacket(pkt);
+        pkt.encode(&mut self.stream)?;
+        Ok(())
     }
 }
 
 #[derive(Serialize, Deserialize)]
 #[serde(tag = "type")]
 pub enum Pkt {
-    #[serde(rename="publish")]
+    #[serde(rename = "publish")]
     Publish(Publish),
-    #[serde(rename="pong")]
+    #[serde(rename = "pong")]
     Pong,
 }
 
 impl Pkt {
     pub fn to_lua(&self) -> Result<AnyLuaValue> {
-        let v = serde_json::to_value(&self)?;
+        let v = serde_json::to_value(self)?;
         let v = LuaJsonValue::from(v).into();
         Ok(v)
     }

sn0int-std/src/sockets/mod.rs

@@ -107,7 +107,7 @@ impl Stream {
         let mut errors = Vec::new();
 
         for addr in addrs {
-            match Stream::connect_addr(host, (addr, port).into(), &options) {
+            match Stream::connect_addr(host, (addr, port).into(), options) {
                 Ok(socket) => {
                     return Ok(socket);
                 },
@@ -129,7 +129,7 @@ impl Stream {
         let socket = if connect_timeout > 0 {
             TcpStream::connect_timeout(&addr, Duration::from_secs(connect_timeout))?
         } else {
-            TcpStream::connect(&addr)?
+            TcpStream::connect(addr)?
         };
         debug!("successfully connected to {:?}", addr);
 
@@ -138,7 +138,7 @@ impl Stream {
         tls::wrap_if_enabled(socket, host, options)
     }
 
-    pub fn connect_socks5_stream(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+    pub fn connect_socks5_stream(proxy: SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
         debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
 
         let addr = match host.parse::<Ipv4Addr>() {
@@ -146,7 +146,7 @@ impl Stream {
             _ => ProxyDest::Domain(host.to_string()),
         };
 
-        let fut = socks5::connect(proxy, addr, port);
+        let fut = socks5::connect(&proxy, addr, port);
 
         let mut rt = Runtime::new()?;
         let socket = rt.block_on(fut)?;
@@ -211,7 +211,7 @@ impl Socket {
         Ok(Socket::new(stream))
     }
 
-    pub fn connect_socks5(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
+    pub fn connect_socks5(proxy: SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
         let stream = Stream::connect_socks5_stream(proxy, host, port, options)?;
         Ok(Socket::new(stream))
     }
@@ -233,7 +233,7 @@ impl Socket {
     }
 
     pub fn send(&mut self, data: &[u8]) -> Result<()> {
-        match str::from_utf8(&data) {
+        match str::from_utf8(data) {
             Ok(data) => debug!("send: {:?}", data),
             Err(_) => debug!("send: {:?}", data),
         };

sn0int-std/src/sockets/tls.rs

@@ -17,7 +17,7 @@ pub struct TlsData {
 
 impl TlsData {
     pub fn to_lua(&self) -> Result<AnyLuaValue> {
-        let v = serde_json::to_value(&self)?;
+        let v = serde_json::to_value(self)?;
         let v = LuaJsonValue::from(v).into();
         Ok(v)
     }
@@ -47,9 +47,9 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
     }
 
     let dns_name = if let Some(v) = &options.sni_value {
-        get_dns_name(&mut config, &v)
+        get_dns_name(&mut config, v)
     } else {
-        get_dns_name(&mut config, &host)
+        get_dns_name(&mut config, host)
     };
 
     let config = Arc::new(config);
@@ -58,7 +58,7 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
 }
 
 fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
-    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(&host) {
+    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(host) {
         debug!("setting sni value to: {:?}", host);
         name.to_owned()
     } else {

sn0int-std/src/web.rs

@@ -8,11 +8,13 @@ pub use chrootable_https::{Client, HttpClient, Resolver, Response};
 use chrootable_https::http::HttpTryFrom;
 use chrootable_https::http::uri::Parts;
 use chrootable_https::http::request::Builder;
+use data_encoding::BASE64;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
 use serde::{Serialize, Deserialize};
 use std::collections::{HashMap, HashSet};
 use std::fmt;
+use std::fmt::Write;
 use std::iter;
 use std::net::SocketAddr;
 use std::ops::Deref;
@@ -65,21 +67,21 @@ impl HttpSession {
 
 #[derive(Debug, Default, Deserialize)]
 pub struct RequestOptions {
-    query: Option<HashMap<String, String>>,
-    headers: Option<HashMap<String, String>>,
-    basic_auth: Option<(String, String)>,
-    user_agent: Option<String>,
-    json: Option<serde_json::Value>,
-    form: Option<serde_json::Value>,
+    pub query: Option<HashMap<String, String>>,
+    pub headers: Option<HashMap<String, String>>,
+    pub basic_auth: Option<(String, String)>,
+    pub user_agent: Option<String>,
+    pub json: Option<serde_json::Value>,
+    pub form: Option<serde_json::Value>,
     #[serde(default)]
-    follow_redirects: usize,
-    body: Option<String>,
-    timeout: Option<u64>,
+    pub follow_redirects: usize,
+    pub body: Option<String>,
+    pub timeout: Option<u64>,
     #[serde(default)]
-    into_blob: bool,
-    proxy: Option<SocketAddr>,
+    pub into_blob: bool,
+    pub proxy: Option<SocketAddr>,
     #[serde(default)]
-    binary: bool,
+    pub binary: bool,
 }
 
 impl RequestOptions {
@@ -110,11 +112,8 @@ pub struct HttpRequest {
 }
 
 impl HttpRequest {
-    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions, default_agent: fn() -> String) -> HttpRequest {
+    pub fn new(session: &HttpSession, method: String, url: String, user_agent: String, options: RequestOptions) -> HttpRequest {
         let cookies = session.cookies.clone();
-
-        let user_agent = options.user_agent.unwrap_or_else(default_agent);
-
         let timeout = options.timeout.map(Duration::from_millis);
 
         let mut request = HttpRequest {
@@ -163,9 +162,9 @@ impl HttpRequest {
         // add headers
         if let Some(ref auth) = self.basic_auth {
             use chrootable_https::header::AUTHORIZATION;
-            let &(ref user, ref password) = auth;
+            let (user, password) = auth;
 
-            let auth = base64::encode(&format!("{}:{}", user, password));
+            let auth = BASE64.encode(format!("{}:{}", user, password).as_bytes());
             let auth = format!("Basic {}", auth);
             req.header(AUTHORIZATION, auth.as_str());
         }
@@ -216,7 +215,7 @@ impl HttpRequest {
             if self.follow_redirects > 0 && res.status >= 300 && res.status < 400 {
                 if let Some(location) = res.headers.get("location") {
                     let base = Url::parse(&url.to_string())?;
-                    let joined = base.join(&location)?;
+                    let joined = base.join(location)?;
                     url = joined.to_string().parse()?;
 
                     req = self.mkrequest("GET", &url).body(Body::empty())?;
@@ -254,7 +253,8 @@ impl HttpRequest {
             if !cookies.is_empty() {
                 cookies += "; ";
             }
-            cookies.push_str(&format!("{}={}", key, value));
+            // it's a write to a String, so panic if re-allocation fails is fine
+            write!(cookies, "{}={}", key, value).expect("out of memory");
         }
 
         if !cookies.is_empty() {
@@ -318,9 +318,9 @@ impl HttpRequest {
     }
 }
 
-impl Into<AnyLuaValue> for HttpRequest {
-    fn into(self) -> AnyLuaValue {
-        let v = serde_json::to_value(&self).unwrap();
+impl From<HttpRequest> for AnyLuaValue {
+    fn from(req: HttpRequest) -> AnyLuaValue {
+        let v = serde_json::to_value(req).unwrap();
         LuaJsonValue::from(v).into()
     }
 }

sn0int-std/src/websockets.rs

@@ -91,9 +91,11 @@ impl WebSocket {
     }
 
     pub fn options(&self, options: &WebSocketOptions) -> Result<()> {
-        let mut o = SocketOptions::default();
-        o.read_timeout = options.read_timeout;
-        o.write_timeout = options.write_timeout;
+        let o = SocketOptions {
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+            ..Default::default()
+        };
         o.apply(self.sock.get_ref())
     }
 

sn0int-std/src/xml.rs

@@ -62,7 +62,7 @@ pub fn decode(x: &str) -> Result<AnyLuaValue> {
 }
 
 #[inline]
-fn append_text(stack: &mut Vec<XmlElement>, text: String) {
+fn append_text(stack: &mut [XmlElement], text: String) {
     if let Some(tail) = stack.last_mut() {
         if let Some(prev) = tail.text.as_mut() {
             prev.push_str(&text);

src/args.rs

@@ -9,7 +9,7 @@ use sn0int_common::ModuleID;
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
     /// Select a different workspace instead of the default
-    #[structopt(short="w", long="workspace")]
+    #[structopt(short="w", long="workspace", env="SN0INT_WORKSPACE")]
     pub workspace: Option<Workspace>,
 
     #[structopt(subcommand)]
@@ -66,6 +66,12 @@ pub enum SubCommand {
     /// Exclude entities from scope
     #[structopt(name="noscope")]
     Noscope(cmd::noscope_cmd::Args),
+    /// Manage autoscope rules
+    Autoscope(cmd::autoscope_cmd::Args),
+    /// Manage autonoscope rules
+    Autonoscope(cmd::autonoscope_cmd::Args),
+    /// Rescope all entities based on autonoscope rules
+    Rescope(cmd::rescope_cmd::Args),
     /// Manage workspaces
     #[structopt(name="workspace")]
     Workspace(cmd::workspace_cmd::Args),
@@ -97,18 +103,11 @@ pub enum SubCommand {
 
 #[derive(Debug, StructOpt)]
 pub struct Run {
-    /// Execute a module that has been installed
-    pub module: String,
+    #[structopt(flatten)]
+    pub run: cmd::run_cmd::Args,
     /// Run a module from a path
     #[structopt(short="f", long="file")]
     pub file: bool,
-    /// Run modules concurrently
-    #[structopt(short="j", long="threads", default_value="1")]
-    pub threads: usize,
-    /// Verbose logging, once to print inserts even if they don't add new
-    /// data, twice to activate the debug() function
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    pub verbose: u64,
     /// Expose stdin to modules
     #[structopt(long="stdin")]
     pub stdin: bool,
@@ -138,7 +137,7 @@ pub struct Run {
 #[derive(Debug, StructOpt)]
 pub struct Sandbox {
     /// This value is only used for process listings
-    label: String,
+    _label: String,
 }
 
 #[derive(Debug, StructOpt)]

src/autonoscope/domain.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 
@@ -41,6 +41,13 @@ impl TryFrom<Autonoscope> for DomainRule {
     }
 }
 
+impl AutoRule<Domain> for DomainRule {
+    #[inline]
+    fn matches(&self, domain: &Domain) -> Result<bool> {
+        self.matches(domain.value.as_str())
+    }
+}
+
 impl AutoRule<NewDomain> for DomainRule {
     #[inline]
     fn matches(&self, domain: &NewDomain) -> Result<bool> {
@@ -48,6 +55,13 @@ impl AutoRule<NewDomain> for DomainRule {
     }
 }
 
+impl AutoRule<Subdomain> for DomainRule {
+    #[inline]
+    fn matches(&self, domain: &Subdomain) -> Result<bool> {
+        self.matches(domain.value.as_str())
+    }
+}
+
 impl AutoRule<NewSubdomain> for DomainRule {
     #[inline]
     fn matches(&self, domain: &NewSubdomain) -> Result<bool> {
@@ -55,6 +69,18 @@ impl AutoRule<NewSubdomain> for DomainRule {
     }
 }
 
+impl AutoRule<Url> for DomainRule {
+    #[inline]
+    fn matches(&self, url: &Url) -> Result<bool> {
+        let url = url.value.parse::<url::Url>()?;
+        if let Some(domain) = url.domain() {
+            self.matches(domain)
+        } else {
+            Ok(false)
+        }
+    }
+}
+
 impl AutoRule<NewUrl> for DomainRule {
     #[inline]
     fn matches(&self, url: &NewUrl) -> Result<bool> {
@@ -94,8 +120,8 @@ impl RulePrecision for DomainRule {
     }
 }
 
-impl IntoRule for DomainRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for DomainRule {
+    fn to_rule(&self) -> (&'static str, String) {
         ("domain", self.to_string())
     }
 }

src/autonoscope/ip.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 use std::net;
@@ -36,12 +36,25 @@ impl TryFrom<Autonoscope> for IpRule {
     }
 }
 
+impl AutoRule<IpAddr> for IpRule {
+    fn matches(&self, ipaddr: &IpAddr) -> Result<bool> {
+        self.matches(ipaddr.value.as_str())
+    }
+}
+
 impl AutoRule<NewIpAddr> for IpRule {
     fn matches(&self, ipaddr: &NewIpAddr) -> Result<bool> {
         self.matches(ipaddr.value.as_str())
     }
 }
 
+impl AutoRule<Port> for IpRule {
+    fn matches(&self, port: &Port) -> Result<bool> {
+        let addr = port.value.parse::<net::SocketAddr>()?;
+        self.matches(&addr.ip())
+    }
+}
+
 impl AutoRule<NewPort> for IpRule {
     fn matches(&self, port: &NewPort) -> Result<bool> {
         let addr = port.value.parse::<net::SocketAddr>()?;
@@ -49,15 +62,25 @@ impl AutoRule<NewPort> for IpRule {
     }
 }
 
+fn match_netblock_str(network: &IpNetwork, netblock: &str) -> Result<bool> {
+    let range = netblock.parse::<ipnetwork::IpNetwork>()?;
+
+    if network.prefix() <= range.prefix() {
+        Ok(network.contains(range.ip()))
+    } else {
+        Ok(false)
+    }
+}
+
+impl AutoRule<Netblock> for IpRule {
+    fn matches(&self, netblock: &Netblock) -> Result<bool> {
+        match_netblock_str(&self.network, &netblock.value)
+    }
+}
+
 impl AutoRule<NewNetblock> for IpRule {
     fn matches(&self, netblock: &NewNetblock) -> Result<bool> {
-        let range = netblock.value.parse::<ipnetwork::IpNetwork>()?;
-
-        if self.network.prefix() <= range.prefix() {
-            Ok(self.network.contains(range.ip()))
-        } else {
-            Ok(false)
-        }
+        match_netblock_str(&self.network, &netblock.value)
     }
 }
 
@@ -80,8 +103,8 @@ impl RulePrecision for IpRule {
     }
 }
 
-impl IntoRule for IpRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for IpRule {
+    fn to_rule(&self) -> (&'static str, String) {
         ("ip", self.to_string())
     }
 }

src/autonoscope/mod.rs

@@ -47,6 +47,18 @@ fn sort_precision_desc<T: RulePrecision>(a: &T, b: &T) -> Ordering {
 }
 
 impl RuleSet {
+    pub fn domains(&self) -> &[Rule<DomainRule>] {
+        &self.domains
+    }
+
+    pub fn ips(&self) -> &[Rule<IpRule>] {
+        &self.ips
+    }
+
+    pub fn urls(&self) -> &[Rule<UrlRule>] {
+        &self.urls
+    }
+
     pub fn load(db: &DatabaseSock) -> Result<Self> {
         use crate::schema::autonoscope::dsl::*;
         let rules = autonoscope.load::<Autonoscope>(db)?;
@@ -112,15 +124,15 @@ impl RuleSet {
         match obj {
             RuleType::Domain => {
                 self.domains.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
             },
             RuleType::Ip => {
                 self.ips.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
             },
             RuleType::Url => {
                 self.urls.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
             },
         }
         Ok(())
@@ -143,10 +155,18 @@ impl RuleSet {
         rules
     }
 
+    pub fn is_empty(&self) -> bool {
+        self.domains.is_empty() && self.ips.is_empty() && self.urls.is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        self.domains.len() + self.ips.len() + self.urls.len()
+    }
+
     #[inline]
-    fn push_rules_display<T: IntoRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
+    fn push_rules_display<T: ToRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
         for rule in rules {
-            let (object, value) = rule.into_rule();
+            let (object, value) = rule.to_rule();
             output.push((object, value, rule.scoped));
         }
     }
@@ -159,10 +179,8 @@ impl RuleSet {
             Insert::Url(url) => {
                 if let Some(result) = Self::matches_any(&self.domains, url)? {
                     Some(result)
-                } else if let Some(result) = Self::matches_any(&self.urls, url)? {
-                    Some(result)
                 } else {
-                    None
+                    Self::matches_any(&self.urls, url)?
                 }
             },
             // Insert::Email(email) => unimplemented!(),
@@ -176,7 +194,7 @@ impl RuleSet {
 
     fn matches_any<T1, T2>(rules: &[Rule<T1>], object: &T2) -> Result<Option<bool>>
         where T1: AutoRule<T2>,
-            T1: IntoRule,
+            T1: ToRule,
     {
         for rule in rules {
             if rule.matches(object)? {
@@ -238,12 +256,12 @@ pub trait RulePrecision {
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Rule<T: IntoRule> {
+pub struct Rule<T: ToRule> {
     rule: T,
-    scoped: bool,
+    pub scoped: bool,
 }
 
-impl<T: IntoRule> Rule<T> {
+impl<T: ToRule> Rule<T> {
     pub fn new(rule: T, scoped: bool) -> Rule<T> {
         Rule {
             rule,
@@ -254,7 +272,7 @@ impl<T: IntoRule> Rule<T> {
 
 // TODO: maybe drop this
 use std::ops::Deref;
-impl<T: IntoRule> Deref for Rule<T> {
+impl<T: ToRule> Deref for Rule<T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -262,23 +280,23 @@ impl<T: IntoRule> Deref for Rule<T> {
     }
 }
 
-impl<T: IntoRule + RulePrecision> RulePrecision for Rule<T> {
+impl<T: ToRule + RulePrecision> RulePrecision for Rule<T> {
     fn precision(&self) -> usize {
         self.rule.precision()
     }
 }
 
-pub trait IntoRule {
-    fn into_rule(&self) -> (&'static str, String);
+pub trait ToRule {
+    fn to_rule(&self) -> (&'static str, String);
 }
 
-impl<T: IntoRule> Into<NewAutonoscope> for &Rule<T> {
-    fn into(self) -> NewAutonoscope {
-        let (object, value) = self.rule.into_rule();
+impl<T: ToRule> From<&Rule<T>> for NewAutonoscope {
+    fn from(rule: &Rule<T>) -> NewAutonoscope {
+        let (object, value) = rule.rule.to_rule();
         NewAutonoscope {
             object: object.to_string(),
             value,
-            scoped: self.scoped,
+            scoped: rule.scoped,
         }
     }
 }

src/autonoscope/url.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 
@@ -47,6 +47,12 @@ impl TryFrom<&str> for UrlRule {
 }
 
 // TODO: there is no way to write a rule that matches all urls
+impl AutoRule<Url> for UrlRule {
+    fn matches(&self, url: &Url) -> Result<bool> {
+        self.matches(url.value.as_str())
+    }
+}
+
 impl AutoRule<NewUrl> for UrlRule {
     fn matches(&self, url: &NewUrl) -> Result<bool> {
         self.matches(url.value.as_str())
@@ -86,8 +92,8 @@ impl RulePrecision for UrlRule {
     }
 }
 
-impl IntoRule for UrlRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for UrlRule {
+    fn to_rule(&self) -> (&'static str, String) {
         ("url", self.to_string())
     }
 }

src/cal/date.rs

@@ -6,6 +6,7 @@ use crate::cal::{ActivityGrade, DateArg};
 use crate::models::*;
 use std::collections::HashMap;
 use std::collections::VecDeque;
+use std::fmt::Write;
 
 const MONTH_LINES: i32 = 7;
 
@@ -54,11 +55,11 @@ fn chunk_months(ctx: &DateContext, months: &[DateSpec]) -> String {
 }
 
 fn days_in_month(year: i32, month: u32) -> i64 {
-    let start = Utc.ymd(year, month, 1);
+    let start = Utc.with_ymd_and_hms(year, month, 1, 0, 0, 0).single().expect("Datetime is not unique");
     let end = if month == 12 {
-        Utc.ymd(year + 1, 1, 1)
+        Utc.with_ymd_and_hms(year + 1, 1, 1, 0, 0, 0).single().expect("Datetime is not unique")
     } else {
-        Utc.ymd(year, month + 1, 1)
+        Utc.with_ymd_and_hms(year, month + 1, 1, 0, 0, 0).single().expect("Datetime is not unique")
     };
     end.signed_duration_since(start).num_days()
 }
@@ -112,7 +113,7 @@ pub struct DateContext {
 
 impl DateContext {
     pub fn new(events: &[Activity], today: NaiveDate) -> DateContext {
-        let (events, max) = setup_graph_map(&events);
+        let (events, max) = setup_graph_map(events);
         DateContext {
             events,
             max,
@@ -151,7 +152,7 @@ impl DateSpec {
             bail!("Too many datespec args");
         }
 
-        let today = Utc::today();
+        let today = Utc::now();
         let ds = match (args.get(0), args.get(1), context) {
             (None, _, None) => DateSpec::YearMonth((today.year(), today.month())),
             (None, _, Some(context)) => DateSpec::YearMonthContext((today.year(), today.month(), context)),
@@ -171,8 +172,8 @@ impl DateSpec {
 
     pub fn start(&self) -> NaiveDate {
         match self {
-            DateSpec::Year(year) => NaiveDate::from_ymd(*year, 1, 1),
-            DateSpec::YearMonth((year, month)) => NaiveDate::from_ymd(*year, *month, 1),
+            DateSpec::Year(year) => NaiveDate::from_ymd_opt(*year, 1, 1).expect("Invalid month/day"),
+            DateSpec::YearMonth((year, month)) => NaiveDate::from_ymd_opt(*year, *month, 1).expect("Invalid month/day"),
             DateSpec::YearMonthContext((year, month, context)) => {
                 let mut year = *year - (*context / 12) as i32;
                 let context = context % 12;
@@ -182,21 +183,21 @@ impl DateSpec {
                 } else {
                     month - context
                 };
-                NaiveDate::from_ymd(year, month, 1)
+                NaiveDate::from_ymd_opt(year, month, 1).expect("Invalid month/day")
             },
         }
     }
 
     pub fn end(&self) -> NaiveDate {
         match self {
-            DateSpec::Year(year) => NaiveDate::from_ymd(year + 1, 1, 1),
+            DateSpec::Year(year) => NaiveDate::from_ymd_opt(year + 1, 1, 1).expect("Invalid month/day"),
             DateSpec::YearMonth((year, month)) => {
                 let (year, month) = if *month == 12 {
                     (*year + 1, 1)
                 } else {
                     (*year, *month + 1)
                 };
-                NaiveDate::from_ymd(year, month, 1)
+                NaiveDate::from_ymd_opt(year, month, 1).expect("Invalid month/day")
             },
             DateSpec::YearMonthContext((year, month, _context)) => {
                 let (year, month) = if *month == 12 {
@@ -204,7 +205,7 @@ impl DateSpec {
                 } else {
                     (*year, *month + 1)
                 };
-                NaiveDate::from_ymd(year, month, 1)
+                NaiveDate::from_ymd_opt(year, month, 1).expect("Invalid month/day")
             },
         }
     }
@@ -220,10 +221,10 @@ impl DateSpec {
             DateSpec::YearMonth((year, month)) => {
                 let mut w = String::new();
 
-                let start = Utc.ymd(*year, *month, 1);
+                let start = Utc.with_ymd_and_hms(*year, *month, 1, 0, 0, 0).single().expect("Datetime is not unique");
                 let days = days_in_month(*year, *month) as u32;
 
-                w.push_str(&format!("{:^21}\n", start.format("%B %Y")));
+                writeln!(w, "{:^21}", start.format("%B %Y")).expect("out of memory");
                 w.push_str(" Su Mo Tu We Th Fr Sa\n");
 
                 let mut cur_week_day = start.weekday();
@@ -232,7 +233,7 @@ impl DateSpec {
 
                 let mut week_written = week_progress * 3;
                 for cur_day in 1..=days {
-                    let date = NaiveDate::from_ymd(*year, *month, cur_day);
+                    let date = NaiveDate::from_ymd_opt(*year, *month, cur_day).expect("Invalid month/day");
 
                     if !ctx.is_future(&date) {
                         let activity = ctx.activity_for_day(&date);
@@ -244,7 +245,7 @@ impl DateSpec {
                     } else {
                         w.push(' ');
                     }
-                    w.push_str(&format!("{:2}", cur_day));
+                    write!(w, "{:2}", cur_day).expect("out of memory");
                     week_written += 3;
                     w.push_str("\x1b[0m");
 
@@ -296,25 +297,25 @@ mod tests {
         DateContext {
             events: HashMap::new(),
             max: 0,
-            today: NaiveDate::from_ymd(2020, 05, 30),
+            today: NaiveDate::from_ymd_opt(2020, 5, 30).unwrap(),
         }
     }
 
     #[test]
     fn test_days_in_month_2020_05() {
-        let days = days_in_month(2020, 05);
+        let days = days_in_month(2020, 5);
         assert_eq!(days, 31);
     }
 
     #[test]
     fn test_days_in_month_2020_04() {
-        let days = days_in_month(2020, 04);
+        let days = days_in_month(2020, 4);
         assert_eq!(days, 30);
     }
 
     #[test]
     fn test_days_in_month_2020_02() {
-        let days = days_in_month(2020, 02);
+        let days = days_in_month(2020, 2);
         assert_eq!(days, 29);
     }
 
@@ -324,15 +325,15 @@ mod tests {
         let ctx = DateContext {
             events,
             max: 0,
-            today: NaiveDate::from_ymd(2020, 06, 06),
+            today: NaiveDate::from_ymd_opt(2020, 6, 6).unwrap(),
         };
-        let grade = ctx.activity_for_day(&NaiveDate::from_ymd(2020, 06, 06));
+        let grade = ctx.activity_for_day(&NaiveDate::from_ymd_opt(2020, 6, 6).unwrap());
         assert_eq!(grade, ActivityGrade::None);
     }
 
     #[test]
     fn test_datespec_year_month() {
-        let ds = DateSpec::YearMonth((2020, 05));
+        let ds = DateSpec::YearMonth((2020, 5));
         let out = ds.to_term_string(&context());
         assert_eq!(out, "      May 2020       
  Su Mo Tu We Th Fr Sa

src/cal/time.rs

@@ -5,6 +5,7 @@ use chrono::prelude::*;
 use crate::cal::{ActivityGrade, DateArg};
 use crate::models::*;
 use std::collections::HashMap;
+use std::fmt::Write;
 
 const MIN_PER_DAY: u32 = 1440;
 
@@ -13,7 +14,7 @@ fn round_to_slice(time: &NaiveDateTime, slice_duration: u32) -> NaiveDateTime {
     let hour = time.hour();
     let mins = time.minute();
     let slice = mins - (mins % slice_duration);
-    date.and_hms(hour, slice, 0)
+    date.and_hms_opt(hour, slice, 0).expect("Invalid hour/min/sec")
 }
 
 fn setup_graph_map(events: &[Activity], slice_duration: u32) -> (HashMap<NaiveDateTime, u64>, u64) {
@@ -68,7 +69,7 @@ pub struct DateTimeContext {
 
 impl DateTimeContext {
     pub fn new(events: &[Activity], now: NaiveDateTime, slice_width: u32, slice_duration: u32) -> DateTimeContext {
-        let (events, max) = setup_graph_map(&events, slice_duration);
+        let (events, max) = setup_graph_map(events, slice_duration);
         DateTimeContext {
             events,
             max,
@@ -109,7 +110,7 @@ pub struct DateTimeSpec {
 
 impl DateTimeSpec {
     pub fn from_args(args: &[DateArg], context: Option<u32>) -> Result<DateTimeSpec> {
-        let today = Utc::today().naive_utc();
+        let today = Utc::now().date_naive();
         if args.is_empty() {
             let mut start = today;
 
@@ -156,7 +157,7 @@ impl DateTimeSpec {
         // add legend
         w.push_str(&" ".repeat(11));
         for x in 0..24 {
-            w.push_str(&format!("{:02}", x));
+            write!(w, "{:02}", x).expect("out of memory");
 
             for i in 0..ctx.hour_width() {
                 if i >= 2 {
@@ -175,7 +176,7 @@ impl DateTimeSpec {
             let mut mins = 0;
 
             for _ in 0..(MIN_PER_DAY / ctx.slice_duration) {
-                let time = date.and_hms(hours, mins, 0);
+                let time = date.and_hms_opt(hours, mins, 0).expect("Invalid hour/min/sec");
 
                 if !ctx.is_future(&time) {
                     let activity = ctx.activity_for_slice(&time);

src/cmd/activity_cmd.rs

@@ -18,7 +18,7 @@ pub struct TimeSpec {
 
 impl TimeSpec {
     fn resolve(s: &str, now: NaiveDateTime) -> Result<Self> {
-        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
+        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms_opt(0, 0, 0).expect("Invalid hour/min/sec"));
 
         let datetime = match s {
             "today" => today,
@@ -122,8 +122,8 @@ mod tests {
     use super::*;
 
     fn datetime() -> NaiveDateTime {
-        let date = chrono::NaiveDate::from_ymd(2020, 3, 14);
-        let time = chrono::NaiveTime::from_hms(16, 20, 23);
+        let date = chrono::NaiveDate::from_ymd_opt(2020, 3, 14).unwrap();
+        let time = chrono::NaiveTime::from_hms_opt(16, 20, 23).unwrap();
         NaiveDateTime::new(date, time)
     }
 

src/cmd/add_cmd.rs

@@ -290,7 +290,7 @@ impl InsertFromString for AddUrl {
         let subdomain = parts.domain()
             .ok_or_else(|| format_err!("url doesn't have a domain host"))?;
 
-        let dns_name = rl.psl()?.parse_dns_name(&subdomain)
+        let dns_name = rl.psl()?.parse_dns_name(subdomain)
             .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
 
         let domain_id = match rl.db().insert_struct(NewDomain {

src/cmd/autonoscope_cmd.rs

@@ -1,6 +1,7 @@
 use crate::errors::*;
 
 use crate::autonoscope;
+use crate::cmd::Cmd;
 use crate::fmt::colors::*;
 use crate::shell::Shell;
 use std::fmt::Write;
@@ -65,7 +66,8 @@ pub fn run_with_scope_param(rl: &mut Shell, args: Args, scoped: bool) -> Result<
     }
 }
 
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    run_with_scope_param(rl, args, false)
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        run_with_scope_param(rl, self, false)
+    }
 }

src/cmd/autoscope_cmd.rs

@@ -1,13 +1,18 @@
 use crate::errors::*;
 
+use crate::cmd::Cmd;
 use crate::cmd::autonoscope_cmd;
 use crate::shell::Shell;
 use structopt::StructOpt;
 
-
-pub type Args = autonoscope_cmd::Args;
-
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    autonoscope_cmd::run_with_scope_param(rl, args, true)
+#[derive(Debug, StructOpt)]
+pub struct Args {
+    #[structopt(flatten)]
+    args: autonoscope_cmd::Args,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        autonoscope_cmd::run_with_scope_param(rl, self.args, true)
+    }
 }

src/cmd/cal_cmd.rs

@@ -34,8 +34,8 @@ impl Cmd for Args {
                 .context("Failed to parse date spec")?;
             let filter = ActivityFilter {
                 topic: None,
-                since: Some(dts.start().and_hms(0, 0, 0)),
-                until: Some(dts.end().and_hms(23, 59, 59)),
+                since: Some(dts.start().and_hms_opt(0, 0, 0).expect("Invalid hour/min/sec")),
+                until: Some(dts.end().and_hms_opt(23, 59, 59).expect("Invalid hour/min/sec")),
                 location: false,
             };
             let events = Activity::query(rl.db(), &filter)?;
@@ -53,12 +53,12 @@ impl Cmd for Args {
                 .context("Failed to parse date spec")?;
             let filter = ActivityFilter {
                 topic: None,
-                since: Some(ds.start().and_hms(0, 0, 0)),
-                until: Some(ds.end().and_hms(23, 59, 59)),
+                since: Some(ds.start().and_hms_opt(0, 0, 0).expect("Invalid hour/min/sec")),
+                until: Some(ds.end().and_hms_opt(23, 59, 59).expect("Invalid hour/min/sec")),
                 location: false,
             };
             let events = Activity::query(rl.db(), &filter)?;
-            let ctx = DateContext::new(&events, Utc::today().naive_utc());
+            let ctx = DateContext::new(&events, Utc::now().date_naive());
             println!("{}", ds.to_term_string(&ctx));
         }
         Ok(())

src/cmd/delete_cmd.rs

@@ -19,20 +19,20 @@ pub struct Args {
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
         let rows = match &self.subcommand {
-            Target::Domains(filter) => delete::<Domain>(rl, &filter),
-            Target::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
-            Target::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
-            Target::Urls(filter) => delete::<Url>(rl, &filter),
-            Target::Emails(filter) => delete::<Email>(rl, &filter),
-            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
-            Target::Devices(filter) => delete::<Device>(rl, &filter),
-            Target::Networks(filter) => delete::<Network>(rl, &filter),
-            Target::Accounts(filter) => delete::<Account>(rl, &filter),
-            Target::Breaches(filter) => delete::<Breach>(rl, &filter),
-            Target::Images(filter) => delete::<Image>(rl, &filter),
-            Target::Ports(filter) => delete::<Port>(rl, &filter),
-            Target::Netblocks(filter) => delete::<Netblock>(rl, &filter),
-            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, &filter),
+            Target::Domains(filter) => delete::<Domain>(rl, filter),
+            Target::Subdomains(filter) => delete::<Subdomain>(rl, filter),
+            Target::IpAddrs(filter) => delete::<IpAddr>(rl, filter),
+            Target::Urls(filter) => delete::<Url>(rl, filter),
+            Target::Emails(filter) => delete::<Email>(rl, filter),
+            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, filter),
+            Target::Devices(filter) => delete::<Device>(rl, filter),
+            Target::Networks(filter) => delete::<Network>(rl, filter),
+            Target::Accounts(filter) => delete::<Account>(rl, filter),
+            Target::Breaches(filter) => delete::<Breach>(rl, filter),
+            Target::Images(filter) => delete::<Image>(rl, filter),
+            Target::Ports(filter) => delete::<Port>(rl, filter),
+            Target::Netblocks(filter) => delete::<Netblock>(rl, filter),
+            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, filter),
         }?;
         term::info(&format!("Deleted {} rows", rows));
         Ok(())

src/cmd/mod.rs

@@ -32,6 +32,7 @@ pub mod keyring_cmd;
 pub mod noscope_cmd;
 pub mod notify_cmd;
 pub mod pkg_cmd;
+pub mod rescope_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;
 pub mod stats_cmd;

src/cmd/notify_cmd.rs

@@ -2,12 +2,12 @@ use crate::errors::*;
 
 use crate::cmd::Cmd;
 use crate::engine::Module;
-// use crate::models::*;
 use crate::notify::{self, Notification};
 use crate::options::{self, Opt};
 use crate::shell::Shell;
 use crate::term;
 use sn0int_std::ratelimits::Ratelimiter;
+use std::fmt::Write;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 
@@ -52,19 +52,19 @@ pub struct ExecArgs {
 }
 
 fn print_summary(module: &Module, sent: usize, errors: usize) {
-        let mut out = if sent == 1 {
-            String::from("Sent 1 notification")
-        } else {
-            format!("Sent {} notifications", sent)
-        };
+    let mut out = if sent == 1 {
+        String::from("Sent 1 notification")
+    } else {
+        format!("Sent {} notifications", sent)
+    };
 
-        out.push_str(&format!(" with {}", module.canonical()));
+    write!(out, " with {}", module.canonical()).expect("out of memory");
 
-        if errors > 0 {
-            out.push_str(&format!(" ({} errors)", errors));
-        }
+    if errors > 0 {
+        write!(out, " ({} errors)", errors).expect("out of memory");
+    }
 
-        term::info(&out);
+    term::info(&out);
 }
 
 fn send(args: SendArgs, rl: &mut Shell) -> Result<()> {

src/cmd/pkg_cmd.rs

@@ -137,7 +137,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
 
             for module in library.list() {
                 if let Some(source) = &list.source {
-                    if !module.source_equals(&source) {
+                    if !module.source_equals(source) {
                         continue;
                     }
                 }
@@ -148,7 +148,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
                 }
 
                 if let Some(stealth) = &list.stealth {
-                    if !module.stealth().equal_or_better(&stealth) {
+                    if !module.stealth().equal_or_better(stealth) {
                         continue;
                     }
                 }
@@ -163,17 +163,17 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
             Ok(ModuleReload::No)
         },
         SubCommand::Install(install) => {
-            registry::run_install(install, &config)?;
+            registry::run_install(install, config)?;
             // trigger reload
             Ok(ModuleReload::Yes)
         },
         SubCommand::Search(search) => {
-            registry::run_search(library, &search, &config)?;
+            registry::run_search(library, &search, config)?;
             Ok(ModuleReload::No)
         },
         SubCommand::Update(_) => {
             let mut autoupdate = AutoUpdater::load()?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let updater = Arc::new(Updater::new(config)?);
 
             let modules = library.list()
                 .into_iter()
@@ -199,14 +199,14 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
             Ok(ModuleReload::Yes)
         },
         SubCommand::Uninstall(uninstall) => {
-            let updater = Updater::new(&config)?;
+            let updater = Updater::new(config)?;
             updater.uninstall(&uninstall.module)?;
             // trigger reload
             Ok(ModuleReload::Yes)
         },
         SubCommand::Quickstart => {
-            let client = Client::new(&config)?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let client = Client::new(config)?;
+            let updater = Arc::new(Updater::new(config)?);
             let mut autoupdate = AutoUpdater::load()?;
 
             let installed = library.list()
@@ -250,7 +250,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
 
 impl LiteCmd for Args {
     fn run(self, config: &Config) -> Result<()> {
-        let library = Library::new(false, &config)?;
+        let library = Library::new(false, config)?;
         run_subcommand(self.subcommand, &library, config)?;
         Ok(())
     }

src/cmd/rescope_cmd.rs

@@ -0,0 +1,268 @@
+use crate::errors::*;
+
+use crate::autonoscope::{AutoRule, ToRule};
+use crate::cmd::Cmd;
+use crate::db::Database;
+use crate::filters::{Filter, Target};
+use crate::shell::Shell;
+use std::collections::HashSet;
+use std::fmt;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+use crate::models::*;
+use crate::utils;
+use crate::term;
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Run rules interactively
+    #[structopt(short, long)]
+    interactive: bool,
+    /// Automatically apply changes to database
+    #[structopt(short="y", long)]
+    auto_confirm: bool,
+    /// Only show changes, do not apply them to the database
+    #[structopt(short="n", long)]
+    dry_run: bool,
+    /// Only rescope entities matching specific filter
+    #[structopt(subcommand)]
+    target: Option<Target>,
+}
+
+enum Entity {
+    Domain(Domain),
+    Subdomain(Subdomain),
+    IpAddr(IpAddr),
+    Url(Url),
+    Port(Port),
+    Netblock(Netblock),
+}
+
+impl Entity {
+    fn set_scoped(&self, db: &Database, value: bool) -> Result<()> {
+        match self {
+            Entity::Domain(entity) => entity.set_scoped(db, value),
+            Entity::Subdomain(entity) => entity.set_scoped(db, value),
+            Entity::IpAddr(entity) => entity.set_scoped(db, value),
+            Entity::Url(entity) => entity.set_scoped(db, value),
+            Entity::Port(entity) => entity.set_scoped(db, value),
+            Entity::Netblock(entity) => entity.set_scoped(db, value),
+        }
+    }
+}
+
+enum Input {
+    Yes,
+    No,
+    Done,
+    Always,
+    Never,
+}
+
+fn get_input() -> Result<Input> {
+    loop {
+        let input = utils::question_opt("Update this entity? [Y/n/d/a/x/?]")?;
+        let input = input.map(|s| s.to_lowercase());
+        match input.as_deref() {
+            Some("y") | None => return Ok(Input::Yes),
+            Some("n") => return Ok(Input::No),
+            Some("d") => return Ok(Input::Done),
+            Some("a") => return Ok(Input::Always),
+            Some("x") => return Ok(Input::Never),
+            Some("?") => {
+                term::success("y -> yes, apply this change");
+                term::success("n -> no, skip this change");
+                term::success("d -> done, skip this and further changes");
+                term::success("a -> always, apply every change caused by this specific rule");
+                term::success("x -> never, skip every change caused by this specific rule");
+            },
+            Some(input) => term::error(&format!("Unrecognized input: {:?}", input)),
+        }
+    }
+}
+
+#[derive(Default)]
+struct Context {
+    update_queue: Vec<(Entity, bool)>,
+    always_rules: HashSet<(&'static str, String)>,
+    never_rules: HashSet<(&'static str, String)>,
+    done: bool,
+    target: Option<Target>,
+}
+
+fn rescope_to_queue<T, F1, F2, F3>(ctx: &mut Context, db: &Database, interactive: bool, get_filter: F1, matches_rule: F2, wrap: F3) -> Result<()>
+    where
+        T: Model + Scopable + fmt::Debug,
+        F1: Fn(&Target) -> Option<&Filter>,
+        F2: Fn(&T) -> Result<Option<((&'static str, String), bool)>>,
+        F3: Fn(T) -> Entity,
+{
+    // do nothing if we're already done
+    if ctx.done {
+        return Ok(());
+    }
+
+    // check if there are filters to be applied
+    let filter = if let Some(target) = &ctx.target {
+        if let Some(filter) = get_filter(target) {
+            // we've selected this specific entity type and there's a filter
+            filter.parse_optional()
+                .context("Filter is invalid")?
+        } else {
+            // we do not wish to process this entity type
+            return Ok(());
+        }
+    } else {
+        // any entity type is fine
+        Filter::any()
+    };
+
+    let entities = db.filter::<T>(&filter)?;
+
+    for entity in entities {
+        let currently_scoped = entity.scoped();
+        debug!("rescoping entity: {:?}", entity);
+
+        if let Some((rule, should_be)) = matches_rule(&entity)? {
+            // check if we're actively ignoring this rule
+            if ctx.never_rules.contains(&rule) {
+                continue;
+            }
+
+            if currently_scoped != should_be {
+                let prefix = if should_be {
+                    "\x1b[1m[\x1b[32m+\x1b[0;1m]\x1b[0m"
+                } else {
+                    "\x1b[1m[\x1b[31m-\x1b[0;1m]\x1b[0m"
+                };
+
+                println!("{} Setting entity {:?} => {:?}: {:?}", prefix, currently_scoped, should_be, entity);
+
+                // check if we're auto-accepting this rule
+                let input = if ctx.always_rules.contains(&rule) {
+                    Input::Yes
+                } else if interactive {
+                    get_input()?
+                } else {
+                    Input::Yes
+                };
+
+                // process user input
+                let input = match input {
+                    Input::Always => {
+                        ctx.always_rules.insert(rule);
+                        Input::Yes
+                    },
+                    Input::Never => {
+                        ctx.never_rules.insert(rule);
+                        Input::No
+                    },
+                    Input::Done => {
+                        ctx.done = true;
+                        break;
+                    },
+                    input => input,
+                };
+
+                if let Input::Yes = input {
+                    ctx.update_queue.push((wrap(entity), should_be));
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let rules = rl.db().autonoscope();
+        term::success(&format!("Loaded {} rules", rules.len()));
+
+        let mut ctx = Context {
+            target: self.target,
+            ..Default::default()
+        };
+
+        rescope_to_queue::<Domain, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.domains(), |entity| {
+            for rule in rules.domains() {
+                if rule.matches(entity.value.as_str())? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Domain)?;
+        rescope_to_queue::<Subdomain, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.subdomains(), |entity| {
+            for rule in rules.domains() {
+                if rule.matches(entity.value.as_str())? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Subdomain)?;
+
+        rescope_to_queue::<IpAddr, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.ipaddrs(), |entity| {
+            for rule in rules.ips() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::IpAddr)?;
+
+        rescope_to_queue::<Url, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.urls(), |entity| {
+            for rule in rules.domains() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            for rule in rules.urls() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Url)?;
+        rescope_to_queue::<Port, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.ports(), |entity| {
+            for rule in rules.ips() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Port)?;
+        rescope_to_queue::<Netblock, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.netblocks(), |entity| {
+            for rule in rules.ips() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Netblock)?;
+
+        if ctx.update_queue.is_empty() {
+            term::success("Nothing has changed, not updating database");
+        } else {
+            let confirm = if self.dry_run {
+                false
+            } else if self.auto_confirm {
+                true
+            } else {
+                utils::no_else_yes(&format!("Apply {} changes to scope now?", ctx.update_queue.len()))?
+            };
+
+            if confirm {
+                term::info(&format!("Applying {} changes to database", ctx.update_queue.len()));
+
+                for (update, value) in ctx.update_queue {
+                    update.set_scoped(rl.db(), value)?;
+                }
+            } else {
+                term::info("Database not updated");
+            }
+        }
+
+        Ok(())
+    }
+}

src/cmd/run_cmd.rs

@@ -17,6 +17,7 @@ use serde::Serialize;
 use sn0int_common::metadata::Source;
 use sn0int_std::ratelimits::Ratelimiter;
 use std::collections::HashMap;
+use std::net::SocketAddr;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 
@@ -24,14 +25,26 @@ use structopt::clap::AppSettings;
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
-    #[structopt(short="j", long="threads", default_value="1")]
-    threads: usize,
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    verbose: u64,
+    /// Execute a module that has been installed
+    pub module: Option<String>,
+    /// Run investigations concurrently
+    #[structopt(short="j", default_value="1")]
+    pub threads: usize,
+    /// Verbose logging, once to print inserts even if they don't add new
+    /// data, twice to activate the debug() function
+    #[structopt(short="v", long, parse(from_occurrences))]
+    pub verbose: u64,
+    /// Set a specific socks5 proxy to use
+    #[structopt(short="X", long)]
+    pub proxy: Option<SocketAddr>,
+    /// Set a different default user agent
+    #[structopt(long)]
+    pub user_agent: Option<String>,
 }
 
 #[derive(Debug, Clone)]
 pub struct Params<'a> {
+    pub module: Option<&'a String>,
     pub threads: usize,
     pub verbose: u64,
     pub stdin: bool,
@@ -39,25 +52,61 @@ pub struct Params<'a> {
     pub grant_full_keyring: bool,
     pub deny_keyring: bool,
     pub exit_on_error: bool,
+    pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<&'a String>,
+}
+
+impl<'a> Params<'a> {
+    pub fn get_module(&self, rl: &Shell) -> Result<Module> {
+        let module = if let Some(module) = self.module {
+            rl.library().get(module)?
+                .clone()
+        } else {
+            rl.module()
+                .map(|m| m.to_owned())
+                .ok_or_else(|| format_err!("No module selected"))?
+        };
+        Ok(module)
+    }
+
+    pub fn get_proxy(&self, rl: &Shell) -> Option<SocketAddr> {
+        if self.proxy.is_some() {
+            self.proxy
+        } else {
+            rl.config().network.proxy
+        }
+    }
+
+    pub fn get_user_agent(&self, rl: &Shell) -> Option<String> {
+        if let Some(user_agent) = self.user_agent {
+            Some(user_agent.to_string())
+        } else {
+            rl.config().network.user_agent.clone()
+        }
+    }
 }
 
 impl<'a> From<&'a args::Run> for Params<'a> {
     fn from(args: &args::Run) -> Params {
         Params {
-            threads: args.threads,
-            verbose: args.verbose,
+            module: args.run.module.as_ref(),
+            threads: args.run.threads,
+            verbose: args.run.verbose,
             stdin: args.stdin,
             grants: &args.grants,
             grant_full_keyring: args.grant_full_keyring,
             deny_keyring: args.deny_keyring,
             exit_on_error: args.exit_on_error,
+            proxy: args.run.proxy,
+            user_agent: args.run.user_agent.as_ref(),
         }
     }
 }
 
-impl From<Args> for Params<'static> {
-    fn from(args: Args) -> Params<'static> {
+impl<'a> From<&'a Args> for Params<'a> {
+    fn from(args: &Args) -> Params {
         Params {
+            module: args.module.as_ref(),
             threads: args.threads,
             verbose: args.verbose,
             stdin: false,
@@ -65,6 +114,8 @@ impl From<Args> for Params<'static> {
             grant_full_keyring: false,
             deny_keyring: false,
             exit_on_error: false,
+            proxy: args.proxy,
+            user_agent: args.user_agent.as_ref(),
         }
     }
 }
@@ -93,7 +144,7 @@ fn prepare_args<T: Scopable + Serialize + Model>(rl: &Shell, filter: &Filter, pa
 }
 
 pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
-    for namespace in keyring.unauthorized_namespaces(&module) {
+    for namespace in keyring.unauthorized_namespaces(module) {
         let grant_access = if params.deny_keyring {
             false
         } else if params.grant_full_keyring || params.grants.contains(namespace) {
@@ -104,7 +155,7 @@ pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params)
         };
 
         if grant_access {
-            keyring.grant_access(&module, namespace.to_string());
+            keyring.grant_access(module, namespace.to_string());
             term::info(&format!("Granted access to {:?}", namespace));
         }
     }
@@ -136,8 +187,8 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
         Some(Source::Notifications) => bail!("Notification modules can't be executed like this"),
         Some(Source::KeyRing(namespace)) => {
             let keyring = rl.keyring();
-            if keyring.is_access_granted(&module, &namespace) {
-                keyring.get_all_for(&namespace).into_iter()
+            if keyring.is_access_granted(module, namespace) {
+                keyring.get_all_for(namespace).into_iter()
                     .map(|key| {
                         let pretty = format!("{}:{}", key.namespace, key.access_key);
                         let arg = serde_json::to_value(key)?;
@@ -153,15 +204,14 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
 }
 
 pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);
 
     prepare_keyring(rl.keyring_mut(), &module, &params)?;
     let keyring = rl.keyring().request_keys(&module);
 
     let dns_config = Resolver::from_system_v4()?;
-    let proxy = rl.config().network.proxy;
 
     let args = get_args(rl, &module)?;
     for (arg, _pretty_arg, blobs) in args {
@@ -169,6 +219,7 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
                                           keyring.clone(),
                                           dns_config.clone(),
                                           proxy,
+                                          user_agent.clone(),
                                           options.clone(),
                                           module.clone(),
                                           arg,
@@ -181,15 +232,15 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
 }
 
 pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);
 
     prepare_keyring(rl.keyring_mut(), &module, &params)?;
     let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
-    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, rl.config().network.proxy, options);
+    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, proxy, user_agent, options);
     rl.signal_register().reset_ctrlc();
 
     if errors > 0 {
@@ -212,6 +263,6 @@ impl Cmd for Args {
             Some(options) => options.clone(),
             _ => HashMap::new(),
         };
-        execute(rl, self.into(), options)
+        execute(rl, Params::from(&self), options)
     }
 }

src/cmd/select_cmd.rs

@@ -106,20 +106,20 @@ impl Cmd for Args {
         let printer = Printer::new(rl, &self);
 
         match &self.subcommand {
-            Target::Domains(filter) => printer.select::<Domain>(&filter),
-            Target::Subdomains(filter) => printer.select::<Subdomain>(&filter),
-            Target::IpAddrs(filter) => printer.select::<IpAddr>(&filter),
-            Target::Urls(filter) => printer.select::<Url>(&filter),
-            Target::Emails(filter) => printer.select::<Email>(&filter),
-            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(&filter),
-            Target::Devices(filter) => printer.select::<Device>(&filter),
-            Target::Networks(filter) => printer.select::<Network>(&filter),
-            Target::Accounts(filter) => printer.select::<Account>(&filter),
-            Target::Breaches(filter) => printer.select::<Breach>(&filter),
-            Target::Images(filter) => printer.select::<Image>(&filter),
-            Target::Ports(filter) => printer.select::<Port>(&filter),
-            Target::Netblocks(filter) => printer.select::<Netblock>(&filter),
-            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(&filter),
+            Target::Domains(filter) => printer.select::<Domain>(filter),
+            Target::Subdomains(filter) => printer.select::<Subdomain>(filter),
+            Target::IpAddrs(filter) => printer.select::<IpAddr>(filter),
+            Target::Urls(filter) => printer.select::<Url>(filter),
+            Target::Emails(filter) => printer.select::<Email>(filter),
+            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(filter),
+            Target::Devices(filter) => printer.select::<Device>(filter),
+            Target::Networks(filter) => printer.select::<Network>(filter),
+            Target::Accounts(filter) => printer.select::<Account>(filter),
+            Target::Breaches(filter) => printer.select::<Breach>(filter),
+            Target::Images(filter) => printer.select::<Image>(filter),
+            Target::Ports(filter) => printer.select::<Port>(filter),
+            Target::Netblocks(filter) => printer.select::<Netblock>(filter),
+            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(filter),
         }
     }
 }

src/cmd/stats_cmd.rs

@@ -89,21 +89,21 @@ impl Stats {
     fn count(workspace: String, db: &Database) -> Result<Stats> {
         Ok(Stats {
             workspace,
-            domains: count_models::<Domain>(&db)?,
-            subdomains: count_models::<Subdomain>(&db)?,
-            ipaddrs: count_models::<IpAddr>(&db)?,
-            urls: count_models::<Url>(&db)?,
-            emails: count_models::<Email>(&db)?,
-            phonenumbers: count_models::<PhoneNumber>(&db)?,
-            devices: count_models::<Device>(&db)?,
-            networks: count_models::<Network>(&db)?,
-            accounts: count_models::<Account>(&db)?,
-            breaches: count_models::<Breach>(&db)?,
-            images: count_models::<Image>(&db)?,
-            ports: count_models::<Port>(&db)?,
-            netblocks: count_models::<Netblock>(&db)?,
-            cryptoaddrs: count_models::<CryptoAddr>(&db)?,
-            activity: Activity::count(&db)?,
+            domains: count_models::<Domain>(db)?,
+            subdomains: count_models::<Subdomain>(db)?,
+            ipaddrs: count_models::<IpAddr>(db)?,
+            urls: count_models::<Url>(db)?,
+            emails: count_models::<Email>(db)?,
+            phonenumbers: count_models::<PhoneNumber>(db)?,
+            devices: count_models::<Device>(db)?,
+            networks: count_models::<Network>(db)?,
+            accounts: count_models::<Account>(db)?,
+            breaches: count_models::<Breach>(db)?,
+            images: count_models::<Image>(db)?,
+            ports: count_models::<Port>(db)?,
+            netblocks: count_models::<Netblock>(db)?,
+            cryptoaddrs: count_models::<CryptoAddr>(db)?,
+            activity: Activity::count(db)?,
             blobs: None,
         })
     }
@@ -162,7 +162,7 @@ impl Cmd for Args {
                 println!("{:>41}", workspace.bold());
             }
 
-            let mut stats = Stats::count(workspace.into(), &db)?;
+            let mut stats = Stats::count(workspace.into(), db)?;
             if !self.short {
                 stats.add_blob_usage(rl.blobs())?;
             }

src/cmd/target_cmd.rs

@@ -81,7 +81,7 @@ fn count_selected(rl: &mut Shell, source: &Source) -> Result<usize> {
         Source::Netblocks => db.filter::<Netblock>(&filter)?.len(),
         Source::CryptoAddrs(currency) => db.filter_with_param::<CryptoAddr>(&filter, currency.as_ref())?.len(),
         Source::Notifications => bail!("Notifications can't be set as target"),
-        Source::KeyRing(namespace) => rl.keyring().list_for(&namespace).len(),
+        Source::KeyRing(namespace) => rl.keyring().list_for(namespace).len(),
     };
     Ok(num)
 }

src/config.rs

@@ -21,7 +21,7 @@ pub struct Config {
 impl Config {
     pub fn load() -> Result<Config> {
         let path = Config::path()?;
-        Config::load_from(&path)
+        Config::load_from(path)
     }
 
     pub fn path() -> Result<PathBuf> {
@@ -75,4 +75,5 @@ fn default_registry() -> String {
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct NetworkConfig {
     pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
 }

src/db/mod.rs

@@ -5,6 +5,7 @@ use diesel::expression::SqlLiteral;
 use diesel::expression::sql_literal::sql;
 use diesel::sql_types::Bool;
 use diesel::prelude::*;
+use std::fmt::Write;
 use strum_macros::{EnumString, IntoStaticStr};
 use crate::autonoscope::{RuleSet, RuleType};
 use crate::models::*;
@@ -164,6 +165,11 @@ impl Database {
         self.autonoscope.rules()
     }
 
+    #[inline(always)]
+    pub fn autonoscope(&self) -> &RuleSet {
+        &self.autonoscope
+    }
+
     /// Returns true if we didn't have this value yet
     pub fn insert_generic(&self, object: Insert) -> Result<Option<(DbChange, i32)>> {
         let scoped = self.autonoscope.matches(&object)?;
@@ -198,14 +204,14 @@ impl Database {
 
             let update = obj.upsert(&existing);
             if update.is_dirty() {
-                update.apply(&self)?;
+                update.apply(self)?;
                 Ok(Some((DbChange::Update(update.generic()), existing.id())))
             } else {
                 Ok(Some((DbChange::None, existing.id())))
             }
         } else {
             obj.set_scoped(scoped);
-            obj.insert(&self)?;
+            obj.insert(self)?;
             let id = M::get_id(self, obj.value())?;
             Ok(Some((DbChange::Insert, id)))
         }
@@ -218,7 +224,7 @@ impl Database {
                 return Ok(false);
             }
         }
-        obj.insert(&self)?;
+        obj.insert(self)?;
         Ok(true)
     }
 
@@ -254,7 +260,7 @@ impl Database {
 
             let update = obj.upsert(&existing);
             if update.is_dirty() {
-                update.apply(&self)?;
+                update.apply(self)?;
                 Ok(Some((DbChange::Update(update.generic()), id)))
             } else {
                 Ok(Some((DbChange::None, id)))
@@ -402,7 +408,7 @@ impl Database {
     }
 
     fn get_opt_typed<T: Model + Scopable>(&self, value: &T::ID) -> Result<Option<i32>> {
-        match T::get_opt(self, &value)? {
+        match T::get_opt(self, value)? {
             Some(ref obj) if obj.scoped() => Ok(Some(obj.id())),
             _ => Ok(None),
         }
@@ -410,23 +416,23 @@ impl Database {
 
     pub fn get_opt(&self, family: &Family, value: &str) -> Result<Option<i32>> {
         match family {
-            Family::Domain => self.get_opt_typed::<Domain>(&value),
-            Family::Subdomain => self.get_opt_typed::<Subdomain>(&value),
-            Family::Ipaddr => self.get_opt_typed::<IpAddr>(&value),
+            Family::Domain => self.get_opt_typed::<Domain>(value),
+            Family::Subdomain => self.get_opt_typed::<Subdomain>(value),
+            Family::Ipaddr => self.get_opt_typed::<IpAddr>(value),
             Family::SubdomainIpaddr => bail!("Unsupported operation"),
-            Family::Url => self.get_opt_typed::<Url>(&value),
-            Family::Email => self.get_opt_typed::<Email>(&value),
-            Family::Phonenumber => self.get_opt_typed::<PhoneNumber>(&value),
-            Family::Device => self.get_opt_typed::<Device>(&value),
-            Family::Network => self.get_opt_typed::<Network>(&value),
+            Family::Url => self.get_opt_typed::<Url>(value),
+            Family::Email => self.get_opt_typed::<Email>(value),
+            Family::Phonenumber => self.get_opt_typed::<PhoneNumber>(value),
+            Family::Device => self.get_opt_typed::<Device>(value),
+            Family::Network => self.get_opt_typed::<Network>(value),
             Family::NetworkDevice => bail!("Unsupported operation"),
-            Family::Account => self.get_opt_typed::<Account>(&value),
-            Family::Breach => self.get_opt_typed::<Breach>(&value),
+            Family::Account => self.get_opt_typed::<Account>(value),
+            Family::Breach => self.get_opt_typed::<Breach>(value),
             Family::BreachEmail => bail!("Unsupported operation"),
-            Family::Image => self.get_opt_typed::<Image>(&value),
-            Family::Port => self.get_opt_typed::<Port>(&value),
-            Family::Netblock => self.get_opt_typed::<Netblock>(&value),
-            Family::Cryptoaddr => self.get_opt_typed::<CryptoAddr>(&value),
+            Family::Image => self.get_opt_typed::<Image>(value),
+            Family::Port => self.get_opt_typed::<Port>(value),
+            Family::Netblock => self.get_opt_typed::<Netblock>(value),
+            Family::Cryptoaddr => self.get_opt_typed::<CryptoAddr>(value),
         }
     }
 
@@ -460,6 +466,11 @@ impl Filter {
         }
     }
 
+    #[inline]
+    pub fn any() -> Filter {
+        Filter::new("1")
+    }
+
     fn escape(value: &str) -> String {
         let mut out = String::from("'");
         for c in value.chars() {
@@ -492,14 +503,14 @@ impl Filter {
         for arg in args {
             if ["=", "!=", "<", ">", "<=", ">=", "like"].contains(&arg.to_lowercase().as_str()) {
                 expect_value = true;
-                query += &format!(" {}", arg);
+                write!(query, " {}", arg)?;
                 continue;
             }
 
             if let Some(idx) = arg.find('=') {
                 if idx != 0 {
                     let (key, value) = arg.split_at(idx);
-                    query += &format!(" {} = {}", key, Self::escape(&value[1..]));
+                    write!(query, " {} = {}", key, Self::escape(&value[1..]))?;
                     continue;
                 }
             }
@@ -509,7 +520,7 @@ impl Filter {
                 query.push_str(&Self::escape(arg));
                 expect_value = false;
             } else {
-                query += &format!(" {}", arg);
+                write!(query, " {}", arg)?;
             }
         }
         debug!("Parsed query: {:?}", query);
@@ -522,7 +533,7 @@ impl Filter {
 
         if args.is_empty() {
             debug!("Using filter with no condition");
-            return Ok(Filter::new("1"));
+            return Ok(Filter::any());
         }
 
         Self::parse(args)

src/db/ttl.rs

@@ -135,7 +135,7 @@ pub fn reap_expired(rl: &mut Shell) -> Result<()> {
 
         let subject = format!("Deleted {} {:?}", &expired.family, &expired.value);
         let topic = &format!("db:{}:{}:delete", &expired.family, &expired.value);
-        if let Err(err) = notify::trigger_notify_event(rl, &mut Term, &mut ratelimit, &topic, &Notification {
+        if let Err(err) = notify::trigger_notify_event(rl, &mut Term, &mut ratelimit, topic, &Notification {
             subject,
             body: None,
         }) {

src/engine/ctx.rs

@@ -215,6 +215,7 @@ pub struct LuaState {
     geoip: Option<Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>>,
     asn: Option<Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>>,
     proxy: Option<SocketAddr>,
+    user_agent: Option<String>,
     options: HashMap<String, String>,
 }
 
@@ -299,7 +300,7 @@ impl State for LuaState {
         let id = self.random_id();
 
         let sock = if let Some(proxy) = self.resolve_proxy_options(&options.proxy)? {
-            Socket::connect_socks5(proxy, host, port, options)?
+            Socket::connect_socks5(*proxy, host, port, options)?
         } else {
             Socket::connect(&self.dns_config, host, port, options)?
         };
@@ -372,7 +373,15 @@ impl State for LuaState {
         let mtx = self.http_sessions.lock().unwrap();
         let session = mtx.get(session_id).expect("Invalid session reference"); // TODO
 
-        HttpRequest::new(&session, method, url, options, || format!("sn0int/{}", env!("CARGO_PKG_VERSION")))
+        let user_agent = if let Some(user_agent) = &options.user_agent {
+            user_agent.to_string()
+        } else if let Some(user_agent) = &self.user_agent {
+            user_agent.to_string()
+        } else {
+            format!("sn0int/{}", env!("CARGO_PKG_VERSION"))
+        };
+
+        HttpRequest::new(session, method, url, user_agent, options)
     }
 
     fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
@@ -472,6 +481,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
         geoip,
         asn,
         proxy: env.proxy,
+        user_agent: env.user_agent,
         options: env.options,
     });
 
@@ -517,6 +527,9 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
     runtime::http_fetch_json(&mut lua, state.clone());
     runtime::img_exif(&mut lua, state.clone());
     runtime::img_load(&mut lua, state.clone());
+    runtime::img_ahash(&mut lua, state.clone());
+    runtime::img_dhash(&mut lua, state.clone());
+    runtime::img_phash(&mut lua, state.clone());
     runtime::img_nudity(&mut lua, state.clone());
     runtime::info(&mut lua, state.clone());
     runtime::intval(&mut lua, state.clone());
@@ -662,6 +675,7 @@ impl Script {
         let keyring = Vec::new();
         let dns_config = Resolver::from_system_v4()?;
         let proxy = None;
+        let user_agent = None;
         let psl = PslReader::String(r#"
 // ===BEGIN ICANN DOMAINS===
 com
@@ -679,6 +693,7 @@ a.prod.fastly.net
             keyring,
             dns_config,
             proxy,
+            user_agent,
             options: HashMap::new(),
             blobs: Vec::new(),
             psl,

src/engine/mod.rs

@@ -34,6 +34,7 @@ pub struct Environment {
     pub keyring: Vec<KeyRingEntry>,
     pub dns_config: Resolver,
     pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
     pub options: HashMap<String, String>,
     pub blobs: Vec<Blob>,
     pub psl: PslReader,
@@ -76,7 +77,7 @@ impl<'a> Library<'a> {
     }
 
     pub fn private_modules(path: &Path) -> Result<bool> {
-        let metadata = fs::symlink_metadata(&path)?.file_type();
+        let metadata = fs::symlink_metadata(path)?.file_type();
         if metadata.is_symlink() {
             debug!("Folder is a symlink, flagging modules as private");
             return Ok(true);
@@ -120,14 +121,14 @@ impl<'a> Library<'a> {
                 folder.to_owned()
             } else {
                 let folder = folder.strip_prefix("~/")
-                    .unwrap_or(&folder);
+                    .unwrap_or(folder);
 
                 dirs_next::home_dir()
                     .ok_or_else(|| format_err!("Failed to find home folder"))?
                     .join(folder)
             };
 
-            self.load_module_folder(&folder, &author, true)?;
+            self.load_module_folder(&folder, author, true)?;
         }
 
         Ok(())
@@ -156,7 +157,7 @@ impl<'a> Library<'a> {
                 continue;
             }
 
-            if let Err(err) = self.load_single_module(&module.path(), &author_name, &module_name, private_modules) {
+            if let Err(err) = self.load_single_module(&module.path(), author_name, module_name, private_modules) {
                 let root = err.find_root_cause();
                 term::warn(&format!("Failed to load {}/{}: {}", author_name, module_name, root));
             }
@@ -167,7 +168,7 @@ impl<'a> Library<'a> {
 
     pub fn load_single_module(&mut self, path: &Path, author_name: &str, module_name: &str, private_module: bool) -> Result<()> {
         let module_name = module_name.to_string();
-        let module = Module::load(path, &author_name, &module_name, private_module)
+        let module = Module::load(path, author_name, &module_name, private_module)
             .context(format!("Failed to parse {}/{}", author_name, module_name))?;
 
         for key in &[&module_name, &format!("{}/{}", author_name, module_name)] {
@@ -332,7 +333,7 @@ impl Module {
     pub fn source_equals(&self, other: &str) -> bool {
         match self.source() {
             Some(source) => source.group_as_str() == other,
-            None => other == "",
+            None => other.is_empty(),
         }
     }
 }

src/filters.rs

@@ -50,12 +50,67 @@ pub enum Target {
     CryptoAddrs(Filter),
 }
 
+impl Target {
+    pub fn domains(&self) -> Option<&Filter> {
+        if let Target::Domains(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn subdomains(&self) -> Option<&Filter> {
+        if let Target::Subdomains(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn ipaddrs(&self) -> Option<&Filter> {
+        if let Target::IpAddrs(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn urls(&self) -> Option<&Filter> {
+        if let Target::Urls(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn ports(&self) -> Option<&Filter> {
+        if let Target::Ports(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn netblocks(&self) -> Option<&Filter> {
+        if let Target::Netblocks(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct Filter {
     args: Vec<String>,
 }
 
 impl Filter {
+    #[inline]
+    pub fn any() -> db::Filter {
+        db::Filter::any()
+    }
+
     pub fn parse_optional(&self) -> Result<db::Filter> {
         db::Filter::parse_optional(&self.args)
     }

src/ipc/child.rs

@@ -86,6 +86,7 @@ pub fn run(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslRea
         keyring: start.keyring,
         dns_config: start.dns_config,
         proxy: start.proxy,
+        user_agent: start.user_agent,
         options: start.options,
         blobs: start.blobs,
         psl,

src/ipc/common.rs

@@ -12,6 +12,7 @@ pub struct StartCommand {
     pub keyring: Vec<KeyRingEntry>,
     pub dns_config: Resolver,
     pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
     pub options: HashMap<String, String>,
     pub module: Module,
     pub arg: serde_json::Value,
@@ -23,6 +24,7 @@ impl StartCommand {
                keyring: Vec<KeyRingEntry>,
                dns_config: Resolver,
                proxy: Option<SocketAddr>,
+               user_agent: Option<String>,
                options: HashMap<String, String>,
                module: Module,
                arg: serde_json::Value,
@@ -33,6 +35,7 @@ impl StartCommand {
             keyring,
             dns_config,
             proxy,
+            user_agent,
             options,
             module,
             arg,

src/ipc/parent.rs

@@ -47,7 +47,7 @@ impl IpcParent {
     }
 
     pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
-        let start = serde_json::to_value(&start)?;
+        let start = serde_json::to_value(start)?;
         self.send(&start)?;
         Ok(())
     }
@@ -71,6 +71,10 @@ impl IpcParent {
         let mut line = String::new();
         let len = self.stdout.read_line(&mut line)?;
 
+        if len == 0 {
+            bail!("Sandbox child has crashed");
+        }
+
         let event = serde_json::from_str(&line[..len])?;
         debug!("IpcParent received: {:?}", event);
         Ok(event)
@@ -105,6 +109,7 @@ pub fn run(module: Module,
            verbose: u64,
            has_stdin: bool,
            proxy: Option<SocketAddr>,
+           user_agent: Option<String>,
            options: HashMap<String, String>,
            blobs: Vec<Blob>,
 ) -> Result<ExitEvent> {
@@ -117,15 +122,15 @@ pub fn run(module: Module,
     };
 
     let mut ipc_parent = IpcParent::setup(&module)?;
-    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
+    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, user_agent, options, module, arg, blobs))?;
 
     let exit = loop {
         match ipc_parent.recv()? {
             Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => ipc_parent.send_event_callback(*object, &tx),
+            Event::Database(object) => ipc_parent.send_event_callback(*object, tx),
             Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
-            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
-            Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),
+            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, tx),
+            Event::Blob(blob) => ipc_parent.send_event_callback(blob, tx),
             Event::Exit(event) => {
                 if let ExitEvent::Err(err) = &event {
                     tx.send(Event2::Log(LogEvent::Error(err.clone())));

src/keyring.rs

@@ -94,7 +94,7 @@ impl KeyRing {
     }
 
     pub fn load(path: &Path) -> Result<KeyRing> {
-        let buf = fs::read(&path)
+        let buf = fs::read(path)
             .context("Failed to read keyring file")?;
         serde_json::from_slice(&buf)
             .map_err(Error::from)
@@ -103,7 +103,7 @@ impl KeyRing {
     pub fn save(&self) -> Result<()> {
         let path = Self::path()?;
         let buf = serde_json::to_string(&self)?;
-        fs::write(&path, buf)
+        fs::write(path, buf)
             .context("Failed to save keyring")?;
         Ok(())
     }
@@ -111,7 +111,7 @@ impl KeyRing {
     pub fn insert(&mut self, key: KeyName, secret: Option<String>) -> Result<()> {
         // get the namespace or create a new one
         let mut x = self.keys.remove(&key.namespace)
-            .unwrap_or_else(HashMap::new);
+            .unwrap_or_default();
         // insert key into namespace
         x.insert(key.name, secret);
         // add namespace backinto keyring
@@ -170,13 +170,13 @@ impl KeyRing {
 
     pub fn unauthorized_namespaces<'a>(&self, module: &'a Module) -> Vec<&'a String> {
         module.keyring_access().iter()
-            .filter(|namespace| !self.is_access_granted(&module, &namespace))
+            .filter(|namespace| !self.is_access_granted(module, namespace))
             .collect()
     }
 
     pub fn grant_access(&mut self, module: &Module, namespace: String) {
         let mut grants = self.grants.remove(&namespace)
-            .unwrap_or_else(HashSet::new);
+            .unwrap_or_default();
         grants.insert(module.id());
         self.grants.insert(namespace, grants);
     }
@@ -192,7 +192,7 @@ impl KeyRing {
     pub fn request_keys(&self, module: &Module) -> Vec<KeyRingEntry> {
         // TODO: we probably want to randomize the order
         module.keyring_access().iter()
-            .filter(|namespace| self.is_access_granted(&module, &namespace))
+            .filter(|namespace| self.is_access_granted(module, namespace))
             .flat_map(|namespace| self.list_for(namespace))
             .flat_map(|x| self.get(&x))
             .collect()
@@ -208,7 +208,7 @@ pub struct KeyRingEntry {
 
 impl KeyRingEntry {
     pub fn to_lua(&self) -> Result<AnyLuaValue> {
-        let v = serde_json::to_value(&self)?;
+        let v = serde_json::to_value(self)?;
         let v = LuaJsonValue::from(v).into();
         Ok(v)
     }

src/lib.rs

@@ -1,6 +1,8 @@
 #![allow(clippy::too_many_arguments)]
 #![allow(clippy::type_complexity)]
 #![allow(clippy::large_enum_variant)]
+// because of diesel
+#![allow(clippy::extra_unused_lifetimes)]
 
 #![warn(unused_extern_crates)]
 use hlua_badtouch as hlua;

src/main.rs

@@ -2,6 +2,7 @@ use env_logger::Env;
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
 use sn0int::cmd::{self, LiteCmd};
+use sn0int::cmd::run_cmd::Params;
 use sn0int::config::Config;
 use sn0int::db;
 use sn0int::errors::*;
@@ -23,34 +24,41 @@ use std::path::Path;
 fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
     let mut rl = shell::init(gargs, config, false)?;
 
+    let module = args.run.module.as_ref()
+        .ok_or_else(|| format_err!("Module is required"))?;
+
     let module = if args.file {
-        let path = Path::new(&args.module);
+        let path = Path::new(&module);
 
         let filename = path.file_stem()
             .ok_or_else(|| format_err!("Failed to decode filename"))?
             .to_str()
             .ok_or_else(|| format_err!("Failed to decode filename"))?;
 
-        Module::load(&path.to_path_buf(), "anonymous", &filename, true)
+        Module::load(path, "anonymous", filename, true)
             .context(format!("Failed to parse {:?}", path))?
     } else {
-        rl.library().get(&args.module)?
+        rl.library().get(module)?
             .clone()
     };
 
     rl.set_module(module);
 
     if let Some(target) = &args.target {
-        let target = shellwords::split(&target)
+        let target = shellwords::split(target)
             .map_err(|_| format_err!("Failed to parse target quotes"))?;
         let target = db::Filter::parse(&target)?;
         rl.set_target(Some(target));
     }
 
+    let mut params = Params::from(args);
+    // The module was already set and loaded
+    params.module = None;
+
     if args.dump_sandbox_init_msg {
-        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, args.into(), Opt::collect(&args.options))
+        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, params, Opt::collect(&args.options))
     } else {
-        cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+        cmd::run_cmd::execute(&mut rl, params, Opt::collect(&args.options))
     }
 }
 
@@ -123,6 +131,9 @@ fn run() -> Result<()> {
         Some(SubCommand::Activity(activity)) => run_cmd(&args, activity, &config),
         Some(SubCommand::Scope(scope)) => run_cmd(&args, scope, &config),
         Some(SubCommand::Noscope(noscope)) => run_cmd(&args, noscope, &config),
+        Some(SubCommand::Autoscope(autoscope)) => run_cmd(&args, autoscope, &config),
+        Some(SubCommand::Autonoscope(autonoscope)) => run_cmd(&args, autonoscope, &config),
+        Some(SubCommand::Rescope(rescope)) => run_cmd(&args, rescope, &config),
         Some(SubCommand::Workspace(workspace)) => workspace.run(&config),
         Some(SubCommand::Fsck(fsck)) => run_cmd(&args, fsck, &config),
         Some(SubCommand::Export(export)) => run_cmd(&args, export, &config),

src/models/account.rs

@@ -115,6 +115,14 @@ impl Scopable for Account {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::accounts::dsl::*;
+        diesel::update(accounts.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::accounts::dsl::*;
 
@@ -342,7 +350,7 @@ impl Upsert for AccountUpdate {
     }
 
     fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_account(&self)
+        db.update_account(self)
     }
 }
 

src/models/breach.rs

@@ -94,6 +94,14 @@ impl Scopable for Breach {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::breaches::dsl::*;
+        diesel::update(breaches.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::breaches::dsl::*;
 

src/models/cryptoaddr.rs

@@ -113,6 +113,14 @@ impl Scopable for CryptoAddr {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::cryptoaddrs::dsl::*;
+        diesel::update(cryptoaddrs.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::cryptoaddrs::dsl::*;
 
@@ -326,7 +334,7 @@ impl Upsert for CryptoAddrUpdate {
     }
 
     fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_cryptoaddr(&self)
+        db.update_cryptoaddr(self)
     }
 }
 

src/models/device.rs

@@ -99,6 +99,14 @@ impl Scopable for Device {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::devices::dsl::*;
+        diesel::update(devices.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::devices::dsl::*;
 

src/models/domain.rs

@@ -94,6 +94,14 @@ impl Scopable for Domain {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::domains::dsl::*;
+        diesel::update(domains.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::domains::dsl::*;
 

src/models/email.rs

@@ -96,6 +96,14 @@ impl Scopable for Email {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::emails::dsl::*;
+        diesel::update(emails.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::emails::dsl::*;
 

src/models/image.rs

@@ -116,6 +116,14 @@ impl Scopable for Image {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::images::dsl::*;
+        diesel::update(images.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::images::dsl::*;
 
@@ -442,7 +450,7 @@ impl Upsert for ImageUpdate {
     }
 
     fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_image(&self)
+        db.update_image(self)
     }
 }
 

src/models/ipaddr.rs

@@ -109,6 +109,14 @@ impl Scopable for IpAddr {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::ipaddrs::dsl::*;
+        diesel::update(ipaddrs.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::ipaddrs::dsl::*;
 

src/models/mod.rs

@@ -243,6 +243,8 @@ pub trait Model: Sized {
 pub trait Scopable: Model {
     fn scoped(&self) -> bool;
 
+    fn set_scoped(&self, _db: &Database, _value: bool) -> Result<()>;
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize>;
 
     fn noscope(db: &Database, filter: &Filter) -> Result<usize>;

src/models/netblock.rs

@@ -98,6 +98,14 @@ impl Scopable for Netblock {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::netblocks::dsl::*;
+        diesel::update(netblocks.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::netblocks::dsl::*;
 

src/models/network.rs

@@ -97,6 +97,14 @@ impl Scopable for Network {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::networks::dsl::*;
+        diesel::update(networks.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::networks::dsl::*;
 

src/models/phonenumber.rs

@@ -105,6 +105,14 @@ impl Scopable for PhoneNumber {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::phonenumbers::dsl::*;
+        diesel::update(phonenumbers.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::phonenumbers::dsl::*;
 

src/models/port.rs

@@ -106,6 +106,14 @@ impl Scopable for Port {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::ports::dsl::*;
+        diesel::update(ports.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::ports::dsl::*;
 
@@ -324,7 +332,7 @@ impl Upsert for PortUpdate {
     }
 
     fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_port(&self)
+        db.update_port(self)
     }
 }
 

src/models/subdomain.rs

@@ -98,6 +98,14 @@ impl Scopable for Subdomain {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::subdomains::dsl::*;
+        diesel::update(subdomains.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::subdomains::dsl::*;
 

src/models/url.rs

@@ -104,6 +104,14 @@ impl Scopable for Url {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::urls::dsl::*;
+        diesel::update(urls.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::urls::dsl::*;
 
@@ -339,7 +347,7 @@ impl Upsert for UrlChangeset {
     }
 
     fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_url(&self)
+        db.update_url(self)
     }
 }
 

src/notify/mod.rs

@@ -88,6 +88,7 @@ pub fn exec(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, option
     }
 
     let params = Params {
+        module: None,
         threads: 1,
         verbose,
         stdin: false,
@@ -95,13 +96,15 @@ pub fn exec(rl: &mut Shell, module: &Module, ratelimit: &mut Ratelimiter, option
         grant_full_keyring: false,
         deny_keyring: false,
         exit_on_error: false,
+        proxy: None,
+        user_agent: None,
     };
 
-    prepare_keyring(rl.keyring_mut(), &module, &params)?;
-    let args = vec![prepare_arg(&notification)?];
+    prepare_keyring(rl.keyring_mut(), module, &params)?;
+    let args = vec![prepare_arg(notification)?];
 
     debug!("Executing notification module {:?}", module_name);
-    let errors = worker::spawn(rl, &module, ratelimit, args, &params, rl.config().network.proxy, options);
+    let errors = worker::spawn(rl, module, ratelimit, args, &params, rl.config().network.proxy, None, options);
     debug!("Notification module {:?} exited with {:?} errors", module_name, errors);
 
     Ok(errors)

src/notify/rules.rs

@@ -18,7 +18,7 @@ impl Glob {
 
         loop {
             match (filter.next(), topic.next()) {
-                (Some(filter), Some(topic)) => if !filter.matches(&topic) {
+                (Some(filter), Some(topic)) => if !filter.matches(topic) {
                     return None;
                 },
                 (None, None) => return Some(!self.inverse),

src/paths.rs

@@ -9,21 +9,21 @@ pub fn sn0int_dir() -> Result<PathBuf> {
         .ok_or_else(|| format_err!("Failed to find data directory"))?;
     let path = path.join("sn0int");
     fs::create_dir_all(&path)
-        .context("Failed to create data directory")?;
+        .context("Failed to create sn0int data directory")?;
     Ok(path)
 }
 
 pub fn history_path() -> Result<PathBuf> {
-    let path = sn0int_dir()?;
-    let path = path.join("history");
+    let path = sn0int_dir()?
+        .join("history");
     Ok(path)
 }
 
 pub fn module_dir() -> Result<PathBuf> {
-    let path = sn0int_dir()?;
-    let path = path.join("modules");
+    let path = sn0int_dir()?
+        .join("modules");
     fs::create_dir_all(&path)
-        .context("Failed to create module directory")?;
+        .context("Failed to create modules directory")?;
     Ok(path)
 }
 
@@ -31,16 +31,15 @@ pub fn data_dir() -> Result<PathBuf> {
     let path = sn0int_dir()?
         .join("data");
     fs::create_dir_all(&path)
-        .context("Failed to create module directory")?;
+        .context("Failed to create data directory")?;
     Ok(path)
 }
 
 pub fn workspace_dir(workspace: &Workspace) -> Result<PathBuf> {
-    let path = sn0int_dir()?
-        .join("data")
+    let path = data_dir()?
         .join(workspace.as_str());
     fs::create_dir_all(&path)
-        .context("Failed to create module directory")?;
+        .context("Failed to create workspace directory")?;
     Ok(path)
 }
 
@@ -48,7 +47,7 @@ pub fn blobs_dir(workspace: &Workspace) -> Result<PathBuf> {
     let path = workspace_dir(workspace)?
         .join("blobs");
     fs::create_dir_all(&path)
-        .context("Failed to create module directory")?;
+        .context("Failed to create blobs directory")?;
     Ok(path)
 }
 
@@ -57,7 +56,7 @@ pub fn cache_dir() -> Result<PathBuf> {
         .ok_or_else(|| format_err!("Failed to find cache directory"))?;
     let path = path.join("sn0int");
     fs::create_dir_all(&path)
-        .context("Failed to create cache directory")?;
+        .context("Failed to create sn0int cache directory")?;
     Ok(path)
 }
 

src/registry.rs

@@ -23,7 +23,7 @@ pub struct Updater {
 
 impl Updater {
     pub fn new(config: &Config) -> Result<Updater> {
-        let client = Client::new(&config)?;
+        let client = Client::new(config)?;
         Ok(Updater {
             client,
         })
@@ -97,7 +97,7 @@ pub fn run_publish(_args: &Args, publish: &Publish, config: &Config) -> Result<(
     let session = auth::load_token()
         .context("Failed to load auth token, login first")?;
 
-    let mut client = Client::new(&config)?;
+    let mut client = Client::new(config)?;
     client.authenticate(session);
 
     for path in &publish.paths {
@@ -247,7 +247,7 @@ fn write_tag(out: &mut String, color: Color, txt: &str) -> Result<()> {
 }
 
 pub fn run_search(library: &Library, search: &Search, config: &Config) -> Result<()> {
-    let client = Client::new(&config)?;
+    let client = Client::new(config)?;
 
     let label = format!("Searching {:?}", search.query);
     let modules = worker::spawn_fn(&label, || {

src/repl/mod.rs

@@ -24,21 +24,19 @@ pub struct Repl<'a> {
 }
 
 impl<'a> Repl<'a> {
-    pub fn new(lua: Lua<'a>, state: Arc<LuaState>) -> Repl<'a> {
-        let rl = Readline::with(ReplCompleter::default());
-        Repl {
+    pub fn new(lua: Lua<'a>, state: Arc<LuaState>) -> Result<Repl<'a>> {
+        let rl = Readline::with(ReplCompleter::default())?;
+        Ok(Repl {
             rl,
             lua,
             state,
-        }
+        })
     }
 
     fn update_globals(&mut self) {
         let mut globals = Vec::new();
-        for item in self.lua.globals_table().iter::<String, AnyLuaValue>() {
-            if let Some((k, _)) = item {
-                globals.push(k);
-            }
+        for (k, _) in self.lua.globals_table().iter::<String, AnyLuaValue>().flatten() {
+            globals.push(k);
         }
         if let Some(helper) = self.rl.helper_mut() {
             debug!("updating globals: {:?}", globals);
@@ -65,7 +63,7 @@ impl<'a> Repl<'a> {
             Ok(val) => {
                 if val != AnyLuaValue::LuaNil {
                     let mut out = String::new();
-                    format_lua(&mut out, &val);
+                    format_lua(&mut out, &val).expect("out of memory");
                     println!("{}", out);
                 }
                 if let Some(err) = self.state.last_error() {
@@ -84,6 +82,7 @@ pub fn run(config: &Config) -> Result<()> {
     let keyring = Vec::new();
     let dns_config = Resolver::from_system_v4()?;
     let proxy = config.network.proxy;
+    let user_agent = config.network.user_agent.clone();
 
     let cache_dir = paths::cache_dir()?;
     let psl = PslReader::open(&cache_dir)?;
@@ -95,6 +94,7 @@ pub fn run(config: &Config) -> Result<()> {
         keyring,
         dns_config,
         proxy,
+        user_agent,
         options: HashMap::new(),
         blobs: Vec::new(),
         psl,
@@ -104,7 +104,7 @@ pub fn run(config: &Config) -> Result<()> {
 
     let tx = DummyIpcChild::create();
     let (lua, state) = ctx::ctx(env, tx);
-    let mut repl = Repl::new(lua, state);
+    let mut repl = Repl::new(lua, state)?;
 
     println!(r#":: sn0int v{} lua repl
 Assign variables with `a = sn0int_version()` and `return a` to print

src/runtime/db.rs

@@ -199,7 +199,7 @@ fn gen_changeset2<T: Model, U: UpdateToChangeset<C>, C: Updateable<T>>(object: L
 }
 
 fn run_update(state: Arc<dyn State>, family: &str, object: AnyLuaValue, update: AnyLuaValue) -> Result<Option<i32>> {
-    let family = Family::from_str(&family)?;
+    let family = Family::from_str(family)?;
     let object = LuaJsonValue::from(object);
     let update = LuaJsonValue::from(update);
 

src/runtime/encoding.rs

@@ -3,7 +3,7 @@ use crate::errors::*;
 use crate::engine::ctx::State;
 use crate::engine::structs::{byte_array, lua_bytes};
 use crate::hlua::{self, AnyLuaValue};
-use data_encoding::{Specification, Encoding};
+use data_encoding::{BASE64, Specification, Encoding};
 use std::sync::Arc;
 
 
@@ -19,7 +19,7 @@ fn spec(symbols: &str, padding: &str) -> Result<Encoding> {
 
 pub fn base64_decode(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("base64_decode", hlua::function1(move |bytes: String| -> Result<AnyLuaValue> {
-        base64::decode(&bytes)
+        BASE64.decode(bytes.as_bytes())
             .map_err(|err| state.set_error(err.into()))
             .map(|bytes| lua_bytes(&bytes))
     }))
@@ -29,7 +29,7 @@ pub fn base64_encode(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("base64_encode", hlua::function1(move |bytes: AnyLuaValue| -> Result<String> {
         byte_array(bytes)
             .map_err(|err| state.set_error(err))
-            .map(|bytes| base64::encode(&bytes))
+            .map(|bytes| BASE64.encode(&bytes))
     }))
 }
 

src/runtime/gfx.rs

@@ -36,6 +36,48 @@ pub fn img_load(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     }))
 }
 
+pub fn img_ahash(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("img_ahash", hlua::function1(move |blob: String| -> Result<String> {
+        let img = state.get_blob(&blob)
+            .map_err(|err| state.set_error(err))?;
+
+        let img = gfx::load(&img.bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        let hash = img.perception_hash(gfx::HashAlg::Mean);
+
+        Ok(hash)
+    }))
+}
+
+pub fn img_dhash(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("img_dhash", hlua::function1(move |blob: String| -> Result<String> {
+        let img = state.get_blob(&blob)
+            .map_err(|err| state.set_error(err))?;
+
+        let img = gfx::load(&img.bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        let hash = img.perception_hash(gfx::HashAlg::Gradient);
+
+        Ok(hash)
+    }))
+}
+
+pub fn img_phash(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("img_phash", hlua::function1(move |blob: String| -> Result<String> {
+        let img = state.get_blob(&blob)
+            .map_err(|err| state.set_error(err))?;
+
+        let img = gfx::load(&img.bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        let hash = img.perception_hash(gfx::HashAlg::Median);
+
+        Ok(hash)
+    }))
+}
+
 #[derive(Debug, Serialize)]
 pub struct Nudity {
     nude: bool,

src/runtime/hashes.rs

@@ -2,9 +2,8 @@ use crate::errors::*;
 use crate::engine::ctx::State;
 use crate::engine::structs::{byte_array, lua_bytes};
 use crate::hlua::{self, AnyLuaValue};
-use digest::{Digest, Update, BlockInput, FixedOutput, Reset};
-use digest::generic_array::ArrayLength;
-use hmac::{Hmac, Mac, NewMac};
+use digest::{Digest, core_api::BlockSizeUser};
+use hmac::{Mac, SimpleHmac};
 use std::sync::Arc;
 
 
@@ -12,7 +11,7 @@ pub fn md5(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("md5", hlua::function1(move |bytes: AnyLuaValue| -> Result<AnyLuaValue> {
         byte_array(bytes)
             .map_err(|err| state.set_error(err))
-            .map(|bytes| lua_bytes(&md5::Md5::digest(&bytes)))
+            .map(|bytes| lua_bytes(&md5::Md5::digest(bytes)))
     }))
 }
 
@@ -20,7 +19,7 @@ pub fn sha1(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("sha1", hlua::function1(move |bytes: AnyLuaValue| -> Result<AnyLuaValue> {
         byte_array(bytes)
             .map_err(|err| state.set_error(err))
-            .map(|bytes| lua_bytes(&sha1::Sha1::digest(&bytes)))
+            .map(|bytes| lua_bytes(&sha1::Sha1::digest(bytes)))
     }))
 }
 
@@ -28,7 +27,7 @@ pub fn sha2_256(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("sha2_256", hlua::function1(move |bytes: AnyLuaValue| -> Result<AnyLuaValue> {
         byte_array(bytes)
             .map_err(|err| state.set_error(err))
-            .map(|bytes| lua_bytes(&sha2::Sha256::digest(&bytes)))
+            .map(|bytes| lua_bytes(&sha2::Sha256::digest(bytes)))
     }))
 }
 
@@ -36,7 +35,7 @@ pub fn sha2_512(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("sha2_512", hlua::function1(move |bytes: AnyLuaValue| -> Result<AnyLuaValue> {
         byte_array(bytes)
             .map_err(|err| state.set_error(err))
-            .map(|bytes| lua_bytes(&sha2::Sha512::digest(&bytes)))
+            .map(|bytes| lua_bytes(&sha2::Sha512::digest(bytes)))
     }))
 }
 
@@ -44,7 +43,7 @@ pub fn sha3_256(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("sha3_256", hlua::function1(move |bytes: AnyLuaValue| -> Result<AnyLuaValue> {
         byte_array(bytes)
             .map_err(|err| state.set_error(err))
-            .map(|bytes| lua_bytes(&sha3::Sha3_256::digest(&bytes)))
+            .map(|bytes| lua_bytes(&sha3::Sha3_256::digest(bytes)))
     }))
 }
 
@@ -52,20 +51,17 @@ pub fn sha3_512(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("sha3_512", hlua::function1(move |bytes: AnyLuaValue| -> Result<AnyLuaValue> {
         byte_array(bytes)
             .map_err(|err| state.set_error(err))
-            .map(|bytes| lua_bytes(&sha3::Sha3_512::digest(&bytes)))
+            .map(|bytes| lua_bytes(&sha3::Sha3_512::digest(bytes)))
     }))
 }
 
 fn hmac<D>(secret: AnyLuaValue, msg: AnyLuaValue) -> Result<AnyLuaValue>
-    where
-        D: Update + BlockInput + FixedOutput + Reset + Default + Clone,
-        D::BlockSize: ArrayLength<u8> + Clone,
-        D::OutputSize: ArrayLength<u8>,
+    where D: Digest + BlockSizeUser
 {
     let secret = byte_array(secret)?;
     let msg = byte_array(msg)?;
 
-    let mut mac = match Hmac::<D>::new_from_slice(&secret) {
+    let mut mac = match SimpleHmac::<D>::new_from_slice(&secret) {
         Ok(mac) => mac,
         Err(_) => bail!("Invalid key length"),
     };

src/runtime/hex.rs

@@ -3,9 +3,9 @@ use crate::errors::*;
 use crate::engine::ctx::State;
 use crate::engine::structs::byte_array;
 use crate::hlua::{self, AnyLuaValue};
+use std::fmt::Write;
 use std::sync::Arc;
 
-
 // TODO: consider deprecating this function, replace with hex_{en,de}code and hex_custom_{en,de}code
 pub fn hex(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("hex", hlua::function1(move |bytes: AnyLuaValue| -> Result<String> {
@@ -15,7 +15,7 @@ pub fn hex(lua: &mut hlua::Lua, state: Arc<dyn State>) {
                 let mut out = String::new();
 
                 for b in bytes {
-                    out += &format!("{:02x}", b);
+                    write!(out, "{:02x}", b).expect("out of memory");
                 }
 
                 out