Merge pull request #269 from Setsuwaa/patch-1

Fixed a typo

.github/FUNDING.yml

@@ -1 +1,2 @@
 github: [kpcyrd]
+patreon: kpcyrd

.github/workflows/docker-release.yml

@@ -0,0 +1,58 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to GitHub Registry
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/kpcyrd/sn0int
+          tags: |
+            type=semver,pattern={{raw}}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: Login to Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push Docker images
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          file: Dockerfile
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+      -
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/docker.yml

@@ -2,9 +2,9 @@ name: Docker
 
 on:
   push:
-    branches: [ master ]
+    branches: [ main ]
   pull_request:
-    branches: [ master ]
+    branches: [ main ]
 
 jobs:
   build:
@@ -22,7 +22,7 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Build the Docker image
-      run: docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
+      run: DOCKER_BUILDKIT=1 docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
     - name: Test the Docker image
       run: docker run --rm ${{ matrix.build.name }} --help
 

.github/workflows/rust.yml

@@ -2,9 +2,11 @@ name: Rust
 
 on:
   push:
-    branches: [ master ]
+    branches: [ main ]
   pull_request:
-    branches: [ master ]
+    branches: [ main ]
+  schedule:
+    - cron: '0 9 * * 1'
 
 env:
   CARGO_TERM_COLOR: always
@@ -49,3 +51,19 @@ jobs:
       run: cd sn0int-std && cargo test --verbose
     - name: Run tests (std, --ignored)
       run: cd sn0int-std && cargo test --verbose -- --ignored
+
+  notify:
+    # forks shouldn't notify
+    if: github.repository == 'kpcyrd/sn0int'
+    runs-on: ubuntu-latest
+    needs:
+    - build
+
+    steps:
+    - name: irc notify
+      uses: rectalogic/notify-irc@v1
+      with:
+        server: irc.hackint.org
+        channel: "#sn0int"
+        nickname: github-ci
+        message: '${{ github.repository }}#${{ github.run_id }}(${{ github.event_name }}): ${{ github.ref }}: tests completed: ${{ needs.build.result }} (https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})'

.travis.yml

@@ -1,61 +0,0 @@
-language: rust
-
-# upload takes too long on windows and gets killed due to inactivity
-#cache: cargo
-
-matrix:
-  include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=std
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=std
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows test"
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows common"
-
-before_install:
-- ci/setup.sh "$TRAVIS_OS_NAME"
-script:
-- df -h
-- ci/run.sh $BUILD_MODE
-- df -h
-
-notifications:
-  irc:
-    template:
-    - "%{repository_slug}#%{build_number} (%{branch} - %{commit} : %{author}): %{message} (%{build_url})"
-    channels:
-    - "ircs://irc.hackint.org:6697/#sn0int"
-    use_notice: true
-    skip_join: true
-    #on_success: change # default: always
-    #on_failure: always # default: always

Cargo.toml

@@ -1,16 +1,13 @@
 [package]
 name = "sn0int"
-version = "0.20.1"
+version = "0.26.1"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
-edition = "2018"
-
-[badges]
-travis-ci = { repository = "kpcyrd/sn0int" }
+edition = "2021"
 
 [workspace]
 members = ["sn0int-common",
@@ -27,7 +24,6 @@ the results in a unified format for followup investigations."""
 section = "utils"
 priority = "optional"
 depends = "$auto, publicsuffix"
-build-depends = "pkg-config, libsqlite3-dev, libseccomp-dev, libsodium-dev, publicsuffix"
 assets = [
     ["target/release/sn0int", "usr/bin/", "755"],
 ]
@@ -36,13 +32,14 @@ assets = [
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.12.0", path="sn0int-common" }
-sn0int-std = { version="=0.20.1", path="sn0int-std" }
-rustyline = "7.0"
+sn0int-common = { version="0.14.0", path="sn0int-common" }
+sn0int-std = { version="=0.26.0", path="sn0int-std" }
+rustyline = "10.0"
 log = "0.4"
-env_logger = "0.8"
+env_logger = "0.11"
 hlua-badtouch = "0.4"
-structopt = "0.3"
+clap = { version = "4.3.11", features = ["derive", "env"] }
+clap_complete = "4.3.2"
 failure = "0.1"
 rand = "0.8"
 colored = "2"
@@ -50,59 +47,58 @@ lazy_static = "1.0"
 shellwords = "1.0"
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-libsqlite3-sys = { version = "0.18.0", features = ["bundled-windows"] }
+libsqlite3-sys = { version = "0.22.0", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
 dirs-next = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
 chrootable-https = "0.16"
-base64 = "0.13"
-data-encoding = "2.1.2"
+data-encoding = "2.3.3"
 serde = { version = "1.0", features = ["derive"] }
 serde_urlencoded = "0.7"
 serde_json = "1.0"
 crossbeam-channel = "0.5"
 ctrlc = "3.1"
-opener = "0.4"
+opener = "0.6"
 separator = "0.4"
 maplit = "1.0.1"
 sloppy-rfc4880 = "0.2"
 regex = "1.0"
-toml = "0.5"
+toml = "0.7"
 threadpool = "1.7"
 atty = "0.2"
-semver = "0.11"
+semver = "1"
 bytes = "0.4"
-bytesize = "1.0"
-ipnetwork = "0.17"
-strum = "0.20"
-strum_macros = "0.20"
+ipnetwork = "0.20"
+strum = "0.25"
+strum_macros = "0.25"
 embedded-triple = "0.1.0"
+humansize = "2"
 
-digest = "0.9"
-md-5 = "0.9"
-sha-1 = "0.9"
-sha2 = "0.9"
-sha3 = "0.9"
-hmac = "0.10"
+digest = "0.10"
+md-5 = "0.10"
+sha-1 = "0.10"
+sha2 = "0.10"
+sha3 = "0.10"
+hmac = "0.12"
 
 walkdir = "2.2"
 nude = "0.3"
 glob = "0.3.0"
-os-version = "0.1.1"
+os-version = "0.2"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.15"
-nix = "0.19"
+syscallz = "0.16"
+nix = { version = "0.27", features = ["fs"] }
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.4"
-unveil = "0.2.0"
+unveil = "0.3"
 
 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.11"
+boxxy = "0.13"
 tempfile = "3.0"

Dockerfile

@@ -1,14 +1,18 @@
-FROM rust:alpine3.11
+FROM rust:alpine3.20
 ENV RUSTFLAGS="-C target-feature=-crt-static"
 RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo build --release --verbose
-RUN strip target/release/sn0int
+RUN --mount=type=cache,target=/var/cache/buildkit \
+    CARGO_HOME=/var/cache/buildkit/cargo \
+    CARGO_TARGET_DIR=/var/cache/buildkit/target \
+    cargo build --release --locked --verbose && \
+    cp -v /var/cache/buildkit/target/release/sn0int /
+RUN strip /sn0int
 
-FROM alpine:3.11
+FROM alpine:3.20
 RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
-COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+COPY --from=0 /sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \
     XDG_CACHE_HOME=/cache

README.md

@@ -1,7 +1,5 @@
-# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
+# sn0int [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
-[travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
-[travis]:       https://travis-ci.org/kpcyrd/sn0int
 [crates-img]:   https://img.shields.io/crates/v/sn0int.svg
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
@@ -16,33 +14,34 @@
 [registry]:     https://sn0int.com/
 
 sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
-package manager. It was built for IT security professionals and bug hunters to
-gather intelligence about a given target or about yourself. sn0int is
-enumerating attack surface by semi-automatically processing public information
-and mapping the results in a unified format for followup investigations.
+package manager. It's used by IT security professionals, bug bounty hunters,
+law enforcement agencies and in security awareness trainings to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.
 
 [ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
 - Harvest subdomains from certificate transparency logs and passive dns
+- Mass resolve collected subdomains and scan for http or https services
 - Enrich ip addresses with asn and geoip info
 - Harvest emails from pgp keyservers and whois
 - Discover compromised logins in breaches
 - Find somebody's profiles across the internet
 - Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Attempt to bypass cloudflare with shodan
-- Harvest data and images from instagram profiles
-- Scan images for nudity
+- Harvest activity and images from social media profiles
+- Basic image processing
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource. None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
+in the source, instead they are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+you to ship updates for your modules on your own instead of pull-requesting
+them into the sn0int codebase.
 
 For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
@@ -50,6 +49,8 @@ For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https:/
 
 ## Installation
 
+<a href="https://repology.org/project/sn0int/versions"><img align="right" src="https://repology.org/badge/vertical-allrepos/sn0int.svg" alt="Packaging status"></a>
+
 Archlinux
 
     pacman -S sn0int
@@ -60,19 +61,16 @@ Mac OSX
 
 Debian/Ubuntu/Kali
 
-There are prebuilt packages signed by a debian maintainer. We can import the
-key for this repository out of the debian keyring.
+There are prebuilt packages signed by a debian maintainer:
 
-    apt install debian-keyring
-    gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
-    apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
-    echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
-    apt update
-    apt install sn0int
+    sudo apt install curl sq
+    curl -sSf https://apt.vulns.sexy/kpcyrd.pgp | sq dearmor | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg > /dev/null
+    echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    sudo apt update
 
 Docker
 
-    docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
+    docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" ghcr.io/kpcyrd/sn0int
 
 Alpine
 
@@ -226,6 +224,9 @@ For everything else please have a look at the [detailed list][1].
   - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
   - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_ahash](https://sn0int.readthedocs.io/en/latest/reference.html#img-ahash)
+  - [img_dhash](https://sn0int.readthedocs.io/en/latest/reference.html#img-dhash)
+  - [img_phash](https://sn0int.readthedocs.io/en/latest/reference.html#img-phash)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
   - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)

ci/integration.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import subprocess
 from subprocess import DEVNULL, PIPE
+import shutil
+from pathlib import Path
 import tempfile
 import json
 import sys
@@ -98,6 +100,11 @@ def main(tempdir, binary):
         ('https://www.example.com/', 200),
     }
 
+    cache = Path.home() / '.cache' / 'sn0int'
+    if cache.exists():
+        print('[*] copying geoip files')
+        shutil.copytree(cache, tempdir + '/.cache/sn0int', dirs_exist_ok=True)
+
     print('[*] running geoip')
     sn0int(tempdir, binary, [
         'use geoip',

ci/run.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    build)
-        cargo build --verbose
-        cargo build --verbose --examples
-        ;;
-    test)
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    common)
-        cd sn0int-common
-        cargo test --verbose
-        ;;
-    std)
-        cd sn0int-std
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    windows)
-        cargo build --verbose
-        cargo build --verbose --examples
-        ;;
-    boxxy)
-        cargo build --verbose --examples
-        if cat ci/boxxy_stage1.txt | RUST_LOG=boxxy cargo run --example boxxy; then
-            echo "SANDOX ERROR: should've crashed"
-            exit 1
-        fi
-        ;;
-esac

ci/setup.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    linux)
-        sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev libsodium-dev
-        ;;
-    osx)
-        brew install libsodium
-        ;;
-esac

docs/build.rst

@@ -44,6 +44,13 @@ Alpine
 
     $ apk add sqlite-dev libseccomp-dev libsodium-dev
 
+Docker
+~~~~~~
+
+.. code-block:: bash
+
+    $ DOCKER_BUILDKIT=1 docker build -t kpcyrd/sn0int .
+
 OpenBSD
 ~~~~~~~
 

docs/install.rst

@@ -19,34 +19,31 @@ Mac OSX
 
     $ brew install sn0int
 
-Debian >= bullseye, Ubuntu >= 20.04, Kali
+Debian >= bookwork, Ubuntu >= 22.10, Kali
 -----------------------------------------
 
-There are prebuilt packages signed by a debian maintainer. We can import the
-key for this repository out of the debian keyring.
+There are prebuilt packages signed by a debian maintainer:
 
 .. code-block:: bash
 
-    $ sudo apt install debian-keyring
-    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg kpcyrd@archlinux.org | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg
+    $ sudo apt install curl sq
+    $ curl -sSf https://apt.vulns.sexy/kpcyrd.pgp | sq keyring filter -B --handle 64B13F7117D6E07D661BBCE0FE763A64F5E54FD6 | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg > /dev/null
     $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
-    $ sudo apt update
-    $ sudo apt install sn0int
+    $ apt update
+    $ apt install sn0int
 
-Debian <= buster, Ubuntu <= 19.10
----------------------------------
+Debian <= bullseye, Ubuntu <= 22.04
+-----------------------------------
 
-There are prebuilt packages signed by a debian maintainer. We can import the
-key for this repository out of the debian keyring.
+There are prebuilt packages signed by a debian maintainer:
 
 .. code-block:: bash
 
-    $ sudo apt install debian-keyring
-    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | sudo apt-key add -
-    $ sudo apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    $ sudo apt install curl sq
+    $ curl -sSf https://apt.vulns.sexy/kpcyrd.pgp | sq dearmor | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg > /dev/null
     $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
-    $ sudo apt update
-    $ sudo apt install sn0int
+    $ apt update
+    $ apt install sn0int
 
 Fedora/CentOS/Redhat
 --------------------

docs/reference.rst

@@ -504,6 +504,39 @@ Extract exif metadata from an image.
     if last_err() then return end
     debug(exif)
 
+img_ahash
+---------
+
+Calculate the Mean (aHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_ahash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_dhash
+---------
+
+Calculate the Gradient (dHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_dhash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_phash
+---------
+
+Calculate the DCT (pHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_phash(blob)
+    if last_err() then return end
+    debug(hash)
+
 img_nudity
 ----------
 

docs/sandbox.rst

@@ -119,7 +119,7 @@ You might experience a sandbox failure, especially on architectures that are
 less popular. This usually looks like this::
 
     [sn0int][example][kpcyrd/ctlogs] > run
-    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [-] Failed "example.com": Sandbox child has crashed
     [+] Finished kpcyrd/ctlogs (1 errors)
 
 A module that never finishes could also mean an IO thread inside the worker got

examples/boxxy.rs

@@ -1,6 +1,7 @@
-#[macro_use] extern crate boxxy;
-extern crate sn0int;
+#[macro_use]
+extern crate boxxy;
 extern crate env_logger;
+extern crate sn0int;
 
 fn stage1(sh: &mut boxxy::Shell, _args: Vec<String>) -> Result<(), boxxy::Error> {
     shprintln!(sh, "[*] starting stage1");
@@ -14,8 +15,6 @@ fn main() {
 
     println!("stage1        activate sandbox");
 
-    let toolbox = boxxy::Toolbox::new().with(vec![
-            ("stage1", stage1),
-        ]);
+    let toolbox = boxxy::Toolbox::new().with(vec![("stage1", stage1)]);
     boxxy::Shell::new(toolbox).run()
 }

examples/maxmind.rs

@@ -1,19 +1,19 @@
+use clap::Parser;
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
 use sn0int::paths;
 use std::net::IpAddr;
 use std::path::Path;
-use structopt::StructOpt;
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Args {
-    #[structopt(name="asn")]
+    #[command(name = "asn")]
     Asn(AsnArgs),
-    #[structopt(name="geoip")]
+    #[command(name = "geoip")]
     GeoIP(GeoIPArgs),
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AsnArgs {
     ip: IpAddr,
 }
@@ -30,7 +30,7 @@ impl AsnArgs {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct GeoIPArgs {
     ip: IpAddr,
 }
@@ -47,9 +47,8 @@ impl GeoIPArgs {
     }
 }
 
-
 fn run() -> Result<()> {
-    let args = Args::from_args();
+    let args = Args::parse();
     debug!("{:?}", args);
     let cache_dir = paths::cache_dir()?;
     match args {

examples/spinners.rs

@@ -1,21 +1,20 @@
+use clap::Parser;
+use sn0int::term::{Spinner, StackedSpinners, SPINNERS};
 use std::thread;
 use std::time::Duration;
-use sn0int::term::{SPINNERS, Spinner, StackedSpinners};
-use structopt::StructOpt;
 
-
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Args {
-    #[structopt(name="single")]
+    #[command(name = "single")]
     Single(Single),
-    #[structopt(name="stacked")]
+    #[command(name = "stacked")]
     Stacked(Stacked),
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Single {
     idx: usize,
-    #[structopt(long="ticks", default_value="100")]
+    #[structopt(long = "ticks", default_value = "100")]
     ticks: usize,
 }
 
@@ -32,9 +31,8 @@ impl Single {
     }
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Stacked {
-}
+#[derive(Debug, Parser)]
+pub struct Stacked {}
 
 impl Stacked {
     fn run(&self) {
@@ -59,7 +57,7 @@ impl Stacked {
 }
 
 fn main() {
-    let args = Args::from_args();
+    let args = Args::parse();
     match args {
         Args::Single(args) => args.run(),
         Args::Stacked(args) => args.run(),

modules/harness/img-hash.lua

@@ -0,0 +1,11 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    debug(arg)
+    info(img_ahash(arg['value']))
+    info(img_dhash(arg['value']))
+    info(img_phash(arg['value']))
+end

modules/harness/import-subdomains.lua

@@ -3,7 +3,7 @@
 -- License: GPL-3.0
 
 function run()
-    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lu
+    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lua
 
     while true do
         local line = stdin_readline()

modules/harness/winkekatze-sub.lua

@@ -10,8 +10,17 @@ function run()
     mqtt_subscribe(sock, '#', 0)
 
     while true do
-        local pkt = mqtt_recv_text(sock)
+        -- read the next mqtt packet
+        local pkt = mqtt_recv(sock)
         if last_err() then return end
-        info(pkt)
+
+        local text
+        if pkt then
+            -- attempt to utf8 decode the body if there was a pkt
+            text = utf8_decode(pkt['body'])
+            if last_err() then clear_err() end
+        end
+
+        info({pkt=pkt, text=text})
     end
 end

sn0int-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.12.0"
+version = "0.14.0"
 description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -12,4 +12,5 @@ serde = { version = "1.0", features=["derive"] }
 #rocket_failure_errors = { path = "../../../rocket_failure/rocket_failure_errors" }
 rocket_failure_errors = "0.2"
 anyhow = "1.0"
-nom = "6.0"
+nom = "7.0"
+clap = { version = "4.3.11", features = ["derive"] }

sn0int-common/src/api.rs

@@ -1,5 +1,5 @@
 use crate::id::ModuleID;
-use serde::{Serialize, Deserialize};
+use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct WhoamiResponse {

sn0int-common/src/errors.rs

@@ -1,2 +1,2 @@
-pub use anyhow::{Error, Context, anyhow, format_err, bail};
+pub use anyhow::{anyhow, bail, format_err, Context, Error};
 pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-common/src/id.rs

@@ -1,10 +1,9 @@
 use crate::errors::*;
-use serde::{de, Serialize, Serializer, Deserialize, Deserializer};
+use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
 use std::fmt;
 use std::result;
 use std::str::FromStr;
 
-
 #[inline(always)]
 fn valid_char(c: char) -> bool {
     nom::character::is_alphanumeric(c as u8) || c == '-'
@@ -19,15 +18,15 @@ pub fn valid_name(name: &str) -> Result<()> {
 }
 
 fn module(s: &str) -> nom::IResult<&str, ModuleID> {
-    let (input, (author, _, name)) = nom::sequence::tuple((
-        token,
-        nom::bytes::complete::tag("/"),
-        token,
-    ))(s)?;
-    Ok((input, ModuleID {
-        author: author.to_string(),
-        name: name.to_string(),
-    }))
+    let (input, (author, _, name)) =
+        nom::sequence::tuple((token, nom::bytes::complete::tag("/"), token))(s)?;
+    Ok((
+        input,
+        ModuleID {
+            author: author.to_string(),
+            name: name.to_string(),
+        },
+    ))
 }
 
 #[inline]
@@ -35,7 +34,7 @@ fn token(s: &str) -> nom::IResult<&str, &str> {
     nom::bytes::complete::take_while1(valid_char)(s)
 }
 
-#[derive(Debug, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct ModuleID {
     pub author: String,
     pub name: String,
@@ -51,8 +50,8 @@ impl FromStr for ModuleID {
     type Err = Error;
 
     fn from_str(s: &str) -> Result<ModuleID> {
-        let (trailing, module) = module(s)
-            .map_err(|err| anyhow!("Failed to parse module id: {:?}", err))?;
+        let (trailing, module) =
+            module(s).map_err(|err| anyhow!("Failed to parse module id: {:?}", err))?;
         if !trailing.is_empty() {
             bail!("Trailing data in module id");
         }
@@ -71,7 +70,8 @@ impl Serialize for ModuleID {
 
 impl<'de> Deserialize<'de> for ModuleID {
     fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-        where D: Deserializer<'de>
+    where
+        D: Deserializer<'de>,
     {
         let s = String::deserialize(deserializer)?;
         FromStr::from_str(&s).map_err(de::Error::custom)
@@ -85,10 +85,13 @@ mod tests {
     #[test]
     fn verify_valid() {
         let result = ModuleID::from_str("kpcyrd/foo").expect("parse");
-        assert_eq!(result, ModuleID {
-            author: "kpcyrd".to_string(),
-            name: "foo".to_string(),
-        });
+        assert_eq!(
+            result,
+            ModuleID {
+                author: "kpcyrd".to_string(),
+                name: "foo".to_string(),
+            }
+        );
     }
 
     #[test]

sn0int-common/src/lib.rs

@@ -1,10 +1,8 @@
-#[macro_use] extern crate nom;
-
 pub mod api;
 pub mod errors;
 pub use crate::errors::*;
-pub mod metadata;
 pub mod id;
+pub mod metadata;
 pub use crate::id::*;
 
 pub use rocket_failure_errors::StrictApiResponse as ApiResponse;

sn0int-common/src/metadata/mod.rs

@@ -1,5 +1,9 @@
 use crate::errors::*;
-use serde::{Serialize, Deserialize};
+use nom::bytes::complete::{tag, take_until};
+use nom::combinator::map_res;
+use nom::multi::fold_many0;
+use nom::IResult;
+use serde::{Deserialize, Serialize};
 use std::str::FromStr;
 
 mod stealth;
@@ -154,8 +158,7 @@ impl FromStr for Metadata {
     type Err = Error;
 
     fn from_str(code: &str) -> Result<Metadata> {
-        let (_, lines) = metalines(code)
-            .map_err(|_| format_err!("Failed to parse header"))?;
+        let (_, lines) = metalines(code).map_err(|_| format_err!("Failed to parse header"))?;
 
         let mut data = NewMetadata::default();
 
@@ -190,24 +193,26 @@ pub struct NewMetadata<'a> {
 
 impl<'a> NewMetadata<'a> {
     fn try_from(self) -> Result<Metadata> {
-        let description = self.description.ok_or_else(|| format_err!("Description is required"))?;
-        let version = self.version.ok_or_else(|| format_err!("Version is required"))?;
+        let description = self
+            .description
+            .ok_or_else(|| format_err!("Description is required"))?;
+        let version = self
+            .version
+            .ok_or_else(|| format_err!("Version is required"))?;
         let source = match self.source {
             Some(x) => Some(x.parse()?),
             _ => None,
         };
-        let keyring_access = self.keyring_access.into_iter()
-            .map(String::from)
-            .collect();
+        let keyring_access = self.keyring_access.into_iter().map(String::from).collect();
         let stealth = match self.stealth {
             Some(x) => x.parse()?,
             _ => Stealth::Normal,
         };
-        let authors = self.authors.into_iter()
-            .map(String::from)
-            .collect();
+        let authors = self.authors.into_iter().map(String::from).collect();
         let repository = self.repository.map(String::from);
-        let license = self.license.ok_or_else(|| format_err!("License is required"))?;
+        let license = self
+            .license
+            .ok_or_else(|| format_err!("License is required"))?;
         let license = license.parse()?;
 
         Ok(Metadata {
@@ -223,23 +228,25 @@ impl<'a> NewMetadata<'a> {
     }
 }
 
-named!(metaline<&str, (EntryType, &str)>, do_parse!(
-    tag!("-- ") >>
-    name: map_res!(take_until!(": "), EntryType::from_str) >>
-    tag!(": ") >>
-    value: take_until!("\n") >>
-    tag!("\n") >>
-    (name, value)
-));
+fn metaline(input: &str) -> IResult<&str, (EntryType, &str)> {
+    let (input, _) = tag("-- ")(input)?;
+    let (input, name) = map_res(take_until(": "), EntryType::from_str)(input)?;
+    let (input, _) = tag(": ")(input)?;
+    let (input, value) = take_until("\n")(input)?;
+    let (input, _) = tag("\n")(input)?;
 
-named!(metalines<&str, Vec<(EntryType, &str)>>, do_parse!(
-    lines: fold_many0!(metaline, Vec::new(), |mut acc: Vec<_>, item| {
+    Ok((input, (name, value)))
+}
+
+fn metalines(input: &str) -> IResult<&str, Vec<(EntryType, &str)>> {
+    let (input, lines) = fold_many0(metaline, Vec::new, |mut acc: Vec<_>, item| {
         acc.push(item);
         acc
-    }) >>
-    tag!("\n") >>
-    (lines)
-));
+    })(input)?;
+    let (input, _) = tag("\n")(input)?;
+
+    Ok((input, lines))
+}
 
 #[cfg(test)]
 mod tests {
@@ -247,27 +254,34 @@ mod tests {
 
     #[test]
     fn verify_simple() {
-        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+        let metadata = Metadata::from_str(
+            r#"-- Description: Hello world, this is my description
 -- Version: 1.0.0
 -- Source: domains
 -- License: WTFPL
 
-"#).expect("parse");
-        assert_eq!(metadata, Metadata {
-            description: "Hello world, this is my description".to_string(),
-            version: "1.0.0".to_string(),
-            license: License::WTFPL,
-            source: Some(Source::Domains),
-            stealth: Stealth::Normal,
-            authors: vec![],
-            repository: None,
-            keyring_access: Vec::new(),
-        });
+"#,
+        )
+        .expect("parse");
+        assert_eq!(
+            metadata,
+            Metadata {
+                description: "Hello world, this is my description".to_string(),
+                version: "1.0.0".to_string(),
+                license: License::WTFPL,
+                source: Some(Source::Domains),
+                stealth: Stealth::Normal,
+                authors: vec![],
+                repository: None,
+                keyring_access: Vec::new(),
+            }
+        );
     }
 
     #[test]
     fn verify_much_metadata() {
-        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+        let metadata = Metadata::from_str(
+            r#"-- Description: Hello world, this is my description
 -- Version: 1.0.0
 -- Source: domains
 -- Stealth: passive
@@ -276,59 +290,74 @@ mod tests {
 -- Repository: https://github.com/kpcyrd/sn0int
 -- License: WTFPL
 
-"#).expect("parse");
-        assert_eq!(metadata, Metadata {
-            description: "Hello world, this is my description".to_string(),
-            version: "1.0.0".to_string(),
-            license: License::WTFPL,
-            source: Some(Source::Domains),
-            stealth: Stealth::Passive,
-            authors: vec![
-                "kpcyrd <git at rxv dot cc>".to_string(),
-                "kpcyrd's cat".to_string(),
-            ],
-            repository: Some("https://github.com/kpcyrd/sn0int".to_string()),
-            keyring_access: Vec::new(),
-        });
+"#,
+        )
+        .expect("parse");
+        assert_eq!(
+            metadata,
+            Metadata {
+                description: "Hello world, this is my description".to_string(),
+                version: "1.0.0".to_string(),
+                license: License::WTFPL,
+                source: Some(Source::Domains),
+                stealth: Stealth::Passive,
+                authors: vec![
+                    "kpcyrd <git at rxv dot cc>".to_string(),
+                    "kpcyrd's cat".to_string(),
+                ],
+                repository: Some("https://github.com/kpcyrd/sn0int".to_string()),
+                keyring_access: Vec::new(),
+            }
+        );
     }
 
     #[test]
     fn verify_no_source() {
-        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+        let metadata = Metadata::from_str(
+            r#"-- Description: Hello world, this is my description
 -- Version: 1.0.0
 -- License: WTFPL
 
-"#).expect("parse");
-        assert_eq!(metadata, Metadata {
-            description: "Hello world, this is my description".to_string(),
-            version: "1.0.0".to_string(),
-            license: License::WTFPL,
-            source: None,
-            stealth: Stealth::Normal,
-            authors: vec![],
-            repository: None,
-            keyring_access: Vec::new(),
-        });
+"#,
+        )
+        .expect("parse");
+        assert_eq!(
+            metadata,
+            Metadata {
+                description: "Hello world, this is my description".to_string(),
+                version: "1.0.0".to_string(),
+                license: License::WTFPL,
+                source: None,
+                stealth: Stealth::Normal,
+                authors: vec![],
+                repository: None,
+                keyring_access: Vec::new(),
+            }
+        );
     }
 
     #[test]
     fn verify_require_license() {
-        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+        let metadata = Metadata::from_str(
+            r#"-- Description: Hello world, this is my description
 -- Version: 1.0.0
 -- Source: domains
 
-"#);
+"#,
+        );
         assert!(metadata.is_err());
     }
 
     #[test]
     fn verify_require_opensource_license() {
-        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+        let metadata = Metadata::from_str(
+            r#"-- Description: Hello world, this is my description
 -- Version: 1.0.0
 -- Source: domains
 -- License: Proprietary
 
-"#);
+"#,
+        );
         assert!(metadata.is_err());
     }
 

sn0int-common/src/metadata/stealth.rs

@@ -1,8 +1,9 @@
 use crate::errors::*;
-use serde::{Serialize, Deserialize};
+use clap::ValueEnum;
+use serde::{Deserialize, Serialize};
 use std::str::FromStr;
 
-#[derive(Debug, Eq, PartialEq, PartialOrd, Clone, Serialize, Deserialize)]
+#[derive(Debug, Eq, PartialEq, PartialOrd, Clone, ValueEnum, Serialize, Deserialize)]
 pub enum Stealth {
     Loud,
     Normal,
@@ -13,12 +14,7 @@ pub enum Stealth {
 impl Stealth {
     #[inline]
     pub fn variants() -> &'static [&'static str] {
-        &[
-            "loud",
-            "normal",
-            "passive",
-            "offline",
-        ]
+        &["loud", "normal", "passive", "offline"]
     }
 
     #[inline(always)]

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.12.0", path="../sn0int-common" }
+sn0int-common = { version="0.14.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }
@@ -23,7 +23,7 @@ oauth2 = "2.0.0"
 failure = "0.1"
 url = "1.0"
 log = "0.4"
-semver = "0.9.0"
+semver = "1"
 lazy_static = "1"
 blake2 = "0.8.0"
 hex = "0.4"
@@ -33,5 +33,5 @@ serde_json = "1.0"
 syntect = "3.3"
 
 dotenv = "0.15"
-env_logger = "0.7"
+env_logger = "0.9"
 structopt = "0.3.21"

sn0int-registry/src/github.rs

@@ -5,6 +5,7 @@ pub fn get_username(oauth_token: &str) -> Result<String> {
     let client = reqwest::Client::new();
     let mut resp = client.get("https://api.github.com/user")
         .header("Authorization", format!("token {}", oauth_token))
+        .header("User-Agent", "sn0int-registry")
         .send()
         .context("Failed to check access_token")?
         .error_for_status()

sn0int-std/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "sn0int-std"
-version = "0.20.1"
+version = "0.26.0"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
 license = "GPL-3.0"
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -21,35 +21,37 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_urlencoded = "0.7"
 webpki = "0.21"
-webpki-roots = "0.20"
+webpki-roots = "0.21"
 ct-logs = "0.7"
 #chrootable-https = { path = "../../chrootable-https" }
 chrootable-https = "0.16"
 http = "0.2"
 bufstream = "0.1.4"
-pem = "0.8"
+pem = "3"
 url = "2.0"
-tungstenite = { version = "0.11", default-features = false }
+tungstenite = { version = "0.13", default-features = false }
 kuchiki = "0.8.0"
-maxminddb = "0.17"
-x509-parser = "0.9"
-der-parser = "5"
-publicsuffix = { version="1.5", default-features=false }
+maxminddb = "0.23"
+x509-parser = "0.13"
+der-parser = "8"
+publicsuffix = { version="2", default-features=false }
 xml-rs = "0.8"
-geo = "0.16"
+geo = "0.25"
 bytes = "0.4"
-base64 = "0.13"
 chrono = { version = "0.4", features = ["serde"] }
-mqtt-protocol = "0.10"
+mqtt-protocol = "0.11"
 sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
 
-image = "0.23.0"
+image = "0.23"
 kamadak-exif = "0.5.1"
+img_hash_median = "4.0.0"
 
-bs58 = "0.4"
-digest = "0.9"
-blake2 = "0.9"
+bs58 = "0.5"
+digest = "0.10"
+blake2 = "0.10"
+data-encoding = "2.3.3"
+thiserror = "1.0.38"
 
 [dev-dependencies]
-env_logger = "0.8"
+env_logger = "0.10"
 maplit = "1.0.1"

sn0int-std/src/blobs.rs

@@ -1,8 +1,9 @@
+use blake2::Blake2bVar;
 use bytes::Bytes;
-use blake2::VarBlake2b;
+use data_encoding::BASE64;
 use digest::{Update, VariableOutput};
-use serde::ser::{Serialize, Serializer};
 use serde::de::{self, Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
 use std::result;
 
 #[derive(Debug, Clone, PartialEq)]
@@ -14,14 +15,11 @@ pub struct Blob {
 impl Blob {
     pub fn create(bytes: Bytes) -> Blob {
         let id = Self::hash(&bytes);
-        Blob {
-            id,
-            bytes,
-        }
+        Blob { id, bytes }
     }
 
     pub fn hash(bytes: &[u8]) -> String {
-        let mut h = VarBlake2b::new(32).unwrap();
+        let mut h = Blake2bVar::new(32).unwrap();
         h.update(bytes);
         let output = h.finalize_boxed();
         Self::encode_hash(&output)
@@ -40,7 +38,7 @@ impl Serialize for Blob {
     where
         S: Serializer,
     {
-        let s = base64::encode(&self.bytes);
+        let s = BASE64.encode(&self.bytes);
         serializer.serialize_str(&s)
     }
 }
@@ -52,8 +50,7 @@ impl<'de> Deserialize<'de> for Blob {
         D: Deserializer<'de>,
     {
         let s = String::deserialize(deserializer)?;
-        let bytes = base64::decode(&s)
-            .map_err(de::Error::custom)?;
+        let bytes = BASE64.decode(s.as_bytes()).map_err(de::Error::custom)?;
         Ok(Blob::create(Bytes::from(bytes)))
     }
 }
@@ -62,7 +59,6 @@ pub trait BlobState {
     fn register_blob(&self, blob: Blob) -> String;
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -77,10 +73,13 @@ mod tests {
     #[test]
     fn verify_create_blob() {
         let (bytes, blob) = blob();
-        assert_eq!(blob, Blob {
-            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
-            bytes,
-        });
+        assert_eq!(
+            blob,
+            Blob {
+                id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
+                bytes,
+            }
+        );
     }
 
     #[test]

sn0int-std/src/crt.rs

@@ -1,10 +1,11 @@
 use crate::errors::*;
-use serde::{Serialize, Deserialize};
+use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 use std::net::IpAddr;
-use x509_parser::x509::X509Version;
 use x509_parser::certificate::X509Certificate;
 use x509_parser::extensions::{GeneralName, ParsedExtension};
+use x509_parser::prelude::*;
+use x509_parser::x509::X509Version;
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
@@ -24,14 +25,14 @@ impl Certificate {
                     bail!("input is not a certificate");
                 }
                 pem
-            },
+            }
             Err(_) => bail!("Failed to parse pem"),
         };
         Certificate::from_bytes(&pem.contents)
     }
 
     pub fn from_bytes(crt: &[u8]) -> Result<Certificate> {
-        let crt = match X509Certificate::from_der(&crt) {
+        let crt = match X509Certificate::from_der(crt) {
             Ok((remaining, der)) => {
                 if !remaining.is_empty() {
                     bail!("input cert has trailing garbage");
@@ -40,7 +41,7 @@ impl Certificate {
                     bail!("unexpected certificate version");
                 }
                 der
-            },
+            }
             Err(_) => bail!("Failed to parse der"),
         };
 
@@ -55,35 +56,33 @@ impl Certificate {
             }
         }
 
-        for (_oid, ext) in crt.tbs_certificate.extensions {
+        for ext in crt.tbs_certificate.extensions() {
             if let ParsedExtension::SubjectAlternativeName(san) = ext.parsed_extension() {
                 for name in &san.general_names {
                     debug!("Certificate is valid for {:?}", name);
                     match name {
                         GeneralName::DNSName(v) => {
                             valid_names.insert(v.to_string());
-                        },
+                        }
                         GeneralName::RFC822Name(v) => {
                             valid_emails.insert(v.to_string());
-                        },
+                        }
                         GeneralName::IPAddress(v) => {
                             let ip = match v.len() {
                                 4 => Some(IpAddr::from([v[0], v[1], v[2], v[3]])),
                                 16 => Some(IpAddr::from([
-                                    v[0], v[1], v[2], v[3],
-                                    v[4], v[5], v[6], v[7],
-                                    v[8], v[9], v[10], v[11],
-                                    v[12], v[13], v[14], v[15],
+                                    v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7], v[8], v[9],
+                                    v[10], v[11], v[12], v[13], v[14], v[15],
                                 ])),
                                 _ => {
                                     info!("Certificate is valid for invalid ip address: {:?}", v);
                                     None
-                                },
+                                }
                             };
                             if let Some(ip) = ip {
                                 valid_ipaddrs.insert(ip);
                             }
-                        },
+                        }
                         _ => (),
                     }
                 }
@@ -107,7 +106,8 @@ mod tests {
 
     #[test]
     fn test_parse_pem_github() {
-        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+        let mut x = Certificate::parse_pem(
+            r#"-----BEGIN CERTIFICATE-----
 MIIHQjCCBiqgAwIBAgIQCgYwQn9bvO1pVzllk7ZFHzANBgkqhkiG9w0BAQsFADB1
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
 d3cuZGlnaWNlcnQuY29tMTQwMgYDVQQDEytEaWdpQ2VydCBTSEEyIEV4dGVuZGVk
@@ -148,18 +148,24 @@ Kqg6LK0Hcq4K0sZnxE8HFxiZ92WpV2AVWjRMEc/2z2shNoDvxvFUYyY1Oe67xINk
 myQKc+ygSBZzyLnXSFVWmHr3u5dcaaQGGAR42v6Ydr4iL38Hd4dOiBma+FXsXBIq
 WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik=
 -----END CERTIFICATE-----
-"#).expect("Failed to parse cert");
+"#,
+        )
+        .expect("Failed to parse cert");
         x.valid_names.sort();
-        assert_eq!(x, Certificate {
-            valid_names: vec!["github.com".into(), "www.github.com".into()],
-            valid_emails: vec![],
-            valid_ipaddrs: vec![],
-        });
+        assert_eq!(
+            x,
+            Certificate {
+                valid_names: vec!["github.com".into(), "www.github.com".into()],
+                valid_emails: vec![],
+                valid_ipaddrs: vec![],
+            }
+        );
     }
 
     #[test]
     fn test_parse_pem_1_1_1_1() {
-        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+        let mut x = Certificate::parse_pem(
+            r#"-----BEGIN CERTIFICATE-----
 MIID9DCCA3qgAwIBAgIQBWzetBRl/ycHFsBukRYuGTAKBggqhkjOPQQDAjBMMQsw
 CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMSYwJAYDVQQDEx1EaWdp
 Q2VydCBFQ0MgU2VjdXJlIFNlcnZlciBDQTAeFw0xODAzMzAwMDAwMDBaFw0yMDAz
@@ -183,27 +189,30 @@ ADBlAjEAjoyy2Ogh1i1/Kh9+psMc1OChlQIvQF6AkojZS8yliar6m8q5nqC3qe0h
 HR0fExwLAjAueWRnHX4QJ9loqMhsPk3NB0Cs0mStsNDNG6/DpCYw7XmjoG3y1LS7
 ZkZZmqNn2Q8=
 -----END CERTIFICATE-----
-"#).expect("Failed to parse cert");
+"#,
+        )
+        .expect("Failed to parse cert");
         x.valid_names.sort();
         x.valid_ipaddrs.sort();
-        assert_eq!(x, Certificate {
-            valid_names: vec![
-                "*.cloudflare-dns.com".into(),
-                "cloudflare-dns.com".into(),
-            ],
-            valid_emails: vec![],
-            valid_ipaddrs: vec![
-                "1.0.0.1".parse().unwrap(),
-                "1.1.1.1".parse().unwrap(),
-                "2606:4700:4700::1001".parse().unwrap(),
-                "2606:4700:4700::1111".parse().unwrap(),
-            ],
-        });
+        assert_eq!(
+            x,
+            Certificate {
+                valid_names: vec!["*.cloudflare-dns.com".into(), "cloudflare-dns.com".into(),],
+                valid_emails: vec![],
+                valid_ipaddrs: vec![
+                    "1.0.0.1".parse().unwrap(),
+                    "1.1.1.1".parse().unwrap(),
+                    "2606:4700:4700::1001".parse().unwrap(),
+                    "2606:4700:4700::1111".parse().unwrap(),
+                ],
+            }
+        );
     }
 
     #[test]
     fn test_long_san_extension() {
-        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+        let mut x = Certificate::parse_pem(
+            r#"-----BEGIN CERTIFICATE-----
 MIII3jCCB8agAwIBAgIQAp1dOviF3mpYKKObx4fjxjANBgkqhkiG9w0BAQsFADBe
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
 d3cuZGlnaWNlcnQuY29tMR0wGwYDVQQDExRHZW9UcnVzdCBSU0EgQ0EgMjAxODAe
@@ -253,56 +262,62 @@ mAlnYDoB0Mj2UIPvIeftkDfF6sURmmZb0/+AMbFDCQYHvZFPI8DFgcagy8og5XJZ
 gQ+70UdJdM3RWyrd9R66aZwNGkcS6C2wtKCRhztWDMru/wNuyOsYS6JttoTYxRsh
 z/6Vy8Ga9kigYVsa8ZFMR+Ex
 -----END CERTIFICATE-----
-"#).expect("Failed to parse cert");
+"#,
+        )
+        .expect("Failed to parse cert");
         x.valid_names.sort();
         x.valid_ipaddrs.sort();
-        assert_eq!(x, Certificate {
-            valid_names: vec![
-                "aboutyou.de".into(),
-                "assets.aboutyou.de".into(),
-                "cdn.aboutstatic.com".into(),
-                "cdn.aboutyou-staging.de".into(),
-                "cdn.aboutyou.de".into(),
-                "cdn.edited.de".into(),
-                "cdn.mary-paul.de".into(),
-                "cdn.youandidol.de".into(),
-                "cdn1.aboutyou.de".into(),
-                "cdn2.aboutyou.de".into(),
-                "cdn3.aboutyou.de".into(),
-                "cdn4.aboutyou.de".into(),
-                "cdn5.aboutyou.de".into(),
-                "co-m.aboutyou.de".into(),
-                "co-mapp.aboutyou.de".into(),
-                "co-t.aboutyou.de".into(),
-                "co.aboutyou.de".into(),
-                "edited.de".into(),
-                "files.aboutstatic.com".into(),
-                "images.aboutstatic.com".into(),
-                "img.aboutstatic.com".into(),
-                "img.aboutyou.de".into(),
-                "m-assets.aboutyou.de".into(),
-                "m.aboutyou.de".into(),
-                "media.aboutyou.de".into(),
-                "static.aboutyou.de".into(),
-                "static1.aboutyou.de".into(),
-                "static2.aboutyou.de".into(),
-                "static3.aboutyou.de".into(),
-                "static4.aboutyou.de".into(),
-                "static5.aboutyou.de".into(),
-                "staticmail-cdn.aboutyou.de".into(),
-                "t.aboutyou.de".into(),
-                "witt-weiden.dam.acme.aboutyou.cloud".into(),
-                "witt-weiden.dam.staging.aboutyou.cloud".into(),
-                "www.aboutyou.de".into(),
-            ],
-            valid_emails: vec![],
-            valid_ipaddrs: vec![],
-        });
+        assert_eq!(
+            x,
+            Certificate {
+                valid_names: vec![
+                    "aboutyou.de".into(),
+                    "assets.aboutyou.de".into(),
+                    "cdn.aboutstatic.com".into(),
+                    "cdn.aboutyou-staging.de".into(),
+                    "cdn.aboutyou.de".into(),
+                    "cdn.edited.de".into(),
+                    "cdn.mary-paul.de".into(),
+                    "cdn.youandidol.de".into(),
+                    "cdn1.aboutyou.de".into(),
+                    "cdn2.aboutyou.de".into(),
+                    "cdn3.aboutyou.de".into(),
+                    "cdn4.aboutyou.de".into(),
+                    "cdn5.aboutyou.de".into(),
+                    "co-m.aboutyou.de".into(),
+                    "co-mapp.aboutyou.de".into(),
+                    "co-t.aboutyou.de".into(),
+                    "co.aboutyou.de".into(),
+                    "edited.de".into(),
+                    "files.aboutstatic.com".into(),
+                    "images.aboutstatic.com".into(),
+                    "img.aboutstatic.com".into(),
+                    "img.aboutyou.de".into(),
+                    "m-assets.aboutyou.de".into(),
+                    "m.aboutyou.de".into(),
+                    "media.aboutyou.de".into(),
+                    "static.aboutyou.de".into(),
+                    "static1.aboutyou.de".into(),
+                    "static2.aboutyou.de".into(),
+                    "static3.aboutyou.de".into(),
+                    "static4.aboutyou.de".into(),
+                    "static5.aboutyou.de".into(),
+                    "staticmail-cdn.aboutyou.de".into(),
+                    "t.aboutyou.de".into(),
+                    "witt-weiden.dam.acme.aboutyou.cloud".into(),
+                    "witt-weiden.dam.staging.aboutyou.cloud".into(),
+                    "www.aboutyou.de".into(),
+                ],
+                valid_emails: vec![],
+                valid_ipaddrs: vec![],
+            }
+        );
     }
 
     #[test]
     fn test_san_email() {
-        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+        let mut x = Certificate::parse_pem(
+            r#"-----BEGIN CERTIFICATE-----
 MIIE5zCCA8+gAwIBAgIQBvsKfZ5AGSW3Vc8Ldto1hTANBgkqhkiG9w0BAQUFADBp
 MSQwIgYJKoZIhvcNAQkBFhVwa2lfYWRtaW5Ac3VuZ2FyZC5jb20xJjAkBgNVBAoT
 HVN1bkdhcmQgQXZhaWxhYmlsaXR5IFNlcnZpY2VzMRkwFwYDVQQDExBTQVMgUHVi
@@ -331,18 +346,19 @@ K4f9GAOcawvNsI//mx99ol/ZGEamydeL9G0qiKrhqSxd2TGmFaIVJdu9fh59hos4
 UT+11L6q7MSIXSIMV8kJSUUYE92P7bnAqViTIuu/hHnfmIhiy6t7AuT2QHEhqDab
 EF4l5MwdUqs8FvM=
 -----END CERTIFICATE-----
-"#).expect("Failed to parse cert");
+"#,
+        )
+        .expect("Failed to parse cert");
         x.valid_names.sort();
         x.valid_emails.sort();
         x.valid_ipaddrs.sort();
-        assert_eq!(x, Certificate {
-            valid_names: vec![
-                "*.hosted.jivesoftware.com".into(),
-            ],
-            valid_emails: vec![
-                "subjectname@example.com".into(),
-            ],
-            valid_ipaddrs: vec![],
-        });
+        assert_eq!(
+            x,
+            Certificate {
+                valid_names: vec!["*.hosted.jivesoftware.com".into(),],
+                valid_emails: vec!["subjectname@example.com".into(),],
+                valid_ipaddrs: vec![],
+            }
+        );
     }
 }

sn0int-std/src/crypto.rs

@@ -17,12 +17,11 @@ pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
         bail!("Encrypted message is too short");
     }
 
-    let key = Key::from_slice(key)
-        .ok_or_else(|| format_err!("Key has wrong length"))?;
+    let key = Key::from_slice(key).ok_or_else(|| format_err!("Key has wrong length"))?;
     let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
         .ok_or_else(|| format_err!("Nonce has wrong length"))?;
     let ciphertext = &encrypted[secretbox::NONCEBYTES..];
-    let plain = secretbox::open(&ciphertext, &nonce, &key)
+    let plain = secretbox::open(ciphertext, &nonce, &key)
         .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
     Ok(plain)
 }

sn0int-std/src/engine/structs.rs

@@ -1,13 +1,13 @@
 use crate::errors::*;
-use crate::hlua::{AnyHashableLuaValue, AnyLuaValue, AnyLuaString};
-use std::collections::{self, HashMap};
+use crate::hlua::{AnyHashableLuaValue, AnyLuaString, AnyLuaValue};
 use crate::json::LuaJsonValue;
+use std::collections::{self, HashMap};
 
 pub fn from_lua<T>(x: LuaJsonValue) -> Result<T>
-    where for<'de> T: serde::Deserialize<'de>
+where
+    for<'de> T: serde::Deserialize<'de>,
 {
-    serde_json::from_value(x.into())
-        .map_err(Error::from)
+    serde_json::from_value(x.into()).map_err(Error::from)
 }
 
 #[derive(Debug, Default)]
@@ -26,22 +26,32 @@ impl LuaMap {
 
     #[inline]
     pub fn insert<K: Into<String>, V: Into<AnyLuaValue>>(&mut self, k: K, v: V) {
-        self.0.insert(AnyHashableLuaValue::LuaString(k.into()), v.into());
+        self.0
+            .insert(AnyHashableLuaValue::LuaString(k.into()), v.into());
     }
 
     #[inline]
     pub fn insert_str<K: Into<String>, V: Into<String>>(&mut self, k: K, v: V) {
-        self.0.insert(AnyHashableLuaValue::LuaString(k.into()), AnyLuaValue::LuaString(v.into()));
+        self.0.insert(
+            AnyHashableLuaValue::LuaString(k.into()),
+            AnyLuaValue::LuaString(v.into()),
+        );
     }
 
     #[inline]
     pub fn insert_num<K: Into<String>>(&mut self, k: K, v: f64) {
-        self.0.insert(AnyHashableLuaValue::LuaString(k.into()), AnyLuaValue::LuaNumber(v));
+        self.0.insert(
+            AnyHashableLuaValue::LuaString(k.into()),
+            AnyLuaValue::LuaNumber(v),
+        );
     }
 
     pub fn insert_serde<K: Into<String>, S: serde::Serialize>(&mut self, k: K, v: S) -> Result<()> {
         let v = serde_json::to_value(v)?;
-        self.0.insert(AnyHashableLuaValue::LuaString(k.into()), LuaJsonValue::from(v).into());
+        self.0.insert(
+            AnyHashableLuaValue::LuaString(k.into()),
+            LuaJsonValue::from(v).into(),
+        );
         Ok(())
     }
 }
@@ -86,23 +96,23 @@ impl From<Vec<(AnyLuaValue, AnyLuaValue)>> for LuaMap {
     }
 }
 
-impl Into<HashMap<AnyHashableLuaValue, AnyLuaValue>> for LuaMap {
-    fn into(self: LuaMap) -> HashMap<AnyHashableLuaValue, AnyLuaValue> {
-        self.0
+impl From<LuaMap> for HashMap<AnyHashableLuaValue, AnyLuaValue> {
+    fn from(map: LuaMap) -> Self {
+        map.0
     }
 }
 
-impl Into<AnyLuaValue> for LuaMap {
-    fn into(self: LuaMap) -> AnyLuaValue {
+impl From<LuaMap> for AnyLuaValue {
+    fn from(map: LuaMap) -> AnyLuaValue {
         AnyLuaValue::LuaArray(
-            self.into_iter()
+            map.into_iter()
                 .filter_map(|(k, v)| {
                     match k {
                         AnyHashableLuaValue::LuaString(x) => Some((AnyLuaValue::LuaString(x), v)),
                         _ => None, // TODO: unknown types are discarded
                     }
                 })
-                .collect()
+                .collect(),
         )
     }
 }
@@ -136,9 +146,9 @@ impl LuaList {
     }
 }
 
-impl Into<AnyLuaValue> for LuaList {
-    fn into(self: LuaList) -> AnyLuaValue {
-        AnyLuaValue::LuaArray(self.0)
+impl From<LuaList> for AnyLuaValue {
+    fn from(list: LuaList) -> AnyLuaValue {
+        AnyLuaValue::LuaArray(list.0)
     }
 }
 
@@ -146,17 +156,20 @@ pub fn byte_array(bytes: AnyLuaValue) -> Result<Vec<u8>> {
     match bytes {
         AnyLuaValue::LuaAnyString(bytes) => Ok(bytes.0),
         AnyLuaValue::LuaString(bytes) => Ok(bytes.into_bytes()),
-        AnyLuaValue::LuaArray(bytes) => {
-            Ok(bytes.into_iter()
-                .map(|num| match num.1 {
-                    AnyLuaValue::LuaNumber(num) if num <= 255.0 && num >= 0.0 && (num % 1.0 == 0.0) =>
-                            Ok(num as u8),
-                    AnyLuaValue::LuaNumber(num) =>
-                            Err(format_err!("number is out of range: {:?}", num)),
-                    _ => Err(format_err!("unexpected type: {:?}", num)),
-                })
-                .collect::<Result<_>>()?)
-        },
+        AnyLuaValue::LuaArray(bytes) => Ok(bytes
+            .into_iter()
+            .map(|num| match num.1 {
+                AnyLuaValue::LuaNumber(num)
+                    if (0.0..=255.0).contains(&num) && (num % 1.0 == 0.0) =>
+                {
+                    Ok(num as u8)
+                }
+                AnyLuaValue::LuaNumber(num) => {
+                    Err(format_err!("number is out of range: {:?}", num))
+                }
+                _ => Err(format_err!("unexpected type: {:?}", num)),
+            })
+            .collect::<Result<_>>()?),
         _ => Err(format_err!("invalid type: {:?}", bytes)),
     }
 }

sn0int-std/src/errors.rs

@@ -1,3 +1,3 @@
-pub use log::{trace, debug, info, warn, error};
-pub use failure::{Error, ResultExt, format_err, bail};
+pub use failure::{bail, format_err, Error, ResultExt};
+pub use log::{debug, error, info, trace, warn};
 pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-std/src/geo.rs

@@ -1,8 +1,8 @@
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
-use geo::{LineString, Polygon, Coordinate};
 use geo::prelude::*;
+use geo::{Coord, LineString, Polygon};
 use serde::Deserialize;
 
 #[derive(Debug, Deserialize)]
@@ -20,8 +20,9 @@ impl Point {
 }
 
 pub fn polygon_contains(ring: &[Point], p: &Point) -> bool {
-    let ring = ring.iter()
-        .map(|p| Coordinate { x: p.lon, y: p.lat })
+    let ring = ring
+        .iter()
+        .map(|p| Coord { x: p.lon, y: p.lat })
         .collect::<Vec<_>>();
 
     let polygon = Polygon::new(LineString::from(ring), vec![]);
@@ -36,75 +37,210 @@ mod tests {
 
     fn hamburg_polygon() -> &'static [Point] {
         &[
-            Point { lat: 53.63975308945899,  lon: 9.764785766601562  },
-            Point { lat: 53.59494998253459,  lon: 9.827270507812     },
-            Point { lat: 53.663153974456456, lon: 9.9151611328125    },
-            Point { lat: 53.65582987649682,  lon: 9.976272583007812  },
-            Point { lat: 53.68613523817129,  lon: 9.992752075195312  },
-            Point { lat: 53.68674518938816,  lon: 10.051460266113281 },
-            Point { lat: 53.72495117617815,  lon: 10.075492858886719 },
-            Point { lat: 53.71946627930625,  lon: 10.118408203125    },
-            Point { lat: 53.743635083157756, lon: 10.164413452148438 },
-            Point { lat: 53.73104466704585,  lon: 10.202865600585938 },
-            Point { lat: 53.676781546441546, lon: 10.16304016113281  },
-            Point { lat: 53.632832079199474, lon: 10.235824584960938 },
-            Point { lat: 53.608803292930894, lon: 10.2008056640625   },
-            Point { lat: 53.578646152866504, lon: 10.208358764648438 },
-            Point { lat: 53.57212285981298,  lon: 10.163726806640625 },
-            Point { lat: 53.52071674896369,  lon: 10.18707275390625  },
-            Point { lat: 53.52643162253097,  lon: 10.224151611328125 },
-            Point { lat: 53.44062753992289,  lon: 10.347747802734375 },
-            Point { lat: 53.38824275010831,  lon: 10.248870849609375 },
-            Point { lat: 53.38824275010831,  lon: 10.15960693359375  },
-            Point { lat: 53.44635321212876,  lon: 10.064849853515625 },
-            Point { lat: 53.40595029739904,  lon: 9.985198974609375  },
-            Point { lat: 53.42385506057106,  lon: 9.951210021972656  },
-            Point { lat: 53.41843327091211,  lon: 9.944171905517578  },
-            Point { lat: 53.41812635648326,  lon: 9.927349090576172  },
-            Point { lat: 53.412294561442884, lon: 9.917736053466797  },
-            Point { lat: 53.41464783813818,  lon: 9.901256561279297  },
-            Point { lat: 53.443490472483326, lon: 9.912586212158201  },
-            Point { lat: 53.45177144115704,  lon: 9.897651672363281  },
-            Point { lat: 53.43633277935392,  lon: 9.866924285888672  },
-            Point { lat: 53.427639673754776, lon: 9.866409301757812  },
-            Point { lat: 53.427639673754776, lon: 9.858856201171875  },
-            Point { lat: 53.46710230573499,  lon: 9.795513153076172  },
-            Point { lat: 53.49039461941655,  lon: 9.795341491699219  },
-            Point { lat: 53.49029248806277,  lon: 9.77903366088867   },
-            Point { lat: 53.49856433088649,  lon: 9.780235290527344  },
-            Point { lat: 53.5078554643033,   lon: 9.758434295654297  },
-            Point { lat: 53.545407634092975, lon: 9.759807586669922  },
-            Point { lat: 53.568147234570084, lon: 9.633293151855469  },
-            Point { lat: 53.58802162343514,  lon: 9.655780792236328  },
-            Point { lat: 53.568351121879815, lon: 9.727706909179688  },
-            Point { lat: 53.60921067445695,  lon: 9.737663269042969  },
+            Point {
+                lat: 53.63975308945899,
+                lon: 9.764785766601562,
+            },
+            Point {
+                lat: 53.59494998253459,
+                lon: 9.827270507812,
+            },
+            Point {
+                lat: 53.663153974456456,
+                lon: 9.9151611328125,
+            },
+            Point {
+                lat: 53.65582987649682,
+                lon: 9.976272583007812,
+            },
+            Point {
+                lat: 53.68613523817129,
+                lon: 9.992752075195312,
+            },
+            Point {
+                lat: 53.68674518938816,
+                lon: 10.051460266113281,
+            },
+            Point {
+                lat: 53.72495117617815,
+                lon: 10.075492858886719,
+            },
+            Point {
+                lat: 53.71946627930625,
+                lon: 10.118408203125,
+            },
+            Point {
+                lat: 53.743635083157756,
+                lon: 10.164413452148438,
+            },
+            Point {
+                lat: 53.73104466704585,
+                lon: 10.202865600585938,
+            },
+            Point {
+                lat: 53.676781546441546,
+                lon: 10.16304016113281,
+            },
+            Point {
+                lat: 53.632832079199474,
+                lon: 10.235824584960938,
+            },
+            Point {
+                lat: 53.608803292930894,
+                lon: 10.2008056640625,
+            },
+            Point {
+                lat: 53.578646152866504,
+                lon: 10.208358764648438,
+            },
+            Point {
+                lat: 53.57212285981298,
+                lon: 10.163726806640625,
+            },
+            Point {
+                lat: 53.52071674896369,
+                lon: 10.18707275390625,
+            },
+            Point {
+                lat: 53.52643162253097,
+                lon: 10.224151611328125,
+            },
+            Point {
+                lat: 53.44062753992289,
+                lon: 10.347747802734375,
+            },
+            Point {
+                lat: 53.38824275010831,
+                lon: 10.248870849609375,
+            },
+            Point {
+                lat: 53.38824275010831,
+                lon: 10.15960693359375,
+            },
+            Point {
+                lat: 53.44635321212876,
+                lon: 10.064849853515625,
+            },
+            Point {
+                lat: 53.40595029739904,
+                lon: 9.985198974609375,
+            },
+            Point {
+                lat: 53.42385506057106,
+                lon: 9.951210021972656,
+            },
+            Point {
+                lat: 53.41843327091211,
+                lon: 9.944171905517578,
+            },
+            Point {
+                lat: 53.41812635648326,
+                lon: 9.927349090576172,
+            },
+            Point {
+                lat: 53.412294561442884,
+                lon: 9.917736053466797,
+            },
+            Point {
+                lat: 53.41464783813818,
+                lon: 9.901256561279297,
+            },
+            Point {
+                lat: 53.443490472483326,
+                lon: 9.912586212158201,
+            },
+            Point {
+                lat: 53.45177144115704,
+                lon: 9.897651672363281,
+            },
+            Point {
+                lat: 53.43633277935392,
+                lon: 9.866924285888672,
+            },
+            Point {
+                lat: 53.427639673754776,
+                lon: 9.866409301757812,
+            },
+            Point {
+                lat: 53.427639673754776,
+                lon: 9.858856201171875,
+            },
+            Point {
+                lat: 53.46710230573499,
+                lon: 9.795513153076172,
+            },
+            Point {
+                lat: 53.49039461941655,
+                lon: 9.795341491699219,
+            },
+            Point {
+                lat: 53.49029248806277,
+                lon: 9.77903366088867,
+            },
+            Point {
+                lat: 53.49856433088649,
+                lon: 9.780235290527344,
+            },
+            Point {
+                lat: 53.5078554643033,
+                lon: 9.758434295654297,
+            },
+            Point {
+                lat: 53.545407634092975,
+                lon: 9.759807586669922,
+            },
+            Point {
+                lat: 53.568147234570084,
+                lon: 9.633293151855469,
+            },
+            Point {
+                lat: 53.58802162343514,
+                lon: 9.655780792236328,
+            },
+            Point {
+                lat: 53.568351121879815,
+                lon: 9.727706909179688,
+            },
+            Point {
+                lat: 53.60921067445695,
+                lon: 9.737663269042969,
+            },
         ]
     }
 
     #[test]
     fn test_polygon_hamburg_contains_hamburg() {
-        let contains = polygon_contains(hamburg_polygon(), &Point {
-            lat: 53.551085,
-            lon: 9.993682,
-        });
+        let contains = polygon_contains(
+            hamburg_polygon(),
+            &Point {
+                lat: 53.551085,
+                lon: 9.993682,
+            },
+        );
         assert!(contains);
     }
 
     #[test]
     fn test_polygon_hamburg_not_contains_berlin() {
-        let contains = polygon_contains(hamburg_polygon(), &Point {
-            lat: 52.52437,
-            lon: 13.41053,
-        });
+        let contains = polygon_contains(
+            hamburg_polygon(),
+            &Point {
+                lat: 52.52437,
+                lon: 13.41053,
+            },
+        );
         assert!(!contains);
     }
 
     #[test]
     fn test_polygon_hamburg_not_contains_ny() {
-        let contains = polygon_contains(hamburg_polygon(), &Point {
-            lat: 40.726662,
-            lon: -74.036677,
-        });
+        let contains = polygon_contains(
+            hamburg_polygon(),
+            &Point {
+                lat: 40.726662,
+                lon: -74.036677,
+            },
+        );
         assert!(!contains);
     }
 }

sn0int-std/src/geoip/mod.rs

@@ -3,15 +3,14 @@ use crate::lazy::LazyInit;
 use maxminddb::{self, geoip2};
 use std::fmt;
 use std::fs::{self, File};
+use std::io::Read;
 use std::net::IpAddr;
 use std::path::{Path, PathBuf};
-use std::io::Read;
 use std::sync::Arc;
 
 pub mod models;
-use self::models::GeoLookup;
 use self::models::AsnLookup;
-
+use self::models::GeoLookup;
 
 pub trait Maxmind: Sized {
     fn filename() -> &'static str;
@@ -37,14 +36,13 @@ pub trait Maxmind: Sized {
         }
 
         // use cache path
-        let path = cache_dir
-            .join(Self::filename());
+        let path = cache_dir.join(Self::filename());
         Ok(path)
     }
 
     fn from_buf(buf: Vec<u8>) -> Result<Self> {
-        let reader = maxminddb::Reader::from_source(buf)
-            .context("Failed to read geoip database")?;
+        let reader =
+            maxminddb::Reader::from_source(buf).context("Failed to read geoip database")?;
         Ok(Self::new(reader))
     }
 
@@ -72,9 +70,7 @@ pub struct MaxmindReader {
 impl MaxmindReader {
     fn open_path<P: AsRef<Path>>(path: P) -> Result<MaxmindReader> {
         let reader = File::open(path)?;
-        Ok(MaxmindReader {
-            reader,
-        })
+        Ok(MaxmindReader { reader })
     }
 }
 
@@ -118,9 +114,7 @@ impl Maxmind for GeoIP {
 
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
-        GeoIP {
-            reader
-        }
+        GeoIP { reader }
     }
 }
 
@@ -150,9 +144,7 @@ impl Maxmind for AsnDB {
 
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
-        AsnDB {
-            reader
-        }
+        AsnDB { reader }
     }
 }
 

sn0int-std/src/geoip/models.rs

@@ -72,7 +72,7 @@ pub struct Continent {
 }
 
 impl Continent {
-    fn from_maxmind(continent: geoip2::model::Continent) -> Option<Self> {
+    fn from_maxmind(continent: geoip2::city::Continent) -> Option<Self> {
         let code = continent.code?;
         let name = from_geoip_model_names(continent.names)?;
 
@@ -90,7 +90,7 @@ pub struct Country {
 }
 
 impl Country {
-    fn from_maxmind(country: geoip2::model::Country) -> Option<Self> {
+    fn from_maxmind(country: geoip2::city::Country) -> Option<Self> {
         let code = country.iso_code?;
         let name = from_geoip_model_names(country.names)?;
 
@@ -108,7 +108,7 @@ pub struct Location {
 }
 
 impl Location {
-    fn from_maxmind(location: &geoip2::model::Location) -> Option<Self> {
+    fn from_maxmind(location: &geoip2::city::Location) -> Option<Self> {
         let latitude = match location.latitude {
             Some(latitude) => latitude,
             _ => return None,
@@ -134,9 +134,11 @@ pub struct AsnLookup {
 impl AsnLookup {
     pub fn try_from(lookup: geoip2::Isp) -> Result<AsnLookup> {
         // parse maxminddb lookup
-        let asn = lookup.autonomous_system_number
+        let asn = lookup
+            .autonomous_system_number
             .ok_or_else(|| format_err!("autonomous_system_number not set"))?;
-        let as_org = lookup.autonomous_system_organization
+        let as_org = lookup
+            .autonomous_system_organization
             .ok_or_else(|| format_err!("autonomous_system_organization not set"))?;
 
         Ok(AsnLookup {

sn0int-std/src/gfx/exif.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use serde::{Serialize, Deserialize};
+use serde::{Deserialize, Serialize};
 use std::io;
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
@@ -9,7 +9,7 @@ pub struct Location {
 }
 
 impl Location {
-    fn try_from_iter<'a, I: IntoIterator<Item=&'a exif::Field>>(iter: I) -> Result<Self> {
+    fn try_from_iter<'a, I: IntoIterator<Item = &'a exif::Field>>(iter: I) -> Result<Self> {
         let mut builder = LocationBuilder::default();
         for f in iter {
             debug!("Exif field: {:?}", f.display_value().to_string());
@@ -31,8 +31,12 @@ impl LocationBuilder {
     fn add_one(&mut self, f: &exif::Field) -> Result<()> {
         debug!("Exif tag: {:?}, {}", f.tag, f.value.display_as(f.tag));
         match f.tag {
-            exif::Tag::GPSLatitudeRef => self.latitude_ref = Some(cardinal_direction_modifier(&f.value)?),
-            exif::Tag::GPSLongitudeRef => self.longitude_ref = Some(cardinal_direction_modifier(&f.value)?),
+            exif::Tag::GPSLatitudeRef => {
+                self.latitude_ref = Some(cardinal_direction_modifier(&f.value)?)
+            }
+            exif::Tag::GPSLongitudeRef => {
+                self.longitude_ref = Some(cardinal_direction_modifier(&f.value)?)
+            }
             exif::Tag::GPSLatitude => self.latitude = Some(dms_to_float(&f.value)?),
             exif::Tag::GPSLongitude => self.longitude = Some(dms_to_float(&f.value)?),
             _ => (),
@@ -41,14 +45,18 @@ impl LocationBuilder {
     }
 
     fn build(self) -> Result<Location> {
-        let latitude = self.latitude
+        let latitude = self
+            .latitude
             .ok_or_else(|| format_err!("Missing latitude field"))?;
-        let latitude_ref = self.latitude_ref
+        let latitude_ref = self
+            .latitude_ref
             .ok_or_else(|| format_err!("Missing latitude field"))?;
 
-        let longitude = self.longitude
+        let longitude = self
+            .longitude
             .ok_or_else(|| format_err!("Missing latitude field"))?;
-        let longitude_ref = self.longitude_ref
+        let longitude_ref = self
+            .longitude_ref
             .ok_or_else(|| format_err!("Missing latitude field"))?;
 
         Ok(Location {
@@ -60,8 +68,7 @@ impl LocationBuilder {
 
 pub fn gps(img: &[u8]) -> Result<Option<Location>> {
     let mut buf = io::Cursor::new(img);
-    let reader = exif::Reader::new()
-        .read_from_container(&mut buf)?;
+    let reader = exif::Reader::new().read_from_container(&mut buf)?;
     let fields = reader.fields();
 
     let location = Location::try_from_iter(fields).ok();
@@ -82,7 +89,7 @@ pub fn dms_to_float(dms: &exif::Value) -> Result<f64> {
     let minutes = dms[1].to_f64();
     let seconds = dms[2].to_f64();
 
-    let float = degrees + minutes/60.0 + seconds/3600.0;
+    let float = degrees + minutes / 60.0 + seconds / 3600.0;
     let float = (float * 1000000.0).round() / 1000000.0;
     Ok(float)
 }
@@ -90,17 +97,18 @@ pub fn dms_to_float(dms: &exif::Value) -> Result<f64> {
 pub fn cardinal_direction_modifier(value: &exif::Value) -> Result<f64> {
     match value {
         exif::Value::Ascii(s) => {
-            let s = s.get(0)
+            let s = s
+                .get(0)
                 .ok_or_else(|| format_err!("Cardinal direction value is empty"))?;
 
-            match s.get(0) {
+            match s.first() {
                 Some(b'N') => Ok(1.0),
                 Some(b'S') => Ok(-1.0),
                 Some(b'E') => Ok(1.0),
                 Some(b'W') => Ok(-1.0),
                 _ => bail!("Unexpected cardinal direction"),
             }
-        },
+        }
         _ => bail!("Unexpected exif value"),
     }
 }
@@ -119,60 +127,62 @@ mod tests {
                 tag: exif::Tag::GPSLatitudeRef,
                 ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Ascii(vec![vec![b'N']]),
-            }, exif::Field {
+            },
+            exif::Field {
                 tag: exif::Tag::GPSLongitudeRef,
                 ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Ascii(vec![vec![b'E']]),
-            }, exif::Field {
+            },
+            exif::Field {
                 tag: exif::Tag::GPSLatitude,
                 ifd_num: exif::In::PRIMARY,
-                value: exif::Value::Rational(vec![exif::Rational {
-                    num: 43,
-                    denom: 1,
-                }, exif::Rational {
-                    num: 28,
-                    denom: 1,
-                }, exif::Rational {
-                    num: 176399999,
-                    denom: 100000000,
-                }]),
-            }, exif::Field {
+                value: exif::Value::Rational(vec![
+                    exif::Rational { num: 43, denom: 1 },
+                    exif::Rational { num: 28, denom: 1 },
+                    exif::Rational {
+                        num: 176399999,
+                        denom: 100000000,
+                    },
+                ]),
+            },
+            exif::Field {
                 tag: exif::Tag::GPSLongitude,
                 ifd_num: exif::In::PRIMARY,
-                value: exif::Value::Rational(vec![exif::Rational {
-                    num: 11,
-                    denom: 1,
-                }, exif::Rational {
-                    num: 53,
-                    denom: 1,
-                }, exif::Rational {
-                    num: 742199999,
-                    denom: 100000000,
-                }]),
+                value: exif::Value::Rational(vec![
+                    exif::Rational { num: 11, denom: 1 },
+                    exif::Rational { num: 53, denom: 1 },
+                    exif::Rational {
+                        num: 742199999,
+                        denom: 100000000,
+                    },
+                ]),
             },
-        ]).unwrap();
+        ])
+        .unwrap();
         println!("{:?}", location);
 
-        assert_eq!(location, Location {
-            latitude: 43.467157,
-            longitude: 11.885395
-        });
+        assert_eq!(
+            location,
+            Location {
+                latitude: 43.467157,
+                longitude: 11.885395
+            }
+        );
     }
 
     #[test]
     fn verify_dms() {
         test_init();
 
-        let latitude = dms_to_float(&exif::Value::Rational(vec![exif::Rational {
-            num: 43,
-            denom: 1,
-        }, exif::Rational {
-            num: 28,
-            denom: 1,
-        }, exif::Rational {
-            num: 176399999,
-            denom: 100000000,
-        }])).unwrap();
+        let latitude = dms_to_float(&exif::Value::Rational(vec![
+            exif::Rational { num: 43, denom: 1 },
+            exif::Rational { num: 28, denom: 1 },
+            exif::Rational {
+                num: 176399999,
+                denom: 100000000,
+            },
+        ]))
+        .unwrap();
 
         assert_eq!(latitude, 43.467157);
     }

sn0int-std/src/gfx/mod.rs

@@ -1,9 +1,9 @@
 use crate::errors::*;
 use image::{self, DynamicImage, GenericImageView};
+pub use img_hash_median::HashAlg;
 
 pub mod exif;
 
-
 #[derive(Debug)]
 pub enum ImageFormat {
     Png,
@@ -45,9 +45,9 @@ impl ImageFormat {
     }
 }
 
-impl Into<image::ImageFormat> for ImageFormat {
-    fn into(self) -> image::ImageFormat {
-        match self {
+impl From<ImageFormat> for image::ImageFormat {
+    fn from(format: ImageFormat) -> image::ImageFormat {
+        match format {
             ImageFormat::Png => image::ImageFormat::Png,
             ImageFormat::Jpeg => image::ImageFormat::Jpeg,
             ImageFormat::Gif => image::ImageFormat::Gif,
@@ -79,6 +79,14 @@ impl Image {
     pub fn height(&self) -> u32 {
         self.image.height()
     }
+
+    pub fn perception_hash(&self, hash_alg: HashAlg) -> String {
+        let hasher = img_hash_median::HasherConfig::new()
+            .hash_alg(hash_alg)
+            .to_hasher();
+        let hash = hasher.hash_image(&self.image);
+        hash.to_base64()
+    }
 }
 
 impl AsRef<DynamicImage> for Image {
@@ -94,18 +102,14 @@ pub fn guess_format(buf: &[u8]) -> Result<ImageFormat> {
 }
 
 pub fn load(buf: &[u8]) -> Result<Image> {
-    let img_format = image::guess_format(&buf)?;
+    let img_format = image::guess_format(buf)?;
     let format = ImageFormat::try_from(&img_format)?;
 
-    let image = image::load_from_memory_with_format(&buf, img_format)?;
+    let image = image::load_from_memory_with_format(buf, img_format)?;
 
-    Ok(Image {
-        image,
-        format,
-    })
+    Ok(Image { image, format })
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;

sn0int-std/src/html.rs

@@ -11,13 +11,13 @@ pub struct Element {
     pub html: String,
 }
 
-impl Into<AnyLuaValue> for Element {
-    fn into(self) -> AnyLuaValue {
+impl From<Element> for AnyLuaValue {
+    fn from(elem: Element) -> AnyLuaValue {
         let mut map = LuaMap::new();
 
-        map.insert_str("text", self.text);
-        map.insert("attrs", LuaMap::from(self.attrs));
-        map.insert_str("html", self.html);
+        map.insert_str("text", elem.text);
+        map.insert("attrs", LuaMap::from(elem.attrs));
+        map.insert_str("html", elem.html);
 
         map.into()
     }
@@ -41,14 +41,10 @@ fn transform_element(entry: &kuchiki::NodeDataRef<kuchiki::ElementData>) -> Elem
         Err(_) => {
             debug!("html serialize failed");
             String::new()
-        },
+        }
     };
 
-    Element {
-        attrs,
-        text,
-        html,
-    }
+    Element { attrs, text, html }
 }
 
 pub fn html_select(html: &str, selector: &str) -> Result<Element> {
@@ -95,7 +91,6 @@ pub fn html_form(html: &str) -> Result<HashMap<String, String>> {
     Ok(form)
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -104,9 +99,10 @@ mod tests {
     #[test]
     fn test_html_select() {
         let elems = html_select(r#"<html><div id="yey">content</div></html>"#, "#yey").unwrap();
-        assert_eq!(elems,
+        assert_eq!(
+            elems,
             Element {
-                attrs: hashmap!{
+                attrs: hashmap! {
                     "id".into() => "yey".into(),
                 },
                 text: "content".into(),
@@ -117,15 +113,17 @@ mod tests {
 
     #[test]
     fn test_html_select_list() {
-        let elems = html_select_list(r#"<html><div id="yey">content</div></html>"#, "#yey").unwrap();
-        assert_eq!(elems, vec![
-            Element {
-                attrs: hashmap!{
+        let elems =
+            html_select_list(r#"<html><div id="yey">content</div></html>"#, "#yey").unwrap();
+        assert_eq!(
+            elems,
+            vec![Element {
+                attrs: hashmap! {
                     "id".into() => "yey".into(),
                 },
                 text: "content".into(),
                 html: r#"<div id="yey">content</div>"#.into(),
-            }
-        ]);
+            }]
+        );
     }
 }

sn0int-std/src/json.rs

@@ -1,14 +1,11 @@
 use crate::errors::*;
 
-use std::iter::FromIterator;
-use std::collections::HashMap;
 use crate::hlua::AnyLuaValue;
-use serde_json::{self, Deserializer, Value, Number, Map};
-
+use serde_json::{self, Deserializer, Map, Number, Value};
+use std::collections::HashMap;
 
 pub fn decode<T: AsRef<[u8]>>(x: T) -> Result<AnyLuaValue> {
-    let v: Value = serde_json::from_slice(x.as_ref())
-                        .context("deserialize failed")?;
+    let v: Value = serde_json::from_slice(x.as_ref()).context("deserialize failed")?;
     let v: LuaJsonValue = v.into();
     Ok(v.into())
 }
@@ -16,13 +13,12 @@ pub fn decode<T: AsRef<[u8]>>(x: T) -> Result<AnyLuaValue> {
 pub fn encode(v: AnyLuaValue) -> Result<String> {
     let v: LuaJsonValue = v.into();
     let v: Value = v.into();
-    let s = serde_json::to_string(&v)
-        .context("serialize failed")?;
+    let s = serde_json::to_string(&v).context("serialize failed")?;
     Ok(s)
 }
 
 pub fn decode_stream(x: &str) -> Result<Vec<AnyLuaValue>> {
-    let stream = Deserializer::from_str(&x).into_iter::<Value>();
+    let stream = Deserializer::from_str(x).into_iter::<Value>();
 
     let list = stream
         .filter_map(|x| x.ok())
@@ -52,21 +48,24 @@ pub enum LuaJsonValue {
     Object(HashMap<String, LuaJsonValue>),
 }
 
-impl Into<AnyLuaValue> for LuaJsonValue {
-    fn into(self) -> AnyLuaValue {
-        match self {
+impl From<LuaJsonValue> for AnyLuaValue {
+    fn from(value: LuaJsonValue) -> AnyLuaValue {
+        match value {
             LuaJsonValue::Null => AnyLuaValue::LuaNil,
             LuaJsonValue::Bool(v) => AnyLuaValue::LuaBoolean(v),
             // TODO: not sure if this might fail
             LuaJsonValue::Number(v) => AnyLuaValue::LuaNumber(v.as_f64().unwrap()),
             LuaJsonValue::String(v) => AnyLuaValue::LuaString(v),
-            LuaJsonValue::Array(v) => AnyLuaValue::LuaArray(v.into_iter().enumerate()
-                .map(|(i, x)| (AnyLuaValue::LuaNumber((i+1) as f64), x.into()))
-                .collect()
+            LuaJsonValue::Array(v) => AnyLuaValue::LuaArray(
+                v.into_iter()
+                    .enumerate()
+                    .map(|(i, x)| (AnyLuaValue::LuaNumber((i + 1) as f64), x.into()))
+                    .collect(),
             ),
-            LuaJsonValue::Object(v) => AnyLuaValue::LuaArray(v.into_iter()
-                .map(|(k, v)| (AnyLuaValue::LuaString(k), v.into()))
-                .collect()
+            LuaJsonValue::Object(v) => AnyLuaValue::LuaArray(
+                v.into_iter()
+                    .map(|(k, v)| (AnyLuaValue::LuaString(k), v.into()))
+                    .collect(),
             ),
         }
     }
@@ -78,9 +77,10 @@ impl From<AnyLuaValue> for LuaJsonValue {
             AnyLuaValue::LuaNil => LuaJsonValue::Null,
             AnyLuaValue::LuaBoolean(v) => LuaJsonValue::Bool(v),
             AnyLuaValue::LuaString(v) => LuaJsonValue::String(v),
-            AnyLuaValue::LuaAnyString(v) => LuaJsonValue::Array(v.0.into_iter()
-                .map(|x| LuaJsonValue::Number(x.into()))
-                .collect()
+            AnyLuaValue::LuaAnyString(v) => LuaJsonValue::Array(
+                v.0.into_iter()
+                    .map(|x| LuaJsonValue::Number(x.into()))
+                    .collect(),
             ),
             AnyLuaValue::LuaNumber(v) => {
                 // this is needed or every number is detected as float
@@ -89,42 +89,39 @@ impl From<AnyLuaValue> for LuaJsonValue {
                 } else {
                     Number::from_f64(v).expect("invalid LuaJson::Number")
                 })
-            },
+            }
             AnyLuaValue::LuaArray(v) => {
                 if lua_array_is_list(&v) {
-                    LuaJsonValue::Array(v.into_iter()
-                        .map(|(_, v)| v.into())
-                        .collect()
-                    )
+                    LuaJsonValue::Array(v.into_iter().map(|(_, v)| v.into()).collect())
                 } else {
-                    LuaJsonValue::Object(v.into_iter()
-                        .filter_map(|(k, v)| match k {
-                            AnyLuaValue::LuaString(k) => Some((k, v.into())),
-                            _ => None,
-                        })
-                        .collect()
+                    LuaJsonValue::Object(
+                        v.into_iter()
+                            .filter_map(|(k, v)| match k {
+                                AnyLuaValue::LuaString(k) => Some((k, v.into())),
+                                _ => None,
+                            })
+                            .collect(),
                     )
                 }
-            },
+            }
             AnyLuaValue::LuaOther => LuaJsonValue::Null,
         }
     }
 }
 
-impl Into<serde_json::Value> for LuaJsonValue {
-    fn into(self) -> serde_json::Value {
-        match self {
+impl From<LuaJsonValue> for serde_json::Value {
+    fn from(value: LuaJsonValue) -> serde_json::Value {
+        match value {
             LuaJsonValue::Null => Value::Null,
             LuaJsonValue::Bool(v) => Value::Bool(v),
             LuaJsonValue::Number(v) => Value::Number(v),
             LuaJsonValue::String(v) => Value::String(v),
-            LuaJsonValue::Array(v) => Value::Array(v.into_iter()
-                .map(|x| x.into())
-                .collect()
+            LuaJsonValue::Array(v) => Value::Array(v.into_iter().map(|x| x.into()).collect()),
+            LuaJsonValue::Object(v) => Value::Object(
+                v.into_iter()
+                    .map(|(k, v)| (k, v.into()))
+                    .collect::<Map<_, _>>(),
             ),
-            LuaJsonValue::Object(v) => Value::Object(Map::from_iter(v.into_iter()
-                .map(|(k, v)| (k, v.into()))
-            )),
         }
     }
 }
@@ -136,13 +133,12 @@ impl From<serde_json::Value> for LuaJsonValue {
             Value::Bool(v) => LuaJsonValue::Bool(v),
             Value::Number(v) => LuaJsonValue::Number(v),
             Value::String(v) => LuaJsonValue::String(v),
-            Value::Array(v) => LuaJsonValue::Array(v.into_iter()
-                .map(|x| x.into())
-                .collect()
+            Value::Array(v) => LuaJsonValue::Array(v.into_iter().map(|x| x.into()).collect()),
+            Value::Object(v) => LuaJsonValue::Object(
+                v.into_iter()
+                    .map(|(k, v)| (k, v.into()))
+                    .collect::<HashMap<_, _>>(),
             ),
-            Value::Object(v) => LuaJsonValue::Object(HashMap::from_iter(v.into_iter()
-                .map(|(k, v)| (k, v.into()))
-            )),
         }
     }
 }

sn0int-std/src/lazy.rs

@@ -13,11 +13,12 @@ impl<T1: LazyInit<T2>, T2> Lazy<T1, T2> {
     pub fn get(&mut self) -> Result<&mut T2> {
         match self {
             Lazy::Init(init) => {
-                let init = init.take()
+                let init = init
+                    .take()
                     .ok_or_else(|| format_err!("Previous initialization failed"))?;
                 *self = Lazy::Active(init.initialize()?);
                 self.get()
-            },
+            }
             Lazy::Active(active) => Ok(active),
         }
     }

sn0int-std/src/lib.rs

@@ -3,8 +3,8 @@ use hlua_badtouch as hlua;
 pub mod blobs;
 pub mod crt;
 pub mod crypto;
-mod errors;
 pub mod engine;
+mod errors;
 pub mod geo;
 pub mod geoip;
 pub mod gfx;

sn0int-std/src/mqtt.rs

@@ -1,20 +1,26 @@
-use chrootable_https::DnsResolver;
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
-use mqtt::packet::VariablePacketError;
 use crate::json::LuaJsonValue;
-use crate::sockets::{Stream, SocketOptions};
-use mqtt::{TopicFilter, QualityOfService};
-use mqtt::control::ConnectReturnCode;
+use crate::sockets::{SocketOptions, Stream};
+use chrootable_https::DnsResolver;
 use mqtt::control::fixed_header::FixedHeaderError;
-use mqtt::encodable::{Encodable, Decodable};
-use mqtt::packet::{Packet, VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
-use serde::{Serialize, Deserialize};
+use mqtt::control::ConnectReturnCode;
+use mqtt::encodable::{Decodable, Encodable};
+use mqtt::packet::VariablePacketError;
+use mqtt::packet::{ConnectPacket, PingreqPacket, SubscribePacket, VariablePacket};
+use mqtt::{QualityOfService, TopicFilter};
+use serde::{Deserialize, Serialize};
 use std::convert::TryFrom;
 use std::io;
 use std::net::SocketAddr;
+use std::time::{Duration, Instant};
 use url::Url;
 
+// a reasonable default for keep-alive
+// some servers reject 0 as invalid with a very confusing error message
+const DEFAULT_PING_INTERVAL: u64 = 90;
+const DEFAULT_KEEP_ALIVE: u16 = 120;
+
 #[derive(Debug, Default, Deserialize)]
 pub struct MqttOptions {
     pub username: Option<String>,
@@ -23,10 +29,12 @@ pub struct MqttOptions {
     pub proxy: Option<SocketAddr>,
     #[serde(default)]
     pub connect_timeout: u64,
-    #[serde(default)]
-    pub read_timeout: u64,
+    pub read_timeout: Option<u64>,
     #[serde(default)]
     pub write_timeout: u64,
+
+    pub ping_interval: Option<u64>,
+    pub keep_alive: Option<u16>,
 }
 
 impl MqttOptions {
@@ -37,25 +45,44 @@ impl MqttOptions {
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum MqttRecvError {
+    #[error("Failed to read mqtt packet: {0:#}")]
+    Recv(#[from] VariablePacketError),
+    #[error("Failed to read mqtt packet: connection disconnected")]
+    RecvDisconnect,
+    #[error("Failed to interact with mqtt: {0:#}")]
+    Error(Error),
+}
+
+impl From<Error> for MqttRecvError {
+    fn from(err: Error) -> Self {
+        MqttRecvError::Error(err)
+    }
+}
+
 pub struct MqttClient {
     stream: Stream,
+    last_ping: Instant,
+    ping_interval: Option<u64>,
 }
 
 impl MqttClient {
     pub fn negotiate(stream: Stream, options: &MqttOptions) -> Result<MqttClient> {
+        // default to DEFAULT_PING_INTERVAL, if an explicit value of 0 was set, disable auto-ping
+        let ping_interval = Some(options.ping_interval.unwrap_or(DEFAULT_PING_INTERVAL));
+        ping_interval.filter(|s| *s != 0);
+
         let mut client = MqttClient {
             stream,
+            last_ping: Instant::now(),
+            ping_interval,
         };
 
         let mut pkt = ConnectPacket::new("sn0int");
         pkt.set_user_name(options.username.clone());
         pkt.set_password(options.password.clone());
-
-        /*
-        if let Some(keep_alive) = msg.keep_alive {
-            packet.set_keep_alive(keep_alive);
-        }
-        */
+        pkt.set_keep_alive(options.keep_alive.unwrap_or(DEFAULT_KEEP_ALIVE));
 
         client.send(pkt.into())?;
         let pkt = client.recv()?;
@@ -72,14 +99,19 @@ impl MqttClient {
         }
     }
 
-    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &MqttOptions) -> Result<MqttClient> {
+    pub fn connect<R: DnsResolver>(
+        resolver: &R,
+        url: Url,
+        options: &MqttOptions,
+    ) -> Result<MqttClient> {
         let tls = match url.scheme() {
             "mqtt" => false,
             "mqtts" => true,
             _ => bail!("Invalid mqtt protocol"),
         };
 
-        let host = url.host_str()
+        let host = url
+            .host_str()
             .ok_or_else(|| format_err!("Missing host in url"))?;
 
         let port = match (url.port(), tls) {
@@ -88,29 +120,57 @@ impl MqttClient {
             (None, false) => 1883,
         };
 
+        // if no read timeout is configured then keep alive won't work
+        let read_timeout = options.read_timeout.unwrap_or(DEFAULT_PING_INTERVAL);
 
-        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
-            tls,
-            sni_value: None,
-            disable_tls_verify: false,
-            proxy: options.proxy,
+        let stream = Stream::connect_stream(
+            resolver,
+            host,
+            port,
+            &SocketOptions {
+                tls,
+                sni_value: None,
+                disable_tls_verify: false,
+                proxy: options.proxy,
 
-            connect_timeout: options.connect_timeout,
-            read_timeout: options.read_timeout,
-            write_timeout: options.write_timeout,
-        })?;
+                connect_timeout: options.connect_timeout,
+                read_timeout,
+                write_timeout: options.write_timeout,
+            },
+        )?;
 
         Self::negotiate(stream, options)
     }
 
+    fn maintain_ping(&mut self) -> Result<()> {
+        if let Some(ping_interval) = self.ping_interval {
+            if self.last_ping.elapsed() >= Duration::from_secs(ping_interval) {
+                self.ping().context("Failed to ping")?;
+                self.last_ping = Instant::now();
+            }
+        }
+        Ok(())
+    }
+
     fn send(&mut self, pkt: VariablePacket) -> Result<()> {
+        self.maintain_ping()?;
         debug!("Sending mqtt packet: {:?}", pkt);
         pkt.encode(&mut self.stream)?;
         Ok(())
     }
 
-    fn recv(&mut self) -> std::result::Result<VariablePacket, VariablePacketError> {
-        let pkt = VariablePacket::decode(&mut self.stream)?;
+    fn recv(&mut self) -> std::result::Result<VariablePacket, MqttRecvError> {
+        self.maintain_ping()?;
+        let pkt = VariablePacket::decode(&mut self.stream).map_err(|err| match err {
+            // search for any io error and check if it's ErrorKind::UnexpectedEof
+            VariablePacketError::IoError(err)
+            | VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))
+                if err.kind() == io::ErrorKind::UnexpectedEof =>
+            {
+                MqttRecvError::RecvDisconnect
+            }
+            _ => MqttRecvError::Recv(err),
+        })?;
         debug!("Received mqtt packet: {:?}", pkt);
         Ok(pkt)
     }
@@ -139,30 +199,35 @@ impl MqttClient {
     pub fn recv_pkt(&mut self) -> Result<Option<Pkt>> {
         match self.recv() {
             Ok(pkt) => Ok(Some(Pkt::try_from(pkt)?)),
-            Err(VariablePacketError::IoError(err)) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
-            Err(VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
-            Err(err) => Err(Error::from(err))
+            // search for any io error and check if it's ErrorKind::WouldBlock
+            Err(MqttRecvError::Recv(
+                VariablePacketError::IoError(err)
+                | VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err)),
+            )) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(err) => Err(err.into()),
         }
     }
 
     pub fn ping(&mut self) -> Result<()> {
         let pkt = PingreqPacket::new();
-        self.send(pkt.into())
+        let pkt = VariablePacket::PingreqPacket(pkt);
+        pkt.encode(&mut self.stream)?;
+        Ok(())
     }
 }
 
 #[derive(Serialize, Deserialize)]
 #[serde(tag = "type")]
 pub enum Pkt {
-    #[serde(rename="publish")]
+    #[serde(rename = "publish")]
     Publish(Publish),
-    #[serde(rename="pong")]
+    #[serde(rename = "pong")]
     Pong,
 }
 
 impl Pkt {
     pub fn to_lua(&self) -> Result<AnyLuaValue> {
-        let v = serde_json::to_value(&self)?;
+        let v = serde_json::to_value(self)?;
         let v = LuaJsonValue::from(v).into();
         Ok(v)
     }
@@ -177,7 +242,7 @@ impl TryFrom<VariablePacket> for Pkt {
             VariablePacket::ConnackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
             VariablePacket::PublishPacket(pkt) => Ok(Pkt::Publish(Publish {
                 topic: pkt.topic_name().to_string(),
-                body: pkt.payload(),
+                body: pkt.payload().to_vec(),
             })),
             VariablePacket::PubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
             VariablePacket::PubrecPacket(_) => bail!("Unsupported pkt: {:?}", pkt),

sn0int-std/src/psl.rs

@@ -1,11 +1,12 @@
 use crate::errors::*;
-
-use chrootable_https::Client;
-use std::fs::{self, File};
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
 use crate::lazy::LazyInit;
-
+use chrootable_https::Client;
+use publicsuffix::{List, Psl as _};
+use std::fs::{self, File};
+use std::io::Read;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+use std::sync::Arc;
 
 #[derive(Debug, PartialEq)]
 pub struct DnsName {
@@ -22,8 +23,8 @@ pub enum PslReader {
 
 impl PslReader {
     pub fn open_or_download<F>(cache_dir: &Path, indicator: F) -> Result<PslReader>
-        where
-            F: Fn(Box<dyn Fn() -> Result<PslReader>>) -> Result<PslReader>
+    where
+        F: Fn(Box<dyn Fn() -> Result<PslReader>>) -> Result<PslReader>,
     {
         let path = Self::path(cache_dir)?;
         let reader = match Self::open_from(&path) {
@@ -54,14 +55,14 @@ impl PslReader {
         }
 
         // else, use local cache
-        let path = cache_dir
-            .join("public_suffix_list.dat");
+        let path = cache_dir.join("public_suffix_list.dat");
         Ok(path)
     }
 
     pub fn download(path: &Path, url: &str) -> Result<()> {
         let client = Client::with_system_resolver_v4()?;
-        let resp = client.get(url)
+        let resp = client
+            .get(url)
             .wait_for_response()
             .context("http request failed")?;
         fs::write(path, &resp.body)?;
@@ -72,47 +73,47 @@ impl PslReader {
 impl LazyInit<Arc<Psl>> for PslReader {
     fn initialize(self) -> Result<Arc<Psl>> {
         let list = match self {
-            PslReader::Reader(file) => publicsuffix::List::from_reader(file),
-            PslReader::String(s) => publicsuffix::List::from_str(&s),
+            PslReader::Reader(mut file) => {
+                let mut buf = String::new();
+                file.read_to_string(&mut buf)?;
+                buf
+            }
+            PslReader::String(s) => s,
         };
 
-        let list = list
+        let list = List::from_str(&list)
             .map_err(|e| format_err!("Failed to load public suffix list: {}", e))?;
 
-        Ok(Arc::new(Psl {
-            list,
-        }))
+        Ok(Arc::new(Psl { list }))
     }
 }
 
 #[derive(Debug)]
 pub struct Psl {
-    list: publicsuffix::List,
+    list: List,
 }
 
 impl Psl {
     pub fn parse_dns_name(&self, name: &str) -> Result<DnsName> {
-        let dns_name = self.list.parse_dns_name(name)
-            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
+        let bytes = name.as_bytes();
 
-        let (root, suffix) = if let Some(domain) = dns_name.domain() {
-            let root = domain.to_string();
-            if let Some(suffix) = domain.suffix() {
-                (root, suffix.to_string())
-            } else {
-                // XXX: not sure if this is reachable
-                (root.clone(), root)
-            }
+        let suffix = self
+            .list
+            .suffix(bytes)
+            .ok_or_else(|| format_err!("Failed to detect suffix"))?;
+        let suffix = String::from_utf8(suffix.as_bytes().to_vec())?;
+
+        let root = if let Some(root) = self.list.domain(bytes) {
+            String::from_utf8(root.as_bytes().to_vec())?
         } else {
             // this is technically a tld, but support eg. a.prod.fastly.net anyway
-            // XXX: consider forcing domain to be in self.list.private()
-            (name.to_string(), name.to_string())
+            name.to_string()
         };
 
-        let fulldomain = if root.as_str() != name {
-            Some(name.to_string())
-        } else {
+        let fulldomain = if root == name {
             None
+        } else {
+            Some(name.to_string())
         };
 
         Ok(DnsName {
@@ -123,60 +124,82 @@ impl Psl {
     }
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;
 
     fn init() -> Arc<Psl> {
-        PslReader::String(r#"
+        PslReader::String(
+            r#"
 // ===BEGIN ICANN DOMAINS===
 com
 // ===END ICANN DOMAINS===
 // ===BEGIN PRIVATE DOMAINS===
 a.prod.fastly.net
 // ===END PRIVATE DOMAINS===
-"#.into()).initialize().unwrap()
+"#
+            .into(),
+        )
+        .initialize()
+        .unwrap()
     }
 
     #[test]
     fn test_psl_example_com() {
-        let x = init().parse_dns_name("example.com").expect("parse_dns_name");
-        assert_eq!(x, DnsName {
-            fulldomain: None,
-            root: "example.com".into(),
-            suffix: "com".into(),
-        });
+        let x = init()
+            .parse_dns_name("example.com")
+            .expect("parse_dns_name");
+        assert_eq!(
+            x,
+            DnsName {
+                fulldomain: None,
+                root: "example.com".into(),
+                suffix: "com".into(),
+            }
+        );
     }
 
     #[test]
     fn test_psl_www_example_com() {
-        let x = init().parse_dns_name("www.example.com").expect("parse_dns_name");
-        assert_eq!(x, DnsName {
-            fulldomain: Some("www.example.com".into()),
-            root: "example.com".into(),
-            suffix: "com".into(),
-        });
+        let x = init()
+            .parse_dns_name("www.example.com")
+            .expect("parse_dns_name");
+        assert_eq!(
+            x,
+            DnsName {
+                fulldomain: Some("www.example.com".into()),
+                root: "example.com".into(),
+                suffix: "com".into(),
+            }
+        );
     }
 
     #[test]
     fn test_psl_com() {
         let x = init().parse_dns_name("com").expect("parse_dns_name");
-        assert_eq!(x, DnsName {
-            fulldomain: None,
-            root: "com".into(),
-            suffix: "com".into(),
-        });
+        assert_eq!(
+            x,
+            DnsName {
+                fulldomain: None,
+                root: "com".into(),
+                suffix: "com".into(),
+            }
+        );
     }
 
     #[test]
     fn test_psl_a_b_c_d_e_f_g_com() {
-        let x = init().parse_dns_name("a.b.c.d.e.f.g.com").expect("parse_dns_name");
-        assert_eq!(x, DnsName {
-            fulldomain: Some("a.b.c.d.e.f.g.com".into()),
-            root: "g.com".into(),
-            suffix: "com".into(),
-        });
+        let x = init()
+            .parse_dns_name("a.b.c.d.e.f.g.com")
+            .expect("parse_dns_name");
+        assert_eq!(
+            x,
+            DnsName {
+                fulldomain: Some("a.b.c.d.e.f.g.com".into()),
+                root: "g.com".into(),
+                suffix: "com".into(),
+            }
+        );
     }
 
     #[test]
@@ -187,41 +210,61 @@ a.prod.fastly.net
 
     #[test]
     fn test_psl_asdfinvalid() {
-        let x = init().parse_dns_name("asdfinvalid").expect("parse_dns_name");
-        assert_eq!(x, DnsName {
-            fulldomain: None,
-            root: "asdfinvalid".into(),
-            suffix: "asdfinvalid".into(),
-        });
+        let x = init()
+            .parse_dns_name("asdfinvalid")
+            .expect("parse_dns_name");
+        assert_eq!(
+            x,
+            DnsName {
+                fulldomain: None,
+                root: "asdfinvalid".into(),
+                suffix: "asdfinvalid".into(),
+            }
+        );
     }
 
     #[test]
     fn test_psl_www_example_asdfinvalid() {
-        let x = init().parse_dns_name("www.example.asdfinvalid").expect("parse_dns_name");
-        assert_eq!(x, DnsName {
-            fulldomain: Some("www.example.asdfinvalid".into()),
-            root: "example.asdfinvalid".into(),
-            suffix: "asdfinvalid".into(),
-        });
+        let x = init()
+            .parse_dns_name("www.example.asdfinvalid")
+            .expect("parse_dns_name");
+        assert_eq!(
+            x,
+            DnsName {
+                fulldomain: Some("www.example.asdfinvalid".into()),
+                root: "example.asdfinvalid".into(),
+                suffix: "asdfinvalid".into(),
+            }
+        );
     }
 
     #[test]
     fn test_psl_a_prod_fastly_net() {
-        let x = init().parse_dns_name("a.prod.fastly.net").expect("parse_dns_name");
-        assert_eq!(x, DnsName {
-            fulldomain: None,
-            root: "a.prod.fastly.net".into(),
-            suffix: "a.prod.fastly.net".into(),
-        });
+        let x = init()
+            .parse_dns_name("a.prod.fastly.net")
+            .expect("parse_dns_name");
+        assert_eq!(
+            x,
+            DnsName {
+                fulldomain: None,
+                root: "a.prod.fastly.net".into(),
+                suffix: "a.prod.fastly.net".into(),
+            }
+        );
     }
 
     #[test]
     fn test_psl_www_a_prod_fastly_net() {
-        let x = init().parse_dns_name("www.a.prod.fastly.net").expect("parse_dns_name");
-        assert_eq!(x, DnsName {
-            fulldomain: None,
-            root: "www.a.prod.fastly.net".into(),
-            suffix: "a.prod.fastly.net".into(),
-        });
+        let x = init()
+            .parse_dns_name("www.a.prod.fastly.net")
+            .expect("parse_dns_name");
+        assert_eq!(
+            x,
+            DnsName {
+                fulldomain: None,
+                root: "www.a.prod.fastly.net".into(),
+                suffix: "a.prod.fastly.net".into(),
+            }
+        );
     }
 }

sn0int-std/src/ratelimits.rs

@@ -1,5 +1,5 @@
 use chrono::prelude::*;
-use serde::{Serialize, Deserialize};
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::result;
 use std::sync::mpsc;
@@ -43,9 +43,7 @@ struct Bucket {
 
 impl Bucket {
     pub fn new() -> Bucket {
-        Bucket {
-            passes: Vec::new(),
-        }
+        Bucket { passes: Vec::new() }
     }
 
     pub fn pass(&mut self, passes: usize, time: u32) -> RatelimitResponse {

sn0int-std/src/sockets/mod.rs

@@ -1,8 +1,8 @@
 use crate::errors::*;
 
-use bufstream::BufStream;
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
+use bufstream::BufStream;
 use chrootable_https::dns::{DnsResolver, RecordType};
 use chrootable_https::socks5::{self, ProxyDest};
 use regex::Regex;
@@ -10,19 +10,18 @@ use serde::Deserialize;
 use tokio::runtime::Runtime;
 
 use std::fmt;
-use std::str;
 use std::io;
 use std::io::prelude::*;
 use std::io::BufRead;
 use std::net::SocketAddr;
 use std::net::TcpStream;
 use std::net::{IpAddr, Ipv4Addr};
+use std::str;
 use std::time::Duration;
 
 mod tls;
 pub use self::tls::TlsData;
 
-
 #[cfg(unix)]
 fn unwrap_socket(socket: tokio::net::TcpStream) -> Result<TcpStream> {
     use std::os::unix::io::AsRawFd;
@@ -49,7 +48,6 @@ pub struct SocketOptions {
     // TODO: enable_sni (default to true)
     // TODO: sni_name
     // TODO: cacert
-
     #[serde(default)]
     pub connect_timeout: u64,
     #[serde(default)]
@@ -96,10 +94,16 @@ pub enum Stream {
 }
 
 impl Stream {
-    pub fn connect_stream<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+    pub fn connect_stream<R: DnsResolver>(
+        resolver: &R,
+        host: &str,
+        port: u16,
+        options: &SocketOptions,
+    ) -> Result<Stream> {
         let addrs = match host.parse::<IpAddr>() {
             Ok(addr) => vec![addr],
-            Err(_) => resolver.resolve(host, RecordType::A)
+            Err(_) => resolver
+                .resolve(host, RecordType::A)
                 .wait_for_response()?
                 .success()?,
         };
@@ -107,10 +111,10 @@ impl Stream {
         let mut errors = Vec::new();
 
         for addr in addrs {
-            match Stream::connect_addr(host, (addr, port).into(), &options) {
+            match Stream::connect_addr(host, (addr, port).into(), options) {
                 Ok(socket) => {
                     return Ok(socket);
-                },
+                }
                 Err(err) => errors.push((addr, err)),
             }
         }
@@ -129,7 +133,7 @@ impl Stream {
         let socket = if connect_timeout > 0 {
             TcpStream::connect_timeout(&addr, Duration::from_secs(connect_timeout))?
         } else {
-            TcpStream::connect(&addr)?
+            TcpStream::connect(addr)?
         };
         debug!("successfully connected to {:?}", addr);
 
@@ -138,15 +142,23 @@ impl Stream {
         tls::wrap_if_enabled(socket, host, options)
     }
 
-    pub fn connect_socks5_stream(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
-        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
+    pub fn connect_socks5_stream(
+        proxy: SocketAddr,
+        host: &str,
+        port: u16,
+        options: &SocketOptions,
+    ) -> Result<Stream> {
+        debug!(
+            "connecting to {:?}:{:?} with socks5 on {:?}",
+            host, port, proxy
+        );
 
         let addr = match host.parse::<Ipv4Addr>() {
             Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
             _ => ProxyDest::Domain(host.to_string()),
         };
 
-        let fut = socks5::connect(proxy, addr, port);
+        let fut = socks5::connect(&proxy, addr, port);
 
         let mut rt = Runtime::new()?;
         let socket = rt.block_on(fut)?;
@@ -206,12 +218,22 @@ impl Socket {
         }
     }
 
-    pub fn connect<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
+    pub fn connect<R: DnsResolver>(
+        resolver: &R,
+        host: &str,
+        port: u16,
+        options: &SocketOptions,
+    ) -> Result<Socket> {
         let stream = Stream::connect_stream(resolver, host, port, options)?;
         Ok(Socket::new(stream))
     }
 
-    pub fn connect_socks5(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
+    pub fn connect_socks5(
+        proxy: SocketAddr,
+        host: &str,
+        port: u16,
+        options: &SocketOptions,
+    ) -> Result<Socket> {
         let stream = Stream::connect_socks5_stream(proxy, host, port, options)?;
         Ok(Socket::new(stream))
     }
@@ -233,7 +255,7 @@ impl Socket {
     }
 
     pub fn send(&mut self, data: &[u8]) -> Result<()> {
-        match str::from_utf8(&data) {
+        match str::from_utf8(data) {
             Ok(data) => debug!("send: {:?}", data),
             Err(_) => debug!("send: {:?}", data),
         };
@@ -266,8 +288,7 @@ impl Socket {
     pub fn recvline(&mut self) -> Result<String> {
         let needle = self.newline.clone();
         let buf = self.recvuntil(needle.as_bytes())?;
-        let line = String::from_utf8(buf)
-            .context("Failed to decode utf8")?;
+        let line = String::from_utf8(buf).context("Failed to decode utf8")?;
         Ok(line)
     }
 
@@ -321,10 +342,13 @@ impl Socket {
                     Ok(n) => n,
                     Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
                     Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(Vec::new()),
-                    Err(e) => return Err(e.into())
+                    Err(e) => return Err(e.into()),
                 };
 
-                match available.windows(delim_len).position(|window| window == delim) {
+                match available
+                    .windows(delim_len)
+                    .position(|window| window == delim)
+                {
                     Some(i) => {
                         buf.extend_from_slice(&available[..i + delim_len]);
                         (true, i + delim_len)
@@ -366,21 +390,34 @@ mod tests {
     #[test]
     fn verify_tls_good() {
         let resolver = Resolver::from_system_v4().unwrap();
-        let _sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
-            tls: true,
-            ..Default::default()
-        }).unwrap();
+        let _sock = Socket::connect(
+            &resolver,
+            "badssl.com",
+            443,
+            &SocketOptions {
+                tls: true,
+                ..Default::default()
+            },
+        )
+        .unwrap();
     }
 
     #[test]
     #[ignore]
     fn verify_tls_good_request() {
         let resolver = Resolver::from_system_v4().unwrap();
-        let mut sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
-            tls: true,
-            ..Default::default()
-        }).unwrap();
-        sock.send(b"GET / HTTP/1.1\r\nHost: badssl.com\r\nConnection: close\r\n\r\n").unwrap();
+        let mut sock = Socket::connect(
+            &resolver,
+            "badssl.com",
+            443,
+            &SocketOptions {
+                tls: true,
+                ..Default::default()
+            },
+        )
+        .unwrap();
+        sock.send(b"GET / HTTP/1.1\r\nHost: badssl.com\r\nConnection: close\r\n\r\n")
+            .unwrap();
         let status = sock.recvline().unwrap();
         assert_eq!(status, "HTTP/1.1 200 OK\r\n");
     }
@@ -389,10 +426,15 @@ mod tests {
     #[ignore]
     fn verify_tls_expired() {
         let resolver = Resolver::from_system_v4().unwrap();
-        let sock = Socket::connect(&resolver, "expired.badssl.com", 443, &SocketOptions{
-            tls: true,
-            ..Default::default()
-        });
+        let sock = Socket::connect(
+            &resolver,
+            "expired.badssl.com",
+            443,
+            &SocketOptions {
+                tls: true,
+                ..Default::default()
+            },
+        );
         assert!(sock.is_err());
     }
 

sn0int-std/src/sockets/tls.rs

@@ -1,13 +1,13 @@
+use super::{SocketOptions, Stream};
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
-use rustls::{self, ClientConfig, Session, ClientSession};
+use rustls::{self, ClientConfig, ClientSession, Session};
 use serde::Serialize;
-use std::str;
-use std::result;
-use std::sync::Arc;
 use std::net::TcpStream;
-use super::{Stream, SocketOptions};
+use std::result;
+use std::str;
+use std::sync::Arc;
 
 #[derive(Debug, Serialize)]
 pub struct TlsData {
@@ -17,7 +17,7 @@ pub struct TlsData {
 
 impl TlsData {
     pub fn to_lua(&self) -> Result<AnyLuaValue> {
-        let v = serde_json::to_value(&self)?;
+        let v = serde_json::to_value(self)?;
         let v = LuaJsonValue::from(v).into();
         Ok(v)
     }
@@ -42,14 +42,15 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
 
     if options.disable_tls_verify {
         info!("tls verification has been disabled");
-        config.dangerous()
+        config
+            .dangerous()
             .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
     }
 
     let dns_name = if let Some(v) = &options.sni_value {
-        get_dns_name(&mut config, &v)
+        get_dns_name(&mut config, v)
     } else {
-        get_dns_name(&mut config, &host)
+        get_dns_name(&mut config, host)
     };
 
     let config = Arc::new(config);
@@ -58,7 +59,7 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
 }
 
 fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
-    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(&host) {
+    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(host) {
         debug!("setting sni value to: {:?}", host);
         name.to_owned()
     } else {
@@ -73,12 +74,14 @@ fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
 fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, TlsData)> {
     info!("starting tls handshake");
     if session.is_handshaking() {
-        session.complete_io(&mut stream)
+        session
+            .complete_io(&mut stream)
             .context("Failed to read reply to tls client hello")?;
     }
 
     if session.wants_write() {
-        session.complete_io(&mut stream)
+        session
+            .complete_io(&mut stream)
             .context("wants_write->complete_io failed")?;
     }
 
@@ -88,19 +91,14 @@ fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, T
     };
 
     if let Some(certs) = session.get_peer_certificates() {
-        tls.cert_chain = certs.into_iter()
+        tls.cert_chain = certs
+            .into_iter()
             .rev()
-            .map(|c| {
-                pem::encode(&pem::Pem {
-                    tag: String::from("CERTIFICATE"),
-                    contents: c.0,
-                })
-            })
+            .map(|c| pem::encode(&pem::Pem::new("CERTIFICATE", c.0)))
             .collect();
     }
 
-    tls.cert = tls.cert_chain.last()
-        .map(|x| x.to_owned());
+    tls.cert = tls.cert_chain.last().map(|x| x.to_owned());
 
     info!("successfully established tls connection");
     let stream = rustls::StreamOwned::new(session, stream);
@@ -111,11 +109,13 @@ fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, T
 pub struct NoCertificateVerification {}
 
 impl rustls::ServerCertVerifier for NoCertificateVerification {
-    fn verify_server_cert(&self,
+    fn verify_server_cert(
+        &self,
         _roots: &rustls::RootCertStore,
         _presented_certs: &[rustls::Certificate],
         _dns_name: webpki::DNSNameRef<'_>,
-        _ocsp: &[u8]) -> result::Result<rustls::ServerCertVerified, rustls::TLSError> {
+        _ocsp: &[u8],
+    ) -> result::Result<rustls::ServerCertVerified, rustls::TLSError> {
         Ok(rustls::ServerCertVerified::assertion())
     }
 }

sn0int-std/src/web.rs

@@ -1,18 +1,20 @@
 use crate::blobs::{Blob, BlobState};
 use crate::engine::structs::LuaMap;
 use crate::errors::*;
-use crate::json::LuaJsonValue;
 use crate::hlua::AnyLuaValue;
-use chrootable_https::{Request, Body, Uri};
-pub use chrootable_https::{Client, HttpClient, Resolver, Response};
-use chrootable_https::http::HttpTryFrom;
-use chrootable_https::http::uri::Parts;
+use crate::json::LuaJsonValue;
 use chrootable_https::http::request::Builder;
-use rand::{Rng, thread_rng};
+use chrootable_https::http::uri::Parts;
+use chrootable_https::http::HttpTryFrom;
+use chrootable_https::{Body, Request, Uri};
+pub use chrootable_https::{Client, HttpClient, Resolver, Response};
+use data_encoding::BASE64;
 use rand::distributions::Alphanumeric;
-use serde::{Serialize, Deserialize};
+use rand::{thread_rng, Rng};
+use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::fmt;
+use std::fmt::Write;
 use std::iter;
 use std::net::SocketAddr;
 use std::ops::Deref;
@@ -25,15 +27,17 @@ pub fn url_set_qs<S: Serialize + fmt::Debug>(url: Uri, query: &S) -> Result<Uri>
 
     let query = serde_urlencoded::to_string(query)?;
 
-    parts.path_and_query = Some(match parts.path_and_query {
-        Some(pq) => {
-            format!("{}?{}", pq.path(), query)
-        },
-        None => format!("/?{}", query),
-    }.parse()?);
+    parts.path_and_query = Some(
+        match parts.path_and_query {
+            Some(pq) => {
+                format!("{}?{}", pq.path(), query)
+            }
+            None => format!("/?{}", query),
+        }
+        .parse()?,
+    );
 
-    Uri::from_parts(parts)
-        .map_err(Error::from)
+    Uri::from_parts(parts).map_err(Error::from)
 }
 
 pub trait WebState {
@@ -56,30 +60,33 @@ impl HttpSession {
             .map(char::from)
             .take(16)
             .collect();
-        (id.clone(), HttpSession {
-            id,
-            cookies: CookieJar::default(),
-        })
+        (
+            id.clone(),
+            HttpSession {
+                id,
+                cookies: CookieJar::default(),
+            },
+        )
     }
 }
 
 #[derive(Debug, Default, Deserialize)]
 pub struct RequestOptions {
-    query: Option<HashMap<String, String>>,
-    headers: Option<HashMap<String, String>>,
-    basic_auth: Option<(String, String)>,
-    user_agent: Option<String>,
-    json: Option<serde_json::Value>,
-    form: Option<serde_json::Value>,
+    pub query: Option<HashMap<String, String>>,
+    pub headers: Option<HashMap<String, String>>,
+    pub basic_auth: Option<(String, String)>,
+    pub user_agent: Option<String>,
+    pub json: Option<serde_json::Value>,
+    pub form: Option<serde_json::Value>,
     #[serde(default)]
-    follow_redirects: usize,
-    body: Option<String>,
-    timeout: Option<u64>,
+    pub follow_redirects: usize,
+    pub body: Option<String>,
+    pub timeout: Option<u64>,
     #[serde(default)]
-    into_blob: bool,
-    proxy: Option<SocketAddr>,
+    pub into_blob: bool,
+    pub proxy: Option<SocketAddr>,
     #[serde(default)]
-    binary: bool,
+    pub binary: bool,
 }
 
 impl RequestOptions {
@@ -110,11 +117,14 @@ pub struct HttpRequest {
 }
 
 impl HttpRequest {
-    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions, default_agent: fn() -> String) -> HttpRequest {
+    pub fn new(
+        session: &HttpSession,
+        method: String,
+        url: String,
+        user_agent: String,
+        options: RequestOptions,
+    ) -> HttpRequest {
         let cookies = session.cookies.clone();
-
-        let user_agent = options.user_agent.unwrap_or_else(default_agent);
-
         let timeout = options.timeout.map(Duration::from_millis);
 
         let mut request = HttpRequest {
@@ -163,9 +173,9 @@ impl HttpRequest {
         // add headers
         if let Some(ref auth) = self.basic_auth {
             use chrootable_https::header::AUTHORIZATION;
-            let &(ref user, ref password) = auth;
+            let (user, password) = auth;
 
-            let auth = base64::encode(&format!("{}:{}", user, password));
+            let auth = BASE64.encode(format!("{}:{}", user, password).as_bytes());
             let auth = format!("Basic {}", auth);
             req.header(AUTHORIZATION, auth.as_str());
         }
@@ -180,21 +190,21 @@ impl HttpRequest {
 
         // finalize request
         let body = match self.body {
-            Some(ReqBody::Raw(ref x))  => { Body::from(x.clone()) },
+            Some(ReqBody::Raw(ref x)) => Body::from(x.clone()),
             Some(ReqBody::Form(ref x)) => {
                 // if Content-Type is not set, set header
                 if !observed_headers.contains("content-type") {
                     req.header("Content-Type", "application/x-www-form-urlencoded");
                 }
                 Body::from(serde_urlencoded::to_string(x)?)
-            },
+            }
             Some(ReqBody::Json(ref x)) => {
                 // if Content-Type is not set, set header
                 if !observed_headers.contains("content-type") {
                     req.header("Content-Type", "application/json");
                 }
                 Body::from(serde_json::to_string(x)?)
-            },
+            }
             None => Body::empty(),
         };
         let mut req = req.body(body)?;
@@ -205,7 +215,8 @@ impl HttpRequest {
         let res = loop {
             // send request
             debug!("Sending http request: {:?}", req);
-            let res = client.request(req)
+            let res = client
+                .request(req)
                 .with_timeout(self.timeout)
                 .wait_for_response()?;
 
@@ -216,7 +227,7 @@ impl HttpRequest {
             if self.follow_redirects > 0 && res.status >= 300 && res.status < 400 {
                 if let Some(location) = res.headers.get("location") {
                     let base = Url::parse(&url.to_string())?;
-                    let joined = base.join(&location)?;
+                    let joined = base.join(location)?;
                     url = joined.to_string().parse()?;
 
                     req = self.mkrequest("GET", &url).body(Body::empty())?;
@@ -233,7 +244,8 @@ impl HttpRequest {
 
     /// create a basic request, reusable when following redirects
     fn mkrequest<T>(&self, method: &str, url: T) -> Builder
-        where Uri: HttpTryFrom<T>,
+    where
+        Uri: HttpTryFrom<T>,
     {
         let mut req = Request::builder();
         req.method(method);
@@ -254,7 +266,8 @@ impl HttpRequest {
             if !cookies.is_empty() {
                 cookies += "; ";
             }
-            cookies.push_str(&format!("{}={}", key, value));
+            // it's a write to a String, so panic if re-allocation fails is fine
+            write!(cookies, "{}={}", key, value).expect("out of memory");
         }
 
         if !cookies.is_empty() {
@@ -263,7 +276,8 @@ impl HttpRequest {
     }
 
     pub fn response_to_lua<S>(&self, state: &S, res: Response) -> Result<LuaMap>
-        where S: WebState + BlobState
+    where
+        S: WebState + BlobState,
     {
         // map result to LuaMap
         let mut resp = LuaMap::new();
@@ -318,9 +332,9 @@ impl HttpRequest {
     }
 }
 
-impl Into<AnyLuaValue> for HttpRequest {
-    fn into(self) -> AnyLuaValue {
-        let v = serde_json::to_value(&self).unwrap();
+impl From<HttpRequest> for AnyLuaValue {
+    fn from(req: HttpRequest) -> AnyLuaValue {
+        let v = serde_json::to_value(req).unwrap();
         LuaJsonValue::from(v).into()
     }
 }

sn0int-std/src/websockets.rs

@@ -1,13 +1,13 @@
-use chrootable_https::DnsResolver;
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
-use crate::sockets::{Stream, SocketOptions};
+use crate::sockets::{SocketOptions, Stream};
+use chrootable_https::DnsResolver;
 use http::Request;
 use serde::Deserialize;
 use std::collections::HashMap;
-use std::net::SocketAddr;
 use std::io;
+use std::net::SocketAddr;
 use tungstenite::protocol::{self, Message};
 use url::Url;
 
@@ -44,7 +44,11 @@ pub struct WebSocket {
 }
 
 impl WebSocket {
-    pub fn negotiate(stream: Stream, url: Url, headers: Option<&HashMap<String, String>>) -> Result<WebSocket> {
+    pub fn negotiate(
+        stream: Stream,
+        url: Url,
+        headers: Option<&HashMap<String, String>>,
+    ) -> Result<WebSocket> {
         let mut req = Request::get(url.to_string()); // TODO: don't re-parse here
 
         if let Some(headers) = headers {
@@ -56,19 +60,22 @@ impl WebSocket {
         let req = req.body(()).unwrap();
 
         let (sock, _resp) = tungstenite::client::client(req, stream)?;
-        Ok(WebSocket {
-            sock,
-        })
+        Ok(WebSocket { sock })
     }
 
-    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &WebSocketOptions) -> Result<WebSocket> {
+    pub fn connect<R: DnsResolver>(
+        resolver: &R,
+        url: Url,
+        options: &WebSocketOptions,
+    ) -> Result<WebSocket> {
         let tls = match url.scheme() {
             "ws" => false,
             "wss" => true,
             _ => bail!("Invalid websocket protocol"),
         };
 
-        let host = url.host_str()
+        let host = url
+            .host_str()
             .ok_or_else(|| format_err!("Missing host in url"))?;
 
         let port = match (url.port(), tls) {
@@ -77,23 +84,30 @@ impl WebSocket {
             (None, false) => 80,
         };
 
-        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
-            tls,
-            sni_value: None,
-            disable_tls_verify: false,
-            proxy: options.proxy,
+        let stream = Stream::connect_stream(
+            resolver,
+            host,
+            port,
+            &SocketOptions {
+                tls,
+                sni_value: None,
+                disable_tls_verify: false,
+                proxy: options.proxy,
 
-            connect_timeout: options.connect_timeout,
-            read_timeout: options.read_timeout,
-            write_timeout: options.write_timeout,
-        })?;
+                connect_timeout: options.connect_timeout,
+                read_timeout: options.read_timeout,
+                write_timeout: options.write_timeout,
+            },
+        )?;
         Self::negotiate(stream, url, options.headers.as_ref())
     }
 
     pub fn options(&self, options: &WebSocketOptions) -> Result<()> {
-        let mut o = SocketOptions::default();
-        o.read_timeout = options.read_timeout;
-        o.write_timeout = options.write_timeout;
+        let o = SocketOptions {
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+            ..Default::default()
+        };
         o.apply(self.sock.get_ref())
     }
 
@@ -105,12 +119,14 @@ impl WebSocket {
                 Ok(Message::Ping(ping)) => {
                     self.sock.write_message(Message::Pong(ping))?;
                     continue;
-                },
+                }
                 Ok(Message::Pong(_)) => continue, // this should never happen
                 Ok(Message::Close(_)) => Event::Close,
                 Err(tungstenite::Error::ConnectionClosed) => Event::Close,
                 Err(tungstenite::Error::AlreadyClosed) => Event::Close,
-                Err(tungstenite::Error::Io(err)) if err.kind() == io::ErrorKind::WouldBlock => Event::Timeout,
+                Err(tungstenite::Error::Io(err)) if err.kind() == io::ErrorKind::WouldBlock => {
+                    Event::Timeout
+                }
                 Err(err) => return Err(err.into()),
             };
             return Ok(msg);

sn0int-std/src/xml.rs

@@ -1,14 +1,13 @@
 use crate::errors::*;
 
-use crate::json::LuaJsonValue;
 use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
 use serde::Serialize;
 use std::collections::HashMap;
 use xml::attribute::OwnedAttribute;
 use xml::name::OwnedName;
 use xml::reader::{EventReader, ParserConfig, XmlEvent};
 
-
 #[derive(Debug, PartialEq, Serialize)]
 pub struct XmlDocument {
     pub children: Vec<XmlElement>,
@@ -41,7 +40,8 @@ impl XmlElement {
     #[inline]
     fn from(name: OwnedName, attributes: Vec<OwnedAttribute>) -> XmlElement {
         let name = name.local_name;
-        let attrs = attributes.into_iter()
+        let attrs = attributes
+            .into_iter()
             .map(|attr| (attr.name.local_name, attr.value))
             .collect();
         XmlElement {
@@ -62,7 +62,7 @@ pub fn decode(x: &str) -> Result<AnyLuaValue> {
 }
 
 #[inline]
-fn append_text(stack: &mut Vec<XmlElement>, text: String) {
+fn append_text(stack: &mut [XmlElement], text: String) {
     if let Some(tail) = stack.last_mut() {
         if let Some(prev) = tail.text.as_mut() {
             prev.push_str(&text);
@@ -91,16 +91,13 @@ fn decode_raw(x: &str) -> Result<XmlDocument> {
 
         match next {
             XmlEvent::StartElement {
-                name,
-                attributes,
-                ..
+                name, attributes, ..
             } => {
                 stack.push(XmlElement::from(name, attributes));
-            },
-            XmlEvent::EndElement {
-                name,
-            } => {
-                let child = stack.pop()
+            }
+            XmlEvent::EndElement { name } => {
+                let child = stack
+                    .pop()
                     .ok_or_else(|| format_err!("end element has no matching start element"))?;
 
                 let name = name.local_name;
@@ -113,7 +110,7 @@ fn decode_raw(x: &str) -> Result<XmlDocument> {
                 } else {
                     doc.children.push(child);
                 }
-            },
+            }
             XmlEvent::CData(text) => append_text(&mut stack, text),
             XmlEvent::Characters(text) => append_text(&mut stack, text),
             _ => (),
@@ -142,63 +139,62 @@ mod tests {
     #[test]
     fn verify_xml_decode_empty_body() {
         let doc = decode_raw("<body></body>").unwrap();
-        assert_eq!(doc, XmlDocument {
-            children: vec![
-                XmlElement {
+        assert_eq!(
+            doc,
+            XmlDocument {
+                children: vec![XmlElement {
                     name: String::from("body"),
                     attrs: HashMap::new(),
                     text: None,
                     children: vec![],
-                }
-            ]
-        });
+                }]
+            }
+        );
     }
 
     #[test]
     fn verify_xml_decode_single_tag() {
         let doc = decode_raw("<body><foo x=\"1\" /></body>").unwrap();
-        assert_eq!(doc, XmlDocument {
-            children: vec![
-                XmlElement {
+        assert_eq!(
+            doc,
+            XmlDocument {
+                children: vec![XmlElement {
                     name: String::from("body"),
                     attrs: HashMap::new(),
                     text: None,
-                    children: vec![
-                        XmlElement {
-                            name: String::from("foo"),
-                            attrs: hashmap!{
-                                String::from("x") => String::from("1"),
-                            },
-                            text: None,
-                            children: vec![],
-                        }
-                    ],
-                }
-            ]
-        });
+                    children: vec![XmlElement {
+                        name: String::from("foo"),
+                        attrs: hashmap! {
+                            String::from("x") => String::from("1"),
+                        },
+                        text: None,
+                        children: vec![],
+                    }],
+                }]
+            }
+        );
     }
 
     #[test]
     fn verify_xml_decode_single_tag_text() {
         let doc = decode_raw("<body><foo x=\"1\">hello world</foo></body>").unwrap();
-        assert_eq!(doc, XmlDocument {
-            children: vec![
-                XmlElement {
+        assert_eq!(
+            doc,
+            XmlDocument {
+                children: vec![XmlElement {
                     name: String::from("body"),
                     attrs: HashMap::new(),
                     text: None,
-                    children: vec![
-                        XmlElement {
-                            name: String::from("foo"),
-                            attrs: hashmap!{
-                                String::from("x") => String::from("1"),
-                            },
-                            text: Some(String::from("hello world")),
-                            children: vec![],
-                        }
-                    ],
-                }
-            ]
-        });
+                    children: vec![XmlElement {
+                        name: String::from("foo"),
+                        attrs: hashmap! {
+                            String::from("x") => String::from("1"),
+                        },
+                        text: Some(String::from("hello world")),
+                        children: vec![],
+                    }],
+                }]
+            }
+        );
     }
 }

src/args.rs

@@ -1,18 +1,20 @@
+use clap::{CommandFactory, Parser};
+use clap_complete::Shell;
 use crate::cmd;
+use crate::errors::*;
 use crate::options;
 use crate::workspaces::Workspace;
-use structopt::StructOpt;
-use structopt::clap::{AppSettings, Shell};
 use sn0int_common::ModuleID;
+use std::io;
 
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
+#[command(version)]
 pub struct Args {
     /// Select a different workspace instead of the default
-    #[structopt(short="w", long="workspace")]
+    #[arg(short = 'w', long="workspace", env="SN0INT_WORKSPACE")]
     pub workspace: Option<Workspace>,
 
-    #[structopt(subcommand)]
+    #[command(subcommand)]
     pub subcommand: Option<SubCommand>,
 }
 
@@ -22,160 +24,174 @@ impl Args {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum SubCommand {
     /// Run a module directly
-    #[structopt(name="run")]
+    #[command(name="run")]
     Run(Run),
     /// For internal use
-    #[structopt(name="sandbox")]
+    #[command(name="sandbox")]
     Sandbox(Sandbox),
     /// Login to the registry for publishing
-    #[structopt(name="login")]
+    #[command(name="login")]
     Login(Login),
     /// Create a new module
-    #[structopt(name="new")]
+    #[command(name="new")]
     New(New),
     /// Publish a script to the registry
-    #[structopt(name="publish")]
+    #[command(name="publish")]
     Publish(Publish),
     /// Install a module from the registry
-    #[structopt(name="install")]
+    #[command(name="install")]
     Install(Install),
     /// Search in the registry
-    #[structopt(name="search")]
+    #[command(name="search")]
     Search(Search),
     /// The sn0int package manager
-    #[structopt(name="pkg")]
+    #[command(name="pkg")]
     Pkg(cmd::pkg_cmd::Args),
     /// Insert into the database
-    #[structopt(name="add")]
+    #[command(name="add")]
     Add(cmd::add_cmd::Args),
     /// Select from the database
-    #[structopt(name="select")]
+    #[command(name="select")]
     Select(cmd::select_cmd::Args),
     /// Delete from the database
-    #[structopt(name="delete")]
+    #[command(name="delete")]
     Delete(cmd::delete_cmd::Args),
     /// Query logged activity
-    #[structopt(name="activity")]
+    #[command(name="activity")]
     Activity(cmd::activity_cmd::Args),
     /// Include entities in the scope
-    #[structopt(name="scope")]
+    #[command(name="scope")]
     Scope(cmd::scope_cmd::Args),
     /// Exclude entities from scope
-    #[structopt(name="noscope")]
+    #[command(name="noscope")]
     Noscope(cmd::noscope_cmd::Args),
+    /// Manage autoscope rules
+    Autoscope(cmd::autoscope_cmd::Args),
+    /// Manage autonoscope rules
+    Autonoscope(cmd::autonoscope_cmd::Args),
+    /// Rescope all entities based on autonoscope rules
+    Rescope(cmd::rescope_cmd::Args),
     /// Manage workspaces
-    #[structopt(name="workspace")]
+    #[command(name="workspace")]
     Workspace(cmd::workspace_cmd::Args),
     /// Calendar
-    #[structopt(name="cal")]
+    #[command(name="cal")]
     Cal(cmd::cal_cmd::Args),
     /// Notify
-    #[structopt(name="notify")]
+    #[command(name="notify")]
     Notify(cmd::notify_cmd::Args),
     /// Verify blob storage for corrupt and dangling blobs
-    #[structopt(name="fsck")]
+    #[command(name="fsck")]
     Fsck(cmd::fsck_cmd::Args),
     /// Export a workspace for external processing
-    #[structopt(name="export")]
+    #[command(name="export")]
     Export(cmd::export_cmd::Args),
+    /// Show statistics about your current workspace
+    #[command(name="stats")]
+    Stats(cmd::stats_cmd::Args),
     /// Run a lua repl
-    #[structopt(name="repl")]
+    #[command(name="repl")]
     Repl,
     /// Show paths of various file system locations
-    #[structopt(name="paths")]
+    #[command(name="paths")]
     Paths,
     /// Generate shell completions
-    #[structopt(name="completions")]
+    #[command(name="completions")]
     Completions(Completions),
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Run {
-    /// Execute a module that has been installed
-    pub module: String,
+    #[command(flatten)]
+    pub run: cmd::run_cmd::Args,
     /// Run a module from a path
-    #[structopt(short="f", long="file")]
+    #[arg(short = 'f', long="file")]
     pub file: bool,
-    /// Run modules concurrently
-    #[structopt(short="j", long="threads", default_value="1")]
-    pub threads: usize,
-    /// Verbose logging, once to print inserts even if they don't add new
-    /// data, twice to activate the debug() function
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    pub verbose: u64,
     /// Expose stdin to modules
-    #[structopt(long="stdin")]
+    #[arg(long="stdin")]
     pub stdin: bool,
     /// Automatically grant access to a keyring namespace
-    #[structopt(long="grant")]
+    #[arg(long="grant")]
     pub grants: Vec<String>,
     /// Automatically grant access to all requested keys
-    #[structopt(long="grant-full-keyring")]
+    #[arg(long="grant-full-keyring")]
     pub grant_full_keyring: bool,
     /// Automatically deny access to all requested keys
-    #[structopt(long="deny-keyring")]
+    #[arg(long="deny-keyring")]
     pub deny_keyring: bool,
     /// Exit on first error and set exit code
-    #[structopt(short="x", long="exit-on-error")]
+    #[arg(short = 'x', long="exit-on-error")]
     pub exit_on_error: bool,
     /// Set an option
-    #[structopt(short="o", long="option")]
+    #[arg(short = 'o', long="option")]
     pub options: Vec<options::Opt>,
     /// Narrow down targeted entities
-    #[structopt(short="t", long="target")]
+    #[arg(short = 't', long="target")]
     pub target: Option<String>,
     /// Dump the sandbox init message to stdout instead of running a child process
-    #[structopt(long="dump-sandbox-init-msg")]
+    #[arg(long="dump-sandbox-init-msg")]
     pub dump_sandbox_init_msg: bool,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Sandbox {
     /// This value is only used for process listings
-    label: String,
+    _label: String,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Login {
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct New {
     /// Path to the new file
     pub path: String,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Publish {
     /// The scripts to publish
-    #[structopt(required = true)]
+    #[arg(required = true)]
     pub paths: Vec<String>,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Install {
     /// The script to install
     pub module: ModuleID,
     /// Specify the version, defaults to the latest version
     pub version: Option<String>,
-    #[structopt(short="f", long="force")]
+    #[arg(short = 'f', long="force")]
     pub force: bool,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Search {
     /// Only show modules that aren't installed yet
-    #[structopt(long="new")]
+    #[arg(long="new")]
     pub new: bool,
     /// The search query
     pub query: String,
 }
 
-#[derive(Debug, StructOpt)]
+/// Generate shell completions
+#[derive(Debug, Parser)]
 pub struct Completions {
-    #[structopt(possible_values=&Shell::variants())]
     pub shell: Shell,
 }
+
+impl Completions {
+    pub fn generate(&self) -> Result<()> {
+        clap_complete::generate(
+            self.shell,
+            &mut Args::command(),
+            "sn0int",
+            &mut io::stdout(),
+        );
+        Ok(())
+    }
+}

src/autonoscope/domain.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 
@@ -41,6 +41,13 @@ impl TryFrom<Autonoscope> for DomainRule {
     }
 }
 
+impl AutoRule<Domain> for DomainRule {
+    #[inline]
+    fn matches(&self, domain: &Domain) -> Result<bool> {
+        self.matches(domain.value.as_str())
+    }
+}
+
 impl AutoRule<NewDomain> for DomainRule {
     #[inline]
     fn matches(&self, domain: &NewDomain) -> Result<bool> {
@@ -48,6 +55,13 @@ impl AutoRule<NewDomain> for DomainRule {
     }
 }
 
+impl AutoRule<Subdomain> for DomainRule {
+    #[inline]
+    fn matches(&self, domain: &Subdomain) -> Result<bool> {
+        self.matches(domain.value.as_str())
+    }
+}
+
 impl AutoRule<NewSubdomain> for DomainRule {
     #[inline]
     fn matches(&self, domain: &NewSubdomain) -> Result<bool> {
@@ -55,6 +69,18 @@ impl AutoRule<NewSubdomain> for DomainRule {
     }
 }
 
+impl AutoRule<Url> for DomainRule {
+    #[inline]
+    fn matches(&self, url: &Url) -> Result<bool> {
+        let url = url.value.parse::<url::Url>()?;
+        if let Some(domain) = url.domain() {
+            self.matches(domain)
+        } else {
+            Ok(false)
+        }
+    }
+}
+
 impl AutoRule<NewUrl> for DomainRule {
     #[inline]
     fn matches(&self, url: &NewUrl) -> Result<bool> {
@@ -94,8 +120,8 @@ impl RulePrecision for DomainRule {
     }
 }
 
-impl IntoRule for DomainRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for DomainRule {
+    fn to_rule(&self) -> (&'static str, String) {
         ("domain", self.to_string())
     }
 }

src/autonoscope/ip.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 use std::net;
@@ -36,12 +36,25 @@ impl TryFrom<Autonoscope> for IpRule {
     }
 }
 
+impl AutoRule<IpAddr> for IpRule {
+    fn matches(&self, ipaddr: &IpAddr) -> Result<bool> {
+        self.matches(ipaddr.value.as_str())
+    }
+}
+
 impl AutoRule<NewIpAddr> for IpRule {
     fn matches(&self, ipaddr: &NewIpAddr) -> Result<bool> {
         self.matches(ipaddr.value.as_str())
     }
 }
 
+impl AutoRule<Port> for IpRule {
+    fn matches(&self, port: &Port) -> Result<bool> {
+        let addr = port.value.parse::<net::SocketAddr>()?;
+        self.matches(&addr.ip())
+    }
+}
+
 impl AutoRule<NewPort> for IpRule {
     fn matches(&self, port: &NewPort) -> Result<bool> {
         let addr = port.value.parse::<net::SocketAddr>()?;
@@ -49,15 +62,25 @@ impl AutoRule<NewPort> for IpRule {
     }
 }
 
+fn match_netblock_str(network: &IpNetwork, netblock: &str) -> Result<bool> {
+    let range = netblock.parse::<ipnetwork::IpNetwork>()?;
+
+    if network.prefix() <= range.prefix() {
+        Ok(network.contains(range.ip()))
+    } else {
+        Ok(false)
+    }
+}
+
+impl AutoRule<Netblock> for IpRule {
+    fn matches(&self, netblock: &Netblock) -> Result<bool> {
+        match_netblock_str(&self.network, &netblock.value)
+    }
+}
+
 impl AutoRule<NewNetblock> for IpRule {
     fn matches(&self, netblock: &NewNetblock) -> Result<bool> {
-        let range = netblock.value.parse::<ipnetwork::IpNetwork>()?;
-
-        if self.network.prefix() <= range.prefix() {
-            Ok(self.network.contains(range.ip()))
-        } else {
-            Ok(false)
-        }
+        match_netblock_str(&self.network, &netblock.value)
     }
 }
 
@@ -80,8 +103,8 @@ impl RulePrecision for IpRule {
     }
 }
 
-impl IntoRule for IpRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for IpRule {
+    fn to_rule(&self) -> (&'static str, String) {
         ("ip", self.to_string())
     }
 }

src/autonoscope/mod.rs

@@ -47,6 +47,18 @@ fn sort_precision_desc<T: RulePrecision>(a: &T, b: &T) -> Ordering {
 }
 
 impl RuleSet {
+    pub fn domains(&self) -> &[Rule<DomainRule>] {
+        &self.domains
+    }
+
+    pub fn ips(&self) -> &[Rule<IpRule>] {
+        &self.ips
+    }
+
+    pub fn urls(&self) -> &[Rule<UrlRule>] {
+        &self.urls
+    }
+
     pub fn load(db: &DatabaseSock) -> Result<Self> {
         use crate::schema::autonoscope::dsl::*;
         let rules = autonoscope.load::<Autonoscope>(db)?;
@@ -112,15 +124,15 @@ impl RuleSet {
         match obj {
             RuleType::Domain => {
                 self.domains.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
             },
             RuleType::Ip => {
                 self.ips.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
             },
             RuleType::Url => {
                 self.urls.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
             },
         }
         Ok(())
@@ -143,10 +155,18 @@ impl RuleSet {
         rules
     }
 
+    pub fn is_empty(&self) -> bool {
+        self.domains.is_empty() && self.ips.is_empty() && self.urls.is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        self.domains.len() + self.ips.len() + self.urls.len()
+    }
+
     #[inline]
-    fn push_rules_display<T: IntoRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
+    fn push_rules_display<T: ToRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
         for rule in rules {
-            let (object, value) = rule.into_rule();
+            let (object, value) = rule.to_rule();
             output.push((object, value, rule.scoped));
         }
     }
@@ -159,10 +179,8 @@ impl RuleSet {
             Insert::Url(url) => {
                 if let Some(result) = Self::matches_any(&self.domains, url)? {
                     Some(result)
-                } else if let Some(result) = Self::matches_any(&self.urls, url)? {
-                    Some(result)
                 } else {
-                    None
+                    Self::matches_any(&self.urls, url)?
                 }
             },
             // Insert::Email(email) => unimplemented!(),
@@ -176,7 +194,7 @@ impl RuleSet {
 
     fn matches_any<T1, T2>(rules: &[Rule<T1>], object: &T2) -> Result<Option<bool>>
         where T1: AutoRule<T2>,
-            T1: IntoRule,
+            T1: ToRule,
     {
         for rule in rules {
             if rule.matches(object)? {
@@ -187,7 +205,7 @@ impl RuleSet {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum RuleType {
     Domain,
     Ip,
@@ -238,12 +256,12 @@ pub trait RulePrecision {
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Rule<T: IntoRule> {
+pub struct Rule<T: ToRule> {
     rule: T,
-    scoped: bool,
+    pub scoped: bool,
 }
 
-impl<T: IntoRule> Rule<T> {
+impl<T: ToRule> Rule<T> {
     pub fn new(rule: T, scoped: bool) -> Rule<T> {
         Rule {
             rule,
@@ -254,7 +272,7 @@ impl<T: IntoRule> Rule<T> {
 
 // TODO: maybe drop this
 use std::ops::Deref;
-impl<T: IntoRule> Deref for Rule<T> {
+impl<T: ToRule> Deref for Rule<T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -262,23 +280,23 @@ impl<T: IntoRule> Deref for Rule<T> {
     }
 }
 
-impl<T: IntoRule + RulePrecision> RulePrecision for Rule<T> {
+impl<T: ToRule + RulePrecision> RulePrecision for Rule<T> {
     fn precision(&self) -> usize {
         self.rule.precision()
     }
 }
 
-pub trait IntoRule {
-    fn into_rule(&self) -> (&'static str, String);
+pub trait ToRule {
+    fn to_rule(&self) -> (&'static str, String);
 }
 
-impl<T: IntoRule> Into<NewAutonoscope> for &Rule<T> {
-    fn into(self) -> NewAutonoscope {
-        let (object, value) = self.rule.into_rule();
+impl<T: ToRule> From<&Rule<T>> for NewAutonoscope {
+    fn from(rule: &Rule<T>) -> NewAutonoscope {
+        let (object, value) = rule.rule.to_rule();
         NewAutonoscope {
             object: object.to_string(),
             value,
-            scoped: self.scoped,
+            scoped: rule.scoped,
         }
     }
 }

src/autonoscope/url.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 
@@ -47,6 +47,12 @@ impl TryFrom<&str> for UrlRule {
 }
 
 // TODO: there is no way to write a rule that matches all urls
+impl AutoRule<Url> for UrlRule {
+    fn matches(&self, url: &Url) -> Result<bool> {
+        self.matches(url.value.as_str())
+    }
+}
+
 impl AutoRule<NewUrl> for UrlRule {
     fn matches(&self, url: &NewUrl) -> Result<bool> {
         self.matches(url.value.as_str())
@@ -86,8 +92,8 @@ impl RulePrecision for UrlRule {
     }
 }
 
-impl IntoRule for UrlRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for UrlRule {
+    fn to_rule(&self) -> (&'static str, String) {
         ("url", self.to_string())
     }
 }

src/blobs.rs

@@ -93,6 +93,14 @@ impl BlobStorage {
         }
         Ok(blobs)
     }
+
+    pub fn stat(&self, id: &str) -> Result<u64> {
+        let path = self.join(id)?;
+        debug!("Stat-ing blob: {:?}", path);
+        let md = fs::metadata(path)
+            .context("Failed to stat blob")?;
+        Ok(md.len())
+    }
 }
 
 

src/cal/date.rs

@@ -6,6 +6,7 @@ use crate::cal::{ActivityGrade, DateArg};
 use crate::models::*;
 use std::collections::HashMap;
 use std::collections::VecDeque;
+use std::fmt::Write;
 
 const MONTH_LINES: i32 = 7;
 
@@ -54,11 +55,11 @@ fn chunk_months(ctx: &DateContext, months: &[DateSpec]) -> String {
 }
 
 fn days_in_month(year: i32, month: u32) -> i64 {
-    let start = Utc.ymd(year, month, 1);
+    let start = Utc.with_ymd_and_hms(year, month, 1, 0, 0, 0).single().expect("Datetime is not unique");
     let end = if month == 12 {
-        Utc.ymd(year + 1, 1, 1)
+        Utc.with_ymd_and_hms(year + 1, 1, 1, 0, 0, 0).single().expect("Datetime is not unique")
     } else {
-        Utc.ymd(year, month + 1, 1)
+        Utc.with_ymd_and_hms(year, month + 1, 1, 0, 0, 0).single().expect("Datetime is not unique")
     };
     end.signed_duration_since(start).num_days()
 }
@@ -112,7 +113,7 @@ pub struct DateContext {
 
 impl DateContext {
     pub fn new(events: &[Activity], today: NaiveDate) -> DateContext {
-        let (events, max) = setup_graph_map(&events);
+        let (events, max) = setup_graph_map(events);
         DateContext {
             events,
             max,
@@ -151,7 +152,7 @@ impl DateSpec {
             bail!("Too many datespec args");
         }
 
-        let today = Utc::today();
+        let today = Utc::now();
         let ds = match (args.get(0), args.get(1), context) {
             (None, _, None) => DateSpec::YearMonth((today.year(), today.month())),
             (None, _, Some(context)) => DateSpec::YearMonthContext((today.year(), today.month(), context)),
@@ -171,8 +172,8 @@ impl DateSpec {
 
     pub fn start(&self) -> NaiveDate {
         match self {
-            DateSpec::Year(year) => NaiveDate::from_ymd(*year, 1, 1),
-            DateSpec::YearMonth((year, month)) => NaiveDate::from_ymd(*year, *month, 1),
+            DateSpec::Year(year) => NaiveDate::from_ymd_opt(*year, 1, 1).expect("Invalid month/day"),
+            DateSpec::YearMonth((year, month)) => NaiveDate::from_ymd_opt(*year, *month, 1).expect("Invalid month/day"),
             DateSpec::YearMonthContext((year, month, context)) => {
                 let mut year = *year - (*context / 12) as i32;
                 let context = context % 12;
@@ -182,21 +183,21 @@ impl DateSpec {
                 } else {
                     month - context
                 };
-                NaiveDate::from_ymd(year, month, 1)
+                NaiveDate::from_ymd_opt(year, month, 1).expect("Invalid month/day")
             },
         }
     }
 
     pub fn end(&self) -> NaiveDate {
         match self {
-            DateSpec::Year(year) => NaiveDate::from_ymd(year + 1, 1, 1),
+            DateSpec::Year(year) => NaiveDate::from_ymd_opt(year + 1, 1, 1).expect("Invalid month/day"),
             DateSpec::YearMonth((year, month)) => {
                 let (year, month) = if *month == 12 {
                     (*year + 1, 1)
                 } else {
                     (*year, *month + 1)
                 };
-                NaiveDate::from_ymd(year, month, 1)
+                NaiveDate::from_ymd_opt(year, month, 1).expect("Invalid month/day")
             },
             DateSpec::YearMonthContext((year, month, _context)) => {
                 let (year, month) = if *month == 12 {
@@ -204,7 +205,7 @@ impl DateSpec {
                 } else {
                     (*year, *month + 1)
                 };
-                NaiveDate::from_ymd(year, month, 1)
+                NaiveDate::from_ymd_opt(year, month, 1).expect("Invalid month/day")
             },
         }
     }
@@ -220,10 +221,10 @@ impl DateSpec {
             DateSpec::YearMonth((year, month)) => {
                 let mut w = String::new();
 
-                let start = Utc.ymd(*year, *month, 1);
+                let start = Utc.with_ymd_and_hms(*year, *month, 1, 0, 0, 0).single().expect("Datetime is not unique");
                 let days = days_in_month(*year, *month) as u32;
 
-                w.push_str(&format!("{:^21}\n", start.format("%B %Y")));
+                writeln!(w, "{:^21}", start.format("%B %Y")).expect("out of memory");
                 w.push_str(" Su Mo Tu We Th Fr Sa\n");
 
                 let mut cur_week_day = start.weekday();
@@ -232,7 +233,7 @@ impl DateSpec {
 
                 let mut week_written = week_progress * 3;
                 for cur_day in 1..=days {
-                    let date = NaiveDate::from_ymd(*year, *month, cur_day);
+                    let date = NaiveDate::from_ymd_opt(*year, *month, cur_day).expect("Invalid month/day");
 
                     if !ctx.is_future(&date) {
                         let activity = ctx.activity_for_day(&date);
@@ -244,7 +245,7 @@ impl DateSpec {
                     } else {
                         w.push(' ');
                     }
-                    w.push_str(&format!("{:2}", cur_day));
+                    write!(w, "{:2}", cur_day).expect("out of memory");
                     week_written += 3;
                     w.push_str("\x1b[0m");
 
@@ -296,25 +297,25 @@ mod tests {
         DateContext {
             events: HashMap::new(),
             max: 0,
-            today: NaiveDate::from_ymd(2020, 05, 30),
+            today: NaiveDate::from_ymd_opt(2020, 5, 30).unwrap(),
         }
     }
 
     #[test]
     fn test_days_in_month_2020_05() {
-        let days = days_in_month(2020, 05);
+        let days = days_in_month(2020, 5);
         assert_eq!(days, 31);
     }
 
     #[test]
     fn test_days_in_month_2020_04() {
-        let days = days_in_month(2020, 04);
+        let days = days_in_month(2020, 4);
         assert_eq!(days, 30);
     }
 
     #[test]
     fn test_days_in_month_2020_02() {
-        let days = days_in_month(2020, 02);
+        let days = days_in_month(2020, 2);
         assert_eq!(days, 29);
     }
 
@@ -324,15 +325,15 @@ mod tests {
         let ctx = DateContext {
             events,
             max: 0,
-            today: NaiveDate::from_ymd(2020, 06, 06),
+            today: NaiveDate::from_ymd_opt(2020, 6, 6).unwrap(),
         };
-        let grade = ctx.activity_for_day(&NaiveDate::from_ymd(2020, 06, 06));
+        let grade = ctx.activity_for_day(&NaiveDate::from_ymd_opt(2020, 6, 6).unwrap());
         assert_eq!(grade, ActivityGrade::None);
     }
 
     #[test]
     fn test_datespec_year_month() {
-        let ds = DateSpec::YearMonth((2020, 05));
+        let ds = DateSpec::YearMonth((2020, 5));
         let out = ds.to_term_string(&context());
         assert_eq!(out, "      May 2020       
  Su Mo Tu We Th Fr Sa

src/cal/mod.rs

@@ -1,10 +1,10 @@
-use crate::errors::*;
-use std::str::FromStr;
-
 pub mod date;
 pub mod time;
 
-#[derive(Debug)]
+use crate::errors::*;
+use std::str::FromStr;
+
+#[derive(Debug, Clone)]
 pub enum DateArg {
     Month(u32),
     Num(i32),

src/cal/time.rs

@@ -5,6 +5,7 @@ use chrono::prelude::*;
 use crate::cal::{ActivityGrade, DateArg};
 use crate::models::*;
 use std::collections::HashMap;
+use std::fmt::Write;
 
 const MIN_PER_DAY: u32 = 1440;
 
@@ -13,7 +14,7 @@ fn round_to_slice(time: &NaiveDateTime, slice_duration: u32) -> NaiveDateTime {
     let hour = time.hour();
     let mins = time.minute();
     let slice = mins - (mins % slice_duration);
-    date.and_hms(hour, slice, 0)
+    date.and_hms_opt(hour, slice, 0).expect("Invalid hour/min/sec")
 }
 
 fn setup_graph_map(events: &[Activity], slice_duration: u32) -> (HashMap<NaiveDateTime, u64>, u64) {
@@ -68,7 +69,7 @@ pub struct DateTimeContext {
 
 impl DateTimeContext {
     pub fn new(events: &[Activity], now: NaiveDateTime, slice_width: u32, slice_duration: u32) -> DateTimeContext {
-        let (events, max) = setup_graph_map(&events, slice_duration);
+        let (events, max) = setup_graph_map(events, slice_duration);
         DateTimeContext {
             events,
             max,
@@ -109,7 +110,7 @@ pub struct DateTimeSpec {
 
 impl DateTimeSpec {
     pub fn from_args(args: &[DateArg], context: Option<u32>) -> Result<DateTimeSpec> {
-        let today = Utc::today().naive_utc();
+        let today = Utc::now().date_naive();
         if args.is_empty() {
             let mut start = today;
 
@@ -156,7 +157,7 @@ impl DateTimeSpec {
         // add legend
         w.push_str(&" ".repeat(11));
         for x in 0..24 {
-            w.push_str(&format!("{:02}", x));
+            write!(w, "{:02}", x).expect("out of memory");
 
             for i in 0..ctx.hour_width() {
                 if i >= 2 {
@@ -175,7 +176,7 @@ impl DateTimeSpec {
             let mut mins = 0;
 
             for _ in 0..(MIN_PER_DAY / ctx.slice_duration) {
-                let time = date.and_hms(hours, mins, 0);
+                let time = date.and_hms_opt(hours, mins, 0).expect("Invalid hour/min/sec");
 
                 if !ctx.is_future(&time) {
                     let activity = ctx.activity_for_slice(&time);

src/cmd/activity_cmd.rs

@@ -1,24 +1,22 @@
-use crate::errors::*;
-
-use crate::cmd::Cmd;
-use crate::shell::Shell;
-use crate::models::*;
 use chrono::{Utc, NaiveDateTime, NaiveTime, Duration};
+use clap::Parser;
+use crate::cmd::Cmd;
+use crate::errors::*;
+use crate::models::*;
+use crate::shell::Shell;
 use regex::Regex;
 use std::convert::TryFrom;
 use std::io;
 use std::str::FromStr;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct TimeSpec {
     datetime: NaiveDateTime,
 }
 
 impl TimeSpec {
     fn resolve(s: &str, now: NaiveDateTime) -> Result<Self> {
-        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
+        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms_opt(0, 0, 0).expect("Invalid hour/min/sec"));
 
         let datetime = match s {
             "today" => today,
@@ -64,23 +62,22 @@ impl FromStr for TimeSpec {
     }
 }
 
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
     /// Only query events for a given topic
-    #[structopt(short="t", long="topic")]
+    #[arg(short = 't', long="topic")]
     topic: Option<String>,
     /// Only query events starting from that datetime
-    #[structopt(long="since")]
+    #[arg(long="since")]
     since: Option<TimeSpec>,
     /// Only query events until this datetime
-    #[structopt(long="until")]
+    #[arg(long="until")]
     until: Option<TimeSpec>,
     /// Try to select the previous event before --since as an initial state
-    #[structopt(short="i", long="initial")]
+    #[arg(short = 'i', long="initial")]
     initial: bool,
     /// Only query events that are tied to a location
-    #[structopt(short="l", long="location")]
+    #[arg(short = 'l', long="location")]
     location: bool,
 }
 
@@ -122,8 +119,8 @@ mod tests {
     use super::*;
 
     fn datetime() -> NaiveDateTime {
-        let date = chrono::NaiveDate::from_ymd(2020, 3, 14);
-        let time = chrono::NaiveTime::from_hms(16, 20, 23);
+        let date = chrono::NaiveDate::from_ymd_opt(2020, 3, 14).unwrap();
+        let time = chrono::NaiveTime::from_hms_opt(16, 20, 23).unwrap();
         NaiveDateTime::new(date, time)
     }
 

src/cmd/add_cmd.rs

@@ -1,13 +1,11 @@
 use crate::errors::*;
-
 use crate::blobs::Blob;
 use crate::cmd::Cmd;
 use crate::db::DbChange;
 use crate::gfx;
 use crate::models::*;
 use crate::shell::Shell;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use clap::Parser;
 use crate::utils;
 use crate::term;
 use std::fmt::Debug;
@@ -18,62 +16,61 @@ use std::net::SocketAddr;
 use std::path::Path;
 use walkdir::WalkDir;
 
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
-    #[structopt(subcommand)]
+    #[command(subcommand)]
     subcommand: Target,
     /// Do not actually insert into database
-    #[structopt(short="n", long="dry-run")]
+    #[arg(short = 'n', long="dry-run")]
     dry_run: bool,
     /// Stream structs from stdin line by line
-    #[structopt(long)]
+    #[arg(long)]
     stdin: bool,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Target {
     /// Insert domain into the database
-    #[structopt(name="domain")]
+    #[command(name="domain")]
     Domain(AddDomain),
     /// Insert subdomain into the database
-    #[structopt(name="subdomain")]
+    #[command(name="subdomain")]
     Subdomain(AddSubdomain),
     /// Insert ip address into the database
-    #[structopt(name="ipaddr")]
+    #[command(name="ipaddr")]
     IpAddr(AddIpAddr),
     /// Insert url into the database
-    #[structopt(name="url")]
+    #[command(name="url")]
     Url(AddUrl),
     /// Insert email into the database
-    #[structopt(name="email")]
+    #[command(name="email")]
     Email(AddEmail),
     /// Insert phonenumber into the database
-    #[structopt(name="phonenumber")]
+    #[command(name="phonenumber")]
     PhoneNumber(AddPhoneNumber),
     /// Insert device into the database
-    #[structopt(name="device")]
+    #[command(name="device")]
     Device(AddDevice),
     /// Insert network into the database
-    #[structopt(name="network")]
+    #[command(name="network")]
     Network(AddNetwork),
     /// Insert account into the database
-    #[structopt(name="account")]
+    #[command(name="account")]
     Account(AddAccount),
     /// Insert breach into the database
-    #[structopt(name="breach")]
+    #[command(name="breach")]
     Breach(AddBreach),
     /// Insert images into the database
-    #[structopt(name="image")]
+    #[command(name="image")]
     Image(AddImage),
     /// Insert ip network into the database
-    #[structopt(name="netblock")]
+    #[command(name="netblock")]
     Netblock(AddNetblock),
     /// Insert port into the database
-    #[structopt(name="port")]
+    #[command(name="port")]
     Port(AddPort),
     /// Insert a crypto currency address into the database
-    #[structopt(name="cryptoaddr")]
+    #[command(name="cryptoaddr")]
     CryptoAddr(AddCryptoAddr),
 }
 
@@ -153,7 +150,7 @@ trait IntoInsert: Sized {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddDomain {
     domain: Option<String>,
 }
@@ -185,7 +182,7 @@ impl InsertFromString for AddDomain {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddSubdomain {
     subdomain: Option<String>,
 }
@@ -222,7 +219,7 @@ impl InsertFromString for AddSubdomain {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddIpAddr {
     ipaddr: Option<net::IpAddr>,
 }
@@ -269,7 +266,7 @@ impl InsertFromString for AddIpAddr {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddUrl {
     url: Option<String>,
 }
@@ -290,7 +287,7 @@ impl InsertFromString for AddUrl {
         let subdomain = parts.domain()
             .ok_or_else(|| format_err!("url doesn't have a domain host"))?;
 
-        let dns_name = rl.psl()?.parse_dns_name(&subdomain)
+        let dns_name = rl.psl()?.parse_dns_name(subdomain)
             .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
 
         let domain_id = match rl.db().insert_struct(NewDomain {
@@ -323,7 +320,7 @@ impl InsertFromString for AddUrl {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddEmail {
     email: Option<String>,
 }
@@ -350,7 +347,7 @@ impl InsertFromString for AddEmail {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddPhoneNumber {
     phonenumber: Option<String>,
     name: Option<String>,
@@ -405,7 +402,7 @@ impl InsertFromString for AddPhoneNumber {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddDevice {
     mac: Option<String>,
     name: Option<String>,
@@ -435,7 +432,7 @@ impl IntoInsert for AddDevice {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddNetwork {
     network: Option<String>,
     latitude: Option<f32>,
@@ -464,7 +461,7 @@ impl IntoInsert for AddNetwork {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddAccount {
     service: Option<String>,
     username: Option<String>,
@@ -506,7 +503,7 @@ impl IntoInsert for AddAccount {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddBreach {
     name: Option<String>,
 }
@@ -525,7 +522,7 @@ impl IntoInsert for AddBreach {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddImage {
     paths: Vec<String>,
 }
@@ -624,7 +621,7 @@ impl IntoInsert for AddImage {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddNetblock {
     ipnet: Option<ipnetwork::IpNetwork>,
 }
@@ -655,7 +652,7 @@ impl IntoInsert for AddNetblock {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddPort {
     protocol: Option<String>,
     addr: Option<SocketAddr>,
@@ -730,7 +727,7 @@ impl InsertFromString for AddPort {
     }
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct AddCryptoAddr {
     address: Option<String>,
 }

src/cmd/autonoscope_cmd.rs

@@ -1,36 +1,35 @@
 use crate::errors::*;
-
 use crate::autonoscope;
+use crate::cmd::Cmd;
 use crate::fmt::colors::*;
 use crate::shell::Shell;
 use std::fmt::Write;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use clap::Parser;
 
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
+#[group(skip)]
 pub struct Args {
-    #[structopt(subcommand)]
+    #[command(subcommand)]
     subcommand: Subcommand,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Subcommand {
-    #[structopt(name="add")]
+    #[command(name="add")]
     Add(Add),
-    #[structopt(name="delete")]
+    #[command(name="delete")]
     Delete(Delete),
-    #[structopt(name="list")]
+    #[command(name="list")]
     List,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Add {
     object: autonoscope::RuleType,
     value: String,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Delete {
     object: autonoscope::RuleType,
     value: String,
@@ -65,7 +64,8 @@ pub fn run_with_scope_param(rl: &mut Shell, args: Args, scoped: bool) -> Result<
     }
 }
 
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    run_with_scope_param(rl, args, false)
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        run_with_scope_param(rl, self, false)
+    }
 }

src/cmd/autoscope_cmd.rs

@@ -1,13 +1,17 @@
-use crate::errors::*;
-
+use clap::Parser;
+use crate::cmd::Cmd;
 use crate::cmd::autonoscope_cmd;
+use crate::errors::*;
 use crate::shell::Shell;
-use structopt::StructOpt;
 
-
-pub type Args = autonoscope_cmd::Args;
-
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    autonoscope_cmd::run_with_scope_param(rl, args, true)
+#[derive(Debug, Parser)]
+pub struct Args {
+    #[structopt(flatten)]
+    args: autonoscope_cmd::Args,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        autonoscope_cmd::run_with_scope_param(rl, self.args, true)
+    }
 }

src/cmd/cal_cmd.rs

@@ -1,5 +1,4 @@
 use crate::errors::*;
-
 use chrono::Utc;
 use crate::cal::DateArg;
 use crate::cal::date::{DateContext, DateSpec};
@@ -7,21 +6,18 @@ use crate::cal::time::{DateTimeContext, DateTimeSpec};
 use crate::cmd::Cmd;
 use crate::models::*;
 use crate::shell::Shell;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use clap::Parser;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
     /// Show additional months for context
-    #[structopt(short="C", long)]
+    #[arg(short = 'C', long)]
     context: Option<u32>,
     /// Group events in 12 min slices
-    #[structopt(short="T", long, group = "view")]
+    #[arg(short = 'T', long, group = "view")]
     time: bool,
     /// Group events by hour
-    #[structopt(short="H", long, group = "view")]
+    #[arg(short = 'H', long, group = "view")]
     hourly: bool,
     args: Vec<DateArg>,
 }
@@ -34,8 +30,8 @@ impl Cmd for Args {
                 .context("Failed to parse date spec")?;
             let filter = ActivityFilter {
                 topic: None,
-                since: Some(dts.start().and_hms(0, 0, 0)),
-                until: Some(dts.end().and_hms(23, 59, 59)),
+                since: Some(dts.start().and_hms_opt(0, 0, 0).expect("Invalid hour/min/sec")),
+                until: Some(dts.end().and_hms_opt(23, 59, 59).expect("Invalid hour/min/sec")),
                 location: false,
             };
             let events = Activity::query(rl.db(), &filter)?;
@@ -53,12 +49,12 @@ impl Cmd for Args {
                 .context("Failed to parse date spec")?;
             let filter = ActivityFilter {
                 topic: None,
-                since: Some(ds.start().and_hms(0, 0, 0)),
-                until: Some(ds.end().and_hms(23, 59, 59)),
+                since: Some(ds.start().and_hms_opt(0, 0, 0).expect("Invalid hour/min/sec")),
+                until: Some(ds.end().and_hms_opt(23, 59, 59).expect("Invalid hour/min/sec")),
                 location: false,
             };
             let events = Activity::query(rl.db(), &filter)?;
-            let ctx = DateContext::new(&events, Utc::today().naive_utc());
+            let ctx = DateContext::new(&events, Utc::now().date_naive());
             println!("{}", ds.to_term_string(&ctx));
         }
         Ok(())

src/cmd/delete_cmd.rs

@@ -1,16 +1,12 @@
 use crate::errors::*;
-
 use crate::cmd::Cmd;
 use crate::filters::{Target, Filter};
 use crate::shell::Shell;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use clap::Parser;
 use crate::models::*;
 use crate::term;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
     #[structopt(subcommand)]
     subcommand: Target,
@@ -19,20 +15,20 @@ pub struct Args {
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
         let rows = match &self.subcommand {
-            Target::Domains(filter) => delete::<Domain>(rl, &filter),
-            Target::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
-            Target::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
-            Target::Urls(filter) => delete::<Url>(rl, &filter),
-            Target::Emails(filter) => delete::<Email>(rl, &filter),
-            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
-            Target::Devices(filter) => delete::<Device>(rl, &filter),
-            Target::Networks(filter) => delete::<Network>(rl, &filter),
-            Target::Accounts(filter) => delete::<Account>(rl, &filter),
-            Target::Breaches(filter) => delete::<Breach>(rl, &filter),
-            Target::Images(filter) => delete::<Image>(rl, &filter),
-            Target::Ports(filter) => delete::<Port>(rl, &filter),
-            Target::Netblocks(filter) => delete::<Netblock>(rl, &filter),
-            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, &filter),
+            Target::Domains(filter) => delete::<Domain>(rl, filter),
+            Target::Subdomains(filter) => delete::<Subdomain>(rl, filter),
+            Target::IpAddrs(filter) => delete::<IpAddr>(rl, filter),
+            Target::Urls(filter) => delete::<Url>(rl, filter),
+            Target::Emails(filter) => delete::<Email>(rl, filter),
+            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, filter),
+            Target::Devices(filter) => delete::<Device>(rl, filter),
+            Target::Networks(filter) => delete::<Network>(rl, filter),
+            Target::Accounts(filter) => delete::<Account>(rl, filter),
+            Target::Breaches(filter) => delete::<Breach>(rl, filter),
+            Target::Images(filter) => delete::<Image>(rl, filter),
+            Target::Ports(filter) => delete::<Port>(rl, filter),
+            Target::Netblocks(filter) => delete::<Netblock>(rl, filter),
+            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, filter),
         }?;
         term::info(&format!("Deleted {} rows", rows));
         Ok(())

src/cmd/export_cmd.rs

@@ -1,21 +1,19 @@
-use crate::errors::*;
-
+use clap::Parser;
+use clap::ValueEnum;
 use crate::blobs::Blob;
 use crate::cmd::Cmd;
 use crate::db::ttl;
+use crate::errors::*;
 use crate::models::*;
 use crate::shell::Shell;
 use serde::{Serialize, Deserialize};
 use std::io::{self, Write};
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
 use strum_macros::{EnumString, IntoStaticStr};
 
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
     /// Specify the export format
-    #[structopt(short="f", long="format", possible_values=Format::variants())]
+    #[arg(short = 'f', long="format", value_enum)]
     format: Format,
 }
 
@@ -37,7 +35,7 @@ fn export<T: ExportFormat + Serialize>(rl: &mut Shell) -> Result<()> {
     Ok(())
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, ValueEnum, Serialize, Deserialize)]
 #[derive(EnumString, IntoStaticStr)]
 #[strum(serialize_all = "kebab_case")]
 pub enum Format {
@@ -45,17 +43,6 @@ pub enum Format {
     JsonBlobs,
 }
 
-impl Format {
-    // TODO: this function should be generated by strum instead
-    #[inline]
-    fn variants() -> &'static [&'static str] {
-        &[
-            "json",
-            "json-blobs",
-        ]
-    }
-}
-
 trait ExportFormat {
     fn load(rl: &mut Shell) -> Result<Box<Self>>;
 }

src/cmd/fsck_cmd.rs

@@ -1,25 +1,23 @@
-use crate::errors::*;
-
+use clap::{ArgAction, Parser};
 use crate::blobs::Blob;
 use crate::cmd::Cmd;
+use crate::errors::*;
+use crate::models::*;
 use crate::shell::Shell;
 use crate::term;
 use crate::worker;
-use crate::models::*;
 use std::collections::HashSet;
-use structopt::StructOpt;
 
-
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Args {
     /// Verbose output
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    verbose: u64,
+    #[arg(short = 'v', long="verbose", action(ArgAction::Count))]
+    verbose: u8,
     /// Delete only dangling blobs
-    #[structopt(long="gc")]
+    #[arg(long="gc")]
     gc: bool,
     /// Delete dangling and corrupted blobs
-    #[structopt(long="gc-all")]
+    #[arg(long="gc-all")]
     gc_all: bool,
 }
 

src/cmd/help_cmd.rs

@@ -21,6 +21,7 @@ pub fn run(_rl: &mut Shell, _args: &[String]) -> Result<()> {
     help("run",         "Run the currently selected module");
     help("scope",       "Include entities in the scope again");
     help("select",      "Select entities from the database");
+    help("stats",       "Show statistics about your current workspace");
     help("target",      "Preview targeted entities or narrow them down");
     help("use",         "Select a module");
     help("workspace",   "Switch to a different workspace");

src/cmd/keyring_cmd.rs

@@ -1,56 +1,47 @@
+use clap::Parser;
 use crate::errors::*;
-
 use crate::keyring::{KeyName, KeyRing};
 use crate::shell::Shell;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
 use crate::utils;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub enum Args {
-    #[structopt(name="add")]
     /// Add a new key to the keyring
     Add(KeyRingAdd),
-    #[structopt(name="delete")]
     /// Delete a key from the keyring
     Delete(KeyRingDelete),
-    #[structopt(name="get")]
     /// Get a key from the keyring
     Get(KeyRingGet),
-    #[structopt(name="list")]
     /// List keys in the keyring
     List(KeyRingList),
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct KeyRingAdd {
     key: KeyName,
     secret: Option<String>,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct KeyRingDelete {
     key: KeyName,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct KeyRingGet {
     key: KeyName,
-    #[structopt(short="q",
-                long="quiet")]
+    #[arg(short = 'q', long="quiet")]
     /// Only output secret key
     quiet: bool,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct KeyRingList {
     namespace: Option<String>,
 }
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
+    let args = Args::try_parse_from(args)?;
     match args {
         Args::Add(add) => keyring_add(rl, add),
         Args::Delete(delete) => keyring_delete(rl, delete),

src/cmd/mod.rs

@@ -2,17 +2,17 @@ use crate::errors::*;
 use crate::shell::Shell;
 use crate::config::Config;
 
-pub trait Cmd: structopt::StructOpt + Sized {
+pub trait Cmd: clap::Parser + Sized {
     fn run(self, rl: &mut Shell) -> Result<()>;
 
     #[inline]
     fn run_str(rl: &mut Shell, args: &[String]) -> Result<()> {
-        let args = Self::from_iter_safe(args)?;
+        let args = Self::try_parse_from(args)?;
         args.run(rl)
     }
 }
 
-pub trait LiteCmd: structopt::StructOpt + Sized {
+pub trait LiteCmd: clap::Parser + Sized {
     fn run(self, config: &Config) -> Result<()>;
 }
 
@@ -32,8 +32,10 @@ pub mod keyring_cmd;
 pub mod noscope_cmd;
 pub mod notify_cmd;
 pub mod pkg_cmd;
+pub mod rescope_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;
+pub mod stats_cmd;
 pub mod target_cmd;
 pub mod quickstart_cmd;
 pub mod workspace_cmd;

src/cmd/noscope_cmd.rs

@@ -1,18 +1,14 @@
-use crate::errors::*;
-
+use clap::Parser;
 use crate::cmd::Cmd;
+use crate::errors::*;
 use crate::filters::{Target, Filter};
-use crate::shell::Shell;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
 use crate::models::*;
+use crate::shell::Shell;
 use crate::term;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
-    #[structopt(subcommand)]
+    #[command(subcommand)]
     subcommand: Target,
 }
 
@@ -40,7 +36,7 @@ impl Cmd for Args {
 }
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
+    let args = Args::try_parse_from(args)?;
     args.run(rl)
 }
 

src/cmd/notify_cmd.rs

@@ -1,24 +1,21 @@
-use crate::errors::*;
-
+use clap::{ArgAction, Parser};
 use crate::cmd::Cmd;
 use crate::engine::Module;
-// use crate::models::*;
+use crate::errors::*;
 use crate::notify::{self, Notification};
 use crate::options::{self, Opt};
 use crate::shell::Shell;
 use crate::term;
 use sn0int_std::ratelimits::Ratelimiter;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use std::fmt::Write;
 
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
-    #[structopt(subcommand)]
+    #[command(subcommand)]
     subcommand: Subcommand,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Subcommand {
     /// Manually add a notification to the outbox
     Send(SendArgs),
@@ -30,41 +27,41 @@ pub enum Subcommand {
     Deliver,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct SendArgs {
     /// Evaluate the routing rules, but do not actually send a notification
-    #[structopt(short="n", long)]
+    #[arg(short = 'n', long)]
     pub dry_run: bool,
     pub topic: String,
-    #[structopt(flatten)]
+    #[command(flatten)]
     pub notification: Notification,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct ExecArgs {
     pub module: String,
-    #[structopt(short="o", long="option")]
+    #[arg(short = 'o', long="option")]
     pub options: Vec<options::Opt>,
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    verbose: u64,
-    #[structopt(flatten)]
+    #[arg(short = 'v', long="verbose", action(ArgAction::Count))]
+    verbose: u8,
+    #[command(flatten)]
     pub notification: Notification,
 }
 
 fn print_summary(module: &Module, sent: usize, errors: usize) {
-        let mut out = if sent == 1 {
-            String::from("Sent 1 notification")
-        } else {
-            format!("Sent {} notifications", sent)
-        };
+    let mut out = if sent == 1 {
+        String::from("Sent 1 notification")
+    } else {
+        format!("Sent {} notifications", sent)
+    };
 
-        out.push_str(&format!(" with {}", module.canonical()));
+    write!(out, " with {}", module.canonical()).expect("out of memory");
 
-        if errors > 0 {
-            out.push_str(&format!(" ({} errors)", errors));
-        }
+    if errors > 0 {
+        write!(out, " ({} errors)", errors).expect("out of memory");
+    }
 
-        term::info(&out);
+    term::info(&out);
 }
 
 fn send(args: SendArgs, rl: &mut Shell) -> Result<()> {

src/cmd/pkg_cmd.rs

@@ -1,5 +1,4 @@
 use crate::errors::*;
-
 use crate::args::Install;
 use crate::api::Client;
 use crate::args;
@@ -16,80 +15,76 @@ use sn0int_common::metadata::Stealth;
 use std::collections::HashSet;
 use std::fmt::Write;
 use std::sync::Arc;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use clap::Parser;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
-    #[structopt(subcommand)]
+    #[command(subcommand)]
     pub subcommand: SubCommand,
 }
 
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct ArgsInteractive {
-    #[structopt(subcommand)]
+    #[command(subcommand)]
     pub subcommand: SubCommandInteractive,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum SubCommand {
     /// List installed modules
-    #[structopt(name="list")]
+    #[command(name="list")]
     List(List),
     /// Install module from registry
-    #[structopt(name="install")]
+    #[command(name="install")]
     Install(args::Install),
     /// Search modules in registry
-    #[structopt(name="search")]
+    #[command(name="search")]
     Search(args::Search),
     /// Update modules
-    #[structopt(name="update")]
+    #[command(name="update")]
     Update(Update),
     /// Uninstall a module
-    #[structopt(name="uninstall")]
+    #[command(name="uninstall")]
     Uninstall(Uninstall),
     /// Install all featured modules
-    #[structopt(name="quickstart")]
+    #[command(name="quickstart")]
     Quickstart,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum SubCommandInteractive {
-    #[structopt(flatten)]
+    #[command(flatten)]
     Base(SubCommand),
     /// Reload modules
-    #[structopt(name="reload")]
+    #[command(name="reload")]
     Reload(Reload),
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct List {
     /// Only show modules with a specific input source
-    #[structopt(long="source")]
+    #[arg(long="source")]
     pub source: Option<String>,
     /// List outdated modules
-    #[structopt(long="outdated")]
+    #[arg(long="outdated")]
     pub outdated_only: bool,
     /// Only show modules with equal or better stealth level
-    #[structopt(long="stealth", possible_values=Stealth::variants())]
+    #[arg(long="stealth", value_enum)]
     pub stealth: Option<Stealth>,
     /// Filter by pattern
-    #[structopt(default_value="*")]
+    #[arg(default_value="*")]
     pub pattern: String,
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Reload {
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Update {
 }
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub struct Uninstall {
     module: ModuleID,
 }
@@ -137,7 +132,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
 
             for module in library.list() {
                 if let Some(source) = &list.source {
-                    if !module.source_equals(&source) {
+                    if !module.source_equals(source) {
                         continue;
                     }
                 }
@@ -148,7 +143,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
                 }
 
                 if let Some(stealth) = &list.stealth {
-                    if !module.stealth().equal_or_better(&stealth) {
+                    if !module.stealth().equal_or_better(stealth) {
                         continue;
                     }
                 }
@@ -163,17 +158,17 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
             Ok(ModuleReload::No)
         },
         SubCommand::Install(install) => {
-            registry::run_install(install, &config)?;
+            registry::run_install(install, config)?;
             // trigger reload
             Ok(ModuleReload::Yes)
         },
         SubCommand::Search(search) => {
-            registry::run_search(library, &search, &config)?;
+            registry::run_search(library, &search, config)?;
             Ok(ModuleReload::No)
         },
         SubCommand::Update(_) => {
             let mut autoupdate = AutoUpdater::load()?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let updater = Arc::new(Updater::new(config)?);
 
             let modules = library.list()
                 .into_iter()
@@ -199,14 +194,14 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
             Ok(ModuleReload::Yes)
         },
         SubCommand::Uninstall(uninstall) => {
-            let updater = Updater::new(&config)?;
+            let updater = Updater::new(config)?;
             updater.uninstall(&uninstall.module)?;
             // trigger reload
             Ok(ModuleReload::Yes)
         },
         SubCommand::Quickstart => {
-            let client = Client::new(&config)?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let client = Client::new(config)?;
+            let updater = Arc::new(Updater::new(config)?);
             let mut autoupdate = AutoUpdater::load()?;
 
             let installed = library.list()
@@ -250,7 +245,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
 
 impl LiteCmd for Args {
     fn run(self, config: &Config) -> Result<()> {
-        let library = Library::new(false, &config)?;
+        let library = Library::new(false, config)?;
         run_subcommand(self.subcommand, &library, config)?;
         Ok(())
     }

src/cmd/quickstart_cmd.rs

@@ -1,20 +1,16 @@
 use crate::errors::*;
-
 use crate::cmd::Cmd;
 use crate::cmd::pkg_cmd::{ArgsInteractive as PkgArgs, SubCommand, SubCommandInteractive};
 use crate::shell::Shell;
 use crate::term;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use clap::Parser;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
 }
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let _args = Args::from_iter_safe(args)?;
+    let _args = Args::try_parse_from(args)?;
 
     term::warn("The \x1b[1mquickstart\x1b[0m command is deprecated, use \x1b[1mpkg quickstart\x1b[0m");
 

src/cmd/rescope_cmd.rs

@@ -0,0 +1,266 @@
+use crate::errors::*;
+
+use crate::autonoscope::{AutoRule, ToRule};
+use crate::cmd::Cmd;
+use crate::db::Database;
+use crate::filters::{Filter, Target};
+use crate::shell::Shell;
+use std::collections::HashSet;
+use std::fmt;
+use clap::Parser;
+use crate::models::*;
+use crate::utils;
+use crate::term;
+
+#[derive(Debug, Parser)]
+pub struct Args {
+    /// Run rules interactively
+    #[arg(short, long)]
+    interactive: bool,
+    /// Automatically apply changes to database
+    #[arg(short='y', long)]
+    auto_confirm: bool,
+    /// Only show changes, do not apply them to the database
+    #[arg(short='n', long)]
+    dry_run: bool,
+    /// Only rescope entities matching specific filter
+    #[command(subcommand)]
+    target: Option<Target>,
+}
+
+enum Entity {
+    Domain(Domain),
+    Subdomain(Subdomain),
+    IpAddr(IpAddr),
+    Url(Url),
+    Port(Port),
+    Netblock(Netblock),
+}
+
+impl Entity {
+    fn set_scoped(&self, db: &Database, value: bool) -> Result<()> {
+        match self {
+            Entity::Domain(entity) => entity.set_scoped(db, value),
+            Entity::Subdomain(entity) => entity.set_scoped(db, value),
+            Entity::IpAddr(entity) => entity.set_scoped(db, value),
+            Entity::Url(entity) => entity.set_scoped(db, value),
+            Entity::Port(entity) => entity.set_scoped(db, value),
+            Entity::Netblock(entity) => entity.set_scoped(db, value),
+        }
+    }
+}
+
+enum Input {
+    Yes,
+    No,
+    Done,
+    Always,
+    Never,
+}
+
+fn get_input() -> Result<Input> {
+    loop {
+        let input = utils::question_opt("Update this entity? [Y/n/d/a/x/?]")?;
+        let input = input.map(|s| s.to_lowercase());
+        match input.as_deref() {
+            Some("y") | None => return Ok(Input::Yes),
+            Some("n") => return Ok(Input::No),
+            Some("d") => return Ok(Input::Done),
+            Some("a") => return Ok(Input::Always),
+            Some("x") => return Ok(Input::Never),
+            Some("?") => {
+                term::success("y -> yes, apply this change");
+                term::success("n -> no, skip this change");
+                term::success("d -> done, skip this and further changes");
+                term::success("a -> always, apply every change caused by this specific rule");
+                term::success("x -> never, skip every change caused by this specific rule");
+            },
+            Some(input) => term::error(&format!("Unrecognized input: {:?}", input)),
+        }
+    }
+}
+
+#[derive(Default)]
+struct Context {
+    update_queue: Vec<(Entity, bool)>,
+    always_rules: HashSet<(&'static str, String)>,
+    never_rules: HashSet<(&'static str, String)>,
+    done: bool,
+    target: Option<Target>,
+}
+
+fn rescope_to_queue<T, F1, F2, F3>(ctx: &mut Context, db: &Database, interactive: bool, get_filter: F1, matches_rule: F2, wrap: F3) -> Result<()>
+    where
+        T: Model + Scopable + fmt::Debug,
+        F1: Fn(&Target) -> Option<&Filter>,
+        F2: Fn(&T) -> Result<Option<((&'static str, String), bool)>>,
+        F3: Fn(T) -> Entity,
+{
+    // do nothing if we're already done
+    if ctx.done {
+        return Ok(());
+    }
+
+    // check if there are filters to be applied
+    let filter = if let Some(target) = &ctx.target {
+        if let Some(filter) = get_filter(target) {
+            // we've selected this specific entity type and there's a filter
+            filter.parse_optional()
+                .context("Filter is invalid")?
+        } else {
+            // we do not wish to process this entity type
+            return Ok(());
+        }
+    } else {
+        // any entity type is fine
+        Filter::any()
+    };
+
+    let entities = db.filter::<T>(&filter)?;
+
+    for entity in entities {
+        let currently_scoped = entity.scoped();
+        debug!("rescoping entity: {:?}", entity);
+
+        if let Some((rule, should_be)) = matches_rule(&entity)? {
+            // check if we're actively ignoring this rule
+            if ctx.never_rules.contains(&rule) {
+                continue;
+            }
+
+            if currently_scoped != should_be {
+                let prefix = if should_be {
+                    "\x1b[1m[\x1b[32m+\x1b[0;1m]\x1b[0m"
+                } else {
+                    "\x1b[1m[\x1b[31m-\x1b[0;1m]\x1b[0m"
+                };
+
+                println!("{} Setting entity {:?} => {:?}: {:?}", prefix, currently_scoped, should_be, entity);
+
+                // check if we're auto-accepting this rule
+                let input = if ctx.always_rules.contains(&rule) {
+                    Input::Yes
+                } else if interactive {
+                    get_input()?
+                } else {
+                    Input::Yes
+                };
+
+                // process user input
+                let input = match input {
+                    Input::Always => {
+                        ctx.always_rules.insert(rule);
+                        Input::Yes
+                    },
+                    Input::Never => {
+                        ctx.never_rules.insert(rule);
+                        Input::No
+                    },
+                    Input::Done => {
+                        ctx.done = true;
+                        break;
+                    },
+                    input => input,
+                };
+
+                if let Input::Yes = input {
+                    ctx.update_queue.push((wrap(entity), should_be));
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let rules = rl.db().autonoscope();
+        term::success(&format!("Loaded {} rules", rules.len()));
+
+        let mut ctx = Context {
+            target: self.target,
+            ..Default::default()
+        };
+
+        rescope_to_queue::<Domain, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.domains(), |entity| {
+            for rule in rules.domains() {
+                if rule.matches(entity.value.as_str())? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Domain)?;
+        rescope_to_queue::<Subdomain, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.subdomains(), |entity| {
+            for rule in rules.domains() {
+                if rule.matches(entity.value.as_str())? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Subdomain)?;
+
+        rescope_to_queue::<IpAddr, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.ipaddrs(), |entity| {
+            for rule in rules.ips() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::IpAddr)?;
+
+        rescope_to_queue::<Url, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.urls(), |entity| {
+            for rule in rules.domains() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            for rule in rules.urls() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Url)?;
+        rescope_to_queue::<Port, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.ports(), |entity| {
+            for rule in rules.ips() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Port)?;
+        rescope_to_queue::<Netblock, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.netblocks(), |entity| {
+            for rule in rules.ips() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Netblock)?;
+
+        if ctx.update_queue.is_empty() {
+            term::success("Nothing has changed, not updating database");
+        } else {
+            let confirm = if self.dry_run {
+                false
+            } else if self.auto_confirm {
+                true
+            } else {
+                utils::no_else_yes(&format!("Apply {} changes to scope now?", ctx.update_queue.len()))?
+            };
+
+            if confirm {
+                term::info(&format!("Applying {} changes to database", ctx.update_queue.len()));
+
+                for (update, value) in ctx.update_queue {
+                    update.set_scoped(rl.db(), value)?;
+                }
+            } else {
+                term::info("Database not updated");
+            }
+        }
+
+        Ok(())
+    }
+}

src/cmd/run_cmd.rs

@@ -1,15 +1,15 @@
-use crate::errors::*;
-
 use chrootable_https::dns::Resolver;
+use clap::{ArgAction, Parser};
 use crate::args;
 use crate::blobs::{Blob, BlobStorage};
 use crate::cmd::Cmd;
 use crate::db::{ttl, Filter};
 use crate::engine::Module;
+use crate::errors::*;
 use crate::ipc::common::StartCommand;
+use crate::keyring::KeyRing;
 use crate::models::*;
 use crate::shell::Shell;
-use crate::keyring::KeyRing;
 use crate::term;
 use crate::utils;
 use crate::worker;
@@ -17,47 +17,92 @@ use serde::Serialize;
 use sn0int_common::metadata::Source;
 use sn0int_std::ratelimits::Ratelimiter;
 use std::collections::HashMap;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use std::net::SocketAddr;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
-    #[structopt(short="j", long="threads", default_value="1")]
-    threads: usize,
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    verbose: u64,
+    /// Execute a module that has been installed
+    pub module: Option<String>,
+    /// Run investigations concurrently
+    #[arg(short = 'j', default_value="1")]
+    pub threads: usize,
+    /// Verbose logging, once to print inserts even if they don't add new
+    /// data, twice to activate the debug() function
+    #[arg(short = 'v', long, action(ArgAction::Count))]
+    pub verbose: u8,
+    /// Set a specific socks5 proxy to use
+    #[arg(short = 'X', long)]
+    pub proxy: Option<SocketAddr>,
+    /// Set a different default user agent
+    #[arg(long)]
+    pub user_agent: Option<String>,
 }
 
 #[derive(Debug, Clone)]
 pub struct Params<'a> {
+    pub module: Option<&'a String>,
     pub threads: usize,
-    pub verbose: u64,
+    pub verbose: u8,
     pub stdin: bool,
     pub grants: &'a [String],
     pub grant_full_keyring: bool,
     pub deny_keyring: bool,
     pub exit_on_error: bool,
+    pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<&'a String>,
+}
+
+impl<'a> Params<'a> {
+    pub fn get_module(&self, rl: &Shell) -> Result<Module> {
+        let module = if let Some(module) = self.module {
+            rl.library().get(module)?
+                .clone()
+        } else {
+            rl.module()
+                .map(|m| m.to_owned())
+                .ok_or_else(|| format_err!("No module selected"))?
+        };
+        Ok(module)
+    }
+
+    pub fn get_proxy(&self, rl: &Shell) -> Option<SocketAddr> {
+        if self.proxy.is_some() {
+            self.proxy
+        } else {
+            rl.config().network.proxy
+        }
+    }
+
+    pub fn get_user_agent(&self, rl: &Shell) -> Option<String> {
+        if let Some(user_agent) = self.user_agent {
+            Some(user_agent.to_string())
+        } else {
+            rl.config().network.user_agent.clone()
+        }
+    }
 }
 
 impl<'a> From<&'a args::Run> for Params<'a> {
     fn from(args: &args::Run) -> Params {
         Params {
-            threads: args.threads,
-            verbose: args.verbose,
+            module: args.run.module.as_ref(),
+            threads: args.run.threads,
+            verbose: args.run.verbose,
             stdin: args.stdin,
             grants: &args.grants,
             grant_full_keyring: args.grant_full_keyring,
             deny_keyring: args.deny_keyring,
             exit_on_error: args.exit_on_error,
+            proxy: args.run.proxy,
+            user_agent: args.run.user_agent.as_ref(),
         }
     }
 }
 
-impl From<Args> for Params<'static> {
-    fn from(args: Args) -> Params<'static> {
+impl<'a> From<&'a Args> for Params<'a> {
+    fn from(args: &Args) -> Params {
         Params {
+            module: args.module.as_ref(),
             threads: args.threads,
             verbose: args.verbose,
             stdin: false,
@@ -65,6 +110,8 @@ impl From<Args> for Params<'static> {
             grant_full_keyring: false,
             deny_keyring: false,
             exit_on_error: false,
+            proxy: args.proxy,
+            user_agent: args.user_agent.as_ref(),
         }
     }
 }
@@ -93,7 +140,7 @@ fn prepare_args<T: Scopable + Serialize + Model>(rl: &Shell, filter: &Filter, pa
 }
 
 pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
-    for namespace in keyring.unauthorized_namespaces(&module) {
+    for namespace in keyring.unauthorized_namespaces(module) {
         let grant_access = if params.deny_keyring {
             false
         } else if params.grant_full_keyring || params.grants.contains(namespace) {
@@ -104,7 +151,7 @@ pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params)
         };
 
         if grant_access {
-            keyring.grant_access(&module, namespace.to_string());
+            keyring.grant_access(module, namespace.to_string());
             term::info(&format!("Granted access to {:?}", namespace));
         }
     }
@@ -136,8 +183,8 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
         Some(Source::Notifications) => bail!("Notification modules can't be executed like this"),
         Some(Source::KeyRing(namespace)) => {
             let keyring = rl.keyring();
-            if keyring.is_access_granted(&module, &namespace) {
-                keyring.get_all_for(&namespace).into_iter()
+            if keyring.is_access_granted(module, namespace) {
+                keyring.get_all_for(namespace).into_iter()
                     .map(|key| {
                         let pretty = format!("{}:{}", key.namespace, key.access_key);
                         let arg = serde_json::to_value(key)?;
@@ -153,15 +200,14 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
 }
 
 pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);
 
     prepare_keyring(rl.keyring_mut(), &module, &params)?;
     let keyring = rl.keyring().request_keys(&module);
 
     let dns_config = Resolver::from_system_v4()?;
-    let proxy = rl.config().network.proxy;
 
     let args = get_args(rl, &module)?;
     for (arg, _pretty_arg, blobs) in args {
@@ -169,6 +215,7 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
                                           keyring.clone(),
                                           dns_config.clone(),
                                           proxy,
+                                          user_agent.clone(),
                                           options.clone(),
                                           module.clone(),
                                           arg,
@@ -181,15 +228,15 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
 }
 
 pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);
 
     prepare_keyring(rl.keyring_mut(), &module, &params)?;
     let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
-    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, rl.config().network.proxy, options);
+    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, proxy, user_agent, options);
     rl.signal_register().reset_ctrlc();
 
     if errors > 0 {
@@ -212,6 +259,6 @@ impl Cmd for Args {
             Some(options) => options.clone(),
             _ => HashMap::new(),
         };
-        execute(rl, self.into(), options)
+        execute(rl, Params::from(&self), options)
     }
 }

src/cmd/scope_cmd.rs

@@ -1,18 +1,14 @@
 use crate::errors::*;
-
 use crate::cmd::Cmd;
 use crate::filters::{Target, Filter};
 use crate::shell::Shell;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use clap::Parser;
 use crate::models::*;
 use crate::term;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
-    #[structopt(subcommand)]
+    #[command(subcommand)]
     subcommand: Target,
 }
 
@@ -40,7 +36,7 @@ impl Cmd for Args {
 }
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
+    let args = Args::try_parse_from(args)?;
     args.run(rl)
 }
 

src/cmd/select_cmd.rs

@@ -1,28 +1,27 @@
-use crate::errors::*;
-
+use clap::Parser;
 use crate::cmd::Cmd;
 use crate::db::ttl;
+use crate::errors::*;
 use crate::filters::{Target, Filter};
+use crate::models::*;
 use crate::shell::Shell;
 use serde::Serialize;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
-use crate::models::*;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
     #[structopt(subcommand)]
     subcommand: Target,
     /// Print json output
-    #[structopt(long="json", group="output")]
+    #[structopt(long, group="output")]
     json: bool,
+    /// Only print the value instead of the whole object
+    #[structopt(long, group="output")]
+    values: bool,
     /// Print paths to blobs
-    #[structopt(long="paths", group="output")]
+    #[structopt(long, group="output")]
     paths: bool,
     /// Count rows returned
-    #[structopt(short="c", long="count", group="output")]
+    #[structopt(short = 'c', group="output")]
     count: bool,
 }
 
@@ -30,6 +29,7 @@ pub struct Args {
 enum Output {
     Normal,
     Json,
+    Values,
     Paths,
     Count,
 }
@@ -43,6 +43,8 @@ impl<'a, 'b> Printer<'a, 'b> {
     pub fn new(rl: &'a mut Shell<'b>, args: &Args) -> Printer<'a, 'b> {
         let output = if args.json {
             Output::Json
+        } else if args.values {
+            Output::Values
         } else if args.paths {
             Output::Paths
         } else if args.count {
@@ -70,6 +72,9 @@ impl<'a, 'b> Printer<'a, 'b> {
                         let v = serde_json::to_string(&obj)?;
                         println!("{}", v);
                     },
+                    Output::Values => {
+                        println!("{}", obj.to_string());
+                    },
                     Output::Paths => {
                         let blob = obj.blob()
                             .ok_or_else(|| format_err!("This model isn't linked to blob storage"))?;
@@ -97,20 +102,20 @@ impl Cmd for Args {
         let printer = Printer::new(rl, &self);
 
         match &self.subcommand {
-            Target::Domains(filter) => printer.select::<Domain>(&filter),
-            Target::Subdomains(filter) => printer.select::<Subdomain>(&filter),
-            Target::IpAddrs(filter) => printer.select::<IpAddr>(&filter),
-            Target::Urls(filter) => printer.select::<Url>(&filter),
-            Target::Emails(filter) => printer.select::<Email>(&filter),
-            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(&filter),
-            Target::Devices(filter) => printer.select::<Device>(&filter),
-            Target::Networks(filter) => printer.select::<Network>(&filter),
-            Target::Accounts(filter) => printer.select::<Account>(&filter),
-            Target::Breaches(filter) => printer.select::<Breach>(&filter),
-            Target::Images(filter) => printer.select::<Image>(&filter),
-            Target::Ports(filter) => printer.select::<Port>(&filter),
-            Target::Netblocks(filter) => printer.select::<Netblock>(&filter),
-            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(&filter),
+            Target::Domains(filter) => printer.select::<Domain>(filter),
+            Target::Subdomains(filter) => printer.select::<Subdomain>(filter),
+            Target::IpAddrs(filter) => printer.select::<IpAddr>(filter),
+            Target::Urls(filter) => printer.select::<Url>(filter),
+            Target::Emails(filter) => printer.select::<Email>(filter),
+            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(filter),
+            Target::Devices(filter) => printer.select::<Device>(filter),
+            Target::Networks(filter) => printer.select::<Network>(filter),
+            Target::Accounts(filter) => printer.select::<Account>(filter),
+            Target::Breaches(filter) => printer.select::<Breach>(filter),
+            Target::Images(filter) => printer.select::<Image>(filter),
+            Target::Ports(filter) => printer.select::<Port>(filter),
+            Target::Netblocks(filter) => printer.select::<Netblock>(filter),
+            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(filter),
         }
     }
 }

src/cmd/set_cmd.rs

@@ -1,12 +1,8 @@
+use clap::Parser;
 use crate::errors::*;
-
 use crate::shell::Shell;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
     key: Option<String>,
     value: Option<String>,
@@ -15,7 +11,7 @@ pub struct Args {
 // TODO: maybe introduce global settings
 // TODO: maybe allow setting jobs here as well in addition to -j
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
+    let args = Args::try_parse_from(args)?;
 
     let options = rl.options_mut()
         .ok_or_else(|| format_err!("Module needs to be selected first"))?;

src/cmd/stats_cmd.rs

@@ -0,0 +1,195 @@
+use crate::args;
+use colored::Colorize;
+use crate::blobs::BlobStorage;
+use crate::cmd::Cmd;
+use crate::db::{ttl, Database};
+use crate::errors::*;
+use crate::models::*;
+use crate::shell::{self, Shell};
+use crate::workspaces;
+use separator::Separatable;
+use serde::{Serialize, Deserialize};
+use clap::Parser;
+
+#[derive(Debug, Clone, Parser)]
+pub struct Args {
+    /// Exclude blob storage
+    #[arg(short, long)]
+    short: bool,
+    /// Exclude categories that don't contain any structs
+    #[arg(short, long)]
+    quiet: bool,
+    /// Show workspace statistics in json
+    #[arg(short, long)]
+    json: bool,
+    /// Go through all workspaces
+    #[arg(short, long)]
+    all: bool,
+}
+
+impl Args {
+    fn show_amount(&self, label: &str, count: usize, amount: &str) {
+        if self.quiet && count == 0 {
+            return;
+        }
+
+        let label = format!("{:20}", label);
+        let amount = format!("{:>20}", amount);
+
+        if count > 0 {
+            println!("{} {}", label.green(), amount.yellow());
+        } else {
+            println!("{} {}", label, amount);
+        }
+    }
+
+    fn show_count(&self, label: &str, count: usize) {
+        self.show_amount(label, count, &count.separated_string());
+    }
+}
+
+fn count_models<T: Model>(db: &Database) -> Result<usize> {
+    let query = db.list::<T>()?;
+    let count = query.len();
+    Ok(count)
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Stats {
+    workspace: String,
+    domains: usize,
+    subdomains: usize,
+    ipaddrs: usize,
+    urls: usize,
+    emails: usize,
+    phonenumbers: usize,
+    devices: usize,
+    networks: usize,
+    accounts: usize,
+    breaches: usize,
+    images: usize,
+    ports: usize,
+    netblocks: usize,
+    cryptoaddrs: usize,
+    activity: usize,
+    blobs: Option<BlobStats>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct BlobStats {
+    count: usize,
+    total_size: u64,
+    total_human_size: String,
+}
+
+impl Stats {
+    fn count(workspace: String, db: &Database) -> Result<Stats> {
+        Ok(Stats {
+            workspace,
+            domains: count_models::<Domain>(db)?,
+            subdomains: count_models::<Subdomain>(db)?,
+            ipaddrs: count_models::<IpAddr>(db)?,
+            urls: count_models::<Url>(db)?,
+            emails: count_models::<Email>(db)?,
+            phonenumbers: count_models::<PhoneNumber>(db)?,
+            devices: count_models::<Device>(db)?,
+            networks: count_models::<Network>(db)?,
+            accounts: count_models::<Account>(db)?,
+            breaches: count_models::<Breach>(db)?,
+            images: count_models::<Image>(db)?,
+            ports: count_models::<Port>(db)?,
+            netblocks: count_models::<Netblock>(db)?,
+            cryptoaddrs: count_models::<CryptoAddr>(db)?,
+            activity: Activity::count(db)?,
+            blobs: None,
+        })
+    }
+
+    fn add_blob_usage(&mut self, storage: &BlobStorage) -> Result<()> {
+        let blobs = storage.list()?;
+
+        let mut total_size = 0;
+        for blob in &blobs {
+            total_size += storage.stat(blob)?;
+        }
+
+        let total_human_size = humansize::format_size(total_size, humansize::BINARY);
+
+        self.blobs = Some(BlobStats {
+            count: blobs.len(),
+            total_size,
+            total_human_size,
+        });
+
+        Ok(())
+    }
+}
+
+impl Cmd for Args {
+    #[inline]
+    fn run(mut self, rl: &mut Shell) -> Result<()> {
+        if self.all {
+            self.all = false;
+            let mut first = true;
+
+            let config = rl.config();
+            for ws in workspaces::list()? {
+                if first {
+                    first = false;
+                } else if !self.json {
+                    println!();
+                }
+
+                let mut rl = shell::init(&args::Args {
+                    workspace: Some(ws),
+                    subcommand: None,
+                }, config, false)?;
+                self.clone().run(&mut rl)?;
+            }
+        } else {
+            ttl::reap_expired(rl)?;
+
+            let db = rl.db();
+
+            let workspace = rl.workspace();
+
+            if !self.json {
+                // print this early in case counting takes a while
+                println!("{:>41}", workspace.bold());
+            }
+
+            let mut stats = Stats::count(workspace.into(), db)?;
+            if !self.short {
+                stats.add_blob_usage(rl.blobs())?;
+            }
+
+            if self.json {
+                let stats = serde_json::to_string(&stats)?;
+                println!("{}", stats);
+            } else {
+                self.show_count("domains", stats.domains);
+                self.show_count("subdomains", stats.subdomains);
+                self.show_count("ipaddrs", stats.ipaddrs);
+                self.show_count("urls", stats.urls);
+                self.show_count("emails", stats.emails);
+                self.show_count("phonenumbers", stats.phonenumbers);
+                self.show_count("devices", stats.devices);
+                self.show_count("networks", stats.networks);
+                self.show_count("accounts", stats.accounts);
+                self.show_count("breaches", stats.breaches);
+                self.show_count("images", stats.images);
+                self.show_count("ports", stats.ports);
+                self.show_count("netblocks", stats.netblocks);
+                self.show_count("cryptoaddrs", stats.cryptoaddrs);
+                self.show_count("activity", stats.activity);
+
+                if let Some(blobs) = stats.blobs {
+                    self.show_count("blobs", blobs.count);
+                    self.show_amount("blobs (size)", blobs.total_size as usize, &blobs.total_human_size);
+                }
+            }
+        }
+
+        Ok(())
+    }
+}

src/cmd/target_cmd.rs

@@ -3,14 +3,12 @@ use crate::errors::*;
 use crate::db;
 use crate::shell::Shell;
 use sn0int_common::metadata::Source;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
+use clap::Parser;
 use crate::term;
 use crate::models::*;
 
 
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
     // TODO: target -p # print current filter
     // TODO: target -c # clear filter
@@ -19,7 +17,7 @@ pub struct Args {
 }
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
+    let args = Args::try_parse_from(args)?;
 
     let source = rl.module()
         .ok_or_else(|| format_err!("No module selected"))
@@ -81,7 +79,7 @@ fn count_selected(rl: &mut Shell, source: &Source) -> Result<usize> {
         Source::Netblocks => db.filter::<Netblock>(&filter)?.len(),
         Source::CryptoAddrs(currency) => db.filter_with_param::<CryptoAddr>(&filter, currency.as_ref())?.len(),
         Source::Notifications => bail!("Notifications can't be set as target"),
-        Source::KeyRing(namespace) => rl.keyring().list_for(&namespace).len(),
+        Source::KeyRing(namespace) => rl.keyring().list_for(namespace).len(),
     };
     Ok(num)
 }

src/cmd/use_cmd.rs

@@ -1,18 +1,14 @@
+use clap::Parser;
 use crate::errors::*;
-
 use crate::shell::Shell;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
     module: String,
 }
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
+    let args = Args::try_parse_from(args)?;
 
     let module = rl.library().get(&args.module)?.clone();
     rl.set_module(module);

src/cmd/workspace_cmd.rs

@@ -1,30 +1,26 @@
-use crate::errors::*;
-
+use clap::Parser;
 use crate::blobs::BlobStorage;
 use crate::cmd::{Cmd, LiteCmd};
 use crate::config::Config;
 use crate::db::Database;
+use crate::errors::*;
 use crate::shell::Shell;
 use crate::term;
 use crate::utils;
-use structopt::StructOpt;
-use structopt::clap::AppSettings;
 use crate::workspaces::{self, Workspace};
 
-
-#[derive(Debug, StructOpt)]
-#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+#[derive(Debug, Parser)]
 pub struct Args {
-    /// Delete a workspaceb
-    #[structopt(long = "delete", group = "action")]
+    /// Delete a workspace
+    #[arg(long = "delete", group = "action")]
     delete: bool,
     /// Show disk usage of workspace
-    #[structopt(long = "usage", group = "action")]
+    #[arg(long = "usage", group = "action")]
     usage: bool,
     /// Skip confirmation
-    #[structopt(short = "f", long = "force")]
+    #[arg(short = 'f', long = "force")]
     force: bool,
-    workspace: Option<Workspace>,
+    workspaces: Vec<Workspace>,
 }
 
 fn delete(workspace: Workspace, force: bool) -> Result<()> {
@@ -38,13 +34,13 @@ fn delete(workspace: Workspace, force: bool) -> Result<()> {
     Ok(())
 }
 
-fn usage(workspace: Option<Workspace>) -> Result<()> {
-    if let Some(ws) = workspace {
-        println!("{}", ws.usage_human()?);
-    } else {
-        for ws in workspaces::list()? {
-            println!("{:50}: {}", ws.as_str(), ws.usage_human()?);
-        }
+fn usage(mut workspaces: Vec<Workspace>) -> Result<()> {
+    if workspaces.is_empty() {
+        workspaces = workspaces::list()?;
+    }
+
+    for ws in workspaces {
+        println!("{:50}: {}", ws.as_str(), ws.usage_human()?);
     }
 
     Ok(())
@@ -67,30 +63,34 @@ fn list() -> Result<()> {
     Ok(())
 }
 
-fn run(args: Args, rl: Option<&mut Shell>) -> Result<()> {
+fn run(mut args: Args, rl: Option<&mut Shell>) -> Result<()> {
     if args.delete {
-        if let Some(workspace) = args.workspace {
-            if let Some(rl) = rl {
+        if args.workspaces.is_empty() {
+            bail!("--delete requires workspace");
+        }
+
+        for workspace in args.workspaces {
+            if let Some(rl) = &rl {
                 if *rl.db().workspace() == workspace {
                     bail!("Can't delete current workspace")
                 }
             }
-
-            delete(workspace, args.force)
-        } else {
-            bail!("--delete requires workspace")
+            delete(workspace, args.force)?;
         }
     } else if args.usage {
-        usage(args.workspace)
-    } else if let Some(workspace) = args.workspace {
-        if let Some(rl) = rl {
-            change(rl, workspace)
-        } else {
-            Ok(())
-        }
+        usage(args.workspaces)?;
     } else {
-        list()
+        match args.workspaces.len() {
+            0 => list()?,
+            1 => if let Some(rl) = rl {
+                // we've already tested there's one item in the list
+                change(rl, args.workspaces.pop().unwrap())?;
+            },
+            _ => bail!("Only one argument allowed when switching workspaces"),
+        }
     }
+
+    Ok(())
 }
 
 impl Cmd for Args {

src/config.rs

@@ -21,7 +21,7 @@ pub struct Config {
 impl Config {
     pub fn load() -> Result<Config> {
         let path = Config::path()?;
-        Config::load_from(&path)
+        Config::load_from(path)
     }
 
     pub fn path() -> Result<PathBuf> {
@@ -41,10 +41,10 @@ impl Config {
     }
 
     pub fn load_from<P: AsRef<Path>>(path: P) -> Result<Config> {
-        let config = fs::read(&path)
+        let config = fs::read_to_string(&path)
             .context("Failed to read config file")?;
 
-        let config = toml::from_slice(&config)?;
+        let config = toml::from_str(&config)?;
 
         Ok(config)
     }
@@ -75,4 +75,5 @@ fn default_registry() -> String {
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct NetworkConfig {
     pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
 }

src/db/mod.rs

@@ -5,6 +5,7 @@ use diesel::expression::SqlLiteral;
 use diesel::expression::sql_literal::sql;
 use diesel::sql_types::Bool;
 use diesel::prelude::*;
+use std::fmt::Write;
 use strum_macros::{EnumString, IntoStaticStr};
 use crate::autonoscope::{RuleSet, RuleType};
 use crate::models::*;
@@ -164,6 +165,11 @@ impl Database {
         self.autonoscope.rules()
     }
 
+    #[inline(always)]
+    pub fn autonoscope(&self) -> &RuleSet {
+        &self.autonoscope
+    }
+
     /// Returns true if we didn't have this value yet
     pub fn insert_generic(&self, object: Insert) -> Result<Option<(DbChange, i32)>> {
         let scoped = self.autonoscope.matches(&object)?;
@@ -198,14 +204,14 @@ impl Database {
 
             let update = obj.upsert(&existing);
             if update.is_dirty() {
-                update.apply(&self)?;
+                update.apply(self)?;
                 Ok(Some((DbChange::Update(update.generic()), existing.id())))
             } else {
                 Ok(Some((DbChange::None, existing.id())))
             }
         } else {
             obj.set_scoped(scoped);
-            obj.insert(&self)?;
+            obj.insert(self)?;
             let id = M::get_id(self, obj.value())?;
             Ok(Some((DbChange::Insert, id)))
         }
@@ -218,7 +224,7 @@ impl Database {
                 return Ok(false);
             }
         }
-        obj.insert(&self)?;
+        obj.insert(self)?;
         Ok(true)
     }
 
@@ -254,7 +260,7 @@ impl Database {
 
             let update = obj.upsert(&existing);
             if update.is_dirty() {
-                update.apply(&self)?;
+                update.apply(self)?;
                 Ok(Some((DbChange::Update(update.generic()), id)))
             } else {
                 Ok(Some((DbChange::None, id)))
@@ -402,7 +408,7 @@ impl Database {
     }
 
     fn get_opt_typed<T: Model + Scopable>(&self, value: &T::ID) -> Result<Option<i32>> {
-        match T::get_opt(self, &value)? {
+        match T::get_opt(self, value)? {
             Some(ref obj) if obj.scoped() => Ok(Some(obj.id())),
             _ => Ok(None),
         }
@@ -410,23 +416,23 @@ impl Database {
 
     pub fn get_opt(&self, family: &Family, value: &str) -> Result<Option<i32>> {
         match family {
-            Family::Domain => self.get_opt_typed::<Domain>(&value),
-            Family::Subdomain => self.get_opt_typed::<Subdomain>(&value),
-            Family::Ipaddr => self.get_opt_typed::<IpAddr>(&value),
+            Family::Domain => self.get_opt_typed::<Domain>(value),
+            Family::Subdomain => self.get_opt_typed::<Subdomain>(value),
+            Family::Ipaddr => self.get_opt_typed::<IpAddr>(value),
             Family::SubdomainIpaddr => bail!("Unsupported operation"),
-            Family::Url => self.get_opt_typed::<Url>(&value),
-            Family::Email => self.get_opt_typed::<Email>(&value),
-            Family::Phonenumber => self.get_opt_typed::<PhoneNumber>(&value),
-            Family::Device => self.get_opt_typed::<Device>(&value),
-            Family::Network => self.get_opt_typed::<Network>(&value),
+            Family::Url => self.get_opt_typed::<Url>(value),
+            Family::Email => self.get_opt_typed::<Email>(value),
+            Family::Phonenumber => self.get_opt_typed::<PhoneNumber>(value),
+            Family::Device => self.get_opt_typed::<Device>(value),
+            Family::Network => self.get_opt_typed::<Network>(value),
             Family::NetworkDevice => bail!("Unsupported operation"),
-            Family::Account => self.get_opt_typed::<Account>(&value),
-            Family::Breach => self.get_opt_typed::<Breach>(&value),
+            Family::Account => self.get_opt_typed::<Account>(value),
+            Family::Breach => self.get_opt_typed::<Breach>(value),
             Family::BreachEmail => bail!("Unsupported operation"),
-            Family::Image => self.get_opt_typed::<Image>(&value),
-            Family::Port => self.get_opt_typed::<Port>(&value),
-            Family::Netblock => self.get_opt_typed::<Netblock>(&value),
-            Family::Cryptoaddr => self.get_opt_typed::<CryptoAddr>(&value),
+            Family::Image => self.get_opt_typed::<Image>(value),
+            Family::Port => self.get_opt_typed::<Port>(value),
+            Family::Netblock => self.get_opt_typed::<Netblock>(value),
+            Family::Cryptoaddr => self.get_opt_typed::<CryptoAddr>(value),
         }
     }
 
@@ -460,6 +466,11 @@ impl Filter {
         }
     }
 
+    #[inline]
+    pub fn any() -> Filter {
+        Filter::new("1")
+    }
+
     fn escape(value: &str) -> String {
         let mut out = String::from("'");
         for c in value.chars() {
@@ -492,14 +503,14 @@ impl Filter {
         for arg in args {
             if ["=", "!=", "<", ">", "<=", ">=", "like"].contains(&arg.to_lowercase().as_str()) {
                 expect_value = true;
-                query += &format!(" {}", arg);
+                write!(query, " {}", arg)?;
                 continue;
             }
 
             if let Some(idx) = arg.find('=') {
                 if idx != 0 {
                     let (key, value) = arg.split_at(idx);
-                    query += &format!(" {} = {}", key, Self::escape(&value[1..]));
+                    write!(query, " {} = {}", key, Self::escape(&value[1..]))?;
                     continue;
                 }
             }
@@ -509,7 +520,7 @@ impl Filter {
                 query.push_str(&Self::escape(arg));
                 expect_value = false;
             } else {
-                query += &format!(" {}", arg);
+                write!(query, " {}", arg)?;
             }
         }
         debug!("Parsed query: {:?}", query);
@@ -522,7 +533,7 @@ impl Filter {
 
         if args.is_empty() {
             debug!("Using filter with no condition");
-            return Ok(Filter::new("1"));
+            return Ok(Filter::any());
         }
 
         Self::parse(args)

src/db/ttl.rs

@@ -135,7 +135,7 @@ pub fn reap_expired(rl: &mut Shell) -> Result<()> {
 
         let subject = format!("Deleted {} {:?}", &expired.family, &expired.value);
         let topic = &format!("db:{}:{}:delete", &expired.family, &expired.value);
-        if let Err(err) = notify::trigger_notify_event(rl, &mut Term, &mut ratelimit, &topic, &Notification {
+        if let Err(err) = notify::trigger_notify_event(rl, &mut Term, &mut ratelimit, topic, &Notification {
             subject,
             body: None,
         }) {

src/engine/ctx.rs

@@ -38,7 +38,7 @@ pub trait State {
 
     fn recv(&self) -> Result<serde_json::Value>;
 
-    fn verbose(&self) -> u64;
+    fn verbose(&self) -> u8;
 
     #[inline]
     fn info(&self, msg: String) {
@@ -208,13 +208,14 @@ pub struct LuaState {
     http_sessions: Mutex<HashMap<String, HttpSession>>,
     http_clients: Mutex<HashMap<String, Arc<chrootable_https::Client<Resolver>>>>,
 
-    verbose: u64,
+    verbose: u8,
     keyring: Vec<KeyRingEntry>, // TODO: maybe hashmap
     dns_config: Resolver,
     psl: Mutex<Lazy<PslReader, Arc<Psl>>>,
     geoip: Option<Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>>,
     asn: Option<Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>>,
     proxy: Option<SocketAddr>,
+    user_agent: Option<String>,
     options: HashMap<String, String>,
 }
 
@@ -246,7 +247,7 @@ impl State for LuaState {
         tx.recv()
     }
 
-    fn verbose(&self) -> u64 {
+    fn verbose(&self) -> u8 {
         self.verbose
     }
 
@@ -299,7 +300,7 @@ impl State for LuaState {
         let id = self.random_id();
 
         let sock = if let Some(proxy) = self.resolve_proxy_options(&options.proxy)? {
-            Socket::connect_socks5(proxy, host, port, options)?
+            Socket::connect_socks5(*proxy, host, port, options)?
         } else {
             Socket::connect(&self.dns_config, host, port, options)?
         };
@@ -372,7 +373,15 @@ impl State for LuaState {
         let mtx = self.http_sessions.lock().unwrap();
         let session = mtx.get(session_id).expect("Invalid session reference"); // TODO
 
-        HttpRequest::new(&session, method, url, options, || format!("sn0int/{}", env!("CARGO_PKG_VERSION")))
+        let user_agent = if let Some(user_agent) = &options.user_agent {
+            user_agent.to_string()
+        } else if let Some(user_agent) = &self.user_agent {
+            user_agent.to_string()
+        } else {
+            format!("sn0int/{}", env!("CARGO_PKG_VERSION"))
+        };
+
+        HttpRequest::new(session, method, url, user_agent, options)
     }
 
     fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
@@ -472,6 +481,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
         geoip,
         asn,
         proxy: env.proxy,
+        user_agent: env.user_agent,
         options: env.options,
     });
 
@@ -517,6 +527,9 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
     runtime::http_fetch_json(&mut lua, state.clone());
     runtime::img_exif(&mut lua, state.clone());
     runtime::img_load(&mut lua, state.clone());
+    runtime::img_ahash(&mut lua, state.clone());
+    runtime::img_dhash(&mut lua, state.clone());
+    runtime::img_phash(&mut lua, state.clone());
     runtime::img_nudity(&mut lua, state.clone());
     runtime::info(&mut lua, state.clone());
     runtime::intval(&mut lua, state.clone());
@@ -662,6 +675,7 @@ impl Script {
         let keyring = Vec::new();
         let dns_config = Resolver::from_system_v4()?;
         let proxy = None;
+        let user_agent = None;
         let psl = PslReader::String(r#"
 // ===BEGIN ICANN DOMAINS===
 com
@@ -679,6 +693,7 @@ a.prod.fastly.net
             keyring,
             dns_config,
             proxy,
+            user_agent,
             options: HashMap::new(),
             blobs: Vec::new(),
             psl,

src/engine/mod.rs

@@ -30,10 +30,11 @@ pub use sn0int_std::engine::structs;
 /// Data that is passed to every script
 #[derive(Debug)]
 pub struct Environment {
-    pub verbose: u64,
+    pub verbose: u8,
     pub keyring: Vec<KeyRingEntry>,
     pub dns_config: Resolver,
     pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
     pub options: HashMap<String, String>,
     pub blobs: Vec<Blob>,
     pub psl: PslReader,
@@ -76,7 +77,7 @@ impl<'a> Library<'a> {
     }
 
     pub fn private_modules(path: &Path) -> Result<bool> {
-        let metadata = fs::symlink_metadata(&path)?.file_type();
+        let metadata = fs::symlink_metadata(path)?.file_type();
         if metadata.is_symlink() {
             debug!("Folder is a symlink, flagging modules as private");
             return Ok(true);
@@ -120,14 +121,14 @@ impl<'a> Library<'a> {
                 folder.to_owned()
             } else {
                 let folder = folder.strip_prefix("~/")
-                    .unwrap_or(&folder);
+                    .unwrap_or(folder);
 
                 dirs_next::home_dir()
                     .ok_or_else(|| format_err!("Failed to find home folder"))?
                     .join(folder)
             };
 
-            self.load_module_folder(&folder, &author, true)?;
+            self.load_module_folder(&folder, author, true)?;
         }
 
         Ok(())
@@ -156,7 +157,7 @@ impl<'a> Library<'a> {
                 continue;
             }
 
-            if let Err(err) = self.load_single_module(&module.path(), &author_name, &module_name, private_modules) {
+            if let Err(err) = self.load_single_module(&module.path(), author_name, module_name, private_modules) {
                 let root = err.find_root_cause();
                 term::warn(&format!("Failed to load {}/{}: {}", author_name, module_name, root));
             }
@@ -167,7 +168,7 @@ impl<'a> Library<'a> {
 
     pub fn load_single_module(&mut self, path: &Path, author_name: &str, module_name: &str, private_module: bool) -> Result<()> {
         let module_name = module_name.to_string();
-        let module = Module::load(path, &author_name, &module_name, private_module)
+        let module = Module::load(path, author_name, &module_name, private_module)
             .context(format!("Failed to parse {}/{}", author_name, module_name))?;
 
         for key in &[&module_name, &format!("{}/{}", author_name, module_name)] {
@@ -332,7 +333,7 @@ impl Module {
     pub fn source_equals(&self, other: &str) -> bool {
         match self.source() {
             Some(source) => source.group_as_str() == other,
-            None => other == "",
+            None => other.is_empty(),
         }
     }
 }

src/filters.rs

@@ -1,61 +1,114 @@
 use crate::errors::*;
-
 use crate::db;
-use structopt::StructOpt;
+use clap::Parser;
 
-
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Parser)]
 pub enum Target {
     /// On domains
-    #[structopt(name="domains")]
+    #[command(name="domains")]
     Domains(Filter),
     /// On subdomains
-    #[structopt(name="subdomains")]
+    #[command(name="subdomains")]
     Subdomains(Filter),
     /// On ipaddrs
-    #[structopt(name="ipaddrs")]
+    #[command(name="ipaddrs")]
     IpAddrs(Filter),
     /// On urls
-    #[structopt(name="urls")]
+    #[command(name="urls")]
     Urls(Filter),
     /// On emails
-    #[structopt(name="emails")]
+    #[command(name="emails")]
     Emails(Filter),
     /// On phone numbers
-    #[structopt(name="phonenumbers")]
+    #[command(name="phonenumbers")]
     PhoneNumbers(Filter),
     /// On devices
-    #[structopt(name="devices")]
+    #[command(name="devices")]
     Devices(Filter),
     /// On networks
-    #[structopt(name="networks")]
+    #[command(name="networks")]
     Networks(Filter),
     /// On accounts
-    #[structopt(name="accounts")]
+    #[command(name="accounts")]
     Accounts(Filter),
     /// On breaches
-    #[structopt(name="breaches")]
+    #[command(name="breaches")]
     Breaches(Filter),
     /// On images
-    #[structopt(name="images")]
+    #[command(name="images")]
     Images(Filter),
     /// On ports
-    #[structopt(name="ports")]
+    #[command(name="ports")]
     Ports(Filter),
     /// On ipnets
-    #[structopt(name="netblocks")]
+    #[command(name="netblocks")]
     Netblocks(Filter),
     /// On crypto currency addresses
-    #[structopt(name="cryptoaddrs")]
+    #[command(name="cryptoaddrs")]
     CryptoAddrs(Filter),
 }
 
-#[derive(Debug, StructOpt)]
+impl Target {
+    pub fn domains(&self) -> Option<&Filter> {
+        if let Target::Domains(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn subdomains(&self) -> Option<&Filter> {
+        if let Target::Subdomains(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn ipaddrs(&self) -> Option<&Filter> {
+        if let Target::IpAddrs(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn urls(&self) -> Option<&Filter> {
+        if let Target::Urls(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn ports(&self) -> Option<&Filter> {
+        if let Target::Ports(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+
+    pub fn netblocks(&self) -> Option<&Filter> {
+        if let Target::Netblocks(f) = self {
+            Some(f)
+        } else {
+            None
+        }
+    }
+}
+
+#[derive(Debug, Parser)]
 pub struct Filter {
     args: Vec<String>,
 }
 
 impl Filter {
+    #[inline]
+    pub fn any() -> db::Filter {
+        db::Filter::any()
+    }
+
     pub fn parse_optional(&self) -> Result<db::Filter> {
         db::Filter::parse_optional(&self.args)
     }

src/ipc/child.rs

@@ -86,6 +86,7 @@ pub fn run(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslRea
         keyring: start.keyring,
         dns_config: start.dns_config,
         proxy: start.proxy,
+        user_agent: start.user_agent,
         options: start.options,
         blobs: start.blobs,
         psl,

src/ipc/common.rs

@@ -8,10 +8,11 @@ use std::net::SocketAddr;
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct StartCommand {
-    pub verbose: u64,
+    pub verbose: u8,
     pub keyring: Vec<KeyRingEntry>,
     pub dns_config: Resolver,
     pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<String>,
     pub options: HashMap<String, String>,
     pub module: Module,
     pub arg: serde_json::Value,
@@ -19,10 +20,11 @@ pub struct StartCommand {
 }
 
 impl StartCommand {
-    pub fn new(verbose: u64,
+    pub fn new(verbose: u8,
                keyring: Vec<KeyRingEntry>,
                dns_config: Resolver,
                proxy: Option<SocketAddr>,
+               user_agent: Option<String>,
                options: HashMap<String, String>,
                module: Module,
                arg: serde_json::Value,
@@ -33,6 +35,7 @@ impl StartCommand {
             keyring,
             dns_config,
             proxy,
+            user_agent,
             options,
             module,
             arg,

src/ipc/parent.rs

@@ -47,7 +47,7 @@ impl IpcParent {
     }
 
     pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
-        let start = serde_json::to_value(&start)?;
+        let start = serde_json::to_value(start)?;
         self.send(&start)?;
         Ok(())
     }
@@ -71,6 +71,10 @@ impl IpcParent {
         let mut line = String::new();
         let len = self.stdout.read_line(&mut line)?;
 
+        if len == 0 {
+            bail!("Sandbox child has crashed");
+        }
+
         let event = serde_json::from_str(&line[..len])?;
         debug!("IpcParent received: {:?}", event);
         Ok(event)
@@ -102,9 +106,10 @@ pub fn run(module: Module,
            tx: &EventSender,
            arg: serde_json::Value,
            keyring: Vec<KeyRingEntry>,
-           verbose: u64,
+           verbose: u8,
            has_stdin: bool,
            proxy: Option<SocketAddr>,
+           user_agent: Option<String>,
            options: HashMap<String, String>,
            blobs: Vec<Blob>,
 ) -> Result<ExitEvent> {
@@ -117,15 +122,15 @@ pub fn run(module: Module,
     };
 
     let mut ipc_parent = IpcParent::setup(&module)?;
-    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
+    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, user_agent, options, module, arg, blobs))?;
 
     let exit = loop {
         match ipc_parent.recv()? {
             Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => ipc_parent.send_event_callback(*object, &tx),
+            Event::Database(object) => ipc_parent.send_event_callback(*object, tx),
             Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
-            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
-            Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),
+            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, tx),
+            Event::Blob(blob) => ipc_parent.send_event_callback(blob, tx),
             Event::Exit(event) => {
                 if let ExitEvent::Err(err) = &event {
                     tx.send(Event2::Log(LogEvent::Error(err.clone())));

src/keyring.rs

@@ -13,7 +13,7 @@ use std::path::{Path, PathBuf};
 use sn0int_common::ModuleID;
 
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct KeyName {
     pub namespace: String,
     pub name: String,
@@ -94,7 +94,7 @@ impl KeyRing {
     }
 
     pub fn load(path: &Path) -> Result<KeyRing> {
-        let buf = fs::read(&path)
+        let buf = fs::read(path)
             .context("Failed to read keyring file")?;
         serde_json::from_slice(&buf)
             .map_err(Error::from)
@@ -103,7 +103,7 @@ impl KeyRing {
     pub fn save(&self) -> Result<()> {
         let path = Self::path()?;
         let buf = serde_json::to_string(&self)?;
-        fs::write(&path, buf)
+        fs::write(path, buf)
             .context("Failed to save keyring")?;
         Ok(())
     }
@@ -111,7 +111,7 @@ impl KeyRing {
     pub fn insert(&mut self, key: KeyName, secret: Option<String>) -> Result<()> {
         // get the namespace or create a new one
         let mut x = self.keys.remove(&key.namespace)
-            .unwrap_or_else(HashMap::new);
+            .unwrap_or_default();
         // insert key into namespace
         x.insert(key.name, secret);
         // add namespace backinto keyring
@@ -170,13 +170,13 @@ impl KeyRing {
 
     pub fn unauthorized_namespaces<'a>(&self, module: &'a Module) -> Vec<&'a String> {
         module.keyring_access().iter()
-            .filter(|namespace| !self.is_access_granted(&module, &namespace))
+            .filter(|namespace| !self.is_access_granted(module, namespace))
             .collect()
     }
 
     pub fn grant_access(&mut self, module: &Module, namespace: String) {
         let mut grants = self.grants.remove(&namespace)
-            .unwrap_or_else(HashSet::new);
+            .unwrap_or_default();
         grants.insert(module.id());
         self.grants.insert(namespace, grants);
     }
@@ -192,7 +192,7 @@ impl KeyRing {
     pub fn request_keys(&self, module: &Module) -> Vec<KeyRingEntry> {
         // TODO: we probably want to randomize the order
         module.keyring_access().iter()
-            .filter(|namespace| self.is_access_granted(&module, &namespace))
+            .filter(|namespace| self.is_access_granted(module, namespace))
             .flat_map(|namespace| self.list_for(namespace))
             .flat_map(|x| self.get(&x))
             .collect()
@@ -208,7 +208,7 @@ pub struct KeyRingEntry {
 
 impl KeyRingEntry {
     pub fn to_lua(&self) -> Result<AnyLuaValue> {
-        let v = serde_json::to_value(&self)?;
+        let v = serde_json::to_value(self)?;
         let v = LuaJsonValue::from(v).into();
         Ok(v)
     }

src/lib.rs

@@ -1,6 +1,8 @@
 #![allow(clippy::too_many_arguments)]
 #![allow(clippy::type_complexity)]
 #![allow(clippy::large_enum_variant)]
+// because of diesel
+#![allow(clippy::extra_unused_lifetimes)]
 
 #![warn(unused_extern_crates)]
 use hlua_badtouch as hlua;

src/main.rs

@@ -1,7 +1,9 @@
-use env_logger::{self, Env};
+use env_logger::Env;
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
+use clap::Parser;
 use sn0int::cmd::{self, LiteCmd};
+use sn0int::cmd::run_cmd::Params;
 use sn0int::config::Config;
 use sn0int::db;
 use sn0int::errors::*;
@@ -14,8 +16,7 @@ use sn0int::psl::PslReader;
 use sn0int::registry;
 use sn0int::repl;
 use sn0int::sandbox;
-use sn0int::shell::{self, complete};
-use structopt::StructOpt;
+use sn0int::shell;
 use std::fs::OpenOptions;
 use std::io::Write;
 use std::path::Path;
@@ -23,34 +24,41 @@ use std::path::Path;
 fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
     let mut rl = shell::init(gargs, config, false)?;
 
+    let module = args.run.module.as_ref()
+        .ok_or_else(|| format_err!("Module is required"))?;
+
     let module = if args.file {
-        let path = Path::new(&args.module);
+        let path = Path::new(&module);
 
         let filename = path.file_stem()
             .ok_or_else(|| format_err!("Failed to decode filename"))?
             .to_str()
             .ok_or_else(|| format_err!("Failed to decode filename"))?;
 
-        Module::load(&path.to_path_buf(), "anonymous", &filename, true)
+        Module::load(path, "anonymous", filename, true)
             .context(format!("Failed to parse {:?}", path))?
     } else {
-        rl.library().get(&args.module)?
+        rl.library().get(module)?
             .clone()
     };
 
     rl.set_module(module);
 
     if let Some(target) = &args.target {
-        let target = shellwords::split(&target)
+        let target = shellwords::split(target)
             .map_err(|_| format_err!("Failed to parse target quotes"))?;
         let target = db::Filter::parse(&target)?;
         rl.set_target(Some(target));
     }
 
+    let mut params = Params::from(args);
+    // The module was already set and loaded
+    params.module = None;
+
     if args.dump_sandbox_init_msg {
-        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, args.into(), Opt::collect(&args.options))
+        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, params, Opt::collect(&args.options))
     } else {
-        cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+        cmd::run_cmd::execute(&mut rl, params, Opt::collect(&args.options))
     }
 }
 
@@ -91,7 +99,7 @@ end
 }
 
 fn run() -> Result<()> {
-    let mut args = Args::from_args();
+    let mut args = Args::parse();
 
     if !args.is_sandbox() {
         sandbox::fasten_seatbelt()?;
@@ -123,14 +131,18 @@ fn run() -> Result<()> {
         Some(SubCommand::Activity(activity)) => run_cmd(&args, activity, &config),
         Some(SubCommand::Scope(scope)) => run_cmd(&args, scope, &config),
         Some(SubCommand::Noscope(noscope)) => run_cmd(&args, noscope, &config),
+        Some(SubCommand::Autoscope(autoscope)) => run_cmd(&args, autoscope, &config),
+        Some(SubCommand::Autonoscope(autonoscope)) => run_cmd(&args, autonoscope, &config),
+        Some(SubCommand::Rescope(rescope)) => run_cmd(&args, rescope, &config),
         Some(SubCommand::Workspace(workspace)) => workspace.run(&config),
         Some(SubCommand::Fsck(fsck)) => run_cmd(&args, fsck, &config),
         Some(SubCommand::Export(export)) => run_cmd(&args, export, &config),
         Some(SubCommand::Cal(cal)) => run_cmd(&args, cal, &config),
         Some(SubCommand::Notify(notify)) => run_cmd(&args, notify, &config),
+        Some(SubCommand::Stats(stats)) => run_cmd(&args, stats, &config),
         Some(SubCommand::Repl) => repl::run(&config),
         Some(SubCommand::Paths) => paths::run(&config),
-        Some(SubCommand::Completions(completions)) => complete::run_generate(&completions),
+        Some(SubCommand::Completions(completions)) => completions.generate(),
         None => shell::run(&args, &config),
     }
 }

src/models/account.rs

@@ -115,6 +115,14 @@ impl Scopable for Account {
         !self.unscoped
     }
 
+    fn set_scoped(&self, db: &Database, my_value: bool) -> Result<()> {
+        use crate::schema::accounts::dsl::*;
+        diesel::update(accounts.filter(id.eq(self.id)))
+            .set(unscoped.eq(!my_value))
+            .execute(db.db())?;
+        Ok(())
+    }
+
     fn scope(db: &Database, filter: &Filter) -> Result<usize> {
         use crate::schema::accounts::dsl::*;
 
@@ -342,7 +350,7 @@ impl Upsert for AccountUpdate {
     }
 
     fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_account(&self)
+        db.update_account(self)
     }
 }