Bump libsqlite3-sys from 0.22.2 to 0.25.1

Bumps [libsqlite3-sys](https://github.com/rusqlite/rusqlite) from 0.22.2 to 0.25.1.
- [Release notes](https://github.com/rusqlite/rusqlite/releases)
- [Changelog](https://github.com/rusqlite/rusqlite/blob/master/Changelog.md)
- [Commits](https://github.com/rusqlite/rusqlite/commits/v0.25.1)

---
updated-dependencies:
- dependency-name: libsqlite3-sys
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/FUNDING.yml

@@ -1 +1,2 @@
 github: [kpcyrd]
+patreon: kpcyrd

.github/workflows/docker-release.yml

@@ -0,0 +1,58 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to GitHub Registry
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/kpcyrd/sn0int
+          tags: |
+            type=semver,pattern={{raw}}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      -
+        name: Login to Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push Docker images
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          file: Dockerfile
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+      -
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/docker.yml

@@ -0,0 +1,30 @@
+name: Docker
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+        - name: sn0int
+          file: Dockerfile
+        - name: registry
+          file: sn0int-registry/Dockerfile
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build the Docker image
+      run: DOCKER_BUILDKIT=1 docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
+    - name: Test the Docker image
+      run: docker run --rm ${{ matrix.build.name }} --help
+
+    - name: Show Docker images
+      run: docker images

.github/workflows/rust.yml

@@ -0,0 +1,69 @@
+name: Rust
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 9 * * 1'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install dependencies (apt)
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install libsqlite3-dev libseccomp-dev libsodium-dev
+    - name: Install dependencies (brew)
+      if: matrix.os == 'macos-latest'
+      run: brew install pkg-config libsodium
+
+    - name: Build (sn0int)
+      run: cargo build --verbose
+    - name: Build (common)
+      run: cd sn0int-common && cargo build --verbose
+    - name: Build (std)
+      run: cd sn0int-std && cargo build --verbose
+    - name: Build (examples)
+      run: cargo build --verbose --examples
+
+    - name: Run tests (sn0int)
+      run: cargo test --verbose
+    - name: Run tests (sn0int, --ignored)
+      run: cargo test --verbose -- --ignored
+    - name: Run tests (common)
+      run: cd sn0int-common && cargo test --verbose
+    - name: Run tests (common, --ignored)
+      run: cd sn0int-common && cargo test --verbose -- --ignored
+    - name: Run tests (std)
+      run: cd sn0int-std && cargo test --verbose
+    - name: Run tests (std, --ignored)
+      run: cd sn0int-std && cargo test --verbose -- --ignored
+
+  notify:
+    # forks shouldn't notify
+    if: github.repository == 'kpcyrd/sn0int'
+    runs-on: ubuntu-latest
+    needs:
+    - build
+
+    steps:
+    - name: irc notify
+      uses: rectalogic/notify-irc@v1
+      with:
+        server: irc.hackint.org
+        channel: "#sn0int"
+        nickname: github-ci
+        message: '${{ github.repository }}#${{ github.run_id }}(${{ github.event_name }}): ${{ github.ref }}: tests completed: ${{ needs.build.result }} (https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})'

.travis.yml

@@ -1,66 +0,0 @@
-language: rust
-
-# upload takes too long on windows and gets killed due to inactivity
-#cache: cargo
-
-matrix:
-  include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker-registry
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=std
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=std
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows test"
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows common"
-
-before_install:
-- ci/setup.sh "$TRAVIS_OS_NAME"
-script:
-- df -h
-- ci/run.sh $BUILD_MODE
-- df -h
-
-notifications:
-  irc:
-    channels:
-    - "ircs://irc.hackint.org:6697/#sn0int"
-    #on_success: change # default: always
-    #on_failure: always # default: always
-    use_notice: true

Cargo.toml

@@ -1,91 +1,104 @@
 [package]
 name = "sn0int"
-version = "0.18.0"
+version = "0.25.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
-edition = "2018"
-
-[badges]
-travis-ci = { repository = "kpcyrd/sn0int" }
+edition = "2021"
 
 [workspace]
 members = ["sn0int-common",
            "sn0int-registry",
            "sn0int-std"]
 
+[package.metadata.deb]
+extended-description = """\
+sn0int (pronounced /snoɪnt/) is a semi-automatic OSINT framework and package
+manager. It was built for IT security professionals and bug hunters to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations."""
+section = "utils"
+priority = "optional"
+depends = "$auto, publicsuffix"
+assets = [
+    ["target/release/sn0int", "usr/bin/", "755"],
+]
+
 [features]
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.10.0", path="sn0int-common" }
-sn0int-std = { version="=0.18.0", path="sn0int-std" }
-rustyline = "6.0"
+sn0int-common = { version="0.13.0", path="sn0int-common" }
+sn0int-std = { version="=0.25.0", path="sn0int-std" }
+rustyline = "10.0"
 log = "0.4"
-env_logger = "0.7"
+env_logger = "0.9"
 hlua-badtouch = "0.4"
 structopt = "0.3"
 failure = "0.1"
-rand = "0.7"
-colored = "1.6"
+rand = "0.8"
+colored = "2"
 lazy_static = "1.0"
 shellwords = "1.0"
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-libsqlite3-sys = "0.16.0"
+libsqlite3-sys = { version = "0.25.1", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
-dirs = "2.0"
+dirs-next = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.14"
-base64 = "0.11"
-data-encoding = "2.1.2"
-serde_urlencoded = "0.6"
-serde = "1.0"
-serde_derive = "1.0"
+chrootable-https = "0.16"
+data-encoding = "2.3.3"
+serde = { version = "1.0", features = ["derive"] }
+serde_urlencoded = "0.7"
 serde_json = "1.0"
-crossbeam-channel = "0.4"
+crossbeam-channel = "0.5"
 ctrlc = "3.1"
-opener = "0.4"
+opener = "0.5"
 separator = "0.4"
 maplit = "1.0.1"
-sloppy-rfc4880 = "0.1.5"
+sloppy-rfc4880 = "0.2"
 regex = "1.0"
 toml = "0.5"
 threadpool = "1.7"
 atty = "0.2"
-semver = "0.9"
+semver = "1"
 bytes = "0.4"
 bytesize = "1.0"
-ipnetwork = "0.16"
-strum = "0.17"
-strum_macros = "0.17"
+ipnetwork = "0.18"
+strum = "0.24"
+strum_macros = "0.24"
+embedded-triple = "0.1.0"
+humansize = "1.1.0"
 
-digest = "0.8.0"
-md-5 = "0.8.0"
-sha-1 = "0.8.1"
-sha2 = "0.8.0"
-sha3 = "0.8.0"
-hmac = "0.7"
+digest = "0.10"
+md-5 = "0.10"
+sha-1 = "0.10"
+sha2 = "0.10"
+sha3 = "0.10"
+hmac = "0.12"
 
 walkdir = "2.2"
 nude = "0.3"
+glob = "0.3.0"
+os-version = "0.2"
 
 [target.'cfg(target_os="linux")'.dependencies]
-caps = "0.3"
+caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.12"
-nix = "0.17"
+syscallz = "0.16"
+nix = "0.24"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.4"
-unveil = "0.2.0"
+unveil = "0.3"
 
 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.11"
+boxxy = "0.13"
 tempfile = "3.0"

Dockerfile

@@ -1,14 +1,18 @@
-FROM rust:alpine3.11
+FROM rust:alpine3.15
 ENV RUSTFLAGS="-C target-feature=-crt-static"
 RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo build --release --verbose
-RUN strip target/release/sn0int
+RUN --mount=type=cache,target=/var/cache/buildkit \
+    CARGO_HOME=/var/cache/buildkit/cargo \
+    CARGO_TARGET_DIR=/var/cache/buildkit/target \
+    cargo build --release --locked --verbose && \
+    cp -v /var/cache/buildkit/target/release/sn0int /
+RUN strip /sn0int
 
-FROM alpine:3.11
+FROM alpine:3.15
 RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
-COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+COPY --from=0 /sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \
     XDG_CACHE_HOME=/cache

README.md

@@ -1,7 +1,5 @@
-# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
+# sn0int [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
 
-[travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
-[travis]:       https://travis-ci.org/kpcyrd/sn0int
 [crates-img]:   https://img.shields.io/crates/v/sn0int.svg
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
@@ -16,33 +14,34 @@
 [registry]:     https://sn0int.com/
 
 sn0int (pronounced [`/snoɪnt/`][ipa]) is a semi-automatic OSINT framework and
-package manager. It was built for IT security professionals and bug hunters to
-gather intelligence about a given target or about yourself. sn0int is
-enumerating attack surface by semi-automatically processing public information
-and mapping the results in a unified format for followup investigations.
+package manager. It's used by IT security professionals, bug bounty hunters,
+law enforcement agencies and in security awareness trainings to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.
 
 [ipa]: http://ipa-reader.xyz/?text=sno%C9%AAnt
 
 Among other things, sn0int is currently able to:
 
 - Harvest subdomains from certificate transparency logs and passive dns
+- Mass resolve collected subdomains and scan for http or https services
 - Enrich ip addresses with asn and geoip info
 - Harvest emails from pgp keyservers and whois
 - Discover compromised logins in breaches
 - Find somebody's profiles across the internet
 - Enumerate local networks with unique techniques like passive arp
 - Gather information about phonenumbers
-- Attempt to bypass cloudflare with shodan
-- Harvest data and images from instagram profiles
-- Scan images for nudity
+- Harvest activity and images from social media profiles
+- Basic image processing
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource. None of the investigations listed above are hardcoded
-in the source, instead those are provided by modules that are executed in a
+in the source, instead they are provided by modules that are executed in a
 sandbox. You can easily extend sn0int by writing your own modules and share
 them with other users by publishing them to the sn0int registry. This allows
-you to ship updates for your modules on your own since you don't need to send a
-pull request.
+you to ship updates for your modules on your own instead of pull-requesting
+them into the sn0int codebase.
 
 For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
@@ -50,6 +49,8 @@ For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https:/
 
 ## Installation
 
+<a href="https://repology.org/project/sn0int/versions"><img align="right" src="https://repology.org/badge/vertical-allrepos/sn0int.svg" alt="Packaging status"></a>
+
 Archlinux
 
     pacman -S sn0int
@@ -58,6 +59,39 @@ Mac OSX
 
     brew install sn0int
 
+Debian/Ubuntu/Kali
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+    apt install debian-keyring
+    gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
+    apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
+    apt update
+    apt install sn0int
+
+Docker
+
+    docker run --rm --init -it -v "$PWD/.cache:/cache" -v "$PWD/.data:/data" kpcyrd/sn0int
+
+Alpine
+
+    apk add sn0int
+
+OpenBSD
+
+    pkg_add sn0int
+
+Gentoo
+
+    layman -a pentoo
+    emerge --ask net-analyzer/sn0int
+
+NixOS
+
+    nix-env -i sn0int
+
 For everything else please have a look at the [detailed list][1].
 
 [1]: https://sn0int.readthedocs.io/en/latest/install.html
@@ -67,11 +101,14 @@ For everything else please have a look at the [detailed list][1].
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
   - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
   - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
-  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Debian &gt;= bullseye, Ubuntu &gt;= 20.04, Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-bullseye-ubuntu-20-04-kali)
+  - [Debian &lt;= buster, Ubuntu &lt;= 19.10](https://sn0int.readthedocs.io/en/latest/install.html#debian-buster-ubuntu-19-10)
+  - [Fedora/CentOS/Redhat](https://sn0int.readthedocs.io/en/latest/install.html#fedora-centos-redhat)
   - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
   - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
   - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
   - [Gentoo](https://sn0int.readthedocs.io/en/latest/install.html#gentoo)
+  - [NixOS](https://sn0int.readthedocs.io/en/latest/install.html#nixos)
   - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
 - [Build from source](https://sn0int.readthedocs.io/en/latest/build.html)
   - [Install dependencies](https://sn0int.readthedocs.io/en/latest/build.html#install-dependencies)
@@ -129,6 +166,18 @@ For everything else please have a look at the [detailed list][1].
   - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
   - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
   - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
+- [Notifications](https://sn0int.readthedocs.io/en/latest/notifications.html)
+  - [Receiving notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#receiving-notifications)
+    - [Telegram](https://sn0int.readthedocs.io/en/latest/notifications.html#telegram)
+    - [Pushover](https://sn0int.readthedocs.io/en/latest/notifications.html#pushover)
+    - [Discord](https://sn0int.readthedocs.io/en/latest/notifications.html#discord)
+    - [Signal](https://sn0int.readthedocs.io/en/latest/notifications.html#signal)
+    - [Writing your own module](https://sn0int.readthedocs.io/en/latest/notifications.html#writing-your-own-module)
+  - [Setting up notification rules](https://sn0int.readthedocs.io/en/latest/notifications.html#setting-up-notification-rules)
+  - [Testing notifications](https://sn0int.readthedocs.io/en/latest/notifications.html#testing-notifications)
+  - [Running sn0int automatically](https://sn0int.readthedocs.io/en/latest/notifications.html#running-sn0int-automatically)
+    - [Monitors](https://sn0int.readthedocs.io/en/latest/notifications.html#monitors)
+    - [Timers](https://sn0int.readthedocs.io/en/latest/notifications.html#timers)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
@@ -178,6 +227,9 @@ For everything else please have a look at the [detailed list][1].
   - [http_fetch_json](https://sn0int.readthedocs.io/en/latest/reference.html#http-fetch-json)
   - [img_load](https://sn0int.readthedocs.io/en/latest/reference.html#img-load)
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
+  - [img_ahash](https://sn0int.readthedocs.io/en/latest/reference.html#img-ahash)
+  - [img_dhash](https://sn0int.readthedocs.io/en/latest/reference.html#img-dhash)
+  - [img_phash](https://sn0int.readthedocs.io/en/latest/reference.html#img-phash)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
   - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)

ci/integration.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import subprocess
 from subprocess import DEVNULL, PIPE
+import shutil
+from pathlib import Path
 import tempfile
 import json
 import sys
@@ -98,6 +100,11 @@ def main(tempdir, binary):
         ('https://www.example.com/', 200),
     }
 
+    cache = Path.home() / '.cache' / 'sn0int'
+    if cache.exists():
+        print('[*] copying geoip files')
+        shutil.copytree(cache, tempdir + '/.cache/sn0int', dirs_exist_ok=True)
+
     print('[*] running geoip')
     sn0int(tempdir, binary, [
         'use geoip',

ci/run.sh

@@ -1,41 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    build)
-        cargo build --verbose
-        cargo build --verbose --examples
-        ;;
-    test)
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    common)
-        cd sn0int-common
-        cargo test --verbose
-        ;;
-    std)
-        cd sn0int-std
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    windows)
-        cargo build --verbose --features=sqlite-bundled
-        cargo build --verbose --examples --features=sqlite-bundled
-        ;;
-    boxxy)
-        cargo build --verbose --examples
-        if cat ci/boxxy_stage1.txt | RUST_LOG=boxxy cargo run --example boxxy; then
-            echo "SANDOX ERROR: should've crashed"
-            exit 1
-        fi
-        ;;
-    docker)
-        docker build -t sn0int .
-        docker images
-        docker run --rm sn0int --help
-        ;;
-    docker-registry)
-        docker build -t sn0int-registry -f sn0int-registry/Dockerfile .
-        docker images
-        ;;
-esac

ci/setup.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    linux)
-        sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev libsodium-dev
-        ;;
-    osx)
-        brew install libsodium
-        ;;
-esac

docs/build.rst

@@ -32,7 +32,7 @@ Debian/Ubuntu/Kali
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix pkg-config
 
 .. warning::
    On a debian based system make sure you've installed rust with rustup.
@@ -44,6 +44,13 @@ Alpine
 
     $ apk add sqlite-dev libseccomp-dev libsodium-dev
 
+Docker
+~~~~~~
+
+.. code-block:: bash
+
+    $ DOCKER_BUILDKIT=1 docker build -t kpcyrd/sn0int .
+
 OpenBSD
 ~~~~~~~
 
@@ -73,10 +80,4 @@ After all dependencies have been installed, simply build the binary:
 
     $ cargo build --release
 
-Note that you need a different command on windows:
-
-.. code-block:: bash
-
-    $ cargo build --release --features=sqlite-bundled
-
 After the build finished the binary is located at ``target/release/sn0int``.

docs/index.rst

@@ -43,6 +43,7 @@ Getting Started
    database
    structs
    activity
+   notifications
    keyring
    config
    sandbox

docs/install.rst

@@ -19,12 +19,37 @@ Mac OSX
 
     $ brew install sn0int
 
-Debian/Ubuntu/Kali
-------------------
+Debian >= bullseye, Ubuntu >= 20.04, Kali
+-----------------------------------------
 
-Note that debian `doesn't ship the geoip2-database
-<https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=757723>`_ so we're going to
-download them automatically during the first run.
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+.. code-block:: bash
+
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg kpcyrd@archlinux.org | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int
+
+Debian <= buster, Ubuntu <= 19.10
+---------------------------------
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+.. code-block:: bash
+
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | sudo apt-key add -
+    $ sudo apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int
+
+Fedora/CentOS/Redhat
+--------------------
 
 Using rust+cargo from the repos might work for you, but we only officially
 support rust+cargo installed with `rustup <https://rustup.rs/>`_. Have a look
@@ -32,7 +57,7 @@ at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
+    $ dnf install @development-tools libsq3-devel libseccomp-devel libsodium-devel publicsuffix-list
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
@@ -63,8 +88,15 @@ Gentoo
 
 .. code-block:: bash
 
-    layman -f -o https://raw.githubusercontent.com/kpcyrd/overlay/master/overlay.xml -a kpcyrd-overlay
-    emerge --ask net-analyzer/sn0int
+    $ layman -a pentoo
+    $ emerge --ask net-analyzer/sn0int
+
+NixOS
+-----
+
+.. code-block:: bash
+
+    $ nix-env -i sn0int
 
 Windows
 -------
@@ -78,4 +110,4 @@ Make sure rust is installed and setup.
 
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
-    $ cargo install -f --path . --features=sqlite-bundled
+    $ cargo install -f --path .

docs/notifications.rst

@@ -0,0 +1,311 @@
+Notifications
+=============
+
+If you run sn0int unattended nobody might see the sn0int output. For cases like
+this you can configure notifications to send you a push notification in case
+something interesting happens. This is also especially useful if you have
+sn0int setup to run automatically.
+
+Receiving notifications
+-----------------------
+
+Notifications are just regular sn0int modules. You can install them just like
+any other module or write your own. This section contains walkthroughs on how
+to setup common integrations.
+
+Telegram
+~~~~~~~~
+
+Install the telegram notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-telegram
+
+Open your telegram app and open a chat with ``@botfather``. Send ``/newbot``
+and answer the questions. Copy ``bot_token`` and open this url in your browser:
+
+.. code-block::
+
+    https://api.telegram.org/bot**your_bot_token**/getUpdates
+
+Back on your app, open the t.me link to start a new chat with your bot, then
+send ``/start``. Reload the page in your browser, you should see the new
+message you sent. Copy the ``chat_id``.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-telegram -o bot_token=1337:foobar -o chat_id=1337 'hello world'
+
+You should receive ``hello world`` from your bot on Telegram.
+
+Pushover
+~~~~~~~~
+
+Install the pushover notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-pushover
+
+Signup for pushover and configure the app on your device. Copy th user key
+visible on the pushover dashboard. Click "Create an Application/API Token". Set
+"sn0int" as name and set an icon if you want to. Copy the api token.
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-pushover -o user_key=asdf1337 -o api_token=asdf1337 'hello world'
+
+You should receive ``hello world`` as a push notification.
+
+Discord
+~~~~~~~
+
+Install the discord notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-discord
+
+Decide which channel should receive notifications (or create a new one). Open
+the "Server Settings" of your discord server. Click on "Webhooks". Click
+"Create Webhook". Configure the Name and Channel. Copy the Webhook URL.
+
+
+Test your tokens are working correctly by sending yourself a notification:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-discord -o url=https://discord.com/api/webhooks/1337/asdf 'hello world'
+
+You should receive ``hello world`` in your discord channel.
+
+Signal
+~~~~~~
+
+Install the sn0int notification module from the registry:
+
+.. code-block:: bash
+
+    sn0int pkg install kpcyrd/notify-signal
+
+This module allows end-to-end encrypted notifications, but it's also difficult
+to setup. You need a second phone number and install both `signal-cli
+<https://github.com/AsamK/signal-cli>`_ and `sn0int-signal
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+After you've registered your second phone number with signal-cli, you can use
+sn0int-signal to expose a minimal api for notify-signal. For more detailed
+instructions and how to start the api at boot, see the `sn0int-signal README
+<https://github.com/kpcyrd/sn0int-signal>`_.
+
+Read the secret key generated at ``/etc/sn0int-signal.key`` and send a
+notification to the signal phone number:
+
+.. code-block:: bash
+
+    sn0int notify exec kpcyrd/notify-signal -o to=+31337 -o secret=asdf 'hello world'
+
+You should receive ``hello world`` from the number signed up with signal-cli.
+
+Writing your own module
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Make sure you've read the detailed instructions on how to get setup with
+`module development <scripting.html>`_.
+
+Create a new sn0int module like this:
+
+.. code-block:: bash
+
+    sn0int new ~/repos/sn0int-modules/notify-custom.lua
+
+Edit the ``-- Source:`` so it takes notifications as input:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        debug(arg)
+        info(arg['subject'])
+        info(arg['body'])
+    end
+
+Execute your script:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom 'hello world'
+
+You most likely need to pass options to avoid hard-coding keys into your
+script. Options can be fetched like this:
+
+.. code-block:: lua
+
+    -- Description: TODO your description here
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+    -- Source: notifications
+
+    function run(arg)
+        -- TODO your code here
+        -- https://sn0int.readthedocs.io/en/stable/reference.html
+
+        local foo = getopt('foo')
+        if not foo then return 'Missing -o foo= option' end
+
+        info('foo: ' .. foo)
+        info('subject: ' .. arg['subject'])
+    end
+
+And passed like this:
+
+.. code-block:: bash
+
+    sn0int notify exec notify-custom -o "foo=hello world" 'ohai'
+
+Setting up notification rules
+-----------------------------
+
+We now know how to trigger notifications manually, but we would rather trigger
+notifications if a module runs into something interesting.
+
+You can setup subscriptions on specific topics and then have a notification
+script execute automatically.
+
+Lookup the location of your sn0int config file:
+
+.. code-block:: bash
+
+    sn0int paths
+
+And open it in an editor of your choice:
+
+.. code-block:: bash
+
+    vim /home/user/.config/sn0int.toml
+
+A basic configuration could look like this:
+
+.. code-block:: toml
+
+    # You can have multiple notification sections, this one is named
+    # `demo-telegram-integration`
+    # The label can be set to whatever you want, but you may need to add
+    # double-quotes to use some characters.
+    [notifications.demo-telegram-integration]
+    # If this option is present, the notification must originate from one of
+    # the following workspaces.
+    workspaces = ["default", "some-workspace"]
+    # If this option is present, the notification must match one of the
+    # filters. You can use `*` as a wildcard to match everything except `:`.
+    topics = ["activity:harness/activity-ping:*"]
+    # Mandatory: the module to execute.
+    script = "kpcyrd/notify-telegram"
+    # The options to pass to the module, if any.
+    # Can be accessed with `getopt`
+    options = [
+        "bot_token=1337:foobar",
+        "chat_id=1337",
+    ]
+
+All options except ``script`` are optional, but setting filters is highly
+recommended.
+
+Testing notifications
+---------------------
+
+To test if your configuration works correctly you can create an event manually:
+
+.. code-block:: bash
+
+    sn0int -w some-workspace notify send activity:harness/activity-ping:dummy "hello world"
+
+If it matches any of your rules you should receive a push notifications.
+
+.. note::
+    If you want to test just the routing without actually sending something, add ``--dry-run``.
+
+Running sn0int automatically
+----------------------------
+
+Support for this is going to improve in the future, but you can already set
+this up if you're ok with a slightly buggy experience.
+
+Monitors
+~~~~~~~~
+
+Some modules are long-running and either wait for an event from a server or
+have custom polling built in that's usually configurable with an ``-o
+interval=`` option. If your module has a non-trivial setup phase, an author may
+take this approach.
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-new-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+    Restart=always
+    RestartSec=0
+
+    [Install]
+    WantedBy=multi-user.target
+
+Enable the service to run on boot:
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-new-service.service
+
+Timers
+~~~~~~
+
+If the module is only one-shot you can set it up to run with a timer:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.service
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Service]
+    User=your-user
+    ExecStart=/usr/bin/sn0int run -w your-workspace example/changeme
+
+Setup the timer like this:
+
+.. code-block::
+
+    # /etc/systemd/system/sn0int-your-other-service.timer
+
+    [Unit]
+    Description=sn0int: run example/changeme
+
+    [Timer]
+    OnBootSec=1min
+    OnUnitActiveSec=1h
+
+    [Install]
+    WantedBy=timers.target
+
+.. code-block:: bash
+
+    systemctl enable --now sn0int-your-other-service.timer

docs/reference.rst

@@ -383,6 +383,10 @@ options are set. The following options are available:
   The request body that should be json encoded.
 ``form``
   The request body that should be form encoded.
+``follow_redirects``
+  Automatically follow redirects, up to the specified number. If set to 1, only
+  one redirect is going to be followed. Defaults to 0 so redirects aren't
+  followed.
 ``body``
   The raw request body as string.
 ``into_blob``
@@ -500,6 +504,39 @@ Extract exif metadata from an image.
     if last_err() then return end
     debug(exif)
 
+img_ahash
+---------
+
+Calculate the Mean (aHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_ahash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_dhash
+---------
+
+Calculate the Gradient (dHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_dhash(blob)
+    if last_err() then return end
+    debug(hash)
+
+img_phash
+---------
+
+Calculate the DCT (pHash) perceptual hash.
+
+.. code-block:: lua
+
+    hash = img_phash(blob)
+    if last_err() then return end
+    debug(hash)
+
 img_nudity
 ----------
 

docs/sandbox.rst

@@ -6,7 +6,7 @@ child process. It's important to note that there's a basic sandbox that's
 active on every operating system, and there's a second line of defense on
 supported operating systems.
 
-The first line of defense is the restrictive stdlib. It's assumed that and
+The first line of defense is the restrictive stdlib. It's assumed that an
 attacker gains full control over the lua code and is able to call any function
 with arbitrary arguments. The stdlib only provides functions that are
 considered safe, so for example it's not possible to start a process or open a
@@ -119,7 +119,7 @@ You might experience a sandbox failure, especially on architectures that are
 less popular. This usually looks like this::
 
     [sn0int][example][kpcyrd/ctlogs] > run
-    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [-] Failed "example.com": Sandbox child has crashed
     [+] Finished kpcyrd/ctlogs (1 errors)
 
 A module that never finishes could also mean an IO thread inside the worker got

migrations/2020-06-12-222250_ttl-values/down.sql

@@ -0,0 +1,18 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, expire)
+    SELECT id, family, key, expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

migrations/2020-06-12-222250_ttl-values/up.sql

@@ -0,0 +1,19 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ttls_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
+
+INSERT INTO _ttls_new (id, family, key, value, expire)
+    SELECT id, family, key, "unknown", expire
+    FROM ttls;
+
+DROP TABLE ttls;
+ALTER TABLE _ttls_new RENAME TO ttls;
+
+PRAGMA foreign_keys=on;

modules/harness/activity-ping-once.lua

@@ -0,0 +1,26 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    db_activity({
+        topic=topic,
+        time=sn0int_time(),
+        uniq=uniq,
+        latitude=lat,
+        longitude=lon,
+        radius=radius,
+        content={
+            msg='ohai',
+        },
+    })
+end

modules/harness/commit-log.lua

@@ -0,0 +1,26 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- git log -s --format='%H %ci'
+
+function run()
+    while true do
+        local x = stdin_readline()
+        if x == nil then
+            break
+        end
+        local m = regex_find('^(\\S+) (.+)', x)
+        if m then
+            time = strptime('%Y-%m-%d %T %z', m[3])
+            time = sn0int_time_from(time)
+
+            db_activity({
+                topic='harness/sn0int-commit:dummy',
+                time=time,
+                uniq=m[2],
+                content={},
+            })
+        end
+    end
+end

modules/harness/dummy-resolve.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: subdomains
+
+function run(arg)
+    db_update('subdomain', arg, {
+        resolvable=true,
+    })
+end

modules/harness/dummy-subdomains.lua

@@ -0,0 +1,13 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    for i=1, 20 do
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=http_mksession() .. '.' .. arg['value'],
+        })
+    end
+end

modules/harness/img-hash.lua

@@ -0,0 +1,11 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: images
+
+function run(arg)
+    debug(arg)
+    info(img_ahash(arg['value']))
+    info(img_dhash(arg['value']))
+    info(img_phash(arg['value']))
+end

modules/harness/import-subdomains.lua

@@ -3,7 +3,7 @@
 -- License: GPL-3.0
 
 function run()
-    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lu
+    -- echo 'www.example.com' | sn0int run -vvf --stdin modules/harness/import-subdomains.lua
 
     while true do
         local line = stdin_readline()

modules/harness/notify-ohai.lua

@@ -0,0 +1,8 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/notify-ratelimit.lua

@@ -0,0 +1,9 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    ratelimit_throttle('foo', 3, 10000)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/notify-slow.lua

@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    -- TODO your code here
+    sleep(1)
+    info('notication!: ' .. json_encode(arg))
+end

modules/harness/onion-http.lua

@@ -1,5 +1,6 @@
 -- Description: Send a request to a hidden service
 -- Version: 0.1.0
+-- Stealth: passive
 -- License: GPL-3.0
 
 function run()

modules/harness/selftest.lua

@@ -1,5 +1,6 @@
 -- Description: basic selftest
 -- Version: 0.1.0
+-- Stealth: offline
 -- License: GPL-3.0
 
 function run()

modules/harness/stdin.lua

@@ -1,5 +1,6 @@
 -- Description: Read from stdin
 -- Version: 0.1.0
+-- Stealth: offline
 -- License: GPL-3.0
 
 function run()

modules/harness/winkekatze-sub.lua

@@ -10,8 +10,17 @@ function run()
     mqtt_subscribe(sock, '#', 0)
 
     while true do
-        local pkt = mqtt_recv_text(sock)
+        -- read the next mqtt packet
+        local pkt = mqtt_recv(sock)
         if last_err() then return end
-        info(pkt)
+
+        local text
+        if pkt then
+            -- attempt to utf8 decode the body if there was a pkt
+            text = utf8_decode(pkt['body'])
+            if last_err() then clear_err() end
+        end
+
+        info({pkt=pkt, text=text})
     end
 end

modules/harness/xss.lua

@@ -1,6 +1,7 @@
 -- Description: jaVasCript:/*-/*`/*\`/*'/*"/**/(/* */oNcliCk=alert() )//%0D%0A%0d%0a//</stYle/</titLe/</teXtarEa/</scRipt/--!>\x3csVg/<sVg/oNloAd=alert()//>\x3e
 -- Version: 0.1.0
 -- License: GPL-3.0
+-- Stealth: loud
 -- Source: domains
 
 function run()

sn0int-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.10.0"
+version = "0.13.0"
 description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -8,9 +8,8 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-serde = "1.0"
-serde_derive = "1.0"
+serde = { version = "1.0", features=["derive"] }
 #rocket_failure_errors = { path = "../../../rocket_failure/rocket_failure_errors" }
 rocket_failure_errors = "0.2"
-failure = "0.1"
-nom = "5.0"
+anyhow = "1.0"
+nom = "7.0"

sn0int-common/src/api.rs

@@ -1,4 +1,5 @@
 use crate::id::ModuleID;
+use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct WhoamiResponse {

sn0int-common/src/errors.rs

@@ -1,2 +1,2 @@
-pub use failure::{Error, ResultExt};
+pub use anyhow::{anyhow, bail, format_err, Context, Error};
 pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-common/src/id.rs

@@ -1,11 +1,9 @@
 use crate::errors::*;
-use nom;
-use serde::{de, Serialize, Serializer, Deserialize, Deserializer};
+use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
 use std::fmt;
 use std::result;
 use std::str::FromStr;
 
-
 #[inline(always)]
 fn valid_char(c: char) -> bool {
     nom::character::is_alphanumeric(c as u8) || c == '-'
@@ -53,7 +51,7 @@ impl FromStr for ModuleID {
 
     fn from_str(s: &str) -> Result<ModuleID> {
         let (trailing, module) = module(s)
-            .map_err(|err| format_err!("Failed to parse module id: {:?}", err))?;
+            .map_err(|err| anyhow!("Failed to parse module id: {:?}", err))?;
         if !trailing.is_empty() {
             bail!("Trailing data in module id");
         }
@@ -72,7 +70,8 @@ impl Serialize for ModuleID {
 
 impl<'de> Deserialize<'de> for ModuleID {
     fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-        where D: Deserializer<'de>
+    where
+        D: Deserializer<'de>,
     {
         let s = String::deserialize(deserializer)?;
         FromStr::from_str(&s).map_err(de::Error::custom)

sn0int-common/src/lib.rs

@@ -1,12 +1,8 @@
-#[macro_use] extern crate serde_derive;
-#[macro_use] extern crate failure;
-#[macro_use] extern crate nom;
-
 pub mod api;
 pub mod errors;
 pub use crate::errors::*;
-pub mod metadata;
 pub mod id;
+pub mod metadata;
 pub use crate::id::*;
 
 pub use rocket_failure_errors::StrictApiResponse as ApiResponse;

sn0int-common/src/metadata/mod.rs

@@ -1,7 +1,13 @@
 use crate::errors::*;
-
+use nom::bytes::complete::{tag, take_until};
+use nom::combinator::map_res;
+use nom::multi::fold_many0;
+use nom::IResult;
+use serde::{Deserialize, Serialize};
 use std::str::FromStr;
 
+mod stealth;
+pub use self::stealth::Stealth;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum EntryType {
@@ -9,6 +15,9 @@ pub enum EntryType {
     Version,
     Source,
     KeyringAccess,
+    Stealth,
+    Author,
+    Repository,
     License,
 }
 
@@ -21,6 +30,9 @@ impl FromStr for EntryType {
             "Version" => Ok(EntryType::Version),
             "Source" => Ok(EntryType::Source),
             "Keyring-Access" => Ok(EntryType::KeyringAccess),
+            "Stealth" => Ok(EntryType::Stealth),
+            "Author" => Ok(EntryType::Author),
+            "Repository" => Ok(EntryType::Repository),
             "License" => Ok(EntryType::License),
             x => bail!("Unknown EntryType: {:?}", x),
         }
@@ -44,6 +56,7 @@ pub enum Source {
     Netblocks,
     CryptoAddrs(Option<String>),
     KeyRing(String),
+    Notifications,
 }
 
 impl Source {
@@ -63,6 +76,7 @@ impl Source {
             Source::Ports => "ports",
             Source::Netblocks => "netblocks",
             Source::CryptoAddrs(_) => "cryptoaddrs",
+            Source::Notifications => "notifications",
             Source::KeyRing(_) => "keyring",
         }
     }
@@ -94,6 +108,7 @@ impl FromStr for Source {
             ("ports", None) => Ok(Source::Ports),
             ("netblocks", None) => Ok(Source::Netblocks),
             ("cryptoaddrs", param) => Ok(Source::CryptoAddrs(param.map(String::from))),
+            ("notifications", None) => Ok(Source::Notifications),
             ("keyring", Some(param)) => Ok(Source::KeyRing(param.to_string())),
             (x, Some(param)) => bail!("Unknown Source: {:?} ({:?})", x, param),
             (x, None) => bail!("Unknown Source: {:?}", x),
@@ -133,6 +148,9 @@ pub struct Metadata {
     pub version: String,
     pub source: Option<Source>,
     pub keyring_access: Vec<String>,
+    pub stealth: Stealth,
+    pub authors: Vec<String>,
+    pub repository: Option<String>,
     pub license: License,
 }
 
@@ -151,6 +169,9 @@ impl FromStr for Metadata {
                 EntryType::Version => data.version = Some(v),
                 EntryType::Source => data.source = Some(v),
                 EntryType::KeyringAccess => data.keyring_access.push(v),
+                EntryType::Stealth => data.stealth = Some(v),
+                EntryType::Author => data.authors.push(v),
+                EntryType::Repository => data.repository = Some(v),
                 EntryType::License => data.license = Some(v),
             }
         }
@@ -165,6 +186,9 @@ pub struct NewMetadata<'a> {
     pub version: Option<&'a str>,
     pub source: Option<&'a str>,
     pub keyring_access: Vec<&'a str>,
+    pub stealth: Option<&'a str>,
+    pub authors: Vec<&'a str>,
+    pub repository: Option<&'a str>,
     pub license: Option<&'a str>,
 }
 
@@ -179,6 +203,14 @@ impl<'a> NewMetadata<'a> {
         let keyring_access = self.keyring_access.into_iter()
             .map(String::from)
             .collect();
+        let stealth = match self.stealth {
+            Some(x) => x.parse()?,
+            _ => Stealth::Normal,
+        };
+        let authors = self.authors.into_iter()
+            .map(String::from)
+            .collect();
+        let repository = self.repository.map(String::from);
         let license = self.license.ok_or_else(|| format_err!("License is required"))?;
         let license = license.parse()?;
 
@@ -187,28 +219,40 @@ impl<'a> NewMetadata<'a> {
             version: version.to_string(),
             source,
             keyring_access,
+            stealth,
+            authors,
+            repository,
             license,
         })
     }
 }
 
-named!(metaline<&str, (EntryType, &str)>, do_parse!(
-    tag!("-- ") >>
-    name: map_res!(take_until!(": "), EntryType::from_str) >>
-    tag!(": ") >>
-    value: take_until!("\n") >>
-    tag!("\n") >>
-    (name, value)
-));
+fn metaline(input: &str) -> IResult<&str, (EntryType, &str)> {
+    let (input, _) = tag("-- ")(input)?;
+    let (input, name) = map_res(
+        take_until(": "),
+        EntryType::from_str
+    )(input)?;
+    let (input, _) = tag(": ")(input)?;
+    let (input, value) = take_until("\n")(input)?;
+    let (input, _) = tag("\n")(input)?;
 
-named!(metalines<&str, Vec<(EntryType, &str)>>, do_parse!(
-    lines: fold_many0!(metaline, Vec::new(), |mut acc: Vec<_>, item| {
-        acc.push(item);
-        acc
-    }) >>
-    tag!("\n") >>
-    (lines)
-));
+    Ok((input, (name, value)))
+}
+
+fn metalines(input: &str) -> IResult<&str, Vec<(EntryType, &str)>> {
+    let (input, lines) = fold_many0(
+        metaline,
+        Vec::new,
+        |mut acc: Vec<_>, item| {
+            acc.push(item);
+            acc
+        }
+    )(input)?;
+    let (input, _) = tag("\n")(input)?;
+
+    Ok((input, lines))
+}
 
 #[cfg(test)]
 mod tests {
@@ -227,6 +271,36 @@ mod tests {
             version: "1.0.0".to_string(),
             license: License::WTFPL,
             source: Some(Source::Domains),
+            stealth: Stealth::Normal,
+            authors: vec![],
+            repository: None,
+            keyring_access: Vec::new(),
+        });
+    }
+
+    #[test]
+    fn verify_much_metadata() {
+        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+-- Version: 1.0.0
+-- Source: domains
+-- Stealth: passive
+-- Author: kpcyrd <git at rxv dot cc>
+-- Author: kpcyrd's cat
+-- Repository: https://github.com/kpcyrd/sn0int
+-- License: WTFPL
+
+"#).expect("parse");
+        assert_eq!(metadata, Metadata {
+            description: "Hello world, this is my description".to_string(),
+            version: "1.0.0".to_string(),
+            license: License::WTFPL,
+            source: Some(Source::Domains),
+            stealth: Stealth::Passive,
+            authors: vec![
+                "kpcyrd <git at rxv dot cc>".to_string(),
+                "kpcyrd's cat".to_string(),
+            ],
+            repository: Some("https://github.com/kpcyrd/sn0int".to_string()),
             keyring_access: Vec::new(),
         });
     }
@@ -243,6 +317,9 @@ mod tests {
             version: "1.0.0".to_string(),
             license: License::WTFPL,
             source: None,
+            stealth: Stealth::Normal,
+            authors: vec![],
+            repository: None,
             keyring_access: Vec::new(),
         });
     }

sn0int-common/src/metadata/stealth.rs

@@ -0,0 +1,53 @@
+use crate::errors::*;
+use serde::{Deserialize, Serialize};
+use std::str::FromStr;
+
+#[derive(Debug, Eq, PartialEq, PartialOrd, Clone, Serialize, Deserialize)]
+pub enum Stealth {
+    Loud,
+    Normal,
+    Passive,
+    Offline,
+}
+
+impl Stealth {
+    #[inline]
+    pub fn variants() -> &'static [&'static str] {
+        &[
+            "loud",
+            "normal",
+            "passive",
+            "offline",
+        ]
+    }
+
+    #[inline(always)]
+    fn as_num(&self) -> u8 {
+        match self {
+            Stealth::Loud => 3,
+            Stealth::Normal => 2,
+            Stealth::Passive => 1,
+            Stealth::Offline => 0,
+        }
+    }
+
+    #[inline]
+    pub fn equal_or_better(&self, other: &Self) -> bool {
+        self.as_num() <= other.as_num()
+    }
+}
+
+impl FromStr for Stealth {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Stealth> {
+        match s {
+            "loud" => Ok(Stealth::Loud),
+            // This is also the default level if none is provided
+            "normal" => Ok(Stealth::Normal),
+            "passive" => Ok(Stealth::Passive),
+            "offline" => Ok(Stealth::Offline),
+            x => bail!("Unknown stealth: {:?}", x),
+        }
+    }
+}

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.10.0", path="../sn0int-common" }
+sn0int-common = { version="0.13.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }
@@ -23,16 +23,15 @@ oauth2 = "2.0.0"
 failure = "0.1"
 url = "1.0"
 log = "0.4"
-semver = "0.9.0"
+semver = "1"
 lazy_static = "1"
 blake2 = "0.8.0"
 hex = "0.4"
 maplit = "1.0.1"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 syntect = "3.3"
 
-serde = "1.0"
-serde_derive = "1.0"
-serde_json = "1.0"
-
 dotenv = "0.15"
-env_logger = "0.7"
+env_logger = "0.9"
+structopt = "0.3.21"

sn0int-registry/src/assets.rs

@@ -1,5 +1,5 @@
 use blake2::{Blake2b, Digest};
-
+use lazy_static::lazy_static;
 
 pub static FAVICON: &[u8] = include_bytes!("../assets/favicon.ico");
 pub static STYLESHEET: &[u8] = include_bytes!("../assets/style.css");

sn0int-registry/src/auth.rs

@@ -1,8 +1,8 @@
+use crate::github;
 use diesel::pg::PgConnection;
 use oauth2::basic::BasicClient;
 use oauth2::prelude::*;
 use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl, TokenResponse};
-use crate::github::GithubAuthenticator;
 use sn0int_registry::errors::*;
 use sn0int_registry::models::AuthToken;
 use url::Url;
@@ -67,8 +67,7 @@ impl Authenticator {
         let access_token = response.access_token();
         let access_token = access_token.secret().to_string();
 
-        let client = GithubAuthenticator::from_env()?;
-        let user = client.get_username(&access_token)?;
+        let user = github::get_username(&access_token)?;
 
         AuthToken::create(&AuthToken {
             id: state,

sn0int-registry/src/auth2.rs

@@ -4,7 +4,7 @@ use sn0int_registry::db::Connection;
 use rocket::http::Status;
 use rocket::{Request, Outcome};
 use rocket::request::{self, FromRequest};
-use crate::github::GithubAuthenticator;
+use crate::github;
 
 
 pub struct AuthHeader(String);
@@ -12,8 +12,7 @@ pub struct AuthHeader(String);
 impl AuthHeader {
     pub fn verify(&self, connection: &Connection) -> Result<String> {
         let session = AuthToken::read(&self.0, &connection)?;
-        let client = GithubAuthenticator::from_env()?;
-        client.get_username(&session.access_token)
+        github::get_username(&session.access_token)
             .map_err(Error::from)
     }
 }

sn0int-registry/src/errors.rs

@@ -1,4 +1,5 @@
-pub use failure::{Error, ResultExt};
+pub use failure::{Error, ResultExt, bail, format_err};
 pub type Result<T> = ::std::result::Result<T, Error>;
+pub use log::{debug, info};
 
 pub use rocket_failure::errors::*;

sn0int-registry/src/github.rs

@@ -1,54 +1,24 @@
+use serde::Deserialize;
 use sn0int_registry::errors::*;
-use std::env;
-use reqwest;
 
+pub fn get_username(oauth_token: &str) -> Result<String> {
+    let client = reqwest::Client::new();
+    let mut resp = client.get("https://api.github.com/user")
+        .header("Authorization", format!("token {}", oauth_token))
+        .header("User-Agent", "sn0int-registry")
+        .send()
+        .context("Failed to check access_token")?
+        .error_for_status()
+        .context("Github returned http error")?;
 
-pub struct GithubAuthenticator {
-    client_id: String,
-    client_secret: String,
-}
+    let data = resp.json::<GithubUser>()
+        .context("Failed to deserialize github reply")?;
 
-impl GithubAuthenticator {
-    pub fn new(client_id: String, client_secret: String) -> GithubAuthenticator {
-        GithubAuthenticator {
-            client_id,
-            client_secret,
-        }
-    }
-
-    pub fn from_env() -> Result<GithubAuthenticator> {
-        let client_id = env::var("GITHUB_CLIENT_ID")
-            .context("GITHUB_CLIENT_ID is not set")?;
-        let client_secret = env::var("GITHUB_CLIENT_SECRET")
-            .context("GITHUB_CLIENT_SECRET is not set")?;
-        Ok(GithubAuthenticator::new(client_id, client_secret))
-    }
-
-    pub fn get_username(&self, oauth_token: &str) -> Result<String> {
-        let url = format!("https://api.github.com/applications/{}/tokens/{}", self.client_id, oauth_token);
-        let client = reqwest::Client::new();
-        let mut resp = client.get(&url)
-            .basic_auth(&self.client_id, Some(&self.client_secret))
-            .send()?;
-
-        if !resp.status().is_success() {
-            bail!("Github returned: {}", resp.status())
-        }
-
-        let data = resp.json::<GithubReply>()
-            .context("Failed to deserialize github reply")?;
-
-        Ok(data.user.login)
-    }
+    Ok(data.login)
 }
 
 #[derive(Debug, Deserialize)]
-pub struct GithubReply {
-    user: GithubUser,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct GithubUser {
+struct GithubUser {
     login: String,
     #[serde(rename="type")]
     user_type: String,

sn0int-registry/src/lib.rs

@@ -1,7 +1,5 @@
 #[macro_use] extern crate diesel;
 #[macro_use] extern crate diesel_migrations;
-#[macro_use] extern crate failure;
-#[macro_use] extern crate serde_derive;
 
 pub mod db;
 pub mod errors;

sn0int-registry/src/main.rs

@@ -5,21 +5,16 @@
 #[macro_use] extern crate rocket;
 #[macro_use] extern crate rocket_contrib;
 #[macro_use] extern crate rocket_failure;
-#[macro_use] extern crate serde_derive;
-#[macro_use] extern crate log;
-#[macro_use] extern crate maplit;
-#[macro_use] extern crate lazy_static;
-#[macro_use] extern crate failure;
 
+use dotenv::dotenv;
 use rocket::fairing::AdHoc;
 use rocket::http::Header;
 use rocket_contrib::json::{Json, JsonValue};
 use rocket_contrib::templates::Template;
-use dotenv::dotenv;
-
-use std::env;
-use sn0int_registry::errors::*;
 use sn0int_registry::db;
+use sn0int_registry::errors::*;
+use std::env;
+use structopt::StructOpt;
 
 pub mod assets;
 pub mod auth;
@@ -49,7 +44,13 @@ fn internal_error() -> Json<JsonValue> {
     }))
 }
 
+#[derive(Debug, StructOpt)]
+pub struct Args {
+}
+
 fn run() -> Result<()> {
+    let _args = Args::from_args();
+
     dotenv().ok();
 
     let database_url = env::var("DATABASE_URL")

sn0int-registry/src/models.rs

@@ -4,6 +4,7 @@ use diesel::pg::PgConnection;
 use diesel::sql_types::BigInt;
 use diesel_full_text_search::{plainto_tsquery, TsQueryExtensions};
 use crate::schema::*;
+use serde::{Serialize, Deserialize};
 use std::collections::HashMap;
 use std::time::SystemTime;
 use sn0int_common::ModuleID;

sn0int-registry/src/routes/auth.rs

@@ -1,12 +1,13 @@
-use sn0int_registry::errors::*;
+use crate::assets::ASSET_REV;
 use crate::auth::Authenticator;
+use serde::{Serialize, Deserialize};
+use serde_json::Value;
 use sn0int_registry::db;
+use sn0int_registry::errors::*;
 use rocket::request::Form;
 use rocket::response::Redirect;
 use rocket_contrib::templates::Template;
-use crate::assets::ASSET_REV;
-use serde_json::{self, Value};
-
+use maplit::hashmap;
 
 #[get("/?<auth..>")]
 pub fn get(auth: Form<OAuth>) -> Template {

sn0int-registry/src/routes/badges.rs

@@ -1,3 +1,4 @@
+use serde::Serialize;
 use sn0int_registry::errors::*;
 use sn0int_registry::db;
 use sn0int_registry::models::*;

sn0int-registry/src/routes/html.rs

@@ -1,10 +1,17 @@
-use sn0int_registry::errors::*;
 use crate::assets::*;
+use lazy_static::lazy_static;
 use sn0int_registry::db;
+use sn0int_registry::errors::*;
 use sn0int_registry::models::*;
-use rocket_contrib::templates::Template;
 use std::cmp::Ordering;
+use syntect::html::ClassedHTMLGenerator;
+use syntect::parsing::{SyntaxSet, SyntaxReference};
+use rocket_contrib::templates::Template;
 
+lazy_static! {
+    pub static ref SYNTAX_SET: SyntaxSet = SyntaxSet::load_defaults_newlines();
+    pub static ref SYNTAX: &'static SyntaxReference = SYNTAX_SET.find_syntax_by_extension("lua").unwrap();
+}
 
 #[get("/")]
 pub fn index(connection: db::Connection) -> ApiResult<Template> {
@@ -30,14 +37,6 @@ pub fn index(connection: db::Connection) -> ApiResult<Template> {
     })))
 }
 
-use syntect::html::ClassedHTMLGenerator;
-use syntect::parsing::{SyntaxSet, SyntaxReference};
-
-lazy_static! {
-    pub static ref SYNTAX_SET: SyntaxSet = SyntaxSet::load_defaults_newlines();
-    pub static ref SYNTAX: &'static SyntaxReference = SYNTAX_SET.find_syntax_by_extension("lua").unwrap();
-}
-
 #[get("/r/<author>/<name>")]
 pub fn details(author: String, name: String,connection: db::Connection) -> ApiResult<Template> {
     let module = Module::find(&author, &name, &connection)

sn0int-std/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "sn0int-std"
-version = "0.18.0"
+version = "0.25.0"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
 license = "GPL-3.0"
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -14,42 +14,44 @@ log = "0.4"
 failure = "0.1"
 hlua-badtouch = "0.4"
 tokio = "0.1.14"
-rand = "0.7"
+rand = "0.8"
 regex = "1.0"
-rustls = { version="0.16", features=["dangerous_configuration"] }
-serde = "1.0"
-serde_derive = "1.0"
+rustls = { version="0.18", features=["dangerous_configuration"] }
+serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-serde_urlencoded = "0.6"
+serde_urlencoded = "0.7"
 webpki = "0.21"
-webpki-roots = "0.19.0"
-ct-logs = "0.6"
-chrootable-https = "0.14"
+webpki-roots = "0.21"
+ct-logs = "0.7"
+#chrootable-https = { path = "../../chrootable-https" }
+chrootable-https = "0.16"
 http = "0.2"
 bufstream = "0.1.4"
-pem = "0.7"
+pem = "1"
 url = "2.0"
-tungstenite = { version = "0.10.1", default-features = false }
+tungstenite = { version = "0.13", default-features = false }
 kuchiki = "0.8.0"
-maxminddb = "0.13"
-x509-parser = "0.6.2"
-der-parser = "3.0"
-publicsuffix = { version="1.5", default-features=false }
+maxminddb = "0.23"
+x509-parser = "0.13"
+der-parser = "8"
+publicsuffix = { version="2", default-features=false }
 xml-rs = "0.8"
-geo = "0.12"
+geo = "0.23"
 bytes = "0.4"
-base64 = "0.11"
 chrono = { version = "0.4", features = ["serde"] }
-mqtt-protocol = "0.8.1"
+mqtt-protocol = "0.11"
 sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
 
-image = "0.23.0"
+image = "0.23"
 kamadak-exif = "0.5.1"
+img_hash_median = "4.0.0"
 
-bs58 = "0.3"
-digest = "0.8.0"
-blake2 = "0.8.0"
+bs58 = "0.4"
+digest = "0.10"
+blake2 = "0.10"
+data-encoding = "2.3.3"
+thiserror = "1.0.38"
 
 [dev-dependencies]
-env_logger = "0.7"
+env_logger = "0.9"
 maplit = "1.0.1"

sn0int-std/src/blobs.rs

@@ -1,8 +1,9 @@
+use blake2::Blake2bVar;
 use bytes::Bytes;
-use blake2::VarBlake2b;
-use digest::{Input, VariableOutput};
-use serde::ser::{Serialize, Serializer};
+use data_encoding::BASE64;
+use digest::{Update, VariableOutput};
 use serde::de::{self, Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
 use std::result;
 
 #[derive(Debug, Clone, PartialEq)]
@@ -21,9 +22,10 @@ impl Blob {
     }
 
     pub fn hash(bytes: &[u8]) -> String {
-        let mut h = VarBlake2b::new(32).unwrap();
-        h.input(bytes);
-        Self::encode_hash(&h.vec_result())
+        let mut h = Blake2bVar::new(32).unwrap();
+        h.update(bytes);
+        let output = h.finalize_boxed();
+        Self::encode_hash(&output)
     }
 
     #[inline]
@@ -39,7 +41,7 @@ impl Serialize for Blob {
     where
         S: Serializer,
     {
-        let s = base64::encode(&self.bytes);
+        let s = BASE64.encode(&self.bytes);
         serializer.serialize_str(&s)
     }
 }
@@ -51,7 +53,7 @@ impl<'de> Deserialize<'de> for Blob {
         D: Deserializer<'de>,
     {
         let s = String::deserialize(deserializer)?;
-        let bytes = base64::decode(&s)
+        let bytes = BASE64.decode(s.as_bytes())
             .map_err(de::Error::custom)?;
         Ok(Blob::create(Bytes::from(bytes)))
     }

sn0int-std/src/crt.rs

@@ -1,78 +1,11 @@
 use crate::errors::*;
-
-use x509_parser;
-use der_parser::der::DerObject;
-use der_parser::ber::{BerObjectContent, BerTag};
-use der_parser::oid::Oid;
+use serde::{Serialize, Deserialize};
 use std::collections::HashSet;
-use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
-
-
-#[derive(Debug, PartialEq)]
-pub enum AlternativeName {
-    DnsName(String),
-    Email(String),
-    IpAddr(IpAddr),
-}
-
-pub fn san_extension(i: &[u8]) -> Result<Vec<AlternativeName>> {
-    let (rem, seq) = der_parser::der::parse_der_sequence(i)
-        .map_err(|_| format_err!("Failed to parse san extension"))?;
-
-    if !rem.is_empty() {
-        bail!("san extension has trailing garbage");
-    }
-
-    debug!("Decoded sequence: {:?}", seq);
-    if let BerObjectContent::Sequence(seq) = seq.content {
-        seq.into_iter()
-            .map(san_value)
-            .collect()
-    } else {
-        bail!("Expected der sequence");
-    }
-}
-
-pub fn san_value(o: DerObject) -> Result<AlternativeName> {
-    debug!("DER object in SAN extension: {:?}", o);
-
-    match (o.class, o.tag, &o.content) {
-        (2, BerTag::Integer, BerObjectContent::Unknown(BerTag::Integer, value)) => san_value_dns(value),
-        (2, BerTag::Boolean, BerObjectContent::Unknown(BerTag::Boolean, value)) => san_value_email(value),
-        (2, BerTag::ObjDescriptor, BerObjectContent::Unknown(BerTag::ObjDescriptor, value)) => san_value_ipaddr(value),
-        _ => bail!("Unexpected object: {:?}", o),
-    }
-}
-
-pub fn san_value_dns(v: &[u8]) -> Result<AlternativeName> {
-    debug!("Reading as dns name: {:?}", v);
-    String::from_utf8(v.to_vec())
-        .map(AlternativeName::DnsName)
-        .map_err(Error::from)
-}
-
-pub fn san_value_email(v: &[u8]) -> Result<AlternativeName> {
-    debug!("Reading as email: {:?}", v);
-    String::from_utf8(v.to_vec())
-        .map(AlternativeName::Email)
-        .map_err(Error::from)
-}
-
-pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
-    debug!("Reading as ipaddr: {:?}", v);
-    match v.len() {
-        4  => Ok(AlternativeName::IpAddr(Ipv4Addr::from([
-            v[0], v[1], v[2], v[3],
-        ]).into())),
-        16 => Ok(AlternativeName::IpAddr(Ipv6Addr::from([
-            v[0],  v[1],  v[2],  v[3],
-            v[4],  v[5],  v[6],  v[7],
-            v[8],  v[9],  v[10], v[11],
-            v[12], v[13], v[14], v[15],
-        ]).into())),
-        _ => Err(format_err!("Invalid ipaddr")),
-    }
-}
+use std::net::IpAddr;
+use x509_parser::x509::X509Version;
+use x509_parser::certificate::X509Certificate;
+use x509_parser::extensions::{GeneralName, ParsedExtension};
+use x509_parser::prelude::*;
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
@@ -83,7 +16,7 @@ pub struct Certificate {
 
 impl Certificate {
     pub fn parse_pem(crt: &str) -> Result<Certificate> {
-        let pem = match x509_parser::pem::pem_to_der(crt.as_bytes()) {
+        let pem = match x509_parser::pem::parse_x509_pem(crt.as_bytes()) {
             Ok((remaining, pem)) => {
                 if !remaining.is_empty() {
                     bail!("input cert has trailing garbage");
@@ -99,12 +32,12 @@ impl Certificate {
     }
 
     pub fn from_bytes(crt: &[u8]) -> Result<Certificate> {
-        let crt = match x509_parser::parse_x509_der(&crt) {
+        let crt = match X509Certificate::from_der(crt) {
             Ok((remaining, der)) => {
                 if !remaining.is_empty() {
                     bail!("input cert has trailing garbage");
                 }
-                if der.tbs_certificate.version != 2 {
+                if der.tbs_certificate.version != X509Version::V3 {
                     bail!("unexpected certificate version");
                 }
                 der
@@ -116,35 +49,45 @@ impl Certificate {
         let mut valid_emails = HashSet::new();
         let mut valid_ipaddrs = HashSet::new();
 
-        for x in crt.tbs_certificate.subject.rdn_seq {
-            for y in x.set {
-                // CommonName
-                if y.attr_type != Oid::from(&[2, 5, 4, 3]) {
-                    continue;
-                }
-
-                if let Ok(x) = y.attr_value.content.as_slice() {
-                    let value = String::from_utf8(x.to_vec())?;
-                    info!("Found CN in Subject: {:?}", value);
-                    valid_names.insert(value);
-                }
+        for attr in crt.subject().iter_common_name() {
+            if let Ok(cn) = attr.as_str() {
+                info!("Found CN in Subject: {:?}", cn);
+                valid_names.insert(cn.to_string());
             }
         }
 
-        for x in crt.tbs_certificate.extensions {
-            if x.oid != Oid::from(&[2, 5, 29, 17]) {
-                continue;
-            }
-
-            debug!("Found san extension: {:?}", x.value);
-            let values = san_extension(x.value)?;
-
-            for v in values {
-                match v {
-                    AlternativeName::DnsName(v) => valid_names.insert(v),
-                    AlternativeName::Email(v) => valid_emails.insert(v),
-                    AlternativeName::IpAddr(v) => valid_ipaddrs.insert(v),
-                };
+        for ext in crt.tbs_certificate.extensions() {
+            if let ParsedExtension::SubjectAlternativeName(san) = ext.parsed_extension() {
+                for name in &san.general_names {
+                    debug!("Certificate is valid for {:?}", name);
+                    match name {
+                        GeneralName::DNSName(v) => {
+                            valid_names.insert(v.to_string());
+                        },
+                        GeneralName::RFC822Name(v) => {
+                            valid_emails.insert(v.to_string());
+                        },
+                        GeneralName::IPAddress(v) => {
+                            let ip = match v.len() {
+                                4 => Some(IpAddr::from([v[0], v[1], v[2], v[3]])),
+                                16 => Some(IpAddr::from([
+                                    v[0], v[1], v[2], v[3],
+                                    v[4], v[5], v[6], v[7],
+                                    v[8], v[9], v[10], v[11],
+                                    v[12], v[13], v[14], v[15],
+                                ])),
+                                _ => {
+                                    info!("Certificate is valid for invalid ip address: {:?}", v);
+                                    None
+                                },
+                            };
+                            if let Some(ip) = ip {
+                                valid_ipaddrs.insert(ip);
+                            }
+                        },
+                        _ => (),
+                    }
+                }
             }
         }
 
@@ -162,7 +105,6 @@ impl Certificate {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use der_parser::parse_der;
 
     #[test]
     fn test_parse_pem_github() {
@@ -260,19 +202,6 @@ ZkZZmqNn2Q8=
         });
     }
 
-    #[test]
-    fn test_san_extension() {
-        let ext = san_extension(&[48, 28,
-                130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109,
-                130, 14, 119, 119, 119, 46, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109
-            ])
-            .expect("Failed to parse extension");
-        assert_eq!(ext, vec![
-            AlternativeName::DnsName(String::from("github.com")),
-            AlternativeName::DnsName(String::from("www.github.com")),
-        ]);
-    }
-
     #[test]
     fn test_long_san_extension() {
         let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
@@ -372,48 +301,6 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
         });
     }
 
-    #[test]
-    fn test_san_value_dns() {
-        let (rem, v) = parse_der(&[130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
-            .expect("Failed to parse san value");
-        assert!(rem.is_empty());
-        println!("{:?}", v);
-        assert_eq!(v, DerObject {
-            class: 2,
-            structured: 0,
-            tag: BerTag::Integer,
-            content: BerObjectContent::Unknown(BerTag::Integer, &[103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
-        });
-        let content = match v.content {
-            BerObjectContent::Unknown(BerTag::Integer, v) => v,
-            _ => panic!("Wrong BerObjectContent"),
-        };
-        let v = san_value_dns(content)
-            .expect("Failed to process san value");
-        assert_eq!(v, AlternativeName::DnsName(String::from("github.com")));
-    }
-
-    #[test]
-    fn test_san_value_ipaddr() {
-        let (rem, v) = parse_der(&[135, 4, 1, 1, 1, 1])
-            .expect("Failed to parse san value");
-        assert!(rem.is_empty());
-        println!("{:?}", v);
-        assert_eq!(v, DerObject {
-            class: 2,
-            structured: 0,
-            tag: BerTag::ObjDescriptor,
-            content: BerObjectContent::Unknown(BerTag::ObjDescriptor, &[1, 1, 1, 1])
-        });
-        let content = match v.content {
-            BerObjectContent::Unknown(BerTag::ObjDescriptor, v) => v,
-            _ => panic!("Wrong BerObjectContent"),
-        };
-        let v = san_value_ipaddr(content)
-            .expect("Failed to process san value");
-        assert_eq!(v, AlternativeName::IpAddr("1.1.1.1".parse().unwrap()));
-    }
-
     #[test]
     fn test_san_email() {
         let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----

sn0int-std/src/crypto.rs

@@ -22,7 +22,7 @@ pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
     let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
         .ok_or_else(|| format_err!("Nonce has wrong length"))?;
     let ciphertext = &encrypted[secretbox::NONCEBYTES..];
-    let plain = secretbox::open(&ciphertext, &nonce, &key)
+    let plain = secretbox::open(ciphertext, &nonce, &key)
         .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
     Ok(plain)
 }

sn0int-std/src/engine/structs.rs

@@ -2,9 +2,6 @@ use crate::errors::*;
 use crate::hlua::{AnyHashableLuaValue, AnyLuaValue, AnyLuaString};
 use std::collections::{self, HashMap};
 use crate::json::LuaJsonValue;
-use serde;
-use serde_json;
-
 
 pub fn from_lua<T>(x: LuaJsonValue) -> Result<T>
     where for<'de> T: serde::Deserialize<'de>
@@ -89,16 +86,16 @@ impl From<Vec<(AnyLuaValue, AnyLuaValue)>> for LuaMap {
     }
 }
 
-impl Into<HashMap<AnyHashableLuaValue, AnyLuaValue>> for LuaMap {
-    fn into(self: LuaMap) -> HashMap<AnyHashableLuaValue, AnyLuaValue> {
-        self.0
+impl From<LuaMap> for HashMap<AnyHashableLuaValue, AnyLuaValue> {
+    fn from(map: LuaMap) -> Self {
+        map.0
     }
 }
 
-impl Into<AnyLuaValue> for LuaMap {
-    fn into(self: LuaMap) -> AnyLuaValue {
+impl From<LuaMap> for AnyLuaValue {
+    fn from(map: LuaMap) -> AnyLuaValue {
         AnyLuaValue::LuaArray(
-            self.into_iter()
+            map.into_iter()
                 .filter_map(|(k, v)| {
                     match k {
                         AnyHashableLuaValue::LuaString(x) => Some((AnyLuaValue::LuaString(x), v)),
@@ -139,9 +136,9 @@ impl LuaList {
     }
 }
 
-impl Into<AnyLuaValue> for LuaList {
-    fn into(self: LuaList) -> AnyLuaValue {
-        AnyLuaValue::LuaArray(self.0)
+impl From<LuaList> for AnyLuaValue {
+    fn from(list: LuaList) -> AnyLuaValue {
+        AnyLuaValue::LuaArray(list.0)
     }
 }
 
@@ -152,7 +149,7 @@ pub fn byte_array(bytes: AnyLuaValue) -> Result<Vec<u8>> {
         AnyLuaValue::LuaArray(bytes) => {
             Ok(bytes.into_iter()
                 .map(|num| match num.1 {
-                    AnyLuaValue::LuaNumber(num) if num <= 255.0 && num >= 0.0 && (num % 1.0 == 0.0) =>
+                    AnyLuaValue::LuaNumber(num) if (0.0..=255.0).contains(&num) && (num % 1.0 == 0.0) =>
                             Ok(num as u8),
                     AnyLuaValue::LuaNumber(num) =>
                             Err(format_err!("number is out of range: {:?}", num)),

sn0int-std/src/errors.rs

@@ -1,3 +1,3 @@
-pub use log::{trace, debug, info, warn, error};
-pub use failure::{Error, ResultExt, format_err, bail};
+pub use failure::{bail, format_err, Error, ResultExt};
+pub use log::{debug, error, info, trace, warn};
 pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-std/src/geo.rs

@@ -1,10 +1,9 @@
 use crate::errors::*;
-
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
-use geo::{LineString, Polygon, Coordinate};
+use geo::{LineString, Polygon, Coord};
 use geo::prelude::*;
-
+use serde::Deserialize;
 
 #[derive(Debug, Deserialize)]
 pub struct Point {
@@ -22,7 +21,7 @@ impl Point {
 
 pub fn polygon_contains(ring: &[Point], p: &Point) -> bool {
     let ring = ring.iter()
-        .map(|p| Coordinate { x: p.lon, y: p.lat })
+        .map(|p| Coord { x: p.lon, y: p.lat })
         .collect::<Vec<_>>();
 
     let polygon = Polygon::new(LineString::from(ring), vec![]);

sn0int-std/src/geoip/models.rs

@@ -1,16 +1,10 @@
 use crate::errors::*;
 use maxminddb::geoip2;
+use serde::Serialize;
 use std::collections::BTreeMap;
 
-
-fn from_geoip_model_names(names: Option<BTreeMap<String, String>>) -> Option<String> {
-    let names = match names {
-        Some(names) => names,
-        _ => return None,
-    };
-
-    names.get("en")
-        .map(|x| x.to_owned())
+fn from_geoip_model_names(names: Option<BTreeMap<&str, &str>>) -> Option<String> {
+    names?.get("en").map(|x| x.to_string())
 }
 
 #[derive(Debug, Serialize)]
@@ -24,7 +18,7 @@ pub struct GeoLookup {
     pub longitude: Option<f64>,
 }
 
-impl From<geoip2::City> for GeoLookup {
+impl<'a> From<geoip2::City<'a>> for GeoLookup {
     fn from(lookup: geoip2::City) -> GeoLookup {
         // parse maxminddb lookup
         let continent = match lookup.continent {
@@ -78,18 +72,12 @@ pub struct Continent {
 }
 
 impl Continent {
-    fn from_maxmind(continent: geoip2::model::Continent) -> Option<Self> {
-        let code = match continent.code {
-            Some(code) => code,
-            _ => return None,
-        };
-        let name = match from_geoip_model_names(continent.names) {
-            Some(name) => name,
-            _ => return None,
-        };
+    fn from_maxmind(continent: geoip2::city::Continent) -> Option<Self> {
+        let code = continent.code?;
+        let name = from_geoip_model_names(continent.names)?;
 
         Some(Continent {
-            code,
+            code: code.to_string(),
             name,
         })
     }
@@ -102,18 +90,12 @@ pub struct Country {
 }
 
 impl Country {
-    fn from_maxmind(country: geoip2::model::Country) -> Option<Self> {
-        let code = match country.iso_code {
-            Some(code) => code,
-            _ => return None,
-        };
-        let name = match from_geoip_model_names(country.names) {
-            Some(name) => name,
-            _ => return None,
-        };
+    fn from_maxmind(country: geoip2::city::Country) -> Option<Self> {
+        let code = country.iso_code?;
+        let name = from_geoip_model_names(country.names)?;
 
         Some(Country {
-            code,
+            code: code.to_string(),
             name,
         })
     }
@@ -126,7 +108,7 @@ pub struct Location {
 }
 
 impl Location {
-    fn from_maxmind(location: &geoip2::model::Location) -> Option<Self> {
+    fn from_maxmind(location: &geoip2::city::Location) -> Option<Self> {
         let latitude = match location.latitude {
             Some(latitude) => latitude,
             _ => return None,
@@ -152,19 +134,14 @@ pub struct AsnLookup {
 impl AsnLookup {
     pub fn try_from(lookup: geoip2::Isp) -> Result<AsnLookup> {
         // parse maxminddb lookup
-        let asn = match lookup.autonomous_system_number {
-            Some(asn) => asn,
-            _ => bail!("autonomous_system_number not set"),
-        };
-        let as_org = match lookup.autonomous_system_organization {
-            Some(org) => org,
-            _ => bail!("autonomous_system_organization not set"),
-        };
+        let asn = lookup.autonomous_system_number
+            .ok_or_else(|| format_err!("autonomous_system_number not set"))?;
+        let as_org = lookup.autonomous_system_organization
+            .ok_or_else(|| format_err!("autonomous_system_organization not set"))?;
 
-        // return result
         Ok(AsnLookup {
             asn,
-            as_org,
+            as_org: as_org.to_string(),
         })
     }
 }

sn0int-std/src/gfx/exif.rs

@@ -1,9 +1,7 @@
 use crate::errors::*;
-
-use exif;
+use serde::{Serialize, Deserialize};
 use std::io;
 
-
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Location {
     latitude: f64,
@@ -95,7 +93,7 @@ pub fn cardinal_direction_modifier(value: &exif::Value) -> Result<f64> {
             let s = s.get(0)
                 .ok_or_else(|| format_err!("Cardinal direction value is empty"))?;
 
-            match s.get(0) {
+            match s.first() {
                 Some(b'N') => Ok(1.0),
                 Some(b'S') => Ok(-1.0),
                 Some(b'E') => Ok(1.0),

sn0int-std/src/gfx/mod.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 use image::{self, DynamicImage, GenericImageView};
+pub use img_hash_median::HashAlg;
 
 pub mod exif;
 
@@ -45,9 +46,9 @@ impl ImageFormat {
     }
 }
 
-impl Into<image::ImageFormat> for ImageFormat {
-    fn into(self) -> image::ImageFormat {
-        match self {
+impl From<ImageFormat> for image::ImageFormat {
+    fn from(format: ImageFormat) -> image::ImageFormat {
+        match format {
             ImageFormat::Png => image::ImageFormat::Png,
             ImageFormat::Jpeg => image::ImageFormat::Jpeg,
             ImageFormat::Gif => image::ImageFormat::Gif,
@@ -79,6 +80,14 @@ impl Image {
     pub fn height(&self) -> u32 {
         self.image.height()
     }
+
+    pub fn perception_hash(&self, hash_alg: HashAlg) -> String {
+        let hasher = img_hash_median::HasherConfig::new()
+            .hash_alg(hash_alg)
+            .to_hasher();
+        let hash = hasher.hash_image(&self.image);
+        hash.to_base64()
+    }
 }
 
 impl AsRef<DynamicImage> for Image {
@@ -94,10 +103,10 @@ pub fn guess_format(buf: &[u8]) -> Result<ImageFormat> {
 }
 
 pub fn load(buf: &[u8]) -> Result<Image> {
-    let img_format = image::guess_format(&buf)?;
+    let img_format = image::guess_format(buf)?;
     let format = ImageFormat::try_from(&img_format)?;
 
-    let image = image::load_from_memory_with_format(&buf, img_format)?;
+    let image = image::load_from_memory_with_format(buf, img_format)?;
 
     Ok(Image {
         image,
@@ -105,7 +114,6 @@ pub fn load(buf: &[u8]) -> Result<Image> {
     })
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;

sn0int-std/src/html.rs

@@ -1,11 +1,8 @@
+use crate::engine::structs::LuaMap;
 use crate::errors::*;
-
-use kuchiki;
+use crate::hlua::AnyLuaValue;
 use kuchiki::traits::TendrilSink;
 use std::collections::HashMap;
-use crate::hlua::AnyLuaValue;
-use crate::engine::structs::LuaMap;
-
 
 #[derive(Debug, PartialEq)]
 pub struct Element {
@@ -14,13 +11,13 @@ pub struct Element {
     pub html: String,
 }
 
-impl Into<AnyLuaValue> for Element {
-    fn into(self) -> AnyLuaValue {
+impl From<Element> for AnyLuaValue {
+    fn from(elem: Element) -> AnyLuaValue {
         let mut map = LuaMap::new();
 
-        map.insert_str("text", self.text);
-        map.insert("attrs", LuaMap::from(self.attrs));
-        map.insert_str("html", self.html);
+        map.insert_str("text", elem.text);
+        map.insert("attrs", LuaMap::from(elem.attrs));
+        map.insert_str("html", elem.html);
 
         map.into()
     }

sn0int-std/src/json.rs

@@ -1,6 +1,5 @@
 use crate::errors::*;
 
-use std::iter::FromIterator;
 use std::collections::HashMap;
 use crate::hlua::AnyLuaValue;
 use serde_json::{self, Deserializer, Value, Number, Map};
@@ -22,7 +21,7 @@ pub fn encode(v: AnyLuaValue) -> Result<String> {
 }
 
 pub fn decode_stream(x: &str) -> Result<Vec<AnyLuaValue>> {
-    let stream = Deserializer::from_str(&x).into_iter::<Value>();
+    let stream = Deserializer::from_str(x).into_iter::<Value>();
 
     let list = stream
         .filter_map(|x| x.ok())
@@ -35,11 +34,7 @@ pub fn decode_stream(x: &str) -> Result<Vec<AnyLuaValue>> {
 pub fn lua_array_is_list(array: &[(AnyLuaValue, AnyLuaValue)]) -> bool {
     if !array.is_empty() {
         let first = &array[0];
-        if let AnyLuaValue::LuaNumber(_) = first.0 {
-            true
-        } else {
-            false
-        }
+        matches!(first.0, AnyLuaValue::LuaNumber(_))
     } else {
         // true // TODO: this breaks unserialize
         false
@@ -56,9 +51,9 @@ pub enum LuaJsonValue {
     Object(HashMap<String, LuaJsonValue>),
 }
 
-impl Into<AnyLuaValue> for LuaJsonValue {
-    fn into(self) -> AnyLuaValue {
-        match self {
+impl From<LuaJsonValue> for AnyLuaValue {
+    fn from(value: LuaJsonValue) -> AnyLuaValue {
+        match value {
             LuaJsonValue::Null => AnyLuaValue::LuaNil,
             LuaJsonValue::Bool(v) => AnyLuaValue::LuaBoolean(v),
             // TODO: not sure if this might fail
@@ -115,9 +110,9 @@ impl From<AnyLuaValue> for LuaJsonValue {
     }
 }
 
-impl Into<serde_json::Value> for LuaJsonValue {
-    fn into(self) -> serde_json::Value {
-        match self {
+impl From<LuaJsonValue> for serde_json::Value {
+    fn from(value: LuaJsonValue) -> serde_json::Value {
+        match value {
             LuaJsonValue::Null => Value::Null,
             LuaJsonValue::Bool(v) => Value::Bool(v),
             LuaJsonValue::Number(v) => Value::Number(v),
@@ -126,9 +121,10 @@ impl Into<serde_json::Value> for LuaJsonValue {
                 .map(|x| x.into())
                 .collect()
             ),
-            LuaJsonValue::Object(v) => Value::Object(Map::from_iter(v.into_iter()
+            LuaJsonValue::Object(v) => Value::Object(v.into_iter()
                 .map(|(k, v)| (k, v.into()))
-            )),
+                .collect::<Map<_, _>>()
+            ),
         }
     }
 }
@@ -144,9 +140,10 @@ impl From<serde_json::Value> for LuaJsonValue {
                 .map(|x| x.into())
                 .collect()
             ),
-            Value::Object(v) => LuaJsonValue::Object(HashMap::from_iter(v.into_iter()
+            Value::Object(v) => LuaJsonValue::Object(v.into_iter()
                 .map(|(k, v)| (k, v.into()))
-            )),
+                .collect::<HashMap<_, _>>()
+            ),
         }
     }
 }

sn0int-std/src/lib.rs

@@ -1,11 +1,10 @@
-#[macro_use] extern crate serde_derive;
 use hlua_badtouch as hlua;
 
 pub mod blobs;
 pub mod crt;
 pub mod crypto;
-mod errors;
 pub mod engine;
+mod errors;
 pub mod geo;
 pub mod geoip;
 pub mod gfx;

sn0int-std/src/mqtt.rs

@@ -1,19 +1,26 @@
-use chrootable_https::DnsResolver;
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
-use mqtt::packet::VariablePacketError;
 use crate::json::LuaJsonValue;
-use crate::sockets::{Stream, SocketOptions};
-use mqtt::{TopicFilter, QualityOfService};
-use mqtt::control::ConnectReturnCode;
+use crate::sockets::{SocketOptions, Stream};
+use chrootable_https::DnsResolver;
 use mqtt::control::fixed_header::FixedHeaderError;
-use mqtt::encodable::{Encodable, Decodable};
-use mqtt::packet::{Packet, VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
+use mqtt::control::ConnectReturnCode;
+use mqtt::encodable::{Decodable, Encodable};
+use mqtt::packet::VariablePacketError;
+use mqtt::packet::{ConnectPacket, PingreqPacket, SubscribePacket, VariablePacket};
+use mqtt::{QualityOfService, TopicFilter};
+use serde::{Deserialize, Serialize};
 use std::convert::TryFrom;
 use std::io;
 use std::net::SocketAddr;
+use std::time::{Duration, Instant};
 use url::Url;
 
+// a reasonable default for keep-alive
+// some servers reject 0 as invalid with a very confusing error message
+const DEFAULT_PING_INTERVAL: u64 = 90;
+const DEFAULT_KEEP_ALIVE: u16 = 120;
+
 #[derive(Debug, Default, Deserialize)]
 pub struct MqttOptions {
     pub username: Option<String>,
@@ -22,10 +29,12 @@ pub struct MqttOptions {
     pub proxy: Option<SocketAddr>,
     #[serde(default)]
     pub connect_timeout: u64,
-    #[serde(default)]
-    pub read_timeout: u64,
+    pub read_timeout: Option<u64>,
     #[serde(default)]
     pub write_timeout: u64,
+
+    pub ping_interval: Option<u64>,
+    pub keep_alive: Option<u16>,
 }
 
 impl MqttOptions {
@@ -36,25 +45,44 @@ impl MqttOptions {
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum MqttRecvError {
+    #[error("Failed to read mqtt packet: {0:#}")]
+    Recv(#[from] VariablePacketError),
+    #[error("Failed to read mqtt packet: connection disconnected")]
+    RecvDisconnect,
+    #[error("Failed to interact with mqtt: {0:#}")]
+    Error(Error),
+}
+
+impl From<Error> for MqttRecvError {
+    fn from(err: Error) -> Self {
+        MqttRecvError::Error(err)
+    }
+}
+
 pub struct MqttClient {
     stream: Stream,
+    last_ping: Instant,
+    ping_interval: Option<u64>,
 }
 
 impl MqttClient {
     pub fn negotiate(stream: Stream, options: &MqttOptions) -> Result<MqttClient> {
+        // default to DEFAULT_PING_INTERVAL, if an explicit value of 0 was set, disable auto-ping
+        let ping_interval = Some(options.ping_interval.unwrap_or(DEFAULT_PING_INTERVAL));
+        ping_interval.filter(|s| *s != 0);
+
         let mut client = MqttClient {
             stream,
+            last_ping: Instant::now(),
+            ping_interval,
         };
 
-        let mut pkt = ConnectPacket::new("MQTT", "sn0int");
+        let mut pkt = ConnectPacket::new("sn0int");
         pkt.set_user_name(options.username.clone());
         pkt.set_password(options.password.clone());
-
-        /*
-        if let Some(keep_alive) = msg.keep_alive {
-            packet.set_keep_alive(keep_alive);
-        }
-        */
+        pkt.set_keep_alive(options.keep_alive.unwrap_or(DEFAULT_KEEP_ALIVE));
 
         client.send(pkt.into())?;
         let pkt = client.recv()?;
@@ -71,14 +99,19 @@ impl MqttClient {
         }
     }
 
-    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &MqttOptions) -> Result<MqttClient> {
+    pub fn connect<R: DnsResolver>(
+        resolver: &R,
+        url: Url,
+        options: &MqttOptions,
+    ) -> Result<MqttClient> {
         let tls = match url.scheme() {
             "mqtt" => false,
             "mqtts" => true,
             _ => bail!("Invalid mqtt protocol"),
         };
 
-        let host = url.host_str()
+        let host = url
+            .host_str()
             .ok_or_else(|| format_err!("Missing host in url"))?;
 
         let port = match (url.port(), tls) {
@@ -87,29 +120,57 @@ impl MqttClient {
             (None, false) => 1883,
         };
 
+        // if no read timeout is configured then keep alive won't work
+        let read_timeout = options.read_timeout.unwrap_or(DEFAULT_PING_INTERVAL);
 
-        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
-            tls,
-            sni_value: None,
-            disable_tls_verify: false,
-            proxy: options.proxy,
+        let stream = Stream::connect_stream(
+            resolver,
+            host,
+            port,
+            &SocketOptions {
+                tls,
+                sni_value: None,
+                disable_tls_verify: false,
+                proxy: options.proxy,
 
-            connect_timeout: options.connect_timeout,
-            read_timeout: options.read_timeout,
-            write_timeout: options.write_timeout,
-        })?;
+                connect_timeout: options.connect_timeout,
+                read_timeout,
+                write_timeout: options.write_timeout,
+            },
+        )?;
 
         Self::negotiate(stream, options)
     }
 
+    fn maintain_ping(&mut self) -> Result<()> {
+        if let Some(ping_interval) = self.ping_interval {
+            if self.last_ping.elapsed() >= Duration::from_secs(ping_interval) {
+                self.ping().context("Failed to ping")?;
+                self.last_ping = Instant::now();
+            }
+        }
+        Ok(())
+    }
+
     fn send(&mut self, pkt: VariablePacket) -> Result<()> {
+        self.maintain_ping()?;
         debug!("Sending mqtt packet: {:?}", pkt);
         pkt.encode(&mut self.stream)?;
         Ok(())
     }
 
-    fn recv(&mut self) -> std::result::Result<VariablePacket, VariablePacketError> {
-        let pkt = VariablePacket::decode(&mut self.stream)?;
+    fn recv(&mut self) -> std::result::Result<VariablePacket, MqttRecvError> {
+        self.maintain_ping()?;
+        let pkt = VariablePacket::decode(&mut self.stream).map_err(|err| match err {
+            // search for any io error and check if it's ErrorKind::UnexpectedEof
+            VariablePacketError::IoError(err)
+            | VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))
+                if err.kind() == io::ErrorKind::UnexpectedEof =>
+            {
+                MqttRecvError::RecvDisconnect
+            }
+            _ => MqttRecvError::Recv(err),
+        })?;
         debug!("Received mqtt packet: {:?}", pkt);
         Ok(pkt)
     }
@@ -138,30 +199,35 @@ impl MqttClient {
     pub fn recv_pkt(&mut self) -> Result<Option<Pkt>> {
         match self.recv() {
             Ok(pkt) => Ok(Some(Pkt::try_from(pkt)?)),
-            Err(VariablePacketError::IoError(err)) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
-            Err(VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
-            Err(err) => Err(Error::from(err))
+            // search for any io error and check if it's ErrorKind::WouldBlock
+            Err(MqttRecvError::Recv(
+                VariablePacketError::IoError(err)
+                | VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err)),
+            )) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(err) => Err(err.into()),
         }
     }
 
     pub fn ping(&mut self) -> Result<()> {
         let pkt = PingreqPacket::new();
-        self.send(pkt.into())
+        let pkt = VariablePacket::PingreqPacket(pkt);
+        pkt.encode(&mut self.stream)?;
+        Ok(())
     }
 }
 
 #[derive(Serialize, Deserialize)]
 #[serde(tag = "type")]
 pub enum Pkt {
-    #[serde(rename="publish")]
+    #[serde(rename = "publish")]
     Publish(Publish),
-    #[serde(rename="pong")]
+    #[serde(rename = "pong")]
     Pong,
 }
 
 impl Pkt {
     pub fn to_lua(&self) -> Result<AnyLuaValue> {
-        let v = serde_json::to_value(&self)?;
+        let v = serde_json::to_value(self)?;
         let v = LuaJsonValue::from(v).into();
         Ok(v)
     }
@@ -176,7 +242,7 @@ impl TryFrom<VariablePacket> for Pkt {
             VariablePacket::ConnackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
             VariablePacket::PublishPacket(pkt) => Ok(Pkt::Publish(Publish {
                 topic: pkt.topic_name().to_string(),
-                body: pkt.payload(),
+                body: pkt.payload().to_vec(),
             })),
             VariablePacket::PubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
             VariablePacket::PubrecPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
@@ -205,7 +271,7 @@ mod tests {
     use chrootable_https::dns::Resolver;
 
     fn connect() -> Result<MqttClient> {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let url = "mqtt://mqtt.winkekatze24.de".parse()?;
         MqttClient::connect(&resolver, url, &MqttOptions::default())
     }

sn0int-std/src/psl.rs

@@ -1,11 +1,12 @@
-use crate::errors::*;
-
 use chrootable_https::Client;
+use crate::errors::*;
+use crate::lazy::LazyInit;
+use publicsuffix::{List, Psl as _};
 use std::fs::{self, File};
+use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use crate::lazy::LazyInit;
-
+use std::str::FromStr;
 
 #[derive(Debug, PartialEq)]
 pub struct DnsName {
@@ -60,7 +61,7 @@ impl PslReader {
     }
 
     pub fn download(path: &Path, url: &str) -> Result<()> {
-        let client = Client::with_system_resolver()?;
+        let client = Client::with_system_resolver_v4()?;
         let resp = client.get(url)
             .wait_for_response()
             .context("http request failed")?;
@@ -72,11 +73,15 @@ impl PslReader {
 impl LazyInit<Arc<Psl>> for PslReader {
     fn initialize(self) -> Result<Arc<Psl>> {
         let list = match self {
-            PslReader::Reader(file) => publicsuffix::List::from_reader(file),
-            PslReader::String(s) => publicsuffix::List::from_str(&s),
+            PslReader::Reader(mut file) => {
+                let mut buf = String::new();
+                file.read_to_string(&mut buf)?;
+                buf
+            },
+            PslReader::String(s) => s,
         };
 
-        let list = list
+        let list = List::from_str(&list)
             .map_err(|e| format_err!("Failed to load public suffix list: {}", e))?;
 
         Ok(Arc::new(Psl {
@@ -87,32 +92,28 @@ impl LazyInit<Arc<Psl>> for PslReader {
 
 #[derive(Debug)]
 pub struct Psl {
-    list: publicsuffix::List,
+    list: List,
 }
 
 impl Psl {
     pub fn parse_dns_name(&self, name: &str) -> Result<DnsName> {
-        let dns_name = self.list.parse_dns_name(name)
-            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
+        let bytes = name.as_bytes();
 
-        let (root, suffix) = if let Some(domain) = dns_name.domain() {
-            let root = domain.to_string();
-            if let Some(suffix) = domain.suffix() {
-                (root, suffix.to_string())
-            } else {
-                // XXX: not sure if this is reachable
-                (root.clone(), root)
-            }
+        let suffix = self.list.suffix(bytes)
+            .ok_or_else(|| format_err!("Failed to detect suffix"))?;
+        let suffix = String::from_utf8(suffix.as_bytes().to_vec())?;
+
+        let root = if let Some(root) = self.list.domain(bytes) {
+            String::from_utf8(root.as_bytes().to_vec())?
         } else {
             // this is technically a tld, but support eg. a.prod.fastly.net anyway
-            // XXX: consider forcing domain to be in self.list.private()
-            (name.to_string(), name.to_string())
+            name.to_string()
         };
 
-        let fulldomain = if root.as_str() != name {
-            Some(name.to_string())
-        } else {
+        let fulldomain = if root == name {
             None
+        } else {
+            Some(name.to_string())
         };
 
         Ok(DnsName {

sn0int-std/src/ratelimits.rs

@@ -1,4 +1,5 @@
 use chrono::prelude::*;
+use serde::{Serialize, Deserialize};
 use std::collections::HashMap;
 use std::result;
 use std::sync::mpsc;
@@ -10,6 +11,12 @@ pub struct Ratelimiter {
     buckets: HashMap<String, Bucket>,
 }
 
+impl Default for Ratelimiter {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl Ratelimiter {
     pub fn new() -> Ratelimiter {
         Ratelimiter {

sn0int-std/src/sockets/mod.rs

@@ -6,6 +6,7 @@ use crate::json::LuaJsonValue;
 use chrootable_https::dns::{DnsResolver, RecordType};
 use chrootable_https::socks5::{self, ProxyDest};
 use regex::Regex;
+use serde::Deserialize;
 use tokio::runtime::Runtime;
 
 use std::fmt;
@@ -91,7 +92,7 @@ impl SocketOptions {
 
 pub enum Stream {
     Tcp(TcpStream),
-    Tls(rustls::StreamOwned<rustls::ClientSession, TcpStream>),
+    Tls(Box<rustls::StreamOwned<rustls::ClientSession, TcpStream>>),
 }
 
 impl Stream {
@@ -106,7 +107,7 @@ impl Stream {
         let mut errors = Vec::new();
 
         for addr in addrs {
-            match Stream::connect_addr(host, (addr, port).into(), &options) {
+            match Stream::connect_addr(host, (addr, port).into(), options) {
                 Ok(socket) => {
                     return Ok(socket);
                 },
@@ -128,7 +129,7 @@ impl Stream {
         let socket = if connect_timeout > 0 {
             TcpStream::connect_timeout(&addr, Duration::from_secs(connect_timeout))?
         } else {
-            TcpStream::connect(&addr)?
+            TcpStream::connect(addr)?
         };
         debug!("successfully connected to {:?}", addr);
 
@@ -137,7 +138,7 @@ impl Stream {
         tls::wrap_if_enabled(socket, host, options)
     }
 
-    pub fn connect_socks5_stream(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+    pub fn connect_socks5_stream(proxy: SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
         debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
 
         let addr = match host.parse::<Ipv4Addr>() {
@@ -145,7 +146,7 @@ impl Stream {
             _ => ProxyDest::Domain(host.to_string()),
         };
 
-        let fut = socks5::connect(proxy, addr, port);
+        let fut = socks5::connect(&proxy, addr, port);
 
         let mut rt = Runtime::new()?;
         let socket = rt.block_on(fut)?;
@@ -210,7 +211,7 @@ impl Socket {
         Ok(Socket::new(stream))
     }
 
-    pub fn connect_socks5(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
+    pub fn connect_socks5(proxy: SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
         let stream = Stream::connect_socks5_stream(proxy, host, port, options)?;
         Ok(Socket::new(stream))
     }
@@ -232,7 +233,7 @@ impl Socket {
     }
 
     pub fn send(&mut self, data: &[u8]) -> Result<()> {
-        match str::from_utf8(&data) {
+        match str::from_utf8(data) {
             Ok(data) => debug!("send: {:?}", data),
             Err(_) => debug!("send: {:?}", data),
         };
@@ -364,7 +365,7 @@ mod tests {
     #[ignore]
     #[test]
     fn verify_tls_good() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let _sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -374,7 +375,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_good_request() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let mut sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -387,7 +388,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_expired() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let sock = Socket::connect(&resolver, "expired.badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -401,7 +402,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_1_1_1_1() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let _sock = Socket::connect(&resolver, "1.1.1.1", 443, &SocketOptions{
             tls: true,
             ..Default::default()

sn0int-std/src/sockets/tls.rs

@@ -1,17 +1,14 @@
 use crate::errors::*;
-
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
 use rustls::{self, ClientConfig, Session, ClientSession};
-
+use serde::Serialize;
 use std::str;
 use std::result;
 use std::sync::Arc;
 use std::net::TcpStream;
-
 use super::{Stream, SocketOptions};
 
-
 #[derive(Debug, Serialize)]
 pub struct TlsData {
     cert: Option<String>,
@@ -20,7 +17,7 @@ pub struct TlsData {
 
 impl TlsData {
     pub fn to_lua(&self) -> Result<AnyLuaValue> {
-        let v = serde_json::to_value(&self)?;
+        let v = serde_json::to_value(self)?;
         let v = LuaJsonValue::from(v).into();
         Ok(v)
     }
@@ -50,9 +47,9 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
     }
 
     let dns_name = if let Some(v) = &options.sni_value {
-        get_dns_name(&mut config, &v)
+        get_dns_name(&mut config, v)
     } else {
-        get_dns_name(&mut config, &host)
+        get_dns_name(&mut config, host)
     };
 
     let config = Arc::new(config);
@@ -61,7 +58,7 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
 }
 
 fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
-    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(&host) {
+    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(host) {
         debug!("setting sni value to: {:?}", host);
         name.to_owned()
     } else {
@@ -107,7 +104,7 @@ fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, T
 
     info!("successfully established tls connection");
     let stream = rustls::StreamOwned::new(session, stream);
-    let stream = Stream::Tls(stream);
+    let stream = Stream::Tls(Box::new(stream));
     Ok((stream, tls))
 }
 

sn0int-std/src/web.rs

@@ -1,25 +1,26 @@
-pub use chrootable_https::{Client, HttpClient, Resolver, Response};
-
 use crate::blobs::{Blob, BlobState};
-use std::collections::{HashMap, HashSet};
-use std::time::Duration;
-use std::ops::Deref;
+use crate::engine::structs::LuaMap;
 use crate::errors::*;
+use crate::json::LuaJsonValue;
 use crate::hlua::AnyLuaValue;
-use serde_json;
+use chrootable_https::{Request, Body, Uri};
+pub use chrootable_https::{Client, HttpClient, Resolver, Response};
+use chrootable_https::http::HttpTryFrom;
+use chrootable_https::http::uri::Parts;
+use chrootable_https::http::request::Builder;
+use data_encoding::BASE64;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
+use serde::{Serialize, Deserialize};
+use std::collections::{HashMap, HashSet};
 use std::fmt;
+use std::fmt::Write;
+use std::iter;
 use std::net::SocketAddr;
+use std::ops::Deref;
 use std::sync::Arc;
-use serde::Serialize;
-use crate::engine::structs::LuaMap;
-use crate::json::LuaJsonValue;
-use chrootable_https::http::uri::Parts;
-use chrootable_https::{Request, Body, Uri};
-use serde_urlencoded;
-use base64;
-
+use std::time::Duration;
+use url::Url;
 
 pub fn url_set_qs<S: Serialize + fmt::Debug>(url: Uri, query: &S) -> Result<Uri> {
     let mut parts = Parts::from(url);
@@ -51,7 +52,12 @@ pub struct HttpSession {
 
 impl HttpSession {
     pub fn new() -> (String, HttpSession) {
-        let id: String = thread_rng().sample_iter(&Alphanumeric).take(16).collect();
+        let mut rng = thread_rng();
+        let id: String = iter::repeat(())
+            .map(|()| rng.sample(Alphanumeric))
+            .map(char::from)
+            .take(16)
+            .collect();
         (id.clone(), HttpSession {
             id,
             cookies: CookieJar::default(),
@@ -61,19 +67,21 @@ impl HttpSession {
 
 #[derive(Debug, Default, Deserialize)]
 pub struct RequestOptions {
-    query: Option<HashMap<String, String>>,
-    headers: Option<HashMap<String, String>>,
-    basic_auth: Option<(String, String)>,
-    user_agent: Option<String>,
-    json: Option<serde_json::Value>,
-    form: Option<serde_json::Value>,
-    body: Option<String>,
-    timeout: Option<u64>,
+    pub query: Option<HashMap<String, String>>,
+    pub headers: Option<HashMap<String, String>>,
+    pub basic_auth: Option<(String, String)>,
+    pub user_agent: Option<String>,
+    pub json: Option<serde_json::Value>,
+    pub form: Option<serde_json::Value>,
     #[serde(default)]
-    into_blob: bool,
-    proxy: Option<SocketAddr>,
+    pub follow_redirects: usize,
+    pub body: Option<String>,
+    pub timeout: Option<u64>,
     #[serde(default)]
-    binary: bool,
+    pub into_blob: bool,
+    pub proxy: Option<SocketAddr>,
+    #[serde(default)]
+    pub binary: bool,
 }
 
 impl RequestOptions {
@@ -84,10 +92,6 @@ impl RequestOptions {
     }
 }
 
-pub fn default_user_agent() -> String {
-    format!("sn0int/{}", env!("CARGO_PKG_VERSION"))
-}
-
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct HttpRequest {
     // reference to the HttpSession
@@ -98,7 +102,8 @@ pub struct HttpRequest {
     query: Option<HashMap<String, String>>,
     headers: Option<HashMap<String, String>>,
     basic_auth: Option<(String, String)>,
-    user_agent: Option<String>,
+    user_agent: String,
+    follow_redirects: usize,
     body: Option<ReqBody>,
     timeout: Option<Duration>,
     into_blob: bool,
@@ -107,10 +112,8 @@ pub struct HttpRequest {
 }
 
 impl HttpRequest {
-    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions) -> HttpRequest {
+    pub fn new(session: &HttpSession, method: String, url: String, user_agent: String, options: RequestOptions) -> HttpRequest {
         let cookies = session.cookies.clone();
-
-        let user_agent = options.user_agent.or_else(|| Some(default_user_agent()));
         let timeout = options.timeout.map(Duration::from_millis);
 
         let mut request = HttpRequest {
@@ -122,6 +125,7 @@ impl HttpRequest {
             headers: options.headers,
             basic_auth: options.basic_auth,
             user_agent,
+            follow_redirects: options.follow_redirects,
             body: None,
             timeout,
             into_blob: options.into_blob,
@@ -144,7 +148,7 @@ impl HttpRequest {
         request
     }
 
-    pub fn send(&self, state: &dyn WebState) -> Result<Response> {
+    pub fn send(&mut self, state: &dyn WebState) -> Result<Response> {
         let mut url = self.url.parse::<Uri>()?;
 
         // set query string
@@ -153,47 +157,19 @@ impl HttpRequest {
         }
 
         // start setting up request
-        let mut req = Request::builder();
-        req.method(self.method.as_str());
-        req.uri(url);
-
-        let mut observed_headers = HashSet::new();
-
-        // set cookies
-        {
-            use chrootable_https::header::COOKIE;
-            let mut cookies = String::new();
-
-            for (key, value) in self.cookies.iter() {
-                if !cookies.is_empty() {
-                    cookies += "; ";
-                }
-                cookies.push_str(&format!("{}={}", key, value));
-            }
-
-            if !cookies.is_empty() {
-                req.header(COOKIE, cookies.as_str());
-                observed_headers.insert(COOKIE.as_str().to_lowercase());
-            }
-        }
+        let mut req = self.mkrequest(self.method.as_str(), &url);
 
         // add headers
-        if let Some(ref agent) = self.user_agent {
-            use chrootable_https::header::USER_AGENT;
-            req.header(USER_AGENT, agent.as_str());
-            observed_headers.insert(USER_AGENT.as_str().to_lowercase());
-        }
-
         if let Some(ref auth) = self.basic_auth {
             use chrootable_https::header::AUTHORIZATION;
-            let &(ref user, ref password) = auth;
+            let (user, password) = auth;
 
-            let auth = base64::encode(&format!("{}:{}", user, password));
+            let auth = BASE64.encode(format!("{}:{}", user, password).as_bytes());
             let auth = format!("Basic {}", auth);
             req.header(AUTHORIZATION, auth.as_str());
-            observed_headers.insert(AUTHORIZATION.as_str().to_lowercase());
         }
 
+        let mut observed_headers = HashSet::new();
         if let Some(ref headers) = self.headers {
             for (k, v) in headers {
                 req.header(k.as_str(), v.as_str());
@@ -220,24 +196,72 @@ impl HttpRequest {
             },
             None => Body::empty(),
         };
-        let req = req.body(body)?;
+        let mut req = req.body(body)?;
 
         debug!("Getting http client");
         let client = state.http(&self.proxy)?;
 
-        // send request
-        debug!("Sending http request: {:?}", req);
-        let res = client.request(req)
-            .with_timeout(self.timeout)
-            .wait_for_response()?;
+        let res = loop {
+            // send request
+            debug!("Sending http request: {:?}", req);
+            let res = client.request(req)
+                .with_timeout(self.timeout)
+                .wait_for_response()?;
 
-        for cookie in &res.cookies {
-            HttpRequest::register_cookies_on_state(&self.session, state, cookie);
-        }
+            for cookie in &res.cookies {
+                HttpRequest::register_cookies_on_state(&self.session, state, cookie);
+            }
+
+            if self.follow_redirects > 0 && res.status >= 300 && res.status < 400 {
+                if let Some(location) = res.headers.get("location") {
+                    let base = Url::parse(&url.to_string())?;
+                    let joined = base.join(location)?;
+                    url = joined.to_string().parse()?;
+
+                    req = self.mkrequest("GET", &url).body(Body::empty())?;
+                    self.follow_redirects -= 1;
+                    continue;
+                }
+            }
+
+            break res;
+        };
 
         Ok(res)
     }
 
+    /// create a basic request, reusable when following redirects
+    fn mkrequest<T>(&self, method: &str, url: T) -> Builder
+        where Uri: HttpTryFrom<T>,
+    {
+        let mut req = Request::builder();
+        req.method(method);
+        req.uri(url);
+        self.attach_cookies(&mut req);
+
+        use chrootable_https::header::USER_AGENT;
+        req.header(USER_AGENT, self.user_agent.as_str());
+
+        req
+    }
+
+    fn attach_cookies(&self, req: &mut Builder) {
+        use chrootable_https::header::COOKIE;
+        let mut cookies = String::new();
+
+        for (key, value) in self.cookies.iter() {
+            if !cookies.is_empty() {
+                cookies += "; ";
+            }
+            // it's a write to a String, so panic if re-allocation fails is fine
+            write!(cookies, "{}={}", key, value).expect("out of memory");
+        }
+
+        if !cookies.is_empty() {
+            req.header(COOKIE, cookies.as_str());
+        }
+    }
+
     pub fn response_to_lua<S>(&self, state: &S, res: Response) -> Result<LuaMap>
         where S: WebState + BlobState
     {
@@ -294,9 +318,9 @@ impl HttpRequest {
     }
 }
 
-impl Into<AnyLuaValue> for HttpRequest {
-    fn into(self) -> AnyLuaValue {
-        let v = serde_json::to_value(&self).unwrap();
+impl From<HttpRequest> for AnyLuaValue {
+    fn from(req: HttpRequest) -> AnyLuaValue {
+        let v = serde_json::to_value(req).unwrap();
         LuaJsonValue::from(v).into()
     }
 }

sn0int-std/src/websockets.rs

@@ -4,6 +4,7 @@ use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
 use crate::sockets::{Stream, SocketOptions};
 use http::Request;
+use serde::Deserialize;
 use std::collections::HashMap;
 use std::net::SocketAddr;
 use std::io;
@@ -90,9 +91,11 @@ impl WebSocket {
     }
 
     pub fn options(&self, options: &WebSocketOptions) -> Result<()> {
-        let mut o = SocketOptions::default();
-        o.read_timeout = options.read_timeout;
-        o.write_timeout = options.write_timeout;
+        let o = SocketOptions {
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+            ..Default::default()
+        };
         o.apply(self.sock.get_ref())
     }
 

sn0int-std/src/xml.rs

@@ -2,6 +2,7 @@ use crate::errors::*;
 
 use crate::json::LuaJsonValue;
 use crate::hlua::AnyLuaValue;
+use serde::Serialize;
 use std::collections::HashMap;
 use xml::attribute::OwnedAttribute;
 use xml::name::OwnedName;
@@ -13,6 +14,12 @@ pub struct XmlDocument {
     pub children: Vec<XmlElement>,
 }
 
+impl Default for XmlDocument {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl XmlDocument {
     #[inline(always)]
     pub fn new() -> XmlDocument {
@@ -55,7 +62,7 @@ pub fn decode(x: &str) -> Result<AnyLuaValue> {
 }
 
 #[inline]
-fn append_text(stack: &mut Vec<XmlElement>, text: String) {
+fn append_text(stack: &mut [XmlElement], text: String) {
     if let Some(tail) = stack.last_mut() {
         if let Some(prev) = tail.text.as_mut() {
             prev.push_str(&text);

src/api.rs

@@ -1,35 +1,42 @@
 use crate::errors::*;
-use std::fmt;
+use crate::web;
 use crate::config::Config;
+use crate::utils;
 use chrootable_https::{self, HttpClient, Body, Request, Uri};
 use chrootable_https::http::request::Builder as RequestBuilder;
 use chrootable_https::header::CONTENT_TYPE;
-use rand::{Rng, thread_rng};
-use rand::distributions::Alphanumeric;
+use maplit::hashmap;
 use serde::de::DeserializeOwned;
 use serde::ser::Serialize;
-use serde_json;
 use sn0int_common::api::*;
 use sn0int_common::{ModuleID, ApiResponse};
-use crate::web;
-
+use std::fmt;
 
 pub struct Client {
     server: String,
     client: chrootable_https::Client<chrootable_https::Resolver>,
     session: Option<String>,
+    os_version: String,
 }
 
 impl Client {
     pub fn new(config: &Config) -> Result<Client> {
-        let client = match config.network.proxy {
-            Some(proxy) => chrootable_https::Client::with_socks5(proxy),
-            _ => chrootable_https::Client::with_system_resolver()?,
+        let client = if let Some(proxy) = config.network.proxy {
+            chrootable_https::Client::with_socks5(proxy)
+        } else {
+            chrootable_https::Client::with_system_resolver_v4()?
         };
+
+        let os_version = match os_version::detect() {
+            Ok(v) => v.to_string(),
+            Err(_) => String::from("unknown"),
+        };
+
         Ok(Client {
             server: config.core.registry.clone(),
             client,
             session: None,
+            os_version,
         })
     }
 
@@ -37,8 +44,18 @@ impl Client {
         self.session = Some(session.into());
     }
 
+    #[inline]
     pub fn random_session() -> String {
-        thread_rng().sample_iter(&Alphanumeric).take(32).collect()
+        utils::random_string(32)
+    }
+
+    #[inline]
+    fn user_agent(&self) -> String {
+        format!("sn0int/{}, {}, {}",
+            env!("CARGO_PKG_VERSION"),
+            embedded_triple::get(),
+            self.os_version,
+        )
     }
 
     pub fn request<T: DeserializeOwned + fmt::Debug>(&self, mut request: RequestBuilder, body: Body) -> Result<T> {
@@ -46,7 +63,7 @@ impl Client {
             info!("Adding session token to request");
             request.header("Auth", session.as_str());
         }
-        request.header("User-Agent", web::default_user_agent());
+        request.header("User-Agent", self.user_agent());
 
         let request = request.body(body)?;
 

src/args.rs

@@ -1,15 +1,15 @@
-use structopt::clap::{AppSettings, Shell};
-use sn0int_common::ModuleID;
 use crate::cmd;
 use crate::options;
 use crate::workspaces::Workspace;
-
+use structopt::StructOpt;
+use structopt::clap::{AppSettings, Shell};
+use sn0int_common::ModuleID;
 
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
     /// Select a different workspace instead of the default
-    #[structopt(short="w", long="workspace")]
+    #[structopt(short="w", long="workspace", env="SN0INT_WORKSPACE")]
     pub workspace: Option<Workspace>,
 
     #[structopt(subcommand)]
@@ -18,10 +18,7 @@ pub struct Args {
 
 impl Args {
     pub fn is_sandbox(&self) -> bool {
-        match self.subcommand {
-            Some(SubCommand::Sandbox(_)) => true,
-            _ => false,
-        }
+        matches!(self.subcommand, Some(SubCommand::Sandbox(_)))
     }
 }
 
@@ -69,15 +66,30 @@ pub enum SubCommand {
     /// Exclude entities from scope
     #[structopt(name="noscope")]
     Noscope(cmd::noscope_cmd::Args),
+    /// Manage autoscope rules
+    Autoscope(cmd::autoscope_cmd::Args),
+    /// Manage autonoscope rules
+    Autonoscope(cmd::autonoscope_cmd::Args),
+    /// Rescope all entities based on autonoscope rules
+    Rescope(cmd::rescope_cmd::Args),
     /// Manage workspaces
     #[structopt(name="workspace")]
     Workspace(cmd::workspace_cmd::Args),
+    /// Calendar
+    #[structopt(name="cal")]
+    Cal(cmd::cal_cmd::Args),
+    /// Notify
+    #[structopt(name="notify")]
+    Notify(cmd::notify_cmd::Args),
     /// Verify blob storage for corrupt and dangling blobs
     #[structopt(name="fsck")]
     Fsck(cmd::fsck_cmd::Args),
     /// Export a workspace for external processing
     #[structopt(name="export")]
     Export(cmd::export_cmd::Args),
+    /// Show statistics about your current workspace
+    #[structopt(name="stats")]
+    Stats(cmd::stats_cmd::Args),
     /// Run a lua repl
     #[structopt(name="repl")]
     Repl,
@@ -91,18 +103,11 @@ pub enum SubCommand {
 
 #[derive(Debug, StructOpt)]
 pub struct Run {
-    /// Execute a module that has been installed
-    pub module: String,
+    #[structopt(flatten)]
+    pub run: cmd::run_cmd::Args,
     /// Run a module from a path
     #[structopt(short="f", long="file")]
     pub file: bool,
-    /// Run modules concurrently
-    #[structopt(short="j", long="threads", default_value="1")]
-    pub threads: usize,
-    /// Verbose logging, once to print inserts even if they don't add new
-    /// data, twice to activate the debug() function
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    pub verbose: u64,
     /// Expose stdin to modules
     #[structopt(long="stdin")]
     pub stdin: bool,
@@ -132,7 +137,7 @@ pub struct Run {
 #[derive(Debug, StructOpt)]
 pub struct Sandbox {
     /// This value is only used for process listings
-    label: String,
+    _label: String,
 }
 
 #[derive(Debug, StructOpt)]

src/auth.rs

@@ -1,5 +1,4 @@
 use crate::errors::*;
-use opener;
 use std::fs;
 use std::path::PathBuf;
 use std::thread;

src/autonoscope/domain.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 
@@ -41,6 +41,13 @@ impl TryFrom<Autonoscope> for DomainRule {
     }
 }
 
+impl AutoRule<Domain> for DomainRule {
+    #[inline]
+    fn matches(&self, domain: &Domain) -> Result<bool> {
+        self.matches(domain.value.as_str())
+    }
+}
+
 impl AutoRule<NewDomain> for DomainRule {
     #[inline]
     fn matches(&self, domain: &NewDomain) -> Result<bool> {
@@ -48,6 +55,13 @@ impl AutoRule<NewDomain> for DomainRule {
     }
 }
 
+impl AutoRule<Subdomain> for DomainRule {
+    #[inline]
+    fn matches(&self, domain: &Subdomain) -> Result<bool> {
+        self.matches(domain.value.as_str())
+    }
+}
+
 impl AutoRule<NewSubdomain> for DomainRule {
     #[inline]
     fn matches(&self, domain: &NewSubdomain) -> Result<bool> {
@@ -55,6 +69,18 @@ impl AutoRule<NewSubdomain> for DomainRule {
     }
 }
 
+impl AutoRule<Url> for DomainRule {
+    #[inline]
+    fn matches(&self, url: &Url) -> Result<bool> {
+        let url = url.value.parse::<url::Url>()?;
+        if let Some(domain) = url.domain() {
+            self.matches(domain)
+        } else {
+            Ok(false)
+        }
+    }
+}
+
 impl AutoRule<NewUrl> for DomainRule {
     #[inline]
     fn matches(&self, url: &NewUrl) -> Result<bool> {
@@ -94,8 +120,8 @@ impl RulePrecision for DomainRule {
     }
 }
 
-impl IntoRule for DomainRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for DomainRule {
+    fn to_rule(&self) -> (&'static str, String) {
         ("domain", self.to_string())
     }
 }

src/autonoscope/ip.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 use std::net;
@@ -36,12 +36,25 @@ impl TryFrom<Autonoscope> for IpRule {
     }
 }
 
+impl AutoRule<IpAddr> for IpRule {
+    fn matches(&self, ipaddr: &IpAddr) -> Result<bool> {
+        self.matches(ipaddr.value.as_str())
+    }
+}
+
 impl AutoRule<NewIpAddr> for IpRule {
     fn matches(&self, ipaddr: &NewIpAddr) -> Result<bool> {
         self.matches(ipaddr.value.as_str())
     }
 }
 
+impl AutoRule<Port> for IpRule {
+    fn matches(&self, port: &Port) -> Result<bool> {
+        let addr = port.value.parse::<net::SocketAddr>()?;
+        self.matches(&addr.ip())
+    }
+}
+
 impl AutoRule<NewPort> for IpRule {
     fn matches(&self, port: &NewPort) -> Result<bool> {
         let addr = port.value.parse::<net::SocketAddr>()?;
@@ -49,15 +62,25 @@ impl AutoRule<NewPort> for IpRule {
     }
 }
 
+fn match_netblock_str(network: &IpNetwork, netblock: &str) -> Result<bool> {
+    let range = netblock.parse::<ipnetwork::IpNetwork>()?;
+
+    if network.prefix() <= range.prefix() {
+        Ok(network.contains(range.ip()))
+    } else {
+        Ok(false)
+    }
+}
+
+impl AutoRule<Netblock> for IpRule {
+    fn matches(&self, netblock: &Netblock) -> Result<bool> {
+        match_netblock_str(&self.network, &netblock.value)
+    }
+}
+
 impl AutoRule<NewNetblock> for IpRule {
     fn matches(&self, netblock: &NewNetblock) -> Result<bool> {
-        let range = netblock.value.parse::<ipnetwork::IpNetwork>()?;
-
-        if self.network.prefix() <= range.prefix() {
-            Ok(self.network.contains(range.ip()))
-        } else {
-            Ok(false)
-        }
+        match_netblock_str(&self.network, &netblock.value)
     }
 }
 
@@ -70,7 +93,7 @@ impl AutoRule<str> for IpRule {
 
 impl AutoRule<net::IpAddr> for IpRule {
     fn matches(&self, ipaddr: &net::IpAddr) -> Result<bool> {
-        Ok(self.network.contains(ipaddr.clone()))
+        Ok(self.network.contains(*ipaddr))
     }
 }
 
@@ -80,8 +103,8 @@ impl RulePrecision for IpRule {
     }
 }
 
-impl IntoRule for IpRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for IpRule {
+    fn to_rule(&self) -> (&'static str, String) {
         ("ip", self.to_string())
     }
 }

src/autonoscope/mod.rs

@@ -1,9 +1,9 @@
 use crate::db::DatabaseSock;
 use crate::errors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::schema::*;
 use crate::models::*;
+use lazy_static::lazy_static;
 use std::cmp::Ordering;
 use std::convert::{TryInto, TryFrom};
 use std::str::FromStr;
@@ -47,6 +47,18 @@ fn sort_precision_desc<T: RulePrecision>(a: &T, b: &T) -> Ordering {
 }
 
 impl RuleSet {
+    pub fn domains(&self) -> &[Rule<DomainRule>] {
+        &self.domains
+    }
+
+    pub fn ips(&self) -> &[Rule<IpRule>] {
+        &self.ips
+    }
+
+    pub fn urls(&self) -> &[Rule<UrlRule>] {
+        &self.urls
+    }
+
     pub fn load(db: &DatabaseSock) -> Result<Self> {
         use crate::schema::autonoscope::dsl::*;
         let rules = autonoscope.load::<Autonoscope>(db)?;
@@ -112,15 +124,15 @@ impl RuleSet {
         match obj {
             RuleType::Domain => {
                 self.domains.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
             },
             RuleType::Ip => {
                 self.ips.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
             },
             RuleType::Url => {
                 self.urls.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
             },
         }
         Ok(())
@@ -143,10 +155,18 @@ impl RuleSet {
         rules
     }
 
+    pub fn is_empty(&self) -> bool {
+        self.domains.is_empty() && self.ips.is_empty() && self.urls.is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        self.domains.len() + self.ips.len() + self.urls.len()
+    }
+
     #[inline]
-    fn push_rules_display<T: IntoRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
+    fn push_rules_display<T: ToRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
         for rule in rules {
-            let (object, value) = rule.into_rule();
+            let (object, value) = rule.to_rule();
             output.push((object, value, rule.scoped));
         }
     }
@@ -159,10 +179,8 @@ impl RuleSet {
             Insert::Url(url) => {
                 if let Some(result) = Self::matches_any(&self.domains, url)? {
                     Some(result)
-                } else if let Some(result) = Self::matches_any(&self.urls, url)? {
-                    Some(result)
                 } else {
-                    None
+                    Self::matches_any(&self.urls, url)?
                 }
             },
             // Insert::Email(email) => unimplemented!(),
@@ -176,7 +194,7 @@ impl RuleSet {
 
     fn matches_any<T1, T2>(rules: &[Rule<T1>], object: &T2) -> Result<Option<bool>>
         where T1: AutoRule<T2>,
-            T1: IntoRule,
+            T1: ToRule,
     {
         for rule in rules {
             if rule.matches(object)? {
@@ -238,12 +256,12 @@ pub trait RulePrecision {
 }
 
 #[derive(Debug, PartialEq)]
-pub struct Rule<T: IntoRule> {
+pub struct Rule<T: ToRule> {
     rule: T,
-    scoped: bool,
+    pub scoped: bool,
 }
 
-impl<T: IntoRule> Rule<T> {
+impl<T: ToRule> Rule<T> {
     pub fn new(rule: T, scoped: bool) -> Rule<T> {
         Rule {
             rule,
@@ -254,7 +272,7 @@ impl<T: IntoRule> Rule<T> {
 
 // TODO: maybe drop this
 use std::ops::Deref;
-impl<T: IntoRule> Deref for Rule<T> {
+impl<T: ToRule> Deref for Rule<T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -262,23 +280,23 @@ impl<T: IntoRule> Deref for Rule<T> {
     }
 }
 
-impl<T: IntoRule + RulePrecision> RulePrecision for Rule<T> {
+impl<T: ToRule + RulePrecision> RulePrecision for Rule<T> {
     fn precision(&self) -> usize {
         self.rule.precision()
     }
 }
 
-pub trait IntoRule {
-    fn into_rule(&self) -> (&'static str, String);
+pub trait ToRule {
+    fn to_rule(&self) -> (&'static str, String);
 }
 
-impl<T: IntoRule> Into<NewAutonoscope> for &Rule<T> {
-    fn into(self) -> NewAutonoscope {
-        let (object, value) = self.rule.into_rule();
+impl<T: ToRule> From<&Rule<T>> for NewAutonoscope {
+    fn from(rule: &Rule<T>) -> NewAutonoscope {
+        let (object, value) = rule.rule.to_rule();
         NewAutonoscope {
             object: object.to_string(),
             value,
-            scoped: self.scoped,
+            scoped: rule.scoped,
         }
     }
 }

src/autonoscope/url.rs

@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 
@@ -47,6 +47,12 @@ impl TryFrom<&str> for UrlRule {
 }
 
 // TODO: there is no way to write a rule that matches all urls
+impl AutoRule<Url> for UrlRule {
+    fn matches(&self, url: &Url) -> Result<bool> {
+        self.matches(url.value.as_str())
+    }
+}
+
 impl AutoRule<NewUrl> for UrlRule {
     fn matches(&self, url: &NewUrl) -> Result<bool> {
         self.matches(url.value.as_str())
@@ -86,8 +92,8 @@ impl RulePrecision for UrlRule {
     }
 }
 
-impl IntoRule for UrlRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for UrlRule {
+    fn to_rule(&self) -> (&'static str, String) {
         ("url", self.to_string())
     }
 }

src/blobs.rs

@@ -93,6 +93,14 @@ impl BlobStorage {
         }
         Ok(blobs)
     }
+
+    pub fn stat(&self, id: &str) -> Result<u64> {
+        let path = self.join(id)?;
+        debug!("Stat-ing blob: {:?}", path);
+        let md = fs::metadata(path)
+            .context("Failed to stat blob")?;
+        Ok(md.len())
+    }
 }
 
 

src/cal/date.rs

@@ -0,0 +1,398 @@
+use crate::errors::*;
+
+use chrono::Utc;
+use chrono::prelude::*;
+use crate::cal::{ActivityGrade, DateArg};
+use crate::models::*;
+use std::collections::HashMap;
+use std::collections::VecDeque;
+use std::fmt::Write;
+
+const MONTH_LINES: i32 = 7;
+
+fn merge_months(ctx: &DateContext, months: &[DateSpec]) -> String {
+    let mut months = months.iter()
+        .map(|ds| {
+            let month = ds.to_term_string(ctx);
+            month.lines()
+                .map(String::from)
+                .collect::<VecDeque<_>>()
+        })
+        .collect::<Vec<_>>();
+
+    let mut out = String::new();
+    for i in 0..=MONTH_LINES {
+        let mut first = true;
+        for m in &mut months {
+            if !first {
+                out.push_str("   ");
+            }
+            if let Some(line) = m.pop_front() {
+                out.push_str(&line);
+            } else {
+                out.push_str(&" ".repeat(21));
+            }
+            first = false;
+        }
+        if i < MONTH_LINES {
+            out.push('\n');
+        }
+    }
+    out
+}
+
+fn chunk_months(ctx: &DateContext, months: &[DateSpec]) -> String {
+    months
+        .chunks(3)
+        .map(|m| merge_months(ctx, m))
+        .fold(String::new(), |a, b| {
+            if a.is_empty() {
+                a + &b
+            } else {
+                a + "\n" + &b
+            }
+        })
+}
+
+fn days_in_month(year: i32, month: u32) -> i64 {
+    let start = Utc.with_ymd_and_hms(year, month, 1, 0, 0, 0).single().expect("Datetime is not unique");
+    let end = if month == 12 {
+        Utc.with_ymd_and_hms(year + 1, 1, 1, 0, 0, 0).single().expect("Datetime is not unique")
+    } else {
+        Utc.with_ymd_and_hms(year, month + 1, 1, 0, 0, 0).single().expect("Datetime is not unique")
+    };
+    end.signed_duration_since(start).num_days()
+}
+
+fn setup_graph_map(events: &[Activity]) -> (HashMap<NaiveDate, u64>, u64) {
+    debug!("Found {} events in selected range", events.len());
+
+    let mut cur = None;
+    let mut ctr = 0;
+    let mut max = 0;
+
+    let mut map = HashMap::new();
+    for event in events {
+        let date = event.time.date();
+        if let Some(cur) = cur.as_mut() {
+            if date == *cur {
+                ctr += 1;
+            } else {
+                if ctr > max {
+                    max = ctr;
+                }
+                map.insert(*cur, ctr);
+                *cur = date;
+                ctr = 1;
+            }
+        } else {
+            cur = Some(date);
+            ctr = 1;
+        }
+    }
+
+    if ctr > 0 {
+        if let Some(cur) = cur.take() {
+            if ctr > max {
+                max = ctr;
+            }
+            map.insert(cur, ctr);
+        }
+    }
+
+    debug!("Maximum events per day is {}", max);
+
+    (map, max)
+}
+
+pub struct DateContext {
+    events: HashMap<NaiveDate, u64>,
+    max: u64,
+    today: NaiveDate,
+}
+
+impl DateContext {
+    pub fn new(events: &[Activity], today: NaiveDate) -> DateContext {
+        let (events, max) = setup_graph_map(events);
+        DateContext {
+            events,
+            max,
+            today,
+        }
+    }
+
+    #[inline]
+    fn is_today(&self, date: &NaiveDate) -> bool {
+        self.today == *date
+    }
+
+    #[inline]
+    fn is_future(&self, date: &NaiveDate) -> bool {
+        self.today < *date
+    }
+
+    fn activity_for_day(&self, date: &NaiveDate) -> ActivityGrade {
+        if let Some(events) = self.events.get(date) {
+            ActivityGrade::from_ratio(*events, self.max)
+        } else {
+            ActivityGrade::None
+        }
+    }
+}
+
+pub enum DateSpec {
+    Year(i32),
+    YearMonth((i32, u32)),
+    YearMonthContext((i32, u32, u32)),
+}
+
+impl DateSpec {
+    pub fn from_args(args: &[DateArg], context: Option<u32>) -> Result<DateSpec> {
+        if args.len() > 2 {
+            bail!("Too many datespec args");
+        }
+
+        let today = Utc::now();
+        let ds = match (args.get(0), args.get(1), context) {
+            (None, _, None) => DateSpec::YearMonth((today.year(), today.month())),
+            (None, _, Some(context)) => DateSpec::YearMonthContext((today.year(), today.month(), context)),
+
+            (Some(DateArg::Month(month)), None, None) => DateSpec::YearMonth((today.year(), *month)),
+            (Some(DateArg::Num(year)), None, None) => DateSpec::Year(*year),
+            (Some(DateArg::Month(month)), Some(DateArg::Num(year)), None) => DateSpec::YearMonth((*year, *month)),
+            (Some(DateArg::Num(year)), Some(DateArg::Month(month)), None) => DateSpec::YearMonth((*year, *month)),
+
+            (Some(DateArg::Month(month)), None, Some(context)) => DateSpec::YearMonthContext((today.year(), *month, context)),
+            (Some(DateArg::Month(month)), Some(DateArg::Num(year)), Some(context)) => DateSpec::YearMonthContext((*year, *month, context)),
+            (Some(DateArg::Num(year)), Some(DateArg::Month(month)), Some(context)) => DateSpec::YearMonthContext((*year, *month, context)),
+            _ => bail!("Combination of datespec args is invalid"),
+        };
+        Ok(ds)
+    }
+
+    pub fn start(&self) -> NaiveDate {
+        match self {
+            DateSpec::Year(year) => NaiveDate::from_ymd_opt(*year, 1, 1).expect("Invalid month/day"),
+            DateSpec::YearMonth((year, month)) => NaiveDate::from_ymd_opt(*year, *month, 1).expect("Invalid month/day"),
+            DateSpec::YearMonthContext((year, month, context)) => {
+                let mut year = *year - (*context / 12) as i32;
+                let context = context % 12;
+                let month = if context >= *month {
+                    year -= 1;
+                    12 - context + month
+                } else {
+                    month - context
+                };
+                NaiveDate::from_ymd_opt(year, month, 1).expect("Invalid month/day")
+            },
+        }
+    }
+
+    pub fn end(&self) -> NaiveDate {
+        match self {
+            DateSpec::Year(year) => NaiveDate::from_ymd_opt(year + 1, 1, 1).expect("Invalid month/day"),
+            DateSpec::YearMonth((year, month)) => {
+                let (year, month) = if *month == 12 {
+                    (*year + 1, 1)
+                } else {
+                    (*year, *month + 1)
+                };
+                NaiveDate::from_ymd_opt(year, month, 1).expect("Invalid month/day")
+            },
+            DateSpec::YearMonthContext((year, month, _context)) => {
+                let (year, month) = if *month == 12 {
+                    (*year + 1, 1)
+                } else {
+                    (*year, *month + 1)
+                };
+                NaiveDate::from_ymd_opt(year, month, 1).expect("Invalid month/day")
+            },
+        }
+    }
+
+    pub fn to_term_string(&self, ctx: &DateContext) -> String {
+        match self {
+            DateSpec::Year(year) => {
+                let months = (1..=12)
+                    .map(|month| DateSpec::YearMonth((*year, month)))
+                    .collect::<Vec<_>>();
+                chunk_months(ctx, &months)
+            },
+            DateSpec::YearMonth((year, month)) => {
+                let mut w = String::new();
+
+                let start = Utc.with_ymd_and_hms(*year, *month, 1, 0, 0, 0).single().expect("Datetime is not unique");
+                let days = days_in_month(*year, *month) as u32;
+
+                writeln!(w, "{:^21}", start.format("%B %Y")).expect("out of memory");
+                w.push_str(" Su Mo Tu We Th Fr Sa\n");
+
+                let mut cur_week_day = start.weekday();
+                let week_progress = cur_week_day.num_days_from_sunday() as usize;
+                w.push_str(&"   ".repeat(week_progress));
+
+                let mut week_written = week_progress * 3;
+                for cur_day in 1..=days {
+                    let date = NaiveDate::from_ymd_opt(*year, *month, cur_day).expect("Invalid month/day");
+
+                    if !ctx.is_future(&date) {
+                        let activity = ctx.activity_for_day(&date);
+                        w.push_str(activity.as_term_str());
+                    }
+
+                    if ctx.is_today(&date) {
+                        w.push_str("\x1b[1m#");
+                    } else {
+                        w.push(' ');
+                    }
+                    write!(w, "{:2}", cur_day).expect("out of memory");
+                    week_written += 3;
+                    w.push_str("\x1b[0m");
+
+                    // detect end of the week
+                    if cur_week_day == Weekday::Sat {
+                        if cur_day != days {
+                            w.push('\n');
+                        }
+                        week_written = 0;
+                    }
+
+                    cur_week_day = cur_week_day.succ();
+                }
+                if week_written != 0 {
+                    w.push_str(&" ".repeat(21 - week_written));
+                }
+
+                w
+            }
+            DateSpec::YearMonthContext((_year, _month, context)) => {
+                let start = self.start();
+                let mut year = start.year();
+                let mut month = start.month();
+
+                let mut months = Vec::new();
+
+                for _ in 0..=*context {
+                    months.push(DateSpec::YearMonth((year, month)));
+
+                    if month == 12 {
+                        year += 1;
+                        month = 1;
+                    } else {
+                        month += 1;
+                    }
+                }
+
+                chunk_months(ctx, &months)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn context() -> DateContext {
+        DateContext {
+            events: HashMap::new(),
+            max: 0,
+            today: NaiveDate::from_ymd_opt(2020, 5, 30).unwrap(),
+        }
+    }
+
+    #[test]
+    fn test_days_in_month_2020_05() {
+        let days = days_in_month(2020, 5);
+        assert_eq!(days, 31);
+    }
+
+    #[test]
+    fn test_days_in_month_2020_04() {
+        let days = days_in_month(2020, 4);
+        assert_eq!(days, 30);
+    }
+
+    #[test]
+    fn test_days_in_month_2020_02() {
+        let days = days_in_month(2020, 2);
+        assert_eq!(days, 29);
+    }
+
+    #[test]
+    fn small_max_activity_0() {
+        let events = HashMap::new();
+        let ctx = DateContext {
+            events,
+            max: 0,
+            today: NaiveDate::from_ymd_opt(2020, 6, 6).unwrap(),
+        };
+        let grade = ctx.activity_for_day(&NaiveDate::from_ymd_opt(2020, 6, 6).unwrap());
+        assert_eq!(grade, ActivityGrade::None);
+    }
+
+    #[test]
+    fn test_datespec_year_month() {
+        let ds = DateSpec::YearMonth((2020, 5));
+        let out = ds.to_term_string(&context());
+        assert_eq!(out, "      May 2020       
+ Su Mo Tu We Th Fr Sa
+               \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m\u{1b}[1m#30\u{1b}[0m
+ 31\u{1b}[0m                  ");
+    }
+
+    #[test]
+    fn test_datespec_year_month_ends_on_sat() {
+        let ds = DateSpec::YearMonth((2020, 10));
+        let out = ds.to_term_string(&context());
+        assert_eq!(out, "    October 2020     
+ Su Mo Tu We Th Fr Sa
+              1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m
+  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m
+ 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m
+ 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m
+ 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m");
+    }
+
+    #[test]
+    fn test_datespec_year() {
+        let ds = DateSpec::Year(2020);
+        let out = ds.to_term_string(&context());
+        assert_eq!(out, "    January 2020            February 2020            March 2020      
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+         \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m                     \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 30\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 31\u{1b}[0m      \u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 30\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 31\u{1b}[0m            
+                                                                     
+     April 2020               May 2020                June 2020      
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+         \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m                  \u{1b}[97m\u{1b}[48;5;238m  1\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  2\u{1b}[0m        1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m  3\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  4\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  5\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  6\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  7\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  8\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m  9\u{1b}[0m     7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 10\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 11\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 12\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 13\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 14\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 15\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 16\u{1b}[0m    14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m   \u{1b}[97m\u{1b}[48;5;238m 17\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 18\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 19\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 20\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 21\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 22\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 23\u{1b}[0m    21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m
+\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 30\u{1b}[0m         \u{1b}[97m\u{1b}[48;5;238m 24\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 25\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 26\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 27\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 28\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m 29\u{1b}[0m\u{1b}[97m\u{1b}[48;5;238m\u{1b}[1m#30\u{1b}[0m    28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m            
+                         31\u{1b}[0m                                          
+      July 2020              August 2020           September 2020    
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+           1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m                       1\u{1b}[0m           1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m
+  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m     2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m     6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m
+ 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m     9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m    13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m
+ 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m    16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m    20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m
+ 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m       23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m    27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m         
+                         30\u{1b}[0m 31\u{1b}[0m                                       
+    October 2020            November 2020           December 2020    
+ Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa    Su Mo Tu We Th Fr Sa
+              1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m     1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m           1\u{1b}[0m  2\u{1b}[0m  3\u{1b}[0m  4\u{1b}[0m  5\u{1b}[0m
+  4\u{1b}[0m  5\u{1b}[0m  6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m     8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m     6\u{1b}[0m  7\u{1b}[0m  8\u{1b}[0m  9\u{1b}[0m 10\u{1b}[0m 11\u{1b}[0m 12\u{1b}[0m
+ 11\u{1b}[0m 12\u{1b}[0m 13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m    15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m    13\u{1b}[0m 14\u{1b}[0m 15\u{1b}[0m 16\u{1b}[0m 17\u{1b}[0m 18\u{1b}[0m 19\u{1b}[0m
+ 18\u{1b}[0m 19\u{1b}[0m 20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m    22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m    20\u{1b}[0m 21\u{1b}[0m 22\u{1b}[0m 23\u{1b}[0m 24\u{1b}[0m 25\u{1b}[0m 26\u{1b}[0m
+ 25\u{1b}[0m 26\u{1b}[0m 27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m    29\u{1b}[0m 30\u{1b}[0m                   27\u{1b}[0m 28\u{1b}[0m 29\u{1b}[0m 30\u{1b}[0m 31\u{1b}[0m      
+                                                                     ");
+    }
+}

src/cal/mod.rs

@@ -0,0 +1,166 @@
+use crate::errors::*;
+use std::str::FromStr;
+
+pub mod date;
+pub mod time;
+
+#[derive(Debug)]
+pub enum DateArg {
+    Month(u32),
+    Num(i32),
+}
+
+impl FromStr for DateArg {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<DateArg> {
+        let ds = match s.to_lowercase().as_str() {
+            "jan" | "january"   => DateArg::Month(1),
+            "feb" | "february"  => DateArg::Month(2),
+            "mar" | "march"     => DateArg::Month(3),
+            "apr" | "april"     => DateArg::Month(4),
+            "may"               => DateArg::Month(5),
+            "jun" | "june"      => DateArg::Month(6),
+            "jul" | "july"      => DateArg::Month(7),
+            "aug" | "august"    => DateArg::Month(8),
+            "sep" | "september" => DateArg::Month(9),
+            "oct" | "october"   => DateArg::Month(10),
+            "nov" | "november"  => DateArg::Month(11),
+            "dec" | "december"  => DateArg::Month(12),
+            _ => {
+                let num = s.parse::<i32>()
+                    .context("Input is not a month and not a number")?;
+                DateArg::Num(num)
+            },
+        };
+        Ok(ds)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub enum ActivityGrade {
+    None,
+    One,
+    Two,
+    Three,
+    Four,
+}
+
+impl ActivityGrade {
+    fn as_term_str(&self) -> &'static str {
+        match self {
+            ActivityGrade::None => "\x1b[97m\x1b[48;5;238m",
+            ActivityGrade::One => "\x1b[30m\x1b[48;5;148m",
+            ActivityGrade::Two => "\x1b[30m\x1b[48;5;71m",
+            ActivityGrade::Three => "\x1b[97m\x1b[48;5;34m",
+            ActivityGrade::Four => "\x1b[97m\x1b[48;5;22m",
+        }
+    }
+
+    pub fn from_ratio(num: u64, max: u64) -> ActivityGrade {
+        let max = max as f64;
+        let num = num as f64;
+        let step = max / 4.0;
+
+        let x = num / step;
+
+        if x <= 1.0 {
+            ActivityGrade::One
+        } else if x <= 2.0 {
+            ActivityGrade::Two
+        } else if x <= 3.0 {
+            ActivityGrade::Three
+        } else {
+            ActivityGrade::Four
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn small_max_activity_1() {
+        let grade = ActivityGrade::from_ratio(1, 1);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_2() {
+        let grade = ActivityGrade::from_ratio(2, 2);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_3() {
+        let grade = ActivityGrade::from_ratio(3, 3);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_4() {
+        let grade = ActivityGrade::from_ratio(4, 4);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_2_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 2);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_3_but_is_2() {
+        let grade = ActivityGrade::from_ratio(2, 3);
+        assert_eq!(grade, ActivityGrade::Three);
+    }
+
+    #[test]
+    fn small_max_activity_3_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 3);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_4_but_is_3() {
+        let grade = ActivityGrade::from_ratio(3, 4);
+        assert_eq!(grade, ActivityGrade::Three);
+    }
+
+    #[test]
+    fn small_max_activity_4_but_is_2() {
+        let grade = ActivityGrade::from_ratio(2, 4);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_4_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 4);
+        assert_eq!(grade, ActivityGrade::One);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_4() {
+        let grade = ActivityGrade::from_ratio(4, 5);
+        assert_eq!(grade, ActivityGrade::Four);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_3() {
+        let grade = ActivityGrade::from_ratio(3, 5);
+        assert_eq!(grade, ActivityGrade::Three);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_2() {
+        let grade = ActivityGrade::from_ratio(2, 5);
+        assert_eq!(grade, ActivityGrade::Two);
+    }
+
+    #[test]
+    fn small_max_activity_5_but_is_1() {
+        let grade = ActivityGrade::from_ratio(1, 5);
+        assert_eq!(grade, ActivityGrade::One);
+    }
+}

src/cal/time.rs

@@ -0,0 +1,210 @@
+use crate::errors::*;
+
+use chrono::Duration;
+use chrono::prelude::*;
+use crate::cal::{ActivityGrade, DateArg};
+use crate::models::*;
+use std::collections::HashMap;
+use std::fmt::Write;
+
+const MIN_PER_DAY: u32 = 1440;
+
+fn round_to_slice(time: &NaiveDateTime, slice_duration: u32) -> NaiveDateTime {
+    let date = time.date();
+    let hour = time.hour();
+    let mins = time.minute();
+    let slice = mins - (mins % slice_duration);
+    date.and_hms_opt(hour, slice, 0).expect("Invalid hour/min/sec")
+}
+
+fn setup_graph_map(events: &[Activity], slice_duration: u32) -> (HashMap<NaiveDateTime, u64>, u64) {
+    debug!("Found {} events in selected range", events.len());
+
+    let mut cur = None;
+    let mut ctr = 0;
+    let mut max = 0;
+
+    let mut map = HashMap::new();
+    for event in events {
+        let time = round_to_slice(&event.time, slice_duration);
+
+        if let Some(cur) = cur.as_mut() {
+            if time == *cur {
+                ctr += 1;
+            } else {
+                if ctr > max {
+                    max = ctr;
+                }
+                map.insert(*cur, ctr);
+                *cur = time;
+                ctr = 1;
+            }
+        } else {
+            cur = Some(time);
+            ctr = 1;
+        }
+    }
+
+    if ctr > 0 {
+        if let Some(cur) = cur.take() {
+            if ctr > max {
+                max = ctr;
+            }
+            map.insert(cur, ctr);
+        }
+    }
+
+    debug!("Maximum events per slice is {}", max);
+
+    (map, max)
+}
+
+pub struct DateTimeContext {
+    events: HashMap<NaiveDateTime, u64>,
+    max: u64,
+    now: NaiveDateTime,
+    pub slice_width: u32,
+    pub slice_duration: u32,
+}
+
+impl DateTimeContext {
+    pub fn new(events: &[Activity], now: NaiveDateTime, slice_width: u32, slice_duration: u32) -> DateTimeContext {
+        let (events, max) = setup_graph_map(events, slice_duration);
+        DateTimeContext {
+            events,
+            max,
+            now: round_to_slice(&now, slice_duration),
+            slice_width,
+            slice_duration,
+        }
+    }
+
+    #[inline]
+    fn is_future(&self, time: &NaiveDateTime) -> bool {
+        self.now < *time
+    }
+
+    #[inline]
+    fn slices_per_hour(&self) -> u32 {
+        60 / self.slice_duration
+    }
+
+    #[inline]
+    fn hour_width(&self) -> u32 {
+        self.slices_per_hour() * self.slice_width
+    }
+
+    fn activity_for_slice(&self, time: &NaiveDateTime) -> ActivityGrade {
+        if let Some(events) = self.events.get(time) {
+            ActivityGrade::from_ratio(*events, self.max)
+        } else {
+            ActivityGrade::None
+        }
+    }
+}
+
+pub struct DateTimeSpec {
+    start: NaiveDate,
+    end: NaiveDate,
+}
+
+impl DateTimeSpec {
+    pub fn from_args(args: &[DateArg], context: Option<u32>) -> Result<DateTimeSpec> {
+        let today = Utc::now().date_naive();
+        if args.is_empty() {
+            let mut start = today;
+
+            if let Some(context) = context {
+                start = start.checked_sub_signed(Duration::days(context as i64))
+                    .ok_or_else(|| format_err!("Failed travel back in time"))?;
+            }
+
+            Ok(DateTimeSpec {
+                start,
+                end: today,
+            })
+        } else {
+            todo!()
+        }
+    }
+
+    pub fn start(&self) -> &NaiveDate {
+        &self.start
+    }
+
+    pub fn end(&self) -> &NaiveDate {
+        &self.end
+    }
+
+    fn push_date(&self, w: &mut String, date: &NaiveDate) {
+        let weekday = date.weekday();
+        let is_weekend = weekday == Weekday::Sat || weekday == Weekday::Sun;
+
+        if is_weekend {
+            w.push_str("\x1b[1m");
+        }
+
+        w.push_str(&date.format("%Y-%m-%d (%a) ").to_string());
+
+        if is_weekend {
+            w.push_str("\x1b[0m");
+        }
+    }
+
+    pub fn to_term_string(&self, ctx: &DateTimeContext) -> String {
+        let mut w = String::new();
+
+        // add legend
+        w.push_str(&" ".repeat(11));
+        for x in 0..24 {
+            write!(w, "{:02}", x).expect("out of memory");
+
+            for i in 0..ctx.hour_width() {
+                if i >= 2 {
+                    w.push(' ');
+                }
+            }
+        }
+        w.push('\n');
+
+        // add days
+        let mut date = self.start;
+        while date <= self.end {
+            self.push_date(&mut w, &date);
+
+            let mut hours = 0;
+            let mut mins = 0;
+
+            for _ in 0..(MIN_PER_DAY / ctx.slice_duration) {
+                let time = date.and_hms_opt(hours, mins, 0).expect("Invalid hour/min/sec");
+
+                if !ctx.is_future(&time) {
+                    let activity = ctx.activity_for_slice(&time);
+                    w.push_str(activity.as_term_str());
+                } else {
+                    w.push_str("\x1b[0m");
+                }
+
+                for _ in 0..ctx.slice_width {
+                    w.push(' ');
+                }
+
+                mins += ctx.slice_duration;
+                if mins >= 60 {
+                    hours += 1;
+                    mins = 0;
+                }
+            }
+            w.push_str("\x1b[0m");
+
+            if date < self.end {
+                w.push('\n');
+            }
+
+            date = date.checked_add_signed(Duration::days(1))
+                .expect("Failed to get next day");
+        }
+
+        w
+    }
+}

src/cmd/activity_cmd.rs

@@ -18,7 +18,7 @@ pub struct TimeSpec {
 
 impl TimeSpec {
     fn resolve(s: &str, now: NaiveDateTime) -> Result<Self> {
-        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
+        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms_opt(0, 0, 0).expect("Invalid hour/min/sec"));
 
         let datetime = match s {
             "today" => today,
@@ -122,8 +122,8 @@ mod tests {
     use super::*;
 
     fn datetime() -> NaiveDateTime {
-        let date = chrono::NaiveDate::from_ymd(2020, 3, 14);
-        let time = chrono::NaiveTime::from_hms(16, 20, 23);
+        let date = chrono::NaiveDate::from_ymd_opt(2020, 3, 14).unwrap();
+        let time = chrono::NaiveTime::from_hms_opt(16, 20, 23).unwrap();
         NaiveDateTime::new(date, time)
     }
 

src/cmd/add_cmd.rs

@@ -2,6 +2,7 @@ use crate::errors::*;
 
 use crate::blobs::Blob;
 use crate::cmd::Cmd;
+use crate::db::DbChange;
 use crate::gfx;
 use crate::models::*;
 use crate::shell::Shell;
@@ -9,10 +10,11 @@ use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::utils;
 use crate::term;
+use std::fmt::Debug;
 use std::fs;
+use std::io::{self, BufRead};
 use std::net;
 use std::net::SocketAddr;
-use ipnetwork;
 use std::path::Path;
 use walkdir::WalkDir;
 
@@ -24,6 +26,9 @@ pub struct Args {
     /// Do not actually insert into database
     #[structopt(short="n", long="dry-run")]
     dry_run: bool,
+    /// Stream structs from stdin line by line
+    #[structopt(long)]
+    stdin: bool,
 }
 
 #[derive(Debug, StructOpt)]
@@ -74,25 +79,68 @@ pub enum Target {
 
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
-        match self.subcommand {
-            Target::Domain(args) => args.insert(rl, self.dry_run),
-            Target::Subdomain(args) => args.insert(rl, self.dry_run),
-            Target::IpAddr(args) => args.insert(rl, self.dry_run),
-            Target::Url(args) => args.insert(rl, self.dry_run),
-            Target::Email(args) => args.insert(rl, self.dry_run),
-            Target::PhoneNumber(args) => args.insert(rl, self.dry_run),
-            Target::Device(args) => args.insert(rl, self.dry_run),
-            Target::Network(args) => args.insert(rl, self.dry_run),
-            Target::Account(args) => args.insert(rl, self.dry_run),
-            Target::Breach(args) => args.insert(rl, self.dry_run),
-            Target::Image(args) => args.insert(rl, self.dry_run),
-            Target::Netblock(args) => args.insert(rl, self.dry_run),
-            Target::Port(args) => args.insert(rl, self.dry_run),
-            Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
+        if self.stdin {
+            match self.subcommand {
+                Target::Domain(_) => stream_stdin::<AddDomain>(rl),
+                Target::Subdomain(_) => stream_stdin::<AddSubdomain>(rl),
+                Target::IpAddr(_) => stream_stdin::<AddIpAddr>(rl),
+                Target::Url(_) => stream_stdin::<AddUrl>(rl),
+                Target::Email(_) => stream_stdin::<AddEmail>(rl),
+                Target::PhoneNumber(_) => stream_stdin::<AddPhoneNumber>(rl),
+                Target::Port(_) => stream_stdin::<AddPort>(rl),
+                Target::CryptoAddr(_) => stream_stdin::<AddCryptoAddr>(rl),
+                Target::Device(_) | Target::Network(_) | Target::Account(_) | Target::Breach(_) | Target::Image(_) | Target::Netblock(_) => {
+                    bail!("This struct doesn't support streaming from stdin")
+                },
+            }
+        } else {
+            match self.subcommand {
+                Target::Domain(args) => args.insert(rl, self.dry_run),
+                Target::Subdomain(args) => args.insert(rl, self.dry_run),
+                Target::IpAddr(args) => args.insert(rl, self.dry_run),
+                Target::Url(args) => args.insert(rl, self.dry_run),
+                Target::Email(args) => args.insert(rl, self.dry_run),
+                Target::PhoneNumber(args) => args.insert(rl, self.dry_run),
+                Target::Device(args) => args.insert(rl, self.dry_run),
+                Target::Network(args) => args.insert(rl, self.dry_run),
+                Target::Account(args) => args.insert(rl, self.dry_run),
+                Target::Breach(args) => args.insert(rl, self.dry_run),
+                Target::Image(args) => args.insert(rl, self.dry_run),
+                Target::Netblock(args) => args.insert(rl, self.dry_run),
+                Target::Port(args) => args.insert(rl, self.dry_run),
+                Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
+            }
         }
     }
 }
 
+fn stream_stdin<T: InsertFromString>(rl: &mut Shell) -> Result<()> {
+    let stdin = io::stdin();
+    for line in stdin.lock().lines() {
+        let line = line?;
+        match T::from_string(rl, line) {
+            Ok(insert) => {
+                debug!("Received {:?}", insert);
+                let result = rl.db().insert_generic(insert.clone())?;
+
+                if let Some((DbChange::Insert, _)) = result {
+                    if let Ok(value) = insert.value(rl.db()) {
+                        let log = format!("Adding {} {:?}", insert.family(), value);
+                        term::success(&log);
+                    }
+                }
+            },
+            Err(err) => term::error(&format!("Error: {:#}", err)),
+        }
+    }
+    Ok(())
+}
+
+trait InsertFromString {
+    // TODO: some implementations run insert_struct instead of just transforming
+    fn from_string(rl: &mut Shell, x: String) -> Result<Insert>;
+}
+
 trait IntoInsert: Sized {
     fn into_insert(self, rl: &mut Shell) -> Result<Insert>;
 
@@ -116,7 +164,12 @@ impl IntoInsert for AddDomain {
             Some(domain) => domain,
             _ => utils::question("Domain")?,
         };
+        Self::from_string(rl, domain)
+    }
+}
 
+impl InsertFromString for AddDomain {
+    fn from_string(rl: &mut Shell, domain: String) -> Result<Insert> {
         // ensure input is a valid domain
         let dns_name = rl.psl()?.parse_dns_name(&domain)
             .map_err(|e| format_err!("Failed to parse domain: {}", e))?;
@@ -143,7 +196,12 @@ impl IntoInsert for AddSubdomain {
             Some(subdomain) => subdomain,
             _ => utils::question("Subdomain")?,
         };
+        Self::from_string(rl, subdomain)
+    }
+}
 
+impl InsertFromString for AddSubdomain {
+    fn from_string(rl: &mut Shell, subdomain: String) -> Result<Insert> {
         let dns_name = rl.psl()?.parse_dns_name(&subdomain)
             .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
 
@@ -170,7 +228,8 @@ pub struct AddIpAddr {
 }
 
 impl IntoInsert for AddIpAddr {
-    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
+        // TODO: change this to Option<String> so we don't need to parse back and forth
         let ipaddr = match self.ipaddr {
             Some(ipaddr) => ipaddr,
             _ => {
@@ -178,6 +237,13 @@ impl IntoInsert for AddIpAddr {
                 ipaddr.parse()?
             },
         };
+        Self::from_string(rl, ipaddr.to_string())
+    }
+}
+
+impl InsertFromString for AddIpAddr {
+    fn from_string(_rl: &mut Shell, ipaddr: String) -> Result<Insert> {
+        let ipaddr = ipaddr.parse()?;
 
         let family = match ipaddr {
             net::IpAddr::V4(_) => "4",
@@ -214,12 +280,17 @@ impl IntoInsert for AddUrl {
             Some(url) => url,
             _ => utils::question("URL")?,
         };
+        Self::from_string(rl, url)
+    }
+}
 
+impl InsertFromString for AddUrl {
+    fn from_string(rl: &mut Shell, url: String) -> Result<Insert> {
         let parts = url::Url::parse(&url)?;
         let subdomain = parts.domain()
             .ok_or_else(|| format_err!("url doesn't have a domain host"))?;
 
-        let dns_name = rl.psl()?.parse_dns_name(&subdomain)
+        let dns_name = rl.psl()?.parse_dns_name(subdomain)
             .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
 
         let domain_id = match rl.db().insert_struct(NewDomain {
@@ -258,12 +329,18 @@ pub struct AddEmail {
 }
 
 impl IntoInsert for AddEmail {
-    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
         let email = match self.email {
             Some(email) => email,
             _ => utils::question("Email")?,
         };
+        Self::from_string(rl, email)
+    }
+}
 
+impl InsertFromString for AddEmail {
+    fn from_string(_rl: &mut Shell, email: String) -> Result<Insert> {
+        // TODO: consider doing basic validation
         Ok(Insert::Email(NewEmail {
             value: email,
             displayname: None,
@@ -290,9 +367,30 @@ impl IntoInsert for AddPhoneNumber {
             },
         };
 
+        // TODO: consider doing basic validation
         Ok(Insert::PhoneNumber(NewPhoneNumber {
             value: phonenumber,
-            name: name,
+            name,
+            valid: None,
+            last_online: None,
+            country: None,
+            carrier: None,
+            line: None,
+            is_ported: None,
+            last_ported: None,
+            caller_name: None,
+            caller_type: None,
+            unscoped: false,
+        }))
+    }
+}
+
+impl InsertFromString for AddPhoneNumber {
+    fn from_string(_rl: &mut Shell, phonenumber: String) -> Result<Insert> {
+        // TODO: consider doing basic validation
+        Ok(Insert::PhoneNumber(NewPhoneNumber {
+            value: phonenumber,
+            name: None,
             valid: None,
             last_online: None,
             country: None,
@@ -328,7 +426,7 @@ impl IntoInsert for AddDevice {
 
         Ok(Insert::Device(NewDevice {
             value: mac,
-            name: name,
+            name,
             hostname: None,
             vendor: None,
             last_seen: None,
@@ -393,9 +491,9 @@ impl IntoInsert for AddAccount {
         let value = format!("{}/{}", service, username);
 
         Ok(Insert::Account(NewAccount {
-            value: value,
-            service: service,
-            username: username,
+            value,
+            service,
+            username,
             displayname: None,
             email: None,
             url: None,
@@ -417,10 +515,7 @@ impl IntoInsert for AddBreach {
     fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
         let name = match self.name {
             Some(name) => name,
-            _ => {
-                let name = utils::question("Name")?;
-                name
-            }
+            _ => utils::question("Name")?,
         };
 
         Ok(Insert::Breach(NewBreach {
@@ -453,7 +548,7 @@ impl IntoInsert for AddImage {
                 let path = match path {
                     Ok(path) => path,
                     Err(err) => {
-                        let path = err.path().unwrap_or(Path::new("")).display();
+                        let path = err.path().unwrap_or_else(|| Path::new("")).display();
 
                         let err = err.io_error()
                             .map(|err| err.to_string())
@@ -575,12 +670,24 @@ impl IntoInsert for AddPort {
         };
 
         let addr = if let Some(addr) = self.addr {
+            // TODO: change this to a regular string so we don't parse->fmt->parse
             addr
         } else {
             let addr = utils::question("IP:Port")?;
             addr.parse()?
         };
 
+        Self::from_string(rl, format!("{}/{}", protocol, addr))
+    }
+}
+
+impl InsertFromString for AddPort {
+    fn from_string(rl: &mut Shell, value: String) -> Result<Insert> {
+        let idx = value.find('/')
+            .ok_or_else(|| format_err!("Could not locate `/` delimiter for protocol/ip:port"))?;
+        let protocol = value[..idx].to_string();
+        let addr = value[idx+1..].parse::<SocketAddr>()?;
+
         let family = match addr.ip() {
             net::IpAddr::V4(_) => "4",
             net::IpAddr::V6(_) => "6",
@@ -606,8 +713,6 @@ impl IntoInsert for AddPort {
             _ => bail!("IpAddr is out out of scope"),
         };
 
-        let value = format!("{}/{}", protocol, addr);
-
         Ok(Insert::Port(NewPort {
             ip_addr_id,
             value,
@@ -631,13 +736,18 @@ pub struct AddCryptoAddr {
 }
 
 impl IntoInsert for AddCryptoAddr {
-    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
         let address = if let Some(address) = self.address {
             address
         } else {
             utils::question("Address")?
         };
+        Self::from_string(rl, address)
+    }
+}
 
+impl InsertFromString for AddCryptoAddr {
+    fn from_string(_rl: &mut Shell, address: String) -> Result<Insert> {
         Ok(Insert::CryptoAddr(NewCryptoAddr {
             value: address,
             currency: None,

src/cmd/autonoscope_cmd.rs

@@ -1,6 +1,7 @@
 use crate::errors::*;
 
 use crate::autonoscope;
+use crate::cmd::Cmd;
 use crate::fmt::colors::*;
 use crate::shell::Shell;
 use std::fmt::Write;
@@ -65,7 +66,8 @@ pub fn run_with_scope_param(rl: &mut Shell, args: Args, scoped: bool) -> Result<
     }
 }
 
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    run_with_scope_param(rl, args, false)
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        run_with_scope_param(rl, self, false)
+    }
 }

src/cmd/autoscope_cmd.rs

@@ -1,13 +1,18 @@
 use crate::errors::*;
 
+use crate::cmd::Cmd;
 use crate::cmd::autonoscope_cmd;
 use crate::shell::Shell;
 use structopt::StructOpt;
 
-
-pub type Args = autonoscope_cmd::Args;
-
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    autonoscope_cmd::run_with_scope_param(rl, args, true)
+#[derive(Debug, StructOpt)]
+pub struct Args {
+    #[structopt(flatten)]
+    args: autonoscope_cmd::Args,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        autonoscope_cmd::run_with_scope_param(rl, self.args, true)
+    }
 }

src/cmd/cal_cmd.rs

@@ -0,0 +1,66 @@
+use crate::errors::*;
+
+use chrono::Utc;
+use crate::cal::DateArg;
+use crate::cal::date::{DateContext, DateSpec};
+use crate::cal::time::{DateTimeContext, DateTimeSpec};
+use crate::cmd::Cmd;
+use crate::models::*;
+use crate::shell::Shell;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Show additional months for context
+    #[structopt(short="C", long)]
+    context: Option<u32>,
+    /// Group events in 12 min slices
+    #[structopt(short="T", long, group = "view")]
+    time: bool,
+    /// Group events by hour
+    #[structopt(short="H", long, group = "view")]
+    hourly: bool,
+    args: Vec<DateArg>,
+}
+
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        if self.time || self.hourly {
+            let dts = DateTimeSpec::from_args(&self.args, self.context)
+                .context("Failed to parse date spec")?;
+            let filter = ActivityFilter {
+                topic: None,
+                since: Some(dts.start().and_hms_opt(0, 0, 0).expect("Invalid hour/min/sec")),
+                until: Some(dts.end().and_hms_opt(23, 59, 59).expect("Invalid hour/min/sec")),
+                location: false,
+            };
+            let events = Activity::query(rl.db(), &filter)?;
+
+            let (slice_width, slice_duration) = if self.hourly {
+                (3, 60)
+            } else {
+                (1, 12)
+            };
+
+            let ctx = DateTimeContext::new(&events, Utc::now().naive_utc(), slice_width, slice_duration);
+            println!("{}", dts.to_term_string(&ctx));
+        } else {
+            let ds = DateSpec::from_args(&self.args, self.context)
+                .context("Failed to parse date spec")?;
+            let filter = ActivityFilter {
+                topic: None,
+                since: Some(ds.start().and_hms_opt(0, 0, 0).expect("Invalid hour/min/sec")),
+                until: Some(ds.end().and_hms_opt(23, 59, 59).expect("Invalid hour/min/sec")),
+                location: false,
+            };
+            let events = Activity::query(rl.db(), &filter)?;
+            let ctx = DateContext::new(&events, Utc::now().date_naive());
+            println!("{}", ds.to_term_string(&ctx));
+        }
+        Ok(())
+    }
+}

src/cmd/delete_cmd.rs

@@ -19,20 +19,20 @@ pub struct Args {
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
         let rows = match &self.subcommand {
-            Target::Domains(filter) => delete::<Domain>(rl, &filter),
-            Target::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
-            Target::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
-            Target::Urls(filter) => delete::<Url>(rl, &filter),
-            Target::Emails(filter) => delete::<Email>(rl, &filter),
-            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
-            Target::Devices(filter) => delete::<Device>(rl, &filter),
-            Target::Networks(filter) => delete::<Network>(rl, &filter),
-            Target::Accounts(filter) => delete::<Account>(rl, &filter),
-            Target::Breaches(filter) => delete::<Breach>(rl, &filter),
-            Target::Images(filter) => delete::<Image>(rl, &filter),
-            Target::Ports(filter) => delete::<Port>(rl, &filter),
-            Target::Netblocks(filter) => delete::<Netblock>(rl, &filter),
-            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, &filter),
+            Target::Domains(filter) => delete::<Domain>(rl, filter),
+            Target::Subdomains(filter) => delete::<Subdomain>(rl, filter),
+            Target::IpAddrs(filter) => delete::<IpAddr>(rl, filter),
+            Target::Urls(filter) => delete::<Url>(rl, filter),
+            Target::Emails(filter) => delete::<Email>(rl, filter),
+            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, filter),
+            Target::Devices(filter) => delete::<Device>(rl, filter),
+            Target::Networks(filter) => delete::<Network>(rl, filter),
+            Target::Accounts(filter) => delete::<Account>(rl, filter),
+            Target::Breaches(filter) => delete::<Breach>(rl, filter),
+            Target::Images(filter) => delete::<Image>(rl, filter),
+            Target::Ports(filter) => delete::<Port>(rl, filter),
+            Target::Netblocks(filter) => delete::<Netblock>(rl, filter),
+            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, filter),
         }?;
         term::info(&format!("Deleted {} rows", rows));
         Ok(())

src/cmd/export_cmd.rs

@@ -3,14 +3,13 @@ use crate::errors::*;
 use crate::blobs::Blob;
 use crate::cmd::Cmd;
 use crate::db::ttl;
+use crate::models::*;
 use crate::shell::Shell;
-use serde_json;
-use serde::Serialize;
+use serde::{Serialize, Deserialize};
 use std::io::{self, Write};
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use strum_macros::{EnumString, IntoStaticStr};
-use crate::models::*;
 
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
@@ -22,7 +21,7 @@ pub struct Args {
 
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
-        ttl::reap_expired(rl.db())?;
+        ttl::reap_expired(rl)?;
         match self.format {
             Format::Json => export::<JsonFormat>(rl),
             Format::JsonBlobs => export::<JsonBlobsFormat>(rl),

src/cmd/help_cmd.rs

@@ -21,6 +21,7 @@ pub fn run(_rl: &mut Shell, _args: &[String]) -> Result<()> {
     help("run",         "Run the currently selected module");
     help("scope",       "Include entities in the scope again");
     help("select",      "Select entities from the database");
+    help("stats",       "Show statistics about your current workspace");
     help("target",      "Preview targeted entities or narrow them down");
     help("use",         "Select a module");
     help("workspace",   "Switch to a different workspace");

src/cmd/mod.rs

@@ -20,6 +20,7 @@ pub mod activity_cmd;
 pub mod add_cmd;
 pub mod autonoscope_cmd;
 pub mod autoscope_cmd;
+pub mod cal_cmd;
 pub mod delete_cmd;
 pub mod export_cmd;
 pub mod fsck_cmd;
@@ -29,9 +30,12 @@ pub mod use_cmd;
 pub mod select_cmd;
 pub mod keyring_cmd;
 pub mod noscope_cmd;
+pub mod notify_cmd;
 pub mod pkg_cmd;
+pub mod rescope_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;
+pub mod stats_cmd;
 pub mod target_cmd;
 pub mod quickstart_cmd;
 pub mod workspace_cmd;

src/cmd/notify_cmd.rs

@@ -0,0 +1,99 @@
+use crate::errors::*;
+
+use crate::cmd::Cmd;
+use crate::engine::Module;
+use crate::notify::{self, Notification};
+use crate::options::{self, Opt};
+use crate::shell::Shell;
+use crate::term;
+use sn0int_std::ratelimits::Ratelimiter;
+use std::fmt::Write;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Subcommand,
+}
+
+#[derive(Debug, StructOpt)]
+pub enum Subcommand {
+    /// Manually add a notification to the outbox
+    Send(SendArgs),
+    /// Show the current outbox
+    Outbox,
+    /// Execute a module directly instead of sending a message
+    Exec(ExecArgs),
+    /// Try to deliver all messages in our outbox
+    Deliver,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct SendArgs {
+    /// Evaluate the routing rules, but do not actually send a notification
+    #[structopt(short="n", long)]
+    pub dry_run: bool,
+    pub topic: String,
+    #[structopt(flatten)]
+    pub notification: Notification,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct ExecArgs {
+    pub module: String,
+    #[structopt(short="o", long="option")]
+    pub options: Vec<options::Opt>,
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
+    verbose: u64,
+    #[structopt(flatten)]
+    pub notification: Notification,
+}
+
+fn print_summary(module: &Module, sent: usize, errors: usize) {
+    let mut out = if sent == 1 {
+        String::from("Sent 1 notification")
+    } else {
+        format!("Sent {} notifications", sent)
+    };
+
+    write!(out, " with {}", module.canonical()).expect("out of memory");
+
+    if errors > 0 {
+        write!(out, " ({} errors)", errors).expect("out of memory");
+    }
+
+    term::info(&out);
+}
+
+fn send(args: SendArgs, rl: &mut Shell) -> Result<()> {
+    rl.signal_register().catch_ctrl();
+    notify::run_router(rl, &mut term::Term, &mut Ratelimiter::new(), args.dry_run, &args.topic, &args.notification)?;
+    rl.signal_register().reset_ctrlc();
+    Ok(())
+}
+
+fn exec(args: ExecArgs, rl: &mut Shell) -> Result<()> {
+    let module = rl.library().get(&args.module)?.clone();
+    let options = Opt::collect(&args.options);
+
+    rl.signal_register().catch_ctrl();
+    let errors = notify::exec(rl, &module, &mut Ratelimiter::new(), options, args.verbose, &args.notification)?;
+    rl.signal_register().reset_ctrlc();
+
+    print_summary(&module, 1, errors);
+    Ok(())
+}
+
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        match self.subcommand {
+            Subcommand::Send(args) => send(args, rl),
+            Subcommand::Outbox => todo!(),
+            Subcommand::Exec(args) => exec(args, rl),
+            Subcommand::Deliver => todo!(),
+        }
+    }
+}

src/cmd/pkg_cmd.rs

@@ -5,13 +5,15 @@ use crate::api::Client;
 use crate::args;
 use crate::config::Config;
 use crate::cmd::{Cmd, LiteCmd};
-use crate::engine::Library;
+use crate::engine::{Library, Module};
 use crate::registry::{self, InstallTask, UpdateTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
 use crate::worker;
-use colored::Colorize;
+use colored::{Color, Colorize};
 use sn0int_common::ModuleID;
+use sn0int_common::metadata::Stealth;
+use std::collections::HashSet;
 use std::fmt::Write;
 use std::sync::Arc;
 use structopt::StructOpt;
@@ -70,7 +72,13 @@ pub struct List {
     pub source: Option<String>,
     /// List outdated modules
     #[structopt(long="outdated")]
-    pub outdated: bool,
+    pub outdated_only: bool,
+    /// Only show modules with equal or better stealth level
+    #[structopt(long="stealth", possible_values=Stealth::variants())]
+    pub stealth: Option<Stealth>,
+    /// Filter by pattern
+    #[structopt(default_value="*")]
+    pub pattern: String,
 }
 
 #[derive(Debug, StructOpt)]
@@ -92,45 +100,80 @@ enum ModuleReload {
     No,
 }
 
+#[inline]
+fn write_tag(out: &mut String, color: Color, txt: &str) -> Result<()> {
+    write!(out, " [{}]", txt.color(color))?;
+    Ok(())
+}
+
+fn print_module(module: &Module, is_outdated: bool) -> Result<()> {
+    let mut out = String::new();
+    write!(&mut out, "{}/{} {}", module.author().purple(),
+                                 module.name(),
+                                 module.version().blue())?;
+
+    match module.stealth() {
+        Stealth::Loud => write_tag(&mut out, Color::Yellow, "LOUD")?,
+        Stealth::Normal => (),
+        Stealth::Passive => write_tag(&mut out, Color::Green, "passive")?,
+        Stealth::Offline => write_tag(&mut out, Color::Green, "offline")?,
+    };
+
+    if is_outdated {
+        write_tag(&mut out, Color::Red, "outdated")?;
+    }
+
+    println!("{}", out.bold());
+    println!("    {}", module.description());
+    Ok(())
+}
+
 fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) -> Result<ModuleReload> {
     match subcommand {
         SubCommand::List(list) => {
             let autoupdate = AutoUpdater::load()?;
 
+            let filter = glob::Pattern::new(&list.pattern)?;
+
             for module in library.list() {
                 if let Some(source) = &list.source {
-                    if !module.source_equals(&source) {
+                    if !module.source_equals(source) {
                         continue;
                     }
                 }
 
                 let canonical = module.canonical();
-
-                let mut out = String::new();
-                write!(&mut out, "{} ({})", canonical.green(),
-                                            module.version().yellow())?;
-                if autoupdate.is_outdated(&canonical) {
-                    write!(&mut out, " {}", "[outdated]".red())?;
-                } else if list.outdated {
+                if !filter.matches(&canonical) {
                     continue;
                 }
-                println!("{}", out);
-                println!("\t{}", module.description());
+
+                if let Some(stealth) = &list.stealth {
+                    if !module.stealth().equal_or_better(stealth) {
+                        continue;
+                    }
+                }
+
+                let is_outdated = autoupdate.is_outdated(&canonical);
+                if list.outdated_only && !is_outdated {
+                    continue;
+                }
+
+                print_module(module, is_outdated)?;
             }
             Ok(ModuleReload::No)
         },
         SubCommand::Install(install) => {
-            registry::run_install(install, &config)?;
+            registry::run_install(install, config)?;
             // trigger reload
             Ok(ModuleReload::Yes)
         },
         SubCommand::Search(search) => {
-            registry::run_search(library, &search, &config)?;
+            registry::run_search(library, &search, config)?;
             Ok(ModuleReload::No)
         },
         SubCommand::Update(_) => {
             let mut autoupdate = AutoUpdater::load()?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let updater = Arc::new(Updater::new(config)?);
 
             let modules = library.list()
                 .into_iter()
@@ -156,27 +199,40 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
             Ok(ModuleReload::Yes)
         },
         SubCommand::Uninstall(uninstall) => {
-            let updater = Updater::new(&config)?;
+            let updater = Updater::new(config)?;
             updater.uninstall(&uninstall.module)?;
             // trigger reload
             Ok(ModuleReload::Yes)
         },
         SubCommand::Quickstart => {
-            let client = Client::new(&config)?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let client = Client::new(config)?;
+            let updater = Arc::new(Updater::new(config)?);
             let mut autoupdate = AutoUpdater::load()?;
 
+            let installed = library.list()
+                .into_iter()
+                .map(|module| module.id())
+                .collect::<HashSet<_>>();
+
             let modules = client.quickstart()?
                 .into_iter()
-                .map(|module| {
-                    InstallTask::new(Install {
-                        module: ModuleID {
-                            author: module.author,
-                            name: module.name,
-                        },
-                        version: None,
-                        force: false,
-                    }, updater.clone())
+                .filter_map(|module| {
+                    let id = ModuleID {
+                        author: module.author,
+                        name: module.name,
+                    };
+
+                    if !installed.contains(&id) {
+                        info!("Queueing for install: {}", id);
+                        Some(InstallTask::new(Install {
+                            module: id,
+                            version: None,
+                            force: false,
+                        }, updater.clone()))
+                    } else {
+                        info!("Skipping already installed module: {}", id);
+                        None
+                    }
                 })
                 .collect::<Vec<_>>();
 
@@ -194,7 +250,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
 
 impl LiteCmd for Args {
     fn run(self, config: &Config) -> Result<()> {
-        let library = Library::new(false, &config)?;
+        let library = Library::new(false, config)?;
         run_subcommand(self.subcommand, &library, config)?;
         Ok(())
     }

src/cmd/rescope_cmd.rs

@@ -0,0 +1,268 @@
+use crate::errors::*;
+
+use crate::autonoscope::{AutoRule, ToRule};
+use crate::cmd::Cmd;
+use crate::db::Database;
+use crate::filters::{Filter, Target};
+use crate::shell::Shell;
+use std::collections::HashSet;
+use std::fmt;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+use crate::models::*;
+use crate::utils;
+use crate::term;
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Run rules interactively
+    #[structopt(short, long)]
+    interactive: bool,
+    /// Automatically apply changes to database
+    #[structopt(short="y", long)]
+    auto_confirm: bool,
+    /// Only show changes, do not apply them to the database
+    #[structopt(short="n", long)]
+    dry_run: bool,
+    /// Only rescope entities matching specific filter
+    #[structopt(subcommand)]
+    target: Option<Target>,
+}
+
+enum Entity {
+    Domain(Domain),
+    Subdomain(Subdomain),
+    IpAddr(IpAddr),
+    Url(Url),
+    Port(Port),
+    Netblock(Netblock),
+}
+
+impl Entity {
+    fn set_scoped(&self, db: &Database, value: bool) -> Result<()> {
+        match self {
+            Entity::Domain(entity) => entity.set_scoped(db, value),
+            Entity::Subdomain(entity) => entity.set_scoped(db, value),
+            Entity::IpAddr(entity) => entity.set_scoped(db, value),
+            Entity::Url(entity) => entity.set_scoped(db, value),
+            Entity::Port(entity) => entity.set_scoped(db, value),
+            Entity::Netblock(entity) => entity.set_scoped(db, value),
+        }
+    }
+}
+
+enum Input {
+    Yes,
+    No,
+    Done,
+    Always,
+    Never,
+}
+
+fn get_input() -> Result<Input> {
+    loop {
+        let input = utils::question_opt("Update this entity? [Y/n/d/a/x/?]")?;
+        let input = input.map(|s| s.to_lowercase());
+        match input.as_deref() {
+            Some("y") | None => return Ok(Input::Yes),
+            Some("n") => return Ok(Input::No),
+            Some("d") => return Ok(Input::Done),
+            Some("a") => return Ok(Input::Always),
+            Some("x") => return Ok(Input::Never),
+            Some("?") => {
+                term::success("y -> yes, apply this change");
+                term::success("n -> no, skip this change");
+                term::success("d -> done, skip this and further changes");
+                term::success("a -> always, apply every change caused by this specific rule");
+                term::success("x -> never, skip every change caused by this specific rule");
+            },
+            Some(input) => term::error(&format!("Unrecognized input: {:?}", input)),
+        }
+    }
+}
+
+#[derive(Default)]
+struct Context {
+    update_queue: Vec<(Entity, bool)>,
+    always_rules: HashSet<(&'static str, String)>,
+    never_rules: HashSet<(&'static str, String)>,
+    done: bool,
+    target: Option<Target>,
+}
+
+fn rescope_to_queue<T, F1, F2, F3>(ctx: &mut Context, db: &Database, interactive: bool, get_filter: F1, matches_rule: F2, wrap: F3) -> Result<()>
+    where
+        T: Model + Scopable + fmt::Debug,
+        F1: Fn(&Target) -> Option<&Filter>,
+        F2: Fn(&T) -> Result<Option<((&'static str, String), bool)>>,
+        F3: Fn(T) -> Entity,
+{
+    // do nothing if we're already done
+    if ctx.done {
+        return Ok(());
+    }
+
+    // check if there are filters to be applied
+    let filter = if let Some(target) = &ctx.target {
+        if let Some(filter) = get_filter(target) {
+            // we've selected this specific entity type and there's a filter
+            filter.parse_optional()
+                .context("Filter is invalid")?
+        } else {
+            // we do not wish to process this entity type
+            return Ok(());
+        }
+    } else {
+        // any entity type is fine
+        Filter::any()
+    };
+
+    let entities = db.filter::<T>(&filter)?;
+
+    for entity in entities {
+        let currently_scoped = entity.scoped();
+        debug!("rescoping entity: {:?}", entity);
+
+        if let Some((rule, should_be)) = matches_rule(&entity)? {
+            // check if we're actively ignoring this rule
+            if ctx.never_rules.contains(&rule) {
+                continue;
+            }
+
+            if currently_scoped != should_be {
+                let prefix = if should_be {
+                    "\x1b[1m[\x1b[32m+\x1b[0;1m]\x1b[0m"
+                } else {
+                    "\x1b[1m[\x1b[31m-\x1b[0;1m]\x1b[0m"
+                };
+
+                println!("{} Setting entity {:?} => {:?}: {:?}", prefix, currently_scoped, should_be, entity);
+
+                // check if we're auto-accepting this rule
+                let input = if ctx.always_rules.contains(&rule) {
+                    Input::Yes
+                } else if interactive {
+                    get_input()?
+                } else {
+                    Input::Yes
+                };
+
+                // process user input
+                let input = match input {
+                    Input::Always => {
+                        ctx.always_rules.insert(rule);
+                        Input::Yes
+                    },
+                    Input::Never => {
+                        ctx.never_rules.insert(rule);
+                        Input::No
+                    },
+                    Input::Done => {
+                        ctx.done = true;
+                        break;
+                    },
+                    input => input,
+                };
+
+                if let Input::Yes = input {
+                    ctx.update_queue.push((wrap(entity), should_be));
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let rules = rl.db().autonoscope();
+        term::success(&format!("Loaded {} rules", rules.len()));
+
+        let mut ctx = Context {
+            target: self.target,
+            ..Default::default()
+        };
+
+        rescope_to_queue::<Domain, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.domains(), |entity| {
+            for rule in rules.domains() {
+                if rule.matches(entity.value.as_str())? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Domain)?;
+        rescope_to_queue::<Subdomain, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.subdomains(), |entity| {
+            for rule in rules.domains() {
+                if rule.matches(entity.value.as_str())? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Subdomain)?;
+
+        rescope_to_queue::<IpAddr, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.ipaddrs(), |entity| {
+            for rule in rules.ips() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::IpAddr)?;
+
+        rescope_to_queue::<Url, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.urls(), |entity| {
+            for rule in rules.domains() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            for rule in rules.urls() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Url)?;
+        rescope_to_queue::<Port, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.ports(), |entity| {
+            for rule in rules.ips() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Port)?;
+        rescope_to_queue::<Netblock, _, _, _>(&mut ctx, rl.db(), self.interactive, |t| t.netblocks(), |entity| {
+            for rule in rules.ips() {
+                if rule.matches(entity)? {
+                    return Ok(Some((rule.to_rule(), rule.scoped)));
+                }
+            }
+            Ok(None)
+        }, Entity::Netblock)?;
+
+        if ctx.update_queue.is_empty() {
+            term::success("Nothing has changed, not updating database");
+        } else {
+            let confirm = if self.dry_run {
+                false
+            } else if self.auto_confirm {
+                true
+            } else {
+                utils::no_else_yes(&format!("Apply {} changes to scope now?", ctx.update_queue.len()))?
+            };
+
+            if confirm {
+                term::info(&format!("Applying {} changes to database", ctx.update_queue.len()));
+
+                for (update, value) in ctx.update_queue {
+                    update.set_scoped(rl.db(), value)?;
+                }
+            } else {
+                term::info("Database not updated");
+            }
+        }
+
+        Ok(())
+    }
+}

src/cmd/run_cmd.rs

@@ -14,9 +14,10 @@ use crate::term;
 use crate::utils;
 use crate::worker;
 use serde::Serialize;
-use serde_json;
 use sn0int_common::metadata::Source;
+use sn0int_std::ratelimits::Ratelimiter;
 use std::collections::HashMap;
+use std::net::SocketAddr;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 
@@ -24,14 +25,26 @@ use structopt::clap::AppSettings;
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
-    #[structopt(short="j", long="threads", default_value="1")]
-    threads: usize,
-    #[structopt(short="v", long="verbose", parse(from_occurrences))]
-    verbose: u64,
+    /// Execute a module that has been installed
+    pub module: Option<String>,
+    /// Run investigations concurrently
+    #[structopt(short="j", default_value="1")]
+    pub threads: usize,
+    /// Verbose logging, once to print inserts even if they don't add new
+    /// data, twice to activate the debug() function
+    #[structopt(short="v", long, parse(from_occurrences))]
+    pub verbose: u64,
+    /// Set a specific socks5 proxy to use
+    #[structopt(short="X", long)]
+    pub proxy: Option<SocketAddr>,
+    /// Set a different default user agent
+    #[structopt(long)]
+    pub user_agent: Option<String>,
 }
 
 #[derive(Debug, Clone)]
 pub struct Params<'a> {
+    pub module: Option<&'a String>,
     pub threads: usize,
     pub verbose: u64,
     pub stdin: bool,
@@ -39,25 +52,61 @@ pub struct Params<'a> {
     pub grant_full_keyring: bool,
     pub deny_keyring: bool,
     pub exit_on_error: bool,
+    pub proxy: Option<SocketAddr>,
+    pub user_agent: Option<&'a String>,
+}
+
+impl<'a> Params<'a> {
+    pub fn get_module(&self, rl: &Shell) -> Result<Module> {
+        let module = if let Some(module) = self.module {
+            rl.library().get(module)?
+                .clone()
+        } else {
+            rl.module()
+                .map(|m| m.to_owned())
+                .ok_or_else(|| format_err!("No module selected"))?
+        };
+        Ok(module)
+    }
+
+    pub fn get_proxy(&self, rl: &Shell) -> Option<SocketAddr> {
+        if self.proxy.is_some() {
+            self.proxy
+        } else {
+            rl.config().network.proxy
+        }
+    }
+
+    pub fn get_user_agent(&self, rl: &Shell) -> Option<String> {
+        if let Some(user_agent) = self.user_agent {
+            Some(user_agent.to_string())
+        } else {
+            rl.config().network.user_agent.clone()
+        }
+    }
 }
 
 impl<'a> From<&'a args::Run> for Params<'a> {
     fn from(args: &args::Run) -> Params {
         Params {
-            threads: args.threads,
-            verbose: args.verbose,
+            module: args.run.module.as_ref(),
+            threads: args.run.threads,
+            verbose: args.run.verbose,
             stdin: args.stdin,
             grants: &args.grants,
             grant_full_keyring: args.grant_full_keyring,
             deny_keyring: args.deny_keyring,
             exit_on_error: args.exit_on_error,
+            proxy: args.run.proxy,
+            user_agent: args.run.user_agent.as_ref(),
         }
     }
 }
 
-impl From<Args> for Params<'static> {
-    fn from(args: Args) -> Params<'static> {
+impl<'a> From<&'a Args> for Params<'a> {
+    fn from(args: &Args) -> Params {
         Params {
+            module: args.module.as_ref(),
             threads: args.threads,
             verbose: args.verbose,
             stdin: false,
@@ -65,6 +114,8 @@ impl From<Args> for Params<'static> {
             grant_full_keyring: false,
             deny_keyring: false,
             exit_on_error: false,
+            proxy: args.proxy,
+            user_agent: args.user_agent.as_ref(),
         }
     }
 }
@@ -92,8 +143,8 @@ fn prepare_args<T: Scopable + Serialize + Model>(rl: &Shell, filter: &Filter, pa
         .collect()
 }
 
-fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
-    for namespace in keyring.unauthorized_namespaces(&module) {
+pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
+    for namespace in keyring.unauthorized_namespaces(module) {
         let grant_access = if params.deny_keyring {
             false
         } else if params.grant_full_keyring || params.grants.contains(namespace) {
@@ -104,7 +155,7 @@ fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> R
         };
 
         if grant_access {
-            keyring.grant_access(&module, namespace.to_string());
+            keyring.grant_access(module, namespace.to_string());
             term::info(&format!("Granted access to {:?}", namespace));
         }
     }
@@ -133,10 +184,11 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
         Some(Source::Ports) => prepare_args::<Port>(rl, &filter, None),
         Some(Source::Netblocks) => prepare_args::<Netblock>(rl, &filter, None),
         Some(Source::CryptoAddrs(currency)) => prepare_args::<CryptoAddr>(rl, &filter, currency.as_ref()),
+        Some(Source::Notifications) => bail!("Notification modules can't be executed like this"),
         Some(Source::KeyRing(namespace)) => {
             let keyring = rl.keyring();
-            if keyring.is_access_granted(&module, &namespace) {
-                keyring.get_all_for(&namespace).into_iter()
+            if keyring.is_access_granted(module, namespace) {
+                keyring.get_all_for(namespace).into_iter()
                     .map(|key| {
                         let pretty = format!("{}:{}", key.namespace, key.access_key);
                         let arg = serde_json::to_value(key)?;
@@ -152,22 +204,22 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
 }
 
 pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);
 
     prepare_keyring(rl.keyring_mut(), &module, &params)?;
     let keyring = rl.keyring().request_keys(&module);
 
-    let dns_config = Resolver::from_system()?;
-    let proxy = rl.config().network.proxy.clone();
+    let dns_config = Resolver::from_system_v4()?;
 
     let args = get_args(rl, &module)?;
     for (arg, _pretty_arg, blobs) in args {
         let start_cmd = StartCommand::new(params.verbose,
                                           keyring.clone(),
                                           dns_config.clone(),
-                                          proxy.clone(),
+                                          proxy,
+                                          user_agent.clone(),
                                           options.clone(),
                                           module.clone(),
                                           arg,
@@ -180,15 +232,15 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
 }
 
 pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
+    let module = params.get_module(rl)?;
+    let proxy = params.get_proxy(rl);
+    let user_agent = params.get_user_agent(rl);
 
     prepare_keyring(rl.keyring_mut(), &module, &params)?;
     let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
-    let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);
+    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, proxy, user_agent, options);
     rl.signal_register().reset_ctrlc();
 
     if errors > 0 {
@@ -206,11 +258,11 @@ pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>)
 
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
-        ttl::reap_expired(rl.db())?;
+        ttl::reap_expired(rl)?;
         let options = match rl.options_mut() {
             Some(options) => options.clone(),
             _ => HashMap::new(),
         };
-        execute(rl, self.into(), options)
+        execute(rl, Params::from(&self), options)
     }
 }

src/cmd/select_cmd.rs

@@ -5,7 +5,6 @@ use crate::db::ttl;
 use crate::filters::{Target, Filter};
 use crate::shell::Shell;
 use serde::Serialize;
-use serde_json;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::models::*;
@@ -17,17 +16,26 @@ pub struct Args {
     #[structopt(subcommand)]
     subcommand: Target,
     /// Print json output
-    #[structopt(long="json", group="output")]
+    #[structopt(long, group="output")]
     json: bool,
+    /// Only print the value instead of the whole object
+    #[structopt(long, group="output")]
+    values: bool,
     /// Print paths to blobs
-    #[structopt(long="paths", group="output")]
+    #[structopt(long, group="output")]
     paths: bool,
+    /// Count rows returned
+    #[structopt(short="c", group="output")]
+    count: bool,
 }
 
+#[derive(PartialEq)]
 enum Output {
     Normal,
     Json,
+    Values,
     Paths,
+    Count,
 }
 
 struct Printer<'a, 'b> {
@@ -39,8 +47,12 @@ impl<'a, 'b> Printer<'a, 'b> {
     pub fn new(rl: &'a mut Shell<'b>, args: &Args) -> Printer<'a, 'b> {
         let output = if args.json {
             Output::Json
+        } else if args.values {
+            Output::Values
         } else if args.paths {
             Output::Paths
+        } else if args.count {
+            Output::Count
         } else {
             Output::Normal
         };
@@ -52,25 +64,35 @@ impl<'a, 'b> Printer<'a, 'b> {
     }
 
     pub fn select<T: Model + Detailed + Serialize>(&self, filter: &Filter) -> Result<()> {
-        for obj in self.rl.db().filter::<T>(&filter.parse_optional()?)? {
-            match self.output {
-                Output::Normal => println!("{}", obj.detailed(self.rl.db())?),
-                Output::Json => {
-                    let v = serde_json::to_string(&obj)?;
-                    println!("{}", v);
-                },
-                Output::Paths => {
-                    let blob = obj.blob()
-                        .ok_or_else(|| format_err!("This model isn't linked to blob storage"))?;
+        let query = self.rl.db().filter::<T>(&filter.parse_optional()?)?;
 
-                    let path = self.rl.blobs()
-                        .join(blob)?;
+        if self.output == Output::Count {
+            println!("{}", query.len());
+        } else {
+            for obj in query {
+                match self.output {
+                    Output::Normal => println!("{}", obj.detailed(self.rl.db())?),
+                    Output::Json => {
+                        let v = serde_json::to_string(&obj)?;
+                        println!("{}", v);
+                    },
+                    Output::Values => {
+                        println!("{}", obj.to_string());
+                    },
+                    Output::Paths => {
+                        let blob = obj.blob()
+                            .ok_or_else(|| format_err!("This model isn't linked to blob storage"))?;
 
-                    let path = path.to_str()
-                        .ok_or_else(|| format_err!("Path is invalid utf-8"))?;
+                        let path = self.rl.blobs()
+                            .join(blob)?;
 
-                    println!("{}", path);
-                },
+                        let path = path.to_str()
+                            .ok_or_else(|| format_err!("Path is invalid utf-8"))?;
+
+                        println!("{}", path);
+                    },
+                    Output::Count => unreachable!(),
+                }
             }
         }
 
@@ -80,24 +102,24 @@ impl<'a, 'b> Printer<'a, 'b> {
 
 impl Cmd for Args {
     fn run(self, rl: &mut Shell) -> Result<()> {
-        ttl::reap_expired(rl.db())?;
+        ttl::reap_expired(rl)?;
         let printer = Printer::new(rl, &self);
 
         match &self.subcommand {
-            Target::Domains(filter) => printer.select::<Domain>(&filter),
-            Target::Subdomains(filter) => printer.select::<Subdomain>(&filter),
-            Target::IpAddrs(filter) => printer.select::<IpAddr>(&filter),
-            Target::Urls(filter) => printer.select::<Url>(&filter),
-            Target::Emails(filter) => printer.select::<Email>(&filter),
-            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(&filter),
-            Target::Devices(filter) => printer.select::<Device>(&filter),
-            Target::Networks(filter) => printer.select::<Network>(&filter),
-            Target::Accounts(filter) => printer.select::<Account>(&filter),
-            Target::Breaches(filter) => printer.select::<Breach>(&filter),
-            Target::Images(filter) => printer.select::<Image>(&filter),
-            Target::Ports(filter) => printer.select::<Port>(&filter),
-            Target::Netblocks(filter) => printer.select::<Netblock>(&filter),
-            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(&filter),
+            Target::Domains(filter) => printer.select::<Domain>(filter),
+            Target::Subdomains(filter) => printer.select::<Subdomain>(filter),
+            Target::IpAddrs(filter) => printer.select::<IpAddr>(filter),
+            Target::Urls(filter) => printer.select::<Url>(filter),
+            Target::Emails(filter) => printer.select::<Email>(filter),
+            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(filter),
+            Target::Devices(filter) => printer.select::<Device>(filter),
+            Target::Networks(filter) => printer.select::<Network>(filter),
+            Target::Accounts(filter) => printer.select::<Account>(filter),
+            Target::Breaches(filter) => printer.select::<Breach>(filter),
+            Target::Images(filter) => printer.select::<Image>(filter),
+            Target::Ports(filter) => printer.select::<Port>(filter),
+            Target::Netblocks(filter) => printer.select::<Netblock>(filter),
+            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(filter),
         }
     }
 }

src/cmd/stats_cmd.rs

@@ -0,0 +1,199 @@
+use crate::args;
+use colored::Colorize;
+use crate::blobs::BlobStorage;
+use crate::cmd::Cmd;
+use crate::db::{ttl, Database};
+use crate::errors::*;
+use crate::models::*;
+use crate::shell::{self, Shell};
+use crate::workspaces;
+use humansize::{FileSize, file_size_opts};
+use separator::Separatable;
+use serde::{Serialize, Deserialize};
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug, Clone, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Exclude blob storage
+    #[structopt(short, long)]
+    short: bool,
+    /// Exclude categories that don't contain any structs
+    #[structopt(short, long)]
+    quiet: bool,
+    /// Show workspace statistics in json
+    #[structopt(short, long)]
+    json: bool,
+    /// Go through all workspaces
+    #[structopt(short, long)]
+    all: bool,
+}
+
+impl Args {
+    fn show_amount(&self, label: &str, count: usize, amount: &str) {
+        if self.quiet && count == 0 {
+            return;
+        }
+
+        let label = format!("{:20}", label);
+        let amount = format!("{:>20}", amount);
+
+        if count > 0 {
+            println!("{} {}", label.green(), amount.yellow());
+        } else {
+            println!("{} {}", label, amount);
+        }
+    }
+
+    fn show_count(&self, label: &str, count: usize) {
+        self.show_amount(label, count, &count.separated_string());
+    }
+}
+
+fn count_models<T: Model>(db: &Database) -> Result<usize> {
+    let query = db.list::<T>()?;
+    let count = query.len();
+    Ok(count)
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Stats {
+    workspace: String,
+    domains: usize,
+    subdomains: usize,
+    ipaddrs: usize,
+    urls: usize,
+    emails: usize,
+    phonenumbers: usize,
+    devices: usize,
+    networks: usize,
+    accounts: usize,
+    breaches: usize,
+    images: usize,
+    ports: usize,
+    netblocks: usize,
+    cryptoaddrs: usize,
+    activity: usize,
+    blobs: Option<BlobStats>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct BlobStats {
+    count: usize,
+    total_size: u64,
+    total_human_size: String,
+}
+
+impl Stats {
+    fn count(workspace: String, db: &Database) -> Result<Stats> {
+        Ok(Stats {
+            workspace,
+            domains: count_models::<Domain>(db)?,
+            subdomains: count_models::<Subdomain>(db)?,
+            ipaddrs: count_models::<IpAddr>(db)?,
+            urls: count_models::<Url>(db)?,
+            emails: count_models::<Email>(db)?,
+            phonenumbers: count_models::<PhoneNumber>(db)?,
+            devices: count_models::<Device>(db)?,
+            networks: count_models::<Network>(db)?,
+            accounts: count_models::<Account>(db)?,
+            breaches: count_models::<Breach>(db)?,
+            images: count_models::<Image>(db)?,
+            ports: count_models::<Port>(db)?,
+            netblocks: count_models::<Netblock>(db)?,
+            cryptoaddrs: count_models::<CryptoAddr>(db)?,
+            activity: Activity::count(db)?,
+            blobs: None,
+        })
+    }
+
+    fn add_blob_usage(&mut self, storage: &BlobStorage) -> Result<()> {
+        let blobs = storage.list()?;
+
+        let mut total_size = 0;
+        for blob in &blobs {
+            total_size += storage.stat(blob)?;
+        }
+
+        let total_human_size = total_size.file_size(file_size_opts::CONVENTIONAL)
+            .map_err(|e| format_err!("Failed to format size: {}", e))?;
+
+        self.blobs = Some(BlobStats {
+            count: blobs.len(),
+            total_size,
+            total_human_size,
+        });
+
+        Ok(())
+    }
+}
+
+impl Cmd for Args {
+    #[inline]
+    fn run(mut self, rl: &mut Shell) -> Result<()> {
+        if self.all {
+            self.all = false;
+            let mut first = true;
+
+            let config = rl.config();
+            for ws in workspaces::list()? {
+                if first {
+                    first = false;
+                } else if !self.json {
+                    println!();
+                }
+
+                let mut rl = shell::init(&args::Args {
+                    workspace: Some(ws),
+                    subcommand: None,
+                }, config, false)?;
+                self.clone().run(&mut rl)?;
+            }
+        } else {
+            ttl::reap_expired(rl)?;
+
+            let db = rl.db();
+
+            let workspace = rl.workspace();
+
+            if !self.json {
+                // print this early in case counting takes a while
+                println!("{:>41}", workspace.bold());
+            }
+
+            let mut stats = Stats::count(workspace.into(), db)?;
+            if !self.short {
+                stats.add_blob_usage(rl.blobs())?;
+            }
+
+            if self.json {
+                let stats = serde_json::to_string(&stats)?;
+                println!("{}", stats);
+            } else {
+                self.show_count("domains", stats.domains);
+                self.show_count("subdomains", stats.subdomains);
+                self.show_count("ipaddrs", stats.ipaddrs);
+                self.show_count("urls", stats.urls);
+                self.show_count("emails", stats.emails);
+                self.show_count("phonenumbers", stats.phonenumbers);
+                self.show_count("devices", stats.devices);
+                self.show_count("networks", stats.networks);
+                self.show_count("accounts", stats.accounts);
+                self.show_count("breaches", stats.breaches);
+                self.show_count("images", stats.images);
+                self.show_count("ports", stats.ports);
+                self.show_count("netblocks", stats.netblocks);
+                self.show_count("cryptoaddrs", stats.cryptoaddrs);
+                self.show_count("activity", stats.activity);
+
+                if let Some(blobs) = stats.blobs {
+                    self.show_count("blobs", blobs.count);
+                    self.show_amount("blobs (size)", blobs.total_size as usize, &blobs.total_human_size);
+                }
+            }
+        }
+
+        Ok(())
+    }
+}