Release v0.22.0

Merge pull request #210 from kpcyrd/bump
Update dependencies
2021-09-08 18:44:54 +02:00 · 2021-09-08 12:12:28 +00:00 · 2021-09-08 13:50:57 +02:00 · 2021-09-06 15:07:53 +00:00 · 2021-09-06 16:26:45 +02:00 · 2021-09-06 15:44:45 +02:00
141 changed files with 3391 additions and 2254 deletions
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,30 @@
+name: Docker
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+        - name: sn0int
+          file: Dockerfile
+        - name: registry
+          file: sn0int-registry/Dockerfile
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build the Docker image
+      run: docker build -t ${{ matrix.build.name }} -f ${{ matrix.build.file }} .
+    - name: Test the Docker image
+      run: docker run --rm ${{ matrix.build.name }} --help
+
+    - name: Show Docker images
+      run: docker images
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -0,0 +1,69 @@
+name: Rust
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 9 * * 1'
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install dependencies (apt)
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get install libsqlite3-dev libseccomp-dev libsodium-dev
+    - name: Install dependencies (brew)
+      if: matrix.os == 'macos-latest'
+      run: brew install pkg-config libsodium
+
+    - name: Build (sn0int)
+      run: cargo build --verbose
+    - name: Build (common)
+      run: cd sn0int-common && cargo build --verbose
+    - name: Build (std)
+      run: cd sn0int-std && cargo build --verbose
+    - name: Build (examples)
+      run: cargo build --verbose --examples
+
+    - name: Run tests (sn0int)
+      run: cargo test --verbose
+    - name: Run tests (sn0int, --ignored)
+      run: cargo test --verbose -- --ignored
+    - name: Run tests (common)
+      run: cd sn0int-common && cargo test --verbose
+    - name: Run tests (common, --ignored)
+      run: cd sn0int-common && cargo test --verbose -- --ignored
+    - name: Run tests (std)
+      run: cd sn0int-std && cargo test --verbose
+    - name: Run tests (std, --ignored)
+      run: cd sn0int-std && cargo test --verbose -- --ignored
+
+  notify:
+    # forks shouldn't notify
+    if: github.repository == 'kpcyrd/sn0int'
+    runs-on: ubuntu-latest
+    needs:
+    - build
+
+    steps:
+    - name: irc notify
+      uses: rectalogic/notify-irc@v1
+      with:
+        server: irc.hackint.org
+        channel: "#sn0int"
+        nickname: github-ci
+        message: '${{ github.repository }}#${{ github.run_id }}(${{ github.event_name }}): ${{ github.ref }}: tests completed: ${{ needs.build.result }} (https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})'
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,69 +0,0 @@
-language: rust
-
-# upload takes too long on windows and gets killed due to inactivity
-#cache: cargo
-
-matrix:
-  include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker-registry
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=std
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: osx
-    rust: stable
-    env:
-    - BUILD_MODE=std
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows test"
-  #- os: windows
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE="windows common"
-
-before_install:
- ci/setup.sh "$TRAVIS_OS_NAME"
-script:
- df -h
- ci/run.sh $BUILD_MODE
- df -h
-
-notifications:
-  irc:
-    template:
-    - "%{repository_slug}#%{build_number} (%{branch} - %{commit} : %{author}): %{message} (%{build_url})"
-    channels:
-    - "ircs://irc.hackint.org:6697/#sn0int"
-    use_notice: true
-    skip_join: true
-    #on_success: change # default: always
-    #on_failure: always # default: always
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.19.0"
+version = "0.22.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -9,9 +9,6 @@ categories = ["command-line-utilities"]
 readme = "README.md"
 edition = "2018"

-[badges]
-travis-ci = { repository = "kpcyrd/sn0int" }
-
 [workspace]
 members = ["sn0int-common",
           "sn0int-registry",
@@ -27,7 +24,6 @@ the results in a unified format for followup investigations."""
 section = "utils"
 priority = "optional"
 depends = "$auto, publicsuffix"
-build-depends = "pkg-config, libsqlite3-dev, libseccomp-dev, libsodium-dev, publicsuffix"
 assets = [
    ["target/release/sn0int", "usr/bin/", "755"],
 ]
@@ -36,74 +32,74 @@ assets = [
 sqlite-bundled = ["libsqlite3-sys/bundled"]

 [dependencies]
-sn0int-common = { version="0.11.0", path="sn0int-common" }
-sn0int-std = { version="=0.19.0", path="sn0int-std" }
-rustyline = "6.0"
+sn0int-common = { version="0.13.0", path="sn0int-common" }
+sn0int-std = { version="=0.22.0", path="sn0int-std" }
+rustyline = "9.0"
 log = "0.4"
-env_logger = "0.7"
+env_logger = "0.9"
 hlua-badtouch = "0.4"
 structopt = "0.3"
 failure = "0.1"
-rand = "0.7"
-colored = "1.6"
+rand = "0.8"
+colored = "2"
 lazy_static = "1.0"
 shellwords = "1.0"
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
-libsqlite3-sys = { version = "0.17.0", features = ["bundled-windows"] }
+libsqlite3-sys = { version = "0.22.0", features = ["bundled-windows"] }
 chrono = { version = "0.4", features = ["serde"] }
-dirs = "2.0"
+dirs-next = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.15.2"
-base64 = "0.12"
+chrootable-https = "0.16"
+base64 = "0.13"
 data-encoding = "2.1.2"
-serde_urlencoded = "0.6"
-serde = "1.0"
-serde_derive = "1.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_urlencoded = "0.7"
 serde_json = "1.0"
-crossbeam-channel = "0.4"
+crossbeam-channel = "0.5"
 ctrlc = "3.1"
-opener = "0.4"
+opener = "0.5"
 separator = "0.4"
 maplit = "1.0.1"
-sloppy-rfc4880 = "0.1.5"
+sloppy-rfc4880 = "0.2"
 regex = "1.0"
 toml = "0.5"
 threadpool = "1.7"
 atty = "0.2"
-semver = "0.9"
+semver = "1"
 bytes = "0.4"
 bytesize = "1.0"
-ipnetwork = "0.16"
-strum = "0.18"
-strum_macros = "0.18"
+ipnetwork = "0.18"
+strum = "0.21"
+strum_macros = "0.21"
 embedded-triple = "0.1.0"
+humansize = "1.1.0"

-digest = "0.8.0"
-md-5 = "0.8.0"
-sha-1 = "0.8.1"
-sha2 = "0.8.0"
-sha3 = "0.8.0"
-hmac = "0.7"
+digest = "0.9"
+md-5 = "0.9"
+sha-1 = "0.9"
+sha2 = "0.9"
+sha3 = "0.9"
+hmac = "0.11"

 walkdir = "2.2"
 nude = "0.3"
 glob = "0.3.0"
-os-version = "0.1.1"
+os-version = "0.2"

 [target.'cfg(target_os="linux")'.dependencies]
-caps = "0.3"
+caps = "0.5"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.14.0"
-nix = "0.17"
+syscallz = "0.16"
+nix = "0.22"

 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.4"
-unveil = "0.2.0"
+unveil = "0.3"

 [dev-dependencies]
 #boxxy = { path = "../boxxy-rs" }
-boxxy = "0.11"
+boxxy = "0.12"
 tempfile = "3.0"
--- a/4
+++ b/4
@@ -1,4 +1,4 @@
-FROM rust:alpine3.11
+FROM rust:alpine3.13
 ENV RUSTFLAGS="-C target-feature=-crt-static"
 RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
@@ -6,7 +6,7 @@ COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int

-FROM alpine:3.11
+FROM alpine:3.13
 RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
--- a/README.md
+++ b/README.md
@@ -1,7 +1,5 @@
-# sn0int [![Build Status][travis-img]][travis] [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]
+# sn0int [![crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs] [![irc.hackint.org:6697/#sn0int][irc-img]][irc] [![@sn0int][twitter-img]][twitter] [![@sn0int@chaos.social][mastodon-img]][mastodon] [![registry status][registry-img]][registry]

-[travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
-[travis]:       https://travis-ci.org/kpcyrd/sn0int
 [crates-img]:   https://img.shields.io/crates/v/sn0int.svg
 [crates]:       https://crates.io/crates/sn0int
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
@@ -50,6 +48,8 @@ For questions and support join us on IRC: [irc.hackint.org:6697/#sn0int](https:/

 ## Installation

+<a href="https://repology.org/project/sn0int/versions"><img align="right" src="https://repology.org/badge/vertical-allrepos/sn0int.svg" alt="Packaging status"></a>
+
 Archlinux

    pacman -S sn0int
@@ -100,7 +100,8 @@ For everything else please have a look at the [detailed list][1].
 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
  - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
  - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
-  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Debian &gt;= bullseye, Ubuntu &gt;= 20.04, Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-bullseye-ubuntu-20-04-kali)
+  - [Debian &lt;= buster, Ubuntu &lt;= 19.10](https://sn0int.readthedocs.io/en/latest/install.html#debian-buster-ubuntu-19-10)
  - [Fedora/CentOS/Redhat](https://sn0int.readthedocs.io/en/latest/install.html#fedora-centos-redhat)
  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
--- a/ci/integration.py
+++ b/ci/integration.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import subprocess
 from subprocess import DEVNULL, PIPE
+import shutil
+from pathlib import Path
 import tempfile
 import json
 import sys
@@ -98,6 +100,11 @@ def main(tempdir, binary):
        ('https://www.example.com/', 200),
    }

+    cache = Path.home() / '.cache' / 'sn0int'
+    if cache.exists():
+        print('[*] copying geoip files')
+        shutil.copytree(cache, tempdir + '/.cache/sn0int', dirs_exist_ok=True)
+
    print('[*] running geoip')
    sn0int(tempdir, binary, [
        'use geoip',
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -1,41 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    build)
-        cargo build --verbose
-        cargo build --verbose --examples
-        ;;
-    test)
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    common)
-        cd sn0int-common
-        cargo test --verbose
-        ;;
-    std)
-        cd sn0int-std
-        cargo test --verbose
-        cargo test --verbose -- --ignored
-        ;;
-    windows)
-        cargo build --verbose
-        cargo build --verbose --examples
-        ;;
-    boxxy)
-        cargo build --verbose --examples
-        if cat ci/boxxy_stage1.txt | RUST_LOG=boxxy cargo run --example boxxy; then
-            echo "SANDOX ERROR: should've crashed"
-            exit 1
-        fi
-        ;;
-    docker)
-        docker build -t sn0int .
-        docker images
-        docker run --rm sn0int --help
-        ;;
-    docker-registry)
-        docker build -t sn0int-registry -f sn0int-registry/Dockerfile .
-        docker images
-        ;;
-esac
--- a/ci/setup.sh
+++ b/ci/setup.sh
@@ -1,11 +0,0 @@
-#!/bin/sh
-set -exu
-case "$1" in
-    linux)
-        sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev libsodium-dev
-        ;;
-    osx)
-        brew install libsodium
-        ;;
-esac
--- a/docs/build.rst
+++ b/docs/build.rst
@@ -32,7 +32,7 @@ Debian/Ubuntu/Kali

 .. code-block:: bash

-    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix pkg-config

 .. warning::
   On a debian based system make sure you've installed rust with rustup.
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -19,20 +19,34 @@ Mac OSX

    $ brew install sn0int

-Debian/Ubuntu/Kali
------------------
+Debian >= bullseye, Ubuntu >= 20.04, Kali
+-----------------------------------------

 There are prebuilt packages signed by a debian maintainer. We can import the
 key for this repository out of the debian keyring.

 .. code-block:: bash

-    $ apt install debian-keyring
-    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | apt-key add -
-    $ apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
-    $ echo deb http://apt.vulns.sexy stable main > /etc/apt/sources.list.d/apt-vulns-sexy.list
-    $ apt update
-    $ apt install sn0int
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg kpcyrd@archlinux.org | sudo tee /etc/apt/trusted.gpg.d/apt-vulns-sexy.gpg
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int
+
+Debian <= buster, Ubuntu <= 19.10
+---------------------------------
+
+There are prebuilt packages signed by a debian maintainer. We can import the
+key for this repository out of the debian keyring.
+
+.. code-block:: bash
+
+    $ sudo apt install debian-keyring
+    $ gpg -a --export --keyring /usr/share/keyrings/debian-maintainers.gpg git@rxv.cc | sudo apt-key add -
+    $ sudo apt-key adv --keyserver keyserver.ubuntu.com --refresh-keys git@rxv.cc
+    $ echo deb http://apt.vulns.sexy stable main | sudo tee /etc/apt/sources.list.d/apt-vulns-sexy.list
+    $ sudo apt update
+    $ sudo apt install sn0int

 Fedora/CentOS/Redhat
 --------------------
--- a/docs/reference.rst
+++ b/docs/reference.rst
@@ -383,6 +383,10 @@ options are set. The following options are available:
  The request body that should be json encoded.
 ``form``
  The request body that should be form encoded.
+``follow_redirects``
+  Automatically follow redirects, up to the specified number. If set to 1, only
+  one redirect is going to be followed. Defaults to 0 so redirects aren't
+  followed.
 ``body``
  The raw request body as string.
 ``into_blob``
--- a/modules/harness/dummy-resolve.lua
+++ b/modules/harness/dummy-resolve.lua
@@ -0,0 +1,10 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: subdomains
+
+function run(arg)
+    db_update('subdomain', arg, {
+        resolvable=true,
+    })
+end
--- a/modules/harness/dummy-subdomains.lua
+++ b/modules/harness/dummy-subdomains.lua
@@ -0,0 +1,13 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: domains
+
+function run(arg)
+    for i=1, 20 do
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=http_mksession() .. '.' .. arg['value'],
+        })
+    end
+end
--- a/modules/harness/notify-ratelimit.lua
+++ b/modules/harness/notify-ratelimit.lua
@@ -0,0 +1,9 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+-- Source: notifications
+
+function run(arg)
+    ratelimit_throttle('foo', 3, 10000)
+    info('notication!: ' .. json_encode(arg))
+end
--- a/modules/harness/onion-http.lua
+++ b/modules/harness/onion-http.lua
@@ -1,5 +1,6 @@
 -- Description: Send a request to a hidden service
 -- Version: 0.1.0
+-- Stealth: passive
 -- License: GPL-3.0

 function run()
--- a/modules/harness/selftest.lua
+++ b/modules/harness/selftest.lua
@@ -1,5 +1,6 @@
 -- Description: basic selftest
 -- Version: 0.1.0
+-- Stealth: offline
 -- License: GPL-3.0

 function run()
--- a/modules/harness/stdin.lua
+++ b/modules/harness/stdin.lua
@@ -1,5 +1,6 @@
 -- Description: Read from stdin
 -- Version: 0.1.0
+-- Stealth: offline
 -- License: GPL-3.0

 function run()
--- a/modules/harness/xss.lua
+++ b/modules/harness/xss.lua
@@ -1,6 +1,7 @@
 -- Description: jaVasCript:/*-/*`/*\`/*'/*"/**/(/* */oNcliCk=alert() )//%0D%0A%0d%0a//</stYle/</titLe/</teXtarEa/</scRipt/--!>\x3csVg/<sVg/oNloAd=alert()//>\x3e
 -- Version: 0.1.0
 -- License: GPL-3.0
+-- Stealth: loud
 -- Source: domains

 function run()
--- a/sn0int-common/Cargo.toml
+++ b/sn0int-common/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-common"
-version = "0.11.0"
+version = "0.13.0"
 description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -8,9 +8,8 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"

 [dependencies]
-serde = "1.0"
-serde_derive = "1.0"
+serde = { version = "1.0", features=["derive"] }
 #rocket_failure_errors = { path = "../../../rocket_failure/rocket_failure_errors" }
 rocket_failure_errors = "0.2"
-failure = "0.1"
-nom = "5.0"
+anyhow = "1.0"
+nom = "7.0"
--- a/sn0int-common/src/api.rs
+++ b/sn0int-common/src/api.rs
@@ -1,4 +1,5 @@
 use crate::id::ModuleID;
+use serde::{Serialize, Deserialize};

 #[derive(Debug, Serialize, Deserialize)]
 pub struct WhoamiResponse {
--- a/sn0int-common/src/errors.rs
+++ b/sn0int-common/src/errors.rs
@@ -1,2 +1,2 @@
-pub use failure::{Error, ResultExt};
+pub use anyhow::{Error, Context, anyhow, format_err, bail};
 pub type Result<T> = ::std::result::Result<T, Error>;
--- a/sn0int-common/src/id.rs
+++ b/sn0int-common/src/id.rs
@@ -1,5 +1,4 @@
 use crate::errors::*;
-use nom;
 use serde::{de, Serialize, Serializer, Deserialize, Deserializer};
 use std::fmt;
 use std::result;
@@ -53,7 +52,7 @@ impl FromStr for ModuleID {

    fn from_str(s: &str) -> Result<ModuleID> {
        let (trailing, module) = module(s)
-            .map_err(|err| format_err!("Failed to parse module id: {:?}", err))?;
+            .map_err(|err| anyhow!("Failed to parse module id: {:?}", err))?;
        if !trailing.is_empty() {
            bail!("Trailing data in module id");
        }
--- a/sn0int-common/src/lib.rs
+++ b/sn0int-common/src/lib.rs
@@ -1,7 +1,3 @@
-#[macro_use] extern crate serde_derive;
-#[macro_use] extern crate failure;
-#[macro_use] extern crate nom;
-
 pub mod api;
 pub mod errors;
 pub use crate::errors::*;
--- a/sn0int-common/src/metadata/mod.rs
+++ b/sn0int-common/src/metadata/mod.rs
@@ -1,7 +1,13 @@
 use crate::errors::*;
-
+use nom::IResult;
+use nom::bytes::complete::{tag, take_until};
+use nom::combinator::map_res;
+use nom::multi::fold_many0;
+use serde::{Serialize, Deserialize};
 use std::str::FromStr;

+mod stealth;
+pub use self::stealth::Stealth;

 #[derive(Debug, PartialEq, Clone)]
 pub enum EntryType {
@@ -9,6 +15,9 @@ pub enum EntryType {
    Version,
    Source,
    KeyringAccess,
+    Stealth,
+    Author,
+    Repository,
    License,
 }

@@ -21,6 +30,9 @@ impl FromStr for EntryType {
            "Version" => Ok(EntryType::Version),
            "Source" => Ok(EntryType::Source),
            "Keyring-Access" => Ok(EntryType::KeyringAccess),
+            "Stealth" => Ok(EntryType::Stealth),
+            "Author" => Ok(EntryType::Author),
+            "Repository" => Ok(EntryType::Repository),
            "License" => Ok(EntryType::License),
            x => bail!("Unknown EntryType: {:?}", x),
        }
@@ -136,6 +148,9 @@ pub struct Metadata {
    pub version: String,
    pub source: Option<Source>,
    pub keyring_access: Vec<String>,
+    pub stealth: Stealth,
+    pub authors: Vec<String>,
+    pub repository: Option<String>,
    pub license: License,
 }

@@ -154,6 +169,9 @@ impl FromStr for Metadata {
                EntryType::Version => data.version = Some(v),
                EntryType::Source => data.source = Some(v),
                EntryType::KeyringAccess => data.keyring_access.push(v),
+                EntryType::Stealth => data.stealth = Some(v),
+                EntryType::Author => data.authors.push(v),
+                EntryType::Repository => data.repository = Some(v),
                EntryType::License => data.license = Some(v),
            }
        }
@@ -168,6 +186,9 @@ pub struct NewMetadata<'a> {
    pub version: Option<&'a str>,
    pub source: Option<&'a str>,
    pub keyring_access: Vec<&'a str>,
+    pub stealth: Option<&'a str>,
+    pub authors: Vec<&'a str>,
+    pub repository: Option<&'a str>,
    pub license: Option<&'a str>,
 }

@@ -182,6 +203,14 @@ impl<'a> NewMetadata<'a> {
        let keyring_access = self.keyring_access.into_iter()
            .map(String::from)
            .collect();
+        let stealth = match self.stealth {
+            Some(x) => x.parse()?,
+            _ => Stealth::Normal,
+        };
+        let authors = self.authors.into_iter()
+            .map(String::from)
+            .collect();
+        let repository = self.repository.map(String::from);
        let license = self.license.ok_or_else(|| format_err!("License is required"))?;
        let license = license.parse()?;

@@ -190,28 +219,40 @@ impl<'a> NewMetadata<'a> {
            version: version.to_string(),
            source,
            keyring_access,
+            stealth,
+            authors,
+            repository,
            license,
        })
    }
 }

-named!(metaline<&str, (EntryType, &str)>, do_parse!(
-    tag!("-- ") >>
-    name: map_res!(take_until!(": "), EntryType::from_str) >>
-    tag!(": ") >>
-    value: take_until!("\n") >>
-    tag!("\n") >>
-    (name, value)
-));
+fn metaline(input: &str) -> IResult<&str, (EntryType, &str)> {
+    let (input, _) = tag("-- ")(input)?;
+    let (input, name) = map_res(
+        take_until(": "),
+        EntryType::from_str
+    )(input)?;
+    let (input, _) = tag(": ")(input)?;
+    let (input, value) = take_until("\n")(input)?;
+    let (input, _) = tag("\n")(input)?;

-named!(metalines<&str, Vec<(EntryType, &str)>>, do_parse!(
-    lines: fold_many0!(metaline, Vec::new(), |mut acc: Vec<_>, item| {
-        acc.push(item);
-        acc
-    }) >>
-    tag!("\n") >>
-    (lines)
-));
+    Ok((input, (name, value)))
+}
+
+fn metalines(input: &str) -> IResult<&str, Vec<(EntryType, &str)>> {
+    let (input, lines) = fold_many0(
+        metaline,
+        Vec::new,
+        |mut acc: Vec<_>, item| {
+            acc.push(item);
+            acc
+        }
+    )(input)?;
+    let (input, _) = tag("\n")(input)?;
+
+    Ok((input, lines))
+}

 #[cfg(test)]
 mod tests {
@@ -230,6 +271,36 @@ mod tests {
            version: "1.0.0".to_string(),
            license: License::WTFPL,
            source: Some(Source::Domains),
+            stealth: Stealth::Normal,
+            authors: vec![],
+            repository: None,
+            keyring_access: Vec::new(),
+        });
+    }
+
+    #[test]
+    fn verify_much_metadata() {
+        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+-- Version: 1.0.0
+-- Source: domains
+-- Stealth: passive
+-- Author: kpcyrd <git at rxv dot cc>
+-- Author: kpcyrd's cat
+-- Repository: https://github.com/kpcyrd/sn0int
+-- License: WTFPL
+
+"#).expect("parse");
+        assert_eq!(metadata, Metadata {
+            description: "Hello world, this is my description".to_string(),
+            version: "1.0.0".to_string(),
+            license: License::WTFPL,
+            source: Some(Source::Domains),
+            stealth: Stealth::Passive,
+            authors: vec![
+                "kpcyrd <git at rxv dot cc>".to_string(),
+                "kpcyrd's cat".to_string(),
+            ],
+            repository: Some("https://github.com/kpcyrd/sn0int".to_string()),
            keyring_access: Vec::new(),
        });
    }
@@ -246,6 +317,9 @@ mod tests {
            version: "1.0.0".to_string(),
            license: License::WTFPL,
            source: None,
+            stealth: Stealth::Normal,
+            authors: vec![],
+            repository: None,
            keyring_access: Vec::new(),
        });
    }
--- a/sn0int-common/src/metadata/stealth.rs
+++ b/sn0int-common/src/metadata/stealth.rs
@@ -0,0 +1,53 @@
+use crate::errors::*;
+use serde::{Serialize, Deserialize};
+use std::str::FromStr;
+
+#[derive(Debug, Eq, PartialEq, PartialOrd, Clone, Serialize, Deserialize)]
+pub enum Stealth {
+    Loud,
+    Normal,
+    Passive,
+    Offline,
+}
+
+impl Stealth {
+    #[inline]
+    pub fn variants() -> &'static [&'static str] {
+        &[
+            "loud",
+            "normal",
+            "passive",
+            "offline",
+        ]
+    }
+
+    #[inline(always)]
+    fn as_num(&self) -> u8 {
+        match self {
+            Stealth::Loud => 3,
+            Stealth::Normal => 2,
+            Stealth::Passive => 1,
+            Stealth::Offline => 0,
+        }
+    }
+
+    #[inline]
+    pub fn equal_or_better(&self, other: &Self) -> bool {
+        self.as_num() <= other.as_num()
+    }
+}
+
+impl FromStr for Stealth {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Stealth> {
+        match s {
+            "loud" => Ok(Stealth::Loud),
+            // This is also the default level if none is provided
+            "normal" => Ok(Stealth::Normal),
+            "passive" => Ok(Stealth::Passive),
+            "offline" => Ok(Stealth::Offline),
+            x => bail!("Unknown stealth: {:?}", x),
+        }
+    }
+}
--- a/sn0int-registry/Cargo.toml
+++ b/sn0int-registry/Cargo.toml
@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"

 [dependencies]
-sn0int-common = { version="0.11.0", path="../sn0int-common" }
+sn0int-common = { version="0.13.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }
@@ -23,16 +23,15 @@ oauth2 = "2.0.0"
 failure = "0.1"
 url = "1.0"
 log = "0.4"
-semver = "0.9.0"
+semver = "1"
 lazy_static = "1"
 blake2 = "0.8.0"
 hex = "0.4"
 maplit = "1.0.1"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 syntect = "3.3"

-serde = "1.0"
-serde_derive = "1.0"
-serde_json = "1.0"
-
 dotenv = "0.15"
-env_logger = "0.7"
+env_logger = "0.9"
+structopt = "0.3.21"
--- a/sn0int-registry/src/assets.rs
+++ b/sn0int-registry/src/assets.rs
@@ -1,5 +1,5 @@
 use blake2::{Blake2b, Digest};
-
+use lazy_static::lazy_static;

 pub static FAVICON: &[u8] = include_bytes!("../assets/favicon.ico");
 pub static STYLESHEET: &[u8] = include_bytes!("../assets/style.css");
--- a/sn0int-registry/src/auth.rs
+++ b/sn0int-registry/src/auth.rs
@@ -1,8 +1,8 @@
+use crate::github;
 use diesel::pg::PgConnection;
 use oauth2::basic::BasicClient;
 use oauth2::prelude::*;
 use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl, TokenResponse};
-use crate::github;
 use sn0int_registry::errors::*;
 use sn0int_registry::models::AuthToken;
 use url::Url;
--- a/sn0int-registry/src/errors.rs
+++ b/sn0int-registry/src/errors.rs
@@ -1,4 +1,5 @@
-pub use failure::{Error, ResultExt};
+pub use failure::{Error, ResultExt, bail, format_err};
 pub type Result<T> = ::std::result::Result<T, Error>;
+pub use log::{debug, info};

 pub use rocket_failure::errors::*;
--- a/sn0int-registry/src/github.rs
+++ b/sn0int-registry/src/github.rs
@@ -1,10 +1,11 @@
+use serde::Deserialize;
 use sn0int_registry::errors::*;
-use reqwest;

 pub fn get_username(oauth_token: &str) -> Result<String> {
    let client = reqwest::Client::new();
    let mut resp = client.get("https://api.github.com/user")
        .header("Authorization", format!("token {}", oauth_token))
+        .header("User-Agent", "sn0int-registry")
        .send()
        .context("Failed to check access_token")?
        .error_for_status()
--- a/sn0int-registry/src/lib.rs
+++ b/sn0int-registry/src/lib.rs
@@ -1,7 +1,5 @@
 #[macro_use] extern crate diesel;
 #[macro_use] extern crate diesel_migrations;
-#[macro_use] extern crate failure;
-#[macro_use] extern crate serde_derive;

 pub mod db;
 pub mod errors;
--- a/sn0int-registry/src/main.rs
+++ b/sn0int-registry/src/main.rs
@@ -5,21 +5,16 @@
 #[macro_use] extern crate rocket;
 #[macro_use] extern crate rocket_contrib;
 #[macro_use] extern crate rocket_failure;
-#[macro_use] extern crate serde_derive;
-#[macro_use] extern crate log;
-#[macro_use] extern crate maplit;
-#[macro_use] extern crate lazy_static;
-#[macro_use] extern crate failure;

+use dotenv::dotenv;
 use rocket::fairing::AdHoc;
 use rocket::http::Header;
 use rocket_contrib::json::{Json, JsonValue};
 use rocket_contrib::templates::Template;
-use dotenv::dotenv;
-
-use std::env;
-use sn0int_registry::errors::*;
 use sn0int_registry::db;
+use sn0int_registry::errors::*;
+use std::env;
+use structopt::StructOpt;

 pub mod assets;
 pub mod auth;
@@ -49,7 +44,13 @@ fn internal_error() -> Json<JsonValue> {
    }))
 }

+#[derive(Debug, StructOpt)]
+pub struct Args {
+}
+
 fn run() -> Result<()> {
+    let _args = Args::from_args();
+
    dotenv().ok();

    let database_url = env::var("DATABASE_URL")
--- a/sn0int-registry/src/models.rs
+++ b/sn0int-registry/src/models.rs
@@ -4,6 +4,7 @@ use diesel::pg::PgConnection;
 use diesel::sql_types::BigInt;
 use diesel_full_text_search::{plainto_tsquery, TsQueryExtensions};
 use crate::schema::*;
+use serde::{Serialize, Deserialize};
 use std::collections::HashMap;
 use std::time::SystemTime;
 use sn0int_common::ModuleID;
--- a/sn0int-registry/src/routes/auth.rs
+++ b/sn0int-registry/src/routes/auth.rs
@@ -1,12 +1,13 @@
-use sn0int_registry::errors::*;
+use crate::assets::ASSET_REV;
 use crate::auth::Authenticator;
+use serde::{Serialize, Deserialize};
+use serde_json::Value;
 use sn0int_registry::db;
+use sn0int_registry::errors::*;
 use rocket::request::Form;
 use rocket::response::Redirect;
 use rocket_contrib::templates::Template;
-use crate::assets::ASSET_REV;
-use serde_json::{self, Value};
-
+use maplit::hashmap;

 #[get("/?<auth..>")]
 pub fn get(auth: Form<OAuth>) -> Template {
--- a/sn0int-registry/src/routes/badges.rs
+++ b/sn0int-registry/src/routes/badges.rs
@@ -1,3 +1,4 @@
+use serde::Serialize;
 use sn0int_registry::errors::*;
 use sn0int_registry::db;
 use sn0int_registry::models::*;
--- a/sn0int-registry/src/routes/html.rs
+++ b/sn0int-registry/src/routes/html.rs
@@ -1,10 +1,17 @@
-use sn0int_registry::errors::*;
 use crate::assets::*;
+use lazy_static::lazy_static;
 use sn0int_registry::db;
+use sn0int_registry::errors::*;
 use sn0int_registry::models::*;
-use rocket_contrib::templates::Template;
 use std::cmp::Ordering;
+use syntect::html::ClassedHTMLGenerator;
+use syntect::parsing::{SyntaxSet, SyntaxReference};
+use rocket_contrib::templates::Template;

+lazy_static! {
+    pub static ref SYNTAX_SET: SyntaxSet = SyntaxSet::load_defaults_newlines();
+    pub static ref SYNTAX: &'static SyntaxReference = SYNTAX_SET.find_syntax_by_extension("lua").unwrap();
+}

 #[get("/")]
 pub fn index(connection: db::Connection) -> ApiResult<Template> {
@@ -30,14 +37,6 @@ pub fn index(connection: db::Connection) -> ApiResult<Template> {
    })))
 }

-use syntect::html::ClassedHTMLGenerator;
-use syntect::parsing::{SyntaxSet, SyntaxReference};
-
-lazy_static! {
-    pub static ref SYNTAX_SET: SyntaxSet = SyntaxSet::load_defaults_newlines();
-    pub static ref SYNTAX: &'static SyntaxReference = SYNTAX_SET.find_syntax_by_extension("lua").unwrap();
-}
-
 #[get("/r/<author>/<name>")]
 pub fn details(author: String, name: String,connection: db::Connection) -> ApiResult<Template> {
    let module = Module::find(&author, &name, &connection)
--- a/sn0int-std/Cargo.toml
+++ b/sn0int-std/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-std"
-version = "0.19.0"
+version = "0.22.0"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
@@ -14,43 +14,42 @@ log = "0.4"
 failure = "0.1"
 hlua-badtouch = "0.4"
 tokio = "0.1.14"
-rand = "0.7"
+rand = "0.8"
 regex = "1.0"
-rustls = { version="0.16", features=["dangerous_configuration"] }
-serde = "1.0"
-serde_derive = "1.0"
+rustls = { version="0.18", features=["dangerous_configuration"] }
+serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-serde_urlencoded = "0.6"
+serde_urlencoded = "0.7"
 webpki = "0.21"
-webpki-roots = "0.19.0"
-ct-logs = "0.6"
+webpki-roots = "0.21"
+ct-logs = "0.7"
 #chrootable-https = { path = "../../chrootable-https" }
-chrootable-https = "0.15.2"
+chrootable-https = "0.16"
 http = "0.2"
 bufstream = "0.1.4"
-pem = "0.7"
+pem = "0.8"
 url = "2.0"
-tungstenite = { version = "0.10.1", default-features = false }
+tungstenite = { version = "0.13", default-features = false }
 kuchiki = "0.8.0"
-maxminddb = "0.14"
-x509-parser = "0.7"
-der-parser = "3.0"
-publicsuffix = { version="1.5", default-features=false }
+maxminddb = "0.21"
+x509-parser = "0.10"
+der-parser = "5"
+publicsuffix = { version="2", default-features=false }
 xml-rs = "0.8"
-geo = "0.13"
+geo = "0.18"
 bytes = "0.4"
-base64 = "0.12"
+base64 = "0.13"
 chrono = { version = "0.4", features = ["serde"] }
-mqtt-protocol = "0.8.1"
+mqtt-protocol = "0.11"
 sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }

 image = "0.23.0"
 kamadak-exif = "0.5.1"

-bs58 = "0.3"
-digest = "0.8.0"
-blake2 = "0.8.0"
+bs58 = "0.4"
+digest = "0.9"
+blake2 = "0.9"

 [dev-dependencies]
-env_logger = "0.7"
+env_logger = "0.9"
 maplit = "1.0.1"
--- a/sn0int-std/src/blobs.rs
+++ b/sn0int-std/src/blobs.rs
@@ -1,6 +1,6 @@
 use bytes::Bytes;
 use blake2::VarBlake2b;
-use digest::{Input, VariableOutput};
+use digest::{Update, VariableOutput};
 use serde::ser::{Serialize, Serializer};
 use serde::de::{self, Deserialize, Deserializer};
 use std::result;
@@ -22,8 +22,9 @@ impl Blob {

    pub fn hash(bytes: &[u8]) -> String {
        let mut h = VarBlake2b::new(32).unwrap();
-        h.input(bytes);
-        Self::encode_hash(&h.vec_result())
+        h.update(bytes);
+        let output = h.finalize_boxed();
+        Self::encode_hash(&output)
    }

    #[inline]
--- a/sn0int-std/src/crt.rs
+++ b/sn0int-std/src/crt.rs
@@ -1,78 +1,11 @@
 use crate::errors::*;
-
-use x509_parser;
-use der_parser::der::DerObject;
-use der_parser::ber::{BerObjectContent, BerTag};
-use der_parser::oid::Oid;
+use serde::{Serialize, Deserialize};
 use std::collections::HashSet;
-use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
-
-
-#[derive(Debug, PartialEq)]
-pub enum AlternativeName {
-    DnsName(String),
-    Email(String),
-    IpAddr(IpAddr),
-}
-
-pub fn san_extension(i: &[u8]) -> Result<Vec<AlternativeName>> {
-    let (rem, seq) = der_parser::der::parse_der_sequence(i)
-        .map_err(|_| format_err!("Failed to parse san extension"))?;
-
-    if !rem.is_empty() {
-        bail!("san extension has trailing garbage");
-    }
-
-    debug!("Decoded sequence: {:?}", seq);
-    if let BerObjectContent::Sequence(seq) = seq.content {
-        seq.into_iter()
-            .map(san_value)
-            .collect()
-    } else {
-        bail!("Expected der sequence");
-    }
-}
-
-pub fn san_value(o: DerObject) -> Result<AlternativeName> {
-    debug!("DER object in SAN extension: {:?}", o);
-
-    match (o.class, o.tag, &o.content) {
-        (2, BerTag::Integer, BerObjectContent::Unknown(BerTag::Integer, value)) => san_value_dns(value),
-        (2, BerTag::Boolean, BerObjectContent::Unknown(BerTag::Boolean, value)) => san_value_email(value),
-        (2, BerTag::ObjDescriptor, BerObjectContent::Unknown(BerTag::ObjDescriptor, value)) => san_value_ipaddr(value),
-        _ => bail!("Unexpected object: {:?}", o),
-    }
-}
-
-pub fn san_value_dns(v: &[u8]) -> Result<AlternativeName> {
-    debug!("Reading as dns name: {:?}", v);
-    String::from_utf8(v.to_vec())
-        .map(AlternativeName::DnsName)
-        .map_err(Error::from)
-}
-
-pub fn san_value_email(v: &[u8]) -> Result<AlternativeName> {
-    debug!("Reading as email: {:?}", v);
-    String::from_utf8(v.to_vec())
-        .map(AlternativeName::Email)
-        .map_err(Error::from)
-}
-
-pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
-    debug!("Reading as ipaddr: {:?}", v);
-    match v.len() {
-        4  => Ok(AlternativeName::IpAddr(Ipv4Addr::from([
-            v[0], v[1], v[2], v[3],
-        ]).into())),
-        16 => Ok(AlternativeName::IpAddr(Ipv6Addr::from([
-            v[0],  v[1],  v[2],  v[3],
-            v[4],  v[5],  v[6],  v[7],
-            v[8],  v[9],  v[10], v[11],
-            v[12], v[13], v[14], v[15],
-        ]).into())),
-        _ => Err(format_err!("Invalid ipaddr")),
-    }
-}
+use std::net::IpAddr;
+use x509_parser::x509::X509Version;
+use x509_parser::certificate::X509Certificate;
+use x509_parser::extensions::{GeneralName, ParsedExtension};
+use x509_parser::prelude::*;

 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
@@ -83,7 +16,7 @@ pub struct Certificate {

 impl Certificate {
    pub fn parse_pem(crt: &str) -> Result<Certificate> {
-        let pem = match x509_parser::pem::pem_to_der(crt.as_bytes()) {
+        let pem = match x509_parser::pem::parse_x509_pem(crt.as_bytes()) {
            Ok((remaining, pem)) => {
                if !remaining.is_empty() {
                    bail!("input cert has trailing garbage");
@@ -99,12 +32,12 @@ impl Certificate {
    }

    pub fn from_bytes(crt: &[u8]) -> Result<Certificate> {
-        let crt = match x509_parser::parse_x509_der(&crt) {
+        let crt = match X509Certificate::from_der(crt) {
            Ok((remaining, der)) => {
                if !remaining.is_empty() {
                    bail!("input cert has trailing garbage");
                }
-                if der.tbs_certificate.version != 2 {
+                if der.tbs_certificate.version != X509Version::V3 {
                    bail!("unexpected certificate version");
                }
                der
@@ -116,35 +49,45 @@ impl Certificate {
        let mut valid_emails = HashSet::new();
        let mut valid_ipaddrs = HashSet::new();

-        for x in crt.tbs_certificate.subject.rdn_seq {
-            for y in x.set {
-                // CommonName
-                if y.attr_type != Oid::from(&[2, 5, 4, 3]) {
-                    continue;
-                }
-
-                if let Ok(x) = y.attr_value.content.as_slice() {
-                    let value = String::from_utf8(x.to_vec())?;
-                    info!("Found CN in Subject: {:?}", value);
-                    valid_names.insert(value);
-                }
+        for attr in crt.subject().iter_common_name() {
+            if let Ok(cn) = attr.as_str() {
+                info!("Found CN in Subject: {:?}", cn);
+                valid_names.insert(cn.to_string());
            }
        }

-        for x in crt.tbs_certificate.extensions {
-            if x.oid != Oid::from(&[2, 5, 29, 17]) {
-                continue;
-            }
-
-            debug!("Found san extension: {:?}", x.value);
-            let values = san_extension(x.value)?;
-
-            for v in values {
-                match v {
-                    AlternativeName::DnsName(v) => valid_names.insert(v),
-                    AlternativeName::Email(v) => valid_emails.insert(v),
-                    AlternativeName::IpAddr(v) => valid_ipaddrs.insert(v),
-                };
+        for ext in crt.tbs_certificate.extensions() {
+            if let ParsedExtension::SubjectAlternativeName(san) = ext.parsed_extension() {
+                for name in &san.general_names {
+                    debug!("Certificate is valid for {:?}", name);
+                    match name {
+                        GeneralName::DNSName(v) => {
+                            valid_names.insert(v.to_string());
+                        },
+                        GeneralName::RFC822Name(v) => {
+                            valid_emails.insert(v.to_string());
+                        },
+                        GeneralName::IPAddress(v) => {
+                            let ip = match v.len() {
+                                4 => Some(IpAddr::from([v[0], v[1], v[2], v[3]])),
+                                16 => Some(IpAddr::from([
+                                    v[0], v[1], v[2], v[3],
+                                    v[4], v[5], v[6], v[7],
+                                    v[8], v[9], v[10], v[11],
+                                    v[12], v[13], v[14], v[15],
+                                ])),
+                                _ => {
+                                    info!("Certificate is valid for invalid ip address: {:?}", v);
+                                    None
+                                },
+                            };
+                            if let Some(ip) = ip {
+                                valid_ipaddrs.insert(ip);
+                            }
+                        },
+                        _ => (),
+                    }
+                }
            }
        }

@@ -162,7 +105,6 @@ impl Certificate {
 #[cfg(test)]
 mod tests {
    use super::*;
-    use der_parser::parse_der;

    #[test]
    fn test_parse_pem_github() {
@@ -260,19 +202,6 @@ ZkZZmqNn2Q8=
        });
    }

-    #[test]
-    fn test_san_extension() {
-        let ext = san_extension(&[48, 28,
-                130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109,
-                130, 14, 119, 119, 119, 46, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109
-            ])
-            .expect("Failed to parse extension");
-        assert_eq!(ext, vec![
-            AlternativeName::DnsName(String::from("github.com")),
-            AlternativeName::DnsName(String::from("www.github.com")),
-        ]);
-    }
-
    #[test]
    fn test_long_san_extension() {
        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
@@ -372,48 +301,6 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
        });
    }

-    #[test]
-    fn test_san_value_dns() {
-        let (rem, v) = parse_der(&[130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
-            .expect("Failed to parse san value");
-        assert!(rem.is_empty());
-        println!("{:?}", v);
-        assert_eq!(v, DerObject {
-            class: 2,
-            structured: 0,
-            tag: BerTag::Integer,
-            content: BerObjectContent::Unknown(BerTag::Integer, &[103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
-        });
-        let content = match v.content {
-            BerObjectContent::Unknown(BerTag::Integer, v) => v,
-            _ => panic!("Wrong BerObjectContent"),
-        };
-        let v = san_value_dns(content)
-            .expect("Failed to process san value");
-        assert_eq!(v, AlternativeName::DnsName(String::from("github.com")));
-    }
-
-    #[test]
-    fn test_san_value_ipaddr() {
-        let (rem, v) = parse_der(&[135, 4, 1, 1, 1, 1])
-            .expect("Failed to parse san value");
-        assert!(rem.is_empty());
-        println!("{:?}", v);
-        assert_eq!(v, DerObject {
-            class: 2,
-            structured: 0,
-            tag: BerTag::ObjDescriptor,
-            content: BerObjectContent::Unknown(BerTag::ObjDescriptor, &[1, 1, 1, 1])
-        });
-        let content = match v.content {
-            BerObjectContent::Unknown(BerTag::ObjDescriptor, v) => v,
-            _ => panic!("Wrong BerObjectContent"),
-        };
-        let v = san_value_ipaddr(content)
-            .expect("Failed to process san value");
-        assert_eq!(v, AlternativeName::IpAddr("1.1.1.1".parse().unwrap()));
-    }
-
    #[test]
    fn test_san_email() {
        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
--- a/sn0int-std/src/crypto.rs
+++ b/sn0int-std/src/crypto.rs
@@ -22,7 +22,7 @@ pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
    let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
        .ok_or_else(|| format_err!("Nonce has wrong length"))?;
    let ciphertext = &encrypted[secretbox::NONCEBYTES..];
-    let plain = secretbox::open(&ciphertext, &nonce, &key)
+    let plain = secretbox::open(ciphertext, &nonce, &key)
        .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
    Ok(plain)
 }
--- a/sn0int-std/src/engine/structs.rs
+++ b/sn0int-std/src/engine/structs.rs
@@ -2,9 +2,6 @@ use crate::errors::*;
 use crate::hlua::{AnyHashableLuaValue, AnyLuaValue, AnyLuaString};
 use std::collections::{self, HashMap};
 use crate::json::LuaJsonValue;
-use serde;
-use serde_json;
-

 pub fn from_lua<T>(x: LuaJsonValue) -> Result<T>
    where for<'de> T: serde::Deserialize<'de>
@@ -89,16 +86,16 @@ impl From<Vec<(AnyLuaValue, AnyLuaValue)>> for LuaMap {
    }
 }

-impl Into<HashMap<AnyHashableLuaValue, AnyLuaValue>> for LuaMap {
-    fn into(self: LuaMap) -> HashMap<AnyHashableLuaValue, AnyLuaValue> {
-        self.0
+impl From<LuaMap> for HashMap<AnyHashableLuaValue, AnyLuaValue> {
+    fn from(map: LuaMap) -> Self {
+        map.0
    }
 }

-impl Into<AnyLuaValue> for LuaMap {
-    fn into(self: LuaMap) -> AnyLuaValue {
+impl From<LuaMap> for AnyLuaValue {
+    fn from(map: LuaMap) -> AnyLuaValue {
        AnyLuaValue::LuaArray(
-            self.into_iter()
+            map.into_iter()
                .filter_map(|(k, v)| {
                    match k {
                        AnyHashableLuaValue::LuaString(x) => Some((AnyLuaValue::LuaString(x), v)),
@@ -139,9 +136,9 @@ impl LuaList {
    }
 }

-impl Into<AnyLuaValue> for LuaList {
-    fn into(self: LuaList) -> AnyLuaValue {
-        AnyLuaValue::LuaArray(self.0)
+impl From<LuaList> for AnyLuaValue {
+    fn from(list: LuaList) -> AnyLuaValue {
+        AnyLuaValue::LuaArray(list.0)
    }
 }

@@ -152,7 +149,7 @@ pub fn byte_array(bytes: AnyLuaValue) -> Result<Vec<u8>> {
        AnyLuaValue::LuaArray(bytes) => {
            Ok(bytes.into_iter()
                .map(|num| match num.1 {
-                    AnyLuaValue::LuaNumber(num) if num <= 255.0 && num >= 0.0 && (num % 1.0 == 0.0) =>
+                    AnyLuaValue::LuaNumber(num) if (0.0..=255.0).contains(&num) && (num % 1.0 == 0.0) =>
                            Ok(num as u8),
                    AnyLuaValue::LuaNumber(num) =>
                            Err(format_err!("number is out of range: {:?}", num)),
--- a/sn0int-std/src/geo.rs
+++ b/sn0int-std/src/geo.rs
@@ -1,10 +1,9 @@
 use crate::errors::*;
-
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
 use geo::{LineString, Polygon, Coordinate};
 use geo::prelude::*;
-
+use serde::Deserialize;

 #[derive(Debug, Deserialize)]
 pub struct Point {
--- a/sn0int-std/src/geoip/models.rs
+++ b/sn0int-std/src/geoip/models.rs
@@ -1,8 +1,8 @@
 use crate::errors::*;
 use maxminddb::geoip2;
+use serde::Serialize;
 use std::collections::BTreeMap;

-
 fn from_geoip_model_names(names: Option<BTreeMap<&str, &str>>) -> Option<String> {
    names?.get("en").map(|x| x.to_string())
 }
--- a/sn0int-std/src/gfx/exif.rs
+++ b/sn0int-std/src/gfx/exif.rs
@@ -1,9 +1,7 @@
 use crate::errors::*;
-
-use exif;
+use serde::{Serialize, Deserialize};
 use std::io;

-
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Location {
    latitude: f64,
--- a/sn0int-std/src/gfx/mod.rs
+++ b/sn0int-std/src/gfx/mod.rs
@@ -45,9 +45,9 @@ impl ImageFormat {
    }
 }

-impl Into<image::ImageFormat> for ImageFormat {
-    fn into(self) -> image::ImageFormat {
-        match self {
+impl From<ImageFormat> for image::ImageFormat {
+    fn from(format: ImageFormat) -> image::ImageFormat {
+        match format {
            ImageFormat::Png => image::ImageFormat::Png,
            ImageFormat::Jpeg => image::ImageFormat::Jpeg,
            ImageFormat::Gif => image::ImageFormat::Gif,
@@ -94,10 +94,10 @@ pub fn guess_format(buf: &[u8]) -> Result<ImageFormat> {
 }

 pub fn load(buf: &[u8]) -> Result<Image> {
-    let img_format = image::guess_format(&buf)?;
+    let img_format = image::guess_format(buf)?;
    let format = ImageFormat::try_from(&img_format)?;

-    let image = image::load_from_memory_with_format(&buf, img_format)?;
+    let image = image::load_from_memory_with_format(buf, img_format)?;

    Ok(Image {
        image,
--- a/sn0int-std/src/html.rs
+++ b/sn0int-std/src/html.rs
@@ -1,11 +1,8 @@
+use crate::engine::structs::LuaMap;
 use crate::errors::*;
-
-use kuchiki;
+use crate::hlua::AnyLuaValue;
 use kuchiki::traits::TendrilSink;
 use std::collections::HashMap;
-use crate::hlua::AnyLuaValue;
-use crate::engine::structs::LuaMap;
-

 #[derive(Debug, PartialEq)]
 pub struct Element {
@@ -14,13 +11,13 @@ pub struct Element {
    pub html: String,
 }

-impl Into<AnyLuaValue> for Element {
-    fn into(self) -> AnyLuaValue {
+impl From<Element> for AnyLuaValue {
+    fn from(elem: Element) -> AnyLuaValue {
        let mut map = LuaMap::new();

-        map.insert_str("text", self.text);
-        map.insert("attrs", LuaMap::from(self.attrs));
-        map.insert_str("html", self.html);
+        map.insert_str("text", elem.text);
+        map.insert("attrs", LuaMap::from(elem.attrs));
+        map.insert_str("html", elem.html);

        map.into()
    }
--- a/sn0int-std/src/json.rs
+++ b/sn0int-std/src/json.rs
@@ -1,6 +1,5 @@
 use crate::errors::*;

-use std::iter::FromIterator;
 use std::collections::HashMap;
 use crate::hlua::AnyLuaValue;
 use serde_json::{self, Deserializer, Value, Number, Map};
@@ -22,7 +21,7 @@ pub fn encode(v: AnyLuaValue) -> Result<String> {
 }

 pub fn decode_stream(x: &str) -> Result<Vec<AnyLuaValue>> {
-    let stream = Deserializer::from_str(&x).into_iter::<Value>();
+    let stream = Deserializer::from_str(x).into_iter::<Value>();

    let list = stream
        .filter_map(|x| x.ok())
@@ -35,11 +34,7 @@ pub fn decode_stream(x: &str) -> Result<Vec<AnyLuaValue>> {
 pub fn lua_array_is_list(array: &[(AnyLuaValue, AnyLuaValue)]) -> bool {
    if !array.is_empty() {
        let first = &array[0];
-        if let AnyLuaValue::LuaNumber(_) = first.0 {
-            true
-        } else {
-            false
-        }
+        matches!(first.0, AnyLuaValue::LuaNumber(_))
    } else {
        // true // TODO: this breaks unserialize
        false
@@ -56,9 +51,9 @@ pub enum LuaJsonValue {
    Object(HashMap<String, LuaJsonValue>),
 }

-impl Into<AnyLuaValue> for LuaJsonValue {
-    fn into(self) -> AnyLuaValue {
-        match self {
+impl From<LuaJsonValue> for AnyLuaValue {
+    fn from(value: LuaJsonValue) -> AnyLuaValue {
+        match value {
            LuaJsonValue::Null => AnyLuaValue::LuaNil,
            LuaJsonValue::Bool(v) => AnyLuaValue::LuaBoolean(v),
            // TODO: not sure if this might fail
@@ -115,9 +110,9 @@ impl From<AnyLuaValue> for LuaJsonValue {
    }
 }

-impl Into<serde_json::Value> for LuaJsonValue {
-    fn into(self) -> serde_json::Value {
-        match self {
+impl From<LuaJsonValue> for serde_json::Value {
+    fn from(value: LuaJsonValue) -> serde_json::Value {
+        match value {
            LuaJsonValue::Null => Value::Null,
            LuaJsonValue::Bool(v) => Value::Bool(v),
            LuaJsonValue::Number(v) => Value::Number(v),
@@ -126,9 +121,10 @@ impl Into<serde_json::Value> for LuaJsonValue {
                .map(|x| x.into())
                .collect()
            ),
-            LuaJsonValue::Object(v) => Value::Object(Map::from_iter(v.into_iter()
+            LuaJsonValue::Object(v) => Value::Object(v.into_iter()
                .map(|(k, v)| (k, v.into()))
-            )),
+                .collect::<Map<_, _>>()
+            ),
        }
    }
 }
@@ -144,9 +140,10 @@ impl From<serde_json::Value> for LuaJsonValue {
                .map(|x| x.into())
                .collect()
            ),
-            Value::Object(v) => LuaJsonValue::Object(HashMap::from_iter(v.into_iter()
+            Value::Object(v) => LuaJsonValue::Object(v.into_iter()
                .map(|(k, v)| (k, v.into()))
-            )),
+                .collect::<HashMap<_, _>>()
+            ),
        }
    }
 }
--- a/sn0int-std/src/lib.rs
+++ b/sn0int-std/src/lib.rs
@@ -1,4 +1,3 @@
-#[macro_use] extern crate serde_derive;
 use hlua_badtouch as hlua;

 pub mod blobs;
--- a/sn0int-std/src/mqtt.rs
+++ b/sn0int-std/src/mqtt.rs
@@ -8,7 +8,8 @@ use mqtt::{TopicFilter, QualityOfService};
 use mqtt::control::ConnectReturnCode;
 use mqtt::control::fixed_header::FixedHeaderError;
 use mqtt::encodable::{Encodable, Decodable};
-use mqtt::packet::{Packet, VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
+use mqtt::packet::{VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
+use serde::{Serialize, Deserialize};
 use std::convert::TryFrom;
 use std::io;
 use std::net::SocketAddr;
@@ -46,7 +47,7 @@ impl MqttClient {
            stream,
        };

-        let mut pkt = ConnectPacket::new("MQTT", "sn0int");
+        let mut pkt = ConnectPacket::new("sn0int");
        pkt.set_user_name(options.username.clone());
        pkt.set_password(options.password.clone());

@@ -176,7 +177,7 @@ impl TryFrom<VariablePacket> for Pkt {
            VariablePacket::ConnackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
            VariablePacket::PublishPacket(pkt) => Ok(Pkt::Publish(Publish {
                topic: pkt.topic_name().to_string(),
-                body: pkt.payload(),
+                body: pkt.payload().to_vec(),
            })),
            VariablePacket::PubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
            VariablePacket::PubrecPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
--- a/sn0int-std/src/psl.rs
+++ b/sn0int-std/src/psl.rs
@@ -1,11 +1,12 @@
-use crate::errors::*;
-
 use chrootable_https::Client;
+use crate::errors::*;
+use crate::lazy::LazyInit;
+use publicsuffix::{List, Psl as _};
 use std::fs::{self, File};
+use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use crate::lazy::LazyInit;
-
+use std::str::FromStr;

 #[derive(Debug, PartialEq)]
 pub struct DnsName {
@@ -72,11 +73,15 @@ impl PslReader {
 impl LazyInit<Arc<Psl>> for PslReader {
    fn initialize(self) -> Result<Arc<Psl>> {
        let list = match self {
-            PslReader::Reader(file) => publicsuffix::List::from_reader(file),
-            PslReader::String(s) => publicsuffix::List::from_str(&s),
+            PslReader::Reader(mut file) => {
+                let mut buf = String::new();
+                file.read_to_string(&mut buf)?;
+                buf
+            },
+            PslReader::String(s) => s,
        };

-        let list = list
+        let list = List::from_str(&list)
            .map_err(|e| format_err!("Failed to load public suffix list: {}", e))?;

        Ok(Arc::new(Psl {
@@ -87,32 +92,28 @@ impl LazyInit<Arc<Psl>> for PslReader {

 #[derive(Debug)]
 pub struct Psl {
-    list: publicsuffix::List,
+    list: List,
 }

 impl Psl {
    pub fn parse_dns_name(&self, name: &str) -> Result<DnsName> {
-        let dns_name = self.list.parse_dns_name(name)
-            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;
+        let bytes = name.as_bytes();

-        let (root, suffix) = if let Some(domain) = dns_name.domain() {
-            let root = domain.to_string();
-            if let Some(suffix) = domain.suffix() {
-                (root, suffix.to_string())
-            } else {
-                // XXX: not sure if this is reachable
-                (root.clone(), root)
-            }
+        let suffix = self.list.suffix(bytes)
+            .ok_or_else(|| format_err!("Failed to detect suffix"))?;
+        let suffix = String::from_utf8(suffix.as_bytes().to_vec())?;
+
+        let root = if let Some(root) = self.list.domain(bytes) {
+            String::from_utf8(root.as_bytes().to_vec())?
        } else {
            // this is technically a tld, but support eg. a.prod.fastly.net anyway
-            // XXX: consider forcing domain to be in self.list.private()
-            (name.to_string(), name.to_string())
+            name.to_string()
        };

-        let fulldomain = if root.as_str() != name {
-            Some(name.to_string())
-        } else {
+        let fulldomain = if root == name {
            None
+        } else {
+            Some(name.to_string())
        };

        Ok(DnsName {
--- a/sn0int-std/src/ratelimits.rs
+++ b/sn0int-std/src/ratelimits.rs
@@ -1,4 +1,5 @@
 use chrono::prelude::*;
+use serde::{Serialize, Deserialize};
 use std::collections::HashMap;
 use std::result;
 use std::sync::mpsc;
@@ -10,6 +11,12 @@ pub struct Ratelimiter {
    buckets: HashMap<String, Bucket>,
 }

+impl Default for Ratelimiter {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl Ratelimiter {
    pub fn new() -> Ratelimiter {
        Ratelimiter {
--- a/sn0int-std/src/sockets/mod.rs
+++ b/sn0int-std/src/sockets/mod.rs
@@ -6,6 +6,7 @@ use crate::json::LuaJsonValue;
 use chrootable_https::dns::{DnsResolver, RecordType};
 use chrootable_https::socks5::{self, ProxyDest};
 use regex::Regex;
+use serde::Deserialize;
 use tokio::runtime::Runtime;

 use std::fmt;
@@ -91,7 +92,7 @@ impl SocketOptions {

 pub enum Stream {
    Tcp(TcpStream),
-    Tls(rustls::StreamOwned<rustls::ClientSession, TcpStream>),
+    Tls(Box<rustls::StreamOwned<rustls::ClientSession, TcpStream>>),
 }

 impl Stream {
@@ -106,7 +107,7 @@ impl Stream {
        let mut errors = Vec::new();

        for addr in addrs {
-            match Stream::connect_addr(host, (addr, port).into(), &options) {
+            match Stream::connect_addr(host, (addr, port).into(), options) {
                Ok(socket) => {
                    return Ok(socket);
                },
@@ -232,7 +233,7 @@ impl Socket {
    }

    pub fn send(&mut self, data: &[u8]) -> Result<()> {
-        match str::from_utf8(&data) {
+        match str::from_utf8(data) {
            Ok(data) => debug!("send: {:?}", data),
            Err(_) => debug!("send: {:?}", data),
        };
--- a/sn0int-std/src/sockets/tls.rs
+++ b/sn0int-std/src/sockets/tls.rs
@@ -1,17 +1,14 @@
 use crate::errors::*;
-
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
 use rustls::{self, ClientConfig, Session, ClientSession};
-
+use serde::Serialize;
 use std::str;
 use std::result;
 use std::sync::Arc;
 use std::net::TcpStream;
-
 use super::{Stream, SocketOptions};

-
 #[derive(Debug, Serialize)]
 pub struct TlsData {
    cert: Option<String>,
@@ -50,9 +47,9 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
    }

    let dns_name = if let Some(v) = &options.sni_value {
-        get_dns_name(&mut config, &v)
+        get_dns_name(&mut config, v)
    } else {
-        get_dns_name(&mut config, &host)
+        get_dns_name(&mut config, host)
    };

    let config = Arc::new(config);
@@ -61,7 +58,7 @@ pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(S
 }

 fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
-    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(&host) {
+    if let Ok(name) = webpki::DNSNameRef::try_from_ascii_str(host) {
        debug!("setting sni value to: {:?}", host);
        name.to_owned()
    } else {
@@ -107,7 +104,7 @@ fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, T

    info!("successfully established tls connection");
    let stream = rustls::StreamOwned::new(session, stream);
-    let stream = Stream::Tls(stream);
+    let stream = Stream::Tls(Box::new(stream));
    Ok((stream, tls))
 }

--- a/sn0int-std/src/web.rs
+++ b/sn0int-std/src/web.rs
@@ -1,25 +1,24 @@
-pub use chrootable_https::{Client, HttpClient, Resolver, Response};
-
 use crate::blobs::{Blob, BlobState};
-use std::collections::{HashMap, HashSet};
-use std::time::Duration;
-use std::ops::Deref;
+use crate::engine::structs::LuaMap;
 use crate::errors::*;
+use crate::json::LuaJsonValue;
 use crate::hlua::AnyLuaValue;
-use serde_json;
+use chrootable_https::{Request, Body, Uri};
+pub use chrootable_https::{Client, HttpClient, Resolver, Response};
+use chrootable_https::http::HttpTryFrom;
+use chrootable_https::http::uri::Parts;
+use chrootable_https::http::request::Builder;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
+use serde::{Serialize, Deserialize};
+use std::collections::{HashMap, HashSet};
 use std::fmt;
+use std::iter;
 use std::net::SocketAddr;
+use std::ops::Deref;
 use std::sync::Arc;
-use serde::Serialize;
-use crate::engine::structs::LuaMap;
-use crate::json::LuaJsonValue;
-use chrootable_https::http::uri::Parts;
-use chrootable_https::{Request, Body, Uri};
-use serde_urlencoded;
-use base64;
-
+use std::time::Duration;
+use url::Url;

 pub fn url_set_qs<S: Serialize + fmt::Debug>(url: Uri, query: &S) -> Result<Uri> {
    let mut parts = Parts::from(url);
@@ -51,7 +50,12 @@ pub struct HttpSession {

 impl HttpSession {
    pub fn new() -> (String, HttpSession) {
-        let id: String = thread_rng().sample_iter(&Alphanumeric).take(16).collect();
+        let mut rng = thread_rng();
+        let id: String = iter::repeat(())
+            .map(|()| rng.sample(Alphanumeric))
+            .map(char::from)
+            .take(16)
+            .collect();
        (id.clone(), HttpSession {
            id,
            cookies: CookieJar::default(),
@@ -67,6 +71,8 @@ pub struct RequestOptions {
    user_agent: Option<String>,
    json: Option<serde_json::Value>,
    form: Option<serde_json::Value>,
+    #[serde(default)]
+    follow_redirects: usize,
    body: Option<String>,
    timeout: Option<u64>,
    #[serde(default)]
@@ -84,10 +90,6 @@ impl RequestOptions {
    }
 }

-pub fn default_user_agent() -> String {
-    format!("sn0int/{}", env!("CARGO_PKG_VERSION"))
-}
-
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct HttpRequest {
    // reference to the HttpSession
@@ -98,7 +100,8 @@ pub struct HttpRequest {
    query: Option<HashMap<String, String>>,
    headers: Option<HashMap<String, String>>,
    basic_auth: Option<(String, String)>,
-    user_agent: Option<String>,
+    user_agent: String,
+    follow_redirects: usize,
    body: Option<ReqBody>,
    timeout: Option<Duration>,
    into_blob: bool,
@@ -107,10 +110,11 @@ pub struct HttpRequest {
 }

 impl HttpRequest {
-    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions) -> HttpRequest {
+    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions, default_agent: fn() -> String) -> HttpRequest {
        let cookies = session.cookies.clone();

-        let user_agent = options.user_agent.or_else(|| Some(default_user_agent()));
+        let user_agent = options.user_agent.unwrap_or_else(default_agent);
+
        let timeout = options.timeout.map(Duration::from_millis);

        let mut request = HttpRequest {
@@ -122,6 +126,7 @@ impl HttpRequest {
            headers: options.headers,
            basic_auth: options.basic_auth,
            user_agent,
+            follow_redirects: options.follow_redirects,
            body: None,
            timeout,
            into_blob: options.into_blob,
@@ -144,7 +149,7 @@ impl HttpRequest {
        request
    }

-    pub fn send(&self, state: &dyn WebState) -> Result<Response> {
+    pub fn send(&mut self, state: &dyn WebState) -> Result<Response> {
        let mut url = self.url.parse::<Uri>()?;

        // set query string
@@ -153,37 +158,9 @@ impl HttpRequest {
        }

        // start setting up request
-        let mut req = Request::builder();
-        req.method(self.method.as_str());
-        req.uri(url);
-
-        let mut observed_headers = HashSet::new();
-
-        // set cookies
-        {
-            use chrootable_https::header::COOKIE;
-            let mut cookies = String::new();
-
-            for (key, value) in self.cookies.iter() {
-                if !cookies.is_empty() {
-                    cookies += "; ";
-                }
-                cookies.push_str(&format!("{}={}", key, value));
-            }
-
-            if !cookies.is_empty() {
-                req.header(COOKIE, cookies.as_str());
-                observed_headers.insert(COOKIE.as_str().to_lowercase());
-            }
-        }
+        let mut req = self.mkrequest(self.method.as_str(), &url);

        // add headers
-        if let Some(ref agent) = self.user_agent {
-            use chrootable_https::header::USER_AGENT;
-            req.header(USER_AGENT, agent.as_str());
-            observed_headers.insert(USER_AGENT.as_str().to_lowercase());
-        }
-
        if let Some(ref auth) = self.basic_auth {
            use chrootable_https::header::AUTHORIZATION;
            let &(ref user, ref password) = auth;
@@ -191,9 +168,9 @@ impl HttpRequest {
            let auth = base64::encode(&format!("{}:{}", user, password));
            let auth = format!("Basic {}", auth);
            req.header(AUTHORIZATION, auth.as_str());
-            observed_headers.insert(AUTHORIZATION.as_str().to_lowercase());
        }

+        let mut observed_headers = HashSet::new();
        if let Some(ref headers) = self.headers {
            for (k, v) in headers {
                req.header(k.as_str(), v.as_str());
@@ -220,24 +197,71 @@ impl HttpRequest {
            },
            None => Body::empty(),
        };
-        let req = req.body(body)?;
+        let mut req = req.body(body)?;

        debug!("Getting http client");
        let client = state.http(&self.proxy)?;

-        // send request
-        debug!("Sending http request: {:?}", req);
-        let res = client.request(req)
-            .with_timeout(self.timeout)
-            .wait_for_response()?;
+        let res = loop {
+            // send request
+            debug!("Sending http request: {:?}", req);
+            let res = client.request(req)
+                .with_timeout(self.timeout)
+                .wait_for_response()?;

-        for cookie in &res.cookies {
-            HttpRequest::register_cookies_on_state(&self.session, state, cookie);
-        }
+            for cookie in &res.cookies {
+                HttpRequest::register_cookies_on_state(&self.session, state, cookie);
+            }
+
+            if self.follow_redirects > 0 && res.status >= 300 && res.status < 400 {
+                if let Some(location) = res.headers.get("location") {
+                    let base = Url::parse(&url.to_string())?;
+                    let joined = base.join(location)?;
+                    url = joined.to_string().parse()?;
+
+                    req = self.mkrequest("GET", &url).body(Body::empty())?;
+                    self.follow_redirects -= 1;
+                    continue;
+                }
+            }
+
+            break res;
+        };

        Ok(res)
    }

+    /// create a basic request, reusable when following redirects
+    fn mkrequest<T>(&self, method: &str, url: T) -> Builder
+        where Uri: HttpTryFrom<T>,
+    {
+        let mut req = Request::builder();
+        req.method(method);
+        req.uri(url);
+        self.attach_cookies(&mut req);
+
+        use chrootable_https::header::USER_AGENT;
+        req.header(USER_AGENT, self.user_agent.as_str());
+
+        req
+    }
+
+    fn attach_cookies(&self, req: &mut Builder) {
+        use chrootable_https::header::COOKIE;
+        let mut cookies = String::new();
+
+        for (key, value) in self.cookies.iter() {
+            if !cookies.is_empty() {
+                cookies += "; ";
+            }
+            cookies.push_str(&format!("{}={}", key, value));
+        }
+
+        if !cookies.is_empty() {
+            req.header(COOKIE, cookies.as_str());
+        }
+    }
+
    pub fn response_to_lua<S>(&self, state: &S, res: Response) -> Result<LuaMap>
        where S: WebState + BlobState
    {
@@ -294,9 +318,9 @@ impl HttpRequest {
    }
 }

-impl Into<AnyLuaValue> for HttpRequest {
-    fn into(self) -> AnyLuaValue {
-        let v = serde_json::to_value(&self).unwrap();
+impl From<HttpRequest> for AnyLuaValue {
+    fn from(req: HttpRequest) -> AnyLuaValue {
+        let v = serde_json::to_value(&req).unwrap();
        LuaJsonValue::from(v).into()
    }
 }
--- a/sn0int-std/src/websockets.rs
+++ b/sn0int-std/src/websockets.rs
@@ -4,6 +4,7 @@ use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
 use crate::sockets::{Stream, SocketOptions};
 use http::Request;
+use serde::Deserialize;
 use std::collections::HashMap;
 use std::net::SocketAddr;
 use std::io;
@@ -90,9 +91,11 @@ impl WebSocket {
    }

    pub fn options(&self, options: &WebSocketOptions) -> Result<()> {
-        let mut o = SocketOptions::default();
-        o.read_timeout = options.read_timeout;
-        o.write_timeout = options.write_timeout;
+        let o = SocketOptions {
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+            ..Default::default()
+        };
        o.apply(self.sock.get_ref())
    }

--- a/sn0int-std/src/xml.rs
+++ b/sn0int-std/src/xml.rs
@@ -2,6 +2,7 @@ use crate::errors::*;

 use crate::json::LuaJsonValue;
 use crate::hlua::AnyLuaValue;
+use serde::Serialize;
 use std::collections::HashMap;
 use xml::attribute::OwnedAttribute;
 use xml::name::OwnedName;
@@ -13,6 +14,12 @@ pub struct XmlDocument {
    pub children: Vec<XmlElement>,
 }

+impl Default for XmlDocument {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl XmlDocument {
    #[inline(always)]
    pub fn new() -> XmlDocument {
--- a/src/api.rs
+++ b/src/api.rs
@@ -1,18 +1,16 @@
 use crate::errors::*;
-use std::fmt;
+use crate::web;
 use crate::config::Config;
+use crate::utils;
 use chrootable_https::{self, HttpClient, Body, Request, Uri};
 use chrootable_https::http::request::Builder as RequestBuilder;
 use chrootable_https::header::CONTENT_TYPE;
-use rand::{Rng, thread_rng};
-use rand::distributions::Alphanumeric;
+use maplit::hashmap;
 use serde::de::DeserializeOwned;
 use serde::ser::Serialize;
-use serde_json;
 use sn0int_common::api::*;
 use sn0int_common::{ModuleID, ApiResponse};
-use crate::web;
-
+use std::fmt;

 pub struct Client {
    server: String,
@@ -46,14 +44,15 @@ impl Client {
        self.session = Some(session.into());
    }

+    #[inline]
    pub fn random_session() -> String {
-        thread_rng().sample_iter(&Alphanumeric).take(32).collect()
+        utils::random_string(32)
    }

    #[inline]
    fn user_agent(&self) -> String {
-        format!("{}, {}, {}",
-            web::default_user_agent(),
+        format!("sn0int/{}, {}, {}",
+            env!("CARGO_PKG_VERSION"),
            embedded_triple::get(),
            self.os_version,
        )
--- a/src/args.rs
+++ b/src/args.rs
@@ -1,15 +1,15 @@
-use structopt::clap::{AppSettings, Shell};
-use sn0int_common::ModuleID;
 use crate::cmd;
 use crate::options;
 use crate::workspaces::Workspace;
-
+use structopt::StructOpt;
+use structopt::clap::{AppSettings, Shell};
+use sn0int_common::ModuleID;

 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
    /// Select a different workspace instead of the default
-    #[structopt(short="w", long="workspace")]
+    #[structopt(short="w", long="workspace", env="SN0INT_WORKSPACE")]
    pub workspace: Option<Workspace>,

    #[structopt(subcommand)]
@@ -18,10 +18,7 @@ pub struct Args {

 impl Args {
    pub fn is_sandbox(&self) -> bool {
-        match self.subcommand {
-            Some(SubCommand::Sandbox(_)) => true,
-            _ => false,
-        }
+        matches!(self.subcommand, Some(SubCommand::Sandbox(_)))
    }
 }

@@ -84,6 +81,9 @@ pub enum SubCommand {
    /// Export a workspace for external processing
    #[structopt(name="export")]
    Export(cmd::export_cmd::Args),
+    /// Show statistics about your current workspace
+    #[structopt(name="stats")]
+    Stats(cmd::stats_cmd::Args),
    /// Run a lua repl
    #[structopt(name="repl")]
    Repl,
--- a/src/auth.rs
+++ b/src/auth.rs
@@ -1,5 +1,4 @@
 use crate::errors::*;
-use opener;
 use std::fs;
 use std::path::PathBuf;
 use std::thread;
--- a/src/autonoscope/domain.rs
+++ b/src/autonoscope/domain.rs
@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;

@@ -94,8 +94,8 @@ impl RulePrecision for DomainRule {
    }
 }

-impl IntoRule for DomainRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for DomainRule {
+    fn to_rule(&self) -> (&'static str, String) {
        ("domain", self.to_string())
    }
 }
--- a/src/autonoscope/ip.rs
+++ b/src/autonoscope/ip.rs
@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;
 use std::net;
@@ -70,7 +70,7 @@ impl AutoRule<str> for IpRule {

 impl AutoRule<net::IpAddr> for IpRule {
    fn matches(&self, ipaddr: &net::IpAddr) -> Result<bool> {
-        Ok(self.network.contains(ipaddr.clone()))
+        Ok(self.network.contains(*ipaddr))
    }
 }

@@ -80,8 +80,8 @@ impl RulePrecision for IpRule {
    }
 }

-impl IntoRule for IpRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for IpRule {
+    fn to_rule(&self) -> (&'static str, String) {
        ("ip", self.to_string())
    }
 }
--- a/src/autonoscope/mod.rs
+++ b/src/autonoscope/mod.rs
@@ -1,9 +1,9 @@
 use crate::db::DatabaseSock;
 use crate::errors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::schema::*;
 use crate::models::*;
+use lazy_static::lazy_static;
 use std::cmp::Ordering;
 use std::convert::{TryInto, TryFrom};
 use std::str::FromStr;
@@ -112,15 +112,15 @@ impl RuleSet {
        match obj {
            RuleType::Domain => {
                self.domains.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
            },
            RuleType::Ip => {
                self.ips.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
            },
            RuleType::Url => {
                self.urls.retain(|x| x.to_string().as_str() != rule);
-                self.db_delete(db, obj, &rule)?;
+                self.db_delete(db, obj, rule)?;
            },
        }
        Ok(())
@@ -144,9 +144,9 @@ impl RuleSet {
    }

    #[inline]
-    fn push_rules_display<T: IntoRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
+    fn push_rules_display<T: ToRule>(output: &mut Vec<(&'static str, String, bool)>, rules: &[Rule<T>]) {
        for rule in rules {
-            let (object, value) = rule.into_rule();
+            let (object, value) = rule.to_rule();
            output.push((object, value, rule.scoped));
        }
    }
@@ -159,10 +159,8 @@ impl RuleSet {
            Insert::Url(url) => {
                if let Some(result) = Self::matches_any(&self.domains, url)? {
                    Some(result)
-                } else if let Some(result) = Self::matches_any(&self.urls, url)? {
-                    Some(result)
                } else {
-                    None
+                    Self::matches_any(&self.urls, url)?
                }
            },
            // Insert::Email(email) => unimplemented!(),
@@ -176,7 +174,7 @@ impl RuleSet {

    fn matches_any<T1, T2>(rules: &[Rule<T1>], object: &T2) -> Result<Option<bool>>
        where T1: AutoRule<T2>,
-            T1: IntoRule,
+            T1: ToRule,
    {
        for rule in rules {
            if rule.matches(object)? {
@@ -238,12 +236,12 @@ pub trait RulePrecision {
 }

 #[derive(Debug, PartialEq)]
-pub struct Rule<T: IntoRule> {
+pub struct Rule<T: ToRule> {
    rule: T,
    scoped: bool,
 }

-impl<T: IntoRule> Rule<T> {
+impl<T: ToRule> Rule<T> {
    pub fn new(rule: T, scoped: bool) -> Rule<T> {
        Rule {
            rule,
@@ -254,7 +252,7 @@ impl<T: IntoRule> Rule<T> {

 // TODO: maybe drop this
 use std::ops::Deref;
-impl<T: IntoRule> Deref for Rule<T> {
+impl<T: ToRule> Deref for Rule<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
@@ -262,23 +260,23 @@ impl<T: IntoRule> Deref for Rule<T> {
    }
 }

-impl<T: IntoRule + RulePrecision> RulePrecision for Rule<T> {
+impl<T: ToRule + RulePrecision> RulePrecision for Rule<T> {
    fn precision(&self) -> usize {
        self.rule.precision()
    }
 }

-pub trait IntoRule {
-    fn into_rule(&self) -> (&'static str, String);
+pub trait ToRule {
+    fn to_rule(&self) -> (&'static str, String);
 }

-impl<T: IntoRule> Into<NewAutonoscope> for &Rule<T> {
-    fn into(self) -> NewAutonoscope {
-        let (object, value) = self.rule.into_rule();
+impl<T: ToRule> From<&Rule<T>> for NewAutonoscope {
+    fn from(rule: &Rule<T>) -> NewAutonoscope {
+        let (object, value) = rule.rule.to_rule();
        NewAutonoscope {
            object: object.to_string(),
            value,
-            scoped: self.scoped,
+            scoped: rule.scoped,
        }
    }
 }
--- a/src/autonoscope/url.rs
+++ b/src/autonoscope/url.rs
@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::autonoscope::{Autonoscope, IntoRule, AutoRule, RulePrecision};
+use crate::autonoscope::{Autonoscope, ToRule, AutoRule, RulePrecision};
 use crate::models::*;
 use std::convert::TryFrom;

@@ -86,8 +86,8 @@ impl RulePrecision for UrlRule {
    }
 }

-impl IntoRule for UrlRule {
-    fn into_rule(&self) -> (&'static str, String) {
+impl ToRule for UrlRule {
+    fn to_rule(&self) -> (&'static str, String) {
        ("url", self.to_string())
    }
 }
--- a/src/blobs.rs
+++ b/src/blobs.rs
@@ -93,6 +93,14 @@ impl BlobStorage {
        }
        Ok(blobs)
    }
+
+    pub fn stat(&self, id: &str) -> Result<u64> {
+        let path = self.join(id)?;
+        debug!("Stat-ing blob: {:?}", path);
+        let md = fs::metadata(path)
+            .context("Failed to stat blob")?;
+        Ok(md.len())
+    }
 }


--- a/src/cal/date.rs
+++ b/src/cal/date.rs
@@ -80,7 +80,7 @@ fn setup_graph_map(events: &[Activity]) -> (HashMap<NaiveDate, u64>, u64) {
                if ctr > max {
                    max = ctr;
                }
-                map.insert(cur.clone(), ctr);
+                map.insert(*cur, ctr);
                *cur = date;
                ctr = 1;
            }
@@ -112,7 +112,7 @@ pub struct DateContext {

 impl DateContext {
    pub fn new(events: &[Activity], today: NaiveDate) -> DateContext {
-        let (events, max) = setup_graph_map(&events);
+        let (events, max) = setup_graph_map(events);
        DateContext {
            events,
            max,
@@ -296,25 +296,25 @@ mod tests {
        DateContext {
            events: HashMap::new(),
            max: 0,
-            today: NaiveDate::from_ymd(2020, 05, 30),
+            today: NaiveDate::from_ymd(2020, 5, 30),
        }
    }

    #[test]
    fn test_days_in_month_2020_05() {
-        let days = days_in_month(2020, 05);
+        let days = days_in_month(2020, 5);
        assert_eq!(days, 31);
    }

    #[test]
    fn test_days_in_month_2020_04() {
-        let days = days_in_month(2020, 04);
+        let days = days_in_month(2020, 4);
        assert_eq!(days, 30);
    }

    #[test]
    fn test_days_in_month_2020_02() {
-        let days = days_in_month(2020, 02);
+        let days = days_in_month(2020, 2);
        assert_eq!(days, 29);
    }

@@ -324,15 +324,15 @@ mod tests {
        let ctx = DateContext {
            events,
            max: 0,
-            today: NaiveDate::from_ymd(2020, 06, 06),
+            today: NaiveDate::from_ymd(2020, 6, 6),
        };
-        let grade = ctx.activity_for_day(&NaiveDate::from_ymd(2020, 06, 06));
+        let grade = ctx.activity_for_day(&NaiveDate::from_ymd(2020, 6, 6));
        assert_eq!(grade, ActivityGrade::None);
    }

    #[test]
    fn test_datespec_year_month() {
-        let ds = DateSpec::YearMonth((2020, 05));
+        let ds = DateSpec::YearMonth((2020, 5));
        let out = ds.to_term_string(&context());
        assert_eq!(out, "      May 2020       
 Su Mo Tu We Th Fr Sa
--- a/src/cal/time.rs
+++ b/src/cal/time.rs
@@ -34,7 +34,7 @@ fn setup_graph_map(events: &[Activity], slice_duration: u32) -> (HashMap<NaiveDa
                if ctr > max {
                    max = ctr;
                }
-                map.insert(cur.clone(), ctr);
+                map.insert(*cur, ctr);
                *cur = time;
                ctr = 1;
            }
@@ -68,7 +68,7 @@ pub struct DateTimeContext {

 impl DateTimeContext {
    pub fn new(events: &[Activity], now: NaiveDateTime, slice_width: u32, slice_duration: u32) -> DateTimeContext {
-        let (events, max) = setup_graph_map(&events, slice_duration);
+        let (events, max) = setup_graph_map(events, slice_duration);
        DateTimeContext {
            events,
            max,
@@ -135,6 +135,21 @@ impl DateTimeSpec {
        &self.end
    }

+    fn push_date(&self, w: &mut String, date: &NaiveDate) {
+        let weekday = date.weekday();
+        let is_weekend = weekday == Weekday::Sat || weekday == Weekday::Sun;
+
+        if is_weekend {
+            w.push_str("\x1b[1m");
+        }
+
+        w.push_str(&date.format("%Y-%m-%d (%a) ").to_string());
+
+        if is_weekend {
+            w.push_str("\x1b[0m");
+        }
+    }
+
    pub fn to_term_string(&self, ctx: &DateTimeContext) -> String {
        let mut w = String::new();

@@ -154,7 +169,7 @@ impl DateTimeSpec {
        // add days
        let mut date = self.start;
        while date <= self.end {
-            w.push_str(&date.format("%Y-%m-%d ").to_string());
+            self.push_date(&mut w, &date);

            let mut hours = 0;
            let mut mins = 0;
--- a/src/cmd/add_cmd.rs
+++ b/src/cmd/add_cmd.rs
@@ -2,6 +2,7 @@ use crate::errors::*;

 use crate::blobs::Blob;
 use crate::cmd::Cmd;
+use crate::db::DbChange;
 use crate::gfx;
 use crate::models::*;
 use crate::shell::Shell;
@@ -9,10 +10,11 @@ use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::utils;
 use crate::term;
+use std::fmt::Debug;
 use std::fs;
+use std::io::{self, BufRead};
 use std::net;
 use std::net::SocketAddr;
-use ipnetwork;
 use std::path::Path;
 use walkdir::WalkDir;

@@ -24,6 +26,9 @@ pub struct Args {
    /// Do not actually insert into database
    #[structopt(short="n", long="dry-run")]
    dry_run: bool,
+    /// Stream structs from stdin line by line
+    #[structopt(long)]
+    stdin: bool,
 }

 #[derive(Debug, StructOpt)]
@@ -74,25 +79,68 @@ pub enum Target {

 impl Cmd for Args {
    fn run(self, rl: &mut Shell) -> Result<()> {
-        match self.subcommand {
-            Target::Domain(args) => args.insert(rl, self.dry_run),
-            Target::Subdomain(args) => args.insert(rl, self.dry_run),
-            Target::IpAddr(args) => args.insert(rl, self.dry_run),
-            Target::Url(args) => args.insert(rl, self.dry_run),
-            Target::Email(args) => args.insert(rl, self.dry_run),
-            Target::PhoneNumber(args) => args.insert(rl, self.dry_run),
-            Target::Device(args) => args.insert(rl, self.dry_run),
-            Target::Network(args) => args.insert(rl, self.dry_run),
-            Target::Account(args) => args.insert(rl, self.dry_run),
-            Target::Breach(args) => args.insert(rl, self.dry_run),
-            Target::Image(args) => args.insert(rl, self.dry_run),
-            Target::Netblock(args) => args.insert(rl, self.dry_run),
-            Target::Port(args) => args.insert(rl, self.dry_run),
-            Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
+        if self.stdin {
+            match self.subcommand {
+                Target::Domain(_) => stream_stdin::<AddDomain>(rl),
+                Target::Subdomain(_) => stream_stdin::<AddSubdomain>(rl),
+                Target::IpAddr(_) => stream_stdin::<AddIpAddr>(rl),
+                Target::Url(_) => stream_stdin::<AddUrl>(rl),
+                Target::Email(_) => stream_stdin::<AddEmail>(rl),
+                Target::PhoneNumber(_) => stream_stdin::<AddPhoneNumber>(rl),
+                Target::Port(_) => stream_stdin::<AddPort>(rl),
+                Target::CryptoAddr(_) => stream_stdin::<AddCryptoAddr>(rl),
+                Target::Device(_) | Target::Network(_) | Target::Account(_) | Target::Breach(_) | Target::Image(_) | Target::Netblock(_) => {
+                    bail!("This struct doesn't support streaming from stdin")
+                },
+            }
+        } else {
+            match self.subcommand {
+                Target::Domain(args) => args.insert(rl, self.dry_run),
+                Target::Subdomain(args) => args.insert(rl, self.dry_run),
+                Target::IpAddr(args) => args.insert(rl, self.dry_run),
+                Target::Url(args) => args.insert(rl, self.dry_run),
+                Target::Email(args) => args.insert(rl, self.dry_run),
+                Target::PhoneNumber(args) => args.insert(rl, self.dry_run),
+                Target::Device(args) => args.insert(rl, self.dry_run),
+                Target::Network(args) => args.insert(rl, self.dry_run),
+                Target::Account(args) => args.insert(rl, self.dry_run),
+                Target::Breach(args) => args.insert(rl, self.dry_run),
+                Target::Image(args) => args.insert(rl, self.dry_run),
+                Target::Netblock(args) => args.insert(rl, self.dry_run),
+                Target::Port(args) => args.insert(rl, self.dry_run),
+                Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
+            }
        }
    }
 }

+fn stream_stdin<T: InsertFromString>(rl: &mut Shell) -> Result<()> {
+    let stdin = io::stdin();
+    for line in stdin.lock().lines() {
+        let line = line?;
+        match T::from_string(rl, line) {
+            Ok(insert) => {
+                debug!("Received {:?}", insert);
+                let result = rl.db().insert_generic(insert.clone())?;
+
+                if let Some((DbChange::Insert, _)) = result {
+                    if let Ok(value) = insert.value(rl.db()) {
+                        let log = format!("Adding {} {:?}", insert.family(), value);
+                        term::success(&log);
+                    }
+                }
+            },
+            Err(err) => term::error(&format!("Error: {:#}", err)),
+        }
+    }
+    Ok(())
+}
+
+trait InsertFromString {
+    // TODO: some implementations run insert_struct instead of just transforming
+    fn from_string(rl: &mut Shell, x: String) -> Result<Insert>;
+}
+
 trait IntoInsert: Sized {
    fn into_insert(self, rl: &mut Shell) -> Result<Insert>;

@@ -116,7 +164,12 @@ impl IntoInsert for AddDomain {
            Some(domain) => domain,
            _ => utils::question("Domain")?,
        };
+        Self::from_string(rl, domain)
+    }
+}

+impl InsertFromString for AddDomain {
+    fn from_string(rl: &mut Shell, domain: String) -> Result<Insert> {
        // ensure input is a valid domain
        let dns_name = rl.psl()?.parse_dns_name(&domain)
            .map_err(|e| format_err!("Failed to parse domain: {}", e))?;
@@ -143,7 +196,12 @@ impl IntoInsert for AddSubdomain {
            Some(subdomain) => subdomain,
            _ => utils::question("Subdomain")?,
        };
+        Self::from_string(rl, subdomain)
+    }
+}

+impl InsertFromString for AddSubdomain {
+    fn from_string(rl: &mut Shell, subdomain: String) -> Result<Insert> {
        let dns_name = rl.psl()?.parse_dns_name(&subdomain)
            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;

@@ -170,7 +228,8 @@ pub struct AddIpAddr {
 }

 impl IntoInsert for AddIpAddr {
-    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
+        // TODO: change this to Option<String> so we don't need to parse back and forth
        let ipaddr = match self.ipaddr {
            Some(ipaddr) => ipaddr,
            _ => {
@@ -178,6 +237,13 @@ impl IntoInsert for AddIpAddr {
                ipaddr.parse()?
            },
        };
+        Self::from_string(rl, ipaddr.to_string())
+    }
+}
+
+impl InsertFromString for AddIpAddr {
+    fn from_string(_rl: &mut Shell, ipaddr: String) -> Result<Insert> {
+        let ipaddr = ipaddr.parse()?;

        let family = match ipaddr {
            net::IpAddr::V4(_) => "4",
@@ -214,12 +280,17 @@ impl IntoInsert for AddUrl {
            Some(url) => url,
            _ => utils::question("URL")?,
        };
+        Self::from_string(rl, url)
+    }
+}

+impl InsertFromString for AddUrl {
+    fn from_string(rl: &mut Shell, url: String) -> Result<Insert> {
        let parts = url::Url::parse(&url)?;
        let subdomain = parts.domain()
            .ok_or_else(|| format_err!("url doesn't have a domain host"))?;

-        let dns_name = rl.psl()?.parse_dns_name(&subdomain)
+        let dns_name = rl.psl()?.parse_dns_name(subdomain)
            .map_err(|e| format_err!("Failed to parse dns_name: {}", e))?;

        let domain_id = match rl.db().insert_struct(NewDomain {
@@ -258,12 +329,18 @@ pub struct AddEmail {
 }

 impl IntoInsert for AddEmail {
-    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
        let email = match self.email {
            Some(email) => email,
            _ => utils::question("Email")?,
        };
+        Self::from_string(rl, email)
+    }
+}

+impl InsertFromString for AddEmail {
+    fn from_string(_rl: &mut Shell, email: String) -> Result<Insert> {
+        // TODO: consider doing basic validation
        Ok(Insert::Email(NewEmail {
            value: email,
            displayname: None,
@@ -290,9 +367,30 @@ impl IntoInsert for AddPhoneNumber {
            },
        };

+        // TODO: consider doing basic validation
        Ok(Insert::PhoneNumber(NewPhoneNumber {
            value: phonenumber,
-            name: name,
+            name,
+            valid: None,
+            last_online: None,
+            country: None,
+            carrier: None,
+            line: None,
+            is_ported: None,
+            last_ported: None,
+            caller_name: None,
+            caller_type: None,
+            unscoped: false,
+        }))
+    }
+}
+
+impl InsertFromString for AddPhoneNumber {
+    fn from_string(_rl: &mut Shell, phonenumber: String) -> Result<Insert> {
+        // TODO: consider doing basic validation
+        Ok(Insert::PhoneNumber(NewPhoneNumber {
+            value: phonenumber,
+            name: None,
            valid: None,
            last_online: None,
            country: None,
@@ -328,7 +426,7 @@ impl IntoInsert for AddDevice {

        Ok(Insert::Device(NewDevice {
            value: mac,
-            name: name,
+            name,
            hostname: None,
            vendor: None,
            last_seen: None,
@@ -393,9 +491,9 @@ impl IntoInsert for AddAccount {
        let value = format!("{}/{}", service, username);

        Ok(Insert::Account(NewAccount {
-            value: value,
-            service: service,
-            username: username,
+            value,
+            service,
+            username,
            displayname: None,
            email: None,
            url: None,
@@ -417,10 +515,7 @@ impl IntoInsert for AddBreach {
    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
        let name = match self.name {
            Some(name) => name,
-            _ => {
-                let name = utils::question("Name")?;
-                name
-            }
+            _ => utils::question("Name")?,
        };

        Ok(Insert::Breach(NewBreach {
@@ -453,7 +548,7 @@ impl IntoInsert for AddImage {
                let path = match path {
                    Ok(path) => path,
                    Err(err) => {
-                        let path = err.path().unwrap_or(Path::new("")).display();
+                        let path = err.path().unwrap_or_else(|| Path::new("")).display();

                        let err = err.io_error()
                            .map(|err| err.to_string())
@@ -575,12 +670,24 @@ impl IntoInsert for AddPort {
        };

        let addr = if let Some(addr) = self.addr {
+            // TODO: change this to a regular string so we don't parse->fmt->parse
            addr
        } else {
            let addr = utils::question("IP:Port")?;
            addr.parse()?
        };

+        Self::from_string(rl, format!("{}/{}", protocol, addr))
+    }
+}
+
+impl InsertFromString for AddPort {
+    fn from_string(rl: &mut Shell, value: String) -> Result<Insert> {
+        let idx = value.find('/')
+            .ok_or_else(|| format_err!("Could not locate `/` delimiter for protocol/ip:port"))?;
+        let protocol = value[..idx].to_string();
+        let addr = value[idx+1..].parse::<SocketAddr>()?;
+
        let family = match addr.ip() {
            net::IpAddr::V4(_) => "4",
            net::IpAddr::V6(_) => "6",
@@ -606,8 +713,6 @@ impl IntoInsert for AddPort {
            _ => bail!("IpAddr is out out of scope"),
        };

-        let value = format!("{}/{}", protocol, addr);
-
        Ok(Insert::Port(NewPort {
            ip_addr_id,
            value,
@@ -631,13 +736,18 @@ pub struct AddCryptoAddr {
 }

 impl IntoInsert for AddCryptoAddr {
-    fn into_insert(self, _rl: &mut Shell) -> Result<Insert> {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
        let address = if let Some(address) = self.address {
            address
        } else {
            utils::question("Address")?
        };
+        Self::from_string(rl, address)
+    }
+}

+impl InsertFromString for AddCryptoAddr {
+    fn from_string(_rl: &mut Shell, address: String) -> Result<Insert> {
        Ok(Insert::CryptoAddr(NewCryptoAddr {
            value: address,
            currency: None,
--- a/src/cmd/delete_cmd.rs
+++ b/src/cmd/delete_cmd.rs
@@ -19,20 +19,20 @@ pub struct Args {
 impl Cmd for Args {
    fn run(self, rl: &mut Shell) -> Result<()> {
        let rows = match &self.subcommand {
-            Target::Domains(filter) => delete::<Domain>(rl, &filter),
-            Target::Subdomains(filter) => delete::<Subdomain>(rl, &filter),
-            Target::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
-            Target::Urls(filter) => delete::<Url>(rl, &filter),
-            Target::Emails(filter) => delete::<Email>(rl, &filter),
-            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
-            Target::Devices(filter) => delete::<Device>(rl, &filter),
-            Target::Networks(filter) => delete::<Network>(rl, &filter),
-            Target::Accounts(filter) => delete::<Account>(rl, &filter),
-            Target::Breaches(filter) => delete::<Breach>(rl, &filter),
-            Target::Images(filter) => delete::<Image>(rl, &filter),
-            Target::Ports(filter) => delete::<Port>(rl, &filter),
-            Target::Netblocks(filter) => delete::<Netblock>(rl, &filter),
-            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, &filter),
+            Target::Domains(filter) => delete::<Domain>(rl, filter),
+            Target::Subdomains(filter) => delete::<Subdomain>(rl, filter),
+            Target::IpAddrs(filter) => delete::<IpAddr>(rl, filter),
+            Target::Urls(filter) => delete::<Url>(rl, filter),
+            Target::Emails(filter) => delete::<Email>(rl, filter),
+            Target::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, filter),
+            Target::Devices(filter) => delete::<Device>(rl, filter),
+            Target::Networks(filter) => delete::<Network>(rl, filter),
+            Target::Accounts(filter) => delete::<Account>(rl, filter),
+            Target::Breaches(filter) => delete::<Breach>(rl, filter),
+            Target::Images(filter) => delete::<Image>(rl, filter),
+            Target::Ports(filter) => delete::<Port>(rl, filter),
+            Target::Netblocks(filter) => delete::<Netblock>(rl, filter),
+            Target::CryptoAddrs(filter) => delete::<CryptoAddr>(rl, filter),
        }?;
        term::info(&format!("Deleted {} rows", rows));
        Ok(())
--- a/src/cmd/export_cmd.rs
+++ b/src/cmd/export_cmd.rs
@@ -5,8 +5,7 @@ use crate::cmd::Cmd;
 use crate::db::ttl;
 use crate::models::*;
 use crate::shell::Shell;
-use serde_json;
-use serde::Serialize;
+use serde::{Serialize, Deserialize};
 use std::io::{self, Write};
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
--- a/src/cmd/help_cmd.rs
+++ b/src/cmd/help_cmd.rs
@@ -21,6 +21,7 @@ pub fn run(_rl: &mut Shell, _args: &[String]) -> Result<()> {
    help("run",         "Run the currently selected module");
    help("scope",       "Include entities in the scope again");
    help("select",      "Select entities from the database");
+    help("stats",       "Show statistics about your current workspace");
    help("target",      "Preview targeted entities or narrow them down");
    help("use",         "Select a module");
    help("workspace",   "Switch to a different workspace");
--- a/src/cmd/mod.rs
+++ b/src/cmd/mod.rs
@@ -34,6 +34,7 @@ pub mod notify_cmd;
 pub mod pkg_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;
+pub mod stats_cmd;
 pub mod target_cmd;
 pub mod quickstart_cmd;
 pub mod workspace_cmd;
--- a/src/cmd/notify_cmd.rs
+++ b/src/cmd/notify_cmd.rs
@@ -7,6 +7,7 @@ use crate::notify::{self, Notification};
 use crate::options::{self, Opt};
 use crate::shell::Shell;
 use crate::term;
+use sn0int_std::ratelimits::Ratelimiter;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;

@@ -67,14 +68,20 @@ fn print_summary(module: &Module, sent: usize, errors: usize) {
 }

 fn send(args: SendArgs, rl: &mut Shell) -> Result<()> {
-    notify::run_router(rl, &mut term::Term, args.dry_run, &args.topic, &args.notification)?;
+    rl.signal_register().catch_ctrl();
+    notify::run_router(rl, &mut term::Term, &mut Ratelimiter::new(), args.dry_run, &args.topic, &args.notification)?;
+    rl.signal_register().reset_ctrlc();
    Ok(())
 }

 fn exec(args: ExecArgs, rl: &mut Shell) -> Result<()> {
    let module = rl.library().get(&args.module)?.clone();
    let options = Opt::collect(&args.options);
-    let errors = notify::exec(rl, &module, options, args.verbose, &args.notification)?;
+
+    rl.signal_register().catch_ctrl();
+    let errors = notify::exec(rl, &module, &mut Ratelimiter::new(), options, args.verbose, &args.notification)?;
+    rl.signal_register().reset_ctrlc();
+
    print_summary(&module, 1, errors);
    Ok(())
 }
--- a/src/cmd/pkg_cmd.rs
+++ b/src/cmd/pkg_cmd.rs
@@ -5,13 +5,14 @@ use crate::api::Client;
 use crate::args;
 use crate::config::Config;
 use crate::cmd::{Cmd, LiteCmd};
-use crate::engine::Library;
+use crate::engine::{Library, Module};
 use crate::registry::{self, InstallTask, UpdateTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
 use crate::worker;
-use colored::Colorize;
+use colored::{Color, Colorize};
 use sn0int_common::ModuleID;
+use sn0int_common::metadata::Stealth;
 use std::collections::HashSet;
 use std::fmt::Write;
 use std::sync::Arc;
@@ -71,7 +72,10 @@ pub struct List {
    pub source: Option<String>,
    /// List outdated modules
    #[structopt(long="outdated")]
-    pub outdated: bool,
+    pub outdated_only: bool,
+    /// Only show modules with equal or better stealth level
+    #[structopt(long="stealth", possible_values=Stealth::variants())]
+    pub stealth: Option<Stealth>,
    /// Filter by pattern
    #[structopt(default_value="*")]
    pub pattern: String,
@@ -96,6 +100,34 @@ enum ModuleReload {
    No,
 }

+#[inline]
+fn write_tag(out: &mut String, color: Color, txt: &str) -> Result<()> {
+    write!(out, " [{}]", txt.color(color))?;
+    Ok(())
+}
+
+fn print_module(module: &Module, is_outdated: bool) -> Result<()> {
+    let mut out = String::new();
+    write!(&mut out, "{}/{} {}", module.author().purple(),
+                                 module.name(),
+                                 module.version().blue())?;
+
+    match module.stealth() {
+        Stealth::Loud => write_tag(&mut out, Color::Yellow, "LOUD")?,
+        Stealth::Normal => (),
+        Stealth::Passive => write_tag(&mut out, Color::Green, "passive")?,
+        Stealth::Offline => write_tag(&mut out, Color::Green, "offline")?,
+    };
+
+    if is_outdated {
+        write_tag(&mut out, Color::Red, "outdated")?;
+    }
+
+    println!("{}", out.bold());
+    println!("    {}", module.description());
+    Ok(())
+}
+
 fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) -> Result<ModuleReload> {
    match subcommand {
        SubCommand::List(list) => {
@@ -105,7 +137,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->

            for module in library.list() {
                if let Some(source) = &list.source {
-                    if !module.source_equals(&source) {
+                    if !module.source_equals(source) {
                        continue;
                    }
                }
@@ -115,31 +147,33 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
                    continue;
                }

-                let mut out = String::new();
-                write!(&mut out, "{} ({})", canonical.green(),
-                                            module.version().yellow())?;
-                if autoupdate.is_outdated(&canonical) {
-                    write!(&mut out, " {}", "[outdated]".red())?;
-                } else if list.outdated {
+                if let Some(stealth) = &list.stealth {
+                    if !module.stealth().equal_or_better(stealth) {
+                        continue;
+                    }
+                }
+
+                let is_outdated = autoupdate.is_outdated(&canonical);
+                if list.outdated_only && !is_outdated {
                    continue;
                }
-                println!("{}", out);
-                println!("\t{}", module.description());
+
+                print_module(module, is_outdated)?;
            }
            Ok(ModuleReload::No)
        },
        SubCommand::Install(install) => {
-            registry::run_install(install, &config)?;
+            registry::run_install(install, config)?;
            // trigger reload
            Ok(ModuleReload::Yes)
        },
        SubCommand::Search(search) => {
-            registry::run_search(library, &search, &config)?;
+            registry::run_search(library, &search, config)?;
            Ok(ModuleReload::No)
        },
        SubCommand::Update(_) => {
            let mut autoupdate = AutoUpdater::load()?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let updater = Arc::new(Updater::new(config)?);

            let modules = library.list()
                .into_iter()
@@ -165,14 +199,14 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
            Ok(ModuleReload::Yes)
        },
        SubCommand::Uninstall(uninstall) => {
-            let updater = Updater::new(&config)?;
+            let updater = Updater::new(config)?;
            updater.uninstall(&uninstall.module)?;
            // trigger reload
            Ok(ModuleReload::Yes)
        },
        SubCommand::Quickstart => {
-            let client = Client::new(&config)?;
-            let updater = Arc::new(Updater::new(&config)?);
+            let client = Client::new(config)?;
+            let updater = Arc::new(Updater::new(config)?);
            let mut autoupdate = AutoUpdater::load()?;

            let installed = library.list()
@@ -216,7 +250,7 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->

 impl LiteCmd for Args {
    fn run(self, config: &Config) -> Result<()> {
-        let library = Library::new(false, &config)?;
+        let library = Library::new(false, config)?;
        run_subcommand(self.subcommand, &library, config)?;
        Ok(())
    }
--- a/src/cmd/run_cmd.rs
+++ b/src/cmd/run_cmd.rs
@@ -14,8 +14,8 @@ use crate::term;
 use crate::utils;
 use crate::worker;
 use serde::Serialize;
-use serde_json;
 use sn0int_common::metadata::Source;
+use sn0int_std::ratelimits::Ratelimiter;
 use std::collections::HashMap;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
@@ -93,7 +93,7 @@ fn prepare_args<T: Scopable + Serialize + Model>(rl: &Shell, filter: &Filter, pa
 }

 pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
-    for namespace in keyring.unauthorized_namespaces(&module) {
+    for namespace in keyring.unauthorized_namespaces(module) {
        let grant_access = if params.deny_keyring {
            false
        } else if params.grant_full_keyring || params.grants.contains(namespace) {
@@ -104,7 +104,7 @@ pub fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params)
        };

        if grant_access {
-            keyring.grant_access(&module, namespace.to_string());
+            keyring.grant_access(module, namespace.to_string());
            term::info(&format!("Granted access to {:?}", namespace));
        }
    }
@@ -136,8 +136,8 @@ fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, O
        Some(Source::Notifications) => bail!("Notification modules can't be executed like this"),
        Some(Source::KeyRing(namespace)) => {
            let keyring = rl.keyring();
-            if keyring.is_access_granted(&module, &namespace) {
-                keyring.get_all_for(&namespace).into_iter()
+            if keyring.is_access_granted(module, namespace) {
+                keyring.get_all_for(namespace).into_iter()
                    .map(|key| {
                        let pretty = format!("{}:{}", key.namespace, key.access_key);
                        let arg = serde_json::to_value(key)?;
@@ -161,14 +161,14 @@ pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<St
    let keyring = rl.keyring().request_keys(&module);

    let dns_config = Resolver::from_system_v4()?;
-    let proxy = rl.config().network.proxy.clone();
+    let proxy = rl.config().network.proxy;

    let args = get_args(rl, &module)?;
    for (arg, _pretty_arg, blobs) in args {
        let start_cmd = StartCommand::new(params.verbose,
                                          keyring.clone(),
                                          dns_config.clone(),
-                                          proxy.clone(),
+                                          proxy,
                                          options.clone(),
                                          module.clone(),
                                          arg,
@@ -189,7 +189,7 @@ pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>)
    let args = get_args(rl, &module)?;

    rl.signal_register().catch_ctrl();
-    let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);
+    let errors = worker::spawn(rl, &module, &mut Ratelimiter::new(), args, &params, rl.config().network.proxy, options);
    rl.signal_register().reset_ctrlc();

    if errors > 0 {
--- a/src/cmd/select_cmd.rs
+++ b/src/cmd/select_cmd.rs
@@ -5,7 +5,6 @@ use crate::db::ttl;
 use crate::filters::{Target, Filter};
 use crate::shell::Shell;
 use serde::Serialize;
-use serde_json;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::models::*;
@@ -17,13 +16,16 @@ pub struct Args {
    #[structopt(subcommand)]
    subcommand: Target,
    /// Print json output
-    #[structopt(long="json", group="output")]
+    #[structopt(long, group="output")]
    json: bool,
+    /// Only print the value instead of the whole object
+    #[structopt(long, group="output")]
+    values: bool,
    /// Print paths to blobs
-    #[structopt(long="paths", group="output")]
+    #[structopt(long, group="output")]
    paths: bool,
    /// Count rows returned
-    #[structopt(short="c", long="count", group="output")]
+    #[structopt(short="c", group="output")]
    count: bool,
 }

@@ -31,6 +33,7 @@ pub struct Args {
 enum Output {
    Normal,
    Json,
+    Values,
    Paths,
    Count,
 }
@@ -44,6 +47,8 @@ impl<'a, 'b> Printer<'a, 'b> {
    pub fn new(rl: &'a mut Shell<'b>, args: &Args) -> Printer<'a, 'b> {
        let output = if args.json {
            Output::Json
+        } else if args.values {
+            Output::Values
        } else if args.paths {
            Output::Paths
        } else if args.count {
@@ -71,6 +76,9 @@ impl<'a, 'b> Printer<'a, 'b> {
                        let v = serde_json::to_string(&obj)?;
                        println!("{}", v);
                    },
+                    Output::Values => {
+                        println!("{}", obj.to_string());
+                    },
                    Output::Paths => {
                        let blob = obj.blob()
                            .ok_or_else(|| format_err!("This model isn't linked to blob storage"))?;
@@ -98,20 +106,20 @@ impl Cmd for Args {
        let printer = Printer::new(rl, &self);

        match &self.subcommand {
-            Target::Domains(filter) => printer.select::<Domain>(&filter),
-            Target::Subdomains(filter) => printer.select::<Subdomain>(&filter),
-            Target::IpAddrs(filter) => printer.select::<IpAddr>(&filter),
-            Target::Urls(filter) => printer.select::<Url>(&filter),
-            Target::Emails(filter) => printer.select::<Email>(&filter),
-            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(&filter),
-            Target::Devices(filter) => printer.select::<Device>(&filter),
-            Target::Networks(filter) => printer.select::<Network>(&filter),
-            Target::Accounts(filter) => printer.select::<Account>(&filter),
-            Target::Breaches(filter) => printer.select::<Breach>(&filter),
-            Target::Images(filter) => printer.select::<Image>(&filter),
-            Target::Ports(filter) => printer.select::<Port>(&filter),
-            Target::Netblocks(filter) => printer.select::<Netblock>(&filter),
-            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(&filter),
+            Target::Domains(filter) => printer.select::<Domain>(filter),
+            Target::Subdomains(filter) => printer.select::<Subdomain>(filter),
+            Target::IpAddrs(filter) => printer.select::<IpAddr>(filter),
+            Target::Urls(filter) => printer.select::<Url>(filter),
+            Target::Emails(filter) => printer.select::<Email>(filter),
+            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(filter),
+            Target::Devices(filter) => printer.select::<Device>(filter),
+            Target::Networks(filter) => printer.select::<Network>(filter),
+            Target::Accounts(filter) => printer.select::<Account>(filter),
+            Target::Breaches(filter) => printer.select::<Breach>(filter),
+            Target::Images(filter) => printer.select::<Image>(filter),
+            Target::Ports(filter) => printer.select::<Port>(filter),
+            Target::Netblocks(filter) => printer.select::<Netblock>(filter),
+            Target::CryptoAddrs(filter) => printer.select::<CryptoAddr>(filter),
        }
    }
 }
--- a/src/cmd/stats_cmd.rs
+++ b/src/cmd/stats_cmd.rs
@@ -0,0 +1,199 @@
+use crate::args;
+use colored::Colorize;
+use crate::blobs::BlobStorage;
+use crate::cmd::Cmd;
+use crate::db::{ttl, Database};
+use crate::errors::*;
+use crate::models::*;
+use crate::shell::{self, Shell};
+use crate::workspaces;
+use humansize::{FileSize, file_size_opts};
+use separator::Separatable;
+use serde::{Serialize, Deserialize};
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug, Clone, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Exclude blob storage
+    #[structopt(short, long)]
+    short: bool,
+    /// Exclude categories that don't contain any structs
+    #[structopt(short, long)]
+    quiet: bool,
+    /// Show workspace statistics in json
+    #[structopt(short, long)]
+    json: bool,
+    /// Go through all workspaces
+    #[structopt(short, long)]
+    all: bool,
+}
+
+impl Args {
+    fn show_amount(&self, label: &str, count: usize, amount: &str) {
+        if self.quiet && count == 0 {
+            return;
+        }
+
+        let label = format!("{:20}", label);
+        let amount = format!("{:>20}", amount);
+
+        if count > 0 {
+            println!("{} {}", label.green(), amount.yellow());
+        } else {
+            println!("{} {}", label, amount);
+        }
+    }
+
+    fn show_count(&self, label: &str, count: usize) {
+        self.show_amount(label, count, &count.separated_string());
+    }
+}
+
+fn count_models<T: Model>(db: &Database) -> Result<usize> {
+    let query = db.list::<T>()?;
+    let count = query.len();
+    Ok(count)
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Stats {
+    workspace: String,
+    domains: usize,
+    subdomains: usize,
+    ipaddrs: usize,
+    urls: usize,
+    emails: usize,
+    phonenumbers: usize,
+    devices: usize,
+    networks: usize,
+    accounts: usize,
+    breaches: usize,
+    images: usize,
+    ports: usize,
+    netblocks: usize,
+    cryptoaddrs: usize,
+    activity: usize,
+    blobs: Option<BlobStats>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct BlobStats {
+    count: usize,
+    total_size: u64,
+    total_human_size: String,
+}
+
+impl Stats {
+    fn count(workspace: String, db: &Database) -> Result<Stats> {
+        Ok(Stats {
+            workspace,
+            domains: count_models::<Domain>(db)?,
+            subdomains: count_models::<Subdomain>(db)?,
+            ipaddrs: count_models::<IpAddr>(db)?,
+            urls: count_models::<Url>(db)?,
+            emails: count_models::<Email>(db)?,
+            phonenumbers: count_models::<PhoneNumber>(db)?,
+            devices: count_models::<Device>(db)?,
+            networks: count_models::<Network>(db)?,
+            accounts: count_models::<Account>(db)?,
+            breaches: count_models::<Breach>(db)?,
+            images: count_models::<Image>(db)?,
+            ports: count_models::<Port>(db)?,
+            netblocks: count_models::<Netblock>(db)?,
+            cryptoaddrs: count_models::<CryptoAddr>(db)?,
+            activity: Activity::count(db)?,
+            blobs: None,
+        })
+    }
+
+    fn add_blob_usage(&mut self, storage: &BlobStorage) -> Result<()> {
+        let blobs = storage.list()?;
+
+        let mut total_size = 0;
+        for blob in &blobs {
+            total_size += storage.stat(blob)?;
+        }
+
+        let total_human_size = total_size.file_size(file_size_opts::CONVENTIONAL)
+            .map_err(|e| format_err!("Failed to format size: {}", e))?;
+
+        self.blobs = Some(BlobStats {
+            count: blobs.len(),
+            total_size,
+            total_human_size,
+        });
+
+        Ok(())
+    }
+}
+
+impl Cmd for Args {
+    #[inline]
+    fn run(mut self, rl: &mut Shell) -> Result<()> {
+        if self.all {
+            self.all = false;
+            let mut first = true;
+
+            let config = rl.config();
+            for ws in workspaces::list()? {
+                if first {
+                    first = false;
+                } else if !self.json {
+                    println!();
+                }
+
+                let mut rl = shell::init(&args::Args {
+                    workspace: Some(ws),
+                    subcommand: None,
+                }, config, false)?;
+                self.clone().run(&mut rl)?;
+            }
+        } else {
+            ttl::reap_expired(rl)?;
+
+            let db = rl.db();
+
+            let workspace = rl.workspace();
+
+            if !self.json {
+                // print this early in case counting takes a while
+                println!("{:>41}", workspace.bold());
+            }
+
+            let mut stats = Stats::count(workspace.into(), db)?;
+            if !self.short {
+                stats.add_blob_usage(rl.blobs())?;
+            }
+
+            if self.json {
+                let stats = serde_json::to_string(&stats)?;
+                println!("{}", stats);
+            } else {
+                self.show_count("domains", stats.domains);
+                self.show_count("subdomains", stats.subdomains);
+                self.show_count("ipaddrs", stats.ipaddrs);
+                self.show_count("urls", stats.urls);
+                self.show_count("emails", stats.emails);
+                self.show_count("phonenumbers", stats.phonenumbers);
+                self.show_count("devices", stats.devices);
+                self.show_count("networks", stats.networks);
+                self.show_count("accounts", stats.accounts);
+                self.show_count("breaches", stats.breaches);
+                self.show_count("images", stats.images);
+                self.show_count("ports", stats.ports);
+                self.show_count("netblocks", stats.netblocks);
+                self.show_count("cryptoaddrs", stats.cryptoaddrs);
+                self.show_count("activity", stats.activity);
+
+                if let Some(blobs) = stats.blobs {
+                    self.show_count("blobs", blobs.count);
+                    self.show_amount("blobs (size)", blobs.total_size as usize, &blobs.total_human_size);
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
--- a/src/cmd/target_cmd.rs
+++ b/src/cmd/target_cmd.rs
@@ -81,7 +81,7 @@ fn count_selected(rl: &mut Shell, source: &Source) -> Result<usize> {
        Source::Netblocks => db.filter::<Netblock>(&filter)?.len(),
        Source::CryptoAddrs(currency) => db.filter_with_param::<CryptoAddr>(&filter, currency.as_ref())?.len(),
        Source::Notifications => bail!("Notifications can't be set as target"),
-        Source::KeyRing(namespace) => rl.keyring().list_for(&namespace).len(),
+        Source::KeyRing(namespace) => rl.keyring().list_for(namespace).len(),
    };
    Ok(num)
 }
--- a/src/cmd/workspace_cmd.rs
+++ b/src/cmd/workspace_cmd.rs
@@ -24,14 +24,12 @@ pub struct Args {
    /// Skip confirmation
    #[structopt(short = "f", long = "force")]
    force: bool,
-    workspace: Option<Workspace>,
+    workspaces: Vec<Workspace>,
 }

 fn delete(workspace: Workspace, force: bool) -> Result<()> {
-    if !force {
-        if !utils::no_else_yes(&format!("Do you really want to delete {:?}", workspace.as_str()))? {
-            return Ok(());
-        }
+    if !force && !utils::no_else_yes(&format!("Do you really want to delete {:?}", workspace.as_str()))? {
+        return Ok(());
    }

    term::info(&format!("Deleting workspace: {:?}", workspace.as_str()));
@@ -40,13 +38,13 @@ fn delete(workspace: Workspace, force: bool) -> Result<()> {
    Ok(())
 }

-fn usage(workspace: Option<Workspace>) -> Result<()> {
-    if let Some(ws) = workspace {
-        println!("{}", ws.usage_human()?);
-    } else {
-        for ws in workspaces::list()? {
-            println!("{:50}: {}", ws.as_str(), ws.usage_human()?);
-        }
+fn usage(mut workspaces: Vec<Workspace>) -> Result<()> {
+    if workspaces.is_empty() {
+        workspaces = workspaces::list()?;
+    }
+
+    for ws in workspaces {
+        println!("{:50}: {}", ws.as_str(), ws.usage_human()?);
    }

    Ok(())
@@ -69,30 +67,34 @@ fn list() -> Result<()> {
    Ok(())
 }

-fn run(args: Args, rl: Option<&mut Shell>) -> Result<()> {
+fn run(mut args: Args, rl: Option<&mut Shell>) -> Result<()> {
    if args.delete {
-        if let Some(workspace) = args.workspace {
-            if let Some(rl) = rl {
+        if args.workspaces.is_empty() {
+            bail!("--delete requires workspace");
+        }
+
+        for workspace in args.workspaces {
+            if let Some(rl) = &rl {
                if *rl.db().workspace() == workspace {
                    bail!("Can't delete current workspace")
                }
            }
-
-            delete(workspace, args.force)
-        } else {
-            bail!("--delete requires workspace")
+            delete(workspace, args.force)?;
        }
    } else if args.usage {
-        usage(args.workspace)
-    } else if let Some(workspace) = args.workspace {
-        if let Some(rl) = rl {
-            change(rl, workspace)
-        } else {
-            Ok(())
-        }
+        usage(args.workspaces)?;
    } else {
-        list()
+        match args.workspaces.len() {
+            0 => list()?,
+            1 => if let Some(rl) = rl {
+                // we've already tested there's one item in the list
+                change(rl, args.workspaces.pop().unwrap())?;
+            },
+            _ => bail!("Only one argument allowed when switching workspaces"),
+        }
    }
+
+    Ok(())
 }

 impl Cmd for Args {
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,12 +1,10 @@
-use dirs;
 use crate::errors::*;
+use serde::{Serialize, Deserialize};
 use crate::notify::NotificationConfig;
 use std::collections::HashMap;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::net::SocketAddr;
-use toml;
-

 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct Config {
@@ -27,7 +25,7 @@ impl Config {
    }

    pub fn path() -> Result<PathBuf> {
-        let path = dirs::config_dir()
+        let path = dirs_next::config_dir()
            .ok_or_else(|| format_err!("Failed to find config directory"))?;
        let path = path.join("sn0int.toml");
        Ok(path)
--- a/src/db/mod.rs
+++ b/src/db/mod.rs
@@ -1,6 +1,6 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};

-use diesel;
 use diesel::expression::SqlLiteral;
 use diesel::expression::sql_literal::sql;
 use diesel::sql_types::Bool;
@@ -25,10 +25,7 @@ pub enum DbChange {

 impl DbChange {
    pub fn is_some(&self) -> bool {
-        match self {
-            DbChange::None => false,
-            _ => true,
-        }
+        !matches!(self, DbChange::None)
    }
 }

@@ -115,15 +112,19 @@ impl Database {

        let db = SqliteConnection::establish(&path)
            .context("Failed to connect to database")?;
-        migrations::run(&db)
-            .context("Failed to run migrations")?;
-        db.execute("PRAGMA journal_mode = WAL")
-            .context("Failed to enable write ahead log")?;
+
+        db.execute("PRAGMA busy_timeout = 10000")
+            .context("Failed to set busy_timeout")?;
        db.execute("PRAGMA foreign_keys = ON")
            .context("Failed to enforce foreign keys")?;
+        db.execute("PRAGMA journal_mode = WAL")
+            .context("Failed to enable write ahead log")?;
        db.execute("PRAGMA synchronous = NORMAL")
            .context("Failed to enforce foreign keys")?;

+        migrations::run(&db)
+            .context("Failed to run migrations")?;
+
        let autonoscope = RuleSet::load(&db)?;

        Ok(Database {
@@ -197,14 +198,14 @@ impl Database {

            let update = obj.upsert(&existing);
            if update.is_dirty() {
-                update.apply(&self)?;
+                update.apply(self)?;
                Ok(Some((DbChange::Update(update.generic()), existing.id())))
            } else {
                Ok(Some((DbChange::None, existing.id())))
            }
        } else {
            obj.set_scoped(scoped);
-            obj.insert(&self)?;
+            obj.insert(self)?;
            let id = M::get_id(self, obj.value())?;
            Ok(Some((DbChange::Insert, id)))
        }
@@ -217,7 +218,7 @@ impl Database {
                return Ok(false);
            }
        }
-        obj.insert(&self)?;
+        obj.insert(self)?;
        Ok(true)
    }

@@ -253,7 +254,7 @@ impl Database {

            let update = obj.upsert(&existing);
            if update.is_dirty() {
-                update.apply(&self)?;
+                update.apply(self)?;
                Ok(Some((DbChange::Update(update.generic()), id)))
            } else {
                Ok(Some((DbChange::None, id)))
@@ -401,7 +402,7 @@ impl Database {
    }

    fn get_opt_typed<T: Model + Scopable>(&self, value: &T::ID) -> Result<Option<i32>> {
-        match T::get_opt(self, &value)? {
+        match T::get_opt(self, value)? {
            Some(ref obj) if obj.scoped() => Ok(Some(obj.id())),
            _ => Ok(None),
        }
@@ -409,23 +410,23 @@ impl Database {

    pub fn get_opt(&self, family: &Family, value: &str) -> Result<Option<i32>> {
        match family {
-            Family::Domain => self.get_opt_typed::<Domain>(&value),
-            Family::Subdomain => self.get_opt_typed::<Subdomain>(&value),
-            Family::Ipaddr => self.get_opt_typed::<IpAddr>(&value),
+            Family::Domain => self.get_opt_typed::<Domain>(value),
+            Family::Subdomain => self.get_opt_typed::<Subdomain>(value),
+            Family::Ipaddr => self.get_opt_typed::<IpAddr>(value),
            Family::SubdomainIpaddr => bail!("Unsupported operation"),
-            Family::Url => self.get_opt_typed::<Url>(&value),
-            Family::Email => self.get_opt_typed::<Email>(&value),
-            Family::Phonenumber => self.get_opt_typed::<PhoneNumber>(&value),
-            Family::Device => self.get_opt_typed::<Device>(&value),
-            Family::Network => self.get_opt_typed::<Network>(&value),
+            Family::Url => self.get_opt_typed::<Url>(value),
+            Family::Email => self.get_opt_typed::<Email>(value),
+            Family::Phonenumber => self.get_opt_typed::<PhoneNumber>(value),
+            Family::Device => self.get_opt_typed::<Device>(value),
+            Family::Network => self.get_opt_typed::<Network>(value),
            Family::NetworkDevice => bail!("Unsupported operation"),
-            Family::Account => self.get_opt_typed::<Account>(&value),
-            Family::Breach => self.get_opt_typed::<Breach>(&value),
+            Family::Account => self.get_opt_typed::<Account>(value),
+            Family::Breach => self.get_opt_typed::<Breach>(value),
            Family::BreachEmail => bail!("Unsupported operation"),
-            Family::Image => self.get_opt_typed::<Image>(&value),
-            Family::Port => self.get_opt_typed::<Port>(&value),
-            Family::Netblock => self.get_opt_typed::<Netblock>(&value),
-            Family::Cryptoaddr => self.get_opt_typed::<CryptoAddr>(&value),
+            Family::Image => self.get_opt_typed::<Image>(value),
+            Family::Port => self.get_opt_typed::<Port>(value),
+            Family::Netblock => self.get_opt_typed::<Netblock>(value),
+            Family::Cryptoaddr => self.get_opt_typed::<CryptoAddr>(value),
        }
    }

--- a/src/db/ttl.rs
+++ b/src/db/ttl.rs
@@ -6,8 +6,8 @@ use crate::schema::*;
 use crate::shell::Shell;
 use crate::term::{self, Term};
 use chrono::{NaiveDateTime, Duration, Utc};
-use diesel;
 use diesel::prelude::*;
+use sn0int_std::ratelimits::Ratelimiter;


 #[derive(Identifiable, Queryable, AsChangeset, PartialEq, Debug)]
@@ -20,25 +20,7 @@ pub struct Ttl {
    pub expire: NaiveDateTime,
 }

-#[derive(Insertable)]
-#[table_name="ttls"]
-pub struct NewTtl<'a> {
-    pub family: &'a str,
-    pub key: i32,
-    pub value: String,
-    pub expire: NaiveDateTime,
-}
-
 impl Ttl {
-    pub fn new(obj: &Insert, key: i32, value: String, expire: NaiveDateTime) -> NewTtl {
-        NewTtl {
-            family: obj.table(),
-            key,
-            value,
-            expire,
-        }
-    }
-
    pub fn find(obj: &Insert, my_key: i32, db: &Database) -> Result<Option<Ttl>> {
        use crate::schema::ttls::dsl::*;

@@ -134,16 +116,26 @@ impl Ttl {
    }
 }

+#[derive(Insertable)]
+#[table_name="ttls"]
+pub struct NewTtl<'a> {
+    pub family: &'a str,
+    pub key: i32,
+    pub value: String,
+    pub expire: NaiveDateTime,
+}
+
 pub fn reap_expired(rl: &mut Shell) -> Result<()> {
    debug!("Reaping expired entities");

+    let mut ratelimit = Ratelimiter::new();
    for expired in Ttl::expired(rl.db())? {
        debug!("Expired: {:?}", expired);
        expired.delete(rl.db())?;

        let subject = format!("Deleted {} {:?}", &expired.family, &expired.value);
        let topic = &format!("db:{}:{}:delete", &expired.family, &expired.value);
-        if let Err(err) = notify::trigger_notify_event(rl, &mut Term, &topic, &Notification {
+        if let Err(err) = notify::trigger_notify_event(rl, &mut Term, &mut ratelimit, topic, &Notification {
            subject,
            body: None,
        }) {
--- a/src/engine/ctx.rs
+++ b/src/engine/ctx.rs
@@ -1,21 +1,22 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};

 use crate::db::Family;
 use crate::engine::{Environment, IpcChild};
 use crate::geoip::{MaxmindReader, GeoIP, AsnDB};
 use crate::hlua::{self, AnyLuaValue};
 use crate::keyring::KeyRingEntry;
+use crate::lazy::Lazy;
 use crate::models::*;
 use crate::psl::{Psl, PslReader};
-use crate::lazy::Lazy;
+use crate::ratelimits::RatelimitResponse;
 use crate::runtime;
 use crate::sockets::{Socket, SocketOptions, TlsData};
+use crate::utils;
 use crate::web::{HttpSession, HttpRequest, RequestOptions};
 use crate::websockets::{WebSocket, WebSocketOptions};
 use crate::worker::{Event, LogEvent, DatabaseEvent, DatabaseResponse, StdioEvent, RatelimitEvent};
-use crate::ratelimits::RatelimitResponse;
 use chrootable_https::{self, Resolver};
-use serde_json;
 use sn0int_std::blobs::{Blob, BlobState};
 use sn0int_std::mqtt::{MqttClient, MqttOptions};
 use sn0int_std::web::WebState;
@@ -24,8 +25,6 @@ use std::result;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
 use std::thread;
-use rand::prelude::*;
-use rand::distributions::Alphanumeric;


 pub trait State {
@@ -80,14 +79,14 @@ pub trait State {
    }

    fn db_insert(&self, object: Insert) -> Result<DatabaseResponse> {
-        self.send(&Event::Database(DatabaseEvent::Insert(object)));
+        self.send(&Event::Database(Box::new(DatabaseEvent::Insert(object))));
        self.db_recv()
            .context("Failed to add to database")
            .map_err(Error::from)
    }

    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<DatabaseResponse> {
-        self.send(&Event::Database(DatabaseEvent::InsertTtl((object, ttl))));
+        self.send(&Event::Database(Box::new(DatabaseEvent::InsertTtl((object, ttl)))));
        self.db_recv()
            .context("Failed to add to database")
            .map_err(Error::from)
@@ -96,7 +95,7 @@ pub trait State {
    fn db_activity(&self, activity: InsertActivity) -> Result<bool> {
        let activity = activity.try_into_new()?;

-        self.send(&Event::Database(DatabaseEvent::Activity(activity)));
+        self.send(&Event::Database(Box::new(DatabaseEvent::Activity(activity))));
        let r = self.db_recv()
            .context("Failed to log activity")?;

@@ -108,14 +107,14 @@ pub trait State {
    }

    fn db_select(&self, family: Family, value: String) -> Result<DatabaseResponse> {
-        self.send(&Event::Database(DatabaseEvent::Select((family, value))));
+        self.send(&Event::Database(Box::new(DatabaseEvent::Select((family, value)))));
        self.db_recv()
            .context("Failed to query database")
            .map_err(Error::from)
    }

    fn db_update(&self, family: Family, value: String, update: Update) -> Result<DatabaseResponse> {
-        self.send(&Event::Database(DatabaseEvent::Update((family, value, update))));
+        self.send(&Event::Database(Box::new(DatabaseEvent::Update((family, value, update)))));
        self.db_recv()
            .context("Failed to update database")
            .map_err(Error::from)
@@ -152,7 +151,7 @@ pub trait State {

    #[inline]
    fn random_id(&self) -> String {
-        thread_rng().sample_iter(&Alphanumeric).take(16).collect()
+        utils::random_string(16)
    }

    fn keyring(&self, namespace: &str) -> Vec<&KeyRingEntry>;
@@ -373,7 +372,7 @@ impl State for LuaState {
        let mtx = self.http_sessions.lock().unwrap();
        let session = mtx.get(session_id).expect("Invalid session reference"); // TODO

-        HttpRequest::new(&session, method, url, options)
+        HttpRequest::new(session, method, url, options, || format!("sn0int/{}", env!("CARGO_PKG_VERSION")))
    }

    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
@@ -620,6 +619,11 @@ impl Script {
        })
    }

+    #[inline]
+    pub fn code(&self) -> &str {
+        &self.code
+    }
+
    pub fn run(&self, env: Environment,
                      tx: Arc<Mutex<Box<dyn IpcChild>>>,
                      arg: AnyLuaValue,
@@ -681,6 +685,6 @@ a.prod.fastly.net
            geoip,
            asn,
        };
-        self.run(env, DummyIpcChild::new(), AnyLuaValue::LuaNil)
+        self.run(env, DummyIpcChild::create(), AnyLuaValue::LuaNil)
    }
 }
--- a/src/engine/mod.rs
+++ b/src/engine/mod.rs
@@ -1,4 +1,5 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};

 use crate::blobs::Blob;
 use crate::config::Config;
@@ -14,14 +15,13 @@ use std::sync::{Arc, Mutex};
 use crate::engine::ctx::Script;
 use crate::ipc::child::IpcChild;
 use sn0int_common::ModuleID;
-use sn0int_common::metadata::{Metadata, Source};
+use sn0int_common::metadata::{Metadata, Source, Stealth};
 use chrootable_https::dns::Resolver;
 use crate::psl::PslReader;
 use crate::paths;
 use std::cmp::Ordering;
 use std::path::Path;
 use crate::term;
-use crate::worker;

 pub mod ctx;
 pub use sn0int_std::engine::structs;
@@ -68,11 +68,9 @@ impl<'a> Library<'a> {
    }

    pub fn reload_modules(&mut self) -> Result<usize> {
-        let modules = worker::spawn_fn("Loading modules", || {
-            self.reload_modules_quiet()
-                .context("Failed to load modules")?;
-            Ok(self.list().len())
-        }, true)?;
+        self.reload_modules_quiet()
+            .context("Failed to load modules")?;
+        let modules = self.list().len();
        term::info(&format!("Loaded {} modules", modules));
        Ok(modules)
    }
@@ -122,14 +120,14 @@ impl<'a> Library<'a> {
                folder.to_owned()
            } else {
                let folder = folder.strip_prefix("~/")
-                    .unwrap_or(&folder);
+                    .unwrap_or(folder);

-                dirs::home_dir()
+                dirs_next::home_dir()
                    .ok_or_else(|| format_err!("Failed to find home folder"))?
                    .join(folder)
            };

-            self.load_module_folder(&folder, &author, true)?;
+            self.load_module_folder(&folder, author, true)?;
        }

        Ok(())
@@ -158,7 +156,7 @@ impl<'a> Library<'a> {
                continue;
            }

-            if let Err(err) = self.load_single_module(&module.path(), &author_name, &module_name, private_modules) {
+            if let Err(err) = self.load_single_module(&module.path(), author_name, module_name, private_modules) {
                let root = err.find_root_cause();
                term::warn(&format!("Failed to load {}/{}: {}", author_name, module_name, root));
            }
@@ -169,7 +167,7 @@ impl<'a> Library<'a> {

    pub fn load_single_module(&mut self, path: &Path, author_name: &str, module_name: &str, private_module: bool) -> Result<()> {
        let module_name = module_name.to_string();
-        let module = Module::load(path, &author_name, &module_name, private_module)
+        let module = Module::load(path, author_name, &module_name, private_module)
            .context(format!("Failed to parse {}/{}", author_name, module_name))?;

        for key in &[&module_name, &format!("{}/{}", author_name, module_name)] {
@@ -229,6 +227,8 @@ pub struct Module {
    version: String,
    source: Option<Source>,
    keyring_access: Vec<String>,
+    stealth: Stealth,
+
    private_module: bool,
    script: Script,
 }
@@ -240,7 +240,7 @@ impl Module {
            .context("Failed to read module")?;

        let metadata = code.parse::<Metadata>()
-            .context("Failed to parse module metadata")?;
+            .map_err(|e| format_err!("Failed to parse module metadata: {}", e))?;

        let script = Script::load_unchecked(code)?;

@@ -251,11 +251,18 @@ impl Module {
            version: metadata.version,
            source: metadata.source,
            keyring_access: metadata.keyring_access,
+            stealth: metadata.stealth,
+
            private_module,
            script,
        })
    }

+    #[inline]
+    pub fn author(&self) -> &str {
+        &self.author
+    }
+
    #[inline]
    pub fn name(&self) -> &str {
        &self.name
@@ -293,11 +300,21 @@ impl Module {
        &self.keyring_access
    }

+    #[inline]
+    pub fn stealth(&self) -> &Stealth {
+        &self.stealth
+    }
+
    #[inline]
    pub fn is_private(&self) -> bool {
        self.private_module
    }

+    #[inline]
+    pub fn code(&self) -> &str {
+        self.script.code()
+    }
+
    pub fn run(&self, env: Environment, ipc_child: Arc<Mutex<Box<dyn IpcChild>>>, arg: LuaJsonValue) -> Result<()> {
        debug!("Executing lua script {}", self.canonical());
        self.script.run(env, ipc_child, arg.into())
@@ -315,7 +332,7 @@ impl Module {
    pub fn source_equals(&self, other: &str) -> bool {
        match self.source() {
            Some(source) => source.group_as_str() == other,
-            None => other == "",
+            None => other.is_empty(),
        }
    }
 }
--- a/src/ipc/child.rs
+++ b/src/ipc/child.rs
@@ -3,15 +3,12 @@ use crate::ipc::common::*;
 use crate::engine::Environment;
 use crate::geoip::MaxmindReader;
 use crate::psl::PslReader;
-use serde_json;
 use crate::worker::Event;
-
 use std::fmt::Debug;
 use std::io::prelude::*;
 use std::io::{self, Stdin, Stdout};
 use std::sync::{Arc, Mutex};

-
 pub trait IpcChild: Debug {
    fn send(&mut self, event: &Event) -> Result<()>;

@@ -65,7 +62,7 @@ impl IpcChild for StdioIpcChild {
 pub struct DummyIpcChild;

 impl DummyIpcChild {
-    pub fn new() -> Arc<Mutex<Box<dyn IpcChild>>> {
+    pub fn create() -> Arc<Mutex<Box<dyn IpcChild>>> {
        Arc::new(Mutex::new(Box::new(DummyIpcChild)))
    }
 }
--- a/src/ipc/common.rs
+++ b/src/ipc/common.rs
@@ -2,12 +2,10 @@ use chrootable_https::dns::Resolver;
 use crate::blobs::Blob;
 use crate::engine::Module;
 use crate::keyring::KeyRingEntry;
-use serde_json;
-
+use serde::{Serialize, Deserialize};
 use std::collections::HashMap;
 use std::net::SocketAddr;

-
 #[derive(Debug, Serialize, Deserialize)]
 pub struct StartCommand {
    pub verbose: u64,
--- a/src/ipc/parent.rs
+++ b/src/ipc/parent.rs
@@ -4,9 +4,7 @@ use chrootable_https::dns::Resolver;
 use crate::blobs::Blob;
 use crate::engine::Module;
 use crate::keyring::KeyRingEntry;
-use serde_json;
 use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
-
 use std::collections::HashMap;
 use std::env;
 use std::ffi::OsString;
@@ -16,7 +14,6 @@ use std::net::SocketAddr;
 use std::sync::mpsc;
 use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};

-
 pub struct IpcParent {
    child: Child,
    stdin: ChildStdin,
@@ -65,7 +62,7 @@ impl IpcParent {

    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
        let value = serde_json::to_value(value).expect("Failed to serialize reply");
-        if let Err(_) = self.send(&value) {
+        if self.send(&value).is_err() {
            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
        }
    }
@@ -125,10 +122,10 @@ pub fn run(module: Module,
    let exit = loop {
        match ipc_parent.recv()? {
            Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => ipc_parent.send_event_callback(object, &tx),
+            Event::Database(object) => ipc_parent.send_event_callback(*object, tx),
            Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
-            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
-            Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),
+            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, tx),
+            Event::Blob(blob) => ipc_parent.send_event_callback(blob, tx),
            Event::Exit(event) => {
                if let ExitEvent::Err(err) = &event {
                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
--- a/src/keyring.rs
+++ b/src/keyring.rs
@@ -1,4 +1,5 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};

 use crate::engine::Module;
 use crate::hlua::AnyLuaValue;
@@ -169,7 +170,7 @@ impl KeyRing {

    pub fn unauthorized_namespaces<'a>(&self, module: &'a Module) -> Vec<&'a String> {
        module.keyring_access().iter()
-            .filter(|namespace| !self.is_access_granted(&module, &namespace))
+            .filter(|namespace| !self.is_access_granted(module, namespace))
            .collect()
    }

@@ -191,7 +192,7 @@ impl KeyRing {
    pub fn request_keys(&self, module: &Module) -> Vec<KeyRingEntry> {
        // TODO: we probably want to randomize the order
        module.keyring_access().iter()
-            .filter(|namespace| self.is_access_granted(&module, &namespace))
+            .filter(|namespace| self.is_access_granted(module, namespace))
            .flat_map(|namespace| self.list_for(namespace))
            .flat_map(|x| self.get(&x))
            .collect()
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,10 +1,9 @@
+#![allow(clippy::too_many_arguments)]
+#![allow(clippy::type_complexity)]
+#![allow(clippy::large_enum_variant)]
+
 #![warn(unused_extern_crates)]
-#[macro_use] extern crate maplit;
-use url;
 use hlua_badtouch as hlua;
-#[macro_use] extern crate serde_derive;
-#[macro_use] extern crate structopt;
-#[macro_use] extern crate lazy_static;
 #[macro_use] extern crate diesel;
 #[macro_use] extern crate diesel_migrations;
 #[macro_use] extern crate crossbeam_channel as channel;
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,4 +1,4 @@
-use env_logger::{self, Env};
+use env_logger::Env;
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
 use sn0int::cmd::{self, LiteCmd};
@@ -27,11 +27,11 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
        let path = Path::new(&args.module);

        let filename = path.file_stem()
-            .ok_or(format_err!("Failed to decode filename"))?
+            .ok_or_else(|| format_err!("Failed to decode filename"))?
            .to_str()
-            .ok_or(format_err!("Failed to decode filename"))?;
+            .ok_or_else(|| format_err!("Failed to decode filename"))?;

-        Module::load(&path.to_path_buf(), "anonymous", &filename, true)
+        Module::load(&path.to_path_buf(), "anonymous", filename, true)
            .context(format!("Failed to parse {:?}", path))?
    } else {
        rl.library().get(&args.module)?
@@ -41,7 +41,7 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
    rl.set_module(module);

    if let Some(target) = &args.target {
-        let target = shellwords::split(&target)
+        let target = shellwords::split(target)
            .map_err(|_| format_err!("Failed to parse target quotes"))?;
        let target = db::Filter::parse(&target)?;
        rl.set_target(Some(target));
@@ -71,6 +71,7 @@ fn run_cmd<T: cmd::Cmd>(gargs: &Args, args: T, config: &Config) -> Result<()> {
 }

 fn run_new(_gargs: &Args, args: &args::New) -> Result<()> {
+    // TODO: Add `-- Author: anonymous`
    let boilerplate = b"-- Description: TODO your description here
 -- Version: 0.1.0
 -- License: GPL-3.0
@@ -127,6 +128,7 @@ fn run() -> Result<()> {
        Some(SubCommand::Export(export)) => run_cmd(&args, export, &config),
        Some(SubCommand::Cal(cal)) => run_cmd(&args, cal, &config),
        Some(SubCommand::Notify(notify)) => run_cmd(&args, notify, &config),
+        Some(SubCommand::Stats(stats)) => run_cmd(&args, stats, &config),
        Some(SubCommand::Repl) => repl::run(&config),
        Some(SubCommand::Paths) => paths::run(&config),
        Some(SubCommand::Completions(completions)) => complete::run_generate(&completions),
--- a/src/models/account.rs
+++ b/src/models/account.rs
@@ -1,11 +1,10 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use chrono::NaiveDateTime;

-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="accounts"]
 pub struct Account {
@@ -210,7 +209,7 @@ impl Detailed for Account {
            displayname: self.displayname.clone(),
            email: self.email.clone(),
            url: self.url.clone(),
-            last_seen: self.last_seen.clone(),
+            last_seen: self.last_seen,
            unscoped: self.unscoped,
            birthday: self.birthday.clone(),
            phonenumber: self.phonenumber.clone(),
@@ -343,7 +342,7 @@ impl Upsert for AccountUpdate {
    }

    fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_account(&self)
+        db.update_account(self)
    }
 }

--- a/src/models/activity.rs
+++ b/src/models/activity.rs
@@ -1,6 +1,6 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};
 use crate::schema::activity;
-use diesel;
 use diesel::prelude::*;
 use diesel::query_builder::BoxedSelectStatement;
 use crate::models::*;
@@ -8,7 +8,6 @@ use chrono::NaiveDateTime;
 use std::convert::TryFrom;
 use std::io::Write;

-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="activity"]
 pub struct Activity {
@@ -84,6 +83,14 @@ impl Activity {
            .optional()
            .map_err(Error::from)
    }
+
+    pub fn count(db: &Database) -> Result<usize> {
+        use crate::schema::activity::dsl::*;
+        activity.count()
+            .get_result::<i64>(db.db())
+            .map(|x| x as usize)
+            .map_err(Error::from)
+    }
 }

 pub struct ActivityFilter {
--- a/src/models/breach.rs
+++ b/src/models/breach.rs
@@ -1,10 +1,9 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;

-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="breaches"]
 pub struct Breach {
--- a/src/models/breach_email.rs
+++ b/src/models/breach_email.rs
@@ -1,9 +1,8 @@
 use crate::errors::*;
-use diesel;
+use serde::{Serialize, Deserialize};
 use diesel::prelude::*;
 use crate::models::*;

-
 #[derive(Identifiable, Queryable, Associations, Serialize, Deserialize)]
 #[belongs_to(Breach)]
 #[belongs_to(Email)]
@@ -136,8 +135,8 @@ impl Printable<PrintableBreachEmail> for BreachEmail {
        let breach = Breach::by_id(db, self.breach_id)?;
        let email = Email::by_id(db, self.email_id)?;
        Ok(PrintableBreachEmail {
-            breach: breach.value.to_string(),
-            email: email.value.to_string(),
+            breach: breach.value,
+            email: email.value,
            password: self.password.clone(),
        })
    }
@@ -167,8 +166,8 @@ impl Printable<PrintableBreachEmail> for NewBreachEmail {
        let breach = Breach::by_id(db, self.breach_id)?;
        let email = Email::by_id(db, self.email_id)?;
        Ok(PrintableBreachEmail {
-            breach: breach.value.to_string(),
-            email: email.value.to_string(),
+            breach: breach.value,
+            email: email.value,
            password: self.password.clone(),
        })
    }
--- a/src/models/cryptoaddr.rs
+++ b/src/models/cryptoaddr.rs
@@ -1,11 +1,10 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use chrono::NaiveDateTime;

-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="cryptoaddrs"]
 pub struct CryptoAddr {
@@ -220,11 +219,11 @@ impl Detailed for CryptoAddr {
            id: self.id,
            value: self.value.to_string(),
            currency: self.currency.clone(),
-            denominator: self.denominator.clone(),
-            balance: self.balance.clone(),
-            received: self.received.clone(),
-            first_seen: self.first_seen.clone(),
-            last_withdrawal: self.last_withdrawal.clone(),
+            denominator: self.denominator,
+            balance: self.balance,
+            received: self.received,
+            first_seen: self.first_seen,
+            last_withdrawal: self.last_withdrawal,
            unscoped: self.unscoped,
            description: self.description.clone(),
        })
@@ -327,7 +326,7 @@ impl Upsert for CryptoAddrUpdate {
    }

    fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_cryptoaddr(&self)
+        db.update_cryptoaddr(self)
    }
 }

--- a/src/models/device.rs
+++ b/src/models/device.rs
@@ -1,11 +1,10 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use chrono::NaiveDateTime;

-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="devices"]
 pub struct Device {
@@ -216,7 +215,7 @@ impl Detailed for Device {
            ipaddr,
            network,
            unscoped: self.unscoped,
-            last_seen: self.last_seen.clone(),
+            last_seen: self.last_seen,
        })
    }
 }
--- a/src/models/domain.rs
+++ b/src/models/domain.rs
@@ -1,10 +1,9 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;

-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="domains"]
 pub struct Domain {
--- a/src/models/email.rs
+++ b/src/models/email.rs
@@ -1,10 +1,9 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};
 use crate::fmt::colors::*;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;

-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="emails"]
 pub struct Email {
--- a/src/models/image.rs
+++ b/src/models/image.rs
@@ -1,13 +1,12 @@
 use crate::errors::*;
+use serde::{Serialize, Deserialize};
 use crate::fmt::colors::*;
 use chrono::NaiveDateTime;
-use diesel;
 use diesel::prelude::*;
 use crate::models::*;
 use std::sync::Arc;
 use crate::engine::ctx::State;

-
 #[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
 #[table_name="images"]
 pub struct Image {
@@ -174,8 +173,8 @@ impl Printable<PrintableImage> for Image {
        Ok(PrintableImage {
            value: self.value.to_string(),
            filename: self.filename.clone(),
-            width: self.width.clone(),
-            height: self.height.clone(),
+            width: self.width,
+            height: self.height,
        })
    }
 }
@@ -256,14 +255,14 @@ impl Detailed for Image {

            filename: self.filename.clone(),
            mime: self.mime.clone(),
-            width: self.width.clone(),
-            height: self.height.clone(),
-            created: self.created.clone(),
+            width: self.width,
+            height: self.height,
+            created: self.created,

-            latitude: self.latitude.clone(),
-            longitude: self.longitude.clone(),
+            latitude: self.latitude,
+            longitude: self.longitude,

-            nudity: self.nudity.clone(),
+            nudity: self.nudity,
            ahash: self.ahash.clone(),
            dhash: self.dhash.clone(),
            phash: self.phash.clone(),
@@ -341,8 +340,8 @@ impl Printable<PrintableImage> for NewImage {
        Ok(PrintableImage {
            value: self.value.to_string(),
            filename: self.filename.clone(),
-            width: self.width.clone(),
-            height: self.height.clone(),
+            width: self.width,
+            height: self.height,
        })
    }
 }
@@ -443,7 +442,7 @@ impl Upsert for ImageUpdate {
    }

    fn apply(&self, db: &Database) -> Result<i32> {
-        db.update_image(&self)
+        db.update_image(self)
    }
 }

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
kpcyrd	8c486b7e6e	Release v0.22.0	2021-09-08 18:44:54 +02:00
kpcyrd	e939a28469	Merge pull request #210 from kpcyrd/bump Update dependencies	2021-09-08 12:12:28 +00:00
kpcyrd	2dec9dd28c	Update dependencies	2021-09-08 13:50:57 +02:00
kpcyrd	06459098b1	Merge pull request #209 from kpcyrd/update Update dependencies, fix clippy warnings	2021-09-06 15:07:53 +00:00
kpcyrd	fc0447725c	Fix clippy warnings	2021-09-06 16:26:45 +02:00
kpcyrd	8312695e46	Update dependencies	2021-09-06 15:44:45 +02:00
kpcyrd	6c8e2df632	Update to nom7	2021-08-21 13:31:09 +02:00
kpcyrd	b1ee2d4ce7	Merge pull request #208 from kpcyrd/update Update dependencies	2021-08-17 20:48:13 +00:00
kpcyrd	4cbb72e4c8	Update dependencies	2021-08-17 21:22:48 +02:00
kpcyrd	9e76935f55	Merge pull request #207 from kpcyrd/readline-exec Support invoking shell commands with `!` from readline	2021-08-01 17:00:10 +00:00
kpcyrd	8b5ad635fa	Support invoking shell commands with `!` from readline	2021-07-29 14:15:00 +02:00
kpcyrd	9f4914f419	Release v0.21.2	2021-06-23 23:12:19 +02:00
kpcyrd	f24b1b2b9b	Update dockerfile baseimage	2021-06-23 23:07:37 +02:00
kpcyrd	735e2dbaf8	Merge pull request #205 from kpcyrd/update Update dependencies, fix seccomp issue on x86_64 musl	2021-06-23 21:03:19 +00:00
kpcyrd	5409ed8cdb	Fix integration tests	2021-06-23 17:33:09 +02:00
kpcyrd	d0a36be95a	Update dependencies	2021-06-23 17:00:11 +02:00
kpcyrd	0ab8aace70	Fix seccomp segfault with open on x86_64 musl	2021-06-23 17:00:11 +02:00
kpcyrd	699ebad23a	registry: Set user agent when authenticating with github	2021-06-23 16:22:31 +02:00
kpcyrd	84e0e62753	Release v0.21.1	2021-05-18 00:49:10 +02:00
kpcyrd	dc8aac1b0f	Merge pull request #202 from kpcyrd/aarch64 Update dependencies to fix aarch64 on musl	2021-05-17 22:46:55 +00:00
kpcyrd	7b4d599951	Update dependencies to fix aarch64 on musl	2021-05-18 00:30:39 +02:00
kpcyrd	d88c722a4b	Release v0.21.0	2021-05-12 19:19:30 +02:00
kpcyrd	d70fc8f4c0	Merge pull request #200 from kpcyrd/update Update dependencies	2021-05-12 16:54:46 +00:00
kpcyrd	da08f3a4bb	Update dependencies	2021-05-12 18:27:44 +02:00
kpcyrd	892fa0d3e4	Merge pull request #199 from kpcyrd/stats Improve stats command	2021-04-07 01:03:21 +00:00
kpcyrd	605d691b28	Add `select --values` as shorthand for `jq -r .value`	2021-04-07 02:34:41 +02:00
kpcyrd	efb458a725	Allow deleting multiple workspaces at once	2021-04-07 02:25:58 +02:00
kpcyrd	84e147f709	Add `stats -q` for more compact output	2021-04-07 02:11:29 +02:00
kpcyrd	1876e9ac8d	Add `stats --all` to show stats for all workspaces	2021-04-07 01:51:57 +02:00
kpcyrd	8525883b91	README: Add packaging status	2021-03-23 01:29:53 +00:00
kpcyrd	02c537c253	Merge pull request #197 from kpcyrd/stats Add sn0int stats	2021-03-20 13:38:12 +00:00
kpcyrd	d84038dcc3	Add irc notifications for github actions	2021-03-18 03:31:07 +01:00
kpcyrd	becb44a5d7	Update dependencies	2021-03-18 02:38:47 +01:00
kpcyrd	6e1904eaf9	Extend interactive completions for pkg list	2021-03-14 19:38:44 +01:00
kpcyrd	d493857011	Add stats --json output	2021-03-14 18:40:47 +01:00
kpcyrd	3af9abde1b	Add stats command	2021-03-14 01:34:42 +01:00
kpcyrd	0ea8c46578	Release v0.20.1	2021-03-07 16:07:34 +01:00
kpcyrd	2e231da60c	Merge pull request #196 from kpcyrd/update Speedup initial database setup	2021-03-07 14:58:12 +00:00
kpcyrd	a97a7c6a4c	Speedup docker build on github actions	2021-03-07 15:40:02 +01:00
kpcyrd	2b09ca997c	Introduce github actions	2021-03-07 14:44:09 +01:00
kpcyrd	66fa77d16e	Speedup initial database setup	2021-03-07 14:11:37 +01:00
kpcyrd	23e06ede3b	Update dependencies	2021-03-07 14:11:25 +01:00
kpcyrd	7b994d7cae	Merge pull request #195 from steev/master Mention pkg-config being needed.	2021-01-03 20:06:05 +00:00
Steev Klimaszewski	59b591d0e8	Mention pkg-config being needed. Not all installations have build tools installed, and libsodium requires calling out to pkg-config so add it to the build instructions on Debian so that the build doesn't fail.	2021-01-03 10:54:13 -06:00
kpcyrd	309be4b22f	Release v0.20.0	2020-12-28 20:47:45 +01:00
kpcyrd	02d0ccce64	Merge pull request #194 from kpcyrd/bump Bump dependencies	2020-12-28 19:03:23 +00:00
kpcyrd	a15d8167b9	Fix clippy warnings	2020-12-28 18:12:45 +01:00
kpcyrd	d125d91f0d	Update dependencies	2020-12-28 17:13:40 +01:00
kpcyrd	01e4f87420	Update dependencies	2020-12-28 15:58:10 +01:00
kpcyrd	037189ec57	Merge pull request #189 from kpcyrd/redirects Add option to follow redirects	2020-12-28 13:17:22 +00:00
kpcyrd	0d77c06a95	Fix default user agent	2020-11-01 23:20:28 +01:00
kpcyrd	247648aff8	Add redirect support	2020-11-01 23:04:06 +01:00
kpcyrd	f73c375d12	Merge pull request #187 from kpcyrd/add-stdin Allow streaming some structs from stdin into db	2020-11-01 20:11:59 +00:00
kpcyrd	484507e033	Merge pull request #188 from kpcyrd/inverse-rules Add inverse rules	2020-11-01 20:04:42 +00:00
kpcyrd	27588f5319	Allow streaming some structs from stdin into db	2020-11-01 19:28:50 +01:00
kpcyrd	c2b535eeed	Merge pull request #186 from kpcyrd/bump Bump dependencies	2020-11-01 16:33:44 +00:00
kpcyrd	410a381643	Update kali install instructions	2020-11-01 03:46:56 +01:00
kpcyrd	f702f48708	Update dependencies	2020-10-27 00:56:26 +01:00
kpcyrd	031a269de0	Update cert parser	2020-10-27 00:49:34 +01:00
kpcyrd	c1b38c8fa6	Add inverse rules	2020-10-27 00:07:00 +01:00
kpcyrd	afe302c101	Update dependencies	2020-10-02 21:23:28 +02:00
kpcyrd	b09c820a11	Merge pull request #183 from kpcyrd/stealth Add support to track module stealth in metadata	2020-09-25 21:20:32 +00:00
kpcyrd	301ad3cf44	Show possible stealth values in help	2020-09-25 22:51:03 +02:00
kpcyrd	53b1e9fd1a	Align pkg-search style with pkg-list	2020-09-25 22:41:33 +02:00
kpcyrd	5c447d259f	Update dependencies	2020-09-25 20:50:14 +02:00
kpcyrd	67082a1002	Allow filtering by stealth level in `pkg list`	2020-09-13 22:40:42 +02:00
kpcyrd	152dd82848	Show stealth status in `pkg list`	2020-09-13 22:19:01 +02:00
kpcyrd	21c70aaf58	Add author and repository meta fields	2020-09-13 21:36:33 +02:00
kpcyrd	a93d9ddbe2	Add `-- Stealth` metadata option	2020-09-13 21:24:17 +02:00
kpcyrd	1de7f1a2d2	Highlight weekends in cal time/hour view	2020-07-08 02:21:41 +02:00
kpcyrd	4a82171ca1	Correctly clear terminal after worker::spawn_multi	2020-06-26 03:18:11 +02:00
kpcyrd	1828b67509	Release v0.19.1	2020-06-24 00:46:45 +02:00
kpcyrd	8f2d71e631	Merge pull request #181 from kpcyrd/notify-fix Fix early exit on notification and make ratelimiter more global	2020-06-23 23:11:59 +02:00
kpcyrd	2277089bda	Share the ratelimit state between main modules and all notifications	2020-06-23 05:41:24 +02:00
kpcyrd	63f226f68f	Prevent notifications from interfering with ctrl-c register	2020-06-23 04:55:16 +02:00