Release v0.18.1

Add select --count

.dockerignore

@@ -1,6 +1,7 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
 docker
 docs
 ci

.travis.yml

@@ -5,18 +5,6 @@ language: rust
 
 matrix:
   include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
   - os: linux
     rust: stable
     env:
@@ -25,6 +13,22 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=docker-registry
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=test
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=common
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=std
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=boxxy
   - os: osx
     rust: stable
     env:
@@ -33,14 +37,18 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=common
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=std
   #- os: windows
   #  rust: stable
   #  env:
   #  - BUILD_MODE="windows test"
-  - os: windows
-    rust: stable
-    env:
-    - BUILD_MODE="windows common"
+  #- os: windows
+  #  rust: stable
+  #  env:
+  #  - BUILD_MODE="windows common"
 
 before_install:
 - ci/setup.sh "$TRAVIS_OS_NAME"
@@ -51,8 +59,11 @@ script:
 
 notifications:
   irc:
+    template:
+    - "%{repository_slug}#%{build_number} (%{branch} - %{commit} : %{author}): %{message} (%{build_url})"
     channels:
     - "ircs://irc.hackint.org:6697/#sn0int"
+    use_notice: true
+    skip_join: true
     #on_success: change # default: always
     #on_failure: always # default: always
-    use_notice: true

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.14.0"
+version = "0.18.1"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -13,15 +13,17 @@ edition = "2018"
 travis-ci = { repository = "kpcyrd/sn0int" }
 
 [workspace]
-members = ["sn0int-registry/sn0int-common",
-           "sn0int-registry"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
 
 [features]
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-registry/sn0int-common" }
-rustyline = "5.0"
+sn0int-common = { version="0.10.0", path="sn0int-common" }
+sn0int-std = { version="=0.18.1", path="sn0int-std" }
+rustyline = "6.0"
 log = "0.4"
 env_logger = "0.7"
 hlua-badtouch = "0.4"
@@ -31,7 +33,6 @@ rand = "0.7"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
 libsqlite3-sys = "0.16.0"
@@ -40,14 +41,9 @@ dirs = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.13"
-rustls = { version="0.16", features=["dangerous_configuration"] }
-webpki = "0.21"
-webpki-roots = "0.18"
-pem = "0.7"
-base64 = "0.11"
+chrootable-https = "0.15"
+base64 = "0.12"
 data-encoding = "2.1.2"
-kuchiki = "0.7.2"
 serde_urlencoded = "0.6"
 serde = "1.0"
 serde_derive = "1.0"
@@ -60,45 +56,35 @@ maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.5"
 regex = "1.0"
 toml = "0.5"
-maxminddb = "0.13"
-tar = "0.4.17"
-libflate = "0.1.14"
 threadpool = "1.7"
-x509-parser = "0.6.0"
-der-parser = "3.0"
 atty = "0.2"
-bufstream = "0.1.4"
-tokio = "0.1.14"
 semver = "0.9"
 bytes = "0.4"
-xml-rs = "0.8"
 bytesize = "1.0"
-ipnetwork = "0.15"
-strum = "0.16"
-strum_macros = "0.16"
+ipnetwork = "0.16"
+strum = "0.18"
+strum_macros = "0.18"
+embedded-triple = "0.1.0"
 
 digest = "0.8.0"
-bs58 = "0.3"
-blake2 = "0.8.0"
 md-5 = "0.8.0"
 sha-1 = "0.8.1"
 sha2 = "0.8.0"
 sha3 = "0.8.0"
 hmac = "0.7"
 
-image = "0.22"
-kamadak-exif = "0.3.1"
 walkdir = "2.2"
-nude = "0.2"
+nude = "0.3"
+glob = "0.3.0"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.11"
-nix = "0.15"
+syscallz = "0.12"
+nix = "0.17"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
-pledge = "0.3.1"
+pledge = "0.4"
 unveil = "0.2.0"
 
 [dev-dependencies]

Dockerfile

@@ -1,14 +1,13 @@
-FROM rust:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

Makefile

@@ -1,19 +1,23 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
 
 update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt

README.md

@@ -89,10 +89,10 @@ For everything else please have a look at the [detailed list][1].
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-  - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope)
-    - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains)
-    - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips)
-    - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls)
+- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains)
+  - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls)
 - [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
   - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
@@ -100,6 +100,8 @@ For everything else please have a look at the [detailed list][1].
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
 - [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
@@ -116,10 +118,17 @@ For everything else please have a look at the [detailed list][1].
   - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
   - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
   - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [CryptoAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#cryptoaddrs)
+  - [Activity](https://sn0int.readthedocs.io/en/latest/structs.html#activity)
   - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
     - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
     - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
     - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Activity](https://sn0int.readthedocs.io/en/latest/activity.html)
+  - [Anatomy of an event](https://sn0int.readthedocs.io/en/latest/activity.html#anatomy-of-an-event)
+  - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
+  - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
+  - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
@@ -147,6 +156,7 @@ For everything else please have a look at the [detailed list][1].
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity)
   - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
@@ -170,16 +180,23 @@ For everything else please have a look at the [detailed list][1].
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [key_trunc_pad](https://sn0int.readthedocs.io/en/latest/reference.html#key-trunc-pad)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
   - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [mqtt_connect](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-connect)
+  - [mqtt_subscribe](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-subscribe)
+  - [mqtt_recv](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-recv)
+  - [mqtt_ping](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-ping)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [ratelimit_throttle](https://sn0int.readthedocs.io/en/latest/reference.html#ratelimit-throttle)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
   - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
@@ -195,6 +212,7 @@ For everything else please have a look at the [detailed list][1].
   - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
   - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_options](https://sn0int.readthedocs.io/en/latest/reference.html#sock-options)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
   - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
@@ -206,11 +224,15 @@ For everything else please have a look at the [detailed list][1].
   - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
   - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [sodium_secretbox_open](https://sn0int.readthedocs.io/en/latest/reference.html#sodium-secretbox-open)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
   - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
   - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
   - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval)
   - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
@@ -221,6 +243,14 @@ For everything else please have a look at the [detailed list][1].
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
   - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
   - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [ws_connect](https://sn0int.readthedocs.io/en/latest/reference.html#ws-connect)
+  - [ws_options](https://sn0int.readthedocs.io/en/latest/reference.html#ws-options)
+  - [ws_recv_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-text)
+  - [ws_recv_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-binary)
+  - [ws_recv_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-json)
+  - [ws_send_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-text)
+  - [ws_send_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-binary)
+  - [ws_send_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-json)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
   - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
   - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)

ci/integration.py

@@ -47,10 +47,10 @@ def main(tempdir, binary):
 
     print('[*] installing modules')
     sn0int(tempdir, binary, [
-        'mod install kpcyrd/ctlogs',
-        'mod install kpcyrd/dns-resolve',
-        'mod install kpcyrd/url-scan',
-        'mod install kpcyrd/geoip',
+        'pkg install kpcyrd/ctlogs',
+        'pkg install kpcyrd/dns-resolve',
+        'pkg install kpcyrd/url-scan',
+        'pkg install kpcyrd/geoip',
     ])
 
     print('[*] running ctlogs')
@@ -63,6 +63,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,6 +92,8 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }

ci/run.sh

@@ -6,18 +6,18 @@ case "$1" in
         cargo build --verbose --examples
         ;;
     test)
-        ci/run.sh build
-        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
-             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;
     common)
-        cd sn0int-registry/sn0int-common
+        cd sn0int-common
         cargo test --verbose
         ;;
+    std)
+        cd sn0int-std
+        cargo test --verbose
+        cargo test --verbose -- --ignored
+        ;;
     windows)
         cargo build --verbose --features=sqlite-bundled
         cargo build --verbose --examples --features=sqlite-bundled
@@ -35,7 +35,7 @@ case "$1" in
         docker run --rm sn0int --help
         ;;
     docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
+        docker build -t sn0int-registry -f sn0int-registry/Dockerfile .
         docker images
         ;;
 esac

ci/setup.sh

@@ -3,6 +3,9 @@ set -exu
 case "$1" in
     linux)
         sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev
+        sudo apt install libsqlite3-dev libseccomp-dev libsodium-dev
+        ;;
+    osx)
+        brew install libsodium
         ;;
 esac

contrib/docker/Dockerfile.alpine

@@ -1,13 +1,13 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

contrib/docker/Dockerfile.debian

@@ -1,13 +1,13 @@
-FROM rust
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]

docs/activity.rst

@@ -0,0 +1,166 @@
+Activity
+========
+
+So far we've learned about regular `structs <structs.html>`_, but activity is
+special.
+
+Activity is an event tied to a specific time and topic and has a small amount
+of data piggybacked to it.
+
+Anatomy of an event
+-------------------
+
+``topic``
+    This is some freestyle text used to group events to a specific topic. This
+    must not conflict with other modules unless there's a very good reason.
+
+    The topic should look like ``kpcyrd/example:something``, with ``something``
+    being a meaningful unique identifier for whatever is generating these
+    events, like a mac address or an account name/id.
+
+    The rules around this might become stricter in the future.
+``time``
+    The most important part of the event: The time and date it happened.
+``initial``
+    This value can not be set but might be present in sn0int output. See `Querying events`_.
+``uniq`` (optional)
+    This is an optional feature to deduplicate events. Assuming you're
+    importing posts by an account, you wouldn't want to store a new event for
+    each post you already imported. If you set this field to the technical post
+    id then sn0int would skip the event if it already has an event with the
+    same ``topic`` and ``uniq`` combination to avoid inserting duplicates.
+``latitude`` (optional)
+    Latitude - if you can tie the event to a specific location.
+``longitude`` (optional)
+    Longitude - if you can tie the event to a specific location.
+``radius`` (optional)
+    The location radius in meters. If the position you got has a precision of
+    100 meters set this value to ``100``.
+``content``
+    Arbitrary data that you want to attach to the event. This doesn't need to
+    be a string and can be an arbitrary object that is then stored as json
+    string.
+
+Logging events
+--------------
+
+An ``activity`` event can be logged with ``db_activity``:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Logging an event that has a location attached could look like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        latitude=40.726662,
+        longitude=-74.036677,
+        radius=50,
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Making sure an event is not logged twice can be done with ``uniq``:
+
+.. code-block:: lua
+
+    -- create the first event
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this does nothing because we already have an event with this topic+uniq combination
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this creates a new event because uniq is different
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+    -- this also creates a new event because topic is different
+    db_activity({
+        topic='harness/activity-ping:something-else',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+Querying events
+---------------
+
+There is a commandline interface that can be used to query all events we've
+logged. To get everything (sorted by time)::
+
+    sn0int activity
+
+To limit the output to a specific topic::
+
+    sn0int activity -t harness/activity-ping:dummy
+
+To limit it to a specific time frame::
+
+    # everything since
+    sn0int activity --since 2020-01-13T04:20:00
+    # everything until
+    sn0int activity --until 2020-01-13T04:20:00
+    # both
+    sn0int activity --since yesterday --until today
+
+When using ``--since`` you might also want to know the previous state and use
+it as an initial value. Consider this example::
+
+    2020-01-13 14:30:00 # user goes offline
+    2020-01-13 23:59:00 # user goes online
+    2020-01-14 09:30:00 # user goes idle
+    2020-01-14 14:20:00 # user goes offline
+
+If we're running a query like ``sn0int activity --since 2020-01-14T00:00:00``
+the program consuming the output wouldn't know that the user is initially
+online because we're only getting this data::
+
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+We can tweak this with ``sn0int activity --initial --since
+2020-01-14T00:00:00`` to include one more event that we only use to populate
+the intial state::
+
+    {"id":7,"initial":true,"topic":"foo/bar:asdf","time":"2020-01-13T23:59:00","content":{"state":"online"}}
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+Visualization
+-------------
+
+There is no visualization built in, there may be external frontends for this in
+the future. You're very welcome to write one!

docs/autonoscope.rst

@@ -0,0 +1,81 @@
+Autonoscope
+===========
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+-------
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+---
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+----
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

docs/build.rst

@@ -18,19 +18,21 @@ Archlinux
 
 .. code-block:: bash
 
-    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+    $ pacman -S geoip2-database libseccomp libsodium publicsuffix-list sqlite
 
 Mac OSX
 ~~~~~~~
 
-None.
+.. code-block:: bash
+
+    $ brew install libsodium
 
 Debian/Ubuntu/Kali
 ~~~~~~~~~~~~~~~~~~
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
 
 .. warning::
    On a debian based system make sure you've installed rust with rustup.
@@ -40,21 +42,21 @@ Alpine
 
 .. code-block:: bash
 
-    $ apk add sqlite-dev libseccomp-dev
+    $ apk add sqlite-dev libseccomp-dev libsodium-dev
 
 OpenBSD
 ~~~~~~~
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3 geolite2-city geolite2-asn
+    $ pkg_add sqlite3 geolite2-city geolite2-asn libsodium
 
 Gentoo
 ~~~~~~
 
 .. code-block:: bash
 
-    emerge --ask sys-libs/libseccomp dev-db/sqlite
+    emerge --ask sys-libs/libseccomp dev-db/sqlite dev-libs/libsodium
 
 Windows
 ~~~~~~~

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'sn0int'
-copyright = '2018, kpcyrd'
+copyright = '2018-2020, kpcyrd'
 author = 'kpcyrd'
 
 # The short X.Y version

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/index.rst

@@ -38,9 +38,11 @@ Getting Started
    install
    build
    usage
+   autonoscope
    scripting
    database
    structs
+   activity
    keyring
    config
    sandbox

docs/install.rst

@@ -32,7 +32,7 @@ at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .
@@ -47,8 +47,6 @@ Docker
 Alpine
 ------
 
-On alpine edge, with enabled testing repositories:
-
 .. code-block:: bash
 
     $ apk add sn0int
@@ -56,8 +54,6 @@ On alpine edge, with enabled testing repositories:
 OpenBSD
 -------
 
-On -current:
-
 .. code-block:: bash
 
     $ pkg_add sn0int

docs/keyring.rst

@@ -58,6 +58,10 @@ If the user granted us access to those keys we can read them with ``keyring``:
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.
 
+If you want to allow the user to select a specific script you can introduce an
+option that is set by the user and then filter ``creds`` until the
+``access_key`` matches.
+
 Using access keys as source argument
 ------------------------------------
 

docs/reference.rst

@@ -153,6 +153,28 @@ ttl.
         device_id=13,
     }, 120)
 
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_select
 ---------
 
@@ -369,6 +391,8 @@ options are set. The following options are available:
 ``proxy``
   Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
   if it doesn't conflict with the global proxy settings.
+``binary``
+  Set to ``true`` to get the http response as raw bytes.
 
 This function may fail.
 
@@ -396,6 +420,8 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``binary``
+  The response body as bytes (if ``binary=true``)
 ``blob``
   If ``into_blob`` was enabled for the request the body is downloaded into blob
   storage with a reference to the body in this field.
@@ -495,6 +521,13 @@ Log an info to the terminal.
 
     info('ohai')
 
+intval
+------
+
+Parse a number from a string.
+
+    x = strval('1234')
+
 json_decode
 -----------
 
@@ -529,6 +562,17 @@ Encode a datastructure into a string.
     })
     print(x)
 
+key_trunc_pad
+-------------
+
+Truncate/pad a key to a given length.
+
+.. code-block:: lua
+
+    -- if longer than 32 bytes: truncate to 32
+    -- if shorter than 32 bytes: pad with \x00
+    local key = key_trunc_pad(password, 32, 0)
+
 keyring
 -------
 
@@ -563,6 +607,64 @@ Hash a byte array with md5 and return the results as bytes.
 
     hex(md5("\x00\xff"))
 
+mqtt_connect
+------------
+
+Connect to an mqtt broker.
+
+.. code-block:: lua
+
+    local sock = mqtt_connect('mqtts://mqtt.example.com', {
+        username='foo',
+        password='secret',
+    })
+    if last_err() then return end
+
+mqtt_subscribe
+--------------
+
+Subscribe to a topic. Right now only QoS 0 is supported.
+
+.. code-block:: lua
+
+    mqtt_subscribe(sock, '#', 0)
+    if last_err() then return end
+
+mqtt_recv
+---------
+
+Receive an mqtt packet. This is not necessarily a publish packet and more
+packets might be added in the future, so you need to check the type
+specifically.
+
+If a read timeout has been set with mqtt_connect_ this function returns ``nil``
+in case of a read timeout.
+
+.. code-block:: lua
+
+    local pkt = mqtt_recv(sock)
+    if last_err() then return end
+    if pkt == nil then
+        -- read timeout, consider sending a ping or disconnect if the previous ping failed
+    elseif pkt['type'] == 'pong' then
+        -- broker sent a pong
+    elseif pkt['type'] == 'publish' then
+        local payload = utf8_decode(pkt['body'])
+        if last_err() then return end
+        info(payload)
+    end
+
+mqtt_ping
+---------
+
+Send a pingreq packet, causing the broker to send a pingresp. This is used to
+make sure the connection is still working correctly.
+
+.. code-block:: lua
+
+    mqtt_ping(sock)
+    if last_err() then return end
+
 pgp_pubkey
 ----------
 
@@ -643,6 +745,22 @@ Returns the parent domain according to the public suffix list. For
     domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
     print(domain == 'example.co.uk')
 
+ratelimit_throttle
+------------------
+
+Create a ratelimit that can only be passed x times every y milliseconds. This
+limit is global for a single ``run`` and also works with threads.
+
+.. code-block:: lua
+
+    -- allow this to pass every 250ms
+    ratelimit_throttle('foo', 1, 250)
+    -- allow this to pass not more than 4 times per second
+    ratelimit_throttle('foo', 4, 1000)
+
+This is useful if you need to coordinate your executions to stay below a
+certain request threshold.
+
 regex_find
 ----------
 
@@ -818,6 +936,13 @@ The following options are available:
 ``proxy``
   Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
   if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
 
 .. code-block:: lua
 
@@ -849,6 +974,23 @@ discarded when using sock_connect_ directly with ``tls=true``.
 
     info(tls)
 
+sock_options
+------------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock_options(sock, {
+        read_timeout=3,
+    })
+
 sock_send
 ---------
 
@@ -950,6 +1092,27 @@ Overwrite the default ``\n`` newline.
 
     sock_newline(sock, "\r\n")
 
+sodium_secretbox_open
+---------------------
+
+Use authenticated symetric crypto to decrypt a given message.
+
+Internally this is ``crypto_secretbox_xsalsa20poly1305``.
+
+The key **must** be 32 bytes, see key_trunc_pad_ if necessary.
+
+The first 24 bytes of the encrypted message are expected to be the nonce.
+
+.. code-block:: lua
+
+    plain = sodium_secretbox_open(encrypted, key)
+    if last_err() then return end
+
+    txt = utf8_decode(plain)
+    if last_err() then return end
+
+    info(txt)
+
 status
 ------
 
@@ -985,6 +1148,32 @@ Read stdin until EOF as a utf-8 string.
 .. note::
    This only works with `sn0int run --stdin`.
 
+str_find
+--------
+
+Returns the byte index of the first character that matches the pattern. This is
+explicitly a literal match instead of a lua pattern.
+
+If no match is found, returns ``nil``.
+
+.. code-block:: lua
+
+    x = str_find('asdf', 'sd')
+    print(x == 2)
+
+str_replace
+-----------
+
+Replaces all matches of a pattern in a string. This is explicitly a literal
+match instead of a lua pattern.
+
+If no match is found, an unmodified copy is returned.
+
+.. code-block:: lua
+
+    x = str_replace('this is old', 'old', 'new')
+    print(x == 'this is new')
+
 strftime
 --------
 
@@ -1005,6 +1194,13 @@ Parse a date into a unix timestamp, see `strftime rules`_.
 
 .. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
 
+strval
+------
+
+Convert a number into a string.
+
+    x = strval(1234)
+
 time_unix
 ---------
 
@@ -1125,6 +1321,111 @@ a ``run`` execution.
     warn_once('ohai')
     warn_once('ohai')
 
+ws_connect
+----------
+
+Create a websocket connection. The url format is ``ws://example.com/asdf``,
+``wss://`` is also supported.
+
+The following options are available:
+
+``headers``
+  A map of additional headers that should be set for the request.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock = ws_connect("wss://example.com/asdf", {})
+
+ws_options
+----------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    ws_options(sock, {
+        read_timeout=3,
+    })
+
+ws_recv_text
+------------
+
+Wait until the server sends a text frame. A binary frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_text(sock)
+
+ws_recv_binary
+--------------
+
+Wait until the server sends a binary frame. A text frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_binary(sock)
+
+ws_recv_json
+------------
+
+Identical to ws_send_text_ but automatically runs json_decode_ on the
+response.
+
+.. code-block:: lua
+
+    msg = ws_recv_json(sock)
+
+ws_send_text
+------------
+
+Send a text frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, "ohai!")
+
+ws_send_binary
+--------------
+
+Send a binary frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_binary(sock, "\x00\x01\x02")
+
+ws_send_json
+------------
+
+Encode the object as json string and send it as a text frame on the websocket
+connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, {
+        foo="ohai!",
+        x={
+            y={1,3,3,7},
+        },
+    })
+
 x509_parse_pem
 --------------
 

docs/structs.rst

@@ -259,6 +259,35 @@ is ``140.82.112.0/20``.
     This field isn't strictly defined and meant to be used as a human
     meaningful name if available.
 
+CryptoAddrs
+-----------
+
+A cryptoaddr is any cryptocurrency address and not tied to a specific currency.
+
+``value``
+    The address string. This looks like ``1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2``.
+``currency``
+    The identifier for a specific currency. This is usually the ticker symbols,
+    like ``xbt``, ``zec`` or ``xmr``.
+``denominator``
+    Balance is tracked internally using 64 bit integers (signed, for technical reasons). Balance is supposed to be the lowest unit, so in case of bitcoin you'd write ``100,000,000`` satoshi instead of ``1`` bitcoin. Since this value is inconvinient to work with we're using the denominator to display values. In case of bitcoin you'd set it to ``8``.
+``balance``
+    The current balance of the address, in the lowest possible unit. In case of bitcoin this would be satoshis.
+``received``
+    The total amount of currency received by this address.
+``first_seen``
+    The first time currency was sent to this address.
+``last_withdrawal``
+    The last time a transaction signed by this address was observed.
+``description``
+    A human readable note for this address.
+
+Activity
+--------
+
+Activity is different from all other structs, have a look at the `Activity
+Section <activity.html>`_.
+
 Relations
 ---------
 

docs/usage.rst

@@ -27,12 +27,12 @@ number of recommended modules::
     [+] Downloading "GeoLite2-City.mmdb"
     [+] Downloading "GeoLite2-ASN.mmdb"
     [+] Loaded 0 modules
-    [*] No modules found, run quickstart to install default modules
+    [*] No modules found, run pkg quickstart to install default modules
     [sn0int][default] >
 
-Typing ``quickstart`` is going to get you a fair number of featured modules::
+Typing ``pkg quickstart`` is going to get you a fair number of featured modules::
 
-    [sn0int][default] > quickstart
+    [sn0int][default] > pkg quickstart
     [+] Installing kpcyrd/asn
     [+] Installing kpcyrd/ctlogs
     [+] Installing kpcyrd/dns-resolve
@@ -105,7 +105,7 @@ Running a module
 Now that we have something to get started with, we can run our first module.
 First lets list all modules we have::
 
-    [sn0int][demo] > mod list
+    [sn0int][demo] > pkg list
     kpcyrd/asn (0.1.0)
         Run a asn lookup for an ip address
     kpcyrd/ctlogs (0.1.0)
@@ -149,7 +149,7 @@ some of them in a browser but hold on, there's a more efficient way to approach
 this.
 
 .. hint::
-   You can run the modules concurrently with ``run -j 8``.
+   You can run the modules concurrently with ``run -j3``.
 
 Running followup modules on the results
 ---------------------------------------
@@ -256,85 +256,3 @@ You can reverse this using the scope command::
 .. hint::
    All entities have this field, you can refer to it in queries using
    ``unscoped=1``.
-
-Autonoscope
------------
-
-Instead of manually unscoping everything you can also define so called
-autonoscope rules. Those are executed from most specific to least specific and
-the first match wins. If no rule matches, the default is in-scope::
-
-    [sn0int][demo] > # add the domain first
-    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
-    [sn0int][demo] > add domain example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > # automatically noscope all subdomains
-    [sn0int][demo] > autonoscope add domain example.com
-    [sn0int][demo] > # except subdomains of prod.example.com 
-    [sn0int][demo] > autoscope add domain prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > autonoscope list 
-      scope domain "prod.example.com"
-    noscope domain "example.com"
-    [sn0int][demo] > 
-    [sn0int][demo] > # this is going to be out-of-scope
-    [sn0int][demo] > add subdomain www.example.com
-    [sn0int][demo] > # this is going to be in-scope
-    [sn0int][demo] > add subdomain db.prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > select subdomains 
-    #1, "www.example.com"
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=0
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=1
-    #1, "www.example.com"
-    [sn0int][demo] > 
-
-Domains
-~~~~~~~
-
-Autonoscope rules for domains are applied to the following structs:
-
-- domains
-- subdomains
-- urls
-
-Example rules::
-
-    autonoscope add domain example.com
-    autonoscope add domain staging.example.com
-    autonoscope add domain com
-    autonoscope add domain .
-
-IPs
-~~~
-
-Autonoscope rules for IPs are applied to the following structs:
-
-- ipaddrs
-- netblocks
-- ports
-
-Example rules::
-
-    autonoscope add ip 0.0.0.0/0
-    autonoscope add ip ::/0
-    autonoscope add ip 192.168.0.0/16
-    autonoscope add ip 10.13.33.37/32
-
-URLs
-~~~~
-
-Autonoscope rules for urls are applied to the following structs:
-
-- urls
-
-Note that these rules are specific to a certain origin (like
-``https://example.com``) and are used to filter paths.
-
-Example rules::
-
-    autonoscope add url https://example.com/
-    autonoscope add url https://example.com/admin/
-    autonoscope add url https://example.com/a/b/c/d

examples/maxmind.rs

@@ -1,61 +1,27 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
 use sn0int::paths;
-use std::fs;
 use std::net::IpAddr;
+use std::path::Path;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
 pub enum Args {
-    #[structopt(name="dl")]
-    Download(Download),
     #[structopt(name="asn")]
     Asn(AsnArgs),
     #[structopt(name="geoip")]
     GeoIP(GeoIPArgs),
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Download {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-impl Download {
-    fn run(&self) -> Result<()> {
-        let path = paths::cache_dir()?.join(&self.target);
-        if self.extract_only {
-            let body = fs::read(&self.url)?;
-            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
-        } else {
-            GeoIP::download(path, &self.filter, &self.url)?;
-        }
-        Ok(())
-    }
-}
-
 #[derive(Debug, StructOpt)]
 pub struct AsnArgs {
     ip: IpAddr,
 }
 
 impl AsnArgs {
-    fn run(&self) -> Result<()> {
-        let asndb = AsnDB::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
+        let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
         println!("{:#?}", asn);
@@ -70,8 +36,9 @@ pub struct GeoIPArgs {
 }
 
 impl GeoIPArgs {
-    fn run(&self) -> Result<()> {
-        let geoip = GeoIP::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
+        let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
         println!("{:#?}", lookup);
@@ -84,10 +51,10 @@ impl GeoIPArgs {
 fn run() -> Result<()> {
     let args = Args::from_args();
     debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
     match args {
-        Args::Download(args) => args.run(),
-        Args::Asn(args) => args.run(),
-        Args::GeoIP(args) => args.run(),
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
     }
 }
 

migrations/2020-01-09-024234_activity/down.sql

@@ -0,0 +1,31 @@
+DROP TABLE activity;
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-01-09-024234_activity/up.sql

@@ -0,0 +1,44 @@
+CREATE TABLE activity (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    topic VARCHAR NOT NULL,
+    time DATETIME NOT NULL,
+    uniq VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    radius INTEGER,
+    content VARCHAR NOT NULL
+);
+CREATE UNIQUE INDEX activity_uniq ON activity(topic, uniq);
+CREATE INDEX activity_topic ON activity(topic);
+CREATE INDEX activity_time ON activity(time);
+CREATE INDEX activity_topic_time ON activity(topic, time);
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

modules/harness/activity-ping.lua

@@ -0,0 +1,29 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    while true do
+        db_activity({
+            topic=topic,
+            time=sn0int_time(),
+            uniq=uniq,
+            latitude=lat,
+            longitude=lon,
+            radius=radius,
+            content={
+                msg='ohai',
+            },
+        })
+        sleep(5)
+    end
+end

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

modules/harness/google-tls.lua

@@ -0,0 +1,29 @@
+-- Description: Test various tls functions
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('sending https request to google.com')
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://google.com/', {})
+    r = http_send(req)
+    if last_err() then return end
+    debug(r)
+
+    info('creating tls socket to google.com')
+    sock = sock_connect('google.com', 443, {
+        tls=true,
+    })
+    if last_err() then return end
+    debug(sock)
+
+    info('creating socket to google.com, wrapping afterwards')
+    sock = sock_connect('google.com', 443, {})
+    if last_err() then return end
+    tls = sock_upgrade_tls(sock, {
+        sni_value='google.com',
+    })
+    if last_err() then return end
+    debug(sock)
+    debug(tls)
+end

modules/harness/pgp-fetch.lua

@@ -0,0 +1,22 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+    url = 'https://openpgpkey.archlinux.org/.well-known/openpgpkey/archlinux.org/hu/in9mwr4s84x7gm51851h343n3at1x61g?l=anthraxx'
+
+    req = http_request(session, 'GET', url, {})
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['text'])
+    info(k)
+
+    req = http_request(session, 'GET', url, {
+        binary=true,
+    })
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['binary'])
+    info(k)
+end

modules/harness/ratelimit.lua

@@ -0,0 +1,11 @@
+-- Description: Run script with a global ratelimit
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- this shouldn't complete in less than 5 seconds
+    for i=1, 20 do
+        ratelimit_throttle('foo', 4, 1000)
+        info(sn0int_time())
+    end
+end

modules/harness/socket-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = sock_connect('127.0.0.1', 4444, {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            sock_send(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        sock_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = sock_recv(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/url-inserter.lua

@@ -0,0 +1,58 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('preparing')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+
+    info('inserting')
+    url1 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+    })
+    url2 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        body='ohai',
+    })
+    url3 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+    })
+
+    info('updating')
+    db_update('url', {
+        id=url1,
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+        path='/',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url2,
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        path='/ohai',
+        body='ohai',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url3,
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        path='/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+        unscoped=false,
+    }, {
+    })
+end

modules/harness/winkekatze-sub.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local sock = mqtt_connect('mqtt://mqtt.winkekatze24.de', {
+        read_timeout=10,
+    })
+    if last_err() then return end
+    mqtt_subscribe(sock, '#', 0)
+
+    while true do
+        local pkt = mqtt_recv_text(sock)
+        if last_err() then return end
+        info(pkt)
+    end
+end

modules/harness/ws-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = ws_connect('ws://127.0.0.1:8080', {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            ws_send_text(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        ws_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = ws_recv_text(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/ws.lua

@@ -0,0 +1,23 @@
+-- Description: Create an echo websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local target = 'ws://echo.websocket.org'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('sending')
+    ws_send_text(sock, 'ohai wurld')
+    if last_err() then return end
+
+    info('recieving')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'ohai wurld' then
+        return 'echo failed, got: ' .. msg
+    end
+end

modules/harness/wss.lua

@@ -0,0 +1,30 @@
+-- Description: Create an encrypted websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- local target = 'ws://echo.websocket.org' -- doesn't support proper ciphers
+    local target = 'wss://rocket.events.ccc.de/sockjs/258/whi0yr1y/websocket'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('recieving 1/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'o' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('recieving 2/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'a["{\\"server_id\\":\\"0\\"}"]' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('handshake succeeded')
+end

sn0int-common/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "sn0int-common"
-version = "0.9.0"
-description = "Common code for sn0int"
+version = "0.10.0"
+description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-common" }
+sn0int-common = { version="0.10.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }

sn0int-registry/Dockerfile

@@ -2,16 +2,16 @@ FROM rust:buster
 RUN apt-get update -q && apt-get install -yq llvm libclang-dev \
     && rm -rf /var/lib/apt/lists/*
 RUN rustup install nightly
-WORKDIR /usr/src/sn0int-registry
+WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo +nightly build --release --verbose
+RUN cd sn0int-registry && cargo +nightly build --release --verbose
 RUN strip target/release/sn0int-registry
 
 FROM debian:buster
 RUN apt-get update -q && apt-get install -yq libcurl4 libpq5 \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=0 /usr/src/sn0int-registry/target/release/sn0int-registry /usr/local/bin/sn0int-registry
-COPY templates /templates
+COPY --from=0 /usr/src/sn0int/target/release/sn0int-registry /usr/local/bin/sn0int-registry
+COPY sn0int-registry/templates /templates
 ENV ROCKET_ENV=prod \
     ROCKET_ADDRESS=0.0.0.0 \
     ROCKET_PORT=8000

sn0int-registry/src/auth.rs

@@ -2,7 +2,7 @@ use diesel::pg::PgConnection;
 use oauth2::basic::BasicClient;
 use oauth2::prelude::*;
 use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl, TokenResponse};
-use crate::github::GithubAuthenticator;
+use crate::github;
 use sn0int_registry::errors::*;
 use sn0int_registry::models::AuthToken;
 use url::Url;
@@ -67,8 +67,7 @@ impl Authenticator {
         let access_token = response.access_token();
         let access_token = access_token.secret().to_string();
 
-        let client = GithubAuthenticator::from_env()?;
-        let user = client.get_username(&access_token)?;
+        let user = github::get_username(&access_token)?;
 
         AuthToken::create(&AuthToken {
             id: state,

sn0int-registry/src/auth2.rs

@@ -4,7 +4,7 @@ use sn0int_registry::db::Connection;
 use rocket::http::Status;
 use rocket::{Request, Outcome};
 use rocket::request::{self, FromRequest};
-use crate::github::GithubAuthenticator;
+use crate::github;
 
 
 pub struct AuthHeader(String);
@@ -12,8 +12,7 @@ pub struct AuthHeader(String);
 impl AuthHeader {
     pub fn verify(&self, connection: &Connection) -> Result<String> {
         let session = AuthToken::read(&self.0, &connection)?;
-        let client = GithubAuthenticator::from_env()?;
-        client.get_username(&session.access_token)
+        github::get_username(&session.access_token)
             .map_err(Error::from)
     }
 }

sn0int-registry/src/github.rs

@@ -1,54 +1,23 @@
 use sn0int_registry::errors::*;
-use std::env;
 use reqwest;
 
+pub fn get_username(oauth_token: &str) -> Result<String> {
+    let client = reqwest::Client::new();
+    let mut resp = client.get("https://api.github.com/user")
+        .header("Authorization", format!("token {}", oauth_token))
+        .send()
+        .context("Failed to check access_token")?
+        .error_for_status()
+        .context("Github returned http error")?;
 
-pub struct GithubAuthenticator {
-    client_id: String,
-    client_secret: String,
-}
+    let data = resp.json::<GithubUser>()
+        .context("Failed to deserialize github reply")?;
 
-impl GithubAuthenticator {
-    pub fn new(client_id: String, client_secret: String) -> GithubAuthenticator {
-        GithubAuthenticator {
-            client_id,
-            client_secret,
-        }
-    }
-
-    pub fn from_env() -> Result<GithubAuthenticator> {
-        let client_id = env::var("GITHUB_CLIENT_ID")
-            .context("GITHUB_CLIENT_ID is not set")?;
-        let client_secret = env::var("GITHUB_CLIENT_SECRET")
-            .context("GITHUB_CLIENT_SECRET is not set")?;
-        Ok(GithubAuthenticator::new(client_id, client_secret))
-    }
-
-    pub fn get_username(&self, oauth_token: &str) -> Result<String> {
-        let url = format!("https://api.github.com/applications/{}/tokens/{}", self.client_id, oauth_token);
-        let client = reqwest::Client::new();
-        let mut resp = client.get(&url)
-            .basic_auth(&self.client_id, Some(&self.client_secret))
-            .send()?;
-
-        if !resp.status().is_success() {
-            bail!("Github returned: {}", resp.status())
-        }
-
-        let data = resp.json::<GithubReply>()
-            .context("Failed to deserialize github reply")?;
-
-        Ok(data.user.login)
-    }
+    Ok(data.login)
 }
 
 #[derive(Debug, Deserialize)]
-pub struct GithubReply {
-    user: GithubUser,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct GithubUser {
+struct GithubUser {
     login: String,
     #[serde(rename="type")]
     user_type: String,

sn0int-registry/templates/index.html.hbs

@@ -21,7 +21,7 @@
     To install a module run:
     </p>
     <p class="code"><code>
-    sn0int install kpcyrd/ctlogs
+    sn0int pkg install kpcyrd/ctlogs
     </code></p>
 
     {{#each modules}}

sn0int-std/Cargo.toml

@@ -0,0 +1,55 @@
+[package]
+name = "sn0int-std"
+version = "0.18.1"
+description = "sn0int - stdlib"
+authors = ["kpcyrd <git@rxv.cc>"]
+repository = "https://github.com/kpcyrd/sn0int"
+license = "GPL-3.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4"
+failure = "0.1"
+hlua-badtouch = "0.4"
+tokio = "0.1.14"
+rand = "0.7"
+regex = "1.0"
+rustls = { version="0.16", features=["dangerous_configuration"] }
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+serde_urlencoded = "0.6"
+webpki = "0.21"
+webpki-roots = "0.19.0"
+ct-logs = "0.6"
+chrootable-https = "0.15"
+http = "0.2"
+bufstream = "0.1.4"
+pem = "0.7"
+url = "2.0"
+tungstenite = { version = "0.10.1", default-features = false }
+kuchiki = "0.8.0"
+maxminddb = "0.13"
+x509-parser = "0.6.2"
+der-parser = "3.0"
+publicsuffix = { version="1.5", default-features=false }
+xml-rs = "0.8"
+geo = "0.12"
+bytes = "0.4"
+base64 = "0.12"
+chrono = { version = "0.4", features = ["serde"] }
+mqtt-protocol = "0.8.1"
+sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
+
+image = "0.23.0"
+kamadak-exif = "0.5.1"
+
+bs58 = "0.3"
+digest = "0.8.0"
+blake2 = "0.8.0"
+
+[dev-dependencies]
+env_logger = "0.7"
+maplit = "1.0.1"

sn0int-std/src/blobs.rs

@@ -0,0 +1,118 @@
+use bytes::Bytes;
+use blake2::VarBlake2b;
+use digest::{Input, VariableOutput};
+use serde::ser::{Serialize, Serializer};
+use serde::de::{self, Deserialize, Deserializer};
+use std::result;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Blob {
+    pub id: String,
+    pub bytes: Bytes,
+}
+
+impl Blob {
+    pub fn create(bytes: Bytes) -> Blob {
+        let id = Self::hash(&bytes);
+        Blob {
+            id,
+            bytes,
+        }
+    }
+
+    pub fn hash(bytes: &[u8]) -> String {
+        let mut h = VarBlake2b::new(32).unwrap();
+        h.input(bytes);
+        Self::encode_hash(&h.vec_result())
+    }
+
+    #[inline]
+    fn encode_hash(bytes: &[u8]) -> String {
+        let x = bs58::encode(bytes).into_string();
+        format!("{:0<44}", x)
+    }
+}
+
+impl Serialize for Blob {
+    #[inline]
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let s = base64::encode(&self.bytes);
+        serializer.serialize_str(&s)
+    }
+}
+
+impl<'de> Deserialize<'de> for Blob {
+    #[inline]
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let bytes = base64::decode(&s)
+            .map_err(de::Error::custom)?;
+        Ok(Blob::create(Bytes::from(bytes)))
+    }
+}
+
+pub trait BlobState {
+    fn register_blob(&self, blob: Blob) -> String;
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json;
+
+    #[inline]
+    fn blob() -> (Bytes, Blob) {
+        let bytes = Bytes::from(&b"asdf"[..]);
+        (bytes.clone(), Blob::create(bytes))
+    }
+
+    #[test]
+    fn verify_create_blob() {
+        let (bytes, blob) = blob();
+        assert_eq!(blob, Blob {
+            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
+            bytes,
+        });
+    }
+
+    #[test]
+    fn test_serialize() {
+        let (_, blob) = blob();
+        let json = serde_json::to_string(&blob).unwrap();
+        assert_eq!(&json, "\"YXNkZg==\"");
+    }
+
+    #[test]
+    fn test_deserialize() {
+        let (_, blob1) = blob();
+        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
+        assert_eq!(blob1, blob2);
+    }
+
+    #[test]
+    fn test_hash_encoding() {
+        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
+    }
+
+    #[test]
+    fn test_hash_encoding_padding() {
+        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
+    }
+}

sn0int-std/src/crt.rs

@@ -11,6 +11,7 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 #[derive(Debug, PartialEq)]
 pub enum AlternativeName {
     DnsName(String),
+    Email(String),
     IpAddr(IpAddr),
 }
 
@@ -37,6 +38,7 @@ pub fn san_value(o: DerObject) -> Result<AlternativeName> {
 
     match (o.class, o.tag, &o.content) {
         (2, BerTag::Integer, BerObjectContent::Unknown(BerTag::Integer, value)) => san_value_dns(value),
+        (2, BerTag::Boolean, BerObjectContent::Unknown(BerTag::Boolean, value)) => san_value_email(value),
         (2, BerTag::ObjDescriptor, BerObjectContent::Unknown(BerTag::ObjDescriptor, value)) => san_value_ipaddr(value),
         _ => bail!("Unexpected object: {:?}", o),
     }
@@ -49,6 +51,13 @@ pub fn san_value_dns(v: &[u8]) -> Result<AlternativeName> {
         .map_err(Error::from)
 }
 
+pub fn san_value_email(v: &[u8]) -> Result<AlternativeName> {
+    debug!("Reading as email: {:?}", v);
+    String::from_utf8(v.to_vec())
+        .map(AlternativeName::Email)
+        .map_err(Error::from)
+}
+
 pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
     debug!("Reading as ipaddr: {:?}", v);
     match v.len() {
@@ -68,7 +77,8 @@ pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
     pub valid_names: Vec<String>,
-    pub valid_ipaddrs: Vec<IpAddr>
+    pub valid_emails: Vec<String>,
+    pub valid_ipaddrs: Vec<IpAddr>,
 }
 
 impl Certificate {
@@ -103,6 +113,7 @@ impl Certificate {
         };
 
         let mut valid_names = HashSet::new();
+        let mut valid_emails = HashSet::new();
         let mut valid_ipaddrs = HashSet::new();
 
         for x in crt.tbs_certificate.subject.rdn_seq {
@@ -131,15 +142,18 @@ impl Certificate {
             for v in values {
                 match v {
                     AlternativeName::DnsName(v) => valid_names.insert(v),
+                    AlternativeName::Email(v) => valid_emails.insert(v),
                     AlternativeName::IpAddr(v) => valid_ipaddrs.insert(v),
                 };
             }
         }
 
         let valid_names = valid_names.into_iter().collect();
+        let valid_emails = valid_emails.into_iter().collect();
         let valid_ipaddrs = valid_ipaddrs.into_iter().collect();
         Ok(Certificate {
             valid_names,
+            valid_emails,
             valid_ipaddrs,
         })
     }
@@ -197,6 +211,7 @@ WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik=
         x.valid_names.sort();
         assert_eq!(x, Certificate {
             valid_names: vec!["github.com".into(), "www.github.com".into()],
+            valid_emails: vec![],
             valid_ipaddrs: vec![],
         });
     }
@@ -235,6 +250,7 @@ ZkZZmqNn2Q8=
                 "*.cloudflare-dns.com".into(),
                 "cloudflare-dns.com".into(),
             ],
+            valid_emails: vec![],
             valid_ipaddrs: vec![
                 "1.0.0.1".parse().unwrap(),
                 "1.1.1.1".parse().unwrap(),
@@ -351,6 +367,7 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
                 "witt-weiden.dam.staging.aboutyou.cloud".into(),
                 "www.aboutyou.de".into(),
             ],
+            valid_emails: vec![],
             valid_ipaddrs: vec![],
         });
     }
@@ -396,4 +413,50 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
             .expect("Failed to process san value");
         assert_eq!(v, AlternativeName::IpAddr("1.1.1.1".parse().unwrap()));
     }
+
+    #[test]
+    fn test_san_email() {
+        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+MIIE5zCCA8+gAwIBAgIQBvsKfZ5AGSW3Vc8Ldto1hTANBgkqhkiG9w0BAQUFADBp
+MSQwIgYJKoZIhvcNAQkBFhVwa2lfYWRtaW5Ac3VuZ2FyZC5jb20xJjAkBgNVBAoT
+HVN1bkdhcmQgQXZhaWxhYmlsaXR5IFNlcnZpY2VzMRkwFwYDVQQDExBTQVMgUHVi
+bGljIENBIHYxMB4XDTEwMDkwMjE2MzY0OVoXDTExMTAwMTE2MzEwMFowgbQxCzAJ
+BgNVBAYTAlVTMQswCQYDVQQIEwJPUjERMA8GA1UEBxMIUG9ydGxhbmQxEzARBgoJ
+kiaJk/IsZAEZFgNjb20xHDAaBgoJkiaJk/IsZAEZFgxqaXZlc29mdHdhcmUxHDAa
+BgNVBAoTE0ppdmUgU29mdHdhcmUsIEluYy4xEDAOBgNVBAsTB0hvc3RpbmcxIjAg
+BgNVBAMTGSouaG9zdGVkLmppdmVzb2Z0d2FyZS5jb20wggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQC0oornTIyL5YjZMpNwy+V2YJbLqaLrbPrbWFCsNJDx
+dnubjfR71aW+YYlUZF8zoq4jFetkblCehyvPEb5tD/l3/WZhiXYOziPDrsEVCngF
+3/b0H3Dyk6mNWBZcNpJkdpOx1YB6Zer8eKzFOr7Qj3aevOR/bEe2NARJIaO0Rjwe
+YIWY0arKRm6z4nJD8fYAvFV6wRWmHsZO9ci7hiGeW3YL6jQYJqLeuwXm64l0jptb
+Qg8r8c1V5BXETlvQJL34gUozEl9jDpzR7KoXtErhlU2ytl9Wg+fOxYuWgx8vER0/
+7Hqc/qD5e7B+NtwgfEio7SvNGA/HhjNxW2Wbrx4qooJRAgMBAAGjggE9MIIBOTAO
+BgNVHQ8BAf8EBAMCA6gwEQYJYIZIAYb4QgEBBAQDAgbAMCIGA1UdEQQbMBmBF3N1
+YmplY3RuYW1lQGV4YW1wbGUuY29tMB8GA1UdIwQYMBaAFDhBxvKFgYP96+IaNpI7
+JmEWgRESMFQGA1UdIARNMEswSQYJKoZIhvcNBQYBMDwwOgYIKwYBBQUHAgEWLmh0
+dHBzOi8vY2VydGlmaWNhdGUuc3VuZ2FyZC5jb20vU0FTX0NBX0NQUy5wZGYwRQYD
+VR0fBD4wPDA6oDigNoY0aHR0cHM6Ly9jZXJ0aWZpY2F0ZS5zdW5nYXJkLmNvbS9T
+QVNfUHVibGljX0NBX3YxLmNybDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4E
+FgQUhJ99py6oeCYBzcePPjhOxHVDBvwwDQYJKoZIhvcNAQEFBQADggEBAF/DAJgX
+f50x8t8Im96AUn4DqC+T0QZIYHihpj2uCwWDbdp5efppqTrk6FrpFOzQy0TRkstb
+Q3zKSgduedQiwii9qh88O1h2gbSTqfi55ApOIGoCiiRCqio2p4tbKyqPV3Q0eyYw
+K4f9GAOcawvNsI//mx99ol/ZGEamydeL9G0qiKrhqSxd2TGmFaIVJdu9fh59hos4
+6t9c4FVyYygdsIeGHkHjpB2bKjZhnJpKRh9dGWctcjdHMITBBqgiRH9OZa/w6SPE
+UT+11L6q7MSIXSIMV8kJSUUYE92P7bnAqViTIuu/hHnfmIhiy6t7AuT2QHEhqDab
+EF4l5MwdUqs8FvM=
+-----END CERTIFICATE-----
+"#).expect("Failed to parse cert");
+        x.valid_names.sort();
+        x.valid_emails.sort();
+        x.valid_ipaddrs.sort();
+        assert_eq!(x, Certificate {
+            valid_names: vec![
+                "*.hosted.jivesoftware.com".into(),
+            ],
+            valid_emails: vec![
+                "subjectname@example.com".into(),
+            ],
+            valid_ipaddrs: vec![],
+        });
+    }
 }

sn0int-std/src/crypto.rs

@@ -0,0 +1,51 @@
+use crate::errors::*;
+use sodiumoxide::crypto::secretbox::{self, Key, Nonce};
+use std::iter;
+
+pub fn key_trunc_pad(mut key: &[u8], len: usize, pad: u8) -> Vec<u8> {
+    if key.len() > len {
+        key = &key[..len];
+    }
+
+    let mut key = key.to_vec();
+    key.extend(iter::repeat(pad).take(len - key.len()));
+    key
+}
+
+pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
+    if encrypted.len() <= secretbox::NONCEBYTES {
+        bail!("Encrypted message is too short");
+    }
+
+    let key = Key::from_slice(key)
+        .ok_or_else(|| format_err!("Key has wrong length"))?;
+    let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
+        .ok_or_else(|| format_err!("Nonce has wrong length"))?;
+    let ciphertext = &encrypted[secretbox::NONCEBYTES..];
+    let plain = secretbox::open(&ciphertext, &nonce, &key)
+        .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
+    Ok(plain)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_key_equal() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_trunc() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5, 6, 7, 8, 9], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_pad() {
+        let key = key_trunc_pad(&[1, 2, 3], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 0, 0]);
+    }
+}

sn0int-std/src/engine/mod.rs

@@ -0,0 +1 @@
+pub mod structs;

sn0int-std/src/engine/structs.rs

@@ -41,6 +41,12 @@ impl LuaMap {
     pub fn insert_num<K: Into<String>>(&mut self, k: K, v: f64) {
         self.0.insert(AnyHashableLuaValue::LuaString(k.into()), AnyLuaValue::LuaNumber(v));
     }
+
+    pub fn insert_serde<K: Into<String>, S: serde::Serialize>(&mut self, k: K, v: S) -> Result<()> {
+        let v = serde_json::to_value(v)?;
+        self.0.insert(AnyHashableLuaValue::LuaString(k.into()), LuaJsonValue::from(v).into());
+        Ok(())
+    }
 }
 
 impl IntoIterator for LuaMap {

sn0int-std/src/errors.rs

@@ -0,0 +1,3 @@
+pub use log::{trace, debug, info, warn, error};
+pub use failure::{Error, ResultExt, format_err, bail};
+pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-std/src/geo.rs

@@ -0,0 +1,111 @@
+use crate::errors::*;
+
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use geo::{LineString, Polygon, Coordinate};
+use geo::prelude::*;
+
+
+#[derive(Debug, Deserialize)]
+pub struct Point {
+    lon: f64,
+    lat: f64,
+}
+
+impl Point {
+    pub fn try_from(x: AnyLuaValue) -> Result<Point> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub fn polygon_contains(ring: &[Point], p: &Point) -> bool {
+    let ring = ring.iter()
+        .map(|p| Coordinate { x: p.lon, y: p.lat })
+        .collect::<Vec<_>>();
+
+    let polygon = Polygon::new(LineString::from(ring), vec![]);
+    let point = geo::Point::new(p.lon, p.lat);
+
+    polygon.contains(&point)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn hamburg_polygon() -> &'static [Point] {
+        &[
+            Point { lat: 53.63975308945899,  lon: 9.764785766601562  },
+            Point { lat: 53.59494998253459,  lon: 9.827270507812     },
+            Point { lat: 53.663153974456456, lon: 9.9151611328125    },
+            Point { lat: 53.65582987649682,  lon: 9.976272583007812  },
+            Point { lat: 53.68613523817129,  lon: 9.992752075195312  },
+            Point { lat: 53.68674518938816,  lon: 10.051460266113281 },
+            Point { lat: 53.72495117617815,  lon: 10.075492858886719 },
+            Point { lat: 53.71946627930625,  lon: 10.118408203125    },
+            Point { lat: 53.743635083157756, lon: 10.164413452148438 },
+            Point { lat: 53.73104466704585,  lon: 10.202865600585938 },
+            Point { lat: 53.676781546441546, lon: 10.16304016113281  },
+            Point { lat: 53.632832079199474, lon: 10.235824584960938 },
+            Point { lat: 53.608803292930894, lon: 10.2008056640625   },
+            Point { lat: 53.578646152866504, lon: 10.208358764648438 },
+            Point { lat: 53.57212285981298,  lon: 10.163726806640625 },
+            Point { lat: 53.52071674896369,  lon: 10.18707275390625  },
+            Point { lat: 53.52643162253097,  lon: 10.224151611328125 },
+            Point { lat: 53.44062753992289,  lon: 10.347747802734375 },
+            Point { lat: 53.38824275010831,  lon: 10.248870849609375 },
+            Point { lat: 53.38824275010831,  lon: 10.15960693359375  },
+            Point { lat: 53.44635321212876,  lon: 10.064849853515625 },
+            Point { lat: 53.40595029739904,  lon: 9.985198974609375  },
+            Point { lat: 53.42385506057106,  lon: 9.951210021972656  },
+            Point { lat: 53.41843327091211,  lon: 9.944171905517578  },
+            Point { lat: 53.41812635648326,  lon: 9.927349090576172  },
+            Point { lat: 53.412294561442884, lon: 9.917736053466797  },
+            Point { lat: 53.41464783813818,  lon: 9.901256561279297  },
+            Point { lat: 53.443490472483326, lon: 9.912586212158201  },
+            Point { lat: 53.45177144115704,  lon: 9.897651672363281  },
+            Point { lat: 53.43633277935392,  lon: 9.866924285888672  },
+            Point { lat: 53.427639673754776, lon: 9.866409301757812  },
+            Point { lat: 53.427639673754776, lon: 9.858856201171875  },
+            Point { lat: 53.46710230573499,  lon: 9.795513153076172  },
+            Point { lat: 53.49039461941655,  lon: 9.795341491699219  },
+            Point { lat: 53.49029248806277,  lon: 9.77903366088867   },
+            Point { lat: 53.49856433088649,  lon: 9.780235290527344  },
+            Point { lat: 53.5078554643033,   lon: 9.758434295654297  },
+            Point { lat: 53.545407634092975, lon: 9.759807586669922  },
+            Point { lat: 53.568147234570084, lon: 9.633293151855469  },
+            Point { lat: 53.58802162343514,  lon: 9.655780792236328  },
+            Point { lat: 53.568351121879815, lon: 9.727706909179688  },
+            Point { lat: 53.60921067445695,  lon: 9.737663269042969  },
+        ]
+    }
+
+    #[test]
+    fn test_polygon_hamburg_contains_hamburg() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 53.551085,
+            lon: 9.993682,
+        });
+        assert!(contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_berlin() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 52.52437,
+            lon: 13.41053,
+        });
+        assert!(!contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_ny() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 40.726662,
+            lon: -74.036677,
+        });
+        assert!(!contains);
+    }
+}

sn0int-std/src/geoip/mod.rs

@@ -1,57 +1,45 @@
-use chrootable_https::Client;
-use crate::archive;
 use crate::errors::*;
 use crate::lazy::LazyInit;
-use crate::paths;
-use crate::worker;
 use maxminddb::{self, geoip2};
 use std::fmt;
 use std::fs::{self, File};
 use std::net::IpAddr;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::io::Read;
 use std::sync::Arc;
 
-pub static GEOIP_CITY_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz";
-pub static GEOIP_ASN_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz";
-
 pub mod models;
 use self::models::GeoLookup;
 use self::models::AsnLookup;
 
 
 pub trait Maxmind: Sized {
-    fn archive_filename() -> &'static str;
-
-    fn archive_url() -> &'static str;
+    fn filename() -> &'static str;
 
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self;
 
-    // TODO: refactor this to return Path
-    fn cache_path() -> Result<String> {
+    fn cache_path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         for path in &[
             // Archlinux
             "/usr/share/GeoIP/",
             // OpenBSD
             "/usr/local/share/examples/libmaxminddb/",
+            // geoipupdate
+            "/var/lib/GeoIP/",
         ] {
             let path = Path::new(path);
-            let path = path.join(Self::archive_filename());
+            let path = path.join(Self::filename());
 
             if path.exists() {
-                let path = path.to_str()
-                    .ok_or_else(|| format_err!("Failed to decode path"))?;
-                return Ok(path.to_string());
+                return Ok(path);
             }
         }
 
         // use cache path
-        let path = paths::cache_dir()?
-            .join(Self::archive_filename());
-        let path = path.to_str()
-            .ok_or_else(|| format_err!("Failed to decode path"))?;
-        Ok(path.to_string())
+        let path = cache_dir
+            .join(Self::filename());
+        Ok(path)
     }
 
     fn from_buf(buf: Vec<u8>) -> Result<Self> {
@@ -60,37 +48,20 @@ pub trait Maxmind: Sized {
         Ok(Self::new(reader))
     }
 
-    fn open(path: &str) -> Result<Self> {
+    fn open(path: &Path) -> Result<Self> {
         let buf = fs::read(path)?;
         Self::from_buf(buf)
     }
 
-    fn open_reader() -> Result<MaxmindReader> {
-        let path = Self::cache_path()?;
-        MaxmindReader::open_path(path)
-    }
+    fn try_open_reader(cache_dir: &Path) -> Result<Option<MaxmindReader>> {
+        let path = Self::cache_path(cache_dir)?;
 
-    fn open_or_download() -> Result<Self> {
-        let path = Self::cache_path()?;
-
-        if File::open(&path).is_err() {
-            worker::spawn_fn(&format!("Downloading {:?}", Self::archive_filename()), || {
-                Self::download(&path, Self::archive_filename(), Self::archive_url())
-            }, false)?;
-        };
-
-        Self::open(&path)
-    }
-
-    fn download<P: AsRef<Path>>(path: P, filter: &str, url: &str) -> Result<()> {
-        debug!("Downloading {:?}...", url);
-        let client = Client::with_system_resolver()?;
-        let resp = client.get(url)
-            .wait_for_response()
-            .context("http request failed")?;
-        debug!("Downloaded {} bytes", resp.body.len());
-        archive::extract(&mut &resp.body[..], filter, path)?;
-        Ok(())
+        if path.exists() {
+            let db = MaxmindReader::open_path(path)?;
+            Ok(Some(db))
+        } else {
+            Ok(None)
+        }
     }
 }
 
@@ -141,15 +112,10 @@ impl fmt::Debug for GeoIP {
 
 impl Maxmind for GeoIP {
     #[inline]
-    fn archive_filename() -> &'static str {
+    fn filename() -> &'static str {
         "GeoLite2-City.mmdb"
     }
 
-    #[inline]
-    fn archive_url() -> &'static str {
-        GEOIP_CITY_URL
-    }
-
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         GeoIP {
@@ -178,15 +144,10 @@ impl fmt::Debug for AsnDB {
 
 impl Maxmind for AsnDB {
     #[inline]
-    fn archive_filename() -> &'static str {
+    fn filename() -> &'static str {
         "GeoLite2-ASN.mmdb"
     }
 
-    #[inline]
-    fn archive_url() -> &'static str {
-        GEOIP_ASN_URL
-    }
-
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         AsnDB {
@@ -205,22 +166,29 @@ impl AsnDB {
 
 #[cfg(test)]
 mod tests {
+    // You need geoip setup on your system to run this
+    /*
     use super::*;
 
     #[test]
+    #[ignore]
     fn test_geoip_lookup() {
         let ip = "1.1.1.1".parse().unwrap();
-        let geoip = GeoIP::open_or_download().expect("Failed to load geoip");
+        let path = GeoIP::cache_path().unwrap();
+        let geoip = GeoIP::open(&path).unwrap();
         let lookup = geoip.lookup(ip).expect("GeoIP lookup failed");
         println!("{:#?}", lookup);
         assert_eq!(lookup.city, None);
     }
 
     #[test]
+    #[ignore]
     fn test_asn_lookup() {
         let ip = "1.1.1.1".parse().unwrap();
-        let asndb = AsnDB::open_or_download().expect("Failed to load asndb");
+        let path = AsnDB::cache_path().unwrap();
+        let asndb = AsnDB::open(&path).unwrap();
         let lookup = asndb.lookup(ip).expect("ASN lookup failed");
         println!("{:#?}", lookup);
     }
+    */
 }

sn0int-std/src/gfx/exif.rs

@@ -11,11 +11,12 @@ pub struct Location {
 }
 
 impl Location {
-    pub fn try_from(fields: &[exif::Field]) -> Result<Location> {
+    fn try_from_iter<'a, I: IntoIterator<Item=&'a exif::Field>>(iter: I) -> Result<Self> {
         let mut builder = LocationBuilder::default();
-        fields.iter()
-            .map(|f| builder.add_one(f))
-            .collect::<Result<()>>()?;
+        for f in iter {
+            debug!("Exif field: {:?}", f.display_value().to_string());
+            builder.add_one(f)?;
+        }
         builder.build()
     }
 }
@@ -60,12 +61,12 @@ impl LocationBuilder {
 }
 
 pub fn gps(img: &[u8]) -> Result<Option<Location>> {
-    let mut buf = io::BufReader::new(img);
-    let reader = exif::Reader::new(&mut buf)?;
+    let mut buf = io::Cursor::new(img);
+    let reader = exif::Reader::new()
+        .read_from_container(&mut buf)?;
     let fields = reader.fields();
-    debug!("Exif fields: {:?}", fields);
 
-    let location = Location::try_from(fields).ok();
+    let location = Location::try_from_iter(fields).ok();
     Ok(location)
 }
 
@@ -115,18 +116,18 @@ mod tests {
     fn verify_exif_location() {
         test_init();
 
-        let location = Location::try_from(&[
+        let location = Location::try_from_iter(&[
             exif::Field {
                 tag: exif::Tag::GPSLatitudeRef,
-                thumbnail: false,
-                value: exif::Value::Ascii(vec![&[b'N']]),
+                ifd_num: exif::In::PRIMARY,
+                value: exif::Value::Ascii(vec![vec![b'N']]),
             }, exif::Field {
                 tag: exif::Tag::GPSLongitudeRef,
-                thumbnail: false,
-                value: exif::Value::Ascii(vec![&[b'E']]),
+                ifd_num: exif::In::PRIMARY,
+                value: exif::Value::Ascii(vec![vec![b'E']]),
             }, exif::Field {
                 tag: exif::Tag::GPSLatitude,
-                thumbnail: false,
+                ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Rational(vec![exif::Rational {
                     num: 43,
                     denom: 1,
@@ -139,7 +140,7 @@ mod tests {
                 }]),
             }, exif::Field {
                 tag: exif::Tag::GPSLongitude,
-                thumbnail: false,
+                ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Rational(vec![exif::Rational {
                     num: 11,
                     denom: 1,

sn0int-std/src/gfx/mod.rs

@@ -6,13 +6,13 @@ pub mod exif;
 
 #[derive(Debug)]
 pub enum ImageFormat {
-    PNG,
-    JPEG,
-    GIF,
-    WEBP,
-    TIFF,
-    BMP,
-    ICO,
+    Png,
+    Jpeg,
+    Gif,
+    WebP,
+    Tiff,
+    Bmp,
+    Ico,
 }
 
 impl ImageFormat {
@@ -20,28 +20,27 @@ impl ImageFormat {
         // https://www.iana.org/assignments/media-types/media-types.xhtml#image
         // /etc/nginx/mime.types
         match self {
-            ImageFormat::PNG => "image/png",
-            ImageFormat::JPEG => "image/jpeg",
-            ImageFormat::GIF => "image/gif",
-            ImageFormat::WEBP => "image/webp",
-            ImageFormat::TIFF => "image/tiff",
-            ImageFormat::BMP => "image/bmp",
-            ImageFormat::ICO => "image/vnd.microsoft.icon",
+            ImageFormat::Png => "image/png",
+            ImageFormat::Jpeg => "image/jpeg",
+            ImageFormat::Gif => "image/gif",
+            ImageFormat::WebP => "image/webp",
+            ImageFormat::Tiff => "image/tiff",
+            ImageFormat::Bmp => "image/bmp",
+            ImageFormat::Ico => "image/vnd.microsoft.icon",
         }
     }
 
     pub fn try_from(format: &image::ImageFormat) -> Result<ImageFormat> {
         use image::ImageFormat::*;
         match format {
-            PNG => Ok(ImageFormat::PNG),
-            JPEG => Ok(ImageFormat::JPEG),
-            GIF => Ok(ImageFormat::GIF),
-            WEBP => Ok(ImageFormat::WEBP),
-            TIFF => Ok(ImageFormat::TIFF),
-            BMP => Ok(ImageFormat::BMP),
-            ICO => Ok(ImageFormat::ICO),
-
-            HDR | PNM | TGA => bail!("Unsupported format: {:?}", format),
+            Png => Ok(ImageFormat::Png),
+            Jpeg => Ok(ImageFormat::Jpeg),
+            Gif => Ok(ImageFormat::Gif),
+            WebP => Ok(ImageFormat::WebP),
+            Tiff => Ok(ImageFormat::Tiff),
+            Bmp => Ok(ImageFormat::Bmp),
+            Ico => Ok(ImageFormat::Ico),
+            _ => bail!("Unsupported format: {:?}", format),
         }
     }
 }
@@ -49,13 +48,13 @@ impl ImageFormat {
 impl Into<image::ImageFormat> for ImageFormat {
     fn into(self) -> image::ImageFormat {
         match self {
-            ImageFormat::PNG => image::ImageFormat::PNG,
-            ImageFormat::JPEG => image::ImageFormat::JPEG,
-            ImageFormat::GIF => image::ImageFormat::GIF,
-            ImageFormat::WEBP => image::ImageFormat::WEBP,
-            ImageFormat::TIFF => image::ImageFormat::TIFF,
-            ImageFormat::BMP => image::ImageFormat::BMP,
-            ImageFormat::ICO => image::ImageFormat::ICO,
+            ImageFormat::Png => image::ImageFormat::Png,
+            ImageFormat::Jpeg => image::ImageFormat::Jpeg,
+            ImageFormat::Gif => image::ImageFormat::Gif,
+            ImageFormat::WebP => image::ImageFormat::WebP,
+            ImageFormat::Tiff => image::ImageFormat::Tiff,
+            ImageFormat::Bmp => image::ImageFormat::Bmp,
+            ImageFormat::Ico => image::ImageFormat::Ico,
         }
     }
 }
@@ -114,7 +113,7 @@ mod tests {
 
     #[test]
     fn verify_gfx_load_ico() {
-        let ico = fs::read("./sn0int-registry/assets/favicon.ico").expect("fs::read");
+        let ico = fs::read("../sn0int-registry/assets/favicon.ico").expect("fs::read");
         let img = load(&ico).expect("gfx::load");
 
         assert_eq!("image/vnd.microsoft.icon", img.mime());

sn0int-std/src/html.rs

@@ -1,4 +1,4 @@
-use crate::errors::Result;
+use crate::errors::*;
 
 use kuchiki;
 use kuchiki::traits::TendrilSink;
@@ -102,6 +102,7 @@ pub fn html_form(html: &str) -> Result<HashMap<String, String>> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn test_html_select() {

sn0int-std/src/lib.rs

@@ -0,0 +1,26 @@
+#[macro_use] extern crate serde_derive;
+use hlua_badtouch as hlua;
+
+pub mod blobs;
+pub mod crt;
+pub mod crypto;
+mod errors;
+pub mod engine;
+pub mod geo;
+pub mod geoip;
+pub mod gfx;
+pub mod html;
+pub mod json;
+pub mod lazy;
+pub mod mqtt;
+pub mod psl;
+pub mod ratelimits;
+pub mod sockets;
+pub mod web;
+pub mod websockets;
+pub mod xml;
+
+#[cfg(test)]
+fn test_init() {
+    let _ = env_logger::builder().is_test(true).try_init();
+}

sn0int-std/src/mqtt.rs

@@ -0,0 +1,228 @@
+use chrootable_https::DnsResolver;
+use crate::errors::*;
+use crate::hlua::AnyLuaValue;
+use mqtt::packet::VariablePacketError;
+use crate::json::LuaJsonValue;
+use crate::sockets::{Stream, SocketOptions};
+use mqtt::{TopicFilter, QualityOfService};
+use mqtt::control::ConnectReturnCode;
+use mqtt::control::fixed_header::FixedHeaderError;
+use mqtt::encodable::{Encodable, Decodable};
+use mqtt::packet::{Packet, VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
+use std::convert::TryFrom;
+use std::io;
+use std::net::SocketAddr;
+use url::Url;
+
+#[derive(Debug, Default, Deserialize)]
+pub struct MqttOptions {
+    pub username: Option<String>,
+    pub password: Option<String>,
+
+    pub proxy: Option<SocketAddr>,
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
+}
+
+impl MqttOptions {
+    pub fn try_from(x: AnyLuaValue) -> Result<MqttOptions> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub struct MqttClient {
+    stream: Stream,
+}
+
+impl MqttClient {
+    pub fn negotiate(stream: Stream, options: &MqttOptions) -> Result<MqttClient> {
+        let mut client = MqttClient {
+            stream,
+        };
+
+        let mut pkt = ConnectPacket::new("MQTT", "sn0int");
+        pkt.set_user_name(options.username.clone());
+        pkt.set_password(options.password.clone());
+
+        /*
+        if let Some(keep_alive) = msg.keep_alive {
+            packet.set_keep_alive(keep_alive);
+        }
+        */
+
+        client.send(pkt.into())?;
+        let pkt = client.recv()?;
+
+        if let VariablePacket::ConnackPacket(pkt) = pkt {
+            let code = pkt.connect_return_code();
+            if code == ConnectReturnCode::ConnectionAccepted {
+                Ok(client)
+            } else {
+                bail!("MQTT negotiation failed: {:?}", code);
+            }
+        } else {
+            bail!("Expected ConnAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &MqttOptions) -> Result<MqttClient> {
+        let tls = match url.scheme() {
+            "mqtt" => false,
+            "mqtts" => true,
+            _ => bail!("Invalid mqtt protocol"),
+        };
+
+        let host = url.host_str()
+            .ok_or_else(|| format_err!("Missing host in url"))?;
+
+        let port = match (url.port(), tls) {
+            (Some(port), _) => port,
+            (None, true) => 8883,
+            (None, false) => 1883,
+        };
+
+
+        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
+            tls,
+            sni_value: None,
+            disable_tls_verify: false,
+            proxy: options.proxy,
+
+            connect_timeout: options.connect_timeout,
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+        })?;
+
+        Self::negotiate(stream, options)
+    }
+
+    fn send(&mut self, pkt: VariablePacket) -> Result<()> {
+        debug!("Sending mqtt packet: {:?}", pkt);
+        pkt.encode(&mut self.stream)?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> std::result::Result<VariablePacket, VariablePacketError> {
+        let pkt = VariablePacket::decode(&mut self.stream)?;
+        debug!("Received mqtt packet: {:?}", pkt);
+        Ok(pkt)
+    }
+
+    pub fn subscribe(&mut self, topic: &str, qos: u8) -> Result<()> {
+        let filter = TopicFilter::new(topic)?;
+
+        let qos = match qos {
+            0 => QualityOfService::Level0,
+            1 => QualityOfService::Level1,
+            2 => QualityOfService::Level2,
+            _ => bail!("Invalid QoS level: {}", qos),
+        };
+
+        let pkt = SubscribePacket::new(1, vec![(filter, qos)]);
+        self.send(pkt.into())?;
+
+        let pkt = self.recv()?;
+        if let VariablePacket::SubackPacket(_pkt) = pkt {
+            Ok(())
+        } else {
+            bail!("Expected SubAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn recv_pkt(&mut self) -> Result<Option<Pkt>> {
+        match self.recv() {
+            Ok(pkt) => Ok(Some(Pkt::try_from(pkt)?)),
+            Err(VariablePacketError::IoError(err)) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(err) => Err(Error::from(err))
+        }
+    }
+
+    pub fn ping(&mut self) -> Result<()> {
+        let pkt = PingreqPacket::new();
+        self.send(pkt.into())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum Pkt {
+    #[serde(rename="publish")]
+    Publish(Publish),
+    #[serde(rename="pong")]
+    Pong,
+}
+
+impl Pkt {
+    pub fn to_lua(&self) -> Result<AnyLuaValue> {
+        let v = serde_json::to_value(&self)?;
+        let v = LuaJsonValue::from(v).into();
+        Ok(v)
+    }
+}
+
+impl TryFrom<VariablePacket> for Pkt {
+    type Error = Error;
+
+    fn try_from(pkt: VariablePacket) -> Result<Pkt> {
+        match pkt {
+            VariablePacket::ConnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::ConnackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PublishPacket(pkt) => Ok(Pkt::Publish(Publish {
+                topic: pkt.topic_name().to_string(),
+                body: pkt.payload(),
+            })),
+            VariablePacket::PubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrecPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrelPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubcompPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingreqPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingrespPacket(_) => Ok(Pkt::Pong),
+            VariablePacket::SubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::SubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::DisconnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Publish {
+    pub topic: String,
+    pub body: Vec<u8>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrootable_https::dns::Resolver;
+
+    fn connect() -> Result<MqttClient> {
+        let resolver = Resolver::from_system_v4().unwrap();
+        let url = "mqtt://mqtt.winkekatze24.de".parse()?;
+        MqttClient::connect(&resolver, url, &MqttOptions::default())
+    }
+
+    #[test]
+    #[ignore]
+    fn test_connect() {
+        connect().expect("Failed to setup connection");
+    }
+
+    // this test is too flaky
+    /*
+    #[test]
+    #[ignore]
+    fn test_subscribe() {
+        let mut c = connect().unwrap();
+        c.subscribe("#", 0).unwrap();
+    }
+    */
+}

sn0int-std/src/psl.rs

@@ -4,9 +4,7 @@ use chrootable_https::Client;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use crate::paths;
 use crate::lazy::LazyInit;
-use crate::worker;
 
 
 #[derive(Debug, PartialEq)]
@@ -23,22 +21,24 @@ pub enum PslReader {
 }
 
 impl PslReader {
-    pub fn open_or_download() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open_or_download<F>(cache_dir: &Path, indicator: F) -> Result<PslReader>
+        where
+            F: Fn(Box<dyn Fn() -> Result<PslReader>>) -> Result<PslReader>
+    {
+        let path = Self::path(cache_dir)?;
         let reader = match Self::open_from(&path) {
             Ok(r) => r,
-            Err(_) => worker::spawn_fn("Downloading public suffix list", || {
+            Err(_) => indicator(Box::new(move || {
                 PslReader::download(&path, publicsuffix::LIST_URL)?;
                 Self::open_from(&path)
-            }, false)?,
+            }))?,
         };
         Ok(reader)
     }
 
-    pub fn open() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open(cache_dir: &Path) -> Result<PslReader> {
+        let path = Self::path(cache_dir)?;
         Self::open_from(&path)
-
     }
 
     pub fn open_from(path: &Path) -> Result<PslReader> {
@@ -46,7 +46,7 @@ impl PslReader {
         Ok(PslReader::Reader(file))
     }
 
-    pub fn path() -> Result<PathBuf> {
+    pub fn path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         let path = Path::new("/usr/share/publicsuffix/public_suffix_list.dat");
         if path.exists() {
@@ -54,7 +54,7 @@ impl PslReader {
         }
 
         // else, use local cache
-        let path = paths::cache_dir()?
+        let path = cache_dir
             .join("public_suffix_list.dat");
         Ok(path)
     }

sn0int-std/src/ratelimits.rs

@@ -0,0 +1,70 @@
+use chrono::prelude::*;
+use std::collections::HashMap;
+use std::result;
+use std::sync::mpsc;
+use std::time::Duration;
+
+pub type RatelimitSender = mpsc::Sender<result::Result<RatelimitResponse, String>>;
+
+pub struct Ratelimiter {
+    buckets: HashMap<String, Bucket>,
+}
+
+impl Ratelimiter {
+    pub fn new() -> Ratelimiter {
+        Ratelimiter {
+            buckets: HashMap::new(),
+        }
+    }
+
+    pub fn pass(&mut self, tx: RatelimitSender, key: &str, passes: u32, time: u32) {
+        let bucket = if let Some(bucket) = self.buckets.get_mut(key) {
+            bucket
+        } else {
+            let bucket = Bucket::new();
+            self.buckets.insert(key.to_string(), bucket);
+            self.buckets.get_mut(key).unwrap()
+        };
+        let reply = bucket.pass(passes as usize, time);
+        tx.send(Ok(reply)).unwrap();
+    }
+}
+
+struct Bucket {
+    passes: Vec<DateTime<Utc>>,
+}
+
+impl Bucket {
+    pub fn new() -> Bucket {
+        Bucket {
+            passes: Vec::new(),
+        }
+    }
+
+    pub fn pass(&mut self, passes: usize, time: u32) -> RatelimitResponse {
+        let now = Utc::now();
+        let time = chrono::Duration::milliseconds(time as i64);
+        let retain = now - time;
+        self.passes.retain(|x| *x >= retain);
+
+        if self.passes.len() >= passes {
+            if let Some(min) = self.passes.iter().min() {
+                let delay = time - (now - *min);
+                RatelimitResponse::Retry(delay.to_std().unwrap())
+            } else {
+                // This should never happen unless passes is zero
+                RatelimitResponse::Retry(Duration::from_millis(100))
+            }
+        } else {
+            let now = Utc::now();
+            self.passes.push(now);
+            RatelimitResponse::Pass
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum RatelimitResponse {
+    Retry(Duration),
+    Pass,
+}

sn0int-std/src/sockets/mod.rs

@@ -16,6 +16,7 @@ use std::io::BufRead;
 use std::net::SocketAddr;
 use std::net::TcpStream;
 use std::net::{IpAddr, Ipv4Addr};
+use std::time::Duration;
 
 mod tls;
 pub use self::tls::TlsData;
@@ -39,14 +40,21 @@ fn unwrap_socket(socket: tokio::net::TcpStream) -> Result<TcpStream> {
 pub struct SocketOptions {
     #[serde(default)]
     pub tls: bool,
-    sni_value: Option<String>,
+    pub sni_value: Option<String>,
     #[serde(default)]
-    disable_tls_verify: bool,
+    pub disable_tls_verify: bool,
     pub proxy: Option<SocketAddr>,
 
     // TODO: enable_sni (default to true)
+    // TODO: sni_name
     // TODO: cacert
-    // TODO: timeout
+
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
 }
 
 impl SocketOptions {
@@ -57,17 +65,97 @@ impl SocketOptions {
     }
 }
 
-#[derive(Debug)]
-pub struct Socket {
-    stream: BufStream<Stream>,
-    newline: String,
+impl SocketOptions {
+    pub fn apply(&self, stream: &Stream) -> Result<()> {
+        let socket = match stream {
+            Stream::Tcp(s) => s,
+            Stream::Tls(s) => s.get_ref(),
+        };
+        self.apply_tcp(socket)
+    }
+
+    pub fn apply_tcp(&self, socket: &TcpStream) -> Result<()> {
+        let read_timeout = self.read_timeout;
+        if read_timeout > 0 {
+            socket.set_read_timeout(Some(Duration::from_secs(read_timeout)))?;
+        }
+
+        let write_timeout = self.write_timeout;
+        if write_timeout > 0 {
+            socket.set_write_timeout(Some(Duration::from_secs(write_timeout)))?;
+        }
+
+        Ok(())
+    }
 }
 
-enum Stream {
+pub enum Stream {
     Tcp(TcpStream),
     Tls(rustls::StreamOwned<rustls::ClientSession, TcpStream>),
 }
 
+impl Stream {
+    pub fn connect_stream<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+        let addrs = match host.parse::<IpAddr>() {
+            Ok(addr) => vec![addr],
+            Err(_) => resolver.resolve(host, RecordType::A)
+                .wait_for_response()?
+                .success()?,
+        };
+
+        let mut errors = Vec::new();
+
+        for addr in addrs {
+            match Stream::connect_addr(host, (addr, port).into(), &options) {
+                Ok(socket) => {
+                    return Ok(socket);
+                },
+                Err(err) => errors.push((addr, err)),
+            }
+        }
+
+        if errors.is_empty() {
+            bail!("no dns records found");
+        } else {
+            bail!("couldn't connect: {:?}", errors);
+        }
+    }
+
+    fn connect_addr(host: &str, addr: SocketAddr, options: &SocketOptions) -> Result<Stream> {
+        debug!("connecting to {}", addr);
+
+        let connect_timeout = options.connect_timeout;
+        let socket = if connect_timeout > 0 {
+            TcpStream::connect_timeout(&addr, Duration::from_secs(connect_timeout))?
+        } else {
+            TcpStream::connect(&addr)?
+        };
+        debug!("successfully connected to {:?}", addr);
+
+        options.apply_tcp(&socket)?;
+
+        tls::wrap_if_enabled(socket, host, options)
+    }
+
+    pub fn connect_socks5_stream(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
+
+        let addr = match host.parse::<Ipv4Addr>() {
+            Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
+            _ => ProxyDest::Domain(host.to_string()),
+        };
+
+        let fut = socks5::connect(proxy, addr, port);
+
+        let mut rt = Runtime::new()?;
+        let socket = rt.block_on(fut)?;
+
+        let socket = unwrap_socket(socket)?;
+
+        tls::wrap_if_enabled(socket, host, options)
+    }
+}
+
 impl fmt::Debug for Stream {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
         match self {
@@ -102,6 +190,12 @@ impl Write for Stream {
     }
 }
 
+#[derive(Debug)]
+pub struct Socket {
+    stream: BufStream<Stream>,
+    newline: String,
+}
+
 impl Socket {
     fn new(stream: Stream) -> Socket {
         let stream = BufStream::new(stream);
@@ -112,60 +206,31 @@ impl Socket {
     }
 
     pub fn connect<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
-        let addrs = match host.parse::<IpAddr>() {
-            Ok(addr) => vec![addr],
-            Err(_) => resolver.resolve(host, RecordType::A)
-                .wait_for_response()?
-                .success()?,
-        };
-
-        let mut errors = Vec::new();
-
-        for addr in addrs {
-            debug!("connecting to {}:{}", addr, port);
-            match TcpStream::connect((addr, port)) {
-                Ok(socket) => {
-                    debug!("successfully connected to {:?}", addr);
-                    return tls::wrap_if_enabled(socket, host, options);
-                },
-                Err(err) => errors.push((addr, err)),
-            }
-        }
-
-        if errors.is_empty() {
-            bail!("no dns records found");
-        } else {
-            bail!("couldn't connect: {:?}", errors);
-        }
+        let stream = Stream::connect_stream(resolver, host, port, options)?;
+        Ok(Socket::new(stream))
     }
 
     pub fn connect_socks5(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
-        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
-
-        let addr = match host.parse::<Ipv4Addr>() {
-            Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
-            _ => ProxyDest::Domain(host.to_string()),
-        };
-
-        let fut = socks5::connect(proxy, addr, port);
-
-        let mut rt = Runtime::new()?;
-        let socket = rt.block_on(fut)?;
-
-        let socket = unwrap_socket(socket)?;
-
-        tls::wrap_if_enabled(socket, host, options)
+        let stream = Stream::connect_socks5_stream(proxy, host, port, options)?;
+        Ok(Socket::new(stream))
     }
 
     pub fn upgrade_to_tls(self, options: &SocketOptions) -> Result<(Socket, TlsData)> {
         let stream = self.stream.into_inner()?;
 
-        match stream {
-            Stream::Tcp(stream) => tls::wrap(stream, "", options),
-            _ => bail!("Only tcp streams can be upgraded"),
+        if let Stream::Tcp(stream) = stream {
+            let (stream, tls) = tls::wrap(stream, "", options)?;
+            let socket = Socket::new(stream);
+            Ok((socket, tls))
+        } else {
+            bail!("Only tcp streams can be upgraded")
         }
     }
 
+    pub fn options(&self, options: &SocketOptions) -> Result<()> {
+        options.apply(self.stream.get_ref())
+    }
+
     pub fn send(&mut self, data: &[u8]) -> Result<()> {
         match str::from_utf8(&data) {
             Ok(data) => debug!("send: {:?}", data),
@@ -178,7 +243,12 @@ impl Socket {
 
     pub fn recv(&mut self) -> Result<Vec<u8>> {
         let mut buf = [0; 4096];
-        let n = self.stream.read(&mut buf)?;
+        let n = match self.stream.read(&mut buf) {
+            Ok(n) if n == 0 => bail!("Connection closed"),
+            Ok(n) => n,
+            Err(err) if err.kind() == io::ErrorKind::WouldBlock => 0,
+            Err(err) => return Err(err.into()),
+        };
         let data = buf[..n].to_vec();
         match str::from_utf8(&data) {
             Ok(data) => debug!("recv: {:?}", data),
@@ -249,6 +319,7 @@ impl Socket {
                 let available = match self.stream.fill_buf() {
                     Ok(n) => n,
                     Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
+                    Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(Vec::new()),
                     Err(e) => return Err(e.into())
                 };
 
@@ -293,7 +364,7 @@ mod tests {
     #[ignore]
     #[test]
     fn verify_tls_good() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let _sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -303,7 +374,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_good_request() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let mut sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -316,7 +387,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_expired() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let sock = Socket::connect(&resolver, "expired.badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -330,7 +401,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_1_1_1_1() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let _sock = Socket::connect(&resolver, "1.1.1.1", 443, &SocketOptions{
             tls: true,
             ..Default::default()

sn0int-std/src/sockets/tls.rs

@@ -2,14 +2,14 @@ use crate::errors::*;
 
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
-use rustls::{self, ClientConfig, Session, ClientSession, RootCertStore};
+use rustls::{self, ClientConfig, Session, ClientSession};
 
 use std::str;
 use std::result;
 use std::sync::Arc;
 use std::net::TcpStream;
 
-use super::{Socket, Stream, SocketOptions};
+use super::{Stream, SocketOptions};
 
 
 #[derive(Debug, Serialize)]
@@ -26,22 +26,22 @@ impl TlsData {
     }
 }
 
-pub fn wrap_if_enabled(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<Socket> {
+pub fn wrap_if_enabled(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<Stream> {
     if !options.tls {
         let stream = Stream::Tcp(stream);
-        return Ok(Socket::new(stream));
+        Ok(stream)
     } else {
         let (socket, _) = wrap(stream, host, options)?;
         Ok(socket)
     }
 }
 
-pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(Socket, TlsData)> {
-    let mut anchors = RootCertStore::empty();
-    anchors.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
-
+pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(Stream, TlsData)> {
     let mut config = ClientConfig::new();
-    config.root_store = anchors;
+    config
+        .root_store
+        .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
+    config.ct_logs = Some(&ct_logs::LOGS);
 
     if options.disable_tls_verify {
         info!("tls verification has been disabled");
@@ -73,14 +73,16 @@ fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
     }
 }
 
-fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Socket, TlsData)> {
+fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, TlsData)> {
     info!("starting tls handshake");
     if session.is_handshaking() {
-        session.complete_io(&mut stream)?;
+        session.complete_io(&mut stream)
+            .context("Failed to read reply to tls client hello")?;
     }
 
     if session.wants_write() {
-        session.complete_io(&mut stream)?;
+        session.complete_io(&mut stream)
+            .context("wants_write->complete_io failed")?;
     }
 
     let mut tls = TlsData {
@@ -106,7 +108,7 @@ fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Socket, T
     info!("successfully established tls connection");
     let stream = rustls::StreamOwned::new(session, stream);
     let stream = Stream::Tls(stream);
-    Ok((Socket::new(stream), tls))
+    Ok((stream, tls))
 }
 
 pub struct NoCertificateVerification {}

sn0int-std/src/web.rs

@@ -1,17 +1,17 @@
 pub use chrootable_https::{Client, HttpClient, Resolver, Response};
 
-use crate::blobs::Blob;
+use crate::blobs::{Blob, BlobState};
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
 use std::ops::Deref;
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
-use crate::engine::ctx::State;
 use serde_json;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
 use std::fmt;
 use std::net::SocketAddr;
+use std::sync::Arc;
 use serde::Serialize;
 use crate::engine::structs::LuaMap;
 use crate::json::LuaJsonValue;
@@ -37,6 +37,12 @@ pub fn url_set_qs<S: Serialize + fmt::Debug>(url: Uri, query: &S) -> Result<Uri>
         .map_err(Error::from)
 }
 
+pub trait WebState {
+    fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>>;
+
+    fn register_in_jar(&self, session: &str, key: String, value: String);
+}
+
 #[derive(Debug)]
 pub struct HttpSession {
     id: String,
@@ -66,6 +72,8 @@ pub struct RequestOptions {
     #[serde(default)]
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    #[serde(default)]
+    binary: bool,
 }
 
 impl RequestOptions {
@@ -95,6 +103,7 @@ pub struct HttpRequest {
     timeout: Option<Duration>,
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    binary: bool,
 }
 
 impl HttpRequest {
@@ -117,6 +126,7 @@ impl HttpRequest {
             timeout,
             into_blob: options.into_blob,
             proxy: options.proxy,
+            binary: options.binary,
         };
 
         if let Some(json) = options.json {
@@ -134,7 +144,7 @@ impl HttpRequest {
         request
     }
 
-    pub fn send(&self, state: &dyn State) -> Result<Response> {
+    pub fn send(&self, state: &dyn WebState) -> Result<Response> {
         let mut url = self.url.parse::<Uri>()?;
 
         // set query string
@@ -228,7 +238,9 @@ impl HttpRequest {
         Ok(res)
     }
 
-    pub fn response_to_lua(&self, state: &dyn State, res: Response) -> Result<LuaMap> {
+    pub fn response_to_lua<S>(&self, state: &S, res: Response) -> Result<LuaMap>
+        where S: WebState + BlobState
+    {
         // map result to LuaMap
         let mut resp = LuaMap::new();
         resp.insert_num("status", f64::from(res.status));
@@ -239,10 +251,16 @@ impl HttpRequest {
         }
         resp.insert("headers", headers);
 
+        if let Some(ipaddr) = res.ipaddr {
+            resp.insert_str("ipaddr", ipaddr.to_string());
+        }
+
         if self.into_blob {
             let blob = Blob::create(res.body);
             let id = state.register_blob(blob);
             resp.insert_str("blob", id);
+        } else if self.binary {
+            resp.insert_serde("binary", &res.body[..])?;
         } else {
             resp.insert_str("text", String::from_utf8_lossy(&res.body));
         }
@@ -250,7 +268,7 @@ impl HttpRequest {
         Ok(resp)
     }
 
-    fn register_cookies_on_state(session: &str, state: &dyn State, cookie: &str) {
+    fn register_cookies_on_state(session: &str, state: &dyn WebState, cookie: &str) {
         let mut key = String::new();
         let mut value = String::new();
         let mut in_key = true;

sn0int-std/src/websockets.rs

@@ -0,0 +1,150 @@
+use chrootable_https::DnsResolver;
+use crate::errors::*;
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use crate::sockets::{Stream, SocketOptions};
+use http::Request;
+use std::collections::HashMap;
+use std::net::SocketAddr;
+use std::io;
+use tungstenite::protocol::{self, Message};
+use url::Url;
+
+#[derive(Debug, Default, Deserialize)]
+pub struct WebSocketOptions {
+    pub headers: Option<HashMap<String, String>>,
+    pub proxy: Option<SocketAddr>,
+
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
+}
+
+impl WebSocketOptions {
+    pub fn try_from(x: AnyLuaValue) -> Result<WebSocketOptions> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub enum Event {
+    Text(String),
+    Binary(Vec<u8>),
+    Close,
+    Timeout,
+}
+
+pub struct WebSocket {
+    sock: protocol::WebSocket<Stream>,
+}
+
+impl WebSocket {
+    pub fn negotiate(stream: Stream, url: Url, headers: Option<&HashMap<String, String>>) -> Result<WebSocket> {
+        let mut req = Request::get(url.to_string()); // TODO: don't re-parse here
+
+        if let Some(headers) = headers {
+            for (k, v) in headers {
+                req = req.header(k, v);
+            }
+        }
+
+        let req = req.body(()).unwrap();
+
+        let (sock, _resp) = tungstenite::client::client(req, stream)?;
+        Ok(WebSocket {
+            sock,
+        })
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &WebSocketOptions) -> Result<WebSocket> {
+        let tls = match url.scheme() {
+            "ws" => false,
+            "wss" => true,
+            _ => bail!("Invalid websocket protocol"),
+        };
+
+        let host = url.host_str()
+            .ok_or_else(|| format_err!("Missing host in url"))?;
+
+        let port = match (url.port(), tls) {
+            (Some(port), _) => port,
+            (None, true) => 443,
+            (None, false) => 80,
+        };
+
+        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
+            tls,
+            sni_value: None,
+            disable_tls_verify: false,
+            proxy: options.proxy,
+
+            connect_timeout: options.connect_timeout,
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+        })?;
+        Self::negotiate(stream, url, options.headers.as_ref())
+    }
+
+    pub fn options(&self, options: &WebSocketOptions) -> Result<()> {
+        let mut o = SocketOptions::default();
+        o.read_timeout = options.read_timeout;
+        o.write_timeout = options.write_timeout;
+        o.apply(self.sock.get_ref())
+    }
+
+    fn read_msg(&mut self) -> Result<Event> {
+        loop {
+            let msg = match self.sock.read_message() {
+                Ok(Message::Text(body)) => Event::Text(body),
+                Ok(Message::Binary(body)) => Event::Binary(body),
+                Ok(Message::Ping(ping)) => {
+                    self.sock.write_message(Message::Pong(ping))?;
+                    continue;
+                },
+                Ok(Message::Pong(_)) => continue, // this should never happen
+                Ok(Message::Close(_)) => Event::Close,
+                Err(tungstenite::Error::ConnectionClosed) => Event::Close,
+                Err(tungstenite::Error::AlreadyClosed) => Event::Close,
+                Err(tungstenite::Error::Io(err)) if err.kind() == io::ErrorKind::WouldBlock => Event::Timeout,
+                Err(err) => return Err(err.into()),
+            };
+            return Ok(msg);
+        }
+    }
+
+    pub fn read_text(&mut self) -> Result<Option<String>> {
+        match self.read_msg()? {
+            Event::Text(text) => Ok(Some(text)),
+            Event::Binary(_) => bail!("Unexpected message type: binary"),
+            Event::Close => bail!("Connection closed"),
+            Event::Timeout => Ok(None),
+        }
+    }
+
+    pub fn read_binary(&mut self) -> Result<Option<Vec<u8>>> {
+        match self.read_msg()? {
+            Event::Text(_) => bail!("Unexpected message type: text"),
+            Event::Binary(body) => Ok(Some(body)),
+            Event::Close => bail!("Connection closed"),
+            Event::Timeout => Ok(None),
+        }
+    }
+
+    fn write_msg(&mut self, msg: Message) -> Result<()> {
+        self.sock.write_message(msg)?;
+        self.sock.write_pending()?;
+        Ok(())
+    }
+
+    pub fn write_text(&mut self, text: String) -> Result<()> {
+        self.write_msg(Message::Text(text))
+    }
+
+    pub fn write_binary(&mut self, binary: Vec<u8>) -> Result<()> {
+        self.write_msg(Message::Binary(binary))
+    }
+}

sn0int-std/src/xml.rs

@@ -124,6 +124,7 @@ fn decode_raw(x: &str) -> Result<XmlDocument> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn verify_xml_decode_empty() {

src/api.rs

@@ -22,10 +22,13 @@ pub struct Client {
 
 impl Client {
     pub fn new(config: &Config) -> Result<Client> {
-        let client = match config.network.proxy {
-            Some(proxy) => chrootable_https::Client::with_socks5(proxy),
-            _ => chrootable_https::Client::with_system_resolver()?,
+        let client = if let Some(proxy) = config.network.proxy {
+            chrootable_https::Client::with_socks5(proxy)
+        } else {
+            chrootable_https::Client::with_system_resolver()
+                .context("Failed to load dns configuration")?
         };
+
         Ok(Client {
             server: config.core.registry.clone(),
             client,
@@ -41,12 +44,20 @@ impl Client {
         thread_rng().sample_iter(&Alphanumeric).take(32).collect()
     }
 
+    #[inline]
+    fn user_agent() -> String {
+        format!("{}, {}",
+            web::default_user_agent(),
+            embedded_triple::get()
+        )
+    }
+
     pub fn request<T: DeserializeOwned + fmt::Debug>(&self, mut request: RequestBuilder, body: Body) -> Result<T> {
         if let Some(session) = &self.session {
             info!("Adding session token to request");
             request.header("Auth", session.as_str());
         }
-        request.header("User-Agent", web::default_user_agent());
+        request.header("User-Agent", Self::user_agent());
 
         let request = request.body(body)?;
 

src/archive.rs

@@ -1,40 +0,0 @@
-use crate::errors::*;
-use tar;
-use libflate::gzip;
-use std::io;
-use std::fs::File;
-use std::path::Path;
-
-
-pub fn extract<R: io::Read, P: AsRef<Path>>(read: &mut R, filter: &str, target: P) -> Result<()> {
-    let file = gzip::Decoder::new(read)?;
-    let mut ar = tar::Archive::new(file);
-
-    for entry in ar.entries()? {
-        let mut entry = entry?;
-        let file_name = {
-            let path = entry.path()?;
-            path.file_name()
-                .ok_or_else(|| format_err!("Invalid path in archive"))?
-                .to_str()
-                .ok_or_else(|| format_err!("Filename is invalid utf8"))?
-                .to_owned()
-        };
-
-        debug!("Found in archive: {:?}", file_name);
-
-        if filter == file_name {
-            debug!("Extracting to {:?}", target.as_ref());
-
-            let mut target = File::create(target)?;
-            let n = io::copy(&mut entry, &mut target)?;
-            debug!("Wrote {:?} bytes", n);
-
-            return Ok(());
-        }
-
-        debug!("Skipping file");
-    }
-
-    bail!("Nothing in archive matched filter")
-}

src/args.rs

@@ -48,6 +48,9 @@ pub enum SubCommand {
     /// Search in the registry
     #[structopt(name="search")]
     Search(Search),
+    /// The sn0int package manager
+    #[structopt(name="pkg")]
+    Pkg(cmd::pkg_cmd::Args),
     /// Insert into the database
     #[structopt(name="add")]
     Add(cmd::add_cmd::Args),
@@ -57,6 +60,9 @@ pub enum SubCommand {
     /// Delete from the database
     #[structopt(name="delete")]
     Delete(cmd::delete_cmd::Args),
+    /// Query logged activity
+    #[structopt(name="activity")]
+    Activity(cmd::activity_cmd::Args),
     /// Include entities in the scope
     #[structopt(name="scope")]
     Scope(cmd::scope_cmd::Args),
@@ -75,6 +81,9 @@ pub enum SubCommand {
     /// Run a lua repl
     #[structopt(name="repl")]
     Repl,
+    /// Show paths of various file system locations
+    #[structopt(name="paths")]
+    Paths,
     /// Generate shell completions
     #[structopt(name="completions")]
     Completions(Completions),
@@ -115,6 +124,9 @@ pub struct Run {
     /// Narrow down targeted entities
     #[structopt(short="t", long="target")]
     pub target: Option<String>,
+    /// Dump the sandbox init message to stdout instead of running a child process
+    #[structopt(long="dump-sandbox-init-msg")]
+    pub dump_sandbox_init_msg: bool,
 }
 
 #[derive(Debug, StructOpt)]

src/auth.rs

@@ -11,7 +11,7 @@ use crate::term;
 
 
 fn path() -> Result<PathBuf> {
-    let path = paths::data_dir()?;
+    let path = paths::sn0int_dir()?;
     Ok(path.join("auth"))
 }
 

src/blobs.rs

@@ -3,44 +3,13 @@ use crate::paths;
 use crate::worker::{EventWithCallback, Event2};
 use crate::workspaces::Workspace;
 
-use blake2::VarBlake2b;
-use digest::{Input, VariableOutput};
 use bytes::Bytes;
-use serde::ser::{Serialize, Serializer};
-use serde::de::{self, Deserialize, Deserializer};
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::mpsc;
 
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct Blob {
-    pub id: String,
-    pub bytes: Bytes,
-}
-
-impl Blob {
-    pub fn create(bytes: Bytes) -> Blob {
-        let id = Self::hash(&bytes);
-        Blob {
-            id,
-            bytes,
-        }
-    }
-
-    pub fn hash(bytes: &[u8]) -> String {
-        let mut h = VarBlake2b::new(32).unwrap();
-        h.input(bytes);
-        Self::encode_hash(&h.vec_result())
-    }
-
-    #[inline]
-    fn encode_hash(bytes: &[u8]) -> String {
-        let x = bs58::encode(bytes).into_string();
-        format!("{:0<44}", x)
-    }
-}
+pub use sn0int_std::blobs::Blob;
 
 impl EventWithCallback for Blob {
     type Payload = ();
@@ -51,30 +20,6 @@ impl EventWithCallback for Blob {
     }
 }
 
-impl Serialize for Blob {
-    #[inline]
-    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let s = base64::encode(&self.bytes);
-        serializer.serialize_str(&s)
-    }
-}
-
-impl<'de> Deserialize<'de> for Blob {
-    #[inline]
-    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let s = String::deserialize(deserializer)?;
-        let bytes = base64::decode(&s)
-            .map_err(de::Error::custom)?;
-        Ok(Blob::create(Bytes::from(bytes)))
-    }
-}
-
 pub struct BlobStorage {
     path: PathBuf,
 }
@@ -155,7 +100,6 @@ impl BlobStorage {
 mod tests {
     use super::*;
     use tempfile;
-    use serde_json;
 
     #[inline]
     fn blob() -> (Bytes, Blob) {
@@ -163,15 +107,6 @@ mod tests {
         (bytes.clone(), Blob::create(bytes))
     }
 
-    #[test]
-    fn verify_create_blob() {
-        let (bytes, blob) = blob();
-        assert_eq!(blob, Blob {
-            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
-            bytes,
-        });
-    }
-
     #[test]
     fn test_blobstorage_save() {
         let dir = tempfile::tempdir().expect("tempdir");
@@ -204,38 +139,4 @@ mod tests {
 
         assert!(result.is_err());
     }
-
-    #[test]
-    fn test_serialize() {
-        let (_, blob) = blob();
-        let json = serde_json::to_string(&blob).unwrap();
-        assert_eq!(&json, "\"YXNkZg==\"");
-    }
-
-    #[test]
-    fn test_deserialize() {
-        let (_, blob1) = blob();
-        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
-        assert_eq!(blob1, blob2);
-    }
-
-    #[test]
-    fn test_hash_encoding() {
-        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
-    }
-
-    #[test]
-    fn test_hash_encoding_padding() {
-        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
-    }
 }

src/cmd/activity_cmd.rs

@@ -0,0 +1,177 @@
+use crate::errors::*;
+
+use crate::cmd::Cmd;
+use crate::shell::Shell;
+use crate::models::*;
+use chrono::{Utc, NaiveDateTime, NaiveTime, Duration};
+use regex::Regex;
+use std::convert::TryFrom;
+use std::io;
+use std::str::FromStr;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug)]
+pub struct TimeSpec {
+    datetime: NaiveDateTime,
+}
+
+impl TimeSpec {
+    fn resolve(s: &str, now: NaiveDateTime) -> Result<Self> {
+        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
+
+        let datetime = match s {
+            "today" => today,
+            "yesterday" => today - Duration::days(1),
+            s if s.ends_with(" ago") => {
+                let re = Regex::new(r"(\d+) ?(s|seconds?|m|min|minutes?|h|hours?|d|days?|w|weeks?|months?|y|years?) ago").unwrap();
+
+                let caps = re.captures(s)
+                    .ok_or_else(|| format_err!("Couldn't parse TimeSpec"))?;
+
+                let n = caps.get(1).unwrap().as_str()
+                    .parse::<i64>()
+                    .context("Failed to parse number in timespec")?;
+                let unit = caps.get(2).unwrap();
+
+                let duration = match unit.as_str() {
+                    "s" | "second" | "seconds" => Duration::seconds(n),
+                    "m" | "min" | "minute" | "minutes" => Duration::minutes(n),
+                    "h" | "hour" | "hours" => Duration::hours(n),
+                    "d" | "day" | "days" => Duration::days(n),
+                    "w" | "week" | "weeks" => Duration::days(n * 7),
+                    "month" | "months" => Duration::days(n * 31),
+                    "y" | "year" | "years" => Duration::days(n * 365),
+                    _ => unreachable!(),
+                };
+                now - duration
+            },
+            s => NaiveDateTime::from_str(s)?,
+        };
+
+        Ok(TimeSpec {
+            datetime,
+        })
+    }
+}
+
+impl FromStr for TimeSpec {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        let now = Utc::now().naive_utc();
+        Self::resolve(s, now)
+    }
+}
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Only query events for a given topic
+    #[structopt(short="t", long="topic")]
+    topic: Option<String>,
+    /// Only query events starting from that datetime
+    #[structopt(long="since")]
+    since: Option<TimeSpec>,
+    /// Only query events until this datetime
+    #[structopt(long="until")]
+    until: Option<TimeSpec>,
+    /// Try to select the previous event before --since as an initial state
+    #[structopt(short="i", long="initial")]
+    initial: bool,
+    /// Only query events that are tied to a location
+    #[structopt(short="l", long="location")]
+    location: bool,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let filter = ActivityFilter {
+            topic: self.topic,
+            since: self.since.map(|t| t.datetime),
+            until: self.until.map(|t| t.datetime),
+            location: self.location,
+        };
+
+        let mut stdout = io::stdout();
+        let events = Activity::query(rl.db(), &filter)?;
+
+        if self.initial {
+            if let Some(first) = events.get(0) {
+                let previous = Activity::previous(rl.db(), first, &filter)?;
+                if let Some(previous) = previous {
+                    let mut previous = JsonActivity::try_from(previous)?;
+                    previous.initial = true;
+                    previous.write_to(&mut stdout)?;
+                }
+            }
+        }
+
+        for activity in events {
+            let activity = JsonActivity::try_from(activity)?;
+            activity.write_to(&mut stdout)?;
+        }
+
+        Ok(())
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn datetime() -> NaiveDateTime {
+        let date = chrono::NaiveDate::from_ymd(2020, 3, 14);
+        let time = chrono::NaiveTime::from_hms(16, 20, 23);
+        NaiveDateTime::new(date, time)
+    }
+
+    #[test]
+    fn test_today() {
+        let x = TimeSpec::resolve("today", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_yesterday() {
+        let x = TimeSpec::resolve("yesterday", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-13T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_20_min_ago() {
+        let x = TimeSpec::resolve("20min ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:00:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_days_ago() {
+        let x = TimeSpec::resolve("3 days ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-11T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_week_ago() {
+        let x = TimeSpec::resolve("1w ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-07T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_months_ago() {
+        let x = TimeSpec::resolve("3 months ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-12-12T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_year_ago() {
+        let x = TimeSpec::resolve("1 year ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-03-15T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_exact_time() {
+        let x = TimeSpec::resolve("2020-03-14T16:20:23", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:20:23").unwrap());
+    }
+}

src/cmd/add_cmd.rs

@@ -11,6 +11,7 @@ use crate::utils;
 use crate::term;
 use std::fs;
 use std::net;
+use std::net::SocketAddr;
 use ipnetwork;
 use std::path::Path;
 use walkdir::WalkDir;
@@ -37,7 +38,7 @@ pub enum Target {
     #[structopt(name="ipaddr")]
     IpAddr(AddIpAddr),
     /// Insert url into the database
-    #[structopt(name="subdomain")]
+    #[structopt(name="url")]
     Url(AddUrl),
     /// Insert email into the database
     #[structopt(name="email")]
@@ -63,6 +64,9 @@ pub enum Target {
     /// Insert ip network into the database
     #[structopt(name="netblock")]
     Netblock(AddNetblock),
+    /// Insert port into the database
+    #[structopt(name="port")]
+    Port(AddPort),
     /// Insert a crypto currency address into the database
     #[structopt(name="cryptoaddr")]
     CryptoAddr(AddCryptoAddr),
@@ -83,6 +87,7 @@ impl Cmd for Args {
             Target::Breach(args) => args.insert(rl, self.dry_run),
             Target::Image(args) => args.insert(rl, self.dry_run),
             Target::Netblock(args) => args.insert(rl, self.dry_run),
+            Target::Port(args) => args.insert(rl, self.dry_run),
             Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
         }
     }
@@ -555,6 +560,71 @@ impl IntoInsert for AddNetblock {
     }
 }
 
+#[derive(Debug, StructOpt)]
+pub struct AddPort {
+    protocol: Option<String>,
+    addr: Option<SocketAddr>,
+}
+
+impl IntoInsert for AddPort {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
+        let protocol = if let Some(protocol) = self.protocol {
+            protocol
+        } else {
+            utils::question("Protocol (tcp/udp)")?
+        };
+
+        let addr = if let Some(addr) = self.addr {
+            addr
+        } else {
+            let addr = utils::question("IP:Port")?;
+            addr.parse()?
+        };
+
+        let family = match addr.ip() {
+            net::IpAddr::V4(_) => "4",
+            net::IpAddr::V6(_) => "6",
+        };
+
+        let ip_addr_id = match rl.db().insert_struct(NewIpAddr {
+            family: family.to_string(),
+            value: addr.ip().to_string(),
+            continent: None,
+            continent_code: None,
+            country: None,
+            country_code: None,
+            city: None,
+            latitude: None,
+            longitude: None,
+            asn: None,
+            as_org: None,
+            description: None,
+            reverse_dns: None,
+            unscoped: false,
+        }, true)? {
+            Some((_, ip_addr_id)) => ip_addr_id,
+            _ => bail!("IpAddr is out out of scope"),
+        };
+
+        let value = format!("{}/{}", protocol, addr);
+
+        Ok(Insert::Port(NewPort {
+            ip_addr_id,
+            value,
+            ip_addr: addr.ip().to_string(),
+            port: addr.port() as i32,
+            protocol,
+            status: None,
+
+            banner: None,
+            service: None,
+            version: None,
+
+            unscoped: false,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddCryptoAddr {
     address: Option<String>,

src/cmd/help_cmd.rs

@@ -16,9 +16,8 @@ pub fn run(_rl: &mut Shell, _args: &[String]) -> Result<()> {
     help("autoscope",   "Manage rules to automatically add entities to scope");
     help("delete",      "Delete entities from the database");
     help("keyring",     "Manage saved credentials");
-    help("mod",         "Manage installed modules");
+    help("pkg",         "Manage installed modules");
     help("noscope",     "Exclude entities from scope");
-    help("quickstart",  "Install all featured modules");
     help("run",         "Run the currently selected module");
     help("scope",       "Include entities in the scope again");
     help("select",      "Select entities from the database");

src/cmd/mod.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 use crate::shell::Shell;
+use crate::config::Config;
 
 pub trait Cmd: structopt::StructOpt + Sized {
     fn run(self, rl: &mut Shell) -> Result<()>;
@@ -11,6 +12,11 @@ pub trait Cmd: structopt::StructOpt + Sized {
     }
 }
 
+pub trait LiteCmd: structopt::StructOpt + Sized {
+    fn run(self, config: &Config) -> Result<()>;
+}
+
+pub mod activity_cmd;
 pub mod add_cmd;
 pub mod autonoscope_cmd;
 pub mod autoscope_cmd;
@@ -22,8 +28,8 @@ pub mod run_cmd;
 pub mod use_cmd;
 pub mod select_cmd;
 pub mod keyring_cmd;
-pub mod mod_cmd;
 pub mod noscope_cmd;
+pub mod pkg_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;
 pub mod target_cmd;

src/cmd/pkg_cmd.rs

@@ -1,7 +1,12 @@
 use crate::errors::*;
 
+use crate::args::Install;
+use crate::api::Client;
 use crate::args;
-use crate::registry::{self, UpdateTask, Updater};
+use crate::config::Config;
+use crate::cmd::{Cmd, LiteCmd};
+use crate::engine::Library;
+use crate::registry::{self, InstallTask, UpdateTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
 use crate::worker;
@@ -20,6 +25,13 @@ pub struct Args {
     pub subcommand: SubCommand,
 }
 
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct ArgsInteractive {
+    #[structopt(subcommand)]
+    pub subcommand: SubCommandInteractive,
+}
+
 #[derive(Debug, StructOpt)]
 pub enum SubCommand {
     /// List installed modules
@@ -31,15 +43,24 @@ pub enum SubCommand {
     /// Search modules in registry
     #[structopt(name="search")]
     Search(args::Search),
-    /// Reload modules
-    #[structopt(name="reload")]
-    Reload(Reload),
     /// Update modules
     #[structopt(name="update")]
     Update(Update),
     /// Uninstall a module
     #[structopt(name="uninstall")]
     Uninstall(Uninstall),
+    /// Install all featured modules
+    #[structopt(name="quickstart")]
+    Quickstart,
+}
+
+#[derive(Debug, StructOpt)]
+pub enum SubCommandInteractive {
+    #[structopt(flatten)]
+    Base(SubCommand),
+    /// Reload modules
+    #[structopt(name="reload")]
+    Reload(Reload),
 }
 
 #[derive(Debug, StructOpt)]
@@ -50,6 +71,9 @@ pub struct List {
     /// List outdated modules
     #[structopt(long="outdated")]
     pub outdated: bool,
+    /// Filter by pattern
+    #[structopt(default_value="*")]
+    pub pattern: String,
 }
 
 #[derive(Debug, StructOpt)]
@@ -65,15 +89,20 @@ pub struct Uninstall {
     module: ModuleID,
 }
 
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    let config = rl.config().clone();
+#[derive(PartialEq)]
+enum ModuleReload {
+    Yes,
+    No,
+}
 
-    match args.subcommand {
+fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) -> Result<ModuleReload> {
+    match subcommand {
         SubCommand::List(list) => {
             let autoupdate = AutoUpdater::load()?;
 
-            for module in rl.engine().list() {
+            let filter = glob::Pattern::new(&list.pattern)?;
+
+            for module in library.list() {
                 if let Some(source) = &list.source {
                     if !module.source_equals(&source) {
                         continue;
@@ -81,6 +110,9 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
                 }
 
                 let canonical = module.canonical();
+                if !filter.matches(&canonical) {
+                    continue;
+                }
 
                 let mut out = String::new();
                 write!(&mut out, "{} ({})", canonical.green(),
@@ -93,31 +125,22 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
                 println!("{}", out);
                 println!("\t{}", module.description());
             }
+            Ok(ModuleReload::No)
         },
         SubCommand::Install(install) => {
             registry::run_install(install, &config)?;
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
-        SubCommand::Search(search) => registry::run_search(rl.engine(), &search, &config)?,
-        SubCommand::Reload(_) => {
-            let current = rl.take_module()
-                            .map(|m| m.canonical());
-
-            rl.engine_mut().reload_modules()?;
-            rl.reload_module_cache();
-
-            if let Some(module) = current {
-                if let Ok(module) = rl.engine().get(&module).map(|x| x.to_owned()) {
-                    rl.set_module(module);
-                }
-            }
+        SubCommand::Search(search) => {
+            registry::run_search(library, &search, &config)?;
+            Ok(ModuleReload::No)
         },
         SubCommand::Update(_) => {
             let mut autoupdate = AutoUpdater::load()?;
             let updater = Arc::new(Updater::new(&config)?);
 
-            let modules = rl.engine().list()
+            let modules = library.list()
                 .into_iter()
                 .filter_map(|module| {
                     let canonical = module.canonical();
@@ -138,15 +161,62 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
             autoupdate.save()?;
 
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
         SubCommand::Uninstall(uninstall) => {
             let updater = Updater::new(&config)?;
             updater.uninstall(&uninstall.module)?;
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
+        },
+        SubCommand::Quickstart => {
+            let client = Client::new(&config)?;
+            let updater = Arc::new(Updater::new(&config)?);
+            let mut autoupdate = AutoUpdater::load()?;
+
+            let modules = client.quickstart()?
+                .into_iter()
+                .map(|module| {
+                    InstallTask::new(Install {
+                        module: ModuleID {
+                            author: module.author,
+                            name: module.name,
+                        },
+                        version: None,
+                        force: false,
+                    }, updater.clone())
+                })
+                .collect::<Vec<_>>();
+
+            worker::spawn_multi(modules, |name| {
+                autoupdate.updated(&name);
+            }, 3)?;
+
+            autoupdate.save()?;
+
+            // trigger reload
+            Ok(ModuleReload::Yes)
         },
     }
-
-    Ok(())
+}
+
+impl LiteCmd for Args {
+    fn run(self, config: &Config) -> Result<()> {
+        let library = Library::new(false, &config)?;
+        run_subcommand(self.subcommand, &library, config)?;
+        Ok(())
+    }
+}
+
+impl Cmd for ArgsInteractive {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let action = match self.subcommand {
+            SubCommandInteractive::Base(subcommand) => run_subcommand(subcommand, rl.library(), rl.config())?,
+            SubCommandInteractive::Reload(_) => ModuleReload::Yes,
+        };
+        if action == ModuleReload::Yes {
+            rl.reload_modules()?;
+        }
+        Ok(())
+    }
 }

src/cmd/quickstart_cmd.rs

@@ -1,16 +1,11 @@
 use crate::errors::*;
 
-use crate::args::Install;
-use crate::api::Client;
-use crate::cmd::mod_cmd;
-use crate::registry::{InstallTask, Updater};
+use crate::cmd::Cmd;
+use crate::cmd::pkg_cmd::{ArgsInteractive as PkgArgs, SubCommand, SubCommandInteractive};
 use crate::shell::Shell;
-use crate::update::AutoUpdater;
-use crate::worker;
-use std::sync::Arc;
+use crate::term;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
-use sn0int_common::ModuleID;
 
 
 #[derive(Debug, StructOpt)]
@@ -20,34 +15,11 @@ pub struct Args {
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     let _args = Args::from_iter_safe(args)?;
-    let config = rl.config().clone();
 
-    let client = Client::new(&config)?;
-    let updater = Arc::new(Updater::new(&config)?);
-    let mut autoupdate = AutoUpdater::load()?;
+    term::warn("The \x1b[1mquickstart\x1b[0m command is deprecated, use \x1b[1mpkg quickstart\x1b[0m");
 
-    let modules = client.quickstart()?
-        .into_iter()
-        .map(|module| {
-            InstallTask::new(Install {
-                module: ModuleID {
-                    author: module.author,
-                    name: module.name,
-                },
-                version: None,
-                force: false,
-            }, updater.clone())
-        })
-        .collect::<Vec<_>>();
-
-    worker::spawn_multi(modules, |name| {
-        autoupdate.updated(&name);
-    }, 3)?;
-
-    autoupdate.save()?;
-
-    // trigger reload
-    mod_cmd::run(rl, &[String::from("mod"), String::from("reload")])?;
-
-    Ok(())
+    let args = PkgArgs {
+        subcommand: SubCommandInteractive::Base(SubCommand::Quickstart),
+    };
+    args.run(rl)
 }

src/cmd/run_cmd.rs

@@ -1,10 +1,12 @@
 use crate::errors::*;
 
+use chrootable_https::dns::Resolver;
 use crate::args;
 use crate::blobs::{Blob, BlobStorage};
 use crate::cmd::Cmd;
 use crate::db::{ttl, Filter};
 use crate::engine::Module;
+use crate::ipc::common::StartCommand;
 use crate::models::*;
 use crate::shell::Shell;
 use crate::keyring::KeyRing;
@@ -113,16 +115,10 @@ fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> R
     Ok(())
 }
 
-pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
-
-    prepare_keyring(rl.keyring_mut(), &module, &params)?;
-
+fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, Option<String>, Vec<Blob>)>> {
     let filter = rl.scoped_targets();
 
-    let args = match module.source() {
+    match module.source() {
         Some(Source::Domains) => prepare_args::<Domain>(rl, &filter, None),
         Some(Source::Subdomains) => prepare_args::<Subdomain>(rl, &filter, None),
         Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl, &filter, None),
@@ -152,7 +148,44 @@ pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>)
             }
         },
         None => Ok(vec![(serde_json::Value::Null, None, vec![])]),
-    }?;
+    }
+}
+
+pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let keyring = rl.keyring().request_keys(&module);
+
+    let dns_config = Resolver::from_system_v4()?;
+    let proxy = rl.config().network.proxy.clone();
+
+    let args = get_args(rl, &module)?;
+    for (arg, _pretty_arg, blobs) in args {
+        let start_cmd = StartCommand::new(params.verbose,
+                                          keyring.clone(),
+                                          dns_config.clone(),
+                                          proxy.clone(),
+                                          options.clone(),
+                                          module.clone(),
+                                          arg,
+                                          blobs);
+        let out = serde_json::to_string(&start_cmd)?;
+        println!("{}", out);
+    }
+
+    Ok(())
+}
+
+pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
     let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);

src/cmd/select_cmd.rs

@@ -22,12 +22,17 @@ pub struct Args {
     /// Print paths to blobs
     #[structopt(long="paths", group="output")]
     paths: bool,
+    /// Count rows returned
+    #[structopt(short="c", long="count", group="output")]
+    count: bool,
 }
 
+#[derive(PartialEq)]
 enum Output {
     Normal,
     Json,
     Paths,
+    Count,
 }
 
 struct Printer<'a, 'b> {
@@ -41,6 +46,8 @@ impl<'a, 'b> Printer<'a, 'b> {
             Output::Json
         } else if args.paths {
             Output::Paths
+        } else if args.count {
+            Output::Count
         } else {
             Output::Normal
         };
@@ -52,25 +59,32 @@ impl<'a, 'b> Printer<'a, 'b> {
     }
 
     pub fn select<T: Model + Detailed + Serialize>(&self, filter: &Filter) -> Result<()> {
-        for obj in self.rl.db().filter::<T>(&filter.parse_optional()?)? {
-            match self.output {
-                Output::Normal => println!("{}", obj.detailed(self.rl.db())?),
-                Output::Json => {
-                    let v = serde_json::to_string(&obj)?;
-                    println!("{}", v);
-                },
-                Output::Paths => {
-                    let blob = obj.blob()
-                        .ok_or_else(|| format_err!("This model isn't linked to blob storage"))?;
+        let query = self.rl.db().filter::<T>(&filter.parse_optional()?)?;
 
-                    let path = self.rl.blobs()
-                        .join(blob)?;
+        if self.output == Output::Count {
+            println!("{}", query.len());
+        } else {
+            for obj in query {
+                match self.output {
+                    Output::Normal => println!("{}", obj.detailed(self.rl.db())?),
+                    Output::Json => {
+                        let v = serde_json::to_string(&obj)?;
+                        println!("{}", v);
+                    },
+                    Output::Paths => {
+                        let blob = obj.blob()
+                            .ok_or_else(|| format_err!("This model isn't linked to blob storage"))?;
 
-                    let path = path.to_str()
-                        .ok_or_else(|| format_err!("Path is invalid utf-8"))?;
+                        let path = self.rl.blobs()
+                            .join(blob)?;
 
-                    println!("{}", path);
-                },
+                        let path = path.to_str()
+                            .ok_or_else(|| format_err!("Path is invalid utf-8"))?;
+
+                        println!("{}", path);
+                    },
+                    Output::Count => unreachable!(),
+                }
             }
         }
 

src/cmd/use_cmd.rs

@@ -14,7 +14,7 @@ pub struct Args {
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
 
-    let module = rl.engine().get(&args.module)?.clone();
+    let module = rl.library().get(&args.module)?.clone();
     rl.set_module(module);
 
     Ok(())

src/cmd/workspace_cmd.rs

@@ -1,7 +1,8 @@
 use crate::errors::*;
 
 use crate::blobs::BlobStorage;
-use crate::cmd::Cmd;
+use crate::cmd::{Cmd, LiteCmd};
+use crate::config::Config;
 use crate::db::Database;
 use crate::shell::Shell;
 use crate::term;
@@ -52,6 +53,8 @@ fn usage(workspace: Option<Workspace>) -> Result<()> {
 }
 
 fn change(rl: &mut Shell, workspace: Workspace) -> Result<()> {
+    workspace.migrate()?;
+
     let blobs = BlobStorage::workspace(&workspace)?;
     let db = Database::establish(workspace)?;
     rl.set_blobstorage(blobs);
@@ -66,29 +69,42 @@ fn list() -> Result<()> {
     Ok(())
 }
 
-impl Cmd for Args {
-    fn run(self, rl: &mut Shell) -> Result<()> {
-        if self.delete {
-            if let Some(workspace) = self.workspace {
+fn run(args: Args, rl: Option<&mut Shell>) -> Result<()> {
+    if args.delete {
+        if let Some(workspace) = args.workspace {
+            if let Some(rl) = rl {
                 if *rl.db().workspace() == workspace {
                     bail!("Can't delete current workspace")
                 }
-
-                delete(workspace, self.force)
-            } else {
-                bail!("--delete requires workspace")
             }
-        } else if self.usage {
-            usage(self.workspace)
-        } else if let Some(workspace) = self.workspace {
+
+            delete(workspace, args.force)
+        } else {
+            bail!("--delete requires workspace")
+        }
+    } else if args.usage {
+        usage(args.workspace)
+    } else if let Some(workspace) = args.workspace {
+        if let Some(rl) = rl {
             change(rl, workspace)
         } else {
-            list()
+            Ok(())
         }
+    } else {
+        list()
     }
 }
 
-#[inline]
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    Args::run_str(rl, args)
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        run(self, Some(rl))
+    }
+}
+
+impl LiteCmd for Args {
+    #[inline]
+    fn run(self, _config: &Config) -> Result<()> {
+        run(self, None)
+    }
 }

src/db/mod.rs

@@ -121,6 +121,8 @@ impl Database {
             .context("Failed to enable write ahead log")?;
         db.execute("PRAGMA foreign_keys = ON")
             .context("Failed to enforce foreign keys")?;
+        db.execute("PRAGMA synchronous = NORMAL")
+            .context("Failed to enforce foreign keys")?;
 
         let autonoscope = RuleSet::load(&db)?;
 
@@ -208,6 +210,17 @@ impl Database {
         }
     }
 
+    pub fn insert_activity(&self, obj: NewActivity) -> Result<bool> {
+        if let Some(uniq) = &obj.uniq {
+            if Activity::uniq(self, uniq)?.is_some() {
+                // unique tag set and event already logged
+                return Ok(false);
+            }
+        }
+        obj.insert(&self)?;
+        Ok(true)
+    }
+
     pub fn insert_subdomain_ipaddr_struct(&self, subdomain_ipaddr: &NewSubdomainIpAddr) -> Result<Option<(DbChange, i32)>> {
         if let Some(subdomain_ipaddr_id) = SubdomainIpAddr::get_id_opt(self, &(subdomain_ipaddr.subdomain_id, subdomain_ipaddr.ip_addr_id))? {
             Ok(Some((DbChange::None, subdomain_ipaddr_id)))
@@ -291,7 +304,7 @@ impl Database {
         Ok(ipaddr_update.id)
     }
 
-    pub fn update_url(&self, url_update: &UrlUpdate) -> Result<i32> {
+    pub fn update_url(&self, url_update: &UrlChangeset) -> Result<i32> {
         use crate::schema::urls::columns::*;
         diesel::update(urls::table.filter(id.eq(url_update.id)))
             .set(url_update)

src/engine/ctx.rs

@@ -1,24 +1,29 @@
 use crate::errors::*;
 
-use crate::blobs::Blob;
 use crate::db::Family;
-use crate::engine::{Environment, Reporter};
+use crate::engine::{Environment, IpcChild};
 use crate::geoip::{MaxmindReader, GeoIP, AsnDB};
 use crate::hlua::{self, AnyLuaValue};
 use crate::keyring::KeyRingEntry;
-use crate::models::{Insert, Update};
+use crate::models::*;
 use crate::psl::{Psl, PslReader};
 use crate::lazy::Lazy;
 use crate::runtime;
 use crate::sockets::{Socket, SocketOptions, TlsData};
 use crate::web::{HttpSession, HttpRequest, RequestOptions};
-use crate::worker::{Event, LogEvent, DatabaseEvent, StdioEvent};
+use crate::websockets::{WebSocket, WebSocketOptions};
+use crate::worker::{Event, LogEvent, DatabaseEvent, DatabaseResponse, StdioEvent, RatelimitEvent};
+use crate::ratelimits::RatelimitResponse;
 use chrootable_https::{self, Resolver};
 use serde_json;
+use sn0int_std::blobs::{Blob, BlobState};
+use sn0int_std::mqtt::{MqttClient, MqttOptions};
+use sn0int_std::web::WebState;
 use std::collections::HashMap;
 use std::result;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
+use std::thread;
 use rand::prelude::*;
 use rand::distributions::Alphanumeric;
 
@@ -68,36 +73,52 @@ pub trait State {
         self.send(&Event::Log(LogEvent::Status(msg)))
     }
 
-    fn db_insert(&self, object: Insert) -> Result<Option<i32>> {
+    fn db_recv(&self) -> Result<DatabaseResponse> {
+        let reply = self.recv()?;
+        let reply: result::Result<DatabaseResponse, String> = serde_json::from_value(reply)?;
+        reply.map_err(|err| format_err!("Database error: {:?}", err))
+    }
+
+    fn db_insert(&self, object: Insert) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Insert(object)));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        self.db_recv()
+            .context("Failed to add to database")
+            .map_err(Error::from)
     }
 
-    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<Option<i32>> {
+    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::InsertTtl((object, ttl))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        self.db_recv()
+            .context("Failed to add to database")
+            .map_err(Error::from)
     }
 
-    fn db_select(&self, family: Family, value: String) -> Result<Option<i32>> {
+    fn db_activity(&self, activity: InsertActivity) -> Result<bool> {
+        let activity = activity.try_into_new()?;
+
+        self.send(&Event::Database(DatabaseEvent::Activity(activity)));
+        let r = self.db_recv()
+            .context("Failed to log activity")?;
+
+        match r {
+            DatabaseResponse::Inserted(_) => Ok(false),
+            DatabaseResponse::NoChange(_) => Ok(true),
+            _ => bail!("Unexpected database response for db_activity: {:?}", r),
+        }
+    }
+
+    fn db_select(&self, family: Family, value: String) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Select((family, value))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to query database: {:?}", err))
+        self.db_recv()
+            .context("Failed to query database")
+            .map_err(Error::from)
     }
 
-    fn db_update(&self, object: String, update: Update) -> Result<Option<i32>> {
+    fn db_update(&self, object: String, update: Update) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Update((object, update))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to update database: {:?}", err))
+        self.db_recv()
+            .context("Failed to update database")
+            .map_err(Error::from)
     }
 
     fn stdin_read_line(&self) -> Result<Option<String>> {
@@ -114,6 +135,21 @@ pub trait State {
         reply.map_err(|err| format_err!("Failed to read stdin: {:?}", err))
     }
 
+    fn ratelimit(&self, key: String, passes: u32, time: u32) -> Result<()> {
+        let ratelimit = Event::Ratelimit(RatelimitEvent::new(key, passes, time));
+        loop {
+            self.send(&ratelimit);
+            let reply = self.recv()?;
+            let reply: result::Result<RatelimitResponse, String> = serde_json::from_value(reply)?;
+            match reply {
+                Ok(RatelimitResponse::Retry(delay)) => thread::sleep(delay),
+                Ok(RatelimitResponse::Pass) => break,
+                Err(err) => bail!("Unexpected error case for ratelimit: {}", err),
+            }
+        }
+        Ok(())
+    }
+
     #[inline]
     fn random_id(&self) -> String {
         thread_rng().sample_iter(&Alphanumeric).take(16).collect()
@@ -139,16 +175,18 @@ pub trait State {
 
     fn sock_upgrade_tls(&self, id: &str, options: &SocketOptions) -> Result<TlsData>;
 
-    fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>>;
+    fn ws_connect(&self, url: url::Url, options: &WebSocketOptions) -> Result<String>;
+
+    fn get_ws(&self, id: &str)-> Arc<Mutex<WebSocket>>;
+
+    fn mqtt_connect(&self, url: url::Url, options: &MqttOptions) -> Result<String>;
+
+    fn get_mqtt(&self, id: &str)-> Arc<Mutex<MqttClient>>;
 
     fn http_mksession(&self) -> String;
 
     fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest;
 
-    fn register_in_jar(&self, session: &str, key: String, value: String);
-
-    fn register_blob(&self, blob: Blob) -> String;
-
     fn get_blob(&self, id: &str) -> Result<Arc<Blob>>;
 
     fn persist_blob(&self, id: &str) -> Result<()> {
@@ -163,8 +201,10 @@ pub trait State {
 // #[derive(Debug)]
 pub struct LuaState {
     error: Mutex<Option<Error>>,
-    logger: Arc<Mutex<Box<dyn Reporter>>>,
+    logger: Arc<Mutex<Box<dyn IpcChild>>>,
     socket_sessions: Mutex<HashMap<String, Arc<Mutex<Socket>>>>,
+    ws_sessions: Mutex<HashMap<String, Arc<Mutex<WebSocket>>>>,
+    mqtt_sessions: Mutex<HashMap<String, Arc<Mutex<MqttClient>>>>,
     blobs: Mutex<HashMap<String, Arc<Blob>>>,
     http_sessions: Mutex<HashMap<String, HttpSession>>,
     http_clients: Mutex<HashMap<String, Arc<chrootable_https::Client<Resolver>>>>,
@@ -173,8 +213,8 @@ pub struct LuaState {
     keyring: Vec<KeyRingEntry>, // TODO: maybe hashmap
     dns_config: Resolver,
     psl: Mutex<Lazy<PslReader, Arc<Psl>>>,
-    geoip: Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>,
-    asn: Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>,
+    geoip: Option<Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>>,
+    asn: Option<Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>>,
     proxy: Option<SocketAddr>,
     options: HashMap<String, String>,
 }
@@ -236,15 +276,23 @@ impl State for LuaState {
     }
 
     fn geoip(&self) -> Result<Arc<GeoIP>> {
-        let mut geoip = self.geoip.lock().unwrap();
-        let geoip = geoip.get()?;
-        Ok(geoip.clone())
+        if let Some(mtx) = &self.geoip {
+            let mut geoip = mtx.lock().unwrap();
+            let geoip = geoip.get()?;
+            Ok(geoip.clone())
+        } else {
+            bail!("No geoip database loaded")
+        }
     }
 
     fn asn(&self) -> Result<Arc<AsnDB>> {
-        let mut asn = self.asn.lock().unwrap();
-        let asn = asn.get()?;
-        Ok(asn.clone())
+        if let Some(mtx) = &self.asn {
+            let mut asn = mtx.lock().unwrap();
+            let asn = asn.get()?;
+            Ok(asn.clone())
+        } else {
+            bail!("No asn database loaded")
+        }
     }
 
     fn sock_connect(&self, host: &str, port: u16, options: &SocketOptions) -> Result<String> {
@@ -264,13 +312,13 @@ impl State for LuaState {
 
     fn get_sock(&self, id: &str)-> Arc<Mutex<Socket>> {
         let mtx = self.socket_sessions.lock().unwrap();
-        let sock = mtx.get(id).expect("invalid session reference"); // TODO
+        let sock = mtx.get(id).expect("Invalid socket reference"); // TODO
         sock.clone()
     }
 
     fn sock_upgrade_tls(&self, id: &str, options: &SocketOptions) -> Result<TlsData> {
         let mut mtx = self.socket_sessions.lock().unwrap();
-        let sock = mtx.remove(id).expect("invalid session reference"); // TODO
+        let sock = mtx.remove(id).expect("Invalid socket reference"); // TODO
 
         let sock = Arc::try_unwrap(sock).unwrap();
         let sock = sock.into_inner().unwrap();
@@ -282,6 +330,61 @@ impl State for LuaState {
         Ok(tls)
     }
 
+    fn ws_connect(&self, url: url::Url, options: &WebSocketOptions) -> Result<String> {
+        let mut mtx = self.ws_sessions.lock().unwrap();
+        let id = self.random_id();
+
+        let sock = WebSocket::connect(&self.dns_config, url, options)?;
+        mtx.insert(id.clone(), Arc::new(Mutex::new(sock)));
+
+        Ok(id)
+    }
+
+    fn get_ws(&self, id: &str)-> Arc<Mutex<WebSocket>> {
+        let mtx = self.ws_sessions.lock().unwrap();
+        let sock = mtx.get(id).expect("Invalid ws reference"); // TODO
+        sock.clone()
+    }
+
+    fn mqtt_connect(&self, url: url::Url, options: &MqttOptions) -> Result<String> {
+        let mut mtx = self.mqtt_sessions.lock().unwrap();
+        let id = self.random_id();
+
+        let sock = MqttClient::connect(&self.dns_config, url, options)?;
+        mtx.insert(id.clone(), Arc::new(Mutex::new(sock)));
+
+        Ok(id)
+    }
+
+    fn get_mqtt(&self, id: &str)-> Arc<Mutex<MqttClient>> {
+        let mtx = self.mqtt_sessions.lock().unwrap();
+        let sock = mtx.get(id).expect("Invalid mqtt reference"); // TODO
+        sock.clone()
+    }
+
+    fn http_mksession(&self) -> String {
+        let mut mtx = self.http_sessions.lock().unwrap();
+        let (id, session) = HttpSession::new();
+        mtx.insert(id.clone(), session);
+        id
+    }
+
+    fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest {
+        let mtx = self.http_sessions.lock().unwrap();
+        let session = mtx.get(session_id).expect("Invalid session reference"); // TODO
+
+        HttpRequest::new(&session, method, url, options)
+    }
+
+    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
+        let mtx = self.blobs.lock().unwrap();
+        let blob = mtx.get(id)
+            .ok_or_else(|| format_err!("Invalid blob reference"))?;
+        Ok(blob.clone())
+    }
+}
+
+impl WebState for LuaState {
     fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>> {
         let proxy = self.resolve_proxy_options(proxy)?;
 
@@ -308,27 +411,15 @@ impl State for LuaState {
         }
     }
 
-    fn http_mksession(&self) -> String {
-        let mut mtx = self.http_sessions.lock().unwrap();
-        let (id, session) = HttpSession::new();
-        mtx.insert(id.clone(), session);
-        id
-    }
-
-    fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest {
-        let mtx = self.http_sessions.lock().unwrap();
-        let session = mtx.get(session_id).expect("invalid session reference"); // TODO
-
-        HttpRequest::new(&session, method, url, options)
-    }
-
     fn register_in_jar(&self, session: &str, key: String, value: String) {
         let mut mtx = self.http_sessions.lock().unwrap();
         if let Some(session) = mtx.get_mut(session) {
             session.cookies.register_in_jar(key, value);
         }
     }
+}
 
+impl BlobState for LuaState {
     fn register_blob(&self, blob: Blob) -> String {
         let id = blob.id.clone();
 
@@ -338,13 +429,6 @@ impl State for LuaState {
 
         id
     }
-
-    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
-        let mtx = self.blobs.lock().unwrap();
-        let blob = mtx.get(id)
-            .ok_or_else(|| format_err!("Invalid blob reference"))?;
-        Ok(blob.clone())
-    }
 }
 
 impl LuaState {
@@ -364,15 +448,20 @@ pub struct Script {
     code: String,
 }
 
-pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
+pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
     debug!("Creating lua context");
     let mut lua = hlua::Lua::new();
     lua.open_string();
 
+    let geoip = env.geoip.map(|db| Mutex::new(Lazy::from(db)));
+    let asn = env.asn.map(|db| Mutex::new(Lazy::from(db)));
+
     let state = Arc::new(LuaState {
         error: Mutex::new(None),
         logger,
         socket_sessions: Mutex::new(HashMap::new()),
+        ws_sessions: Mutex::new(HashMap::new()),
+        mqtt_sessions: Mutex::new(HashMap::new()),
         blobs: Mutex::new(HashMap::new()),
         http_sessions: Mutex::new(HashMap::new()),
         http_clients: Mutex::new(HashMap::new()),
@@ -381,8 +470,8 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
         keyring: env.keyring,
         dns_config: env.dns_config,
         psl: Mutex::new(Lazy::from(env.psl)),
-        geoip: Mutex::new(Lazy::from(env.geoip)),
-        asn: Mutex::new(Lazy::from(env.asn)),
+        geoip,
+        asn,
         proxy: env.proxy,
         options: env.options,
     });
@@ -404,11 +493,13 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::datetime(&mut lua, state.clone());
     runtime::db_add(&mut lua, state.clone());
     runtime::db_add_ttl(&mut lua, state.clone());
+    runtime::db_activity(&mut lua, state.clone());
     runtime::db_select(&mut lua, state.clone());
     runtime::db_update(&mut lua, state.clone());
     runtime::debug(&mut lua, state.clone());
     runtime::dns(&mut lua, state.clone());
     runtime::error(&mut lua, state.clone());
+    runtime::geo_polygon_contains(&mut lua, state.clone());
     runtime::geoip_lookup(&mut lua, state.clone());
     runtime::getopt(&mut lua, state.clone());
     runtime::hex(&mut lua, state.clone());
@@ -433,13 +524,19 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::json_decode(&mut lua, state.clone());
     runtime::json_decode_stream(&mut lua, state.clone());
     runtime::json_encode(&mut lua, state.clone());
+    runtime::key_trunc_pad(&mut lua, state.clone());
     runtime::keyring(&mut lua, state.clone());
     runtime::last_err(&mut lua, state.clone());
     runtime::md5(&mut lua, state.clone());
+    runtime::mqtt_connect(&mut lua, state.clone());
+    runtime::mqtt_subscribe(&mut lua, state.clone());
+    runtime::mqtt_recv(&mut lua, state.clone());
+    runtime::mqtt_ping(&mut lua, state.clone());
     runtime::pgp_pubkey(&mut lua, state.clone());
     runtime::pgp_pubkey_armored(&mut lua, state.clone());
     runtime::print(&mut lua, state.clone());
     runtime::psl_domain_from_dns_name(&mut lua, state.clone());
+    runtime::ratelimit_throttle(&mut lua, state.clone());
     runtime::regex_find(&mut lua, state.clone());
     runtime::regex_find_all(&mut lua, state.clone());
     runtime::semver_match(&mut lua, state.clone());
@@ -455,6 +552,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::sn0int_version(&mut lua, state.clone());
     runtime::sock_connect(&mut lua, state.clone());
     runtime::sock_upgrade_tls(&mut lua, state.clone());
+    runtime::sock_options(&mut lua, state.clone());
     runtime::sock_send(&mut lua, state.clone());
     runtime::sock_recv(&mut lua, state.clone());
     runtime::sock_sendline(&mut lua, state.clone());
@@ -466,9 +564,13 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::sock_recvuntil(&mut lua, state.clone());
     runtime::sock_sendafter(&mut lua, state.clone());
     runtime::sock_newline(&mut lua, state.clone());
+    runtime::sodium_secretbox_open(&mut lua, state.clone());
     runtime::status(&mut lua, state.clone());
     runtime::stdin_read_line(&mut lua, state.clone());
     runtime::stdin_read_to_end(&mut lua, state.clone());
+    runtime::str_find(&mut lua, state.clone());
+    runtime::str_replace(&mut lua, state.clone());
+    runtime::strval(&mut lua, state.clone());
     runtime::strftime(&mut lua, state.clone());
     runtime::strptime(&mut lua, state.clone());
     runtime::time_unix(&mut lua, state.clone());
@@ -481,6 +583,14 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::utf8_decode(&mut lua, state.clone());
     runtime::warn(&mut lua, state.clone());
     runtime::warn_once(&mut lua, state.clone());
+    runtime::ws_connect(&mut lua, state.clone());
+    runtime::ws_options(&mut lua, state.clone());
+    runtime::ws_recv_text(&mut lua, state.clone());
+    runtime::ws_recv_binary(&mut lua, state.clone());
+    runtime::ws_recv_json(&mut lua, state.clone());
+    runtime::ws_send_text(&mut lua, state.clone());
+    runtime::ws_send_binary(&mut lua, state.clone());
+    runtime::ws_send_json(&mut lua, state.clone());
     runtime::x509_parse_pem(&mut lua, state.clone());
     runtime::xml_decode(&mut lua, state.clone());
     runtime::xml_named(&mut lua, state.clone());
@@ -511,7 +621,7 @@ impl Script {
     }
 
     pub fn run(&self, env: Environment,
-                      tx: Arc<Mutex<Box<dyn Reporter>>>,
+                      tx: Arc<Mutex<Box<dyn IpcChild>>>,
                       arg: AnyLuaValue,
     ) -> Result<()> {
         let (mut lua, state) = ctx(env, tx);
@@ -542,10 +652,11 @@ impl Script {
 
     #[cfg(test)]
     pub fn test(&self) -> Result<()> {
-        use crate::engine::DummyReporter;
+        use crate::ipc::child::DummyIpcChild;
         use crate::geoip::Maxmind;
+        use crate::paths;
         let keyring = Vec::new();
-        let dns_config = Resolver::from_system()?;
+        let dns_config = Resolver::from_system_v4()?;
         let proxy = None;
         let psl = PslReader::String(r#"
 // ===BEGIN ICANN DOMAINS===
@@ -555,8 +666,9 @@ com
 a.prod.fastly.net
 // ===END PRIVATE DOMAINS===
 "#.into());
-        let geoip = GeoIP::open_reader()?;
-        let asn = AsnDB::open_reader()?;
+        let cache_dir = paths::cache_dir()?;
+        let geoip = GeoIP::try_open_reader(&cache_dir)?;
+        let asn = AsnDB::try_open_reader(&cache_dir)?;
 
         let env = Environment {
             verbose: 0,
@@ -569,6 +681,6 @@ a.prod.fastly.net
             geoip,
             asn,
         };
-        self.run(env, DummyReporter::new(), AnyLuaValue::LuaNil)
+        self.run(env, DummyIpcChild::new(), AnyLuaValue::LuaNil)
     }
 }

src/engine/isolation.rs

@@ -1,254 +0,0 @@
-use crate::errors::*;
-use chrootable_https::dns::Resolver;
-use crate::blobs::Blob;
-use crate::engine::{Environment, Module, Reporter};
-use crate::geoip::MaxmindReader;
-use crate::keyring::KeyRingEntry;
-use crate::psl::PslReader;
-use serde_json;
-use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
-
-use std::collections::HashMap;
-use std::env;
-use std::ffi::OsString;
-use std::io::prelude::*;
-use std::io::{self, BufReader, BufRead, stdin, Stdin, Stdout};
-use std::net::SocketAddr;
-use std::sync::{mpsc, Arc, Mutex};
-use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
-
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct StartCommand {
-    verbose: u64,
-    keyring: Vec<KeyRingEntry>,
-    dns_config: Resolver,
-    proxy: Option<SocketAddr>,
-    options: HashMap<String, String>,
-    module: Module,
-    arg: serde_json::Value,
-    blobs: Vec<Blob>,
-}
-
-impl StartCommand {
-    pub fn new(verbose: u64,
-               keyring: Vec<KeyRingEntry>,
-               dns_config: Resolver,
-               proxy: Option<SocketAddr>,
-               options: HashMap<String, String>,
-               module: Module,
-               arg: serde_json::Value,
-               blobs: Vec<Blob>,
-    ) -> StartCommand {
-        StartCommand {
-            verbose,
-            keyring,
-            dns_config,
-            proxy,
-            options,
-            module,
-            arg,
-            blobs,
-        }
-    }
-}
-
-pub struct Supervisor {
-    child: Child,
-    stdin: ChildStdin,
-    stdout: BufReader<ChildStdout>,
-}
-
-impl Supervisor {
-    pub fn setup(module: &Module) -> Result<Supervisor> {
-        let exe = match env::current_exe() {
-            Ok(exe) => exe.into_os_string(),
-            _ => OsString::from("sn0int"),
-        };
-
-        let mut child = Command::new(exe)
-            .arg("sandbox")
-            .arg(&module.canonical())
-            .stdin(Stdio::piped())
-            .stdout(Stdio::piped())
-            .spawn()
-            .context("Failed to spawn child process")?;
-
-        let stdin = child.stdin.take().expect("Failed to take child stdin");
-        let stdout = child.stdout.take().expect("Failed to take child stdout");
-        let stdout = BufReader::new(stdout);
-
-        Ok(Supervisor {
-            child,
-            stdin,
-            stdout,
-        })
-    }
-
-    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
-        let start = serde_json::to_value(&start)?;
-        self.send(&start)?;
-        Ok(())
-    }
-
-    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
-        let mut value = serde_json::to_string(value)?;
-        value.push('\n');
-        self.stdin.write_all(value.as_bytes())?;
-        debug!("Supervisor sent: {:?}", value);
-        Ok(())
-    }
-
-    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
-        let value = serde_json::to_value(value).expect("Failed to serialize reply");
-        if let Err(_) = self.send(&value) {
-            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
-        }
-    }
-
-    pub fn recv(&mut self) -> Result<Event> {
-        let mut line = String::new();
-        let len = self.stdout.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Supervisor received: {:?}", event);
-        Ok(event)
-    }
-
-    pub fn wait(&mut self) -> Result<()> {
-        let exit = self.child.wait()
-            .context("Failed to wait for child")?;
-
-        if exit.success() {
-            Ok(())
-        } else {
-            bail!("Child signaled error")
-        }
-    }
-
-    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
-        where <T as EventWithCallback>::Payload: serde::Serialize
-    {
-        let (tx2, rx2) = mpsc::channel();
-        tx.send(event.with_callback(tx2));
-        let reply = rx2.recv().unwrap();
-
-        self.send_struct(reply, tx);
-    }
-}
-
-#[derive(Debug)]
-pub struct StdioReporter {
-    stdin: Stdin,
-    stdout: Stdout,
-}
-
-impl StdioReporter {
-    pub fn setup() -> StdioReporter {
-        let stdin = io::stdin();
-        let stdout = io::stdout();
-
-        StdioReporter {
-            stdin,
-            stdout,
-        }
-    }
-
-    pub fn recv_start(&mut self) -> Result<StartCommand> {
-        let value = self.recv()?;
-        let event = serde_json::from_value(value)?;
-        Ok(event)
-    }
-}
-
-impl Reporter for StdioReporter {
-    fn send(&mut self, event: &Event) -> Result<()> {
-        let mut event = serde_json::to_string(&event)?;
-        event.push('\n');
-        self.stdout.write_all(event.as_bytes())?;
-        debug!("Reporter sent: {:?}", event);
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        let mut line = String::new();
-        let len = self.stdin.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Reporter received: {:?}", event);
-        Ok(event)
-    }
-}
-
-pub fn spawn_module(module: Module,
-                    tx: &EventSender,
-                    arg: serde_json::Value,
-                    keyring: Vec<KeyRingEntry>,
-                    verbose: u64,
-                    has_stdin: bool,
-                    proxy: Option<SocketAddr>,
-                    options: HashMap<String, String>,
-                    blobs: Vec<Blob>,
-) -> Result<ExitEvent> {
-    let dns_config = Resolver::from_system()?;
-
-    let mut reader = if has_stdin {
-        Some(BufReader::new(stdin()))
-    } else {
-        None
-    };
-
-    let mut supervisor = Supervisor::setup(&module)?;
-    supervisor.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
-
-    let exit = loop {
-        match supervisor.recv()? {
-            Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => supervisor.send_event_callback(object, &tx),
-            Event::Stdio(object) => object.apply(&mut supervisor, tx, &mut reader),
-            Event::Blob(blob) => supervisor.send_event_callback(blob, &tx),
-            Event::Exit(event) => {
-                if let ExitEvent::Err(err) = &event {
-                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
-                }
-                break event;
-            },
-        }
-    };
-
-    supervisor.wait()?;
-
-    Ok(exit)
-}
-
-pub fn run_worker(geoip: MaxmindReader, asn: MaxmindReader, psl: PslReader) -> Result<()> {
-    let mut reporter = StdioReporter::setup();
-    let start = reporter.recv_start()?;
-
-    let environment = Environment {
-        verbose: start.verbose,
-        keyring: start.keyring,
-        dns_config: start.dns_config,
-        proxy: start.proxy,
-        options: start.options,
-        blobs: start.blobs,
-        psl,
-        geoip,
-        asn,
-    };
-
-    let mtx: Arc<Mutex<Box<dyn Reporter>>> = Arc::new(Mutex::new(Box::new(reporter)));
-    let result = start.module.run(environment,
-                                  mtx.clone(),
-                                  start.arg.into());
-    let mut reporter = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
-                        .into_inner().expect("Failed to consume Mutex");
-
-    let event = match result {
-        Ok(_) => ExitEvent::Ok,
-        Err(err) => ExitEvent::Err(err.to_string()),
-    };
-    reporter.send(&Event::Exit(event))?;
-
-    Ok(())
-}

src/engine/mod.rs

@@ -5,7 +5,6 @@ use crate::config::Config;
 use crate::geoip::MaxmindReader;
 use crate::json::LuaJsonValue;
 use crate::keyring::KeyRingEntry;
-use serde_json;
 use std::fs;
 use std::fmt::Debug;
 use std::path::PathBuf;
@@ -13,6 +12,7 @@ use std::collections::HashMap;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
 use crate::engine::ctx::Script;
+use crate::ipc::child::IpcChild;
 use sn0int_common::ModuleID;
 use sn0int_common::metadata::{Metadata, Source};
 use chrootable_https::dns::Resolver;
@@ -21,11 +21,10 @@ use crate::paths;
 use std::cmp::Ordering;
 use std::path::Path;
 use crate::term;
-use crate::worker::{self, Event};
+use crate::worker;
 
 pub mod ctx;
-pub mod isolation;
-pub mod structs;
+pub use sn0int_std::engine::structs;
 
 
 /// Data that is passed to every script
@@ -38,34 +37,34 @@ pub struct Environment {
     pub options: HashMap<String, String>,
     pub blobs: Vec<Blob>,
     pub psl: PslReader,
-    pub geoip: MaxmindReader,
-    pub asn: MaxmindReader,
+    pub geoip: Option<MaxmindReader>,
+    pub asn: Option<MaxmindReader>,
 }
 
 #[derive(Debug)]
-pub struct Engine<'a> {
+pub struct Library<'a> {
     path: PathBuf,
     modules: HashMap<String, Vec<Module>>,
     config: &'a Config
 }
 
-impl<'a> Engine<'a> {
-    pub fn new(verbose_init: bool, config: &'a Config) -> Result<Engine> {
+impl<'a> Library<'a> {
+    pub fn new(verbose_init: bool, config: &'a Config) -> Result<Library> {
         let path = paths::module_dir()?;
 
-        let mut engine = Engine {
+        let mut library = Library {
             path,
             modules: HashMap::new(),
             config,
         };
 
         if verbose_init {
-            engine.reload_modules()?;
+            library.reload_modules()?;
         } else {
-            engine.reload_modules_quiet()?;
+            library.reload_modules_quiet()?;
         }
 
-        Ok(engine)
+        Ok(library)
     }
 
     pub fn reload_modules(&mut self) -> Result<usize> {
@@ -299,9 +298,9 @@ impl Module {
         self.private_module
     }
 
-    pub fn run(&self, env: Environment, reporter: Arc<Mutex<Box<dyn Reporter>>>, arg: LuaJsonValue) -> Result<()> {
+    pub fn run(&self, env: Environment, ipc_child: Arc<Mutex<Box<dyn IpcChild>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
-        self.script.run(env, reporter, arg.into())
+        self.script.run(env, ipc_child, arg.into())
     }
 
     #[inline]
@@ -320,28 +319,3 @@ impl Module {
         }
     }
 }
-
-pub trait Reporter: Debug {
-    fn send(&mut self, event: &Event) -> Result<()>;
-
-    fn recv(&mut self) -> Result<serde_json::Value>;
-}
-
-#[derive(Debug)]
-pub struct DummyReporter;
-
-impl DummyReporter {
-    pub fn new() -> Arc<Mutex<Box<dyn Reporter>>> {
-        Arc::new(Mutex::new(Box::new(DummyReporter)))
-    }
-}
-
-impl Reporter for DummyReporter {
-    fn send(&mut self, _event: &Event) -> Result<()> {
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        unimplemented!("DummyReporter::recv doesn't exist")
-    }
-}

src/errors.rs

@@ -1,2 +1,3 @@
-pub use failure::{Error, ResultExt};
+pub use log::{trace, debug, info, warn, error};
+pub use failure::{Error, ResultExt, format_err, bail};
 pub type Result<T> = ::std::result::Result<T, Error>;

src/fmt.rs

@@ -141,8 +141,7 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
     }
 
     #[inline]
-    pub fn display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
-        self.push_into_group()?;
+    pub fn direct_display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
         if self.scoped {
             C::display(self, v)
         } else {
@@ -151,8 +150,19 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
     }
 
     #[inline]
-    pub fn debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
+    pub fn display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
         self.push_into_group()?;
+        self.direct_display::<C, _>(v)
+    }
+
+    pub fn display_label<C: Color, D: fmt::Display>(&mut self, label: &str, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        write!(self, "{}=", label)?;
+        self.direct_display::<C, _>(v)
+    }
+
+    #[inline]
+    pub fn direct_debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
         if self.scoped {
             C::debug(self, v)
         } else {
@@ -160,6 +170,12 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
         }
     }
 
+    #[inline]
+    pub fn debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        self.direct_debug::<C, _>(v)
+    }
+
     pub fn opt_debug<C: Color, D: fmt::Debug>(&mut self, v: &Option<D>) -> fmt::Result {
         if let Some(v) = &v {
             self.debug::<C, _>(v)?;
@@ -167,6 +183,15 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
         Ok(())
     }
 
+    pub fn opt_debug_label<C: Color, D: fmt::Debug>(&mut self, label: &str, v: &Option<D>) -> fmt::Result {
+        if let Some(v) = &v {
+            self.push_into_group()?;
+            write!(self, "{}=", label)?;
+            self.direct_debug::<C, _>(v)?;
+        }
+        Ok(())
+    }
+
     #[inline]
     pub fn clear(&mut self) -> fmt::Result {
         if self.scoped {

src/ipc/child.rs

@@ -0,0 +1,110 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use crate::engine::Environment;
+use crate::geoip::MaxmindReader;
+use crate::psl::PslReader;
+use serde_json;
+use crate::worker::Event;
+
+use std::fmt::Debug;
+use std::io::prelude::*;
+use std::io::{self, Stdin, Stdout};
+use std::sync::{Arc, Mutex};
+
+
+pub trait IpcChild: Debug {
+    fn send(&mut self, event: &Event) -> Result<()>;
+
+    fn recv(&mut self) -> Result<serde_json::Value>;
+}
+
+#[derive(Debug)]
+pub struct StdioIpcChild {
+    stdin: Stdin,
+    stdout: Stdout,
+}
+
+impl StdioIpcChild {
+    pub fn setup() -> StdioIpcChild {
+        let stdin = io::stdin();
+        let stdout = io::stdout();
+
+        StdioIpcChild {
+            stdin,
+            stdout,
+        }
+    }
+
+    pub fn recv_start(&mut self) -> Result<StartCommand> {
+        let value = self.recv()?;
+        let event = serde_json::from_value(value)?;
+        Ok(event)
+    }
+}
+
+impl IpcChild for StdioIpcChild {
+    fn send(&mut self, event: &Event) -> Result<()> {
+        let mut event = serde_json::to_string(&event)?;
+        event.push('\n');
+        debug!("IpcChild sends: {:?}", event);
+        self.stdout.write_all(event.as_bytes())?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        let mut line = String::new();
+        let len = self.stdin.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcChild received: {:?}", event);
+        Ok(event)
+    }
+}
+
+#[derive(Debug)]
+pub struct DummyIpcChild;
+
+impl DummyIpcChild {
+    pub fn new() -> Arc<Mutex<Box<dyn IpcChild>>> {
+        Arc::new(Mutex::new(Box::new(DummyIpcChild)))
+    }
+}
+
+impl IpcChild for DummyIpcChild {
+    fn send(&mut self, _event: &Event) -> Result<()> {
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        unimplemented!("DummyIpcChild::recv doesn't exist")
+    }
+}
+
+pub fn run(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslReader) -> Result<()> {
+    let mut ipc_child = StdioIpcChild::setup();
+    let start = ipc_child.recv_start()?;
+
+    let environment = Environment {
+        verbose: start.verbose,
+        keyring: start.keyring,
+        dns_config: start.dns_config,
+        proxy: start.proxy,
+        options: start.options,
+        blobs: start.blobs,
+        psl,
+        geoip,
+        asn,
+    };
+
+    let mtx: Arc<Mutex<Box<dyn IpcChild>>> = Arc::new(Mutex::new(Box::new(ipc_child)));
+    let result = start.module.run(environment,
+                                  mtx.clone(),
+                                  start.arg.into());
+    let mut ipc_child = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
+                        .into_inner().expect("Failed to consume Mutex");
+
+    let event = result.into();
+    ipc_child.send(&Event::Exit(event))?;
+
+    Ok(())
+}

src/ipc/common.rs

@@ -0,0 +1,44 @@
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+
+use std::collections::HashMap;
+use std::net::SocketAddr;
+
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct StartCommand {
+    pub verbose: u64,
+    pub keyring: Vec<KeyRingEntry>,
+    pub dns_config: Resolver,
+    pub proxy: Option<SocketAddr>,
+    pub options: HashMap<String, String>,
+    pub module: Module,
+    pub arg: serde_json::Value,
+    pub blobs: Vec<Blob>,
+}
+
+impl StartCommand {
+    pub fn new(verbose: u64,
+               keyring: Vec<KeyRingEntry>,
+               dns_config: Resolver,
+               proxy: Option<SocketAddr>,
+               options: HashMap<String, String>,
+               module: Module,
+               arg: serde_json::Value,
+               blobs: Vec<Blob>,
+    ) -> StartCommand {
+        StartCommand {
+            verbose,
+            keyring,
+            dns_config,
+            proxy,
+            options,
+            module,
+            arg,
+            blobs,
+        }
+    }
+}

src/ipc/mod.rs

@@ -0,0 +1,3 @@
+pub mod child;
+pub mod common;
+pub mod parent;

src/ipc/parent.rs

@@ -0,0 +1,144 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
+
+use std::collections::HashMap;
+use std::env;
+use std::ffi::OsString;
+use std::io::prelude::*;
+use std::io::{BufReader, BufRead, stdin};
+use std::net::SocketAddr;
+use std::sync::mpsc;
+use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
+
+
+pub struct IpcParent {
+    child: Child,
+    stdin: ChildStdin,
+    stdout: BufReader<ChildStdout>,
+}
+
+impl IpcParent {
+    pub fn setup(module: &Module) -> Result<IpcParent> {
+        let exe = match env::current_exe() {
+            Ok(exe) => exe.into_os_string(),
+            _ => OsString::from("sn0int"),
+        };
+
+        let mut child = Command::new(exe)
+            .arg("sandbox")
+            .arg(&module.canonical())
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .context("Failed to spawn child process")?;
+
+        let stdin = child.stdin.take().expect("Failed to take child stdin");
+        let stdout = child.stdout.take().expect("Failed to take child stdout");
+        let stdout = BufReader::new(stdout);
+
+        Ok(IpcParent {
+            child,
+            stdin,
+            stdout,
+        })
+    }
+
+    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
+        let start = serde_json::to_value(&start)?;
+        self.send(&start)?;
+        Ok(())
+    }
+
+    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
+        let mut value = serde_json::to_string(value)?;
+        value.push('\n');
+        self.stdin.write_all(value.as_bytes())?;
+        debug!("IpcParent sent: {:?}", value);
+        Ok(())
+    }
+
+    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
+        let value = serde_json::to_value(value).expect("Failed to serialize reply");
+        if let Err(_) = self.send(&value) {
+            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
+        }
+    }
+
+    pub fn recv(&mut self) -> Result<Event> {
+        let mut line = String::new();
+        let len = self.stdout.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcParent received: {:?}", event);
+        Ok(event)
+    }
+
+    pub fn wait(&mut self) -> Result<()> {
+        let exit = self.child.wait()
+            .context("Failed to wait for child")?;
+
+        if exit.success() {
+            Ok(())
+        } else {
+            bail!("Child signaled error")
+        }
+    }
+
+    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
+        where <T as EventWithCallback>::Payload: serde::Serialize
+    {
+        let (tx2, rx2) = mpsc::channel();
+        tx.send(event.with_callback(tx2));
+        let reply = rx2.recv().unwrap();
+
+        self.send_struct(reply, tx);
+    }
+}
+
+pub fn run(module: Module,
+           tx: &EventSender,
+           arg: serde_json::Value,
+           keyring: Vec<KeyRingEntry>,
+           verbose: u64,
+           has_stdin: bool,
+           proxy: Option<SocketAddr>,
+           options: HashMap<String, String>,
+           blobs: Vec<Blob>,
+) -> Result<ExitEvent> {
+    let dns_config = Resolver::from_system_v4()?;
+
+    let mut reader = if has_stdin {
+        Some(BufReader::new(stdin()))
+    } else {
+        None
+    };
+
+    let mut ipc_parent = IpcParent::setup(&module)?;
+    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
+
+    let exit = loop {
+        match ipc_parent.recv()? {
+            Event::Log(event) => tx.send(Event2::Log(event)),
+            Event::Database(object) => ipc_parent.send_event_callback(object, &tx),
+            Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
+            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
+            Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),
+            Event::Exit(event) => {
+                if let ExitEvent::Err(err) = &event {
+                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
+                }
+                break event;
+            },
+        }
+    };
+
+    ipc_parent.wait()?;
+
+    Ok(exit)
+}

src/keyring.rs

@@ -66,7 +66,7 @@ impl fmt::Display for KeyName {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Default, Serialize, Deserialize)]
 pub struct KeyRing {
     keys: HashMap<String, HashMap<String, Option<String>>>,
     grants: HashMap<String, HashSet<ModuleID>>,
@@ -74,7 +74,7 @@ pub struct KeyRing {
 
 impl KeyRing {
     pub fn path() -> Result<PathBuf> {
-        let path = paths::data_dir()?;
+        let path = paths::sn0int_dir()?;
         let path = path.join("keyring.json");
         Ok(path)
     }
@@ -82,18 +82,19 @@ impl KeyRing {
     pub fn init() -> Result<KeyRing> {
         let path = Self::path()?;
 
-        if path.exists() {
+        let keyring = if path.exists() {
             Self::load(&path)
+                .context("Failed to load keyring")?
         } else {
-            Ok(KeyRing {
-                keys: HashMap::new(),
-                grants: HashMap::new(),
-            })
-        }
+            KeyRing::default()
+        };
+
+        Ok(keyring)
     }
 
     pub fn load(path: &Path) -> Result<KeyRing> {
-        let buf = fs::read(&path)?;
+        let buf = fs::read(&path)
+            .context("Failed to read keyring file")?;
         serde_json::from_slice(&buf)
             .map_err(Error::from)
     }
@@ -101,7 +102,8 @@ impl KeyRing {
     pub fn save(&self) -> Result<()> {
         let path = Self::path()?;
         let buf = serde_json::to_string(&self)?;
-        fs::write(&path, buf)?;
+        fs::write(&path, buf)
+            .context("Failed to save keyring")?;
         Ok(())
     }
 

src/lib.rs

@@ -1,10 +1,8 @@
 #![warn(unused_extern_crates)]
-#[macro_use] extern crate failure;
 #[macro_use] extern crate maplit;
 use url;
 use hlua_badtouch as hlua;
 #[macro_use] extern crate serde_derive;
-#[macro_use] extern crate log;
 #[macro_use] extern crate structopt;
 #[macro_use] extern crate lazy_static;
 #[macro_use] extern crate diesel;
@@ -12,30 +10,33 @@ use hlua_badtouch as hlua;
 #[macro_use] extern crate crossbeam_channel as channel;
 
 pub mod api;
-pub mod archive;
 pub mod args;
 pub mod auth;
 pub mod autonoscope;
 pub mod blobs;
 pub mod cmd;
 pub mod config;
-pub mod crt;
+use sn0int_std::crt;
 pub mod db;
 pub mod errors;
 pub mod engine;
 pub mod filters;
 pub mod fmt;
-pub mod geoip;
-pub mod gfx;
-pub mod html;
-pub mod json;
+use sn0int_std::geo;
+pub use sn0int_std::geoip;
+use sn0int_std::gfx;
+use sn0int_std::html;
+use sn0int_std::json;
+pub mod ipc;
 pub mod keyring;
-pub mod lazy;
+use sn0int_std::lazy;
 pub mod migrations;
 pub mod models;
+use sn0int_std::mqtt;
 pub mod paths;
-pub mod psl;
+pub use sn0int_std::psl;
 pub mod options;
+use sn0int_std::ratelimits;
 pub mod registry;
 pub mod repl;
 pub mod runtime;
@@ -43,16 +44,12 @@ pub mod sandbox;
 pub mod schema;
 pub mod ser;
 pub mod shell;
-pub mod sockets;
+use sn0int_std::sockets;
 pub mod term;
 pub mod update;
 pub mod utils;
-pub mod web;
+use sn0int_std::web;
+use sn0int_std::websockets;
 pub mod worker;
 pub mod workspaces;
-pub mod xml;
-
-#[cfg(test)]
-fn test_init() {
-    let _ = env_logger::builder().is_test(true).try_init();
-}
+use sn0int_std::xml;