Release v0.18.0

Properly support inserting urls with no body

.dockerignore

@@ -1,6 +1,7 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
 docker
 docs
 ci

.travis.yml

@@ -5,6 +5,14 @@ language: rust
 
 matrix:
   include:
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=docker
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=docker-registry
   - os: linux
     rust: stable
     env:
@@ -13,19 +21,14 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=common
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=std
   - os: linux
     rust: stable
     env:
     - BUILD_MODE=boxxy
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=docker
-  # https://github.com/rust-lang/rust/issues/68264
-  #- os: linux
-  #  rust: stable
-  #  env:
-  #  - BUILD_MODE=docker-registry
   - os: osx
     rust: stable
     env:
@@ -34,6 +37,10 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=common
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=std
   #- os: windows
   #  rust: stable
   #  env:

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.15.0"
+version = "0.18.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -13,15 +13,17 @@ edition = "2018"
 travis-ci = { repository = "kpcyrd/sn0int" }
 
 [workspace]
-members = ["sn0int-registry/sn0int-common",
-           "sn0int-registry"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
 
 [features]
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-registry/sn0int-common" }
-rustyline = "5.0"
+sn0int-common = { version="0.10.0", path="sn0int-common" }
+sn0int-std = { version="=0.18.0", path="sn0int-std" }
+rustyline = "6.0"
 log = "0.4"
 env_logger = "0.7"
 hlua-badtouch = "0.4"
@@ -31,7 +33,6 @@ rand = "0.7"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
 libsqlite3-sys = "0.16.0"
@@ -40,14 +41,9 @@ dirs = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.13"
-rustls = { version="0.16", features=["dangerous_configuration"] }
-webpki = "0.21"
-webpki-roots = "0.18"
-pem = "0.7"
+chrootable-https = "0.14"
 base64 = "0.11"
 data-encoding = "2.1.2"
-kuchiki = "0.7.2"
 serde_urlencoded = "0.6"
 serde = "1.0"
 serde_derive = "1.0"
@@ -60,43 +56,33 @@ maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.5"
 regex = "1.0"
 toml = "0.5"
-maxminddb = "0.13"
 threadpool = "1.7"
-x509-parser = "0.6.0"
-der-parser = "3.0"
 atty = "0.2"
-bufstream = "0.1.4"
-tokio = "0.1.14"
 semver = "0.9"
 bytes = "0.4"
-xml-rs = "0.8"
 bytesize = "1.0"
-ipnetwork = "0.15"
+ipnetwork = "0.16"
 strum = "0.17"
 strum_macros = "0.17"
 
 digest = "0.8.0"
-bs58 = "0.3"
-blake2 = "0.8.0"
 md-5 = "0.8.0"
 sha-1 = "0.8.1"
 sha2 = "0.8.0"
 sha3 = "0.8.0"
 hmac = "0.7"
 
-image = "0.22"
-kamadak-exif = "0.4"
 walkdir = "2.2"
-nude = "0.2"
+nude = "0.3"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
 #syscallz = { path="../syscallz-rs" }
 syscallz = "0.12"
-nix = "0.16"
+nix = "0.17"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
-pledge = "0.3.1"
+pledge = "0.4"
 unveil = "0.2.0"
 
 [dev-dependencies]

Dockerfile

@@ -1,14 +1,13 @@
-FROM rust:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

Makefile

@@ -1,19 +1,23 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
 
 update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt

README.md

@@ -89,10 +89,10 @@ For everything else please have a look at the [detailed list][1].
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-  - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope)
-    - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains)
-    - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips)
-    - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls)
+- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains)
+  - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls)
 - [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
   - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
@@ -100,6 +100,8 @@ For everything else please have a look at the [detailed list][1].
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
 - [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
@@ -154,6 +156,7 @@ For everything else please have a look at the [detailed list][1].
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity)
   - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
@@ -177,12 +180,18 @@ For everything else please have a look at the [detailed list][1].
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [key_trunc_pad](https://sn0int.readthedocs.io/en/latest/reference.html#key-trunc-pad)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
   - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [mqtt_connect](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-connect)
+  - [mqtt_subscribe](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-subscribe)
+  - [mqtt_recv](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-recv)
+  - [mqtt_ping](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-ping)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
@@ -203,6 +212,7 @@ For everything else please have a look at the [detailed list][1].
   - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
   - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_options](https://sn0int.readthedocs.io/en/latest/reference.html#sock-options)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
   - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
@@ -214,11 +224,15 @@ For everything else please have a look at the [detailed list][1].
   - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
   - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [sodium_secretbox_open](https://sn0int.readthedocs.io/en/latest/reference.html#sodium-secretbox-open)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
   - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
   - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
   - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval)
   - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
@@ -229,6 +243,14 @@ For everything else please have a look at the [detailed list][1].
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
   - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
   - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [ws_connect](https://sn0int.readthedocs.io/en/latest/reference.html#ws-connect)
+  - [ws_options](https://sn0int.readthedocs.io/en/latest/reference.html#ws-options)
+  - [ws_recv_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-text)
+  - [ws_recv_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-binary)
+  - [ws_recv_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-json)
+  - [ws_send_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-text)
+  - [ws_send_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-binary)
+  - [ws_send_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-json)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
   - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
   - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)

ci/integration.py

@@ -47,10 +47,10 @@ def main(tempdir, binary):
 
     print('[*] installing modules')
     sn0int(tempdir, binary, [
-        'mod install kpcyrd/ctlogs',
-        'mod install kpcyrd/dns-resolve',
-        'mod install kpcyrd/url-scan',
-        'mod install kpcyrd/geoip',
+        'pkg install kpcyrd/ctlogs',
+        'pkg install kpcyrd/dns-resolve',
+        'pkg install kpcyrd/url-scan',
+        'pkg install kpcyrd/geoip',
     ])
 
     print('[*] running ctlogs')
@@ -63,6 +63,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,6 +92,8 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }

ci/run.sh

@@ -6,14 +6,18 @@ case "$1" in
         cargo build --verbose --examples
         ;;
     test)
-        ci/run.sh build
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;
     common)
-        cd sn0int-registry/sn0int-common
+        cd sn0int-common
         cargo test --verbose
         ;;
+    std)
+        cd sn0int-std
+        cargo test --verbose
+        cargo test --verbose -- --ignored
+        ;;
     windows)
         cargo build --verbose --features=sqlite-bundled
         cargo build --verbose --examples --features=sqlite-bundled
@@ -31,7 +35,7 @@ case "$1" in
         docker run --rm sn0int --help
         ;;
     docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
+        docker build -t sn0int-registry -f sn0int-registry/Dockerfile .
         docker images
         ;;
 esac

ci/setup.sh

@@ -3,6 +3,9 @@ set -exu
 case "$1" in
     linux)
         sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev
+        sudo apt install libsqlite3-dev libseccomp-dev libsodium-dev
+        ;;
+    osx)
+        brew install libsodium
         ;;
 esac

contrib/docker/Dockerfile.alpine

@@ -1,13 +1,13 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

contrib/docker/Dockerfile.debian

@@ -1,13 +1,13 @@
-FROM rust
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]

docs/activity.rst

@@ -68,7 +68,7 @@ Logging an event that has a location attached could look like this:
         topic='harness/activity-ping:dummy',
         time=sn0int_time(),
         latitude=40.726662,
-        longitude=-74.036677
+        longitude=-74.036677,
         radius=50,
         content={
             a='b',

docs/autonoscope.rst

@@ -0,0 +1,81 @@
+Autonoscope
+===========
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+-------
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+---
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+----
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

docs/build.rst

@@ -18,19 +18,21 @@ Archlinux
 
 .. code-block:: bash
 
-    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+    $ pacman -S geoip2-database libseccomp libsodium publicsuffix-list sqlite
 
 Mac OSX
 ~~~~~~~
 
-None.
+.. code-block:: bash
+
+    $ brew install libsodium
 
 Debian/Ubuntu/Kali
 ~~~~~~~~~~~~~~~~~~
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
 
 .. warning::
    On a debian based system make sure you've installed rust with rustup.
@@ -40,21 +42,21 @@ Alpine
 
 .. code-block:: bash
 
-    $ apk add sqlite-dev libseccomp-dev
+    $ apk add sqlite-dev libseccomp-dev libsodium-dev
 
 OpenBSD
 ~~~~~~~
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3 geolite2-city geolite2-asn
+    $ pkg_add sqlite3 geolite2-city geolite2-asn libsodium
 
 Gentoo
 ~~~~~~
 
 .. code-block:: bash
 
-    emerge --ask sys-libs/libseccomp dev-db/sqlite
+    emerge --ask sys-libs/libseccomp dev-db/sqlite dev-libs/libsodium
 
 Windows
 ~~~~~~~

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/index.rst

@@ -38,6 +38,7 @@ Getting Started
    install
    build
    usage
+   autonoscope
    scripting
    database
    structs

docs/install.rst

@@ -32,7 +32,7 @@ at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .

docs/reference.rst

@@ -153,6 +153,28 @@ ttl.
         device_id=13,
     }, 120)
 
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_select
 ---------
 
@@ -369,6 +391,8 @@ options are set. The following options are available:
 ``proxy``
   Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
   if it doesn't conflict with the global proxy settings.
+``binary``
+  Set to ``true`` to get the http response as raw bytes.
 
 This function may fail.
 
@@ -396,6 +420,8 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``binary``
+  The response body as bytes (if ``binary=true``)
 ``blob``
   If ``into_blob`` was enabled for the request the body is downloaded into blob
   storage with a reference to the body in this field.
@@ -495,6 +521,13 @@ Log an info to the terminal.
 
     info('ohai')
 
+intval
+------
+
+Parse a number from a string.
+
+    x = strval('1234')
+
 json_decode
 -----------
 
@@ -529,6 +562,17 @@ Encode a datastructure into a string.
     })
     print(x)
 
+key_trunc_pad
+-------------
+
+Truncate/pad a key to a given length.
+
+.. code-block:: lua
+
+    -- if longer than 32 bytes: truncate to 32
+    -- if shorter than 32 bytes: pad with \x00
+    local key = key_trunc_pad(password, 32, 0)
+
 keyring
 -------
 
@@ -563,6 +607,64 @@ Hash a byte array with md5 and return the results as bytes.
 
     hex(md5("\x00\xff"))
 
+mqtt_connect
+------------
+
+Connect to an mqtt broker.
+
+.. code-block:: lua
+
+    local sock = mqtt_connect('mqtts://mqtt.example.com', {
+        username='foo',
+        password='secret',
+    })
+    if last_err() then return end
+
+mqtt_subscribe
+--------------
+
+Subscribe to a topic. Right now only QoS 0 is supported.
+
+.. code-block:: lua
+
+    mqtt_subscribe(sock, '#', 0)
+    if last_err() then return end
+
+mqtt_recv
+---------
+
+Receive an mqtt packet. This is not necessarily a publish packet and more
+packets might be added in the future, so you need to check the type
+specifically.
+
+If a read timeout has been set with mqtt_connect_ this function returns ``nil``
+in case of a read timeout.
+
+.. code-block:: lua
+
+    local pkt = mqtt_recv(sock)
+    if last_err() then return end
+    if pkt == nil then
+        -- read timeout, consider sending a ping or disconnect if the previous ping failed
+    elseif pkt['type'] == 'pong' then
+        -- broker sent a pong
+    elseif pkt['type'] == 'publish' then
+        local payload = utf8_decode(pkt['body'])
+        if last_err() then return end
+        info(payload)
+    end
+
+mqtt_ping
+---------
+
+Send a pingreq packet, causing the broker to send a pingresp. This is used to
+make sure the connection is still working correctly.
+
+.. code-block:: lua
+
+    mqtt_ping(sock)
+    if last_err() then return end
+
 pgp_pubkey
 ----------
 
@@ -834,6 +936,13 @@ The following options are available:
 ``proxy``
   Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
   if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
 
 .. code-block:: lua
 
@@ -865,6 +974,23 @@ discarded when using sock_connect_ directly with ``tls=true``.
 
     info(tls)
 
+sock_options
+------------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock_options(sock, {
+        read_timeout=3,
+    })
+
 sock_send
 ---------
 
@@ -966,6 +1092,27 @@ Overwrite the default ``\n`` newline.
 
     sock_newline(sock, "\r\n")
 
+sodium_secretbox_open
+---------------------
+
+Use authenticated symetric crypto to decrypt a given message.
+
+Internally this is ``crypto_secretbox_xsalsa20poly1305``.
+
+The key **must** be 32 bytes, see key_trunc_pad_ if necessary.
+
+The first 24 bytes of the encrypted message are expected to be the nonce.
+
+.. code-block:: lua
+
+    plain = sodium_secretbox_open(encrypted, key)
+    if last_err() then return end
+
+    txt = utf8_decode(plain)
+    if last_err() then return end
+
+    info(txt)
+
 status
 ------
 
@@ -1001,6 +1148,32 @@ Read stdin until EOF as a utf-8 string.
 .. note::
    This only works with `sn0int run --stdin`.
 
+str_find
+--------
+
+Returns the byte index of the first character that matches the pattern. This is
+explicitly a literal match instead of a lua pattern.
+
+If no match is found, returns ``nil``.
+
+.. code-block:: lua
+
+    x = str_find('asdf', 'sd')
+    print(x == 2)
+
+str_replace
+-----------
+
+Replaces all matches of a pattern in a string. This is explicitly a literal
+match instead of a lua pattern.
+
+If no match is found, an unmodified copy is returned.
+
+.. code-block:: lua
+
+    x = str_replace('this is old', 'old', 'new')
+    print(x == 'this is new')
+
 strftime
 --------
 
@@ -1021,6 +1194,13 @@ Parse a date into a unix timestamp, see `strftime rules`_.
 
 .. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
 
+strval
+------
+
+Convert a number into a string.
+
+    x = strval(1234)
+
 time_unix
 ---------
 
@@ -1141,6 +1321,111 @@ a ``run`` execution.
     warn_once('ohai')
     warn_once('ohai')
 
+ws_connect
+----------
+
+Create a websocket connection. The url format is ``ws://example.com/asdf``,
+``wss://`` is also supported.
+
+The following options are available:
+
+``headers``
+  A map of additional headers that should be set for the request.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock = ws_connect("wss://example.com/asdf", {})
+
+ws_options
+----------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    ws_options(sock, {
+        read_timeout=3,
+    })
+
+ws_recv_text
+------------
+
+Wait until the server sends a text frame. A binary frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_text(sock)
+
+ws_recv_binary
+--------------
+
+Wait until the server sends a binary frame. A text frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_binary(sock)
+
+ws_recv_json
+------------
+
+Identical to ws_send_text_ but automatically runs json_decode_ on the
+response.
+
+.. code-block:: lua
+
+    msg = ws_recv_json(sock)
+
+ws_send_text
+------------
+
+Send a text frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, "ohai!")
+
+ws_send_binary
+--------------
+
+Send a binary frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_binary(sock, "\x00\x01\x02")
+
+ws_send_json
+------------
+
+Encode the object as json string and send it as a text frame on the websocket
+connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, {
+        foo="ohai!",
+        x={
+            y={1,3,3,7},
+        },
+    })
+
 x509_parse_pem
 --------------
 

docs/usage.rst

@@ -27,12 +27,12 @@ number of recommended modules::
     [+] Downloading "GeoLite2-City.mmdb"
     [+] Downloading "GeoLite2-ASN.mmdb"
     [+] Loaded 0 modules
-    [*] No modules found, run quickstart to install default modules
+    [*] No modules found, run pkg quickstart to install default modules
     [sn0int][default] >
 
-Typing ``quickstart`` is going to get you a fair number of featured modules::
+Typing ``pkg quickstart`` is going to get you a fair number of featured modules::
 
-    [sn0int][default] > quickstart
+    [sn0int][default] > pkg quickstart
     [+] Installing kpcyrd/asn
     [+] Installing kpcyrd/ctlogs
     [+] Installing kpcyrd/dns-resolve
@@ -105,7 +105,7 @@ Running a module
 Now that we have something to get started with, we can run our first module.
 First lets list all modules we have::
 
-    [sn0int][demo] > mod list
+    [sn0int][demo] > pkg list
     kpcyrd/asn (0.1.0)
         Run a asn lookup for an ip address
     kpcyrd/ctlogs (0.1.0)
@@ -149,7 +149,7 @@ some of them in a browser but hold on, there's a more efficient way to approach
 this.
 
 .. hint::
-   You can run the modules concurrently with ``run -j 8``.
+   You can run the modules concurrently with ``run -j3``.
 
 Running followup modules on the results
 ---------------------------------------
@@ -256,85 +256,3 @@ You can reverse this using the scope command::
 .. hint::
    All entities have this field, you can refer to it in queries using
    ``unscoped=1``.
-
-Autonoscope
------------
-
-Instead of manually unscoping everything you can also define so called
-autonoscope rules. Those are executed from most specific to least specific and
-the first match wins. If no rule matches, the default is in-scope::
-
-    [sn0int][demo] > # add the domain first
-    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
-    [sn0int][demo] > add domain example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > # automatically noscope all subdomains
-    [sn0int][demo] > autonoscope add domain example.com
-    [sn0int][demo] > # except subdomains of prod.example.com 
-    [sn0int][demo] > autoscope add domain prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > autonoscope list 
-      scope domain "prod.example.com"
-    noscope domain "example.com"
-    [sn0int][demo] > 
-    [sn0int][demo] > # this is going to be out-of-scope
-    [sn0int][demo] > add subdomain www.example.com
-    [sn0int][demo] > # this is going to be in-scope
-    [sn0int][demo] > add subdomain db.prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > select subdomains 
-    #1, "www.example.com"
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=0
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=1
-    #1, "www.example.com"
-    [sn0int][demo] > 
-
-Domains
-~~~~~~~
-
-Autonoscope rules for domains are applied to the following structs:
-
-- domains
-- subdomains
-- urls
-
-Example rules::
-
-    autonoscope add domain example.com
-    autonoscope add domain staging.example.com
-    autonoscope add domain com
-    autonoscope add domain .
-
-IPs
-~~~
-
-Autonoscope rules for IPs are applied to the following structs:
-
-- ipaddrs
-- netblocks
-- ports
-
-Example rules::
-
-    autonoscope add ip 0.0.0.0/0
-    autonoscope add ip ::/0
-    autonoscope add ip 192.168.0.0/16
-    autonoscope add ip 10.13.33.37/32
-
-URLs
-~~~~
-
-Autonoscope rules for urls are applied to the following structs:
-
-- urls
-
-Note that these rules are specific to a certain origin (like
-``https://example.com``) and are used to filter paths.
-
-Example rules::
-
-    autonoscope add url https://example.com/
-    autonoscope add url https://example.com/admin/
-    autonoscope add url https://example.com/a/b/c/d

examples/maxmind.rs

@@ -1,16 +1,8 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
+use sn0int::paths;
 use std::net::IpAddr;
+use std::path::Path;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
@@ -27,8 +19,8 @@ pub struct AsnArgs {
 }
 
 impl AsnArgs {
-    fn run(&self) -> Result<()> {
-        let path = AsnDB::cache_path()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
         let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
@@ -44,8 +36,8 @@ pub struct GeoIPArgs {
 }
 
 impl GeoIPArgs {
-    fn run(&self) -> Result<()> {
-        let path = GeoIP::cache_path()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
         let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
@@ -59,9 +51,10 @@ impl GeoIPArgs {
 fn run() -> Result<()> {
     let args = Args::from_args();
     debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
     match args {
-        Args::Asn(args) => args.run(),
-        Args::GeoIP(args) => args.run(),
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
     }
 }
 

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

modules/harness/google-tls.lua

@@ -0,0 +1,29 @@
+-- Description: Test various tls functions
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('sending https request to google.com')
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://google.com/', {})
+    r = http_send(req)
+    if last_err() then return end
+    debug(r)
+
+    info('creating tls socket to google.com')
+    sock = sock_connect('google.com', 443, {
+        tls=true,
+    })
+    if last_err() then return end
+    debug(sock)
+
+    info('creating socket to google.com, wrapping afterwards')
+    sock = sock_connect('google.com', 443, {})
+    if last_err() then return end
+    tls = sock_upgrade_tls(sock, {
+        sni_value='google.com',
+    })
+    if last_err() then return end
+    debug(sock)
+    debug(tls)
+end

modules/harness/pgp-fetch.lua

@@ -0,0 +1,22 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+    url = 'https://openpgpkey.archlinux.org/.well-known/openpgpkey/archlinux.org/hu/in9mwr4s84x7gm51851h343n3at1x61g?l=anthraxx'
+
+    req = http_request(session, 'GET', url, {})
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['text'])
+    info(k)
+
+    req = http_request(session, 'GET', url, {
+        binary=true,
+    })
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['binary'])
+    info(k)
+end

modules/harness/socket-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = sock_connect('127.0.0.1', 4444, {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            sock_send(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        sock_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = sock_recv(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/url-inserter.lua

@@ -0,0 +1,58 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('preparing')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+
+    info('inserting')
+    url1 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+    })
+    url2 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        body='ohai',
+    })
+    url3 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+    })
+
+    info('updating')
+    db_update('url', {
+        id=url1,
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+        path='/',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url2,
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        path='/ohai',
+        body='ohai',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url3,
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        path='/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+        unscoped=false,
+    }, {
+    })
+end

modules/harness/winkekatze-sub.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local sock = mqtt_connect('mqtt://mqtt.winkekatze24.de', {
+        read_timeout=10,
+    })
+    if last_err() then return end
+    mqtt_subscribe(sock, '#', 0)
+
+    while true do
+        local pkt = mqtt_recv_text(sock)
+        if last_err() then return end
+        info(pkt)
+    end
+end

modules/harness/ws-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = ws_connect('ws://127.0.0.1:8080', {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            ws_send_text(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        ws_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = ws_recv_text(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/ws.lua

@@ -0,0 +1,23 @@
+-- Description: Create an echo websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local target = 'ws://echo.websocket.org'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('sending')
+    ws_send_text(sock, 'ohai wurld')
+    if last_err() then return end
+
+    info('recieving')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'ohai wurld' then
+        return 'echo failed, got: ' .. msg
+    end
+end

modules/harness/wss.lua

@@ -0,0 +1,30 @@
+-- Description: Create an encrypted websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- local target = 'ws://echo.websocket.org' -- doesn't support proper ciphers
+    local target = 'wss://rocket.events.ccc.de/sockjs/258/whi0yr1y/websocket'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('recieving 1/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'o' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('recieving 2/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'a["{\\"server_id\\":\\"0\\"}"]' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('handshake succeeded')
+end

sn0int-common/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "sn0int-common"
-version = "0.9.0"
-description = "Common code for sn0int"
+version = "0.10.0"
+description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-common" }
+sn0int-common = { version="0.10.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }

sn0int-registry/Dockerfile

@@ -2,16 +2,16 @@ FROM rust:buster
 RUN apt-get update -q && apt-get install -yq llvm libclang-dev \
     && rm -rf /var/lib/apt/lists/*
 RUN rustup install nightly
-WORKDIR /usr/src/sn0int-registry
+WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo +nightly build --release --verbose
+RUN cd sn0int-registry && cargo +nightly build --release --verbose
 RUN strip target/release/sn0int-registry
 
 FROM debian:buster
 RUN apt-get update -q && apt-get install -yq libcurl4 libpq5 \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=0 /usr/src/sn0int-registry/target/release/sn0int-registry /usr/local/bin/sn0int-registry
-COPY templates /templates
+COPY --from=0 /usr/src/sn0int/target/release/sn0int-registry /usr/local/bin/sn0int-registry
+COPY sn0int-registry/templates /templates
 ENV ROCKET_ENV=prod \
     ROCKET_ADDRESS=0.0.0.0 \
     ROCKET_PORT=8000

sn0int-registry/templates/index.html.hbs

@@ -21,7 +21,7 @@
     To install a module run:
     </p>
     <p class="code"><code>
-    sn0int install kpcyrd/ctlogs
+    sn0int pkg install kpcyrd/ctlogs
     </code></p>
 
     {{#each modules}}

sn0int-std/Cargo.toml

@@ -0,0 +1,55 @@
+[package]
+name = "sn0int-std"
+version = "0.18.0"
+description = "sn0int - stdlib"
+authors = ["kpcyrd <git@rxv.cc>"]
+repository = "https://github.com/kpcyrd/sn0int"
+license = "GPL-3.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4"
+failure = "0.1"
+hlua-badtouch = "0.4"
+tokio = "0.1.14"
+rand = "0.7"
+regex = "1.0"
+rustls = { version="0.16", features=["dangerous_configuration"] }
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+serde_urlencoded = "0.6"
+webpki = "0.21"
+webpki-roots = "0.19.0"
+ct-logs = "0.6"
+chrootable-https = "0.14"
+http = "0.2"
+bufstream = "0.1.4"
+pem = "0.7"
+url = "2.0"
+tungstenite = { version = "0.10.1", default-features = false }
+kuchiki = "0.8.0"
+maxminddb = "0.13"
+x509-parser = "0.6.2"
+der-parser = "3.0"
+publicsuffix = { version="1.5", default-features=false }
+xml-rs = "0.8"
+geo = "0.12"
+bytes = "0.4"
+base64 = "0.11"
+chrono = { version = "0.4", features = ["serde"] }
+mqtt-protocol = "0.8.1"
+sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
+
+image = "0.23.0"
+kamadak-exif = "0.5.1"
+
+bs58 = "0.3"
+digest = "0.8.0"
+blake2 = "0.8.0"
+
+[dev-dependencies]
+env_logger = "0.7"
+maplit = "1.0.1"

sn0int-std/src/blobs.rs

@@ -0,0 +1,118 @@
+use bytes::Bytes;
+use blake2::VarBlake2b;
+use digest::{Input, VariableOutput};
+use serde::ser::{Serialize, Serializer};
+use serde::de::{self, Deserialize, Deserializer};
+use std::result;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Blob {
+    pub id: String,
+    pub bytes: Bytes,
+}
+
+impl Blob {
+    pub fn create(bytes: Bytes) -> Blob {
+        let id = Self::hash(&bytes);
+        Blob {
+            id,
+            bytes,
+        }
+    }
+
+    pub fn hash(bytes: &[u8]) -> String {
+        let mut h = VarBlake2b::new(32).unwrap();
+        h.input(bytes);
+        Self::encode_hash(&h.vec_result())
+    }
+
+    #[inline]
+    fn encode_hash(bytes: &[u8]) -> String {
+        let x = bs58::encode(bytes).into_string();
+        format!("{:0<44}", x)
+    }
+}
+
+impl Serialize for Blob {
+    #[inline]
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let s = base64::encode(&self.bytes);
+        serializer.serialize_str(&s)
+    }
+}
+
+impl<'de> Deserialize<'de> for Blob {
+    #[inline]
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let bytes = base64::decode(&s)
+            .map_err(de::Error::custom)?;
+        Ok(Blob::create(Bytes::from(bytes)))
+    }
+}
+
+pub trait BlobState {
+    fn register_blob(&self, blob: Blob) -> String;
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json;
+
+    #[inline]
+    fn blob() -> (Bytes, Blob) {
+        let bytes = Bytes::from(&b"asdf"[..]);
+        (bytes.clone(), Blob::create(bytes))
+    }
+
+    #[test]
+    fn verify_create_blob() {
+        let (bytes, blob) = blob();
+        assert_eq!(blob, Blob {
+            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
+            bytes,
+        });
+    }
+
+    #[test]
+    fn test_serialize() {
+        let (_, blob) = blob();
+        let json = serde_json::to_string(&blob).unwrap();
+        assert_eq!(&json, "\"YXNkZg==\"");
+    }
+
+    #[test]
+    fn test_deserialize() {
+        let (_, blob1) = blob();
+        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
+        assert_eq!(blob1, blob2);
+    }
+
+    #[test]
+    fn test_hash_encoding() {
+        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
+    }
+
+    #[test]
+    fn test_hash_encoding_padding() {
+        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
+    }
+}

sn0int-std/src/crypto.rs

@@ -0,0 +1,51 @@
+use crate::errors::*;
+use sodiumoxide::crypto::secretbox::{self, Key, Nonce};
+use std::iter;
+
+pub fn key_trunc_pad(mut key: &[u8], len: usize, pad: u8) -> Vec<u8> {
+    if key.len() > len {
+        key = &key[..len];
+    }
+
+    let mut key = key.to_vec();
+    key.extend(iter::repeat(pad).take(len - key.len()));
+    key
+}
+
+pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
+    if encrypted.len() <= secretbox::NONCEBYTES {
+        bail!("Encrypted message is too short");
+    }
+
+    let key = Key::from_slice(key)
+        .ok_or_else(|| format_err!("Key has wrong length"))?;
+    let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
+        .ok_or_else(|| format_err!("Nonce has wrong length"))?;
+    let ciphertext = &encrypted[secretbox::NONCEBYTES..];
+    let plain = secretbox::open(&ciphertext, &nonce, &key)
+        .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
+    Ok(plain)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_key_equal() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_trunc() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5, 6, 7, 8, 9], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_pad() {
+        let key = key_trunc_pad(&[1, 2, 3], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 0, 0]);
+    }
+}

sn0int-std/src/engine/mod.rs

@@ -0,0 +1 @@
+pub mod structs;

sn0int-std/src/engine/structs.rs

@@ -41,6 +41,12 @@ impl LuaMap {
     pub fn insert_num<K: Into<String>>(&mut self, k: K, v: f64) {
         self.0.insert(AnyHashableLuaValue::LuaString(k.into()), AnyLuaValue::LuaNumber(v));
     }
+
+    pub fn insert_serde<K: Into<String>, S: serde::Serialize>(&mut self, k: K, v: S) -> Result<()> {
+        let v = serde_json::to_value(v)?;
+        self.0.insert(AnyHashableLuaValue::LuaString(k.into()), LuaJsonValue::from(v).into());
+        Ok(())
+    }
 }
 
 impl IntoIterator for LuaMap {

sn0int-std/src/errors.rs

@@ -0,0 +1,3 @@
+pub use log::{trace, debug, info, warn, error};
+pub use failure::{Error, ResultExt, format_err, bail};
+pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-std/src/geo.rs

@@ -0,0 +1,111 @@
+use crate::errors::*;
+
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use geo::{LineString, Polygon, Coordinate};
+use geo::prelude::*;
+
+
+#[derive(Debug, Deserialize)]
+pub struct Point {
+    lon: f64,
+    lat: f64,
+}
+
+impl Point {
+    pub fn try_from(x: AnyLuaValue) -> Result<Point> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub fn polygon_contains(ring: &[Point], p: &Point) -> bool {
+    let ring = ring.iter()
+        .map(|p| Coordinate { x: p.lon, y: p.lat })
+        .collect::<Vec<_>>();
+
+    let polygon = Polygon::new(LineString::from(ring), vec![]);
+    let point = geo::Point::new(p.lon, p.lat);
+
+    polygon.contains(&point)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn hamburg_polygon() -> &'static [Point] {
+        &[
+            Point { lat: 53.63975308945899,  lon: 9.764785766601562  },
+            Point { lat: 53.59494998253459,  lon: 9.827270507812     },
+            Point { lat: 53.663153974456456, lon: 9.9151611328125    },
+            Point { lat: 53.65582987649682,  lon: 9.976272583007812  },
+            Point { lat: 53.68613523817129,  lon: 9.992752075195312  },
+            Point { lat: 53.68674518938816,  lon: 10.051460266113281 },
+            Point { lat: 53.72495117617815,  lon: 10.075492858886719 },
+            Point { lat: 53.71946627930625,  lon: 10.118408203125    },
+            Point { lat: 53.743635083157756, lon: 10.164413452148438 },
+            Point { lat: 53.73104466704585,  lon: 10.202865600585938 },
+            Point { lat: 53.676781546441546, lon: 10.16304016113281  },
+            Point { lat: 53.632832079199474, lon: 10.235824584960938 },
+            Point { lat: 53.608803292930894, lon: 10.2008056640625   },
+            Point { lat: 53.578646152866504, lon: 10.208358764648438 },
+            Point { lat: 53.57212285981298,  lon: 10.163726806640625 },
+            Point { lat: 53.52071674896369,  lon: 10.18707275390625  },
+            Point { lat: 53.52643162253097,  lon: 10.224151611328125 },
+            Point { lat: 53.44062753992289,  lon: 10.347747802734375 },
+            Point { lat: 53.38824275010831,  lon: 10.248870849609375 },
+            Point { lat: 53.38824275010831,  lon: 10.15960693359375  },
+            Point { lat: 53.44635321212876,  lon: 10.064849853515625 },
+            Point { lat: 53.40595029739904,  lon: 9.985198974609375  },
+            Point { lat: 53.42385506057106,  lon: 9.951210021972656  },
+            Point { lat: 53.41843327091211,  lon: 9.944171905517578  },
+            Point { lat: 53.41812635648326,  lon: 9.927349090576172  },
+            Point { lat: 53.412294561442884, lon: 9.917736053466797  },
+            Point { lat: 53.41464783813818,  lon: 9.901256561279297  },
+            Point { lat: 53.443490472483326, lon: 9.912586212158201  },
+            Point { lat: 53.45177144115704,  lon: 9.897651672363281  },
+            Point { lat: 53.43633277935392,  lon: 9.866924285888672  },
+            Point { lat: 53.427639673754776, lon: 9.866409301757812  },
+            Point { lat: 53.427639673754776, lon: 9.858856201171875  },
+            Point { lat: 53.46710230573499,  lon: 9.795513153076172  },
+            Point { lat: 53.49039461941655,  lon: 9.795341491699219  },
+            Point { lat: 53.49029248806277,  lon: 9.77903366088867   },
+            Point { lat: 53.49856433088649,  lon: 9.780235290527344  },
+            Point { lat: 53.5078554643033,   lon: 9.758434295654297  },
+            Point { lat: 53.545407634092975, lon: 9.759807586669922  },
+            Point { lat: 53.568147234570084, lon: 9.633293151855469  },
+            Point { lat: 53.58802162343514,  lon: 9.655780792236328  },
+            Point { lat: 53.568351121879815, lon: 9.727706909179688  },
+            Point { lat: 53.60921067445695,  lon: 9.737663269042969  },
+        ]
+    }
+
+    #[test]
+    fn test_polygon_hamburg_contains_hamburg() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 53.551085,
+            lon: 9.993682,
+        });
+        assert!(contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_berlin() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 52.52437,
+            lon: 13.41053,
+        });
+        assert!(!contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_ny() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 40.726662,
+            lon: -74.036677,
+        });
+        assert!(!contains);
+    }
+}

sn0int-std/src/geoip/mod.rs

@@ -1,6 +1,5 @@
 use crate::errors::*;
 use crate::lazy::LazyInit;
-use crate::paths;
 use maxminddb::{self, geoip2};
 use std::fmt;
 use std::fs::{self, File};
@@ -19,13 +18,15 @@ pub trait Maxmind: Sized {
 
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self;
 
-    fn cache_path() -> Result<PathBuf> {
+    fn cache_path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         for path in &[
             // Archlinux
             "/usr/share/GeoIP/",
             // OpenBSD
             "/usr/local/share/examples/libmaxminddb/",
+            // geoipupdate
+            "/var/lib/GeoIP/",
         ] {
             let path = Path::new(path);
             let path = path.join(Self::filename());
@@ -36,7 +37,7 @@ pub trait Maxmind: Sized {
         }
 
         // use cache path
-        let path = paths::cache_dir()?
+        let path = cache_dir
             .join(Self::filename());
         Ok(path)
     }
@@ -52,8 +53,8 @@ pub trait Maxmind: Sized {
         Self::from_buf(buf)
     }
 
-    fn try_open_reader() -> Result<Option<MaxmindReader>> {
-        let path = Self::cache_path()?;
+    fn try_open_reader(cache_dir: &Path) -> Result<Option<MaxmindReader>> {
+        let path = Self::cache_path(cache_dir)?;
 
         if path.exists() {
             let db = MaxmindReader::open_path(path)?;

sn0int-std/src/gfx/exif.rs

@@ -61,8 +61,9 @@ impl LocationBuilder {
 }
 
 pub fn gps(img: &[u8]) -> Result<Option<Location>> {
-    let mut buf = io::BufReader::new(img);
-    let reader = exif::Reader::new(&mut buf)?;
+    let mut buf = io::Cursor::new(img);
+    let reader = exif::Reader::new()
+        .read_from_container(&mut buf)?;
     let fields = reader.fields();
 
     let location = Location::try_from_iter(fields).ok();

sn0int-std/src/gfx/mod.rs

@@ -6,13 +6,13 @@ pub mod exif;
 
 #[derive(Debug)]
 pub enum ImageFormat {
-    PNG,
-    JPEG,
-    GIF,
-    WEBP,
-    TIFF,
-    BMP,
-    ICO,
+    Png,
+    Jpeg,
+    Gif,
+    WebP,
+    Tiff,
+    Bmp,
+    Ico,
 }
 
 impl ImageFormat {
@@ -20,28 +20,27 @@ impl ImageFormat {
         // https://www.iana.org/assignments/media-types/media-types.xhtml#image
         // /etc/nginx/mime.types
         match self {
-            ImageFormat::PNG => "image/png",
-            ImageFormat::JPEG => "image/jpeg",
-            ImageFormat::GIF => "image/gif",
-            ImageFormat::WEBP => "image/webp",
-            ImageFormat::TIFF => "image/tiff",
-            ImageFormat::BMP => "image/bmp",
-            ImageFormat::ICO => "image/vnd.microsoft.icon",
+            ImageFormat::Png => "image/png",
+            ImageFormat::Jpeg => "image/jpeg",
+            ImageFormat::Gif => "image/gif",
+            ImageFormat::WebP => "image/webp",
+            ImageFormat::Tiff => "image/tiff",
+            ImageFormat::Bmp => "image/bmp",
+            ImageFormat::Ico => "image/vnd.microsoft.icon",
         }
     }
 
     pub fn try_from(format: &image::ImageFormat) -> Result<ImageFormat> {
         use image::ImageFormat::*;
         match format {
-            PNG => Ok(ImageFormat::PNG),
-            JPEG => Ok(ImageFormat::JPEG),
-            GIF => Ok(ImageFormat::GIF),
-            WEBP => Ok(ImageFormat::WEBP),
-            TIFF => Ok(ImageFormat::TIFF),
-            BMP => Ok(ImageFormat::BMP),
-            ICO => Ok(ImageFormat::ICO),
-
-            HDR | PNM | TGA => bail!("Unsupported format: {:?}", format),
+            Png => Ok(ImageFormat::Png),
+            Jpeg => Ok(ImageFormat::Jpeg),
+            Gif => Ok(ImageFormat::Gif),
+            WebP => Ok(ImageFormat::WebP),
+            Tiff => Ok(ImageFormat::Tiff),
+            Bmp => Ok(ImageFormat::Bmp),
+            Ico => Ok(ImageFormat::Ico),
+            _ => bail!("Unsupported format: {:?}", format),
         }
     }
 }
@@ -49,13 +48,13 @@ impl ImageFormat {
 impl Into<image::ImageFormat> for ImageFormat {
     fn into(self) -> image::ImageFormat {
         match self {
-            ImageFormat::PNG => image::ImageFormat::PNG,
-            ImageFormat::JPEG => image::ImageFormat::JPEG,
-            ImageFormat::GIF => image::ImageFormat::GIF,
-            ImageFormat::WEBP => image::ImageFormat::WEBP,
-            ImageFormat::TIFF => image::ImageFormat::TIFF,
-            ImageFormat::BMP => image::ImageFormat::BMP,
-            ImageFormat::ICO => image::ImageFormat::ICO,
+            ImageFormat::Png => image::ImageFormat::Png,
+            ImageFormat::Jpeg => image::ImageFormat::Jpeg,
+            ImageFormat::Gif => image::ImageFormat::Gif,
+            ImageFormat::WebP => image::ImageFormat::WebP,
+            ImageFormat::Tiff => image::ImageFormat::Tiff,
+            ImageFormat::Bmp => image::ImageFormat::Bmp,
+            ImageFormat::Ico => image::ImageFormat::Ico,
         }
     }
 }
@@ -114,7 +113,7 @@ mod tests {
 
     #[test]
     fn verify_gfx_load_ico() {
-        let ico = fs::read("./sn0int-registry/assets/favicon.ico").expect("fs::read");
+        let ico = fs::read("../sn0int-registry/assets/favicon.ico").expect("fs::read");
         let img = load(&ico).expect("gfx::load");
 
         assert_eq!("image/vnd.microsoft.icon", img.mime());

sn0int-std/src/html.rs

@@ -102,6 +102,7 @@ pub fn html_form(html: &str) -> Result<HashMap<String, String>> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn test_html_select() {

sn0int-std/src/lib.rs

@@ -0,0 +1,26 @@
+#[macro_use] extern crate serde_derive;
+use hlua_badtouch as hlua;
+
+pub mod blobs;
+pub mod crt;
+pub mod crypto;
+mod errors;
+pub mod engine;
+pub mod geo;
+pub mod geoip;
+pub mod gfx;
+pub mod html;
+pub mod json;
+pub mod lazy;
+pub mod mqtt;
+pub mod psl;
+pub mod ratelimits;
+pub mod sockets;
+pub mod web;
+pub mod websockets;
+pub mod xml;
+
+#[cfg(test)]
+fn test_init() {
+    let _ = env_logger::builder().is_test(true).try_init();
+}

sn0int-std/src/mqtt.rs

@@ -0,0 +1,228 @@
+use chrootable_https::DnsResolver;
+use crate::errors::*;
+use crate::hlua::AnyLuaValue;
+use mqtt::packet::VariablePacketError;
+use crate::json::LuaJsonValue;
+use crate::sockets::{Stream, SocketOptions};
+use mqtt::{TopicFilter, QualityOfService};
+use mqtt::control::ConnectReturnCode;
+use mqtt::control::fixed_header::FixedHeaderError;
+use mqtt::encodable::{Encodable, Decodable};
+use mqtt::packet::{Packet, VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
+use std::convert::TryFrom;
+use std::io;
+use std::net::SocketAddr;
+use url::Url;
+
+#[derive(Debug, Default, Deserialize)]
+pub struct MqttOptions {
+    pub username: Option<String>,
+    pub password: Option<String>,
+
+    pub proxy: Option<SocketAddr>,
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
+}
+
+impl MqttOptions {
+    pub fn try_from(x: AnyLuaValue) -> Result<MqttOptions> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub struct MqttClient {
+    stream: Stream,
+}
+
+impl MqttClient {
+    pub fn negotiate(stream: Stream, options: &MqttOptions) -> Result<MqttClient> {
+        let mut client = MqttClient {
+            stream,
+        };
+
+        let mut pkt = ConnectPacket::new("MQTT", "sn0int");
+        pkt.set_user_name(options.username.clone());
+        pkt.set_password(options.password.clone());
+
+        /*
+        if let Some(keep_alive) = msg.keep_alive {
+            packet.set_keep_alive(keep_alive);
+        }
+        */
+
+        client.send(pkt.into())?;
+        let pkt = client.recv()?;
+
+        if let VariablePacket::ConnackPacket(pkt) = pkt {
+            let code = pkt.connect_return_code();
+            if code == ConnectReturnCode::ConnectionAccepted {
+                Ok(client)
+            } else {
+                bail!("MQTT negotiation failed: {:?}", code);
+            }
+        } else {
+            bail!("Expected ConnAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &MqttOptions) -> Result<MqttClient> {
+        let tls = match url.scheme() {
+            "mqtt" => false,
+            "mqtts" => true,
+            _ => bail!("Invalid mqtt protocol"),
+        };
+
+        let host = url.host_str()
+            .ok_or_else(|| format_err!("Missing host in url"))?;
+
+        let port = match (url.port(), tls) {
+            (Some(port), _) => port,
+            (None, true) => 8883,
+            (None, false) => 1883,
+        };
+
+
+        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
+            tls,
+            sni_value: None,
+            disable_tls_verify: false,
+            proxy: options.proxy,
+
+            connect_timeout: options.connect_timeout,
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+        })?;
+
+        Self::negotiate(stream, options)
+    }
+
+    fn send(&mut self, pkt: VariablePacket) -> Result<()> {
+        debug!("Sending mqtt packet: {:?}", pkt);
+        pkt.encode(&mut self.stream)?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> std::result::Result<VariablePacket, VariablePacketError> {
+        let pkt = VariablePacket::decode(&mut self.stream)?;
+        debug!("Received mqtt packet: {:?}", pkt);
+        Ok(pkt)
+    }
+
+    pub fn subscribe(&mut self, topic: &str, qos: u8) -> Result<()> {
+        let filter = TopicFilter::new(topic)?;
+
+        let qos = match qos {
+            0 => QualityOfService::Level0,
+            1 => QualityOfService::Level1,
+            2 => QualityOfService::Level2,
+            _ => bail!("Invalid QoS level: {}", qos),
+        };
+
+        let pkt = SubscribePacket::new(1, vec![(filter, qos)]);
+        self.send(pkt.into())?;
+
+        let pkt = self.recv()?;
+        if let VariablePacket::SubackPacket(_pkt) = pkt {
+            Ok(())
+        } else {
+            bail!("Expected SubAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn recv_pkt(&mut self) -> Result<Option<Pkt>> {
+        match self.recv() {
+            Ok(pkt) => Ok(Some(Pkt::try_from(pkt)?)),
+            Err(VariablePacketError::IoError(err)) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(err) => Err(Error::from(err))
+        }
+    }
+
+    pub fn ping(&mut self) -> Result<()> {
+        let pkt = PingreqPacket::new();
+        self.send(pkt.into())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum Pkt {
+    #[serde(rename="publish")]
+    Publish(Publish),
+    #[serde(rename="pong")]
+    Pong,
+}
+
+impl Pkt {
+    pub fn to_lua(&self) -> Result<AnyLuaValue> {
+        let v = serde_json::to_value(&self)?;
+        let v = LuaJsonValue::from(v).into();
+        Ok(v)
+    }
+}
+
+impl TryFrom<VariablePacket> for Pkt {
+    type Error = Error;
+
+    fn try_from(pkt: VariablePacket) -> Result<Pkt> {
+        match pkt {
+            VariablePacket::ConnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::ConnackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PublishPacket(pkt) => Ok(Pkt::Publish(Publish {
+                topic: pkt.topic_name().to_string(),
+                body: pkt.payload(),
+            })),
+            VariablePacket::PubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrecPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrelPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubcompPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingreqPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingrespPacket(_) => Ok(Pkt::Pong),
+            VariablePacket::SubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::SubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::DisconnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Publish {
+    pub topic: String,
+    pub body: Vec<u8>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrootable_https::dns::Resolver;
+
+    fn connect() -> Result<MqttClient> {
+        let resolver = Resolver::from_system().unwrap();
+        let url = "mqtt://mqtt.winkekatze24.de".parse()?;
+        MqttClient::connect(&resolver, url, &MqttOptions::default())
+    }
+
+    #[test]
+    #[ignore]
+    fn test_connect() {
+        connect().expect("Failed to setup connection");
+    }
+
+    // this test is too flaky
+    /*
+    #[test]
+    #[ignore]
+    fn test_subscribe() {
+        let mut c = connect().unwrap();
+        c.subscribe("#", 0).unwrap();
+    }
+    */
+}

sn0int-std/src/psl.rs

@@ -4,9 +4,7 @@ use chrootable_https::Client;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use crate::paths;
 use crate::lazy::LazyInit;
-use crate::worker;
 
 
 #[derive(Debug, PartialEq)]
@@ -23,22 +21,24 @@ pub enum PslReader {
 }
 
 impl PslReader {
-    pub fn open_or_download() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open_or_download<F>(cache_dir: &Path, indicator: F) -> Result<PslReader>
+        where
+            F: Fn(Box<dyn Fn() -> Result<PslReader>>) -> Result<PslReader>
+    {
+        let path = Self::path(cache_dir)?;
         let reader = match Self::open_from(&path) {
             Ok(r) => r,
-            Err(_) => worker::spawn_fn("Downloading public suffix list", || {
+            Err(_) => indicator(Box::new(move || {
                 PslReader::download(&path, publicsuffix::LIST_URL)?;
                 Self::open_from(&path)
-            }, false)?,
+            }))?,
         };
         Ok(reader)
     }
 
-    pub fn open() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open(cache_dir: &Path) -> Result<PslReader> {
+        let path = Self::path(cache_dir)?;
         Self::open_from(&path)
-
     }
 
     pub fn open_from(path: &Path) -> Result<PslReader> {
@@ -46,7 +46,7 @@ impl PslReader {
         Ok(PslReader::Reader(file))
     }
 
-    pub fn path() -> Result<PathBuf> {
+    pub fn path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         let path = Path::new("/usr/share/publicsuffix/public_suffix_list.dat");
         if path.exists() {
@@ -54,7 +54,7 @@ impl PslReader {
         }
 
         // else, use local cache
-        let path = paths::cache_dir()?
+        let path = cache_dir
             .join("public_suffix_list.dat");
         Ok(path)
     }

sn0int-std/src/ratelimits.rs

@@ -1,8 +1,11 @@
 use chrono::prelude::*;
-use crate::worker::RatelimitSender;
 use std::collections::HashMap;
+use std::result;
+use std::sync::mpsc;
 use std::time::Duration;
 
+pub type RatelimitSender = mpsc::Sender<result::Result<RatelimitResponse, String>>;
+
 pub struct Ratelimiter {
     buckets: HashMap<String, Bucket>,
 }

sn0int-std/src/sockets/mod.rs

@@ -16,6 +16,7 @@ use std::io::BufRead;
 use std::net::SocketAddr;
 use std::net::TcpStream;
 use std::net::{IpAddr, Ipv4Addr};
+use std::time::Duration;
 
 mod tls;
 pub use self::tls::TlsData;
@@ -39,14 +40,21 @@ fn unwrap_socket(socket: tokio::net::TcpStream) -> Result<TcpStream> {
 pub struct SocketOptions {
     #[serde(default)]
     pub tls: bool,
-    sni_value: Option<String>,
+    pub sni_value: Option<String>,
     #[serde(default)]
-    disable_tls_verify: bool,
+    pub disable_tls_verify: bool,
     pub proxy: Option<SocketAddr>,
 
     // TODO: enable_sni (default to true)
+    // TODO: sni_name
     // TODO: cacert
-    // TODO: timeout
+
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
 }
 
 impl SocketOptions {
@@ -57,17 +65,97 @@ impl SocketOptions {
     }
 }
 
-#[derive(Debug)]
-pub struct Socket {
-    stream: BufStream<Stream>,
-    newline: String,
+impl SocketOptions {
+    pub fn apply(&self, stream: &Stream) -> Result<()> {
+        let socket = match stream {
+            Stream::Tcp(s) => s,
+            Stream::Tls(s) => s.get_ref(),
+        };
+        self.apply_tcp(socket)
+    }
+
+    pub fn apply_tcp(&self, socket: &TcpStream) -> Result<()> {
+        let read_timeout = self.read_timeout;
+        if read_timeout > 0 {
+            socket.set_read_timeout(Some(Duration::from_secs(read_timeout)))?;
+        }
+
+        let write_timeout = self.write_timeout;
+        if write_timeout > 0 {
+            socket.set_write_timeout(Some(Duration::from_secs(write_timeout)))?;
+        }
+
+        Ok(())
+    }
 }
 
-enum Stream {
+pub enum Stream {
     Tcp(TcpStream),
     Tls(rustls::StreamOwned<rustls::ClientSession, TcpStream>),
 }
 
+impl Stream {
+    pub fn connect_stream<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+        let addrs = match host.parse::<IpAddr>() {
+            Ok(addr) => vec![addr],
+            Err(_) => resolver.resolve(host, RecordType::A)
+                .wait_for_response()?
+                .success()?,
+        };
+
+        let mut errors = Vec::new();
+
+        for addr in addrs {
+            match Stream::connect_addr(host, (addr, port).into(), &options) {
+                Ok(socket) => {
+                    return Ok(socket);
+                },
+                Err(err) => errors.push((addr, err)),
+            }
+        }
+
+        if errors.is_empty() {
+            bail!("no dns records found");
+        } else {
+            bail!("couldn't connect: {:?}", errors);
+        }
+    }
+
+    fn connect_addr(host: &str, addr: SocketAddr, options: &SocketOptions) -> Result<Stream> {
+        debug!("connecting to {}", addr);
+
+        let connect_timeout = options.connect_timeout;
+        let socket = if connect_timeout > 0 {
+            TcpStream::connect_timeout(&addr, Duration::from_secs(connect_timeout))?
+        } else {
+            TcpStream::connect(&addr)?
+        };
+        debug!("successfully connected to {:?}", addr);
+
+        options.apply_tcp(&socket)?;
+
+        tls::wrap_if_enabled(socket, host, options)
+    }
+
+    pub fn connect_socks5_stream(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
+
+        let addr = match host.parse::<Ipv4Addr>() {
+            Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
+            _ => ProxyDest::Domain(host.to_string()),
+        };
+
+        let fut = socks5::connect(proxy, addr, port);
+
+        let mut rt = Runtime::new()?;
+        let socket = rt.block_on(fut)?;
+
+        let socket = unwrap_socket(socket)?;
+
+        tls::wrap_if_enabled(socket, host, options)
+    }
+}
+
 impl fmt::Debug for Stream {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
         match self {
@@ -102,6 +190,12 @@ impl Write for Stream {
     }
 }
 
+#[derive(Debug)]
+pub struct Socket {
+    stream: BufStream<Stream>,
+    newline: String,
+}
+
 impl Socket {
     fn new(stream: Stream) -> Socket {
         let stream = BufStream::new(stream);
@@ -112,60 +206,31 @@ impl Socket {
     }
 
     pub fn connect<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
-        let addrs = match host.parse::<IpAddr>() {
-            Ok(addr) => vec![addr],
-            Err(_) => resolver.resolve(host, RecordType::A)
-                .wait_for_response()?
-                .success()?,
-        };
-
-        let mut errors = Vec::new();
-
-        for addr in addrs {
-            debug!("connecting to {}:{}", addr, port);
-            match TcpStream::connect((addr, port)) {
-                Ok(socket) => {
-                    debug!("successfully connected to {:?}", addr);
-                    return tls::wrap_if_enabled(socket, host, options);
-                },
-                Err(err) => errors.push((addr, err)),
-            }
-        }
-
-        if errors.is_empty() {
-            bail!("no dns records found");
-        } else {
-            bail!("couldn't connect: {:?}", errors);
-        }
+        let stream = Stream::connect_stream(resolver, host, port, options)?;
+        Ok(Socket::new(stream))
     }
 
     pub fn connect_socks5(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
-        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
-
-        let addr = match host.parse::<Ipv4Addr>() {
-            Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
-            _ => ProxyDest::Domain(host.to_string()),
-        };
-
-        let fut = socks5::connect(proxy, addr, port);
-
-        let mut rt = Runtime::new()?;
-        let socket = rt.block_on(fut)?;
-
-        let socket = unwrap_socket(socket)?;
-
-        tls::wrap_if_enabled(socket, host, options)
+        let stream = Stream::connect_socks5_stream(proxy, host, port, options)?;
+        Ok(Socket::new(stream))
     }
 
     pub fn upgrade_to_tls(self, options: &SocketOptions) -> Result<(Socket, TlsData)> {
         let stream = self.stream.into_inner()?;
 
-        match stream {
-            Stream::Tcp(stream) => tls::wrap(stream, "", options),
-            _ => bail!("Only tcp streams can be upgraded"),
+        if let Stream::Tcp(stream) = stream {
+            let (stream, tls) = tls::wrap(stream, "", options)?;
+            let socket = Socket::new(stream);
+            Ok((socket, tls))
+        } else {
+            bail!("Only tcp streams can be upgraded")
         }
     }
 
+    pub fn options(&self, options: &SocketOptions) -> Result<()> {
+        options.apply(self.stream.get_ref())
+    }
+
     pub fn send(&mut self, data: &[u8]) -> Result<()> {
         match str::from_utf8(&data) {
             Ok(data) => debug!("send: {:?}", data),
@@ -178,7 +243,12 @@ impl Socket {
 
     pub fn recv(&mut self) -> Result<Vec<u8>> {
         let mut buf = [0; 4096];
-        let n = self.stream.read(&mut buf)?;
+        let n = match self.stream.read(&mut buf) {
+            Ok(n) if n == 0 => bail!("Connection closed"),
+            Ok(n) => n,
+            Err(err) if err.kind() == io::ErrorKind::WouldBlock => 0,
+            Err(err) => return Err(err.into()),
+        };
         let data = buf[..n].to_vec();
         match str::from_utf8(&data) {
             Ok(data) => debug!("recv: {:?}", data),
@@ -249,6 +319,7 @@ impl Socket {
                 let available = match self.stream.fill_buf() {
                     Ok(n) => n,
                     Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
+                    Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(Vec::new()),
                     Err(e) => return Err(e.into())
                 };
 

sn0int-std/src/sockets/tls.rs

@@ -2,14 +2,14 @@ use crate::errors::*;
 
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
-use rustls::{self, ClientConfig, Session, ClientSession, RootCertStore};
+use rustls::{self, ClientConfig, Session, ClientSession};
 
 use std::str;
 use std::result;
 use std::sync::Arc;
 use std::net::TcpStream;
 
-use super::{Socket, Stream, SocketOptions};
+use super::{Stream, SocketOptions};
 
 
 #[derive(Debug, Serialize)]
@@ -26,22 +26,22 @@ impl TlsData {
     }
 }
 
-pub fn wrap_if_enabled(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<Socket> {
+pub fn wrap_if_enabled(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<Stream> {
     if !options.tls {
         let stream = Stream::Tcp(stream);
-        return Ok(Socket::new(stream));
+        Ok(stream)
     } else {
         let (socket, _) = wrap(stream, host, options)?;
         Ok(socket)
     }
 }
 
-pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(Socket, TlsData)> {
-    let mut anchors = RootCertStore::empty();
-    anchors.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
-
+pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(Stream, TlsData)> {
     let mut config = ClientConfig::new();
-    config.root_store = anchors;
+    config
+        .root_store
+        .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
+    config.ct_logs = Some(&ct_logs::LOGS);
 
     if options.disable_tls_verify {
         info!("tls verification has been disabled");
@@ -73,14 +73,16 @@ fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
     }
 }
 
-fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Socket, TlsData)> {
+fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, TlsData)> {
     info!("starting tls handshake");
     if session.is_handshaking() {
-        session.complete_io(&mut stream)?;
+        session.complete_io(&mut stream)
+            .context("Failed to read reply to tls client hello")?;
     }
 
     if session.wants_write() {
-        session.complete_io(&mut stream)?;
+        session.complete_io(&mut stream)
+            .context("wants_write->complete_io failed")?;
     }
 
     let mut tls = TlsData {
@@ -106,7 +108,7 @@ fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Socket, T
     info!("successfully established tls connection");
     let stream = rustls::StreamOwned::new(session, stream);
     let stream = Stream::Tls(stream);
-    Ok((Socket::new(stream), tls))
+    Ok((stream, tls))
 }
 
 pub struct NoCertificateVerification {}

sn0int-std/src/web.rs

@@ -1,17 +1,17 @@
 pub use chrootable_https::{Client, HttpClient, Resolver, Response};
 
-use crate::blobs::Blob;
+use crate::blobs::{Blob, BlobState};
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
 use std::ops::Deref;
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
-use crate::engine::ctx::State;
 use serde_json;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
 use std::fmt;
 use std::net::SocketAddr;
+use std::sync::Arc;
 use serde::Serialize;
 use crate::engine::structs::LuaMap;
 use crate::json::LuaJsonValue;
@@ -37,6 +37,12 @@ pub fn url_set_qs<S: Serialize + fmt::Debug>(url: Uri, query: &S) -> Result<Uri>
         .map_err(Error::from)
 }
 
+pub trait WebState {
+    fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>>;
+
+    fn register_in_jar(&self, session: &str, key: String, value: String);
+}
+
 #[derive(Debug)]
 pub struct HttpSession {
     id: String,
@@ -66,6 +72,8 @@ pub struct RequestOptions {
     #[serde(default)]
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    #[serde(default)]
+    binary: bool,
 }
 
 impl RequestOptions {
@@ -95,6 +103,7 @@ pub struct HttpRequest {
     timeout: Option<Duration>,
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    binary: bool,
 }
 
 impl HttpRequest {
@@ -117,6 +126,7 @@ impl HttpRequest {
             timeout,
             into_blob: options.into_blob,
             proxy: options.proxy,
+            binary: options.binary,
         };
 
         if let Some(json) = options.json {
@@ -134,7 +144,7 @@ impl HttpRequest {
         request
     }
 
-    pub fn send(&self, state: &dyn State) -> Result<Response> {
+    pub fn send(&self, state: &dyn WebState) -> Result<Response> {
         let mut url = self.url.parse::<Uri>()?;
 
         // set query string
@@ -228,7 +238,9 @@ impl HttpRequest {
         Ok(res)
     }
 
-    pub fn response_to_lua(&self, state: &dyn State, res: Response) -> Result<LuaMap> {
+    pub fn response_to_lua<S>(&self, state: &S, res: Response) -> Result<LuaMap>
+        where S: WebState + BlobState
+    {
         // map result to LuaMap
         let mut resp = LuaMap::new();
         resp.insert_num("status", f64::from(res.status));
@@ -239,10 +251,16 @@ impl HttpRequest {
         }
         resp.insert("headers", headers);
 
+        if let Some(ipaddr) = res.ipaddr {
+            resp.insert_str("ipaddr", ipaddr.to_string());
+        }
+
         if self.into_blob {
             let blob = Blob::create(res.body);
             let id = state.register_blob(blob);
             resp.insert_str("blob", id);
+        } else if self.binary {
+            resp.insert_serde("binary", &res.body[..])?;
         } else {
             resp.insert_str("text", String::from_utf8_lossy(&res.body));
         }
@@ -250,7 +268,7 @@ impl HttpRequest {
         Ok(resp)
     }
 
-    fn register_cookies_on_state(session: &str, state: &dyn State, cookie: &str) {
+    fn register_cookies_on_state(session: &str, state: &dyn WebState, cookie: &str) {
         let mut key = String::new();
         let mut value = String::new();
         let mut in_key = true;

sn0int-std/src/websockets.rs

@@ -0,0 +1,150 @@
+use chrootable_https::DnsResolver;
+use crate::errors::*;
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use crate::sockets::{Stream, SocketOptions};
+use http::Request;
+use std::collections::HashMap;
+use std::net::SocketAddr;
+use std::io;
+use tungstenite::protocol::{self, Message};
+use url::Url;
+
+#[derive(Debug, Default, Deserialize)]
+pub struct WebSocketOptions {
+    pub headers: Option<HashMap<String, String>>,
+    pub proxy: Option<SocketAddr>,
+
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
+}
+
+impl WebSocketOptions {
+    pub fn try_from(x: AnyLuaValue) -> Result<WebSocketOptions> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub enum Event {
+    Text(String),
+    Binary(Vec<u8>),
+    Close,
+    Timeout,
+}
+
+pub struct WebSocket {
+    sock: protocol::WebSocket<Stream>,
+}
+
+impl WebSocket {
+    pub fn negotiate(stream: Stream, url: Url, headers: Option<&HashMap<String, String>>) -> Result<WebSocket> {
+        let mut req = Request::get(url.to_string()); // TODO: don't re-parse here
+
+        if let Some(headers) = headers {
+            for (k, v) in headers {
+                req = req.header(k, v);
+            }
+        }
+
+        let req = req.body(()).unwrap();
+
+        let (sock, _resp) = tungstenite::client::client(req, stream)?;
+        Ok(WebSocket {
+            sock,
+        })
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &WebSocketOptions) -> Result<WebSocket> {
+        let tls = match url.scheme() {
+            "ws" => false,
+            "wss" => true,
+            _ => bail!("Invalid websocket protocol"),
+        };
+
+        let host = url.host_str()
+            .ok_or_else(|| format_err!("Missing host in url"))?;
+
+        let port = match (url.port(), tls) {
+            (Some(port), _) => port,
+            (None, true) => 443,
+            (None, false) => 80,
+        };
+
+        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
+            tls,
+            sni_value: None,
+            disable_tls_verify: false,
+            proxy: options.proxy,
+
+            connect_timeout: options.connect_timeout,
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+        })?;
+        Self::negotiate(stream, url, options.headers.as_ref())
+    }
+
+    pub fn options(&self, options: &WebSocketOptions) -> Result<()> {
+        let mut o = SocketOptions::default();
+        o.read_timeout = options.read_timeout;
+        o.write_timeout = options.write_timeout;
+        o.apply(self.sock.get_ref())
+    }
+
+    fn read_msg(&mut self) -> Result<Event> {
+        loop {
+            let msg = match self.sock.read_message() {
+                Ok(Message::Text(body)) => Event::Text(body),
+                Ok(Message::Binary(body)) => Event::Binary(body),
+                Ok(Message::Ping(ping)) => {
+                    self.sock.write_message(Message::Pong(ping))?;
+                    continue;
+                },
+                Ok(Message::Pong(_)) => continue, // this should never happen
+                Ok(Message::Close(_)) => Event::Close,
+                Err(tungstenite::Error::ConnectionClosed) => Event::Close,
+                Err(tungstenite::Error::AlreadyClosed) => Event::Close,
+                Err(tungstenite::Error::Io(err)) if err.kind() == io::ErrorKind::WouldBlock => Event::Timeout,
+                Err(err) => return Err(err.into()),
+            };
+            return Ok(msg);
+        }
+    }
+
+    pub fn read_text(&mut self) -> Result<Option<String>> {
+        match self.read_msg()? {
+            Event::Text(text) => Ok(Some(text)),
+            Event::Binary(_) => bail!("Unexpected message type: binary"),
+            Event::Close => bail!("Connection closed"),
+            Event::Timeout => Ok(None),
+        }
+    }
+
+    pub fn read_binary(&mut self) -> Result<Option<Vec<u8>>> {
+        match self.read_msg()? {
+            Event::Text(_) => bail!("Unexpected message type: text"),
+            Event::Binary(body) => Ok(Some(body)),
+            Event::Close => bail!("Connection closed"),
+            Event::Timeout => Ok(None),
+        }
+    }
+
+    fn write_msg(&mut self, msg: Message) -> Result<()> {
+        self.sock.write_message(msg)?;
+        self.sock.write_pending()?;
+        Ok(())
+    }
+
+    pub fn write_text(&mut self, text: String) -> Result<()> {
+        self.write_msg(Message::Text(text))
+    }
+
+    pub fn write_binary(&mut self, binary: Vec<u8>) -> Result<()> {
+        self.write_msg(Message::Binary(binary))
+    }
+}

sn0int-std/src/xml.rs

@@ -124,6 +124,7 @@ fn decode_raw(x: &str) -> Result<XmlDocument> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn verify_xml_decode_empty() {

src/args.rs

@@ -48,6 +48,9 @@ pub enum SubCommand {
     /// Search in the registry
     #[structopt(name="search")]
     Search(Search),
+    /// The sn0int package manager
+    #[structopt(name="pkg")]
+    Pkg(cmd::pkg_cmd::Args),
     /// Insert into the database
     #[structopt(name="add")]
     Add(cmd::add_cmd::Args),
@@ -78,6 +81,9 @@ pub enum SubCommand {
     /// Run a lua repl
     #[structopt(name="repl")]
     Repl,
+    /// Show paths of various file system locations
+    #[structopt(name="paths")]
+    Paths,
     /// Generate shell completions
     #[structopt(name="completions")]
     Completions(Completions),
@@ -118,6 +124,9 @@ pub struct Run {
     /// Narrow down targeted entities
     #[structopt(short="t", long="target")]
     pub target: Option<String>,
+    /// Dump the sandbox init message to stdout instead of running a child process
+    #[structopt(long="dump-sandbox-init-msg")]
+    pub dump_sandbox_init_msg: bool,
 }
 
 #[derive(Debug, StructOpt)]

src/auth.rs

@@ -11,7 +11,7 @@ use crate::term;
 
 
 fn path() -> Result<PathBuf> {
-    let path = paths::data_dir()?;
+    let path = paths::sn0int_dir()?;
     Ok(path.join("auth"))
 }
 

src/blobs.rs

@@ -3,44 +3,13 @@ use crate::paths;
 use crate::worker::{EventWithCallback, Event2};
 use crate::workspaces::Workspace;
 
-use blake2::VarBlake2b;
-use digest::{Input, VariableOutput};
 use bytes::Bytes;
-use serde::ser::{Serialize, Serializer};
-use serde::de::{self, Deserialize, Deserializer};
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::mpsc;
 
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct Blob {
-    pub id: String,
-    pub bytes: Bytes,
-}
-
-impl Blob {
-    pub fn create(bytes: Bytes) -> Blob {
-        let id = Self::hash(&bytes);
-        Blob {
-            id,
-            bytes,
-        }
-    }
-
-    pub fn hash(bytes: &[u8]) -> String {
-        let mut h = VarBlake2b::new(32).unwrap();
-        h.input(bytes);
-        Self::encode_hash(&h.vec_result())
-    }
-
-    #[inline]
-    fn encode_hash(bytes: &[u8]) -> String {
-        let x = bs58::encode(bytes).into_string();
-        format!("{:0<44}", x)
-    }
-}
+pub use sn0int_std::blobs::Blob;
 
 impl EventWithCallback for Blob {
     type Payload = ();
@@ -51,30 +20,6 @@ impl EventWithCallback for Blob {
     }
 }
 
-impl Serialize for Blob {
-    #[inline]
-    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let s = base64::encode(&self.bytes);
-        serializer.serialize_str(&s)
-    }
-}
-
-impl<'de> Deserialize<'de> for Blob {
-    #[inline]
-    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let s = String::deserialize(deserializer)?;
-        let bytes = base64::decode(&s)
-            .map_err(de::Error::custom)?;
-        Ok(Blob::create(Bytes::from(bytes)))
-    }
-}
-
 pub struct BlobStorage {
     path: PathBuf,
 }
@@ -155,7 +100,6 @@ impl BlobStorage {
 mod tests {
     use super::*;
     use tempfile;
-    use serde_json;
 
     #[inline]
     fn blob() -> (Bytes, Blob) {
@@ -163,15 +107,6 @@ mod tests {
         (bytes.clone(), Blob::create(bytes))
     }
 
-    #[test]
-    fn verify_create_blob() {
-        let (bytes, blob) = blob();
-        assert_eq!(blob, Blob {
-            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
-            bytes,
-        });
-    }
-
     #[test]
     fn test_blobstorage_save() {
         let dir = tempfile::tempdir().expect("tempdir");
@@ -204,38 +139,4 @@ mod tests {
 
         assert!(result.is_err());
     }
-
-    #[test]
-    fn test_serialize() {
-        let (_, blob) = blob();
-        let json = serde_json::to_string(&blob).unwrap();
-        assert_eq!(&json, "\"YXNkZg==\"");
-    }
-
-    #[test]
-    fn test_deserialize() {
-        let (_, blob1) = blob();
-        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
-        assert_eq!(blob1, blob2);
-    }
-
-    #[test]
-    fn test_hash_encoding() {
-        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
-    }
-
-    #[test]
-    fn test_hash_encoding_padding() {
-        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
-    }
 }

src/cmd/activity_cmd.rs

@@ -4,6 +4,7 @@ use crate::cmd::Cmd;
 use crate::shell::Shell;
 use crate::models::*;
 use chrono::{Utc, NaiveDateTime, NaiveTime, Duration};
+use regex::Regex;
 use std::convert::TryFrom;
 use std::io;
 use std::str::FromStr;
@@ -15,17 +16,36 @@ pub struct TimeSpec {
     datetime: NaiveDateTime,
 }
 
-impl FromStr for TimeSpec {
-    type Err = Error;
-
-    fn from_str(s: &str) -> Result<Self> {
-        let now = Utc::now().naive_utc();
+impl TimeSpec {
+    fn resolve(s: &str, now: NaiveDateTime) -> Result<Self> {
         let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
 
         let datetime = match s {
             "today" => today,
             "yesterday" => today - Duration::days(1),
-            // x {second,minute,hour,day,week,month,year}s? ago
+            s if s.ends_with(" ago") => {
+                let re = Regex::new(r"(\d+) ?(s|seconds?|m|min|minutes?|h|hours?|d|days?|w|weeks?|months?|y|years?) ago").unwrap();
+
+                let caps = re.captures(s)
+                    .ok_or_else(|| format_err!("Couldn't parse TimeSpec"))?;
+
+                let n = caps.get(1).unwrap().as_str()
+                    .parse::<i64>()
+                    .context("Failed to parse number in timespec")?;
+                let unit = caps.get(2).unwrap();
+
+                let duration = match unit.as_str() {
+                    "s" | "second" | "seconds" => Duration::seconds(n),
+                    "m" | "min" | "minute" | "minutes" => Duration::minutes(n),
+                    "h" | "hour" | "hours" => Duration::hours(n),
+                    "d" | "day" | "days" => Duration::days(n),
+                    "w" | "week" | "weeks" => Duration::days(n * 7),
+                    "month" | "months" => Duration::days(n * 31),
+                    "y" | "year" | "years" => Duration::days(n * 365),
+                    _ => unreachable!(),
+                };
+                now - duration
+            },
             s => NaiveDateTime::from_str(s)?,
         };
 
@@ -35,6 +55,15 @@ impl FromStr for TimeSpec {
     }
 }
 
+impl FromStr for TimeSpec {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        let now = Utc::now().naive_utc();
+        Self::resolve(s, now)
+    }
+}
+
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
@@ -86,3 +115,63 @@ impl Cmd for Args {
         Ok(())
     }
 }
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn datetime() -> NaiveDateTime {
+        let date = chrono::NaiveDate::from_ymd(2020, 3, 14);
+        let time = chrono::NaiveTime::from_hms(16, 20, 23);
+        NaiveDateTime::new(date, time)
+    }
+
+    #[test]
+    fn test_today() {
+        let x = TimeSpec::resolve("today", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_yesterday() {
+        let x = TimeSpec::resolve("yesterday", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-13T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_20_min_ago() {
+        let x = TimeSpec::resolve("20min ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:00:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_days_ago() {
+        let x = TimeSpec::resolve("3 days ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-11T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_week_ago() {
+        let x = TimeSpec::resolve("1w ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-07T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_months_ago() {
+        let x = TimeSpec::resolve("3 months ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-12-12T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_year_ago() {
+        let x = TimeSpec::resolve("1 year ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-03-15T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_exact_time() {
+        let x = TimeSpec::resolve("2020-03-14T16:20:23", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:20:23").unwrap());
+    }
+}

src/cmd/help_cmd.rs

@@ -16,9 +16,8 @@ pub fn run(_rl: &mut Shell, _args: &[String]) -> Result<()> {
     help("autoscope",   "Manage rules to automatically add entities to scope");
     help("delete",      "Delete entities from the database");
     help("keyring",     "Manage saved credentials");
-    help("mod",         "Manage installed modules");
+    help("pkg",         "Manage installed modules");
     help("noscope",     "Exclude entities from scope");
-    help("quickstart",  "Install all featured modules");
     help("run",         "Run the currently selected module");
     help("scope",       "Include entities in the scope again");
     help("select",      "Select entities from the database");

src/cmd/mod.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 use crate::shell::Shell;
+use crate::config::Config;
 
 pub trait Cmd: structopt::StructOpt + Sized {
     fn run(self, rl: &mut Shell) -> Result<()>;
@@ -11,6 +12,10 @@ pub trait Cmd: structopt::StructOpt + Sized {
     }
 }
 
+pub trait LiteCmd: structopt::StructOpt + Sized {
+    fn run(self, config: &Config) -> Result<()>;
+}
+
 pub mod activity_cmd;
 pub mod add_cmd;
 pub mod autonoscope_cmd;
@@ -23,8 +28,8 @@ pub mod run_cmd;
 pub mod use_cmd;
 pub mod select_cmd;
 pub mod keyring_cmd;
-pub mod mod_cmd;
 pub mod noscope_cmd;
+pub mod pkg_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;
 pub mod target_cmd;

src/cmd/pkg_cmd.rs

@@ -1,7 +1,12 @@
 use crate::errors::*;
 
+use crate::args::Install;
+use crate::api::Client;
 use crate::args;
-use crate::registry::{self, UpdateTask, Updater};
+use crate::config::Config;
+use crate::cmd::{Cmd, LiteCmd};
+use crate::engine::Library;
+use crate::registry::{self, InstallTask, UpdateTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
 use crate::worker;
@@ -20,6 +25,13 @@ pub struct Args {
     pub subcommand: SubCommand,
 }
 
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct ArgsInteractive {
+    #[structopt(subcommand)]
+    pub subcommand: SubCommandInteractive,
+}
+
 #[derive(Debug, StructOpt)]
 pub enum SubCommand {
     /// List installed modules
@@ -31,15 +43,24 @@ pub enum SubCommand {
     /// Search modules in registry
     #[structopt(name="search")]
     Search(args::Search),
-    /// Reload modules
-    #[structopt(name="reload")]
-    Reload(Reload),
     /// Update modules
     #[structopt(name="update")]
     Update(Update),
     /// Uninstall a module
     #[structopt(name="uninstall")]
     Uninstall(Uninstall),
+    /// Install all featured modules
+    #[structopt(name="quickstart")]
+    Quickstart,
+}
+
+#[derive(Debug, StructOpt)]
+pub enum SubCommandInteractive {
+    #[structopt(flatten)]
+    Base(SubCommand),
+    /// Reload modules
+    #[structopt(name="reload")]
+    Reload(Reload),
 }
 
 #[derive(Debug, StructOpt)]
@@ -65,15 +86,18 @@ pub struct Uninstall {
     module: ModuleID,
 }
 
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    let config = rl.config().clone();
+#[derive(PartialEq)]
+enum ModuleReload {
+    Yes,
+    No,
+}
 
-    match args.subcommand {
+fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) -> Result<ModuleReload> {
+    match subcommand {
         SubCommand::List(list) => {
             let autoupdate = AutoUpdater::load()?;
 
-            for module in rl.engine().list() {
+            for module in library.list() {
                 if let Some(source) = &list.source {
                     if !module.source_equals(&source) {
                         continue;
@@ -93,31 +117,22 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
                 println!("{}", out);
                 println!("\t{}", module.description());
             }
+            Ok(ModuleReload::No)
         },
         SubCommand::Install(install) => {
             registry::run_install(install, &config)?;
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
-        SubCommand::Search(search) => registry::run_search(rl.engine(), &search, &config)?,
-        SubCommand::Reload(_) => {
-            let current = rl.take_module()
-                            .map(|m| m.canonical());
-
-            rl.engine_mut().reload_modules()?;
-            rl.reload_module_cache();
-
-            if let Some(module) = current {
-                if let Ok(module) = rl.engine().get(&module).map(|x| x.to_owned()) {
-                    rl.set_module(module);
-                }
-            }
+        SubCommand::Search(search) => {
+            registry::run_search(library, &search, &config)?;
+            Ok(ModuleReload::No)
         },
         SubCommand::Update(_) => {
             let mut autoupdate = AutoUpdater::load()?;
             let updater = Arc::new(Updater::new(&config)?);
 
-            let modules = rl.engine().list()
+            let modules = library.list()
                 .into_iter()
                 .filter_map(|module| {
                     let canonical = module.canonical();
@@ -138,15 +153,62 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
             autoupdate.save()?;
 
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
         SubCommand::Uninstall(uninstall) => {
             let updater = Updater::new(&config)?;
             updater.uninstall(&uninstall.module)?;
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
+        },
+        SubCommand::Quickstart => {
+            let client = Client::new(&config)?;
+            let updater = Arc::new(Updater::new(&config)?);
+            let mut autoupdate = AutoUpdater::load()?;
+
+            let modules = client.quickstart()?
+                .into_iter()
+                .map(|module| {
+                    InstallTask::new(Install {
+                        module: ModuleID {
+                            author: module.author,
+                            name: module.name,
+                        },
+                        version: None,
+                        force: false,
+                    }, updater.clone())
+                })
+                .collect::<Vec<_>>();
+
+            worker::spawn_multi(modules, |name| {
+                autoupdate.updated(&name);
+            }, 3)?;
+
+            autoupdate.save()?;
+
+            // trigger reload
+            Ok(ModuleReload::Yes)
         },
     }
-
-    Ok(())
+}
+
+impl LiteCmd for Args {
+    fn run(self, config: &Config) -> Result<()> {
+        let library = Library::new(false, &config)?;
+        run_subcommand(self.subcommand, &library, config)?;
+        Ok(())
+    }
+}
+
+impl Cmd for ArgsInteractive {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let action = match self.subcommand {
+            SubCommandInteractive::Base(subcommand) => run_subcommand(subcommand, rl.library(), rl.config())?,
+            SubCommandInteractive::Reload(_) => ModuleReload::Yes,
+        };
+        if action == ModuleReload::Yes {
+            rl.reload_modules()?;
+        }
+        Ok(())
+    }
 }

src/cmd/quickstart_cmd.rs

@@ -1,16 +1,11 @@
 use crate::errors::*;
 
-use crate::args::Install;
-use crate::api::Client;
-use crate::cmd::mod_cmd;
-use crate::registry::{InstallTask, Updater};
+use crate::cmd::Cmd;
+use crate::cmd::pkg_cmd::{ArgsInteractive as PkgArgs, SubCommand, SubCommandInteractive};
 use crate::shell::Shell;
-use crate::update::AutoUpdater;
-use crate::worker;
-use std::sync::Arc;
+use crate::term;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
-use sn0int_common::ModuleID;
 
 
 #[derive(Debug, StructOpt)]
@@ -20,34 +15,11 @@ pub struct Args {
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     let _args = Args::from_iter_safe(args)?;
-    let config = rl.config().clone();
 
-    let client = Client::new(&config)?;
-    let updater = Arc::new(Updater::new(&config)?);
-    let mut autoupdate = AutoUpdater::load()?;
+    term::warn("The \x1b[1mquickstart\x1b[0m command is deprecated, use \x1b[1mpkg quickstart\x1b[0m");
 
-    let modules = client.quickstart()?
-        .into_iter()
-        .map(|module| {
-            InstallTask::new(Install {
-                module: ModuleID {
-                    author: module.author,
-                    name: module.name,
-                },
-                version: None,
-                force: false,
-            }, updater.clone())
-        })
-        .collect::<Vec<_>>();
-
-    worker::spawn_multi(modules, |name| {
-        autoupdate.updated(&name);
-    }, 3)?;
-
-    autoupdate.save()?;
-
-    // trigger reload
-    mod_cmd::run(rl, &[String::from("mod"), String::from("reload")])?;
-
-    Ok(())
+    let args = PkgArgs {
+        subcommand: SubCommandInteractive::Base(SubCommand::Quickstart),
+    };
+    args.run(rl)
 }

src/cmd/run_cmd.rs

@@ -1,10 +1,12 @@
 use crate::errors::*;
 
+use chrootable_https::dns::Resolver;
 use crate::args;
 use crate::blobs::{Blob, BlobStorage};
 use crate::cmd::Cmd;
 use crate::db::{ttl, Filter};
 use crate::engine::Module;
+use crate::ipc::common::StartCommand;
 use crate::models::*;
 use crate::shell::Shell;
 use crate::keyring::KeyRing;
@@ -113,16 +115,10 @@ fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> R
     Ok(())
 }
 
-pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
-
-    prepare_keyring(rl.keyring_mut(), &module, &params)?;
-
+fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, Option<String>, Vec<Blob>)>> {
     let filter = rl.scoped_targets();
 
-    let args = match module.source() {
+    match module.source() {
         Some(Source::Domains) => prepare_args::<Domain>(rl, &filter, None),
         Some(Source::Subdomains) => prepare_args::<Subdomain>(rl, &filter, None),
         Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl, &filter, None),
@@ -152,7 +148,44 @@ pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>)
             }
         },
         None => Ok(vec![(serde_json::Value::Null, None, vec![])]),
-    }?;
+    }
+}
+
+pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let keyring = rl.keyring().request_keys(&module);
+
+    let dns_config = Resolver::from_system()?;
+    let proxy = rl.config().network.proxy.clone();
+
+    let args = get_args(rl, &module)?;
+    for (arg, _pretty_arg, blobs) in args {
+        let start_cmd = StartCommand::new(params.verbose,
+                                          keyring.clone(),
+                                          dns_config.clone(),
+                                          proxy.clone(),
+                                          options.clone(),
+                                          module.clone(),
+                                          arg,
+                                          blobs);
+        let out = serde_json::to_string(&start_cmd)?;
+        println!("{}", out);
+    }
+
+    Ok(())
+}
+
+pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
     let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);

src/cmd/use_cmd.rs

@@ -14,7 +14,7 @@ pub struct Args {
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
 
-    let module = rl.engine().get(&args.module)?.clone();
+    let module = rl.library().get(&args.module)?.clone();
     rl.set_module(module);
 
     Ok(())

src/cmd/workspace_cmd.rs

@@ -1,7 +1,8 @@
 use crate::errors::*;
 
 use crate::blobs::BlobStorage;
-use crate::cmd::Cmd;
+use crate::cmd::{Cmd, LiteCmd};
+use crate::config::Config;
 use crate::db::Database;
 use crate::shell::Shell;
 use crate::term;
@@ -52,6 +53,8 @@ fn usage(workspace: Option<Workspace>) -> Result<()> {
 }
 
 fn change(rl: &mut Shell, workspace: Workspace) -> Result<()> {
+    workspace.migrate()?;
+
     let blobs = BlobStorage::workspace(&workspace)?;
     let db = Database::establish(workspace)?;
     rl.set_blobstorage(blobs);
@@ -66,29 +69,42 @@ fn list() -> Result<()> {
     Ok(())
 }
 
-impl Cmd for Args {
-    fn run(self, rl: &mut Shell) -> Result<()> {
-        if self.delete {
-            if let Some(workspace) = self.workspace {
+fn run(args: Args, rl: Option<&mut Shell>) -> Result<()> {
+    if args.delete {
+        if let Some(workspace) = args.workspace {
+            if let Some(rl) = rl {
                 if *rl.db().workspace() == workspace {
                     bail!("Can't delete current workspace")
                 }
-
-                delete(workspace, self.force)
-            } else {
-                bail!("--delete requires workspace")
             }
-        } else if self.usage {
-            usage(self.workspace)
-        } else if let Some(workspace) = self.workspace {
+
+            delete(workspace, args.force)
+        } else {
+            bail!("--delete requires workspace")
+        }
+    } else if args.usage {
+        usage(args.workspace)
+    } else if let Some(workspace) = args.workspace {
+        if let Some(rl) = rl {
             change(rl, workspace)
         } else {
-            list()
+            Ok(())
         }
+    } else {
+        list()
     }
 }
 
-#[inline]
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    Args::run_str(rl, args)
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        run(self, Some(rl))
+    }
+}
+
+impl LiteCmd for Args {
+    #[inline]
+    fn run(self, _config: &Config) -> Result<()> {
+        run(self, None)
+    }
 }

src/db/mod.rs

@@ -121,6 +121,8 @@ impl Database {
             .context("Failed to enable write ahead log")?;
         db.execute("PRAGMA foreign_keys = ON")
             .context("Failed to enforce foreign keys")?;
+        db.execute("PRAGMA synchronous = NORMAL")
+            .context("Failed to enforce foreign keys")?;
 
         let autonoscope = RuleSet::load(&db)?;
 
@@ -302,7 +304,7 @@ impl Database {
         Ok(ipaddr_update.id)
     }
 
-    pub fn update_url(&self, url_update: &UrlUpdate) -> Result<i32> {
+    pub fn update_url(&self, url_update: &UrlChangeset) -> Result<i32> {
         use crate::schema::urls::columns::*;
         diesel::update(urls::table.filter(id.eq(url_update.id)))
             .set(url_update)

src/engine/ctx.rs

@@ -1,8 +1,7 @@
 use crate::errors::*;
 
-use crate::blobs::Blob;
 use crate::db::Family;
-use crate::engine::{Environment, Reporter};
+use crate::engine::{Environment, IpcChild};
 use crate::geoip::{MaxmindReader, GeoIP, AsnDB};
 use crate::hlua::{self, AnyLuaValue};
 use crate::keyring::KeyRingEntry;
@@ -12,10 +11,14 @@ use crate::lazy::Lazy;
 use crate::runtime;
 use crate::sockets::{Socket, SocketOptions, TlsData};
 use crate::web::{HttpSession, HttpRequest, RequestOptions};
-use crate::worker::{Event, LogEvent, DatabaseEvent, StdioEvent, RatelimitEvent};
+use crate::websockets::{WebSocket, WebSocketOptions};
+use crate::worker::{Event, LogEvent, DatabaseEvent, DatabaseResponse, StdioEvent, RatelimitEvent};
 use crate::ratelimits::RatelimitResponse;
 use chrootable_https::{self, Resolver};
 use serde_json;
+use sn0int_std::blobs::{Blob, BlobState};
+use sn0int_std::mqtt::{MqttClient, MqttOptions};
+use sn0int_std::web::WebState;
 use std::collections::HashMap;
 use std::result;
 use std::net::SocketAddr;
@@ -70,48 +73,52 @@ pub trait State {
         self.send(&Event::Log(LogEvent::Status(msg)))
     }
 
-    fn db_insert(&self, object: Insert) -> Result<Option<i32>> {
+    fn db_recv(&self) -> Result<DatabaseResponse> {
+        let reply = self.recv()?;
+        let reply: result::Result<DatabaseResponse, String> = serde_json::from_value(reply)?;
+        reply.map_err(|err| format_err!("Database error: {:?}", err))
+    }
+
+    fn db_insert(&self, object: Insert) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Insert(object)));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        self.db_recv()
+            .context("Failed to add to database")
+            .map_err(Error::from)
     }
 
-    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<Option<i32>> {
+    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::InsertTtl((object, ttl))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        self.db_recv()
+            .context("Failed to add to database")
+            .map_err(Error::from)
     }
 
-    fn db_activity(&self, activity: InsertActivity) -> Result<()> {
+    fn db_activity(&self, activity: InsertActivity) -> Result<bool> {
         let activity = activity.try_into_new()?;
 
         self.send(&Event::Database(DatabaseEvent::Activity(activity)));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
+        let r = self.db_recv()
+            .context("Failed to log activity")?;
 
-        reply
-            .map(|_| ())
-            .map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        match r {
+            DatabaseResponse::Inserted(_) => Ok(false),
+            DatabaseResponse::NoChange(_) => Ok(true),
+            _ => bail!("Unexpected database response for db_activity: {:?}", r),
+        }
     }
 
-    fn db_select(&self, family: Family, value: String) -> Result<Option<i32>> {
+    fn db_select(&self, family: Family, value: String) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Select((family, value))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to query database: {:?}", err))
+        self.db_recv()
+            .context("Failed to query database")
+            .map_err(Error::from)
     }
 
-    fn db_update(&self, object: String, update: Update) -> Result<Option<i32>> {
+    fn db_update(&self, object: String, update: Update) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Update((object, update))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to update database: {:?}", err))
+        self.db_recv()
+            .context("Failed to update database")
+            .map_err(Error::from)
     }
 
     fn stdin_read_line(&self) -> Result<Option<String>> {
@@ -168,16 +175,18 @@ pub trait State {
 
     fn sock_upgrade_tls(&self, id: &str, options: &SocketOptions) -> Result<TlsData>;
 
-    fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>>;
+    fn ws_connect(&self, url: url::Url, options: &WebSocketOptions) -> Result<String>;
+
+    fn get_ws(&self, id: &str)-> Arc<Mutex<WebSocket>>;
+
+    fn mqtt_connect(&self, url: url::Url, options: &MqttOptions) -> Result<String>;
+
+    fn get_mqtt(&self, id: &str)-> Arc<Mutex<MqttClient>>;
 
     fn http_mksession(&self) -> String;
 
     fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest;
 
-    fn register_in_jar(&self, session: &str, key: String, value: String);
-
-    fn register_blob(&self, blob: Blob) -> String;
-
     fn get_blob(&self, id: &str) -> Result<Arc<Blob>>;
 
     fn persist_blob(&self, id: &str) -> Result<()> {
@@ -192,8 +201,10 @@ pub trait State {
 // #[derive(Debug)]
 pub struct LuaState {
     error: Mutex<Option<Error>>,
-    logger: Arc<Mutex<Box<dyn Reporter>>>,
+    logger: Arc<Mutex<Box<dyn IpcChild>>>,
     socket_sessions: Mutex<HashMap<String, Arc<Mutex<Socket>>>>,
+    ws_sessions: Mutex<HashMap<String, Arc<Mutex<WebSocket>>>>,
+    mqtt_sessions: Mutex<HashMap<String, Arc<Mutex<MqttClient>>>>,
     blobs: Mutex<HashMap<String, Arc<Blob>>>,
     http_sessions: Mutex<HashMap<String, HttpSession>>,
     http_clients: Mutex<HashMap<String, Arc<chrootable_https::Client<Resolver>>>>,
@@ -301,13 +312,13 @@ impl State for LuaState {
 
     fn get_sock(&self, id: &str)-> Arc<Mutex<Socket>> {
         let mtx = self.socket_sessions.lock().unwrap();
-        let sock = mtx.get(id).expect("invalid session reference"); // TODO
+        let sock = mtx.get(id).expect("Invalid socket reference"); // TODO
         sock.clone()
     }
 
     fn sock_upgrade_tls(&self, id: &str, options: &SocketOptions) -> Result<TlsData> {
         let mut mtx = self.socket_sessions.lock().unwrap();
-        let sock = mtx.remove(id).expect("invalid session reference"); // TODO
+        let sock = mtx.remove(id).expect("Invalid socket reference"); // TODO
 
         let sock = Arc::try_unwrap(sock).unwrap();
         let sock = sock.into_inner().unwrap();
@@ -319,6 +330,61 @@ impl State for LuaState {
         Ok(tls)
     }
 
+    fn ws_connect(&self, url: url::Url, options: &WebSocketOptions) -> Result<String> {
+        let mut mtx = self.ws_sessions.lock().unwrap();
+        let id = self.random_id();
+
+        let sock = WebSocket::connect(&self.dns_config, url, options)?;
+        mtx.insert(id.clone(), Arc::new(Mutex::new(sock)));
+
+        Ok(id)
+    }
+
+    fn get_ws(&self, id: &str)-> Arc<Mutex<WebSocket>> {
+        let mtx = self.ws_sessions.lock().unwrap();
+        let sock = mtx.get(id).expect("Invalid ws reference"); // TODO
+        sock.clone()
+    }
+
+    fn mqtt_connect(&self, url: url::Url, options: &MqttOptions) -> Result<String> {
+        let mut mtx = self.mqtt_sessions.lock().unwrap();
+        let id = self.random_id();
+
+        let sock = MqttClient::connect(&self.dns_config, url, options)?;
+        mtx.insert(id.clone(), Arc::new(Mutex::new(sock)));
+
+        Ok(id)
+    }
+
+    fn get_mqtt(&self, id: &str)-> Arc<Mutex<MqttClient>> {
+        let mtx = self.mqtt_sessions.lock().unwrap();
+        let sock = mtx.get(id).expect("Invalid mqtt reference"); // TODO
+        sock.clone()
+    }
+
+    fn http_mksession(&self) -> String {
+        let mut mtx = self.http_sessions.lock().unwrap();
+        let (id, session) = HttpSession::new();
+        mtx.insert(id.clone(), session);
+        id
+    }
+
+    fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest {
+        let mtx = self.http_sessions.lock().unwrap();
+        let session = mtx.get(session_id).expect("Invalid session reference"); // TODO
+
+        HttpRequest::new(&session, method, url, options)
+    }
+
+    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
+        let mtx = self.blobs.lock().unwrap();
+        let blob = mtx.get(id)
+            .ok_or_else(|| format_err!("Invalid blob reference"))?;
+        Ok(blob.clone())
+    }
+}
+
+impl WebState for LuaState {
     fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>> {
         let proxy = self.resolve_proxy_options(proxy)?;
 
@@ -345,27 +411,15 @@ impl State for LuaState {
         }
     }
 
-    fn http_mksession(&self) -> String {
-        let mut mtx = self.http_sessions.lock().unwrap();
-        let (id, session) = HttpSession::new();
-        mtx.insert(id.clone(), session);
-        id
-    }
-
-    fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest {
-        let mtx = self.http_sessions.lock().unwrap();
-        let session = mtx.get(session_id).expect("invalid session reference"); // TODO
-
-        HttpRequest::new(&session, method, url, options)
-    }
-
     fn register_in_jar(&self, session: &str, key: String, value: String) {
         let mut mtx = self.http_sessions.lock().unwrap();
         if let Some(session) = mtx.get_mut(session) {
             session.cookies.register_in_jar(key, value);
         }
     }
+}
 
+impl BlobState for LuaState {
     fn register_blob(&self, blob: Blob) -> String {
         let id = blob.id.clone();
 
@@ -375,13 +429,6 @@ impl State for LuaState {
 
         id
     }
-
-    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
-        let mtx = self.blobs.lock().unwrap();
-        let blob = mtx.get(id)
-            .ok_or_else(|| format_err!("Invalid blob reference"))?;
-        Ok(blob.clone())
-    }
 }
 
 impl LuaState {
@@ -401,7 +448,7 @@ pub struct Script {
     code: String,
 }
 
-pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
+pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
     debug!("Creating lua context");
     let mut lua = hlua::Lua::new();
     lua.open_string();
@@ -413,6 +460,8 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
         error: Mutex::new(None),
         logger,
         socket_sessions: Mutex::new(HashMap::new()),
+        ws_sessions: Mutex::new(HashMap::new()),
+        mqtt_sessions: Mutex::new(HashMap::new()),
         blobs: Mutex::new(HashMap::new()),
         http_sessions: Mutex::new(HashMap::new()),
         http_clients: Mutex::new(HashMap::new()),
@@ -450,6 +499,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::debug(&mut lua, state.clone());
     runtime::dns(&mut lua, state.clone());
     runtime::error(&mut lua, state.clone());
+    runtime::geo_polygon_contains(&mut lua, state.clone());
     runtime::geoip_lookup(&mut lua, state.clone());
     runtime::getopt(&mut lua, state.clone());
     runtime::hex(&mut lua, state.clone());
@@ -474,9 +524,14 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::json_decode(&mut lua, state.clone());
     runtime::json_decode_stream(&mut lua, state.clone());
     runtime::json_encode(&mut lua, state.clone());
+    runtime::key_trunc_pad(&mut lua, state.clone());
     runtime::keyring(&mut lua, state.clone());
     runtime::last_err(&mut lua, state.clone());
     runtime::md5(&mut lua, state.clone());
+    runtime::mqtt_connect(&mut lua, state.clone());
+    runtime::mqtt_subscribe(&mut lua, state.clone());
+    runtime::mqtt_recv(&mut lua, state.clone());
+    runtime::mqtt_ping(&mut lua, state.clone());
     runtime::pgp_pubkey(&mut lua, state.clone());
     runtime::pgp_pubkey_armored(&mut lua, state.clone());
     runtime::print(&mut lua, state.clone());
@@ -497,6 +552,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::sn0int_version(&mut lua, state.clone());
     runtime::sock_connect(&mut lua, state.clone());
     runtime::sock_upgrade_tls(&mut lua, state.clone());
+    runtime::sock_options(&mut lua, state.clone());
     runtime::sock_send(&mut lua, state.clone());
     runtime::sock_recv(&mut lua, state.clone());
     runtime::sock_sendline(&mut lua, state.clone());
@@ -508,9 +564,13 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::sock_recvuntil(&mut lua, state.clone());
     runtime::sock_sendafter(&mut lua, state.clone());
     runtime::sock_newline(&mut lua, state.clone());
+    runtime::sodium_secretbox_open(&mut lua, state.clone());
     runtime::status(&mut lua, state.clone());
     runtime::stdin_read_line(&mut lua, state.clone());
     runtime::stdin_read_to_end(&mut lua, state.clone());
+    runtime::str_find(&mut lua, state.clone());
+    runtime::str_replace(&mut lua, state.clone());
+    runtime::strval(&mut lua, state.clone());
     runtime::strftime(&mut lua, state.clone());
     runtime::strptime(&mut lua, state.clone());
     runtime::time_unix(&mut lua, state.clone());
@@ -523,6 +583,14 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::utf8_decode(&mut lua, state.clone());
     runtime::warn(&mut lua, state.clone());
     runtime::warn_once(&mut lua, state.clone());
+    runtime::ws_connect(&mut lua, state.clone());
+    runtime::ws_options(&mut lua, state.clone());
+    runtime::ws_recv_text(&mut lua, state.clone());
+    runtime::ws_recv_binary(&mut lua, state.clone());
+    runtime::ws_recv_json(&mut lua, state.clone());
+    runtime::ws_send_text(&mut lua, state.clone());
+    runtime::ws_send_binary(&mut lua, state.clone());
+    runtime::ws_send_json(&mut lua, state.clone());
     runtime::x509_parse_pem(&mut lua, state.clone());
     runtime::xml_decode(&mut lua, state.clone());
     runtime::xml_named(&mut lua, state.clone());
@@ -553,7 +621,7 @@ impl Script {
     }
 
     pub fn run(&self, env: Environment,
-                      tx: Arc<Mutex<Box<dyn Reporter>>>,
+                      tx: Arc<Mutex<Box<dyn IpcChild>>>,
                       arg: AnyLuaValue,
     ) -> Result<()> {
         let (mut lua, state) = ctx(env, tx);
@@ -584,8 +652,9 @@ impl Script {
 
     #[cfg(test)]
     pub fn test(&self) -> Result<()> {
-        use crate::engine::DummyReporter;
+        use crate::ipc::child::DummyIpcChild;
         use crate::geoip::Maxmind;
+        use crate::paths;
         let keyring = Vec::new();
         let dns_config = Resolver::from_system()?;
         let proxy = None;
@@ -597,8 +666,9 @@ com
 a.prod.fastly.net
 // ===END PRIVATE DOMAINS===
 "#.into());
-        let geoip = GeoIP::try_open_reader()?;
-        let asn = AsnDB::try_open_reader()?;
+        let cache_dir = paths::cache_dir()?;
+        let geoip = GeoIP::try_open_reader(&cache_dir)?;
+        let asn = AsnDB::try_open_reader(&cache_dir)?;
 
         let env = Environment {
             verbose: 0,
@@ -611,6 +681,6 @@ a.prod.fastly.net
             geoip,
             asn,
         };
-        self.run(env, DummyReporter::new(), AnyLuaValue::LuaNil)
+        self.run(env, DummyIpcChild::new(), AnyLuaValue::LuaNil)
     }
 }

src/engine/isolation.rs

@@ -1,255 +0,0 @@
-use crate::errors::*;
-use chrootable_https::dns::Resolver;
-use crate::blobs::Blob;
-use crate::engine::{Environment, Module, Reporter};
-use crate::geoip::MaxmindReader;
-use crate::keyring::KeyRingEntry;
-use crate::psl::PslReader;
-use serde_json;
-use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
-
-use std::collections::HashMap;
-use std::env;
-use std::ffi::OsString;
-use std::io::prelude::*;
-use std::io::{self, BufReader, BufRead, stdin, Stdin, Stdout};
-use std::net::SocketAddr;
-use std::sync::{mpsc, Arc, Mutex};
-use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
-
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct StartCommand {
-    verbose: u64,
-    keyring: Vec<KeyRingEntry>,
-    dns_config: Resolver,
-    proxy: Option<SocketAddr>,
-    options: HashMap<String, String>,
-    module: Module,
-    arg: serde_json::Value,
-    blobs: Vec<Blob>,
-}
-
-impl StartCommand {
-    pub fn new(verbose: u64,
-               keyring: Vec<KeyRingEntry>,
-               dns_config: Resolver,
-               proxy: Option<SocketAddr>,
-               options: HashMap<String, String>,
-               module: Module,
-               arg: serde_json::Value,
-               blobs: Vec<Blob>,
-    ) -> StartCommand {
-        StartCommand {
-            verbose,
-            keyring,
-            dns_config,
-            proxy,
-            options,
-            module,
-            arg,
-            blobs,
-        }
-    }
-}
-
-pub struct Supervisor {
-    child: Child,
-    stdin: ChildStdin,
-    stdout: BufReader<ChildStdout>,
-}
-
-impl Supervisor {
-    pub fn setup(module: &Module) -> Result<Supervisor> {
-        let exe = match env::current_exe() {
-            Ok(exe) => exe.into_os_string(),
-            _ => OsString::from("sn0int"),
-        };
-
-        let mut child = Command::new(exe)
-            .arg("sandbox")
-            .arg(&module.canonical())
-            .stdin(Stdio::piped())
-            .stdout(Stdio::piped())
-            .spawn()
-            .context("Failed to spawn child process")?;
-
-        let stdin = child.stdin.take().expect("Failed to take child stdin");
-        let stdout = child.stdout.take().expect("Failed to take child stdout");
-        let stdout = BufReader::new(stdout);
-
-        Ok(Supervisor {
-            child,
-            stdin,
-            stdout,
-        })
-    }
-
-    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
-        let start = serde_json::to_value(&start)?;
-        self.send(&start)?;
-        Ok(())
-    }
-
-    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
-        let mut value = serde_json::to_string(value)?;
-        value.push('\n');
-        self.stdin.write_all(value.as_bytes())?;
-        debug!("Supervisor sent: {:?}", value);
-        Ok(())
-    }
-
-    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
-        let value = serde_json::to_value(value).expect("Failed to serialize reply");
-        if let Err(_) = self.send(&value) {
-            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
-        }
-    }
-
-    pub fn recv(&mut self) -> Result<Event> {
-        let mut line = String::new();
-        let len = self.stdout.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Supervisor received: {:?}", event);
-        Ok(event)
-    }
-
-    pub fn wait(&mut self) -> Result<()> {
-        let exit = self.child.wait()
-            .context("Failed to wait for child")?;
-
-        if exit.success() {
-            Ok(())
-        } else {
-            bail!("Child signaled error")
-        }
-    }
-
-    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
-        where <T as EventWithCallback>::Payload: serde::Serialize
-    {
-        let (tx2, rx2) = mpsc::channel();
-        tx.send(event.with_callback(tx2));
-        let reply = rx2.recv().unwrap();
-
-        self.send_struct(reply, tx);
-    }
-}
-
-#[derive(Debug)]
-pub struct StdioReporter {
-    stdin: Stdin,
-    stdout: Stdout,
-}
-
-impl StdioReporter {
-    pub fn setup() -> StdioReporter {
-        let stdin = io::stdin();
-        let stdout = io::stdout();
-
-        StdioReporter {
-            stdin,
-            stdout,
-        }
-    }
-
-    pub fn recv_start(&mut self) -> Result<StartCommand> {
-        let value = self.recv()?;
-        let event = serde_json::from_value(value)?;
-        Ok(event)
-    }
-}
-
-impl Reporter for StdioReporter {
-    fn send(&mut self, event: &Event) -> Result<()> {
-        let mut event = serde_json::to_string(&event)?;
-        event.push('\n');
-        self.stdout.write_all(event.as_bytes())?;
-        debug!("Reporter sent: {:?}", event);
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        let mut line = String::new();
-        let len = self.stdin.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Reporter received: {:?}", event);
-        Ok(event)
-    }
-}
-
-pub fn spawn_module(module: Module,
-                    tx: &EventSender,
-                    arg: serde_json::Value,
-                    keyring: Vec<KeyRingEntry>,
-                    verbose: u64,
-                    has_stdin: bool,
-                    proxy: Option<SocketAddr>,
-                    options: HashMap<String, String>,
-                    blobs: Vec<Blob>,
-) -> Result<ExitEvent> {
-    let dns_config = Resolver::from_system()?;
-
-    let mut reader = if has_stdin {
-        Some(BufReader::new(stdin()))
-    } else {
-        None
-    };
-
-    let mut supervisor = Supervisor::setup(&module)?;
-    supervisor.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
-
-    let exit = loop {
-        match supervisor.recv()? {
-            Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => supervisor.send_event_callback(object, &tx),
-            Event::Stdio(object) => object.apply(&mut supervisor, tx, &mut reader),
-            Event::Ratelimit(req) => supervisor.send_event_callback(req, &tx),
-            Event::Blob(blob) => supervisor.send_event_callback(blob, &tx),
-            Event::Exit(event) => {
-                if let ExitEvent::Err(err) = &event {
-                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
-                }
-                break event;
-            },
-        }
-    };
-
-    supervisor.wait()?;
-
-    Ok(exit)
-}
-
-pub fn run_worker(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslReader) -> Result<()> {
-    let mut reporter = StdioReporter::setup();
-    let start = reporter.recv_start()?;
-
-    let environment = Environment {
-        verbose: start.verbose,
-        keyring: start.keyring,
-        dns_config: start.dns_config,
-        proxy: start.proxy,
-        options: start.options,
-        blobs: start.blobs,
-        psl,
-        geoip,
-        asn,
-    };
-
-    let mtx: Arc<Mutex<Box<dyn Reporter>>> = Arc::new(Mutex::new(Box::new(reporter)));
-    let result = start.module.run(environment,
-                                  mtx.clone(),
-                                  start.arg.into());
-    let mut reporter = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
-                        .into_inner().expect("Failed to consume Mutex");
-
-    let event = match result {
-        Ok(_) => ExitEvent::Ok,
-        Err(err) => ExitEvent::Err(err.to_string()),
-    };
-    reporter.send(&Event::Exit(event))?;
-
-    Ok(())
-}

src/engine/mod.rs

@@ -5,7 +5,6 @@ use crate::config::Config;
 use crate::geoip::MaxmindReader;
 use crate::json::LuaJsonValue;
 use crate::keyring::KeyRingEntry;
-use serde_json;
 use std::fs;
 use std::fmt::Debug;
 use std::path::PathBuf;
@@ -13,6 +12,7 @@ use std::collections::HashMap;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
 use crate::engine::ctx::Script;
+use crate::ipc::child::IpcChild;
 use sn0int_common::ModuleID;
 use sn0int_common::metadata::{Metadata, Source};
 use chrootable_https::dns::Resolver;
@@ -21,11 +21,10 @@ use crate::paths;
 use std::cmp::Ordering;
 use std::path::Path;
 use crate::term;
-use crate::worker::{self, Event};
+use crate::worker;
 
 pub mod ctx;
-pub mod isolation;
-pub mod structs;
+pub use sn0int_std::engine::structs;
 
 
 /// Data that is passed to every script
@@ -43,29 +42,29 @@ pub struct Environment {
 }
 
 #[derive(Debug)]
-pub struct Engine<'a> {
+pub struct Library<'a> {
     path: PathBuf,
     modules: HashMap<String, Vec<Module>>,
     config: &'a Config
 }
 
-impl<'a> Engine<'a> {
-    pub fn new(verbose_init: bool, config: &'a Config) -> Result<Engine> {
+impl<'a> Library<'a> {
+    pub fn new(verbose_init: bool, config: &'a Config) -> Result<Library> {
         let path = paths::module_dir()?;
 
-        let mut engine = Engine {
+        let mut library = Library {
             path,
             modules: HashMap::new(),
             config,
         };
 
         if verbose_init {
-            engine.reload_modules()?;
+            library.reload_modules()?;
         } else {
-            engine.reload_modules_quiet()?;
+            library.reload_modules_quiet()?;
         }
 
-        Ok(engine)
+        Ok(library)
     }
 
     pub fn reload_modules(&mut self) -> Result<usize> {
@@ -299,9 +298,9 @@ impl Module {
         self.private_module
     }
 
-    pub fn run(&self, env: Environment, reporter: Arc<Mutex<Box<dyn Reporter>>>, arg: LuaJsonValue) -> Result<()> {
+    pub fn run(&self, env: Environment, ipc_child: Arc<Mutex<Box<dyn IpcChild>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
-        self.script.run(env, reporter, arg.into())
+        self.script.run(env, ipc_child, arg.into())
     }
 
     #[inline]
@@ -320,28 +319,3 @@ impl Module {
         }
     }
 }
-
-pub trait Reporter: Debug {
-    fn send(&mut self, event: &Event) -> Result<()>;
-
-    fn recv(&mut self) -> Result<serde_json::Value>;
-}
-
-#[derive(Debug)]
-pub struct DummyReporter;
-
-impl DummyReporter {
-    pub fn new() -> Arc<Mutex<Box<dyn Reporter>>> {
-        Arc::new(Mutex::new(Box::new(DummyReporter)))
-    }
-}
-
-impl Reporter for DummyReporter {
-    fn send(&mut self, _event: &Event) -> Result<()> {
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        unimplemented!("DummyReporter::recv doesn't exist")
-    }
-}

src/fmt.rs

@@ -141,8 +141,7 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
     }
 
     #[inline]
-    pub fn display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
-        self.push_into_group()?;
+    pub fn direct_display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
         if self.scoped {
             C::display(self, v)
         } else {
@@ -151,8 +150,19 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
     }
 
     #[inline]
-    pub fn debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
+    pub fn display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
         self.push_into_group()?;
+        self.direct_display::<C, _>(v)
+    }
+
+    pub fn display_label<C: Color, D: fmt::Display>(&mut self, label: &str, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        write!(self, "{}=", label)?;
+        self.direct_display::<C, _>(v)
+    }
+
+    #[inline]
+    pub fn direct_debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
         if self.scoped {
             C::debug(self, v)
         } else {
@@ -160,6 +170,12 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
         }
     }
 
+    #[inline]
+    pub fn debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        self.direct_debug::<C, _>(v)
+    }
+
     pub fn opt_debug<C: Color, D: fmt::Debug>(&mut self, v: &Option<D>) -> fmt::Result {
         if let Some(v) = &v {
             self.debug::<C, _>(v)?;
@@ -167,6 +183,15 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
         Ok(())
     }
 
+    pub fn opt_debug_label<C: Color, D: fmt::Debug>(&mut self, label: &str, v: &Option<D>) -> fmt::Result {
+        if let Some(v) = &v {
+            self.push_into_group()?;
+            write!(self, "{}=", label)?;
+            self.direct_debug::<C, _>(v)?;
+        }
+        Ok(())
+    }
+
     #[inline]
     pub fn clear(&mut self) -> fmt::Result {
         if self.scoped {

src/ipc/child.rs

@@ -0,0 +1,110 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use crate::engine::Environment;
+use crate::geoip::MaxmindReader;
+use crate::psl::PslReader;
+use serde_json;
+use crate::worker::Event;
+
+use std::fmt::Debug;
+use std::io::prelude::*;
+use std::io::{self, Stdin, Stdout};
+use std::sync::{Arc, Mutex};
+
+
+pub trait IpcChild: Debug {
+    fn send(&mut self, event: &Event) -> Result<()>;
+
+    fn recv(&mut self) -> Result<serde_json::Value>;
+}
+
+#[derive(Debug)]
+pub struct StdioIpcChild {
+    stdin: Stdin,
+    stdout: Stdout,
+}
+
+impl StdioIpcChild {
+    pub fn setup() -> StdioIpcChild {
+        let stdin = io::stdin();
+        let stdout = io::stdout();
+
+        StdioIpcChild {
+            stdin,
+            stdout,
+        }
+    }
+
+    pub fn recv_start(&mut self) -> Result<StartCommand> {
+        let value = self.recv()?;
+        let event = serde_json::from_value(value)?;
+        Ok(event)
+    }
+}
+
+impl IpcChild for StdioIpcChild {
+    fn send(&mut self, event: &Event) -> Result<()> {
+        let mut event = serde_json::to_string(&event)?;
+        event.push('\n');
+        debug!("IpcChild sends: {:?}", event);
+        self.stdout.write_all(event.as_bytes())?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        let mut line = String::new();
+        let len = self.stdin.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcChild received: {:?}", event);
+        Ok(event)
+    }
+}
+
+#[derive(Debug)]
+pub struct DummyIpcChild;
+
+impl DummyIpcChild {
+    pub fn new() -> Arc<Mutex<Box<dyn IpcChild>>> {
+        Arc::new(Mutex::new(Box::new(DummyIpcChild)))
+    }
+}
+
+impl IpcChild for DummyIpcChild {
+    fn send(&mut self, _event: &Event) -> Result<()> {
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        unimplemented!("DummyIpcChild::recv doesn't exist")
+    }
+}
+
+pub fn run(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslReader) -> Result<()> {
+    let mut ipc_child = StdioIpcChild::setup();
+    let start = ipc_child.recv_start()?;
+
+    let environment = Environment {
+        verbose: start.verbose,
+        keyring: start.keyring,
+        dns_config: start.dns_config,
+        proxy: start.proxy,
+        options: start.options,
+        blobs: start.blobs,
+        psl,
+        geoip,
+        asn,
+    };
+
+    let mtx: Arc<Mutex<Box<dyn IpcChild>>> = Arc::new(Mutex::new(Box::new(ipc_child)));
+    let result = start.module.run(environment,
+                                  mtx.clone(),
+                                  start.arg.into());
+    let mut ipc_child = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
+                        .into_inner().expect("Failed to consume Mutex");
+
+    let event = result.into();
+    ipc_child.send(&Event::Exit(event))?;
+
+    Ok(())
+}

src/ipc/common.rs

@@ -0,0 +1,44 @@
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+
+use std::collections::HashMap;
+use std::net::SocketAddr;
+
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct StartCommand {
+    pub verbose: u64,
+    pub keyring: Vec<KeyRingEntry>,
+    pub dns_config: Resolver,
+    pub proxy: Option<SocketAddr>,
+    pub options: HashMap<String, String>,
+    pub module: Module,
+    pub arg: serde_json::Value,
+    pub blobs: Vec<Blob>,
+}
+
+impl StartCommand {
+    pub fn new(verbose: u64,
+               keyring: Vec<KeyRingEntry>,
+               dns_config: Resolver,
+               proxy: Option<SocketAddr>,
+               options: HashMap<String, String>,
+               module: Module,
+               arg: serde_json::Value,
+               blobs: Vec<Blob>,
+    ) -> StartCommand {
+        StartCommand {
+            verbose,
+            keyring,
+            dns_config,
+            proxy,
+            options,
+            module,
+            arg,
+            blobs,
+        }
+    }
+}

src/ipc/mod.rs

@@ -0,0 +1,3 @@
+pub mod child;
+pub mod common;
+pub mod parent;

src/ipc/parent.rs

@@ -0,0 +1,144 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
+
+use std::collections::HashMap;
+use std::env;
+use std::ffi::OsString;
+use std::io::prelude::*;
+use std::io::{BufReader, BufRead, stdin};
+use std::net::SocketAddr;
+use std::sync::mpsc;
+use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
+
+
+pub struct IpcParent {
+    child: Child,
+    stdin: ChildStdin,
+    stdout: BufReader<ChildStdout>,
+}
+
+impl IpcParent {
+    pub fn setup(module: &Module) -> Result<IpcParent> {
+        let exe = match env::current_exe() {
+            Ok(exe) => exe.into_os_string(),
+            _ => OsString::from("sn0int"),
+        };
+
+        let mut child = Command::new(exe)
+            .arg("sandbox")
+            .arg(&module.canonical())
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .context("Failed to spawn child process")?;
+
+        let stdin = child.stdin.take().expect("Failed to take child stdin");
+        let stdout = child.stdout.take().expect("Failed to take child stdout");
+        let stdout = BufReader::new(stdout);
+
+        Ok(IpcParent {
+            child,
+            stdin,
+            stdout,
+        })
+    }
+
+    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
+        let start = serde_json::to_value(&start)?;
+        self.send(&start)?;
+        Ok(())
+    }
+
+    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
+        let mut value = serde_json::to_string(value)?;
+        value.push('\n');
+        self.stdin.write_all(value.as_bytes())?;
+        debug!("IpcParent sent: {:?}", value);
+        Ok(())
+    }
+
+    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
+        let value = serde_json::to_value(value).expect("Failed to serialize reply");
+        if let Err(_) = self.send(&value) {
+            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
+        }
+    }
+
+    pub fn recv(&mut self) -> Result<Event> {
+        let mut line = String::new();
+        let len = self.stdout.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcParent received: {:?}", event);
+        Ok(event)
+    }
+
+    pub fn wait(&mut self) -> Result<()> {
+        let exit = self.child.wait()
+            .context("Failed to wait for child")?;
+
+        if exit.success() {
+            Ok(())
+        } else {
+            bail!("Child signaled error")
+        }
+    }
+
+    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
+        where <T as EventWithCallback>::Payload: serde::Serialize
+    {
+        let (tx2, rx2) = mpsc::channel();
+        tx.send(event.with_callback(tx2));
+        let reply = rx2.recv().unwrap();
+
+        self.send_struct(reply, tx);
+    }
+}
+
+pub fn run(module: Module,
+           tx: &EventSender,
+           arg: serde_json::Value,
+           keyring: Vec<KeyRingEntry>,
+           verbose: u64,
+           has_stdin: bool,
+           proxy: Option<SocketAddr>,
+           options: HashMap<String, String>,
+           blobs: Vec<Blob>,
+) -> Result<ExitEvent> {
+    let dns_config = Resolver::from_system()?;
+
+    let mut reader = if has_stdin {
+        Some(BufReader::new(stdin()))
+    } else {
+        None
+    };
+
+    let mut ipc_parent = IpcParent::setup(&module)?;
+    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
+
+    let exit = loop {
+        match ipc_parent.recv()? {
+            Event::Log(event) => tx.send(Event2::Log(event)),
+            Event::Database(object) => ipc_parent.send_event_callback(object, &tx),
+            Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
+            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
+            Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),
+            Event::Exit(event) => {
+                if let ExitEvent::Err(err) = &event {
+                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
+                }
+                break event;
+            },
+        }
+    };
+
+    ipc_parent.wait()?;
+
+    Ok(exit)
+}

src/keyring.rs

@@ -74,7 +74,7 @@ pub struct KeyRing {
 
 impl KeyRing {
     pub fn path() -> Result<PathBuf> {
-        let path = paths::data_dir()?;
+        let path = paths::sn0int_dir()?;
         let path = path.join("keyring.json");
         Ok(path)
     }

src/lib.rs

@@ -16,24 +16,27 @@ pub mod autonoscope;
 pub mod blobs;
 pub mod cmd;
 pub mod config;
-pub mod crt;
+use sn0int_std::crt;
 pub mod db;
 pub mod errors;
 pub mod engine;
 pub mod filters;
 pub mod fmt;
-pub mod geoip;
-pub mod gfx;
-pub mod html;
-pub mod json;
+use sn0int_std::geo;
+pub use sn0int_std::geoip;
+use sn0int_std::gfx;
+use sn0int_std::html;
+use sn0int_std::json;
+pub mod ipc;
 pub mod keyring;
-pub mod lazy;
+use sn0int_std::lazy;
 pub mod migrations;
 pub mod models;
+use sn0int_std::mqtt;
 pub mod paths;
-pub mod psl;
+pub use sn0int_std::psl;
 pub mod options;
-pub mod ratelimits;
+use sn0int_std::ratelimits;
 pub mod registry;
 pub mod repl;
 pub mod runtime;
@@ -41,16 +44,12 @@ pub mod sandbox;
 pub mod schema;
 pub mod ser;
 pub mod shell;
-pub mod sockets;
+use sn0int_std::sockets;
 pub mod term;
 pub mod update;
 pub mod utils;
-pub mod web;
+use sn0int_std::web;
+use sn0int_std::websockets;
 pub mod worker;
 pub mod workspaces;
-pub mod xml;
-
-#[cfg(test)]
-fn test_init() {
-    let _ = env_logger::builder().is_test(true).try_init();
-}
+use sn0int_std::xml;

src/main.rs

@@ -1,13 +1,15 @@
 use env_logger::{self, Env};
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
-use sn0int::cmd;
+use sn0int::cmd::{self, LiteCmd};
 use sn0int::config::Config;
 use sn0int::db;
 use sn0int::errors::*;
-use sn0int::engine::{self, Engine, Module};
+use sn0int::engine::Module;
 use sn0int::geoip::{GeoIP, AsnDB, Maxmind};
+use sn0int::ipc;
 use sn0int::options::Opt;
+use sn0int::paths;
 use sn0int::psl::PslReader;
 use sn0int::registry;
 use sn0int::repl;
@@ -32,7 +34,7 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         Module::load(&path.to_path_buf(), "anonymous", &filename, true)
             .context(format!("Failed to parse {:?}", path))?
     } else {
-        rl.engine().get(&args.module)?
+        rl.library().get(&args.module)?
             .clone()
     };
 
@@ -45,17 +47,22 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         rl.set_target(Some(target));
     }
 
-    cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+    if args.dump_sandbox_init_msg {
+        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, args.into(), Opt::collect(&args.options))
+    } else {
+        cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+    }
 }
 
 fn run_sandbox() -> Result<()> {
-    let geoip = GeoIP::try_open_reader()?;
-    let asn = AsnDB::try_open_reader()?;
-    let psl = PslReader::open()?;
+    let cache_dir = paths::cache_dir()?;
+    let geoip = GeoIP::try_open_reader(&cache_dir)?;
+    let asn = AsnDB::try_open_reader(&cache_dir)?;
+    let psl = PslReader::open(&cache_dir)?;
 
     sandbox::init()
         .context("Failed to init sandbox")?;
-    engine::isolation::run_worker(geoip, asn, psl)
+    ipc::child::run(geoip, asn, psl)
 }
 
 fn run_cmd<T: cmd::Cmd>(gargs: &Args, args: T, config: &Config) -> Result<()> {
@@ -100,21 +107,26 @@ fn run() -> Result<()> {
         Some(SubCommand::Login(_)) => auth::run_login(&config),
         Some(SubCommand::New(new)) => run_new(&args, &new),
         Some(SubCommand::Publish(publish)) => registry::run_publish(&args, &publish, &config),
-        Some(SubCommand::Install(install)) => registry::run_install(install, &config),
-        Some(SubCommand::Search(search)) => {
-            let engine = Engine::new(false, &config)?;
-            registry::run_search(&engine, &search, &config)
-        },
+        // TODO: deprecate
+        Some(SubCommand::Install(install)) => cmd::pkg_cmd::Args {
+            subcommand: cmd::pkg_cmd::SubCommand::Install(install),
+        }.run(&config),
+        // TODO: deprecate
+        Some(SubCommand::Search(search)) => cmd::pkg_cmd::Args {
+            subcommand: cmd::pkg_cmd::SubCommand::Search(search),
+        }.run(&config),
+        Some(SubCommand::Pkg(pkg)) => pkg.run(&config),
         Some(SubCommand::Add(add)) => run_cmd(&args, add, &config),
         Some(SubCommand::Select(select)) => run_cmd(&args, select, &config),
         Some(SubCommand::Delete(delete)) => run_cmd(&args, delete, &config),
         Some(SubCommand::Activity(activity)) => run_cmd(&args, activity, &config),
         Some(SubCommand::Scope(scope)) => run_cmd(&args, scope, &config),
         Some(SubCommand::Noscope(noscope)) => run_cmd(&args, noscope, &config),
-        Some(SubCommand::Workspace(workspace)) => run_cmd(&args, workspace, &config),
+        Some(SubCommand::Workspace(workspace)) => workspace.run(&config),
         Some(SubCommand::Fsck(fsck)) => run_cmd(&args, fsck, &config),
         Some(SubCommand::Export(export)) => run_cmd(&args, export, &config),
         Some(SubCommand::Repl) => repl::run(&config),
+        Some(SubCommand::Paths) => paths::run(&config),
         Some(SubCommand::Completions(completions)) => complete::run_generate(&completions),
         None => shell::run(&args, &config),
     }

src/models/account.rs

@@ -179,12 +179,14 @@ impl DisplayableDetailed for DetailedAccount {
 
         w.start_group();
         w.opt_debug::<Yellow, _>(&self.displayname)?;
-        w.opt_debug::<Yellow, _>(&self.email)?;
         w.opt_debug::<Yellow, _>(&self.url)?;
-        w.opt_debug::<Yellow, _>(&self.last_seen)?;
-        w.opt_debug::<Yellow, _>(&self.birthday)?;
-        w.opt_debug::<Yellow, _>(&self.phonenumber)?;
-        w.opt_debug::<Yellow, _>(&self.profile_pic)?;
+        w.end_group()?;
+        w.start_group();
+        w.opt_debug_label::<Yellow, _>("last_seen", &self.last_seen)?;
+        w.opt_debug_label::<Yellow, _>("email", &self.email)?;
+        w.opt_debug_label::<Yellow, _>("phonenumber", &self.phonenumber)?;
+        w.opt_debug_label::<Yellow, _>("birthday", &self.birthday)?;
+        w.opt_debug_label::<Yellow, _>("profile_pic", &self.profile_pic)?;
         w.end_group()?;
 
         Ok(())

src/models/breach_email.rs

@@ -118,11 +118,16 @@ impl BreachEmail {
 pub struct PrintableBreachEmail {
     breach: String,
     email: String,
+    password: Option<String>,
 }
 
 impl fmt::Display for PrintableBreachEmail {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        write!(w, "{:?} -> {:?}", self.breach, self.email)
+        write!(w, "{:?} -> {:?}", self.breach, self.email)?;
+        if let Some(password) = &self.password {
+            write!(w, " ({:?})", password)?;
+        }
+        Ok(())
     }
 }
 
@@ -133,6 +138,7 @@ impl Printable<PrintableBreachEmail> for BreachEmail {
         Ok(PrintableBreachEmail {
             breach: breach.value.to_string(),
             email: email.value.to_string(),
+            password: self.password.clone(),
         })
     }
 }
@@ -163,6 +169,7 @@ impl Printable<PrintableBreachEmail> for NewBreachEmail {
         Ok(PrintableBreachEmail {
             breach: breach.value.to_string(),
             email: email.value.to_string(),
+            password: self.password.clone(),
         })
     }
 }

src/models/cryptoaddr.rs

@@ -165,14 +165,14 @@ pub struct DetailedCryptoAddr {
 }
 
 #[inline]
-fn add_currency(w: &mut fmt::DetailFormatter, num: &Option<i64>, denominator: &Option<i32>) -> fmt::Result {
+fn add_currency(w: &mut fmt::DetailFormatter, label: &str, num: &Option<i64>, denominator: &Option<i32>) -> fmt::Result {
     if let Some(&num) = num.as_ref() {
         let denominator = denominator.unwrap_or(0);
         let display = display_currency(num as u64, denominator as usize);
         if num > 0 {
-            w.display::<Green, _>(display)?;
+            w.display_label::<Green, _>(label, display)?;
         } else {
-            w.display::<Red, _>(display)?;
+            w.display_label::<Red, _>(label, display)?;
         }
     }
     Ok(())
@@ -191,14 +191,14 @@ impl DisplayableDetailed for DetailedCryptoAddr {
 
         w.start_group();
         w.opt_debug::<Yellow, _>(&self.currency)?;
-        add_currency(w, &self.balance, &self.denominator)?;
-        add_currency(w, &self.received, &self.denominator)?;
+        add_currency(w, "balance", &self.balance, &self.denominator)?;
+        add_currency(w, "received", &self.received, &self.denominator)?;
         w.end_group()?;
 
         w.start_group();
-        w.opt_debug::<Yellow, _>(&self.first_seen)?;
-        w.opt_debug::<Yellow, _>(&self.last_withdrawal)?;
-        w.opt_debug::<Yellow, _>(&self.description)?;
+        w.opt_debug_label::<Yellow, _>("first_seen", &self.first_seen)?;
+        w.opt_debug_label::<Yellow, _>("last_withdrawal", &self.last_withdrawal)?;
+        w.opt_debug_label::<Yellow, _>("description", &self.description)?;
         w.end_group()?;
 
         Ok(())

src/models/device.rs

@@ -171,15 +171,15 @@ impl DisplayableDetailed for DetailedDevice {
         w.debug::<Green, _>(&self.value)?;
 
         w.start_group();
-        w.opt_debug::<Yellow, _>(&self.name)?;
-        w.opt_debug::<Yellow, _>(&self.hostname)?;
-        w.opt_debug::<Yellow, _>(&self.vendor)?;
-        w.opt_debug::<Yellow, _>(&self.last_seen)?;
+        w.opt_debug_label::<Yellow, _>("name", &self.name)?;
+        w.opt_debug_label::<Yellow, _>("hostname", &self.hostname)?;
+        w.opt_debug_label::<Yellow, _>("vendor", &self.vendor)?;
+        w.opt_debug_label::<Yellow, _>("last_seen", &self.last_seen)?;
         w.end_group()?;
 
         w.start_group();
-        w.opt_debug::<Yellow, _>(&self.network)?;
-        w.opt_debug::<Yellow, _>(&self.ipaddr)?;
+        w.opt_debug_label::<Yellow, _>("network", &self.network)?;
+        w.opt_debug_label::<Yellow, _>("ipaddr", &self.ipaddr)?;
         w.end_group()?;
 
         Ok(())

src/models/mod.rs

@@ -119,7 +119,7 @@ impl From<&Insert> for Table {
 pub enum Update {
     Subdomain(SubdomainUpdate),
     IpAddr(IpAddrUpdate),
-    Url(UrlUpdate),
+    Url(UrlChangeset),
     Email(EmailUpdate),
     PhoneNumber(PhoneNumberUpdate),
     Device(DeviceUpdate),
@@ -374,6 +374,10 @@ impl<T: InsertToNew> LuaInsertToNew for T {
     }
 }
 
+pub trait UpdateToChangeset<T> {
+    fn try_into_changeset(self) -> Result<T>;
+}
+
 mod domain;
 pub use self::domain::*;
 

src/models/url.rs

@@ -1,10 +1,10 @@
+use crate::ser::StringOrBytes;
 use crate::errors::*;
 use crate::fmt::Write;
 use crate::fmt::colors::*;
 use crate::models::*;
 use diesel;
 use diesel::prelude::*;
-use crate::ser;
 use crate::url;
 
 
@@ -228,7 +228,6 @@ pub struct NewUrl {
     pub value: String,
     pub path: String,
     pub status: Option<i32>,
-    #[serde(deserialize_with="ser::opt_string_or_bytes")]
     pub body: Option<Vec<u8>>,
     pub online: Option<bool>,
     pub title: Option<String>,
@@ -254,7 +253,7 @@ impl InsertableStruct<Url> for NewUrl {
 }
 
 impl Upsertable<Url> for NewUrl {
-    type Update = UrlUpdate;
+    type Update = UrlChangeset;
 
     fn upsert(self, existing: &Url) -> Self::Update {
         Self::Update {
@@ -283,8 +282,7 @@ pub struct InsertUrl {
     pub subdomain_id: i32,
     pub value: String,
     pub status: Option<i32>,
-    #[serde(deserialize_with="ser::opt_string_or_bytes")]
-    pub body: Option<Vec<u8>>,
+    pub body: Option<StringOrBytes>,
     pub online: Option<bool>,
     pub title: Option<String>,
     pub redirect: Option<String>,
@@ -309,7 +307,7 @@ impl InsertToNew for InsertUrl {
             value: self.value,
             path,
             status: self.status,
-            body: self.body,
+            body: self.body.map(|x| x.0),
             online: self.online,
             title: self.title,
             redirect,
@@ -320,7 +318,7 @@ impl InsertToNew for InsertUrl {
 
 #[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
 #[table_name="urls"]
-pub struct UrlUpdate {
+pub struct UrlChangeset {
     pub id: i32,
     pub status: Option<i32>,
     pub body: Option<Vec<u8>>,
@@ -329,7 +327,7 @@ pub struct UrlUpdate {
     pub redirect: Option<String>,
 }
 
-impl Upsert for UrlUpdate {
+impl Upsert for UrlChangeset {
     fn is_dirty(&self) -> bool {
         self.status.is_some() ||
         self.body.is_some() ||
@@ -347,7 +345,7 @@ impl Upsert for UrlUpdate {
     }
 }
 
-impl Updateable<Url> for UrlUpdate {
+impl Updateable<Url> for UrlChangeset {
     fn changeset(&mut self, existing: &Url) {
         Self::clear_if_equal(&mut self.online, &existing.online);
         Self::clear_if_equal(&mut self.status, &existing.status);
@@ -365,6 +363,30 @@ impl Updateable<Url> for UrlUpdate {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct UrlUpdate {
+    pub id: i32,
+    pub status: Option<i32>,
+    pub body: Option<StringOrBytes>,
+    pub online: Option<bool>,
+    pub title: Option<String>,
+    pub redirect: Option<String>,
+}
+
+impl UpdateToChangeset<UrlChangeset> for UrlUpdate {
+    fn try_into_changeset(self) -> Result<UrlChangeset> {
+        // TODO: redirect needs pre-processing
+        Ok(UrlChangeset {
+            id: self.id,
+            status: self.status,
+            body: self.body.map(|x| x.0),
+            online: self.online,
+            title: self.title,
+            redirect: self.redirect,
+        })
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/paths.rs

@@ -1,3 +1,4 @@
+use crate::config::Config;
 use crate::errors::*;
 use crate::workspaces::Workspace;
 
@@ -6,7 +7,7 @@ use std::fs;
 use std::path::PathBuf;
 
 
-pub fn data_dir() -> Result<PathBuf> {
+pub fn sn0int_dir() -> Result<PathBuf> {
     let path = dirs::data_dir()
         .ok_or_else(|| format_err!("Failed to find data directory"))?;
     let path = path.join("sn0int");
@@ -16,24 +17,39 @@ pub fn data_dir() -> Result<PathBuf> {
 }
 
 pub fn history_path() -> Result<PathBuf> {
-    let path = data_dir()?;
+    let path = sn0int_dir()?;
     let path = path.join("history");
     Ok(path)
 }
 
 pub fn module_dir() -> Result<PathBuf> {
-    let path = data_dir()?;
+    let path = sn0int_dir()?;
     let path = path.join("modules");
     fs::create_dir_all(&path)
         .context("Failed to create module directory")?;
     Ok(path)
 }
 
+pub fn data_dir() -> Result<PathBuf> {
+    let path = sn0int_dir()?
+        .join("data");
+    fs::create_dir_all(&path)
+        .context("Failed to create module directory")?;
+    Ok(path)
+}
+
+pub fn workspace_dir(workspace: &Workspace) -> Result<PathBuf> {
+    let path = sn0int_dir()?
+        .join("data")
+        .join(workspace.as_str());
+    fs::create_dir_all(&path)
+        .context("Failed to create module directory")?;
+    Ok(path)
+}
+
 pub fn blobs_dir(workspace: &Workspace) -> Result<PathBuf> {
-    let path = data_dir()?;
-    let path = path
-        .join("blobs")
-        .join(workspace.to_string());
+    let path = workspace_dir(workspace)?
+        .join("blobs");
     fs::create_dir_all(&path)
         .context("Failed to create module directory")?;
     Ok(path)
@@ -47,3 +63,20 @@ pub fn cache_dir() -> Result<PathBuf> {
         .context("Failed to create cache directory")?;
     Ok(path)
 }
+
+fn print_path<D: std::fmt::Debug>(k: &str, v: D) {
+    println!("{:30}: {:?}", k, v);
+}
+
+pub fn run(config: &Config) -> Result<()> {
+    print_path("config_file", Config::path()?);
+    print_path("data_dir", data_dir()?);
+    print_path("module_dir", module_dir()?);
+
+    for (k, v) in &config.namespaces {
+        print_path(&format!("modules({})", k), v);
+    }
+
+    print_path("cache_dir", cache_dir()?);
+    Ok(())
+}

src/registry.rs

@@ -3,7 +3,7 @@ use crate::args::{Args, Publish, Install, Search};
 use crate::api::Client;
 use crate::auth;
 use crate::config::Config;
-use crate::engine::{Engine, Module};
+use crate::engine::{Library, Module};
 use colored::Colorize;
 use separator::Separatable;
 use sn0int_common::ModuleID;
@@ -242,7 +242,7 @@ impl Task for UpdateTask {
     }
 }
 
-pub fn run_search(engine: &Engine, search: &Search, config: &Config) -> Result<()> {
+pub fn run_search(library: &Library, search: &Search, config: &Config) -> Result<()> {
     let client = Client::new(&config)?;
 
     let label = format!("Searching {:?}", search.query);
@@ -252,7 +252,7 @@ pub fn run_search(engine: &Engine, search: &Search, config: &Config) -> Result<(
 
     for module in &modules {
         let canonical = module.canonical();
-        let installed = engine.get_opt(&canonical)?;
+        let installed = library.get_opt(&canonical)?;
 
         if search.new && installed.is_some() {
             continue;

src/repl/complete.rs

@@ -18,6 +18,7 @@ impl ReplCompleter {
 }
 
 impl rustyline::Helper for ReplCompleter {}
+impl rustyline::validate::Validator for ReplCompleter {}
 
 impl Completer for ReplCompleter {
     type Candidate = String;

src/repl/mod.rs

@@ -1,9 +1,11 @@
 use crate::config::Config;
 use crate::errors::*;
-use crate::engine::{ctx, Environment, DummyReporter};
+use crate::engine::{ctx, Environment};
+use crate::ipc::child::DummyIpcChild;
 use crate::engine::ctx::{State, LuaState};
 use crate::geoip::{Maxmind, AsnDB, GeoIP};
 use crate::hlua::{Lua, AnyLuaValue};
+use crate::paths;
 use crate::psl::PslReader;
 use crate::shell::readline::Readline;
 use crate::runtime::format_lua;
@@ -82,12 +84,14 @@ pub fn run(config: &Config) -> Result<()> {
     let keyring = Vec::new();
     let dns_config = Resolver::from_system()?;
     let proxy = config.network.proxy;
-    let psl = PslReader::open()?;
-    let geoip = GeoIP::try_open_reader()?;
-    let asn = AsnDB::try_open_reader()?;
+
+    let cache_dir = paths::cache_dir()?;
+    let psl = PslReader::open(&cache_dir)?;
+    let geoip = GeoIP::try_open_reader(&cache_dir)?;
+    let asn = AsnDB::try_open_reader(&cache_dir)?;
 
     let env = Environment {
-        verbose: 0, // this doesn't do anything since we use a dummy reporter
+        verbose: 0, // this doesn't do anything since we use a dummy ipc child
         keyring,
         dns_config,
         proxy,
@@ -98,7 +102,7 @@ pub fn run(config: &Config) -> Result<()> {
         asn,
     };
 
-    let tx = DummyReporter::new();
+    let tx = DummyIpcChild::new();
     let (lua, state) = ctx::ctx(env, tx);
     let mut repl = Repl::new(lua, state);
 

src/runtime/blobs.rs

@@ -1,12 +1,14 @@
 use crate::errors::*;
-use crate::blobs::Blob;
 use crate::engine::ctx::State;
 use crate::engine::structs::byte_array;
 use crate::hlua::{self, AnyLuaValue};
+use sn0int_std::blobs::{Blob, BlobState};
 use std::sync::Arc;
 
 
-pub fn create_blob(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+pub fn create_blob<S>(lua: &mut hlua::Lua, state: Arc<S>)
+    where S: State + BlobState + 'static
+{
     lua.set("create_blob", hlua::function1(move |bytes: AnyLuaValue| -> Result<String> {
         let bytes = byte_array(bytes)
             .map_err(|err| state.set_error(err))?;

src/runtime/crypto.rs

@@ -0,0 +1,30 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::engine::structs::{byte_array, lua_bytes};
+use crate::hlua::{self, AnyLuaValue};
+use sn0int_std::crypto;
+use std::sync::Arc;
+
+pub fn key_trunc_pad(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("key_trunc_pad", hlua::function3(move |bytes: AnyLuaValue, len: u32, pad: u8| -> Result<AnyLuaValue> {
+        let bytes = byte_array(bytes)
+            .map_err(|err| state.set_error(err))?;
+        let bytes = crypto::key_trunc_pad(&bytes, len as usize, pad);
+        Ok(lua_bytes(&bytes))
+    }))
+}
+
+pub fn sodium_secretbox_open(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("sodium_secretbox_open", hlua::function2(move |encrypted: AnyLuaValue, key: AnyLuaValue| -> Result<AnyLuaValue> {
+        let encrypted = byte_array(encrypted)
+            .map_err(|err| state.set_error(err))?;
+        let key = byte_array(key)
+            .map_err(|err| state.set_error(err))?;
+
+        let plain = crypto::sodium_secretbox_open(&encrypted, &key)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(lua_bytes(&plain))
+    }))
+}

src/runtime/db.rs

@@ -8,6 +8,7 @@ use crate::hlua::{self, AnyLuaValue};
 use std::str::FromStr;
 use std::sync::Arc;
 use crate::models::*;
+use crate::worker::DatabaseResponse;
 use crate::json::LuaJsonValue;
 
 
@@ -85,8 +86,18 @@ pub fn db_add(lua: &mut hlua::Lua, state: Arc<dyn State>) {
         let object = into_insert(family, object, &state)
             .map_err(|e| state.set_error(e))?;
 
-        state.db_insert(object)
-            .map_err(|e| state.set_error(e))
+        let r = state.db_insert(object)
+            .map_err(|e| state.set_error(e))?;
+
+        let r = match r {
+            // Found is technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
     }))
 }
 
@@ -99,13 +110,23 @@ pub fn db_add_ttl(lua: &mut hlua::Lua, state: Arc<dyn State>) {
         let object = into_insert(family, object, &state)
             .map_err(|e| state.set_error(e))?;
 
-        state.db_insert_ttl(object, ttl)
-            .map_err(|e| state.set_error(e))
+        let r = state.db_insert_ttl(object, ttl)
+            .map_err(|e| state.set_error(e))?;
+
+        let r = match r {
+            // Found is technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
     }))
 }
 
 pub fn db_activity(lua: &mut hlua::Lua, state: Arc<dyn State>) {
-    lua.set("db_activity", hlua::function1(move |v: AnyLuaValue| -> Result<()> {
+    lua.set("db_activity", hlua::function1(move |v: AnyLuaValue| -> Result<bool> {
         let v: LuaJsonValue = v.into();
         let v: serde_json::Value = v.into();
         let activity: InsertActivity = serde_json::from_value(v)
@@ -121,8 +142,18 @@ pub fn db_select(lua: &mut hlua::Lua, state: Arc<dyn State>) {
         let family = Family::from_str(&family)
             .map_err(|e| state.set_error(e.into()))?;
 
-        state.db_select(family, value)
-            .map_err(|e| state.set_error(e))
+        let r = state.db_select(family, value)
+            .map_err(|e| state.set_error(e))?;
+
+        let r = match r {
+            // Everything except Found and None are technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
     }))
 }
 
@@ -146,6 +177,28 @@ fn gen_changeset<T: Model, U: Updateable<T>>(object: LuaJsonValue, mut update: L
     Ok((existing.id(), value, update))
 }
 
+fn gen_changeset2<T: Model, U: UpdateToChangeset<C>, C: Updateable<T>>(object: LuaJsonValue, mut update: LuaJsonValue) -> Result<(i32, String, C)>
+    where
+        for<'de> T: serde::Deserialize<'de>,
+        for<'de> U: serde::Deserialize<'de>,
+        for<'de> C: serde::Deserialize<'de>,
+{
+    let existing = structs::from_lua::<T>(object)?;
+
+    // copy the id over to the update struct so we can identify the row
+    if let LuaJsonValue::Object(ref mut update) = update {
+        update.insert("id".into(), LuaJsonValue::Number(existing.id().into()));
+    }
+
+    let update = structs::from_lua::<U>(update)?;
+    let mut update = update.try_into_changeset()?;
+
+    let value = existing.to_string();
+    update.changeset(&existing);
+
+    Ok((existing.id(), value, update))
+}
+
 pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("db_update", hlua::function3(move |family: String, object: AnyLuaValue, update: AnyLuaValue| -> Result<Option<i32>> {
         let family = Family::from_str(&family)
@@ -160,7 +213,7 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             Family::Ipaddr => gen_changeset::<IpAddr, IpAddrUpdate>(object, update)
                 .map(|(id, v, u)| (id, v, Update::IpAddr(u))),
             Family::SubdomainIpaddr => bail!("Subdomain-IpAddr doesn't have mutable fields"),
-            Family::Url => gen_changeset::<Url, UrlUpdate>(object, update)
+            Family::Url => gen_changeset2::<Url, UrlUpdate, UrlChangeset>(object, update)
                 .map(|(id, v, u)| (id, v, Update::Url(u))),
             Family::Email => gen_changeset::<Email, EmailUpdate>(object, update)
                 .map(|(id, v, u)| (id, v, Update::Email(u))),
@@ -191,8 +244,18 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             .map_err(|e| state.set_error(e))?;
 
         if update.is_dirty() {
-            state.db_update(value, update)
-                .map_err(|e| state.set_error(e))
+            let r = state.db_update(value, update)
+                .map_err(|e| state.set_error(e))?;
+
+            let r = match r {
+                // Inserted and Found are technically unreachable
+                DatabaseResponse::Inserted(id) => Some(id),
+                DatabaseResponse::Updated(id) => Some(id),
+                DatabaseResponse::NoChange(id) => Some(id),
+                DatabaseResponse::Found(id) => Some(id),
+                DatabaseResponse::None => None,
+            };
+            Ok(r)
         } else {
             Ok(Some(id))
         }

src/runtime/geo.rs

@@ -0,0 +1,126 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::geo::{self, Point};
+use crate::hlua::{self, AnyLuaValue};
+use std::sync::Arc;
+
+
+pub fn geo_polygon_contains(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("geo_polygon_contains", hlua::function2(move |polygon: Vec<AnyLuaValue>, point: AnyLuaValue| -> Result<bool> {
+        let polygon = polygon.into_iter()
+            .map(|p| Point::try_from(p))
+            .collect::<Result<Vec<_>>>()
+            .context("Invalid polygon")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        let point = Point::try_from(point)
+            .context("Invalid point")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        Ok(geo::polygon_contains(&polygon, &point))
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::engine::ctx::Script;
+
+    fn hamburg_polygon() -> &'static str {
+        "
+            { lat=53.63975308945899,  lon=9.764785766601562  },
+            { lat=53.59494998253459,  lon=9.827270507812     },
+            { lat=53.663153974456456, lon=9.9151611328125    },
+            { lat=53.65582987649682,  lon=9.976272583007812  },
+            { lat=53.68613523817129,  lon=9.992752075195312  },
+            { lat=53.68674518938816,  lon=10.051460266113281 },
+            { lat=53.72495117617815,  lon=10.075492858886719 },
+            { lat=53.71946627930625,  lon=10.118408203125    },
+            { lat=53.743635083157756, lon=10.164413452148438 },
+            { lat=53.73104466704585,  lon=10.202865600585938 },
+            { lat=53.676781546441546, lon=10.16304016113281  },
+            { lat=53.632832079199474, lon=10.235824584960938 },
+            { lat=53.608803292930894, lon=10.2008056640625   },
+            { lat=53.578646152866504, lon=10.208358764648438 },
+            { lat=53.57212285981298,  lon=10.163726806640625 },
+            { lat=53.52071674896369,  lon=10.18707275390625  },
+            { lat=53.52643162253097,  lon=10.224151611328125 },
+            { lat=53.44062753992289,  lon=10.347747802734375 },
+            { lat=53.38824275010831,  lon=10.248870849609375 },
+            { lat=53.38824275010831,  lon=10.15960693359375  },
+            { lat=53.44635321212876,  lon=10.064849853515625 },
+            { lat=53.40595029739904,  lon=9.985198974609375  },
+            { lat=53.42385506057106,  lon=9.951210021972656  },
+            { lat=53.41843327091211,  lon=9.944171905517578  },
+            { lat=53.41812635648326,  lon=9.927349090576172  },
+            { lat=53.412294561442884, lon=9.917736053466797  },
+            { lat=53.41464783813818,  lon=9.901256561279297  },
+            { lat=53.443490472483326, lon=9.912586212158201  },
+            { lat=53.45177144115704,  lon=9.897651672363281  },
+            { lat=53.43633277935392,  lon=9.866924285888672  },
+            { lat=53.427639673754776, lon=9.866409301757812  },
+            { lat=53.427639673754776, lon=9.858856201171875  },
+            { lat=53.46710230573499,  lon=9.795513153076172  },
+            { lat=53.49039461941655,  lon=9.795341491699219  },
+            { lat=53.49029248806277,  lon=9.77903366088867   },
+            { lat=53.49856433088649,  lon=9.780235290527344  },
+            { lat=53.5078554643033,   lon=9.758434295654297  },
+            { lat=53.545407634092975, lon=9.759807586669922  },
+            { lat=53.568147234570084, lon=9.633293151855469  },
+            { lat=53.58802162343514,  lon=9.655780792236328  },
+            { lat=53.568351121879815, lon=9.727706909179688  },
+            { lat=53.60921067445695,  lon=9.737663269042969  },
+        "
+    }
+
+    #[test]
+    fn verify_polygon_hamburg_contains_hamburg() {
+        let script = Script::load_unchecked(format!(r#"
+        function run()
+            x = geo_polygon_contains({{{}}}, {{
+                lat=53.551085,
+                lon=9.993682,
+            }})
+            if last_err() then return end
+            if not x then
+                return 'hamburg should be in hamburg'
+            end
+        end
+        "#, hamburg_polygon())).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    fn verify_polygon_hamburg_not_contains_berlin() {
+        let script = Script::load_unchecked(format!(r#"
+        function run()
+            x = geo_polygon_contains({{{}}}, {{
+                lat=52.52437,
+                lon=13.41053,
+            }})
+            if last_err() then return end
+            if x then
+                return 'berlin should not be in hamburg'
+            end
+        end
+        "#, hamburg_polygon())).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    fn verify_polygon_hamburg_not_contains_ny() {
+        let script = Script::load_unchecked(format!(r#"
+        function run()
+            x = geo_polygon_contains({{{}}}, {{
+                lat=40.726662,
+                lon=-74.036677,
+            }})
+            if last_err() then return end
+            if x then
+                return 'ny should not be in hamburg'
+            end
+        end
+        "#, hamburg_polygon())).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+}