Release v0.18.1

Add select --count

.travis.yml

@@ -59,8 +59,11 @@ script:
 
 notifications:
   irc:
+    template:
+    - "%{repository_slug}#%{build_number} (%{branch} - %{commit} : %{author}): %{message} (%{build_url})"
     channels:
     - "ircs://irc.hackint.org:6697/#sn0int"
+    use_notice: true
+    skip_join: true
     #on_success: change # default: always
     #on_failure: always # default: always
-    use_notice: true

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.17.0"
+version = "0.18.1"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -22,7 +22,7 @@ sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
 sn0int-common = { version="0.10.0", path="sn0int-common" }
-sn0int-std = { version="0.17.0", path="sn0int-std" }
+sn0int-std = { version="=0.18.1", path="sn0int-std" }
 rustyline = "6.0"
 log = "0.4"
 env_logger = "0.7"
@@ -41,8 +41,8 @@ dirs = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.14"
-base64 = "0.11"
+chrootable-https = "0.15"
+base64 = "0.12"
 data-encoding = "2.1.2"
 serde_urlencoded = "0.6"
 serde = "1.0"
@@ -62,8 +62,9 @@ semver = "0.9"
 bytes = "0.4"
 bytesize = "1.0"
 ipnetwork = "0.16"
-strum = "0.17"
-strum_macros = "0.17"
+strum = "0.18"
+strum_macros = "0.18"
+embedded-triple = "0.1.0"
 
 digest = "0.8.0"
 md-5 = "0.8.0"
@@ -74,6 +75,7 @@ hmac = "0.7"
 
 walkdir = "2.2"
 nude = "0.3"
+glob = "0.3.0"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
@@ -82,7 +84,7 @@ syscallz = "0.12"
 nix = "0.17"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
-pledge = "0.3.1"
+pledge = "0.4"
 unveil = "0.2.0"
 
 [dev-dependencies]

Dockerfile

@@ -1,14 +1,13 @@
-FROM rust:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

README.md

@@ -89,10 +89,10 @@ For everything else please have a look at the [detailed list][1].
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-  - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope)
-    - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains)
-    - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips)
-    - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls)
+- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains)
+  - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls)
 - [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
   - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
@@ -100,6 +100,8 @@ For everything else please have a look at the [detailed list][1].
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
 - [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
@@ -154,6 +156,7 @@ For everything else please have a look at the [detailed list][1].
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity)
   - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
@@ -177,12 +180,18 @@ For everything else please have a look at the [detailed list][1].
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [key_trunc_pad](https://sn0int.readthedocs.io/en/latest/reference.html#key-trunc-pad)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
   - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [mqtt_connect](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-connect)
+  - [mqtt_subscribe](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-subscribe)
+  - [mqtt_recv](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-recv)
+  - [mqtt_ping](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-ping)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
@@ -215,6 +224,7 @@ For everything else please have a look at the [detailed list][1].
   - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
   - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [sodium_secretbox_open](https://sn0int.readthedocs.io/en/latest/reference.html#sodium-secretbox-open)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
   - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
@@ -222,6 +232,7 @@ For everything else please have a look at the [detailed list][1].
   - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
   - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
   - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval)
   - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)

ci/integration.py

@@ -47,10 +47,10 @@ def main(tempdir, binary):
 
     print('[*] installing modules')
     sn0int(tempdir, binary, [
-        'mod install kpcyrd/ctlogs',
-        'mod install kpcyrd/dns-resolve',
-        'mod install kpcyrd/url-scan',
-        'mod install kpcyrd/geoip',
+        'pkg install kpcyrd/ctlogs',
+        'pkg install kpcyrd/dns-resolve',
+        'pkg install kpcyrd/url-scan',
+        'pkg install kpcyrd/geoip',
     ])
 
     print('[*] running ctlogs')
@@ -63,6 +63,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,6 +92,8 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }

ci/setup.sh

@@ -3,6 +3,9 @@ set -exu
 case "$1" in
     linux)
         sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev
+        sudo apt install libsqlite3-dev libseccomp-dev libsodium-dev
+        ;;
+    osx)
+        brew install libsodium
         ;;
 esac

contrib/docker/Dockerfile.alpine

@@ -1,13 +1,13 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

contrib/docker/Dockerfile.debian

@@ -1,13 +1,13 @@
-FROM rust
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]

docs/autonoscope.rst

@@ -0,0 +1,81 @@
+Autonoscope
+===========
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+-------
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+---
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+----
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

docs/build.rst

@@ -18,19 +18,21 @@ Archlinux
 
 .. code-block:: bash
 
-    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+    $ pacman -S geoip2-database libseccomp libsodium publicsuffix-list sqlite
 
 Mac OSX
 ~~~~~~~
 
-None.
+.. code-block:: bash
+
+    $ brew install libsodium
 
 Debian/Ubuntu/Kali
 ~~~~~~~~~~~~~~~~~~
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
 
 .. warning::
    On a debian based system make sure you've installed rust with rustup.
@@ -40,21 +42,21 @@ Alpine
 
 .. code-block:: bash
 
-    $ apk add sqlite-dev libseccomp-dev
+    $ apk add sqlite-dev libseccomp-dev libsodium-dev
 
 OpenBSD
 ~~~~~~~
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3 geolite2-city geolite2-asn
+    $ pkg_add sqlite3 geolite2-city geolite2-asn libsodium
 
 Gentoo
 ~~~~~~
 
 .. code-block:: bash
 
-    emerge --ask sys-libs/libseccomp dev-db/sqlite
+    emerge --ask sys-libs/libseccomp dev-db/sqlite dev-libs/libsodium
 
 Windows
 ~~~~~~~

docs/index.rst

@@ -38,6 +38,7 @@ Getting Started
    install
    build
    usage
+   autonoscope
    scripting
    database
    structs

docs/install.rst

@@ -32,7 +32,7 @@ at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .

docs/reference.rst

@@ -391,6 +391,8 @@ options are set. The following options are available:
 ``proxy``
   Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
   if it doesn't conflict with the global proxy settings.
+``binary``
+  Set to ``true`` to get the http response as raw bytes.
 
 This function may fail.
 
@@ -418,6 +420,8 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``binary``
+  The response body as bytes (if ``binary=true``)
 ``blob``
   If ``into_blob`` was enabled for the request the body is downloaded into blob
   storage with a reference to the body in this field.
@@ -517,6 +521,13 @@ Log an info to the terminal.
 
     info('ohai')
 
+intval
+------
+
+Parse a number from a string.
+
+    x = strval('1234')
+
 json_decode
 -----------
 
@@ -551,6 +562,17 @@ Encode a datastructure into a string.
     })
     print(x)
 
+key_trunc_pad
+-------------
+
+Truncate/pad a key to a given length.
+
+.. code-block:: lua
+
+    -- if longer than 32 bytes: truncate to 32
+    -- if shorter than 32 bytes: pad with \x00
+    local key = key_trunc_pad(password, 32, 0)
+
 keyring
 -------
 
@@ -585,6 +607,64 @@ Hash a byte array with md5 and return the results as bytes.
 
     hex(md5("\x00\xff"))
 
+mqtt_connect
+------------
+
+Connect to an mqtt broker.
+
+.. code-block:: lua
+
+    local sock = mqtt_connect('mqtts://mqtt.example.com', {
+        username='foo',
+        password='secret',
+    })
+    if last_err() then return end
+
+mqtt_subscribe
+--------------
+
+Subscribe to a topic. Right now only QoS 0 is supported.
+
+.. code-block:: lua
+
+    mqtt_subscribe(sock, '#', 0)
+    if last_err() then return end
+
+mqtt_recv
+---------
+
+Receive an mqtt packet. This is not necessarily a publish packet and more
+packets might be added in the future, so you need to check the type
+specifically.
+
+If a read timeout has been set with mqtt_connect_ this function returns ``nil``
+in case of a read timeout.
+
+.. code-block:: lua
+
+    local pkt = mqtt_recv(sock)
+    if last_err() then return end
+    if pkt == nil then
+        -- read timeout, consider sending a ping or disconnect if the previous ping failed
+    elseif pkt['type'] == 'pong' then
+        -- broker sent a pong
+    elseif pkt['type'] == 'publish' then
+        local payload = utf8_decode(pkt['body'])
+        if last_err() then return end
+        info(payload)
+    end
+
+mqtt_ping
+---------
+
+Send a pingreq packet, causing the broker to send a pingresp. This is used to
+make sure the connection is still working correctly.
+
+.. code-block:: lua
+
+    mqtt_ping(sock)
+    if last_err() then return end
+
 pgp_pubkey
 ----------
 
@@ -1012,6 +1092,27 @@ Overwrite the default ``\n`` newline.
 
     sock_newline(sock, "\r\n")
 
+sodium_secretbox_open
+---------------------
+
+Use authenticated symetric crypto to decrypt a given message.
+
+Internally this is ``crypto_secretbox_xsalsa20poly1305``.
+
+The key **must** be 32 bytes, see key_trunc_pad_ if necessary.
+
+The first 24 bytes of the encrypted message are expected to be the nonce.
+
+.. code-block:: lua
+
+    plain = sodium_secretbox_open(encrypted, key)
+    if last_err() then return end
+
+    txt = utf8_decode(plain)
+    if last_err() then return end
+
+    info(txt)
+
 status
 ------
 
@@ -1093,6 +1194,13 @@ Parse a date into a unix timestamp, see `strftime rules`_.
 
 .. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
 
+strval
+------
+
+Convert a number into a string.
+
+    x = strval(1234)
+
 time_unix
 ---------
 

docs/usage.rst

@@ -27,12 +27,12 @@ number of recommended modules::
     [+] Downloading "GeoLite2-City.mmdb"
     [+] Downloading "GeoLite2-ASN.mmdb"
     [+] Loaded 0 modules
-    [*] No modules found, run quickstart to install default modules
+    [*] No modules found, run pkg quickstart to install default modules
     [sn0int][default] >
 
-Typing ``quickstart`` is going to get you a fair number of featured modules::
+Typing ``pkg quickstart`` is going to get you a fair number of featured modules::
 
-    [sn0int][default] > quickstart
+    [sn0int][default] > pkg quickstart
     [+] Installing kpcyrd/asn
     [+] Installing kpcyrd/ctlogs
     [+] Installing kpcyrd/dns-resolve
@@ -105,7 +105,7 @@ Running a module
 Now that we have something to get started with, we can run our first module.
 First lets list all modules we have::
 
-    [sn0int][demo] > mod list
+    [sn0int][demo] > pkg list
     kpcyrd/asn (0.1.0)
         Run a asn lookup for an ip address
     kpcyrd/ctlogs (0.1.0)
@@ -149,7 +149,7 @@ some of them in a browser but hold on, there's a more efficient way to approach
 this.
 
 .. hint::
-   You can run the modules concurrently with ``run -j 8``.
+   You can run the modules concurrently with ``run -j3``.
 
 Running followup modules on the results
 ---------------------------------------
@@ -256,85 +256,3 @@ You can reverse this using the scope command::
 .. hint::
    All entities have this field, you can refer to it in queries using
    ``unscoped=1``.
-
-Autonoscope
------------
-
-Instead of manually unscoping everything you can also define so called
-autonoscope rules. Those are executed from most specific to least specific and
-the first match wins. If no rule matches, the default is in-scope::
-
-    [sn0int][demo] > # add the domain first
-    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
-    [sn0int][demo] > add domain example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > # automatically noscope all subdomains
-    [sn0int][demo] > autonoscope add domain example.com
-    [sn0int][demo] > # except subdomains of prod.example.com 
-    [sn0int][demo] > autoscope add domain prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > autonoscope list 
-      scope domain "prod.example.com"
-    noscope domain "example.com"
-    [sn0int][demo] > 
-    [sn0int][demo] > # this is going to be out-of-scope
-    [sn0int][demo] > add subdomain www.example.com
-    [sn0int][demo] > # this is going to be in-scope
-    [sn0int][demo] > add subdomain db.prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > select subdomains 
-    #1, "www.example.com"
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=0
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=1
-    #1, "www.example.com"
-    [sn0int][demo] > 
-
-Domains
-~~~~~~~
-
-Autonoscope rules for domains are applied to the following structs:
-
-- domains
-- subdomains
-- urls
-
-Example rules::
-
-    autonoscope add domain example.com
-    autonoscope add domain staging.example.com
-    autonoscope add domain com
-    autonoscope add domain .
-
-IPs
-~~~
-
-Autonoscope rules for IPs are applied to the following structs:
-
-- ipaddrs
-- netblocks
-- ports
-
-Example rules::
-
-    autonoscope add ip 0.0.0.0/0
-    autonoscope add ip ::/0
-    autonoscope add ip 192.168.0.0/16
-    autonoscope add ip 10.13.33.37/32
-
-URLs
-~~~~
-
-Autonoscope rules for urls are applied to the following structs:
-
-- urls
-
-Note that these rules are specific to a certain origin (like
-``https://example.com``) and are used to filter paths.
-
-Example rules::
-
-    autonoscope add url https://example.com/
-    autonoscope add url https://example.com/admin/
-    autonoscope add url https://example.com/a/b/c/d

modules/harness/pgp-fetch.lua

@@ -0,0 +1,22 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+    url = 'https://openpgpkey.archlinux.org/.well-known/openpgpkey/archlinux.org/hu/in9mwr4s84x7gm51851h343n3at1x61g?l=anthraxx'
+
+    req = http_request(session, 'GET', url, {})
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['text'])
+    info(k)
+
+    req = http_request(session, 'GET', url, {
+        binary=true,
+    })
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['binary'])
+    info(k)
+end

modules/harness/url-inserter.lua

@@ -0,0 +1,58 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('preparing')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+
+    info('inserting')
+    url1 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+    })
+    url2 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        body='ohai',
+    })
+    url3 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+    })
+
+    info('updating')
+    db_update('url', {
+        id=url1,
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+        path='/',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url2,
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        path='/ohai',
+        body='ohai',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url3,
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        path='/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+        unscoped=false,
+    }, {
+    })
+end

modules/harness/winkekatze-sub.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local sock = mqtt_connect('mqtt://mqtt.winkekatze24.de', {
+        read_timeout=10,
+    })
+    if last_err() then return end
+    mqtt_subscribe(sock, '#', 0)
+
+    while true do
+        local pkt = mqtt_recv_text(sock)
+        if last_err() then return end
+        info(pkt)
+    end
+end

sn0int-registry/src/auth.rs

@@ -2,7 +2,7 @@ use diesel::pg::PgConnection;
 use oauth2::basic::BasicClient;
 use oauth2::prelude::*;
 use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl, TokenResponse};
-use crate::github::GithubAuthenticator;
+use crate::github;
 use sn0int_registry::errors::*;
 use sn0int_registry::models::AuthToken;
 use url::Url;
@@ -67,8 +67,7 @@ impl Authenticator {
         let access_token = response.access_token();
         let access_token = access_token.secret().to_string();
 
-        let client = GithubAuthenticator::from_env()?;
-        let user = client.get_username(&access_token)?;
+        let user = github::get_username(&access_token)?;
 
         AuthToken::create(&AuthToken {
             id: state,

sn0int-registry/src/auth2.rs

@@ -4,7 +4,7 @@ use sn0int_registry::db::Connection;
 use rocket::http::Status;
 use rocket::{Request, Outcome};
 use rocket::request::{self, FromRequest};
-use crate::github::GithubAuthenticator;
+use crate::github;
 
 
 pub struct AuthHeader(String);
@@ -12,8 +12,7 @@ pub struct AuthHeader(String);
 impl AuthHeader {
     pub fn verify(&self, connection: &Connection) -> Result<String> {
         let session = AuthToken::read(&self.0, &connection)?;
-        let client = GithubAuthenticator::from_env()?;
-        client.get_username(&session.access_token)
+        github::get_username(&session.access_token)
             .map_err(Error::from)
     }
 }

sn0int-registry/src/github.rs

@@ -1,54 +1,23 @@
 use sn0int_registry::errors::*;
-use std::env;
 use reqwest;
 
+pub fn get_username(oauth_token: &str) -> Result<String> {
+    let client = reqwest::Client::new();
+    let mut resp = client.get("https://api.github.com/user")
+        .header("Authorization", format!("token {}", oauth_token))
+        .send()
+        .context("Failed to check access_token")?
+        .error_for_status()
+        .context("Github returned http error")?;
 
-pub struct GithubAuthenticator {
-    client_id: String,
-    client_secret: String,
-}
+    let data = resp.json::<GithubUser>()
+        .context("Failed to deserialize github reply")?;
 
-impl GithubAuthenticator {
-    pub fn new(client_id: String, client_secret: String) -> GithubAuthenticator {
-        GithubAuthenticator {
-            client_id,
-            client_secret,
-        }
-    }
-
-    pub fn from_env() -> Result<GithubAuthenticator> {
-        let client_id = env::var("GITHUB_CLIENT_ID")
-            .context("GITHUB_CLIENT_ID is not set")?;
-        let client_secret = env::var("GITHUB_CLIENT_SECRET")
-            .context("GITHUB_CLIENT_SECRET is not set")?;
-        Ok(GithubAuthenticator::new(client_id, client_secret))
-    }
-
-    pub fn get_username(&self, oauth_token: &str) -> Result<String> {
-        let url = format!("https://api.github.com/applications/{}/tokens/{}", self.client_id, oauth_token);
-        let client = reqwest::Client::new();
-        let mut resp = client.get(&url)
-            .basic_auth(&self.client_id, Some(&self.client_secret))
-            .send()?;
-
-        if !resp.status().is_success() {
-            bail!("Github returned: {}", resp.status())
-        }
-
-        let data = resp.json::<GithubReply>()
-            .context("Failed to deserialize github reply")?;
-
-        Ok(data.user.login)
-    }
+    Ok(data.login)
 }
 
 #[derive(Debug, Deserialize)]
-pub struct GithubReply {
-    user: GithubUser,
-}
-
-#[derive(Debug, Deserialize)]
-pub struct GithubUser {
+struct GithubUser {
     login: String,
     #[serde(rename="type")]
     user_type: String,

sn0int-registry/templates/index.html.hbs

@@ -21,7 +21,7 @@
     To install a module run:
     </p>
     <p class="code"><code>
-    sn0int install kpcyrd/ctlogs
+    sn0int pkg install kpcyrd/ctlogs
     </code></p>
 
     {{#each modules}}

sn0int-std/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-std"
-version = "0.17.0"
+version = "0.18.1"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
@@ -24,7 +24,7 @@ serde_urlencoded = "0.6"
 webpki = "0.21"
 webpki-roots = "0.19.0"
 ct-logs = "0.6"
-chrootable-https = "0.14"
+chrootable-https = "0.15"
 http = "0.2"
 bufstream = "0.1.4"
 pem = "0.7"
@@ -32,14 +32,16 @@ url = "2.0"
 tungstenite = { version = "0.10.1", default-features = false }
 kuchiki = "0.8.0"
 maxminddb = "0.13"
-x509-parser = "0.6.0"
+x509-parser = "0.6.2"
 der-parser = "3.0"
 publicsuffix = { version="1.5", default-features=false }
 xml-rs = "0.8"
 geo = "0.12"
 bytes = "0.4"
-base64 = "0.11"
+base64 = "0.12"
 chrono = { version = "0.4", features = ["serde"] }
+mqtt-protocol = "0.8.1"
+sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
 
 image = "0.23.0"
 kamadak-exif = "0.5.1"

sn0int-std/src/crypto.rs

@@ -0,0 +1,51 @@
+use crate::errors::*;
+use sodiumoxide::crypto::secretbox::{self, Key, Nonce};
+use std::iter;
+
+pub fn key_trunc_pad(mut key: &[u8], len: usize, pad: u8) -> Vec<u8> {
+    if key.len() > len {
+        key = &key[..len];
+    }
+
+    let mut key = key.to_vec();
+    key.extend(iter::repeat(pad).take(len - key.len()));
+    key
+}
+
+pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
+    if encrypted.len() <= secretbox::NONCEBYTES {
+        bail!("Encrypted message is too short");
+    }
+
+    let key = Key::from_slice(key)
+        .ok_or_else(|| format_err!("Key has wrong length"))?;
+    let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
+        .ok_or_else(|| format_err!("Nonce has wrong length"))?;
+    let ciphertext = &encrypted[secretbox::NONCEBYTES..];
+    let plain = secretbox::open(&ciphertext, &nonce, &key)
+        .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
+    Ok(plain)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_key_equal() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_trunc() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5, 6, 7, 8, 9], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_pad() {
+        let key = key_trunc_pad(&[1, 2, 3], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 0, 0]);
+    }
+}

sn0int-std/src/engine/structs.rs

@@ -41,6 +41,12 @@ impl LuaMap {
     pub fn insert_num<K: Into<String>>(&mut self, k: K, v: f64) {
         self.0.insert(AnyHashableLuaValue::LuaString(k.into()), AnyLuaValue::LuaNumber(v));
     }
+
+    pub fn insert_serde<K: Into<String>, S: serde::Serialize>(&mut self, k: K, v: S) -> Result<()> {
+        let v = serde_json::to_value(v)?;
+        self.0.insert(AnyHashableLuaValue::LuaString(k.into()), LuaJsonValue::from(v).into());
+        Ok(())
+    }
 }
 
 impl IntoIterator for LuaMap {

sn0int-std/src/geoip/mod.rs

@@ -1,6 +1,5 @@
 use crate::errors::*;
 use crate::lazy::LazyInit;
-// use crate::paths;
 use maxminddb::{self, geoip2};
 use std::fmt;
 use std::fs::{self, File};
@@ -26,6 +25,8 @@ pub trait Maxmind: Sized {
             "/usr/share/GeoIP/",
             // OpenBSD
             "/usr/local/share/examples/libmaxminddb/",
+            // geoipupdate
+            "/var/lib/GeoIP/",
         ] {
             let path = Path::new(path);
             let path = path.join(Self::filename());

sn0int-std/src/lib.rs

@@ -3,15 +3,16 @@ use hlua_badtouch as hlua;
 
 pub mod blobs;
 pub mod crt;
+pub mod crypto;
 mod errors;
 pub mod engine;
 pub mod geo;
 pub mod geoip;
 pub mod gfx;
 pub mod html;
-pub mod ipc;
 pub mod json;
 pub mod lazy;
+pub mod mqtt;
 pub mod psl;
 pub mod ratelimits;
 pub mod sockets;

sn0int-std/src/mqtt.rs

@@ -0,0 +1,228 @@
+use chrootable_https::DnsResolver;
+use crate::errors::*;
+use crate::hlua::AnyLuaValue;
+use mqtt::packet::VariablePacketError;
+use crate::json::LuaJsonValue;
+use crate::sockets::{Stream, SocketOptions};
+use mqtt::{TopicFilter, QualityOfService};
+use mqtt::control::ConnectReturnCode;
+use mqtt::control::fixed_header::FixedHeaderError;
+use mqtt::encodable::{Encodable, Decodable};
+use mqtt::packet::{Packet, VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
+use std::convert::TryFrom;
+use std::io;
+use std::net::SocketAddr;
+use url::Url;
+
+#[derive(Debug, Default, Deserialize)]
+pub struct MqttOptions {
+    pub username: Option<String>,
+    pub password: Option<String>,
+
+    pub proxy: Option<SocketAddr>,
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
+}
+
+impl MqttOptions {
+    pub fn try_from(x: AnyLuaValue) -> Result<MqttOptions> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub struct MqttClient {
+    stream: Stream,
+}
+
+impl MqttClient {
+    pub fn negotiate(stream: Stream, options: &MqttOptions) -> Result<MqttClient> {
+        let mut client = MqttClient {
+            stream,
+        };
+
+        let mut pkt = ConnectPacket::new("MQTT", "sn0int");
+        pkt.set_user_name(options.username.clone());
+        pkt.set_password(options.password.clone());
+
+        /*
+        if let Some(keep_alive) = msg.keep_alive {
+            packet.set_keep_alive(keep_alive);
+        }
+        */
+
+        client.send(pkt.into())?;
+        let pkt = client.recv()?;
+
+        if let VariablePacket::ConnackPacket(pkt) = pkt {
+            let code = pkt.connect_return_code();
+            if code == ConnectReturnCode::ConnectionAccepted {
+                Ok(client)
+            } else {
+                bail!("MQTT negotiation failed: {:?}", code);
+            }
+        } else {
+            bail!("Expected ConnAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &MqttOptions) -> Result<MqttClient> {
+        let tls = match url.scheme() {
+            "mqtt" => false,
+            "mqtts" => true,
+            _ => bail!("Invalid mqtt protocol"),
+        };
+
+        let host = url.host_str()
+            .ok_or_else(|| format_err!("Missing host in url"))?;
+
+        let port = match (url.port(), tls) {
+            (Some(port), _) => port,
+            (None, true) => 8883,
+            (None, false) => 1883,
+        };
+
+
+        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
+            tls,
+            sni_value: None,
+            disable_tls_verify: false,
+            proxy: options.proxy,
+
+            connect_timeout: options.connect_timeout,
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+        })?;
+
+        Self::negotiate(stream, options)
+    }
+
+    fn send(&mut self, pkt: VariablePacket) -> Result<()> {
+        debug!("Sending mqtt packet: {:?}", pkt);
+        pkt.encode(&mut self.stream)?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> std::result::Result<VariablePacket, VariablePacketError> {
+        let pkt = VariablePacket::decode(&mut self.stream)?;
+        debug!("Received mqtt packet: {:?}", pkt);
+        Ok(pkt)
+    }
+
+    pub fn subscribe(&mut self, topic: &str, qos: u8) -> Result<()> {
+        let filter = TopicFilter::new(topic)?;
+
+        let qos = match qos {
+            0 => QualityOfService::Level0,
+            1 => QualityOfService::Level1,
+            2 => QualityOfService::Level2,
+            _ => bail!("Invalid QoS level: {}", qos),
+        };
+
+        let pkt = SubscribePacket::new(1, vec![(filter, qos)]);
+        self.send(pkt.into())?;
+
+        let pkt = self.recv()?;
+        if let VariablePacket::SubackPacket(_pkt) = pkt {
+            Ok(())
+        } else {
+            bail!("Expected SubAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn recv_pkt(&mut self) -> Result<Option<Pkt>> {
+        match self.recv() {
+            Ok(pkt) => Ok(Some(Pkt::try_from(pkt)?)),
+            Err(VariablePacketError::IoError(err)) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(err) => Err(Error::from(err))
+        }
+    }
+
+    pub fn ping(&mut self) -> Result<()> {
+        let pkt = PingreqPacket::new();
+        self.send(pkt.into())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum Pkt {
+    #[serde(rename="publish")]
+    Publish(Publish),
+    #[serde(rename="pong")]
+    Pong,
+}
+
+impl Pkt {
+    pub fn to_lua(&self) -> Result<AnyLuaValue> {
+        let v = serde_json::to_value(&self)?;
+        let v = LuaJsonValue::from(v).into();
+        Ok(v)
+    }
+}
+
+impl TryFrom<VariablePacket> for Pkt {
+    type Error = Error;
+
+    fn try_from(pkt: VariablePacket) -> Result<Pkt> {
+        match pkt {
+            VariablePacket::ConnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::ConnackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PublishPacket(pkt) => Ok(Pkt::Publish(Publish {
+                topic: pkt.topic_name().to_string(),
+                body: pkt.payload(),
+            })),
+            VariablePacket::PubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrecPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrelPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubcompPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingreqPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingrespPacket(_) => Ok(Pkt::Pong),
+            VariablePacket::SubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::SubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::DisconnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Publish {
+    pub topic: String,
+    pub body: Vec<u8>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrootable_https::dns::Resolver;
+
+    fn connect() -> Result<MqttClient> {
+        let resolver = Resolver::from_system_v4().unwrap();
+        let url = "mqtt://mqtt.winkekatze24.de".parse()?;
+        MqttClient::connect(&resolver, url, &MqttOptions::default())
+    }
+
+    #[test]
+    #[ignore]
+    fn test_connect() {
+        connect().expect("Failed to setup connection");
+    }
+
+    // this test is too flaky
+    /*
+    #[test]
+    #[ignore]
+    fn test_subscribe() {
+        let mut c = connect().unwrap();
+        c.subscribe("#", 0).unwrap();
+    }
+    */
+}

sn0int-std/src/sockets/mod.rs

@@ -319,6 +319,7 @@ impl Socket {
                 let available = match self.stream.fill_buf() {
                     Ok(n) => n,
                     Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
+                    Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(Vec::new()),
                     Err(e) => return Err(e.into())
                 };
 
@@ -363,7 +364,7 @@ mod tests {
     #[ignore]
     #[test]
     fn verify_tls_good() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let _sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -373,7 +374,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_good_request() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let mut sock = Socket::connect(&resolver, "badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -386,7 +387,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_expired() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let sock = Socket::connect(&resolver, "expired.badssl.com", 443, &SocketOptions{
             tls: true,
             ..Default::default()
@@ -400,7 +401,7 @@ mod tests {
     #[test]
     #[ignore]
     fn verify_tls_1_1_1_1() {
-        let resolver = Resolver::from_system().unwrap();
+        let resolver = Resolver::from_system_v4().unwrap();
         let _sock = Socket::connect(&resolver, "1.1.1.1", 443, &SocketOptions{
             tls: true,
             ..Default::default()

sn0int-std/src/web.rs

@@ -72,6 +72,8 @@ pub struct RequestOptions {
     #[serde(default)]
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    #[serde(default)]
+    binary: bool,
 }
 
 impl RequestOptions {
@@ -101,6 +103,7 @@ pub struct HttpRequest {
     timeout: Option<Duration>,
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    binary: bool,
 }
 
 impl HttpRequest {
@@ -123,6 +126,7 @@ impl HttpRequest {
             timeout,
             into_blob: options.into_blob,
             proxy: options.proxy,
+            binary: options.binary,
         };
 
         if let Some(json) = options.json {
@@ -255,6 +259,8 @@ impl HttpRequest {
             let blob = Blob::create(res.body);
             let id = state.register_blob(blob);
             resp.insert_str("blob", id);
+        } else if self.binary {
+            resp.insert_serde("binary", &res.body[..])?;
         } else {
             resp.insert_str("text", String::from_utf8_lossy(&res.body));
         }

src/api.rs

@@ -22,10 +22,13 @@ pub struct Client {
 
 impl Client {
     pub fn new(config: &Config) -> Result<Client> {
-        let client = match config.network.proxy {
-            Some(proxy) => chrootable_https::Client::with_socks5(proxy),
-            _ => chrootable_https::Client::with_system_resolver()?,
+        let client = if let Some(proxy) = config.network.proxy {
+            chrootable_https::Client::with_socks5(proxy)
+        } else {
+            chrootable_https::Client::with_system_resolver()
+                .context("Failed to load dns configuration")?
         };
+
         Ok(Client {
             server: config.core.registry.clone(),
             client,
@@ -41,12 +44,20 @@ impl Client {
         thread_rng().sample_iter(&Alphanumeric).take(32).collect()
     }
 
+    #[inline]
+    fn user_agent() -> String {
+        format!("{}, {}",
+            web::default_user_agent(),
+            embedded_triple::get()
+        )
+    }
+
     pub fn request<T: DeserializeOwned + fmt::Debug>(&self, mut request: RequestBuilder, body: Body) -> Result<T> {
         if let Some(session) = &self.session {
             info!("Adding session token to request");
             request.header("Auth", session.as_str());
         }
-        request.header("User-Agent", web::default_user_agent());
+        request.header("User-Agent", Self::user_agent());
 
         let request = request.body(body)?;
 

src/args.rs

@@ -124,6 +124,9 @@ pub struct Run {
     /// Narrow down targeted entities
     #[structopt(short="t", long="target")]
     pub target: Option<String>,
+    /// Dump the sandbox init message to stdout instead of running a child process
+    #[structopt(long="dump-sandbox-init-msg")]
+    pub dump_sandbox_init_msg: bool,
 }
 
 #[derive(Debug, StructOpt)]

src/cmd/activity_cmd.rs

@@ -4,6 +4,7 @@ use crate::cmd::Cmd;
 use crate::shell::Shell;
 use crate::models::*;
 use chrono::{Utc, NaiveDateTime, NaiveTime, Duration};
+use regex::Regex;
 use std::convert::TryFrom;
 use std::io;
 use std::str::FromStr;
@@ -15,17 +16,36 @@ pub struct TimeSpec {
     datetime: NaiveDateTime,
 }
 
-impl FromStr for TimeSpec {
-    type Err = Error;
-
-    fn from_str(s: &str) -> Result<Self> {
-        let now = Utc::now().naive_utc();
+impl TimeSpec {
+    fn resolve(s: &str, now: NaiveDateTime) -> Result<Self> {
         let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
 
         let datetime = match s {
             "today" => today,
             "yesterday" => today - Duration::days(1),
-            // x {second,minute,hour,day,week,month,year}s? ago
+            s if s.ends_with(" ago") => {
+                let re = Regex::new(r"(\d+) ?(s|seconds?|m|min|minutes?|h|hours?|d|days?|w|weeks?|months?|y|years?) ago").unwrap();
+
+                let caps = re.captures(s)
+                    .ok_or_else(|| format_err!("Couldn't parse TimeSpec"))?;
+
+                let n = caps.get(1).unwrap().as_str()
+                    .parse::<i64>()
+                    .context("Failed to parse number in timespec")?;
+                let unit = caps.get(2).unwrap();
+
+                let duration = match unit.as_str() {
+                    "s" | "second" | "seconds" => Duration::seconds(n),
+                    "m" | "min" | "minute" | "minutes" => Duration::minutes(n),
+                    "h" | "hour" | "hours" => Duration::hours(n),
+                    "d" | "day" | "days" => Duration::days(n),
+                    "w" | "week" | "weeks" => Duration::days(n * 7),
+                    "month" | "months" => Duration::days(n * 31),
+                    "y" | "year" | "years" => Duration::days(n * 365),
+                    _ => unreachable!(),
+                };
+                now - duration
+            },
             s => NaiveDateTime::from_str(s)?,
         };
 
@@ -35,6 +55,15 @@ impl FromStr for TimeSpec {
     }
 }
 
+impl FromStr for TimeSpec {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        let now = Utc::now().naive_utc();
+        Self::resolve(s, now)
+    }
+}
+
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
@@ -86,3 +115,63 @@ impl Cmd for Args {
         Ok(())
     }
 }
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn datetime() -> NaiveDateTime {
+        let date = chrono::NaiveDate::from_ymd(2020, 3, 14);
+        let time = chrono::NaiveTime::from_hms(16, 20, 23);
+        NaiveDateTime::new(date, time)
+    }
+
+    #[test]
+    fn test_today() {
+        let x = TimeSpec::resolve("today", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_yesterday() {
+        let x = TimeSpec::resolve("yesterday", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-13T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_20_min_ago() {
+        let x = TimeSpec::resolve("20min ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:00:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_days_ago() {
+        let x = TimeSpec::resolve("3 days ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-11T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_week_ago() {
+        let x = TimeSpec::resolve("1w ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-07T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_months_ago() {
+        let x = TimeSpec::resolve("3 months ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-12-12T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_year_ago() {
+        let x = TimeSpec::resolve("1 year ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-03-15T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_exact_time() {
+        let x = TimeSpec::resolve("2020-03-14T16:20:23", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:20:23").unwrap());
+    }
+}

src/cmd/help_cmd.rs

@@ -16,9 +16,8 @@ pub fn run(_rl: &mut Shell, _args: &[String]) -> Result<()> {
     help("autoscope",   "Manage rules to automatically add entities to scope");
     help("delete",      "Delete entities from the database");
     help("keyring",     "Manage saved credentials");
-    help("mod",         "Manage installed modules");
+    help("pkg",         "Manage installed modules");
     help("noscope",     "Exclude entities from scope");
-    help("quickstart",  "Install all featured modules");
     help("run",         "Run the currently selected module");
     help("scope",       "Include entities in the scope again");
     help("select",      "Select entities from the database");

src/cmd/pkg_cmd.rs

@@ -1,10 +1,12 @@
 use crate::errors::*;
 
+use crate::args::Install;
+use crate::api::Client;
 use crate::args;
 use crate::config::Config;
 use crate::cmd::{Cmd, LiteCmd};
 use crate::engine::Library;
-use crate::registry::{self, UpdateTask, Updater};
+use crate::registry::{self, InstallTask, UpdateTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
 use crate::worker;
@@ -47,6 +49,9 @@ pub enum SubCommand {
     /// Uninstall a module
     #[structopt(name="uninstall")]
     Uninstall(Uninstall),
+    /// Install all featured modules
+    #[structopt(name="quickstart")]
+    Quickstart,
 }
 
 #[derive(Debug, StructOpt)]
@@ -66,6 +71,9 @@ pub struct List {
     /// List outdated modules
     #[structopt(long="outdated")]
     pub outdated: bool,
+    /// Filter by pattern
+    #[structopt(default_value="*")]
+    pub pattern: String,
 }
 
 #[derive(Debug, StructOpt)]
@@ -92,6 +100,8 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
         SubCommand::List(list) => {
             let autoupdate = AutoUpdater::load()?;
 
+            let filter = glob::Pattern::new(&list.pattern)?;
+
             for module in library.list() {
                 if let Some(source) = &list.source {
                     if !module.source_equals(&source) {
@@ -100,6 +110,9 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
                 }
 
                 let canonical = module.canonical();
+                if !filter.matches(&canonical) {
+                    continue;
+                }
 
                 let mut out = String::new();
                 write!(&mut out, "{} ({})", canonical.green(),
@@ -156,6 +169,34 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
             // trigger reload
             Ok(ModuleReload::Yes)
         },
+        SubCommand::Quickstart => {
+            let client = Client::new(&config)?;
+            let updater = Arc::new(Updater::new(&config)?);
+            let mut autoupdate = AutoUpdater::load()?;
+
+            let modules = client.quickstart()?
+                .into_iter()
+                .map(|module| {
+                    InstallTask::new(Install {
+                        module: ModuleID {
+                            author: module.author,
+                            name: module.name,
+                        },
+                        version: None,
+                        force: false,
+                    }, updater.clone())
+                })
+                .collect::<Vec<_>>();
+
+            worker::spawn_multi(modules, |name| {
+                autoupdate.updated(&name);
+            }, 3)?;
+
+            autoupdate.save()?;
+
+            // trigger reload
+            Ok(ModuleReload::Yes)
+        },
     }
 }
 

src/cmd/quickstart_cmd.rs

@@ -1,15 +1,11 @@
 use crate::errors::*;
 
-use crate::args::Install;
-use crate::api::Client;
-use crate::registry::{InstallTask, Updater};
+use crate::cmd::Cmd;
+use crate::cmd::pkg_cmd::{ArgsInteractive as PkgArgs, SubCommand, SubCommandInteractive};
 use crate::shell::Shell;
-use crate::update::AutoUpdater;
-use crate::worker;
-use std::sync::Arc;
+use crate::term;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
-use sn0int_common::ModuleID;
 
 
 #[derive(Debug, StructOpt)]
@@ -19,34 +15,11 @@ pub struct Args {
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     let _args = Args::from_iter_safe(args)?;
-    let config = rl.config().clone();
 
-    let client = Client::new(&config)?;
-    let updater = Arc::new(Updater::new(&config)?);
-    let mut autoupdate = AutoUpdater::load()?;
+    term::warn("The \x1b[1mquickstart\x1b[0m command is deprecated, use \x1b[1mpkg quickstart\x1b[0m");
 
-    let modules = client.quickstart()?
-        .into_iter()
-        .map(|module| {
-            InstallTask::new(Install {
-                module: ModuleID {
-                    author: module.author,
-                    name: module.name,
-                },
-                version: None,
-                force: false,
-            }, updater.clone())
-        })
-        .collect::<Vec<_>>();
-
-    worker::spawn_multi(modules, |name| {
-        autoupdate.updated(&name);
-    }, 3)?;
-
-    autoupdate.save()?;
-
-    // trigger reload
-    rl.reload_modules()?;
-
-    Ok(())
+    let args = PkgArgs {
+        subcommand: SubCommandInteractive::Base(SubCommand::Quickstart),
+    };
+    args.run(rl)
 }

src/cmd/run_cmd.rs

@@ -1,10 +1,12 @@
 use crate::errors::*;
 
+use chrootable_https::dns::Resolver;
 use crate::args;
 use crate::blobs::{Blob, BlobStorage};
 use crate::cmd::Cmd;
 use crate::db::{ttl, Filter};
 use crate::engine::Module;
+use crate::ipc::common::StartCommand;
 use crate::models::*;
 use crate::shell::Shell;
 use crate::keyring::KeyRing;
@@ -113,16 +115,10 @@ fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> R
     Ok(())
 }
 
-pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
-
-    prepare_keyring(rl.keyring_mut(), &module, &params)?;
-
+fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, Option<String>, Vec<Blob>)>> {
     let filter = rl.scoped_targets();
 
-    let args = match module.source() {
+    match module.source() {
         Some(Source::Domains) => prepare_args::<Domain>(rl, &filter, None),
         Some(Source::Subdomains) => prepare_args::<Subdomain>(rl, &filter, None),
         Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl, &filter, None),
@@ -152,7 +148,44 @@ pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>)
             }
         },
         None => Ok(vec![(serde_json::Value::Null, None, vec![])]),
-    }?;
+    }
+}
+
+pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let keyring = rl.keyring().request_keys(&module);
+
+    let dns_config = Resolver::from_system_v4()?;
+    let proxy = rl.config().network.proxy.clone();
+
+    let args = get_args(rl, &module)?;
+    for (arg, _pretty_arg, blobs) in args {
+        let start_cmd = StartCommand::new(params.verbose,
+                                          keyring.clone(),
+                                          dns_config.clone(),
+                                          proxy.clone(),
+                                          options.clone(),
+                                          module.clone(),
+                                          arg,
+                                          blobs);
+        let out = serde_json::to_string(&start_cmd)?;
+        println!("{}", out);
+    }
+
+    Ok(())
+}
+
+pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
     let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);

src/cmd/select_cmd.rs

@@ -22,12 +22,17 @@ pub struct Args {
     /// Print paths to blobs
     #[structopt(long="paths", group="output")]
     paths: bool,
+    /// Count rows returned
+    #[structopt(short="c", long="count", group="output")]
+    count: bool,
 }
 
+#[derive(PartialEq)]
 enum Output {
     Normal,
     Json,
     Paths,
+    Count,
 }
 
 struct Printer<'a, 'b> {
@@ -41,6 +46,8 @@ impl<'a, 'b> Printer<'a, 'b> {
             Output::Json
         } else if args.paths {
             Output::Paths
+        } else if args.count {
+            Output::Count
         } else {
             Output::Normal
         };
@@ -52,25 +59,32 @@ impl<'a, 'b> Printer<'a, 'b> {
     }
 
     pub fn select<T: Model + Detailed + Serialize>(&self, filter: &Filter) -> Result<()> {
-        for obj in self.rl.db().filter::<T>(&filter.parse_optional()?)? {
-            match self.output {
-                Output::Normal => println!("{}", obj.detailed(self.rl.db())?),
-                Output::Json => {
-                    let v = serde_json::to_string(&obj)?;
-                    println!("{}", v);
-                },
-                Output::Paths => {
-                    let blob = obj.blob()
-                        .ok_or_else(|| format_err!("This model isn't linked to blob storage"))?;
+        let query = self.rl.db().filter::<T>(&filter.parse_optional()?)?;
 
-                    let path = self.rl.blobs()
-                        .join(blob)?;
+        if self.output == Output::Count {
+            println!("{}", query.len());
+        } else {
+            for obj in query {
+                match self.output {
+                    Output::Normal => println!("{}", obj.detailed(self.rl.db())?),
+                    Output::Json => {
+                        let v = serde_json::to_string(&obj)?;
+                        println!("{}", v);
+                    },
+                    Output::Paths => {
+                        let blob = obj.blob()
+                            .ok_or_else(|| format_err!("This model isn't linked to blob storage"))?;
 
-                    let path = path.to_str()
-                        .ok_or_else(|| format_err!("Path is invalid utf-8"))?;
+                        let path = self.rl.blobs()
+                            .join(blob)?;
 
-                    println!("{}", path);
-                },
+                        let path = path.to_str()
+                            .ok_or_else(|| format_err!("Path is invalid utf-8"))?;
+
+                        println!("{}", path);
+                    },
+                    Output::Count => unreachable!(),
+                }
             }
         }
 

src/db/mod.rs

@@ -304,7 +304,7 @@ impl Database {
         Ok(ipaddr_update.id)
     }
 
-    pub fn update_url(&self, url_update: &UrlUpdate) -> Result<i32> {
+    pub fn update_url(&self, url_update: &UrlChangeset) -> Result<i32> {
         use crate::schema::urls::columns::*;
         diesel::update(urls::table.filter(id.eq(url_update.id)))
             .set(url_update)

src/engine/ctx.rs

@@ -1,7 +1,7 @@
 use crate::errors::*;
 
 use crate::db::Family;
-use crate::engine::{Environment, Reporter};
+use crate::engine::{Environment, IpcChild};
 use crate::geoip::{MaxmindReader, GeoIP, AsnDB};
 use crate::hlua::{self, AnyLuaValue};
 use crate::keyring::KeyRingEntry;
@@ -17,6 +17,7 @@ use crate::ratelimits::RatelimitResponse;
 use chrootable_https::{self, Resolver};
 use serde_json;
 use sn0int_std::blobs::{Blob, BlobState};
+use sn0int_std::mqtt::{MqttClient, MqttOptions};
 use sn0int_std::web::WebState;
 use std::collections::HashMap;
 use std::result;
@@ -178,6 +179,10 @@ pub trait State {
 
     fn get_ws(&self, id: &str)-> Arc<Mutex<WebSocket>>;
 
+    fn mqtt_connect(&self, url: url::Url, options: &MqttOptions) -> Result<String>;
+
+    fn get_mqtt(&self, id: &str)-> Arc<Mutex<MqttClient>>;
+
     fn http_mksession(&self) -> String;
 
     fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest;
@@ -196,9 +201,10 @@ pub trait State {
 // #[derive(Debug)]
 pub struct LuaState {
     error: Mutex<Option<Error>>,
-    logger: Arc<Mutex<Box<dyn Reporter>>>,
+    logger: Arc<Mutex<Box<dyn IpcChild>>>,
     socket_sessions: Mutex<HashMap<String, Arc<Mutex<Socket>>>>,
     ws_sessions: Mutex<HashMap<String, Arc<Mutex<WebSocket>>>>,
+    mqtt_sessions: Mutex<HashMap<String, Arc<Mutex<MqttClient>>>>,
     blobs: Mutex<HashMap<String, Arc<Blob>>>,
     http_sessions: Mutex<HashMap<String, HttpSession>>,
     http_clients: Mutex<HashMap<String, Arc<chrootable_https::Client<Resolver>>>>,
@@ -340,6 +346,22 @@ impl State for LuaState {
         sock.clone()
     }
 
+    fn mqtt_connect(&self, url: url::Url, options: &MqttOptions) -> Result<String> {
+        let mut mtx = self.mqtt_sessions.lock().unwrap();
+        let id = self.random_id();
+
+        let sock = MqttClient::connect(&self.dns_config, url, options)?;
+        mtx.insert(id.clone(), Arc::new(Mutex::new(sock)));
+
+        Ok(id)
+    }
+
+    fn get_mqtt(&self, id: &str)-> Arc<Mutex<MqttClient>> {
+        let mtx = self.mqtt_sessions.lock().unwrap();
+        let sock = mtx.get(id).expect("Invalid mqtt reference"); // TODO
+        sock.clone()
+    }
+
     fn http_mksession(&self) -> String {
         let mut mtx = self.http_sessions.lock().unwrap();
         let (id, session) = HttpSession::new();
@@ -426,7 +448,7 @@ pub struct Script {
     code: String,
 }
 
-pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
+pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
     debug!("Creating lua context");
     let mut lua = hlua::Lua::new();
     lua.open_string();
@@ -439,6 +461,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
         logger,
         socket_sessions: Mutex::new(HashMap::new()),
         ws_sessions: Mutex::new(HashMap::new()),
+        mqtt_sessions: Mutex::new(HashMap::new()),
         blobs: Mutex::new(HashMap::new()),
         http_sessions: Mutex::new(HashMap::new()),
         http_clients: Mutex::new(HashMap::new()),
@@ -501,9 +524,14 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::json_decode(&mut lua, state.clone());
     runtime::json_decode_stream(&mut lua, state.clone());
     runtime::json_encode(&mut lua, state.clone());
+    runtime::key_trunc_pad(&mut lua, state.clone());
     runtime::keyring(&mut lua, state.clone());
     runtime::last_err(&mut lua, state.clone());
     runtime::md5(&mut lua, state.clone());
+    runtime::mqtt_connect(&mut lua, state.clone());
+    runtime::mqtt_subscribe(&mut lua, state.clone());
+    runtime::mqtt_recv(&mut lua, state.clone());
+    runtime::mqtt_ping(&mut lua, state.clone());
     runtime::pgp_pubkey(&mut lua, state.clone());
     runtime::pgp_pubkey_armored(&mut lua, state.clone());
     runtime::print(&mut lua, state.clone());
@@ -536,6 +564,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::sock_recvuntil(&mut lua, state.clone());
     runtime::sock_sendafter(&mut lua, state.clone());
     runtime::sock_newline(&mut lua, state.clone());
+    runtime::sodium_secretbox_open(&mut lua, state.clone());
     runtime::status(&mut lua, state.clone());
     runtime::stdin_read_line(&mut lua, state.clone());
     runtime::stdin_read_to_end(&mut lua, state.clone());
@@ -592,7 +621,7 @@ impl Script {
     }
 
     pub fn run(&self, env: Environment,
-                      tx: Arc<Mutex<Box<dyn Reporter>>>,
+                      tx: Arc<Mutex<Box<dyn IpcChild>>>,
                       arg: AnyLuaValue,
     ) -> Result<()> {
         let (mut lua, state) = ctx(env, tx);
@@ -623,11 +652,11 @@ impl Script {
 
     #[cfg(test)]
     pub fn test(&self) -> Result<()> {
-        use crate::engine::DummyReporter;
+        use crate::ipc::child::DummyIpcChild;
         use crate::geoip::Maxmind;
         use crate::paths;
         let keyring = Vec::new();
-        let dns_config = Resolver::from_system()?;
+        let dns_config = Resolver::from_system_v4()?;
         let proxy = None;
         let psl = PslReader::String(r#"
 // ===BEGIN ICANN DOMAINS===
@@ -652,6 +681,6 @@ a.prod.fastly.net
             geoip,
             asn,
         };
-        self.run(env, DummyReporter::new(), AnyLuaValue::LuaNil)
+        self.run(env, DummyIpcChild::new(), AnyLuaValue::LuaNil)
     }
 }

src/engine/isolation.rs

@@ -1,252 +0,0 @@
-use crate::errors::*;
-use chrootable_https::dns::Resolver;
-use crate::blobs::Blob;
-use crate::engine::{Environment, Module, Reporter};
-use crate::geoip::MaxmindReader;
-use crate::keyring::KeyRingEntry;
-use crate::psl::PslReader;
-use serde_json;
-use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
-
-use std::collections::HashMap;
-use std::env;
-use std::ffi::OsString;
-use std::io::prelude::*;
-use std::io::{self, BufReader, BufRead, stdin, Stdin, Stdout};
-use std::net::SocketAddr;
-use std::sync::{mpsc, Arc, Mutex};
-use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
-
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct StartCommand {
-    verbose: u64,
-    keyring: Vec<KeyRingEntry>,
-    dns_config: Resolver,
-    proxy: Option<SocketAddr>,
-    options: HashMap<String, String>,
-    module: Module,
-    arg: serde_json::Value,
-    blobs: Vec<Blob>,
-}
-
-impl StartCommand {
-    pub fn new(verbose: u64,
-               keyring: Vec<KeyRingEntry>,
-               dns_config: Resolver,
-               proxy: Option<SocketAddr>,
-               options: HashMap<String, String>,
-               module: Module,
-               arg: serde_json::Value,
-               blobs: Vec<Blob>,
-    ) -> StartCommand {
-        StartCommand {
-            verbose,
-            keyring,
-            dns_config,
-            proxy,
-            options,
-            module,
-            arg,
-            blobs,
-        }
-    }
-}
-
-pub struct Supervisor {
-    child: Child,
-    stdin: ChildStdin,
-    stdout: BufReader<ChildStdout>,
-}
-
-impl Supervisor {
-    pub fn setup(module: &Module) -> Result<Supervisor> {
-        let exe = match env::current_exe() {
-            Ok(exe) => exe.into_os_string(),
-            _ => OsString::from("sn0int"),
-        };
-
-        let mut child = Command::new(exe)
-            .arg("sandbox")
-            .arg(&module.canonical())
-            .stdin(Stdio::piped())
-            .stdout(Stdio::piped())
-            .spawn()
-            .context("Failed to spawn child process")?;
-
-        let stdin = child.stdin.take().expect("Failed to take child stdin");
-        let stdout = child.stdout.take().expect("Failed to take child stdout");
-        let stdout = BufReader::new(stdout);
-
-        Ok(Supervisor {
-            child,
-            stdin,
-            stdout,
-        })
-    }
-
-    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
-        let start = serde_json::to_value(&start)?;
-        self.send(&start)?;
-        Ok(())
-    }
-
-    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
-        let mut value = serde_json::to_string(value)?;
-        value.push('\n');
-        self.stdin.write_all(value.as_bytes())?;
-        debug!("Supervisor sent: {:?}", value);
-        Ok(())
-    }
-
-    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
-        let value = serde_json::to_value(value).expect("Failed to serialize reply");
-        if let Err(_) = self.send(&value) {
-            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
-        }
-    }
-
-    pub fn recv(&mut self) -> Result<Event> {
-        let mut line = String::new();
-        let len = self.stdout.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Supervisor received: {:?}", event);
-        Ok(event)
-    }
-
-    pub fn wait(&mut self) -> Result<()> {
-        let exit = self.child.wait()
-            .context("Failed to wait for child")?;
-
-        if exit.success() {
-            Ok(())
-        } else {
-            bail!("Child signaled error")
-        }
-    }
-
-    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
-        where <T as EventWithCallback>::Payload: serde::Serialize
-    {
-        let (tx2, rx2) = mpsc::channel();
-        tx.send(event.with_callback(tx2));
-        let reply = rx2.recv().unwrap();
-
-        self.send_struct(reply, tx);
-    }
-}
-
-#[derive(Debug)]
-pub struct StdioReporter {
-    stdin: Stdin,
-    stdout: Stdout,
-}
-
-impl StdioReporter {
-    pub fn setup() -> StdioReporter {
-        let stdin = io::stdin();
-        let stdout = io::stdout();
-
-        StdioReporter {
-            stdin,
-            stdout,
-        }
-    }
-
-    pub fn recv_start(&mut self) -> Result<StartCommand> {
-        let value = self.recv()?;
-        let event = serde_json::from_value(value)?;
-        Ok(event)
-    }
-}
-
-impl Reporter for StdioReporter {
-    fn send(&mut self, event: &Event) -> Result<()> {
-        let mut event = serde_json::to_string(&event)?;
-        event.push('\n');
-        self.stdout.write_all(event.as_bytes())?;
-        debug!("Reporter sent: {:?}", event);
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        let mut line = String::new();
-        let len = self.stdin.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Reporter received: {:?}", event);
-        Ok(event)
-    }
-}
-
-pub fn spawn_module(module: Module,
-                    tx: &EventSender,
-                    arg: serde_json::Value,
-                    keyring: Vec<KeyRingEntry>,
-                    verbose: u64,
-                    has_stdin: bool,
-                    proxy: Option<SocketAddr>,
-                    options: HashMap<String, String>,
-                    blobs: Vec<Blob>,
-) -> Result<ExitEvent> {
-    let dns_config = Resolver::from_system()?;
-
-    let mut reader = if has_stdin {
-        Some(BufReader::new(stdin()))
-    } else {
-        None
-    };
-
-    let mut supervisor = Supervisor::setup(&module)?;
-    supervisor.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
-
-    let exit = loop {
-        match supervisor.recv()? {
-            Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => supervisor.send_event_callback(object, &tx),
-            Event::Stdio(object) => object.apply(&mut supervisor, tx, &mut reader),
-            Event::Ratelimit(req) => supervisor.send_event_callback(req, &tx),
-            Event::Blob(blob) => supervisor.send_event_callback(blob, &tx),
-            Event::Exit(event) => {
-                if let ExitEvent::Err(err) = &event {
-                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
-                }
-                break event;
-            },
-        }
-    };
-
-    supervisor.wait()?;
-
-    Ok(exit)
-}
-
-pub fn run_worker(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslReader) -> Result<()> {
-    let mut reporter = StdioReporter::setup();
-    let start = reporter.recv_start()?;
-
-    let environment = Environment {
-        verbose: start.verbose,
-        keyring: start.keyring,
-        dns_config: start.dns_config,
-        proxy: start.proxy,
-        options: start.options,
-        blobs: start.blobs,
-        psl,
-        geoip,
-        asn,
-    };
-
-    let mtx: Arc<Mutex<Box<dyn Reporter>>> = Arc::new(Mutex::new(Box::new(reporter)));
-    let result = start.module.run(environment,
-                                  mtx.clone(),
-                                  start.arg.into());
-    let mut reporter = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
-                        .into_inner().expect("Failed to consume Mutex");
-
-    let event = result.into();
-    reporter.send(&Event::Exit(event))?;
-
-    Ok(())
-}

src/engine/mod.rs

@@ -5,7 +5,6 @@ use crate::config::Config;
 use crate::geoip::MaxmindReader;
 use crate::json::LuaJsonValue;
 use crate::keyring::KeyRingEntry;
-use serde_json;
 use std::fs;
 use std::fmt::Debug;
 use std::path::PathBuf;
@@ -13,6 +12,7 @@ use std::collections::HashMap;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
 use crate::engine::ctx::Script;
+use crate::ipc::child::IpcChild;
 use sn0int_common::ModuleID;
 use sn0int_common::metadata::{Metadata, Source};
 use chrootable_https::dns::Resolver;
@@ -21,10 +21,9 @@ use crate::paths;
 use std::cmp::Ordering;
 use std::path::Path;
 use crate::term;
-use crate::worker::{self, Event};
+use crate::worker;
 
 pub mod ctx;
-pub mod isolation;
 pub use sn0int_std::engine::structs;
 
 
@@ -299,9 +298,9 @@ impl Module {
         self.private_module
     }
 
-    pub fn run(&self, env: Environment, reporter: Arc<Mutex<Box<dyn Reporter>>>, arg: LuaJsonValue) -> Result<()> {
+    pub fn run(&self, env: Environment, ipc_child: Arc<Mutex<Box<dyn IpcChild>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
-        self.script.run(env, reporter, arg.into())
+        self.script.run(env, ipc_child, arg.into())
     }
 
     #[inline]
@@ -320,28 +319,3 @@ impl Module {
         }
     }
 }
-
-pub trait Reporter: Debug {
-    fn send(&mut self, event: &Event) -> Result<()>;
-
-    fn recv(&mut self) -> Result<serde_json::Value>;
-}
-
-#[derive(Debug)]
-pub struct DummyReporter;
-
-impl DummyReporter {
-    pub fn new() -> Arc<Mutex<Box<dyn Reporter>>> {
-        Arc::new(Mutex::new(Box::new(DummyReporter)))
-    }
-}
-
-impl Reporter for DummyReporter {
-    fn send(&mut self, _event: &Event) -> Result<()> {
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        unimplemented!("DummyReporter::recv doesn't exist")
-    }
-}

src/ipc/child.rs

@@ -0,0 +1,110 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use crate::engine::Environment;
+use crate::geoip::MaxmindReader;
+use crate::psl::PslReader;
+use serde_json;
+use crate::worker::Event;
+
+use std::fmt::Debug;
+use std::io::prelude::*;
+use std::io::{self, Stdin, Stdout};
+use std::sync::{Arc, Mutex};
+
+
+pub trait IpcChild: Debug {
+    fn send(&mut self, event: &Event) -> Result<()>;
+
+    fn recv(&mut self) -> Result<serde_json::Value>;
+}
+
+#[derive(Debug)]
+pub struct StdioIpcChild {
+    stdin: Stdin,
+    stdout: Stdout,
+}
+
+impl StdioIpcChild {
+    pub fn setup() -> StdioIpcChild {
+        let stdin = io::stdin();
+        let stdout = io::stdout();
+
+        StdioIpcChild {
+            stdin,
+            stdout,
+        }
+    }
+
+    pub fn recv_start(&mut self) -> Result<StartCommand> {
+        let value = self.recv()?;
+        let event = serde_json::from_value(value)?;
+        Ok(event)
+    }
+}
+
+impl IpcChild for StdioIpcChild {
+    fn send(&mut self, event: &Event) -> Result<()> {
+        let mut event = serde_json::to_string(&event)?;
+        event.push('\n');
+        debug!("IpcChild sends: {:?}", event);
+        self.stdout.write_all(event.as_bytes())?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        let mut line = String::new();
+        let len = self.stdin.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcChild received: {:?}", event);
+        Ok(event)
+    }
+}
+
+#[derive(Debug)]
+pub struct DummyIpcChild;
+
+impl DummyIpcChild {
+    pub fn new() -> Arc<Mutex<Box<dyn IpcChild>>> {
+        Arc::new(Mutex::new(Box::new(DummyIpcChild)))
+    }
+}
+
+impl IpcChild for DummyIpcChild {
+    fn send(&mut self, _event: &Event) -> Result<()> {
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        unimplemented!("DummyIpcChild::recv doesn't exist")
+    }
+}
+
+pub fn run(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslReader) -> Result<()> {
+    let mut ipc_child = StdioIpcChild::setup();
+    let start = ipc_child.recv_start()?;
+
+    let environment = Environment {
+        verbose: start.verbose,
+        keyring: start.keyring,
+        dns_config: start.dns_config,
+        proxy: start.proxy,
+        options: start.options,
+        blobs: start.blobs,
+        psl,
+        geoip,
+        asn,
+    };
+
+    let mtx: Arc<Mutex<Box<dyn IpcChild>>> = Arc::new(Mutex::new(Box::new(ipc_child)));
+    let result = start.module.run(environment,
+                                  mtx.clone(),
+                                  start.arg.into());
+    let mut ipc_child = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
+                        .into_inner().expect("Failed to consume Mutex");
+
+    let event = result.into();
+    ipc_child.send(&Event::Exit(event))?;
+
+    Ok(())
+}

src/ipc/common.rs

@@ -0,0 +1,44 @@
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+
+use std::collections::HashMap;
+use std::net::SocketAddr;
+
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct StartCommand {
+    pub verbose: u64,
+    pub keyring: Vec<KeyRingEntry>,
+    pub dns_config: Resolver,
+    pub proxy: Option<SocketAddr>,
+    pub options: HashMap<String, String>,
+    pub module: Module,
+    pub arg: serde_json::Value,
+    pub blobs: Vec<Blob>,
+}
+
+impl StartCommand {
+    pub fn new(verbose: u64,
+               keyring: Vec<KeyRingEntry>,
+               dns_config: Resolver,
+               proxy: Option<SocketAddr>,
+               options: HashMap<String, String>,
+               module: Module,
+               arg: serde_json::Value,
+               blobs: Vec<Blob>,
+    ) -> StartCommand {
+        StartCommand {
+            verbose,
+            keyring,
+            dns_config,
+            proxy,
+            options,
+            module,
+            arg,
+            blobs,
+        }
+    }
+}

src/ipc/mod.rs

@@ -0,0 +1,3 @@
+pub mod child;
+pub mod common;
+pub mod parent;

src/ipc/parent.rs

@@ -0,0 +1,144 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
+
+use std::collections::HashMap;
+use std::env;
+use std::ffi::OsString;
+use std::io::prelude::*;
+use std::io::{BufReader, BufRead, stdin};
+use std::net::SocketAddr;
+use std::sync::mpsc;
+use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
+
+
+pub struct IpcParent {
+    child: Child,
+    stdin: ChildStdin,
+    stdout: BufReader<ChildStdout>,
+}
+
+impl IpcParent {
+    pub fn setup(module: &Module) -> Result<IpcParent> {
+        let exe = match env::current_exe() {
+            Ok(exe) => exe.into_os_string(),
+            _ => OsString::from("sn0int"),
+        };
+
+        let mut child = Command::new(exe)
+            .arg("sandbox")
+            .arg(&module.canonical())
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .context("Failed to spawn child process")?;
+
+        let stdin = child.stdin.take().expect("Failed to take child stdin");
+        let stdout = child.stdout.take().expect("Failed to take child stdout");
+        let stdout = BufReader::new(stdout);
+
+        Ok(IpcParent {
+            child,
+            stdin,
+            stdout,
+        })
+    }
+
+    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
+        let start = serde_json::to_value(&start)?;
+        self.send(&start)?;
+        Ok(())
+    }
+
+    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
+        let mut value = serde_json::to_string(value)?;
+        value.push('\n');
+        self.stdin.write_all(value.as_bytes())?;
+        debug!("IpcParent sent: {:?}", value);
+        Ok(())
+    }
+
+    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
+        let value = serde_json::to_value(value).expect("Failed to serialize reply");
+        if let Err(_) = self.send(&value) {
+            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
+        }
+    }
+
+    pub fn recv(&mut self) -> Result<Event> {
+        let mut line = String::new();
+        let len = self.stdout.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcParent received: {:?}", event);
+        Ok(event)
+    }
+
+    pub fn wait(&mut self) -> Result<()> {
+        let exit = self.child.wait()
+            .context("Failed to wait for child")?;
+
+        if exit.success() {
+            Ok(())
+        } else {
+            bail!("Child signaled error")
+        }
+    }
+
+    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
+        where <T as EventWithCallback>::Payload: serde::Serialize
+    {
+        let (tx2, rx2) = mpsc::channel();
+        tx.send(event.with_callback(tx2));
+        let reply = rx2.recv().unwrap();
+
+        self.send_struct(reply, tx);
+    }
+}
+
+pub fn run(module: Module,
+           tx: &EventSender,
+           arg: serde_json::Value,
+           keyring: Vec<KeyRingEntry>,
+           verbose: u64,
+           has_stdin: bool,
+           proxy: Option<SocketAddr>,
+           options: HashMap<String, String>,
+           blobs: Vec<Blob>,
+) -> Result<ExitEvent> {
+    let dns_config = Resolver::from_system_v4()?;
+
+    let mut reader = if has_stdin {
+        Some(BufReader::new(stdin()))
+    } else {
+        None
+    };
+
+    let mut ipc_parent = IpcParent::setup(&module)?;
+    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
+
+    let exit = loop {
+        match ipc_parent.recv()? {
+            Event::Log(event) => tx.send(Event2::Log(event)),
+            Event::Database(object) => ipc_parent.send_event_callback(object, &tx),
+            Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
+            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
+            Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),
+            Event::Exit(event) => {
+                if let ExitEvent::Err(err) = &event {
+                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
+                }
+                break event;
+            },
+        }
+    };
+
+    ipc_parent.wait()?;
+
+    Ok(exit)
+}

src/keyring.rs

@@ -66,7 +66,7 @@ impl fmt::Display for KeyName {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Default, Serialize, Deserialize)]
 pub struct KeyRing {
     keys: HashMap<String, HashMap<String, Option<String>>>,
     grants: HashMap<String, HashSet<ModuleID>>,
@@ -82,18 +82,19 @@ impl KeyRing {
     pub fn init() -> Result<KeyRing> {
         let path = Self::path()?;
 
-        if path.exists() {
+        let keyring = if path.exists() {
             Self::load(&path)
+                .context("Failed to load keyring")?
         } else {
-            Ok(KeyRing {
-                keys: HashMap::new(),
-                grants: HashMap::new(),
-            })
-        }
+            KeyRing::default()
+        };
+
+        Ok(keyring)
     }
 
     pub fn load(path: &Path) -> Result<KeyRing> {
-        let buf = fs::read(&path)?;
+        let buf = fs::read(&path)
+            .context("Failed to read keyring file")?;
         serde_json::from_slice(&buf)
             .map_err(Error::from)
     }
@@ -101,7 +102,8 @@ impl KeyRing {
     pub fn save(&self) -> Result<()> {
         let path = Self::path()?;
         let buf = serde_json::to_string(&self)?;
-        fs::write(&path, buf)?;
+        fs::write(&path, buf)
+            .context("Failed to save keyring")?;
         Ok(())
     }
 

src/lib.rs

@@ -27,10 +27,12 @@ pub use sn0int_std::geoip;
 use sn0int_std::gfx;
 use sn0int_std::html;
 use sn0int_std::json;
+pub mod ipc;
 pub mod keyring;
 use sn0int_std::lazy;
 pub mod migrations;
 pub mod models;
+use sn0int_std::mqtt;
 pub mod paths;
 pub use sn0int_std::psl;
 pub mod options;

src/main.rs

@@ -5,8 +5,9 @@ use sn0int::cmd::{self, LiteCmd};
 use sn0int::config::Config;
 use sn0int::db;
 use sn0int::errors::*;
-use sn0int::engine::{self, Module};
+use sn0int::engine::Module;
 use sn0int::geoip::{GeoIP, AsnDB, Maxmind};
+use sn0int::ipc;
 use sn0int::options::Opt;
 use sn0int::paths;
 use sn0int::psl::PslReader;
@@ -46,7 +47,11 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         rl.set_target(Some(target));
     }
 
-    cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+    if args.dump_sandbox_init_msg {
+        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, args.into(), Opt::collect(&args.options))
+    } else {
+        cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+    }
 }
 
 fn run_sandbox() -> Result<()> {
@@ -57,7 +62,7 @@ fn run_sandbox() -> Result<()> {
 
     sandbox::init()
         .context("Failed to init sandbox")?;
-    engine::isolation::run_worker(geoip, asn, psl)
+    ipc::child::run(geoip, asn, psl)
 }
 
 fn run_cmd<T: cmd::Cmd>(gargs: &Args, args: T, config: &Config) -> Result<()> {

src/models/mod.rs

@@ -119,7 +119,7 @@ impl From<&Insert> for Table {
 pub enum Update {
     Subdomain(SubdomainUpdate),
     IpAddr(IpAddrUpdate),
-    Url(UrlUpdate),
+    Url(UrlChangeset),
     Email(EmailUpdate),
     PhoneNumber(PhoneNumberUpdate),
     Device(DeviceUpdate),
@@ -374,6 +374,10 @@ impl<T: InsertToNew> LuaInsertToNew for T {
     }
 }
 
+pub trait UpdateToChangeset<T> {
+    fn try_into_changeset(self) -> Result<T>;
+}
+
 mod domain;
 pub use self::domain::*;
 

src/models/netblock.rs

@@ -159,7 +159,7 @@ impl DisplayableDetailed for DetailedNetblock {
     #[inline]
     fn print(&self, w: &mut fmt::DetailFormatter) -> fmt::Result {
         w.id(self.id)?;
-        w.debug::<Green, _>(&self.value)?;
+        w.display::<Green, _>(&self.value)?;
 
         w.start_group();
         w.opt_debug::<Yellow, _>(&self.asn)?;

src/models/url.rs

@@ -1,10 +1,10 @@
+use crate::ser::StringOrBytes;
 use crate::errors::*;
 use crate::fmt::Write;
 use crate::fmt::colors::*;
 use crate::models::*;
 use diesel;
 use diesel::prelude::*;
-use crate::ser;
 use crate::url;
 
 
@@ -228,7 +228,6 @@ pub struct NewUrl {
     pub value: String,
     pub path: String,
     pub status: Option<i32>,
-    #[serde(deserialize_with="ser::opt_string_or_bytes")]
     pub body: Option<Vec<u8>>,
     pub online: Option<bool>,
     pub title: Option<String>,
@@ -254,7 +253,7 @@ impl InsertableStruct<Url> for NewUrl {
 }
 
 impl Upsertable<Url> for NewUrl {
-    type Update = UrlUpdate;
+    type Update = UrlChangeset;
 
     fn upsert(self, existing: &Url) -> Self::Update {
         Self::Update {
@@ -283,8 +282,7 @@ pub struct InsertUrl {
     pub subdomain_id: i32,
     pub value: String,
     pub status: Option<i32>,
-    #[serde(deserialize_with="ser::opt_string_or_bytes")]
-    pub body: Option<Vec<u8>>,
+    pub body: Option<StringOrBytes>,
     pub online: Option<bool>,
     pub title: Option<String>,
     pub redirect: Option<String>,
@@ -309,7 +307,7 @@ impl InsertToNew for InsertUrl {
             value: self.value,
             path,
             status: self.status,
-            body: self.body,
+            body: self.body.map(|x| x.0),
             online: self.online,
             title: self.title,
             redirect,
@@ -320,7 +318,7 @@ impl InsertToNew for InsertUrl {
 
 #[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
 #[table_name="urls"]
-pub struct UrlUpdate {
+pub struct UrlChangeset {
     pub id: i32,
     pub status: Option<i32>,
     pub body: Option<Vec<u8>>,
@@ -329,7 +327,7 @@ pub struct UrlUpdate {
     pub redirect: Option<String>,
 }
 
-impl Upsert for UrlUpdate {
+impl Upsert for UrlChangeset {
     fn is_dirty(&self) -> bool {
         self.status.is_some() ||
         self.body.is_some() ||
@@ -347,7 +345,7 @@ impl Upsert for UrlUpdate {
     }
 }
 
-impl Updateable<Url> for UrlUpdate {
+impl Updateable<Url> for UrlChangeset {
     fn changeset(&mut self, existing: &Url) {
         Self::clear_if_equal(&mut self.online, &existing.online);
         Self::clear_if_equal(&mut self.status, &existing.status);
@@ -365,6 +363,30 @@ impl Updateable<Url> for UrlUpdate {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct UrlUpdate {
+    pub id: i32,
+    pub status: Option<i32>,
+    pub body: Option<StringOrBytes>,
+    pub online: Option<bool>,
+    pub title: Option<String>,
+    pub redirect: Option<String>,
+}
+
+impl UpdateToChangeset<UrlChangeset> for UrlUpdate {
+    fn try_into_changeset(self) -> Result<UrlChangeset> {
+        // TODO: redirect needs pre-processing
+        Ok(UrlChangeset {
+            id: self.id,
+            status: self.status,
+            body: self.body.map(|x| x.0),
+            online: self.online,
+            title: self.title,
+            redirect: self.redirect,
+        })
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/repl/mod.rs

@@ -1,6 +1,7 @@
 use crate::config::Config;
 use crate::errors::*;
-use crate::engine::{ctx, Environment, DummyReporter};
+use crate::engine::{ctx, Environment};
+use crate::ipc::child::DummyIpcChild;
 use crate::engine::ctx::{State, LuaState};
 use crate::geoip::{Maxmind, AsnDB, GeoIP};
 use crate::hlua::{Lua, AnyLuaValue};
@@ -81,7 +82,7 @@ impl<'a> Repl<'a> {
 
 pub fn run(config: &Config) -> Result<()> {
     let keyring = Vec::new();
-    let dns_config = Resolver::from_system()?;
+    let dns_config = Resolver::from_system_v4()?;
     let proxy = config.network.proxy;
 
     let cache_dir = paths::cache_dir()?;
@@ -90,7 +91,7 @@ pub fn run(config: &Config) -> Result<()> {
     let asn = AsnDB::try_open_reader(&cache_dir)?;
 
     let env = Environment {
-        verbose: 0, // this doesn't do anything since we use a dummy reporter
+        verbose: 0, // this doesn't do anything since we use a dummy ipc child
         keyring,
         dns_config,
         proxy,
@@ -101,7 +102,7 @@ pub fn run(config: &Config) -> Result<()> {
         asn,
     };
 
-    let tx = DummyReporter::new();
+    let tx = DummyIpcChild::new();
     let (lua, state) = ctx::ctx(env, tx);
     let mut repl = Repl::new(lua, state);
 

src/runtime/crypto.rs

@@ -0,0 +1,30 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::engine::structs::{byte_array, lua_bytes};
+use crate::hlua::{self, AnyLuaValue};
+use sn0int_std::crypto;
+use std::sync::Arc;
+
+pub fn key_trunc_pad(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("key_trunc_pad", hlua::function3(move |bytes: AnyLuaValue, len: u32, pad: u8| -> Result<AnyLuaValue> {
+        let bytes = byte_array(bytes)
+            .map_err(|err| state.set_error(err))?;
+        let bytes = crypto::key_trunc_pad(&bytes, len as usize, pad);
+        Ok(lua_bytes(&bytes))
+    }))
+}
+
+pub fn sodium_secretbox_open(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("sodium_secretbox_open", hlua::function2(move |encrypted: AnyLuaValue, key: AnyLuaValue| -> Result<AnyLuaValue> {
+        let encrypted = byte_array(encrypted)
+            .map_err(|err| state.set_error(err))?;
+        let key = byte_array(key)
+            .map_err(|err| state.set_error(err))?;
+
+        let plain = crypto::sodium_secretbox_open(&encrypted, &key)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(lua_bytes(&plain))
+    }))
+}

src/runtime/db.rs

@@ -177,6 +177,28 @@ fn gen_changeset<T: Model, U: Updateable<T>>(object: LuaJsonValue, mut update: L
     Ok((existing.id(), value, update))
 }
 
+fn gen_changeset2<T: Model, U: UpdateToChangeset<C>, C: Updateable<T>>(object: LuaJsonValue, mut update: LuaJsonValue) -> Result<(i32, String, C)>
+    where
+        for<'de> T: serde::Deserialize<'de>,
+        for<'de> U: serde::Deserialize<'de>,
+        for<'de> C: serde::Deserialize<'de>,
+{
+    let existing = structs::from_lua::<T>(object)?;
+
+    // copy the id over to the update struct so we can identify the row
+    if let LuaJsonValue::Object(ref mut update) = update {
+        update.insert("id".into(), LuaJsonValue::Number(existing.id().into()));
+    }
+
+    let update = structs::from_lua::<U>(update)?;
+    let mut update = update.try_into_changeset()?;
+
+    let value = existing.to_string();
+    update.changeset(&existing);
+
+    Ok((existing.id(), value, update))
+}
+
 pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("db_update", hlua::function3(move |family: String, object: AnyLuaValue, update: AnyLuaValue| -> Result<Option<i32>> {
         let family = Family::from_str(&family)
@@ -191,7 +213,7 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             Family::Ipaddr => gen_changeset::<IpAddr, IpAddrUpdate>(object, update)
                 .map(|(id, v, u)| (id, v, Update::IpAddr(u))),
             Family::SubdomainIpaddr => bail!("Subdomain-IpAddr doesn't have mutable fields"),
-            Family::Url => gen_changeset::<Url, UrlUpdate>(object, update)
+            Family::Url => gen_changeset2::<Url, UrlUpdate, UrlChangeset>(object, update)
                 .map(|(id, v, u)| (id, v, Update::Url(u))),
             Family::Email => gen_changeset::<Email, EmailUpdate>(object, update)
                 .map(|(id, v, u)| (id, v, Update::Email(u))),

src/runtime/mod.rs

@@ -6,6 +6,7 @@ macro_rules! import_fns {
 }
 
 import_fns!(blobs);
+import_fns!(crypto);
 import_fns!(datetime);
 import_fns!(db);
 import_fns!(dns);
@@ -22,6 +23,7 @@ import_fns!(int);
 import_fns!(json);
 import_fns!(keyring);
 import_fns!(logger);
+import_fns!(mqtt);
 import_fns!(options);
 import_fns!(pgp);
 import_fns!(psl);

src/runtime/mqtt.rs

@@ -0,0 +1,59 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::hlua::{self, AnyLuaValue};
+use crate::mqtt::MqttOptions;
+use std::sync::Arc;
+use url::Url;
+
+pub fn mqtt_connect(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("mqtt_connect", hlua::function2(move |url: String, options: AnyLuaValue| -> Result<String> {
+        let options = MqttOptions::try_from(options)
+            .context("Invalid mqtt options")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        let url = Url::parse(&url)
+            .context("Failed to parse url")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        state.mqtt_connect(url, &options)
+            .map_err(|err| state.set_error(err))
+    }))
+}
+
+pub fn mqtt_subscribe(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("mqtt_subscribe", hlua::function3(move |sock: String, topic: String, level: u8| -> Result<()> {
+        let sock = state.get_mqtt(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        sock.subscribe(&topic, level)
+            .map_err(|err| state.set_error(err))
+    }))
+}
+
+pub fn mqtt_recv(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("mqtt_recv", hlua::function1(move |sock: String| -> Result<AnyLuaValue> {
+        let sock = state.get_mqtt(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let pkt = sock.recv_pkt()
+            .map_err(|err| state.set_error(err))?;
+
+        if let Some(pkt) = pkt {
+            pkt.to_lua()
+                .map_err(|err| state.set_error(err))
+        } else {
+            Ok(AnyLuaValue::LuaNil)
+        }
+    }))
+}
+
+pub fn mqtt_ping(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("mqtt_ping", hlua::function1(move |sock: String| -> Result<()> {
+        let sock = state.get_mqtt(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        sock.ping()
+            .map_err(|err| state.set_error(err))
+    }))
+}

src/runtime/pgp.rs

@@ -158,4 +158,25 @@ vA==
         "#).expect("Failed to load script");
         script.test().expect("Script failed");
     }
+
+    #[test]
+    #[ignore]
+    fn verify_pgp_fetch_wkd() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            session = http_mksession()
+            url = 'https://openpgpkey.archlinux.org/.well-known/openpgpkey/archlinux.org/hu/in9mwr4s84x7gm51851h343n3at1x61g?l=anthraxx'
+            req = http_request(session, 'GET', url, {
+                binary=true,
+            })
+            r = http_fetch(req)
+            k = pgp_pubkey(r['binary'])
+            info(k)
+            if k['fingerprint'] ~= 'E240B57E2C4630BA768E2F26FC1B547C8D8172C8' then
+                return 'wrong fingerprint: ' .. k['fingerprint']
+            end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
 }

src/sandbox/mod.rs

@@ -5,7 +5,7 @@ use caps::{self, CapSet};
 use nix;
 
 #[cfg(target_os = "openbsd")]
-use pledge::{pledge, Promise, ToPromiseString};
+use pledge::pledge;
 #[cfg(target_os = "openbsd")]
 use unveil::unveil;
 
@@ -72,7 +72,7 @@ pub fn init_openbsd() -> Result<()> {
     unveil("", "")
         .map_err(|_| format_err!("Failed to call unveil"))?;
 
-    pledge![Stdio, RPath, Dns, Inet]?;
+    pledge![Stdio Rpath Dns Inet,]?;
 
     Ok(())
 }

src/sandbox/seccomp.rs

@@ -17,7 +17,10 @@ pub fn init() -> Result<()> {
     ctx.allow_syscall(Syscall::fcntl64)?;
     ctx.allow_syscall(Syscall::uname)?;
     ctx.allow_syscall(Syscall::close)?;
+    #[cfg(not(target_arch = "aarch64"))]
     ctx.allow_syscall(Syscall::poll)?;
+    #[cfg(target_arch = "aarch64")]
+    ctx.allow_syscall(Syscall::ppoll)?;
     ctx.allow_syscall(Syscall::epoll_create1)?;
     ctx.allow_syscall(Syscall::pipe2)?;
     ctx.allow_syscall(Syscall::epoll_ctl)?;

src/ser.rs

@@ -8,7 +8,7 @@ use std::marker::PhantomData;
 
 
 #[derive(Debug, Serialize, Deserialize)]
-struct StringOrBytes(#[serde(deserialize_with="string_or_bytes")] Vec<u8>);
+pub struct StringOrBytes(#[serde(deserialize_with="string_or_bytes")] pub Vec<u8>);
 
 pub fn string_or_bytes<'de, D>(deserializer: D) -> result::Result<Vec<u8>, D::Error> where D: Deserializer<'de> {
     struct StringOrBytes(PhantomData<fn() -> Vec<u8>>);

src/shell/complete.rs

@@ -202,6 +202,7 @@ impl Completer for CmdCompleter {
                             "reload",
                             "update",
                             "uninstall",
+                            "quickstart",
                         ], &cmd[1]))
                     }
                 },

src/shell/mod.rs

@@ -49,6 +49,9 @@ pub enum Command {
     Quickstart,
     Workspace,
 
+    Exit,
+    Quit,
+
     Interrupt,
 }
 
@@ -61,6 +64,7 @@ impl Command {
             Command::Autoscope => "autoscope",
             Command::Back => "back",
             Command::Delete => "delete",
+            Command::Exit => "exit",
             Command::Help => "help",
             Command::Keyring => "keyring",
             Command::Mod => "mod",
@@ -73,6 +77,7 @@ impl Command {
             Command::Target => "target",
             Command::Use => "use",
             Command::Quickstart => "quickstart",
+            Command::Quit => "quit",
             Command::Workspace => "workspace",
             Command::Interrupt => unreachable!(),
         }
@@ -118,6 +123,7 @@ impl FromStr for Command {
             "autoscope" => Ok(Command::Autoscope),
             "back" => Ok(Command::Back),
             "delete" => Ok(Command::Delete),
+            "exit" => Ok(Command::Exit),
             "help" => Ok(Command::Help),
             "keyring" => Ok(Command::Keyring),
             "mod" => Ok(Command::Mod),
@@ -130,6 +136,7 @@ impl FromStr for Command {
             "target"  => Ok(Command::Target),
             "use"  => Ok(Command::Use),
             "quickstart"  => Ok(Command::Quickstart),
+            "quit" => Ok(Command::Quit),
             "workspace" => Ok(Command::Workspace),
             x => bail!("unknown command: {:?}, try \"help\"", x),
         }
@@ -465,6 +472,9 @@ pub fn run_once(rl: &mut Shell) -> Result<bool> {
         Some((Command::Use, args)) => use_cmd::run(rl, &args)?,
         Some((Command::Quickstart, args)) => quickstart_cmd::run(rl, &args)?,
         Some((Command::Workspace, args)) => cmd::<workspace_cmd::Args>(rl, &args)?,
+
+        Some((Command::Exit, _)) => return Ok(true),
+        Some((Command::Quit, _)) => return Ok(true),
         Some((Command::Interrupt, _)) => return Ok(true),
         None => (),
     }
@@ -496,12 +506,13 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
     let keyring = KeyRing::init()?;
 
     if verbose_init && library.list().is_empty() {
-        term::success("No modules found, run quickstart to install default modules");
+        term::success("No modules found, run \x1b[1mpkg quickstart\x1b[0m to install default modules");
+        term::success("New to sn0int? Follow https://sn0int.rtfd.io/en/stable/usage.html");
     }
 
     let autoupdate = AutoUpdater::load()?;
     if autoupdate.outdated() > 0 {
-        term::warn(&format!("{} modules are outdated, run: mod update", autoupdate.outdated()));
+        term::warn(&format!("{} modules are outdated, run: \x1b[1mpkg update\x1b[0m", autoupdate.outdated()));
     }
     autoupdate.check_background(&config, library.list());
 

src/update.rs

@@ -9,8 +9,8 @@ use std::path::PathBuf;
 use std::time::{SystemTime, UNIX_EPOCH};
 use std::thread;
 
-// 1 week
-const UPDATE_INTERVAL: u64 = 3600 * 24 * 7;
+// 1 day
+const UPDATE_INTERVAL: u64 = 3600 * 24;
 
 
 #[derive(Debug, Default, Serialize, Deserialize)]
@@ -65,7 +65,8 @@ impl AutoUpdater {
 
     pub fn save(&self) -> Result<()> {
         let config = serde_json::to_string(&self)?;
-        fs::write(AutoUpdater::path()?, &config)?;
+        fs::write(AutoUpdater::path()?, &config)
+            .context("Failed to write auto-update state")?;
         Ok(())
     }
 
@@ -100,7 +101,8 @@ impl AutoUpdater {
     }
 
     pub fn check_updates(&mut self, client: Client, modules: Vec<engine::Module>) -> Result<()> {
-        let latest = client.latest_release()?;
+        let latest = client.latest_release()
+            .context("Failed to get latest release timestamp")?;
 
         if latest.time != self.registry {
             let mut outdated = HashSet::new();

src/worker.rs

@@ -5,8 +5,9 @@ use crate::channel;
 use crate::cmd::run_cmd::Params;
 use crate::db::{Database, DbChange, Family};
 use crate::db::ttl::Ttl;
-use crate::engine::{self, Module};
-use crate::engine::isolation::Supervisor;
+use crate::engine::Module;
+use crate::ipc;
+use crate::ipc::parent::IpcParent;
 use crate::models::*;
 use serde_json;
 use crate::ratelimits::{Ratelimiter, RatelimitResponse};
@@ -243,7 +244,11 @@ impl DatabaseEvent {
                 log.push_str(&format!("@ {}", object.time));
 
                 if let (Some(ref lat), Some(ref lon)) = (object.latitude, object.longitude) {
-                    log.push_str(&format!(" ({}, {})", lat, lon));
+                    log.push_str(&format!(" ({}, {}", lat, lon));
+                    if let Some(radius) = &object.radius {
+                        log.push_str(&format!(" | {}m", radius));
+                    }
+                    log.push_str(")");
                 }
 
                 if verbose > 0 {
@@ -340,13 +345,13 @@ impl StdioEvent {
         }
     }
 
-    pub fn apply(self, supervisor: &mut Supervisor, tx: &EventSender, reader: &mut Option<BufReader<Stdin>>) {
+    pub fn apply(self, ipc_parent: &mut IpcParent, tx: &EventSender, reader: &mut Option<BufReader<Stdin>>) {
         let reply = match self {
             StdioEvent::Readline => Self::read_line(reader),
             StdioEvent::ToEnd => Self::read_to_end(reader),
         };
         let reply = reply.map_err(|e| e.to_string());
-        supervisor.send_struct(reply, tx);
+        ipc_parent.send_struct(reply, tx);
     }
 }
 
@@ -412,7 +417,7 @@ pub fn spawn(rl: &mut Shell, module: &Module, args: Vec<(serde_json::Value, Opti
             }
 
             tx.send(Event2::Start);
-            let event = match engine::isolation::spawn_module(module, &tx, arg, keyring, verbose, has_stdin, proxy, options, blobs) {
+            let event = match ipc::parent::run(module, &tx, arg, keyring, verbose, has_stdin, proxy, options, blobs) {
                 Ok(exit) => exit,
                 Err(err) => ExitEvent::SetupFailed(err.to_string()),
             };