Release v0.18.0

Properly support inserting urls with no body

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.17.1"
+version = "0.18.0"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -22,7 +22,7 @@ sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
 sn0int-common = { version="0.10.0", path="sn0int-common" }
-sn0int-std = { version="0.17.1", path="sn0int-std" }
+sn0int-std = { version="=0.18.0", path="sn0int-std" }
 rustyline = "6.0"
 log = "0.4"
 env_logger = "0.7"
@@ -82,7 +82,7 @@ syscallz = "0.12"
 nix = "0.17"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
-pledge = "0.3.1"
+pledge = "0.4"
 unveil = "0.2.0"
 
 [dev-dependencies]

Dockerfile

@@ -1,14 +1,13 @@
-FROM rust:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian:buster
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
-    && rm -rf /var/lib/apt/lists/*
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

README.md

@@ -89,10 +89,10 @@ For everything else please have a look at the [detailed list][1].
   - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
   - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
   - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
-  - [Autonoscope](https://sn0int.readthedocs.io/en/latest/usage.html#autonoscope)
-    - [Domains](https://sn0int.readthedocs.io/en/latest/usage.html#domains)
-    - [IPs](https://sn0int.readthedocs.io/en/latest/usage.html#ips)
-    - [URLs](https://sn0int.readthedocs.io/en/latest/usage.html#urls)
+- [Autonoscope](https://sn0int.readthedocs.io/en/latest/autonoscope.html)
+  - [Domains](https://sn0int.readthedocs.io/en/latest/autonoscope.html#domains)
+  - [IPs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#ips)
+  - [URLs](https://sn0int.readthedocs.io/en/latest/autonoscope.html#urls)
 - [Writing your first module](https://sn0int.readthedocs.io/en/latest/scripting.html)
   - [Creating a repository](https://sn0int.readthedocs.io/en/latest/scripting.html#creating-a-repository)
   - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
@@ -100,6 +100,8 @@ For everything else please have a look at the [detailed list][1].
   - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
   - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/database.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/database.html#db-activity)
   - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
   - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
 - [Structs](https://sn0int.readthedocs.io/en/latest/structs.html)
@@ -154,6 +156,7 @@ For everything else please have a look at the [detailed list][1].
   - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
   - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
   - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
+  - [db_activity](https://sn0int.readthedocs.io/en/latest/reference.html#db-activity)
   - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
   - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
   - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
@@ -177,12 +180,18 @@ For everything else please have a look at the [detailed list][1].
   - [img_exif](https://sn0int.readthedocs.io/en/latest/reference.html#img-exif)
   - [img_nudity](https://sn0int.readthedocs.io/en/latest/reference.html#img-nudity)
   - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [intval](https://sn0int.readthedocs.io/en/latest/reference.html#intval)
   - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
   - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
   - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [key_trunc_pad](https://sn0int.readthedocs.io/en/latest/reference.html#key-trunc-pad)
   - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
   - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
   - [md5](https://sn0int.readthedocs.io/en/latest/reference.html#md5)
+  - [mqtt_connect](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-connect)
+  - [mqtt_subscribe](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-subscribe)
+  - [mqtt_recv](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-recv)
+  - [mqtt_ping](https://sn0int.readthedocs.io/en/latest/reference.html#mqtt-ping)
   - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
@@ -215,6 +224,7 @@ For everything else please have a look at the [detailed list][1].
   - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
   - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
   - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
+  - [sodium_secretbox_open](https://sn0int.readthedocs.io/en/latest/reference.html#sodium-secretbox-open)
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
   - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
@@ -222,6 +232,7 @@ For everything else please have a look at the [detailed list][1].
   - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
   - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
   - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
+  - [strval](https://sn0int.readthedocs.io/en/latest/reference.html#strval)
   - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
   - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
   - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)

ci/integration.py

@@ -47,10 +47,10 @@ def main(tempdir, binary):
 
     print('[*] installing modules')
     sn0int(tempdir, binary, [
-        'mod install kpcyrd/ctlogs',
-        'mod install kpcyrd/dns-resolve',
-        'mod install kpcyrd/url-scan',
-        'mod install kpcyrd/geoip',
+        'pkg install kpcyrd/ctlogs',
+        'pkg install kpcyrd/dns-resolve',
+        'pkg install kpcyrd/url-scan',
+        'pkg install kpcyrd/geoip',
     ])
 
     print('[*] running ctlogs')

ci/setup.sh

@@ -3,6 +3,9 @@ set -exu
 case "$1" in
     linux)
         sudo apt update
-        sudo apt install libsqlite3-dev libseccomp-dev
+        sudo apt install libsqlite3-dev libseccomp-dev libsodium-dev
+        ;;
+    osx)
+        brew install libsodium
         ;;
 esac

contrib/docker/Dockerfile.alpine

@@ -1,13 +1,13 @@
-FROM alpine:edge
-RUN apk add --no-cache sqlite-dev libseccomp-dev
-RUN apk add --no-cache --virtual .build-rust rust cargo
+FROM rust:alpine3.11
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+RUN apk add --no-cache musl-dev sqlite-dev libseccomp-dev libsodium-dev
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM alpine:edge
-RUN apk add --no-cache libgcc sqlite-libs libseccomp
+FROM alpine:3.11
+RUN apk add --no-cache libgcc sqlite-libs libseccomp libsodium
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]
 ENV XDG_DATA_HOME=/data \

contrib/docker/Dockerfile.debian

@@ -1,13 +1,13 @@
-FROM rust
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM rust:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/sn0int
 COPY . .
 RUN cargo build --release --verbose
 RUN strip target/release/sn0int
 
-FROM debian
-RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+FROM debian:buster
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev libsodium-dev \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
 VOLUME ["/data", "/cache"]

docs/autonoscope.rst

@@ -0,0 +1,81 @@
+Autonoscope
+===========
+
+Instead of manually unscoping everything you can also define so called
+autonoscope rules. Those are executed from most specific to least specific and
+the first match wins. If no rule matches, the default is in-scope::
+
+    [sn0int][demo] > # add the domain first
+    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
+    [sn0int][demo] > add domain example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > # automatically noscope all subdomains
+    [sn0int][demo] > autonoscope add domain example.com
+    [sn0int][demo] > # except subdomains of prod.example.com 
+    [sn0int][demo] > autoscope add domain prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > autonoscope list 
+      scope domain "prod.example.com"
+    noscope domain "example.com"
+    [sn0int][demo] > 
+    [sn0int][demo] > # this is going to be out-of-scope
+    [sn0int][demo] > add subdomain www.example.com
+    [sn0int][demo] > # this is going to be in-scope
+    [sn0int][demo] > add subdomain db.prod.example.com
+    [sn0int][demo] > 
+    [sn0int][demo] > select subdomains 
+    #1, "www.example.com"
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=0
+    #2, "db.prod.example.com"
+    [sn0int][demo] > select subdomains where unscoped=1
+    #1, "www.example.com"
+    [sn0int][demo] > 
+
+Domains
+-------
+
+Autonoscope rules for domains are applied to the following structs:
+
+- domains
+- subdomains
+- urls
+
+Example rules::
+
+    autonoscope add domain example.com
+    autonoscope add domain staging.example.com
+    autonoscope add domain com
+    autonoscope add domain .
+
+IPs
+---
+
+Autonoscope rules for IPs are applied to the following structs:
+
+- ipaddrs
+- netblocks
+- ports
+
+Example rules::
+
+    autonoscope add ip 0.0.0.0/0
+    autonoscope add ip ::/0
+    autonoscope add ip 192.168.0.0/16
+    autonoscope add ip 10.13.33.37/32
+
+URLs
+----
+
+Autonoscope rules for urls are applied to the following structs:
+
+- urls
+
+Note that these rules are specific to a certain origin (like
+``https://example.com``) and are used to filter paths.
+
+Example rules::
+
+    autonoscope add url https://example.com/
+    autonoscope add url https://example.com/admin/
+    autonoscope add url https://example.com/a/b/c/d

docs/build.rst

@@ -18,19 +18,21 @@ Archlinux
 
 .. code-block:: bash
 
-    $ pacman -S geoip2-database libseccomp publicsuffix-list sqlite
+    $ pacman -S geoip2-database libseccomp libsodium publicsuffix-list sqlite
 
 Mac OSX
 ~~~~~~~
 
-None.
+.. code-block:: bash
+
+    $ brew install libsodium
 
 Debian/Ubuntu/Kali
 ~~~~~~~~~~~~~~~~~~
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
 
 .. warning::
    On a debian based system make sure you've installed rust with rustup.
@@ -40,21 +42,21 @@ Alpine
 
 .. code-block:: bash
 
-    $ apk add sqlite-dev libseccomp-dev
+    $ apk add sqlite-dev libseccomp-dev libsodium-dev
 
 OpenBSD
 ~~~~~~~
 
 .. code-block:: bash
 
-    $ pkg_add sqlite3 geolite2-city geolite2-asn
+    $ pkg_add sqlite3 geolite2-city geolite2-asn libsodium
 
 Gentoo
 ~~~~~~
 
 .. code-block:: bash
 
-    emerge --ask sys-libs/libseccomp dev-db/sqlite
+    emerge --ask sys-libs/libseccomp dev-db/sqlite dev-libs/libsodium
 
 Windows
 ~~~~~~~

docs/index.rst

@@ -38,6 +38,7 @@ Getting Started
    install
    build
    usage
+   autonoscope
    scripting
    database
    structs

docs/install.rst

@@ -32,7 +32,7 @@ at the docker image as an alternative.
 
 .. code-block:: bash
 
-    $ apt install build-essential libsqlite3-dev libseccomp-dev publicsuffix
+    $ apt install build-essential libsqlite3-dev libseccomp-dev libsodium-dev publicsuffix
     $ git clone https://github.com/kpcyrd/sn0int.git
     $ cd sn0int
     $ cargo install -f --path .

docs/reference.rst

@@ -391,6 +391,8 @@ options are set. The following options are available:
 ``proxy``
   Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
   if it doesn't conflict with the global proxy settings.
+``binary``
+  Set to ``true`` to get the http response as raw bytes.
 
 This function may fail.
 
@@ -418,6 +420,8 @@ the following keys:
   A table of headers
 ``text``
   The response body as string
+``binary``
+  The response body as bytes (if ``binary=true``)
 ``blob``
   If ``into_blob`` was enabled for the request the body is downloaded into blob
   storage with a reference to the body in this field.
@@ -517,6 +521,13 @@ Log an info to the terminal.
 
     info('ohai')
 
+intval
+------
+
+Parse a number from a string.
+
+    x = strval('1234')
+
 json_decode
 -----------
 
@@ -551,6 +562,17 @@ Encode a datastructure into a string.
     })
     print(x)
 
+key_trunc_pad
+-------------
+
+Truncate/pad a key to a given length.
+
+.. code-block:: lua
+
+    -- if longer than 32 bytes: truncate to 32
+    -- if shorter than 32 bytes: pad with \x00
+    local key = key_trunc_pad(password, 32, 0)
+
 keyring
 -------
 
@@ -585,6 +607,64 @@ Hash a byte array with md5 and return the results as bytes.
 
     hex(md5("\x00\xff"))
 
+mqtt_connect
+------------
+
+Connect to an mqtt broker.
+
+.. code-block:: lua
+
+    local sock = mqtt_connect('mqtts://mqtt.example.com', {
+        username='foo',
+        password='secret',
+    })
+    if last_err() then return end
+
+mqtt_subscribe
+--------------
+
+Subscribe to a topic. Right now only QoS 0 is supported.
+
+.. code-block:: lua
+
+    mqtt_subscribe(sock, '#', 0)
+    if last_err() then return end
+
+mqtt_recv
+---------
+
+Receive an mqtt packet. This is not necessarily a publish packet and more
+packets might be added in the future, so you need to check the type
+specifically.
+
+If a read timeout has been set with mqtt_connect_ this function returns ``nil``
+in case of a read timeout.
+
+.. code-block:: lua
+
+    local pkt = mqtt_recv(sock)
+    if last_err() then return end
+    if pkt == nil then
+        -- read timeout, consider sending a ping or disconnect if the previous ping failed
+    elseif pkt['type'] == 'pong' then
+        -- broker sent a pong
+    elseif pkt['type'] == 'publish' then
+        local payload = utf8_decode(pkt['body'])
+        if last_err() then return end
+        info(payload)
+    end
+
+mqtt_ping
+---------
+
+Send a pingreq packet, causing the broker to send a pingresp. This is used to
+make sure the connection is still working correctly.
+
+.. code-block:: lua
+
+    mqtt_ping(sock)
+    if last_err() then return end
+
 pgp_pubkey
 ----------
 
@@ -1012,6 +1092,27 @@ Overwrite the default ``\n`` newline.
 
     sock_newline(sock, "\r\n")
 
+sodium_secretbox_open
+---------------------
+
+Use authenticated symetric crypto to decrypt a given message.
+
+Internally this is ``crypto_secretbox_xsalsa20poly1305``.
+
+The key **must** be 32 bytes, see key_trunc_pad_ if necessary.
+
+The first 24 bytes of the encrypted message are expected to be the nonce.
+
+.. code-block:: lua
+
+    plain = sodium_secretbox_open(encrypted, key)
+    if last_err() then return end
+
+    txt = utf8_decode(plain)
+    if last_err() then return end
+
+    info(txt)
+
 status
 ------
 
@@ -1093,6 +1194,13 @@ Parse a date into a unix timestamp, see `strftime rules`_.
 
 .. _strftime rules: https://docs.rs/chrono/0.4.6/chrono/format/strftime/index.html
 
+strval
+------
+
+Convert a number into a string.
+
+    x = strval(1234)
+
 time_unix
 ---------
 

docs/usage.rst

@@ -27,12 +27,12 @@ number of recommended modules::
     [+] Downloading "GeoLite2-City.mmdb"
     [+] Downloading "GeoLite2-ASN.mmdb"
     [+] Loaded 0 modules
-    [*] No modules found, run quickstart to install default modules
+    [*] No modules found, run pkg quickstart to install default modules
     [sn0int][default] >
 
-Typing ``quickstart`` is going to get you a fair number of featured modules::
+Typing ``pkg quickstart`` is going to get you a fair number of featured modules::
 
-    [sn0int][default] > quickstart
+    [sn0int][default] > pkg quickstart
     [+] Installing kpcyrd/asn
     [+] Installing kpcyrd/ctlogs
     [+] Installing kpcyrd/dns-resolve
@@ -105,7 +105,7 @@ Running a module
 Now that we have something to get started with, we can run our first module.
 First lets list all modules we have::
 
-    [sn0int][demo] > mod list
+    [sn0int][demo] > pkg list
     kpcyrd/asn (0.1.0)
         Run a asn lookup for an ip address
     kpcyrd/ctlogs (0.1.0)
@@ -149,7 +149,7 @@ some of them in a browser but hold on, there's a more efficient way to approach
 this.
 
 .. hint::
-   You can run the modules concurrently with ``run -j 8``.
+   You can run the modules concurrently with ``run -j3``.
 
 Running followup modules on the results
 ---------------------------------------
@@ -256,85 +256,3 @@ You can reverse this using the scope command::
 .. hint::
    All entities have this field, you can refer to it in queries using
    ``unscoped=1``.
-
-Autonoscope
------------
-
-Instead of manually unscoping everything you can also define so called
-autonoscope rules. Those are executed from most specific to least specific and
-the first match wins. If no rule matches, the default is in-scope::
-
-    [sn0int][demo] > # add the domain first
-    [sn0int][demo] > # this is necessary because we only want to partially unscope example.com
-    [sn0int][demo] > add domain example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > # automatically noscope all subdomains
-    [sn0int][demo] > autonoscope add domain example.com
-    [sn0int][demo] > # except subdomains of prod.example.com 
-    [sn0int][demo] > autoscope add domain prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > autonoscope list 
-      scope domain "prod.example.com"
-    noscope domain "example.com"
-    [sn0int][demo] > 
-    [sn0int][demo] > # this is going to be out-of-scope
-    [sn0int][demo] > add subdomain www.example.com
-    [sn0int][demo] > # this is going to be in-scope
-    [sn0int][demo] > add subdomain db.prod.example.com
-    [sn0int][demo] > 
-    [sn0int][demo] > select subdomains 
-    #1, "www.example.com"
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=0
-    #2, "db.prod.example.com"
-    [sn0int][demo] > select subdomains where unscoped=1
-    #1, "www.example.com"
-    [sn0int][demo] > 
-
-Domains
-~~~~~~~
-
-Autonoscope rules for domains are applied to the following structs:
-
-- domains
-- subdomains
-- urls
-
-Example rules::
-
-    autonoscope add domain example.com
-    autonoscope add domain staging.example.com
-    autonoscope add domain com
-    autonoscope add domain .
-
-IPs
-~~~
-
-Autonoscope rules for IPs are applied to the following structs:
-
-- ipaddrs
-- netblocks
-- ports
-
-Example rules::
-
-    autonoscope add ip 0.0.0.0/0
-    autonoscope add ip ::/0
-    autonoscope add ip 192.168.0.0/16
-    autonoscope add ip 10.13.33.37/32
-
-URLs
-~~~~
-
-Autonoscope rules for urls are applied to the following structs:
-
-- urls
-
-Note that these rules are specific to a certain origin (like
-``https://example.com``) and are used to filter paths.
-
-Example rules::
-
-    autonoscope add url https://example.com/
-    autonoscope add url https://example.com/admin/
-    autonoscope add url https://example.com/a/b/c/d

modules/harness/pgp-fetch.lua

@@ -0,0 +1,22 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    session = http_mksession()
+    url = 'https://openpgpkey.archlinux.org/.well-known/openpgpkey/archlinux.org/hu/in9mwr4s84x7gm51851h343n3at1x61g?l=anthraxx'
+
+    req = http_request(session, 'GET', url, {})
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['text'])
+    info(k)
+
+    req = http_request(session, 'GET', url, {
+        binary=true,
+    })
+    r = http_fetch(req)
+    -- debug(r)
+    k = pgp_pubkey(r['binary'])
+    info(k)
+end

modules/harness/url-inserter.lua

@@ -0,0 +1,58 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('preparing')
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value='example.com',
+    })
+
+    info('inserting')
+    url1 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+    })
+    url2 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        body='ohai',
+    })
+    url3 = db_add('url', {
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+    })
+
+    info('updating')
+    db_update('url', {
+        id=url1,
+        subdomain_id=subdomain_id,
+        value='https://example.com',
+        path='/',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url2,
+        subdomain_id=subdomain_id,
+        value='https://example.com/ohai',
+        path='/ohai',
+        body='ohai',
+        unscoped=false,
+    }, {
+    })
+    db_update('url', {
+        id=url3,
+        subdomain_id=subdomain_id,
+        value='https://example.com/world',
+        path='/world',
+        body={0x77, 0x6f, 0x72, 0x6c, 0x64},
+        unscoped=false,
+    }, {
+    })
+end

modules/harness/winkekatze-sub.lua

@@ -0,0 +1,17 @@
+-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local sock = mqtt_connect('mqtt://mqtt.winkekatze24.de', {
+        read_timeout=10,
+    })
+    if last_err() then return end
+    mqtt_subscribe(sock, '#', 0)
+
+    while true do
+        local pkt = mqtt_recv_text(sock)
+        if last_err() then return end
+        info(pkt)
+    end
+end

sn0int-registry/templates/index.html.hbs

@@ -21,7 +21,7 @@
     To install a module run:
     </p>
     <p class="code"><code>
-    sn0int install kpcyrd/ctlogs
+    sn0int pkg install kpcyrd/ctlogs
     </code></p>
 
     {{#each modules}}

sn0int-std/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int-std"
-version = "0.17.1"
+version = "0.18.0"
 description = "sn0int - stdlib"
 authors = ["kpcyrd <git@rxv.cc>"]
 repository = "https://github.com/kpcyrd/sn0int"
@@ -32,15 +32,16 @@ url = "2.0"
 tungstenite = { version = "0.10.1", default-features = false }
 kuchiki = "0.8.0"
 maxminddb = "0.13"
-# x509-parser 0.6.1 is broken
-x509-parser = "0.5.1"
-der-parser = "2.0"
+x509-parser = "0.6.2"
+der-parser = "3.0"
 publicsuffix = { version="1.5", default-features=false }
 xml-rs = "0.8"
 geo = "0.12"
 bytes = "0.4"
 base64 = "0.11"
 chrono = { version = "0.4", features = ["serde"] }
+mqtt-protocol = "0.8.1"
+sodiumoxide = { version="0.2.5", features=["use-pkg-config"] }
 
 image = "0.23.0"
 kamadak-exif = "0.5.1"

sn0int-std/src/crypto.rs

@@ -0,0 +1,51 @@
+use crate::errors::*;
+use sodiumoxide::crypto::secretbox::{self, Key, Nonce};
+use std::iter;
+
+pub fn key_trunc_pad(mut key: &[u8], len: usize, pad: u8) -> Vec<u8> {
+    if key.len() > len {
+        key = &key[..len];
+    }
+
+    let mut key = key.to_vec();
+    key.extend(iter::repeat(pad).take(len - key.len()));
+    key
+}
+
+pub fn sodium_secretbox_open(encrypted: &[u8], key: &[u8]) -> Result<Vec<u8>> {
+    if encrypted.len() <= secretbox::NONCEBYTES {
+        bail!("Encrypted message is too short");
+    }
+
+    let key = Key::from_slice(key)
+        .ok_or_else(|| format_err!("Key has wrong length"))?;
+    let nonce = Nonce::from_slice(&encrypted[..secretbox::NONCEBYTES])
+        .ok_or_else(|| format_err!("Nonce has wrong length"))?;
+    let ciphertext = &encrypted[secretbox::NONCEBYTES..];
+    let plain = secretbox::open(&ciphertext, &nonce, &key)
+        .map_err(|_| format_err!("Failed to decrypt secretbox"))?;
+    Ok(plain)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_key_equal() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_trunc() {
+        let key = key_trunc_pad(&[1, 2, 3, 4, 5, 6, 7, 8, 9], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 4, 5]);
+    }
+
+    #[test]
+    fn test_key_pad() {
+        let key = key_trunc_pad(&[1, 2, 3], 5, 0);
+        assert_eq!(key, &[1, 2, 3, 0, 0]);
+    }
+}

sn0int-std/src/engine/structs.rs

@@ -41,6 +41,12 @@ impl LuaMap {
     pub fn insert_num<K: Into<String>>(&mut self, k: K, v: f64) {
         self.0.insert(AnyHashableLuaValue::LuaString(k.into()), AnyLuaValue::LuaNumber(v));
     }
+
+    pub fn insert_serde<K: Into<String>, S: serde::Serialize>(&mut self, k: K, v: S) -> Result<()> {
+        let v = serde_json::to_value(v)?;
+        self.0.insert(AnyHashableLuaValue::LuaString(k.into()), LuaJsonValue::from(v).into());
+        Ok(())
+    }
 }
 
 impl IntoIterator for LuaMap {

sn0int-std/src/geoip/mod.rs

@@ -1,6 +1,5 @@
 use crate::errors::*;
 use crate::lazy::LazyInit;
-// use crate::paths;
 use maxminddb::{self, geoip2};
 use std::fmt;
 use std::fs::{self, File};
@@ -26,6 +25,8 @@ pub trait Maxmind: Sized {
             "/usr/share/GeoIP/",
             // OpenBSD
             "/usr/local/share/examples/libmaxminddb/",
+            // geoipupdate
+            "/var/lib/GeoIP/",
         ] {
             let path = Path::new(path);
             let path = path.join(Self::filename());

sn0int-std/src/lib.rs

@@ -3,6 +3,7 @@ use hlua_badtouch as hlua;
 
 pub mod blobs;
 pub mod crt;
+pub mod crypto;
 mod errors;
 pub mod engine;
 pub mod geo;
@@ -11,6 +12,7 @@ pub mod gfx;
 pub mod html;
 pub mod json;
 pub mod lazy;
+pub mod mqtt;
 pub mod psl;
 pub mod ratelimits;
 pub mod sockets;

sn0int-std/src/mqtt.rs

@@ -0,0 +1,228 @@
+use chrootable_https::DnsResolver;
+use crate::errors::*;
+use crate::hlua::AnyLuaValue;
+use mqtt::packet::VariablePacketError;
+use crate::json::LuaJsonValue;
+use crate::sockets::{Stream, SocketOptions};
+use mqtt::{TopicFilter, QualityOfService};
+use mqtt::control::ConnectReturnCode;
+use mqtt::control::fixed_header::FixedHeaderError;
+use mqtt::encodable::{Encodable, Decodable};
+use mqtt::packet::{Packet, VariablePacket, ConnectPacket, SubscribePacket, PingreqPacket};
+use std::convert::TryFrom;
+use std::io;
+use std::net::SocketAddr;
+use url::Url;
+
+#[derive(Debug, Default, Deserialize)]
+pub struct MqttOptions {
+    pub username: Option<String>,
+    pub password: Option<String>,
+
+    pub proxy: Option<SocketAddr>,
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
+}
+
+impl MqttOptions {
+    pub fn try_from(x: AnyLuaValue) -> Result<MqttOptions> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub struct MqttClient {
+    stream: Stream,
+}
+
+impl MqttClient {
+    pub fn negotiate(stream: Stream, options: &MqttOptions) -> Result<MqttClient> {
+        let mut client = MqttClient {
+            stream,
+        };
+
+        let mut pkt = ConnectPacket::new("MQTT", "sn0int");
+        pkt.set_user_name(options.username.clone());
+        pkt.set_password(options.password.clone());
+
+        /*
+        if let Some(keep_alive) = msg.keep_alive {
+            packet.set_keep_alive(keep_alive);
+        }
+        */
+
+        client.send(pkt.into())?;
+        let pkt = client.recv()?;
+
+        if let VariablePacket::ConnackPacket(pkt) = pkt {
+            let code = pkt.connect_return_code();
+            if code == ConnectReturnCode::ConnectionAccepted {
+                Ok(client)
+            } else {
+                bail!("MQTT negotiation failed: {:?}", code);
+            }
+        } else {
+            bail!("Expected ConnAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &MqttOptions) -> Result<MqttClient> {
+        let tls = match url.scheme() {
+            "mqtt" => false,
+            "mqtts" => true,
+            _ => bail!("Invalid mqtt protocol"),
+        };
+
+        let host = url.host_str()
+            .ok_or_else(|| format_err!("Missing host in url"))?;
+
+        let port = match (url.port(), tls) {
+            (Some(port), _) => port,
+            (None, true) => 8883,
+            (None, false) => 1883,
+        };
+
+
+        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
+            tls,
+            sni_value: None,
+            disable_tls_verify: false,
+            proxy: options.proxy,
+
+            connect_timeout: options.connect_timeout,
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+        })?;
+
+        Self::negotiate(stream, options)
+    }
+
+    fn send(&mut self, pkt: VariablePacket) -> Result<()> {
+        debug!("Sending mqtt packet: {:?}", pkt);
+        pkt.encode(&mut self.stream)?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> std::result::Result<VariablePacket, VariablePacketError> {
+        let pkt = VariablePacket::decode(&mut self.stream)?;
+        debug!("Received mqtt packet: {:?}", pkt);
+        Ok(pkt)
+    }
+
+    pub fn subscribe(&mut self, topic: &str, qos: u8) -> Result<()> {
+        let filter = TopicFilter::new(topic)?;
+
+        let qos = match qos {
+            0 => QualityOfService::Level0,
+            1 => QualityOfService::Level1,
+            2 => QualityOfService::Level2,
+            _ => bail!("Invalid QoS level: {}", qos),
+        };
+
+        let pkt = SubscribePacket::new(1, vec![(filter, qos)]);
+        self.send(pkt.into())?;
+
+        let pkt = self.recv()?;
+        if let VariablePacket::SubackPacket(_pkt) = pkt {
+            Ok(())
+        } else {
+            bail!("Expected SubAck, received {:?}", pkt);
+        }
+    }
+
+    pub fn recv_pkt(&mut self) -> Result<Option<Pkt>> {
+        match self.recv() {
+            Ok(pkt) => Ok(Some(Pkt::try_from(pkt)?)),
+            Err(VariablePacketError::IoError(err)) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(VariablePacketError::FixedHeaderError(FixedHeaderError::IoError(err))) if err.kind() == io::ErrorKind::WouldBlock => Ok(None),
+            Err(err) => Err(Error::from(err))
+        }
+    }
+
+    pub fn ping(&mut self) -> Result<()> {
+        let pkt = PingreqPacket::new();
+        self.send(pkt.into())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum Pkt {
+    #[serde(rename="publish")]
+    Publish(Publish),
+    #[serde(rename="pong")]
+    Pong,
+}
+
+impl Pkt {
+    pub fn to_lua(&self) -> Result<AnyLuaValue> {
+        let v = serde_json::to_value(&self)?;
+        let v = LuaJsonValue::from(v).into();
+        Ok(v)
+    }
+}
+
+impl TryFrom<VariablePacket> for Pkt {
+    type Error = Error;
+
+    fn try_from(pkt: VariablePacket) -> Result<Pkt> {
+        match pkt {
+            VariablePacket::ConnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::ConnackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PublishPacket(pkt) => Ok(Pkt::Publish(Publish {
+                topic: pkt.topic_name().to_string(),
+                body: pkt.payload(),
+            })),
+            VariablePacket::PubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrecPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubrelPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PubcompPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingreqPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::PingrespPacket(_) => Ok(Pkt::Pong),
+            VariablePacket::SubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::SubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubscribePacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::UnsubackPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+            VariablePacket::DisconnectPacket(_) => bail!("Unsupported pkt: {:?}", pkt),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Publish {
+    pub topic: String,
+    pub body: Vec<u8>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrootable_https::dns::Resolver;
+
+    fn connect() -> Result<MqttClient> {
+        let resolver = Resolver::from_system().unwrap();
+        let url = "mqtt://mqtt.winkekatze24.de".parse()?;
+        MqttClient::connect(&resolver, url, &MqttOptions::default())
+    }
+
+    #[test]
+    #[ignore]
+    fn test_connect() {
+        connect().expect("Failed to setup connection");
+    }
+
+    // this test is too flaky
+    /*
+    #[test]
+    #[ignore]
+    fn test_subscribe() {
+        let mut c = connect().unwrap();
+        c.subscribe("#", 0).unwrap();
+    }
+    */
+}

sn0int-std/src/sockets/mod.rs

@@ -319,6 +319,7 @@ impl Socket {
                 let available = match self.stream.fill_buf() {
                     Ok(n) => n,
                     Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
+                    Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(Vec::new()),
                     Err(e) => return Err(e.into())
                 };
 

sn0int-std/src/web.rs

@@ -72,6 +72,8 @@ pub struct RequestOptions {
     #[serde(default)]
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    #[serde(default)]
+    binary: bool,
 }
 
 impl RequestOptions {
@@ -101,6 +103,7 @@ pub struct HttpRequest {
     timeout: Option<Duration>,
     into_blob: bool,
     proxy: Option<SocketAddr>,
+    binary: bool,
 }
 
 impl HttpRequest {
@@ -123,6 +126,7 @@ impl HttpRequest {
             timeout,
             into_blob: options.into_blob,
             proxy: options.proxy,
+            binary: options.binary,
         };
 
         if let Some(json) = options.json {
@@ -255,6 +259,8 @@ impl HttpRequest {
             let blob = Blob::create(res.body);
             let id = state.register_blob(blob);
             resp.insert_str("blob", id);
+        } else if self.binary {
+            resp.insert_serde("binary", &res.body[..])?;
         } else {
             resp.insert_str("text", String::from_utf8_lossy(&res.body));
         }

src/cmd/activity_cmd.rs

@@ -4,6 +4,7 @@ use crate::cmd::Cmd;
 use crate::shell::Shell;
 use crate::models::*;
 use chrono::{Utc, NaiveDateTime, NaiveTime, Duration};
+use regex::Regex;
 use std::convert::TryFrom;
 use std::io;
 use std::str::FromStr;
@@ -15,17 +16,36 @@ pub struct TimeSpec {
     datetime: NaiveDateTime,
 }
 
-impl FromStr for TimeSpec {
-    type Err = Error;
-
-    fn from_str(s: &str) -> Result<Self> {
-        let now = Utc::now().naive_utc();
+impl TimeSpec {
+    fn resolve(s: &str, now: NaiveDateTime) -> Result<Self> {
         let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
 
         let datetime = match s {
             "today" => today,
             "yesterday" => today - Duration::days(1),
-            // x {second,minute,hour,day,week,month,year}s? ago
+            s if s.ends_with(" ago") => {
+                let re = Regex::new(r"(\d+) ?(s|seconds?|m|min|minutes?|h|hours?|d|days?|w|weeks?|months?|y|years?) ago").unwrap();
+
+                let caps = re.captures(s)
+                    .ok_or_else(|| format_err!("Couldn't parse TimeSpec"))?;
+
+                let n = caps.get(1).unwrap().as_str()
+                    .parse::<i64>()
+                    .context("Failed to parse number in timespec")?;
+                let unit = caps.get(2).unwrap();
+
+                let duration = match unit.as_str() {
+                    "s" | "second" | "seconds" => Duration::seconds(n),
+                    "m" | "min" | "minute" | "minutes" => Duration::minutes(n),
+                    "h" | "hour" | "hours" => Duration::hours(n),
+                    "d" | "day" | "days" => Duration::days(n),
+                    "w" | "week" | "weeks" => Duration::days(n * 7),
+                    "month" | "months" => Duration::days(n * 31),
+                    "y" | "year" | "years" => Duration::days(n * 365),
+                    _ => unreachable!(),
+                };
+                now - duration
+            },
             s => NaiveDateTime::from_str(s)?,
         };
 
@@ -35,6 +55,15 @@ impl FromStr for TimeSpec {
     }
 }
 
+impl FromStr for TimeSpec {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        let now = Utc::now().naive_utc();
+        Self::resolve(s, now)
+    }
+}
+
 #[derive(Debug, StructOpt)]
 #[structopt(global_settings = &[AppSettings::ColoredHelp])]
 pub struct Args {
@@ -86,3 +115,63 @@ impl Cmd for Args {
         Ok(())
     }
 }
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn datetime() -> NaiveDateTime {
+        let date = chrono::NaiveDate::from_ymd(2020, 3, 14);
+        let time = chrono::NaiveTime::from_hms(16, 20, 23);
+        NaiveDateTime::new(date, time)
+    }
+
+    #[test]
+    fn test_today() {
+        let x = TimeSpec::resolve("today", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_yesterday() {
+        let x = TimeSpec::resolve("yesterday", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-13T00:00:00").unwrap());
+    }
+
+    #[test]
+    fn test_20_min_ago() {
+        let x = TimeSpec::resolve("20min ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:00:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_days_ago() {
+        let x = TimeSpec::resolve("3 days ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-11T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_week_ago() {
+        let x = TimeSpec::resolve("1w ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-07T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_3_months_ago() {
+        let x = TimeSpec::resolve("3 months ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-12-12T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_1_year_ago() {
+        let x = TimeSpec::resolve("1 year ago", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2019-03-15T16:20:23").unwrap());
+    }
+
+    #[test]
+    fn test_exact_time() {
+        let x = TimeSpec::resolve("2020-03-14T16:20:23", datetime()).unwrap();
+        assert_eq!(x.datetime, NaiveDateTime::from_str("2020-03-14T16:20:23").unwrap());
+    }
+}

src/cmd/help_cmd.rs

@@ -16,9 +16,8 @@ pub fn run(_rl: &mut Shell, _args: &[String]) -> Result<()> {
     help("autoscope",   "Manage rules to automatically add entities to scope");
     help("delete",      "Delete entities from the database");
     help("keyring",     "Manage saved credentials");
-    help("mod",         "Manage installed modules");
+    help("pkg",         "Manage installed modules");
     help("noscope",     "Exclude entities from scope");
-    help("quickstart",  "Install all featured modules");
     help("run",         "Run the currently selected module");
     help("scope",       "Include entities in the scope again");
     help("select",      "Select entities from the database");

src/cmd/pkg_cmd.rs

@@ -1,10 +1,12 @@
 use crate::errors::*;
 
+use crate::args::Install;
+use crate::api::Client;
 use crate::args;
 use crate::config::Config;
 use crate::cmd::{Cmd, LiteCmd};
 use crate::engine::Library;
-use crate::registry::{self, UpdateTask, Updater};
+use crate::registry::{self, InstallTask, UpdateTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
 use crate::worker;
@@ -47,6 +49,9 @@ pub enum SubCommand {
     /// Uninstall a module
     #[structopt(name="uninstall")]
     Uninstall(Uninstall),
+    /// Install all featured modules
+    #[structopt(name="quickstart")]
+    Quickstart,
 }
 
 #[derive(Debug, StructOpt)]
@@ -156,6 +161,34 @@ fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) ->
             // trigger reload
             Ok(ModuleReload::Yes)
         },
+        SubCommand::Quickstart => {
+            let client = Client::new(&config)?;
+            let updater = Arc::new(Updater::new(&config)?);
+            let mut autoupdate = AutoUpdater::load()?;
+
+            let modules = client.quickstart()?
+                .into_iter()
+                .map(|module| {
+                    InstallTask::new(Install {
+                        module: ModuleID {
+                            author: module.author,
+                            name: module.name,
+                        },
+                        version: None,
+                        force: false,
+                    }, updater.clone())
+                })
+                .collect::<Vec<_>>();
+
+            worker::spawn_multi(modules, |name| {
+                autoupdate.updated(&name);
+            }, 3)?;
+
+            autoupdate.save()?;
+
+            // trigger reload
+            Ok(ModuleReload::Yes)
+        },
     }
 }
 

src/cmd/quickstart_cmd.rs

@@ -1,15 +1,11 @@
 use crate::errors::*;
 
-use crate::args::Install;
-use crate::api::Client;
-use crate::registry::{InstallTask, Updater};
+use crate::cmd::Cmd;
+use crate::cmd::pkg_cmd::{ArgsInteractive as PkgArgs, SubCommand, SubCommandInteractive};
 use crate::shell::Shell;
-use crate::update::AutoUpdater;
-use crate::worker;
-use std::sync::Arc;
+use crate::term;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
-use sn0int_common::ModuleID;
 
 
 #[derive(Debug, StructOpt)]
@@ -19,34 +15,11 @@ pub struct Args {
 
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     let _args = Args::from_iter_safe(args)?;
-    let config = rl.config().clone();
 
-    let client = Client::new(&config)?;
-    let updater = Arc::new(Updater::new(&config)?);
-    let mut autoupdate = AutoUpdater::load()?;
+    term::warn("The \x1b[1mquickstart\x1b[0m command is deprecated, use \x1b[1mpkg quickstart\x1b[0m");
 
-    let modules = client.quickstart()?
-        .into_iter()
-        .map(|module| {
-            InstallTask::new(Install {
-                module: ModuleID {
-                    author: module.author,
-                    name: module.name,
-                },
-                version: None,
-                force: false,
-            }, updater.clone())
-        })
-        .collect::<Vec<_>>();
-
-    worker::spawn_multi(modules, |name| {
-        autoupdate.updated(&name);
-    }, 3)?;
-
-    autoupdate.save()?;
-
-    // trigger reload
-    rl.reload_modules()?;
-
-    Ok(())
+    let args = PkgArgs {
+        subcommand: SubCommandInteractive::Base(SubCommand::Quickstart),
+    };
+    args.run(rl)
 }

src/db/mod.rs

@@ -304,7 +304,7 @@ impl Database {
         Ok(ipaddr_update.id)
     }
 
-    pub fn update_url(&self, url_update: &UrlUpdate) -> Result<i32> {
+    pub fn update_url(&self, url_update: &UrlChangeset) -> Result<i32> {
         use crate::schema::urls::columns::*;
         diesel::update(urls::table.filter(id.eq(url_update.id)))
             .set(url_update)

src/engine/ctx.rs

@@ -17,6 +17,7 @@ use crate::ratelimits::RatelimitResponse;
 use chrootable_https::{self, Resolver};
 use serde_json;
 use sn0int_std::blobs::{Blob, BlobState};
+use sn0int_std::mqtt::{MqttClient, MqttOptions};
 use sn0int_std::web::WebState;
 use std::collections::HashMap;
 use std::result;
@@ -178,6 +179,10 @@ pub trait State {
 
     fn get_ws(&self, id: &str)-> Arc<Mutex<WebSocket>>;
 
+    fn mqtt_connect(&self, url: url::Url, options: &MqttOptions) -> Result<String>;
+
+    fn get_mqtt(&self, id: &str)-> Arc<Mutex<MqttClient>>;
+
     fn http_mksession(&self) -> String;
 
     fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest;
@@ -199,6 +204,7 @@ pub struct LuaState {
     logger: Arc<Mutex<Box<dyn IpcChild>>>,
     socket_sessions: Mutex<HashMap<String, Arc<Mutex<Socket>>>>,
     ws_sessions: Mutex<HashMap<String, Arc<Mutex<WebSocket>>>>,
+    mqtt_sessions: Mutex<HashMap<String, Arc<Mutex<MqttClient>>>>,
     blobs: Mutex<HashMap<String, Arc<Blob>>>,
     http_sessions: Mutex<HashMap<String, HttpSession>>,
     http_clients: Mutex<HashMap<String, Arc<chrootable_https::Client<Resolver>>>>,
@@ -340,6 +346,22 @@ impl State for LuaState {
         sock.clone()
     }
 
+    fn mqtt_connect(&self, url: url::Url, options: &MqttOptions) -> Result<String> {
+        let mut mtx = self.mqtt_sessions.lock().unwrap();
+        let id = self.random_id();
+
+        let sock = MqttClient::connect(&self.dns_config, url, options)?;
+        mtx.insert(id.clone(), Arc::new(Mutex::new(sock)));
+
+        Ok(id)
+    }
+
+    fn get_mqtt(&self, id: &str)-> Arc<Mutex<MqttClient>> {
+        let mtx = self.mqtt_sessions.lock().unwrap();
+        let sock = mtx.get(id).expect("Invalid mqtt reference"); // TODO
+        sock.clone()
+    }
+
     fn http_mksession(&self) -> String {
         let mut mtx = self.http_sessions.lock().unwrap();
         let (id, session) = HttpSession::new();
@@ -439,6 +461,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
         logger,
         socket_sessions: Mutex::new(HashMap::new()),
         ws_sessions: Mutex::new(HashMap::new()),
+        mqtt_sessions: Mutex::new(HashMap::new()),
         blobs: Mutex::new(HashMap::new()),
         http_sessions: Mutex::new(HashMap::new()),
         http_clients: Mutex::new(HashMap::new()),
@@ -501,9 +524,14 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
     runtime::json_decode(&mut lua, state.clone());
     runtime::json_decode_stream(&mut lua, state.clone());
     runtime::json_encode(&mut lua, state.clone());
+    runtime::key_trunc_pad(&mut lua, state.clone());
     runtime::keyring(&mut lua, state.clone());
     runtime::last_err(&mut lua, state.clone());
     runtime::md5(&mut lua, state.clone());
+    runtime::mqtt_connect(&mut lua, state.clone());
+    runtime::mqtt_subscribe(&mut lua, state.clone());
+    runtime::mqtt_recv(&mut lua, state.clone());
+    runtime::mqtt_ping(&mut lua, state.clone());
     runtime::pgp_pubkey(&mut lua, state.clone());
     runtime::pgp_pubkey_armored(&mut lua, state.clone());
     runtime::print(&mut lua, state.clone());
@@ -536,6 +564,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua
     runtime::sock_recvuntil(&mut lua, state.clone());
     runtime::sock_sendafter(&mut lua, state.clone());
     runtime::sock_newline(&mut lua, state.clone());
+    runtime::sodium_secretbox_open(&mut lua, state.clone());
     runtime::status(&mut lua, state.clone());
     runtime::stdin_read_line(&mut lua, state.clone());
     runtime::stdin_read_to_end(&mut lua, state.clone());

src/lib.rs

@@ -32,6 +32,7 @@ pub mod keyring;
 use sn0int_std::lazy;
 pub mod migrations;
 pub mod models;
+use sn0int_std::mqtt;
 pub mod paths;
 pub use sn0int_std::psl;
 pub mod options;

src/models/mod.rs

@@ -119,7 +119,7 @@ impl From<&Insert> for Table {
 pub enum Update {
     Subdomain(SubdomainUpdate),
     IpAddr(IpAddrUpdate),
-    Url(UrlUpdate),
+    Url(UrlChangeset),
     Email(EmailUpdate),
     PhoneNumber(PhoneNumberUpdate),
     Device(DeviceUpdate),
@@ -374,6 +374,10 @@ impl<T: InsertToNew> LuaInsertToNew for T {
     }
 }
 
+pub trait UpdateToChangeset<T> {
+    fn try_into_changeset(self) -> Result<T>;
+}
+
 mod domain;
 pub use self::domain::*;
 

src/models/url.rs

@@ -1,10 +1,10 @@
+use crate::ser::StringOrBytes;
 use crate::errors::*;
 use crate::fmt::Write;
 use crate::fmt::colors::*;
 use crate::models::*;
 use diesel;
 use diesel::prelude::*;
-use crate::ser;
 use crate::url;
 
 
@@ -228,7 +228,6 @@ pub struct NewUrl {
     pub value: String,
     pub path: String,
     pub status: Option<i32>,
-    #[serde(deserialize_with="ser::opt_string_or_bytes")]
     pub body: Option<Vec<u8>>,
     pub online: Option<bool>,
     pub title: Option<String>,
@@ -254,7 +253,7 @@ impl InsertableStruct<Url> for NewUrl {
 }
 
 impl Upsertable<Url> for NewUrl {
-    type Update = UrlUpdate;
+    type Update = UrlChangeset;
 
     fn upsert(self, existing: &Url) -> Self::Update {
         Self::Update {
@@ -283,8 +282,7 @@ pub struct InsertUrl {
     pub subdomain_id: i32,
     pub value: String,
     pub status: Option<i32>,
-    #[serde(deserialize_with="ser::opt_string_or_bytes")]
-    pub body: Option<Vec<u8>>,
+    pub body: Option<StringOrBytes>,
     pub online: Option<bool>,
     pub title: Option<String>,
     pub redirect: Option<String>,
@@ -309,7 +307,7 @@ impl InsertToNew for InsertUrl {
             value: self.value,
             path,
             status: self.status,
-            body: self.body,
+            body: self.body.map(|x| x.0),
             online: self.online,
             title: self.title,
             redirect,
@@ -320,7 +318,7 @@ impl InsertToNew for InsertUrl {
 
 #[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
 #[table_name="urls"]
-pub struct UrlUpdate {
+pub struct UrlChangeset {
     pub id: i32,
     pub status: Option<i32>,
     pub body: Option<Vec<u8>>,
@@ -329,7 +327,7 @@ pub struct UrlUpdate {
     pub redirect: Option<String>,
 }
 
-impl Upsert for UrlUpdate {
+impl Upsert for UrlChangeset {
     fn is_dirty(&self) -> bool {
         self.status.is_some() ||
         self.body.is_some() ||
@@ -347,7 +345,7 @@ impl Upsert for UrlUpdate {
     }
 }
 
-impl Updateable<Url> for UrlUpdate {
+impl Updateable<Url> for UrlChangeset {
     fn changeset(&mut self, existing: &Url) {
         Self::clear_if_equal(&mut self.online, &existing.online);
         Self::clear_if_equal(&mut self.status, &existing.status);
@@ -365,6 +363,30 @@ impl Updateable<Url> for UrlUpdate {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct UrlUpdate {
+    pub id: i32,
+    pub status: Option<i32>,
+    pub body: Option<StringOrBytes>,
+    pub online: Option<bool>,
+    pub title: Option<String>,
+    pub redirect: Option<String>,
+}
+
+impl UpdateToChangeset<UrlChangeset> for UrlUpdate {
+    fn try_into_changeset(self) -> Result<UrlChangeset> {
+        // TODO: redirect needs pre-processing
+        Ok(UrlChangeset {
+            id: self.id,
+            status: self.status,
+            body: self.body.map(|x| x.0),
+            online: self.online,
+            title: self.title,
+            redirect: self.redirect,
+        })
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/runtime/crypto.rs

@@ -0,0 +1,30 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::engine::structs::{byte_array, lua_bytes};
+use crate::hlua::{self, AnyLuaValue};
+use sn0int_std::crypto;
+use std::sync::Arc;
+
+pub fn key_trunc_pad(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("key_trunc_pad", hlua::function3(move |bytes: AnyLuaValue, len: u32, pad: u8| -> Result<AnyLuaValue> {
+        let bytes = byte_array(bytes)
+            .map_err(|err| state.set_error(err))?;
+        let bytes = crypto::key_trunc_pad(&bytes, len as usize, pad);
+        Ok(lua_bytes(&bytes))
+    }))
+}
+
+pub fn sodium_secretbox_open(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("sodium_secretbox_open", hlua::function2(move |encrypted: AnyLuaValue, key: AnyLuaValue| -> Result<AnyLuaValue> {
+        let encrypted = byte_array(encrypted)
+            .map_err(|err| state.set_error(err))?;
+        let key = byte_array(key)
+            .map_err(|err| state.set_error(err))?;
+
+        let plain = crypto::sodium_secretbox_open(&encrypted, &key)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(lua_bytes(&plain))
+    }))
+}

src/runtime/db.rs

@@ -177,6 +177,28 @@ fn gen_changeset<T: Model, U: Updateable<T>>(object: LuaJsonValue, mut update: L
     Ok((existing.id(), value, update))
 }
 
+fn gen_changeset2<T: Model, U: UpdateToChangeset<C>, C: Updateable<T>>(object: LuaJsonValue, mut update: LuaJsonValue) -> Result<(i32, String, C)>
+    where
+        for<'de> T: serde::Deserialize<'de>,
+        for<'de> U: serde::Deserialize<'de>,
+        for<'de> C: serde::Deserialize<'de>,
+{
+    let existing = structs::from_lua::<T>(object)?;
+
+    // copy the id over to the update struct so we can identify the row
+    if let LuaJsonValue::Object(ref mut update) = update {
+        update.insert("id".into(), LuaJsonValue::Number(existing.id().into()));
+    }
+
+    let update = structs::from_lua::<U>(update)?;
+    let mut update = update.try_into_changeset()?;
+
+    let value = existing.to_string();
+    update.changeset(&existing);
+
+    Ok((existing.id(), value, update))
+}
+
 pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     lua.set("db_update", hlua::function3(move |family: String, object: AnyLuaValue, update: AnyLuaValue| -> Result<Option<i32>> {
         let family = Family::from_str(&family)
@@ -191,7 +213,7 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             Family::Ipaddr => gen_changeset::<IpAddr, IpAddrUpdate>(object, update)
                 .map(|(id, v, u)| (id, v, Update::IpAddr(u))),
             Family::SubdomainIpaddr => bail!("Subdomain-IpAddr doesn't have mutable fields"),
-            Family::Url => gen_changeset::<Url, UrlUpdate>(object, update)
+            Family::Url => gen_changeset2::<Url, UrlUpdate, UrlChangeset>(object, update)
                 .map(|(id, v, u)| (id, v, Update::Url(u))),
             Family::Email => gen_changeset::<Email, EmailUpdate>(object, update)
                 .map(|(id, v, u)| (id, v, Update::Email(u))),

src/runtime/mod.rs

@@ -6,6 +6,7 @@ macro_rules! import_fns {
 }
 
 import_fns!(blobs);
+import_fns!(crypto);
 import_fns!(datetime);
 import_fns!(db);
 import_fns!(dns);
@@ -22,6 +23,7 @@ import_fns!(int);
 import_fns!(json);
 import_fns!(keyring);
 import_fns!(logger);
+import_fns!(mqtt);
 import_fns!(options);
 import_fns!(pgp);
 import_fns!(psl);

src/runtime/mqtt.rs

@@ -0,0 +1,59 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::hlua::{self, AnyLuaValue};
+use crate::mqtt::MqttOptions;
+use std::sync::Arc;
+use url::Url;
+
+pub fn mqtt_connect(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("mqtt_connect", hlua::function2(move |url: String, options: AnyLuaValue| -> Result<String> {
+        let options = MqttOptions::try_from(options)
+            .context("Invalid mqtt options")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        let url = Url::parse(&url)
+            .context("Failed to parse url")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        state.mqtt_connect(url, &options)
+            .map_err(|err| state.set_error(err))
+    }))
+}
+
+pub fn mqtt_subscribe(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("mqtt_subscribe", hlua::function3(move |sock: String, topic: String, level: u8| -> Result<()> {
+        let sock = state.get_mqtt(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        sock.subscribe(&topic, level)
+            .map_err(|err| state.set_error(err))
+    }))
+}
+
+pub fn mqtt_recv(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("mqtt_recv", hlua::function1(move |sock: String| -> Result<AnyLuaValue> {
+        let sock = state.get_mqtt(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let pkt = sock.recv_pkt()
+            .map_err(|err| state.set_error(err))?;
+
+        if let Some(pkt) = pkt {
+            pkt.to_lua()
+                .map_err(|err| state.set_error(err))
+        } else {
+            Ok(AnyLuaValue::LuaNil)
+        }
+    }))
+}
+
+pub fn mqtt_ping(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("mqtt_ping", hlua::function1(move |sock: String| -> Result<()> {
+        let sock = state.get_mqtt(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        sock.ping()
+            .map_err(|err| state.set_error(err))
+    }))
+}

src/runtime/pgp.rs

@@ -158,4 +158,25 @@ vA==
         "#).expect("Failed to load script");
         script.test().expect("Script failed");
     }
+
+    #[test]
+    #[ignore]
+    fn verify_pgp_fetch_wkd() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            session = http_mksession()
+            url = 'https://openpgpkey.archlinux.org/.well-known/openpgpkey/archlinux.org/hu/in9mwr4s84x7gm51851h343n3at1x61g?l=anthraxx'
+            req = http_request(session, 'GET', url, {
+                binary=true,
+            })
+            r = http_fetch(req)
+            k = pgp_pubkey(r['binary'])
+            info(k)
+            if k['fingerprint'] ~= 'E240B57E2C4630BA768E2F26FC1B547C8D8172C8' then
+                return 'wrong fingerprint: ' .. k['fingerprint']
+            end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
 }

src/sandbox/mod.rs

@@ -5,7 +5,7 @@ use caps::{self, CapSet};
 use nix;
 
 #[cfg(target_os = "openbsd")]
-use pledge::{pledge, Promise, ToPromiseString};
+use pledge::pledge;
 #[cfg(target_os = "openbsd")]
 use unveil::unveil;
 
@@ -72,7 +72,7 @@ pub fn init_openbsd() -> Result<()> {
     unveil("", "")
         .map_err(|_| format_err!("Failed to call unveil"))?;
 
-    pledge![Stdio, RPath, Dns, Inet]?;
+    pledge![Stdio Rpath Dns Inet,]?;
 
     Ok(())
 }

src/ser.rs

@@ -8,7 +8,7 @@ use std::marker::PhantomData;
 
 
 #[derive(Debug, Serialize, Deserialize)]
-struct StringOrBytes(#[serde(deserialize_with="string_or_bytes")] Vec<u8>);
+pub struct StringOrBytes(#[serde(deserialize_with="string_or_bytes")] pub Vec<u8>);
 
 pub fn string_or_bytes<'de, D>(deserializer: D) -> result::Result<Vec<u8>, D::Error> where D: Deserializer<'de> {
     struct StringOrBytes(PhantomData<fn() -> Vec<u8>>);

src/shell/complete.rs

@@ -202,6 +202,7 @@ impl Completer for CmdCompleter {
                             "reload",
                             "update",
                             "uninstall",
+                            "quickstart",
                         ], &cmd[1]))
                     }
                 },

src/shell/mod.rs

@@ -506,12 +506,13 @@ pub fn init<'a>(args: &Args, config: &'a Config, verbose_init: bool) -> Result<S
     let keyring = KeyRing::init()?;
 
     if verbose_init && library.list().is_empty() {
-        term::success("No modules found, run quickstart to install default modules");
+        term::success("No modules found, run \x1b[1mpkg quickstart\x1b[0m to install default modules");
+        term::success("New to sn0int? Follow https://sn0int.rtfd.io/en/stable/usage.html");
     }
 
     let autoupdate = AutoUpdater::load()?;
     if autoupdate.outdated() > 0 {
-        term::warn(&format!("{} modules are outdated, run: mod update", autoupdate.outdated()));
+        term::warn(&format!("{} modules are outdated, run: \x1b[1mpkg update\x1b[0m", autoupdate.outdated()));
     }
     autoupdate.check_background(&config, library.list());
 

src/update.rs

@@ -9,8 +9,8 @@ use std::path::PathBuf;
 use std::time::{SystemTime, UNIX_EPOCH};
 use std::thread;
 
-// 1 week
-const UPDATE_INTERVAL: u64 = 3600 * 24 * 7;
+// 1 day
+const UPDATE_INTERVAL: u64 = 3600 * 24;
 
 
 #[derive(Debug, Default, Serialize, Deserialize)]

src/worker.rs

@@ -244,7 +244,11 @@ impl DatabaseEvent {
                 log.push_str(&format!("@ {}", object.time));
 
                 if let (Some(ref lat), Some(ref lon)) = (object.latitude, object.longitude) {
-                    log.push_str(&format!(" ({}, {})", lat, lon));
+                    log.push_str(&format!(" ({}, {}", lat, lon));
+                    if let Some(radius) = &object.radius {
+                        log.push_str(&format!(" | {}m", radius));
+                    }
+                    log.push_str(")");
                 }
 
                 if verbose > 0 {