Release v0.17.1

seccomp: whitelist ppoll instead of poll on aarch64

.dockerignore

@@ -1,6 +1,7 @@
 target
 Dockerfile
 .dockerignore
+docker-compose.yml
 docker
 docs
 ci

.travis.yml

@@ -5,18 +5,6 @@ language: rust
 
 matrix:
   include:
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=test
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=common
-  - os: linux
-    rust: stable
-    env:
-    - BUILD_MODE=boxxy
   - os: linux
     rust: stable
     env:
@@ -25,6 +13,22 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=docker-registry
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=test
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=common
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=std
+  - os: linux
+    rust: stable
+    env:
+    - BUILD_MODE=boxxy
   - os: osx
     rust: stable
     env:
@@ -33,14 +37,18 @@ matrix:
     rust: stable
     env:
     - BUILD_MODE=common
+  - os: osx
+    rust: stable
+    env:
+    - BUILD_MODE=std
   #- os: windows
   #  rust: stable
   #  env:
   #  - BUILD_MODE="windows test"
-  - os: windows
-    rust: stable
-    env:
-    - BUILD_MODE="windows common"
+  #- os: windows
+  #  rust: stable
+  #  env:
+  #  - BUILD_MODE="windows common"
 
 before_install:
 - ci/setup.sh "$TRAVIS_OS_NAME"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.14.0"
+version = "0.17.1"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -13,15 +13,17 @@ edition = "2018"
 travis-ci = { repository = "kpcyrd/sn0int" }
 
 [workspace]
-members = ["sn0int-registry/sn0int-common",
-           "sn0int-registry"]
+members = ["sn0int-common",
+           "sn0int-registry",
+           "sn0int-std"]
 
 [features]
 sqlite-bundled = ["libsqlite3-sys/bundled"]
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-registry/sn0int-common" }
-rustyline = "5.0"
+sn0int-common = { version="0.10.0", path="sn0int-common" }
+sn0int-std = { version="0.17.1", path="sn0int-std" }
+rustyline = "6.0"
 log = "0.4"
 env_logger = "0.7"
 hlua-badtouch = "0.4"
@@ -31,7 +33,6 @@ rand = "0.7"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
-publicsuffix = { version="1.5", default-features=false }
 diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
 libsqlite3-sys = "0.16.0"
@@ -40,14 +41,9 @@ dirs = "2.0"
 url = "2.0"
 percent-encoding = "2.1"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.13"
-rustls = { version="0.16", features=["dangerous_configuration"] }
-webpki = "0.21"
-webpki-roots = "0.18"
-pem = "0.7"
+chrootable-https = "0.14"
 base64 = "0.11"
 data-encoding = "2.1.2"
-kuchiki = "0.7.2"
 serde_urlencoded = "0.6"
 serde = "1.0"
 serde_derive = "1.0"
@@ -60,42 +56,30 @@ maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.5"
 regex = "1.0"
 toml = "0.5"
-maxminddb = "0.13"
-tar = "0.4.17"
-libflate = "0.1.14"
 threadpool = "1.7"
-x509-parser = "0.6.0"
-der-parser = "3.0"
 atty = "0.2"
-bufstream = "0.1.4"
-tokio = "0.1.14"
 semver = "0.9"
 bytes = "0.4"
-xml-rs = "0.8"
 bytesize = "1.0"
-ipnetwork = "0.15"
-strum = "0.16"
-strum_macros = "0.16"
+ipnetwork = "0.16"
+strum = "0.17"
+strum_macros = "0.17"
 
 digest = "0.8.0"
-bs58 = "0.3"
-blake2 = "0.8.0"
 md-5 = "0.8.0"
 sha-1 = "0.8.1"
 sha2 = "0.8.0"
 sha3 = "0.8.0"
 hmac = "0.7"
 
-image = "0.22"
-kamadak-exif = "0.3.1"
 walkdir = "2.2"
-nude = "0.2"
+nude = "0.3"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
 #syscallz = { path="../syscallz-rs" }
-syscallz = "0.11"
-nix = "0.15"
+syscallz = "0.12"
+nix = "0.17"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.3.1"

Makefile

@@ -1,19 +1,23 @@
 check:
-	(cd sn0int-registry/sn0int-common; cargo check)
+	(cd sn0int-common; cargo check)
 	(cd sn0int-registry; cargo check)
+	(cd sn0int-std; cargo check)
 	cargo check
 
 force-check:
-	(cd sn0int-registry/sn0int-common; touch src/lib.rs; cargo check)
+	(cd sn0int-common; touch src/lib.rs; cargo check)
 	(cd sn0int-registry; touch src/main.rs; cargo check)
+	(cd sn0int-std; touch src/lib.rs; cargo check)
 	touch src/lib.rs
 	cargo check
 
 test:
-	(cd sn0int-registry/sn0int-common; cargo test)
+	(cd sn0int-common; cargo test)
 	(cd sn0int-registry; cargo test)
-	cargo test
-	cargo test -- --ignored
+	(cd sn0int-std; cargo test)
+	(cd sn0int-std; cargo test -- --ignored)
+	cargo test --lib
+	cargo test --lib -- --ignored
 
 update:
 	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt

README.md

@@ -116,10 +116,17 @@ For everything else please have a look at the [detailed list][1].
   - [Images](https://sn0int.readthedocs.io/en/latest/structs.html#images)
   - [Ports](https://sn0int.readthedocs.io/en/latest/structs.html#ports)
   - [Netblocks](https://sn0int.readthedocs.io/en/latest/structs.html#netblocks)
+  - [CryptoAddrs](https://sn0int.readthedocs.io/en/latest/structs.html#cryptoaddrs)
+  - [Activity](https://sn0int.readthedocs.io/en/latest/structs.html#activity)
   - [Relations](https://sn0int.readthedocs.io/en/latest/structs.html#relations)
     - [subdomain_ipaddr](https://sn0int.readthedocs.io/en/latest/structs.html#subdomain-ipaddr)
     - [network_device](https://sn0int.readthedocs.io/en/latest/structs.html#network-device)
     - [breach_email](https://sn0int.readthedocs.io/en/latest/structs.html#breach-email)
+- [Activity](https://sn0int.readthedocs.io/en/latest/activity.html)
+  - [Anatomy of an event](https://sn0int.readthedocs.io/en/latest/activity.html#anatomy-of-an-event)
+  - [Logging events](https://sn0int.readthedocs.io/en/latest/activity.html#logging-events)
+  - [Querying events](https://sn0int.readthedocs.io/en/latest/activity.html#querying-events)
+  - [Visualization](https://sn0int.readthedocs.io/en/latest/activity.html#visualization)
 - [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
   - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
   - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
@@ -180,6 +187,7 @@ For everything else please have a look at the [detailed list][1].
   - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
   - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
   - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [ratelimit_throttle](https://sn0int.readthedocs.io/en/latest/reference.html#ratelimit-throttle)
   - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
   - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
   - [semver_match](https://sn0int.readthedocs.io/en/latest/reference.html#semver-match)
@@ -195,6 +203,7 @@ For everything else please have a look at the [detailed list][1].
   - [sn0int_version](https://sn0int.readthedocs.io/en/latest/reference.html#sn0int-version)
   - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
   - [sock_upgrade_tls](https://sn0int.readthedocs.io/en/latest/reference.html#sock-upgrade-tls)
+  - [sock_options](https://sn0int.readthedocs.io/en/latest/reference.html#sock-options)
   - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
   - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
   - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
@@ -209,6 +218,8 @@ For everything else please have a look at the [detailed list][1].
   - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
   - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
   - [stdin_read_to_end](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-read-to-end)
+  - [str_find](https://sn0int.readthedocs.io/en/latest/reference.html#str-find)
+  - [str_replace](https://sn0int.readthedocs.io/en/latest/reference.html#str-replace)
   - [strftime](https://sn0int.readthedocs.io/en/latest/reference.html#strftime)
   - [strptime](https://sn0int.readthedocs.io/en/latest/reference.html#strptime)
   - [time_unix](https://sn0int.readthedocs.io/en/latest/reference.html#time-unix)
@@ -221,6 +232,14 @@ For everything else please have a look at the [detailed list][1].
   - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
   - [warn](https://sn0int.readthedocs.io/en/latest/reference.html#warn)
   - [warn_once](https://sn0int.readthedocs.io/en/latest/reference.html#warn-once)
+  - [ws_connect](https://sn0int.readthedocs.io/en/latest/reference.html#ws-connect)
+  - [ws_options](https://sn0int.readthedocs.io/en/latest/reference.html#ws-options)
+  - [ws_recv_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-text)
+  - [ws_recv_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-binary)
+  - [ws_recv_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-recv-json)
+  - [ws_send_text](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-text)
+  - [ws_send_binary](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-binary)
+  - [ws_send_json](https://sn0int.readthedocs.io/en/latest/reference.html#ws-send-json)
   - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
   - [xml_decode](https://sn0int.readthedocs.io/en/latest/reference.html#xml-decode)
   - [xml_named](https://sn0int.readthedocs.io/en/latest/reference.html#xml-named)

ci/integration.py

@@ -63,6 +63,7 @@ def main(tempdir, binary):
     print('[*] testing db for subdomains')
     subdomains = sn0int_select(tempdir, binary, ['subdomains'])
     assert {x['value'] for x in subdomains} == {
+        'example.com',
         'www.example.com',
         'm.example.com',
         'dev.example.com',
@@ -91,6 +92,8 @@ def main(tempdir, binary):
     print('[*] testing db for urls')
     urls = sn0int_select(tempdir, binary, ['urls'])
     assert {(x['value'], x['status']) for x in urls} == {
+        ('http://example.com/', 200),
+        ('https://example.com/', 200),
         ('http://www.example.com/', 200),
         ('https://www.example.com/', 200),
     }

ci/run.sh

@@ -6,18 +6,18 @@ case "$1" in
         cargo build --verbose --examples
         ;;
     test)
-        ci/run.sh build
-        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
-             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;
     common)
-        cd sn0int-registry/sn0int-common
+        cd sn0int-common
         cargo test --verbose
         ;;
+    std)
+        cd sn0int-std
+        cargo test --verbose
+        cargo test --verbose -- --ignored
+        ;;
     windows)
         cargo build --verbose --features=sqlite-bundled
         cargo build --verbose --examples --features=sqlite-bundled
@@ -35,7 +35,7 @@ case "$1" in
         docker run --rm sn0int --help
         ;;
     docker-registry)
-        docker build -t sn0int-registry sn0int-registry/
+        docker build -t sn0int-registry -f sn0int-registry/Dockerfile .
         docker images
         ;;
 esac

docs/activity.rst

@@ -0,0 +1,166 @@
+Activity
+========
+
+So far we've learned about regular `structs <structs.html>`_, but activity is
+special.
+
+Activity is an event tied to a specific time and topic and has a small amount
+of data piggybacked to it.
+
+Anatomy of an event
+-------------------
+
+``topic``
+    This is some freestyle text used to group events to a specific topic. This
+    must not conflict with other modules unless there's a very good reason.
+
+    The topic should look like ``kpcyrd/example:something``, with ``something``
+    being a meaningful unique identifier for whatever is generating these
+    events, like a mac address or an account name/id.
+
+    The rules around this might become stricter in the future.
+``time``
+    The most important part of the event: The time and date it happened.
+``initial``
+    This value can not be set but might be present in sn0int output. See `Querying events`_.
+``uniq`` (optional)
+    This is an optional feature to deduplicate events. Assuming you're
+    importing posts by an account, you wouldn't want to store a new event for
+    each post you already imported. If you set this field to the technical post
+    id then sn0int would skip the event if it already has an event with the
+    same ``topic`` and ``uniq`` combination to avoid inserting duplicates.
+``latitude`` (optional)
+    Latitude - if you can tie the event to a specific location.
+``longitude`` (optional)
+    Longitude - if you can tie the event to a specific location.
+``radius`` (optional)
+    The location radius in meters. If the position you got has a precision of
+    100 meters set this value to ``100``.
+``content``
+    Arbitrary data that you want to attach to the event. This doesn't need to
+    be a string and can be an arbitrary object that is then stored as json
+    string.
+
+Logging events
+--------------
+
+An ``activity`` event can be logged with ``db_activity``:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Logging an event that has a location attached could look like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        latitude=40.726662,
+        longitude=-74.036677,
+        radius=50,
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+Making sure an event is not logged twice can be done with ``uniq``:
+
+.. code-block:: lua
+
+    -- create the first event
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this does nothing because we already have an event with this topic+uniq combination
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='12345',
+        content='ohai',
+    })
+
+    -- this creates a new event because uniq is different
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+    -- this also creates a new event because topic is different
+    db_activity({
+        topic='harness/activity-ping:something-else',
+        time=sn0int_time(),
+        uniq='6789',
+        content='ohai',
+    })
+
+Querying events
+---------------
+
+There is a commandline interface that can be used to query all events we've
+logged. To get everything (sorted by time)::
+
+    sn0int activity
+
+To limit the output to a specific topic::
+
+    sn0int activity -t harness/activity-ping:dummy
+
+To limit it to a specific time frame::
+
+    # everything since
+    sn0int activity --since 2020-01-13T04:20:00
+    # everything until
+    sn0int activity --until 2020-01-13T04:20:00
+    # both
+    sn0int activity --since yesterday --until today
+
+When using ``--since`` you might also want to know the previous state and use
+it as an initial value. Consider this example::
+
+    2020-01-13 14:30:00 # user goes offline
+    2020-01-13 23:59:00 # user goes online
+    2020-01-14 09:30:00 # user goes idle
+    2020-01-14 14:20:00 # user goes offline
+
+If we're running a query like ``sn0int activity --since 2020-01-14T00:00:00``
+the program consuming the output wouldn't know that the user is initially
+online because we're only getting this data::
+
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+We can tweak this with ``sn0int activity --initial --since
+2020-01-14T00:00:00`` to include one more event that we only use to populate
+the intial state::
+
+    {"id":7,"initial":true,"topic":"foo/bar:asdf","time":"2020-01-13T23:59:00","content":{"state":"online"}}
+    {"id":8,"topic":"foo/bar:asdf","time":"2020-01-14T09:30:00","content":{"state":"idle"}}
+    {"id":9,"topic":"foo/bar:asdf","time":"2020-01-14T14:20:00","content":{"state":"offline"}}
+
+Visualization
+-------------
+
+There is no visualization built in, there may be external frontends for this in
+the future. You're very welcome to write one!

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'sn0int'
-copyright = '2018, kpcyrd'
+copyright = '2018-2020, kpcyrd'
 author = 'kpcyrd'
 
 # The short X.Y version

docs/database.rst

@@ -46,6 +46,45 @@ triggered and an db_update is performed instead.
    removed from scope with ``noscope``. Everytime you use ``db_add`` you need
    to make sure that the ID that has been returned is not ``nil``.
 
+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add_ttl('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_update
 ---------
 

docs/index.rst

@@ -41,6 +41,7 @@ Getting Started
    scripting
    database
    structs
+   activity
    keyring
    config
    sandbox

docs/install.rst

@@ -47,8 +47,6 @@ Docker
 Alpine
 ------
 
-On alpine edge, with enabled testing repositories:
-
 .. code-block:: bash
 
     $ apk add sn0int
@@ -56,8 +54,6 @@ On alpine edge, with enabled testing repositories:
 OpenBSD
 -------
 
-On -current:
-
 .. code-block:: bash
 
     $ pkg_add sn0int

docs/keyring.rst

@@ -58,6 +58,10 @@ If the user granted us access to those keys we can read them with ``keyring``:
 This returns a list of all keys in that namespace. Any empty list is returned
 if the user doesn't have any keys in that namespace.
 
+If you want to allow the user to select a specific script you can introduce an
+option that is set by the user and then filter ``creds`` until the
+``access_key`` matches.
+
 Using access keys as source argument
 ------------------------------------
 

docs/reference.rst

@@ -153,6 +153,28 @@ ttl.
         device_id=13,
     }, 120)
 
+db_activity
+-----------
+
+Log an activity event. A basic event looks like this:
+
+.. code-block:: lua
+
+    db_activity({
+        topic='harness/activity-ping:dummy',
+        time=sn0int_time(),
+        content={
+            a='b',
+            foo={
+                bar=1337,
+            },
+            msg='ohai',
+        },
+    })
+
+This function is explained in detail in the `activity <activity.html>`_
+section.
+
 db_select
 ---------
 
@@ -643,6 +665,22 @@ Returns the parent domain according to the public suffix list. For
     domain = psl_domain_from_dns_name('www.a.b.c.d.example.co.uk')
     print(domain == 'example.co.uk')
 
+ratelimit_throttle
+------------------
+
+Create a ratelimit that can only be passed x times every y milliseconds. This
+limit is global for a single ``run`` and also works with threads.
+
+.. code-block:: lua
+
+    -- allow this to pass every 250ms
+    ratelimit_throttle('foo', 1, 250)
+    -- allow this to pass not more than 4 times per second
+    ratelimit_throttle('foo', 4, 1000)
+
+This is useful if you need to coordinate your executions to stay below a
+certain request threshold.
+
 regex_find
 ----------
 
@@ -818,6 +856,13 @@ The following options are available:
 ``proxy``
   Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
   if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
 
 .. code-block:: lua
 
@@ -849,6 +894,23 @@ discarded when using sock_connect_ directly with ``tls=true``.
 
     info(tls)
 
+sock_options
+------------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock_options(sock, {
+        read_timeout=3,
+    })
+
 sock_send
 ---------
 
@@ -985,6 +1047,32 @@ Read stdin until EOF as a utf-8 string.
 .. note::
    This only works with `sn0int run --stdin`.
 
+str_find
+--------
+
+Returns the byte index of the first character that matches the pattern. This is
+explicitly a literal match instead of a lua pattern.
+
+If no match is found, returns ``nil``.
+
+.. code-block:: lua
+
+    x = str_find('asdf', 'sd')
+    print(x == 2)
+
+str_replace
+-----------
+
+Replaces all matches of a pattern in a string. This is explicitly a literal
+match instead of a lua pattern.
+
+If no match is found, an unmodified copy is returned.
+
+.. code-block:: lua
+
+    x = str_replace('this is old', 'old', 'new')
+    print(x == 'this is new')
+
 strftime
 --------
 
@@ -1125,6 +1213,111 @@ a ``run`` execution.
     warn_once('ohai')
     warn_once('ohai')
 
+ws_connect
+----------
+
+Create a websocket connection. The url format is ``ws://example.com/asdf``,
+``wss://`` is also supported.
+
+The following options are available:
+
+``headers``
+  A map of additional headers that should be set for the request.
+``proxy``
+  Use a socks5 proxy in the format ``127.0.0.1:9050``. This option only works
+  if it doesn't conflict with the global proxy settings.
+``connect_timeout``
+  Abort tcp connection attempts after ``n`` seconds.
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    sock = ws_connect("wss://example.com/asdf", {})
+
+ws_options
+----------
+
+Update options of an existing connection:
+
+``read_timeout``
+  Abort read attempts after ``n`` seconds. This can be used to wake up
+  connections periodically.
+``write_timeout``
+  Abort write attempts after ``n`` seconds.
+
+.. code-block:: lua
+
+    ws_options(sock, {
+        read_timeout=3,
+    })
+
+ws_recv_text
+------------
+
+Wait until the server sends a text frame. A binary frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_text(sock)
+
+ws_recv_binary
+--------------
+
+Wait until the server sends a binary frame. A text frame is considered an
+error. Ping requests are answered automatically.
+
+.. code-block:: lua
+
+    msg = ws_recv_binary(sock)
+
+ws_recv_json
+------------
+
+Identical to ws_send_text_ but automatically runs json_decode_ on the
+response.
+
+.. code-block:: lua
+
+    msg = ws_recv_json(sock)
+
+ws_send_text
+------------
+
+Send a text frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, "ohai!")
+
+ws_send_binary
+--------------
+
+Send a binary frame on the websocket connection.
+
+.. code-block:: lua
+
+    ws_send_binary(sock, "\x00\x01\x02")
+
+ws_send_json
+------------
+
+Encode the object as json string and send it as a text frame on the websocket
+connection.
+
+.. code-block:: lua
+
+    ws_send_text(sock, {
+        foo="ohai!",
+        x={
+            y={1,3,3,7},
+        },
+    })
+
 x509_parse_pem
 --------------
 

docs/structs.rst

@@ -259,6 +259,35 @@ is ``140.82.112.0/20``.
     This field isn't strictly defined and meant to be used as a human
     meaningful name if available.
 
+CryptoAddrs
+-----------
+
+A cryptoaddr is any cryptocurrency address and not tied to a specific currency.
+
+``value``
+    The address string. This looks like ``1BvBMSEYstWetqTFn5Au4m4GFg7xJaNVN2``.
+``currency``
+    The identifier for a specific currency. This is usually the ticker symbols,
+    like ``xbt``, ``zec`` or ``xmr``.
+``denominator``
+    Balance is tracked internally using 64 bit integers (signed, for technical reasons). Balance is supposed to be the lowest unit, so in case of bitcoin you'd write ``100,000,000`` satoshi instead of ``1`` bitcoin. Since this value is inconvinient to work with we're using the denominator to display values. In case of bitcoin you'd set it to ``8``.
+``balance``
+    The current balance of the address, in the lowest possible unit. In case of bitcoin this would be satoshis.
+``received``
+    The total amount of currency received by this address.
+``first_seen``
+    The first time currency was sent to this address.
+``last_withdrawal``
+    The last time a transaction signed by this address was observed.
+``description``
+    A human readable note for this address.
+
+Activity
+--------
+
+Activity is different from all other structs, have a look at the `Activity
+Section <activity.html>`_.
+
 Relations
 ---------
 

examples/maxmind.rs

@@ -1,61 +1,27 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
 use sn0int::errors::*;
 use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
 use sn0int::paths;
-use std::fs;
 use std::net::IpAddr;
+use std::path::Path;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
 pub enum Args {
-    #[structopt(name="dl")]
-    Download(Download),
     #[structopt(name="asn")]
     Asn(AsnArgs),
     #[structopt(name="geoip")]
     GeoIP(GeoIPArgs),
 }
 
-#[derive(Debug, StructOpt)]
-pub struct Download {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-impl Download {
-    fn run(&self) -> Result<()> {
-        let path = paths::cache_dir()?.join(&self.target);
-        if self.extract_only {
-            let body = fs::read(&self.url)?;
-            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
-        } else {
-            GeoIP::download(path, &self.filter, &self.url)?;
-        }
-        Ok(())
-    }
-}
-
 #[derive(Debug, StructOpt)]
 pub struct AsnArgs {
     ip: IpAddr,
 }
 
 impl AsnArgs {
-    fn run(&self) -> Result<()> {
-        let asndb = AsnDB::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = AsnDB::cache_path(cache_dir)?;
+        let asndb = AsnDB::open(&path)?;
 
         let asn = asndb.lookup(self.ip)?;
         println!("{:#?}", asn);
@@ -70,8 +36,9 @@ pub struct GeoIPArgs {
 }
 
 impl GeoIPArgs {
-    fn run(&self) -> Result<()> {
-        let geoip = GeoIP::open_or_download()?;
+    fn run(&self, cache_dir: &Path) -> Result<()> {
+        let path = GeoIP::cache_path(cache_dir)?;
+        let geoip = GeoIP::open(&path)?;
 
         let lookup = geoip.lookup(self.ip)?;
         println!("{:#?}", lookup);
@@ -84,10 +51,10 @@ impl GeoIPArgs {
 fn run() -> Result<()> {
     let args = Args::from_args();
     debug!("{:?}", args);
+    let cache_dir = paths::cache_dir()?;
     match args {
-        Args::Download(args) => args.run(),
-        Args::Asn(args) => args.run(),
-        Args::GeoIP(args) => args.run(),
+        Args::Asn(args) => args.run(&cache_dir),
+        Args::GeoIP(args) => args.run(&cache_dir),
     }
 }
 

migrations/2020-01-09-024234_activity/down.sql

@@ -0,0 +1,31 @@
+DROP TABLE activity;
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

migrations/2020-01-09-024234_activity/up.sql

@@ -0,0 +1,44 @@
+CREATE TABLE activity (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    topic VARCHAR NOT NULL,
+    time DATETIME NOT NULL,
+    uniq VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    radius INTEGER,
+    content VARCHAR NOT NULL
+);
+CREATE UNIQUE INDEX activity_uniq ON activity(topic, uniq);
+CREATE INDEX activity_topic ON activity(topic);
+CREATE INDEX activity_time ON activity(time);
+CREATE INDEX activity_topic_time ON activity(topic, time);
+
+PRAGMA foreign_keys=off;
+
+-- ports
+CREATE TABLE _ports_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    ip_addr VARCHAR NOT NULL,
+    port INTEGER NOT NULL,
+    protocol VARCHAR NOT NULL,
+    status VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+
+    banner VARCHAR,
+    service VARCHAR,
+    version VARCHAR,
+
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT port_unique UNIQUE (value)
+);
+
+INSERT INTO _ports_new (id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version)
+    SELECT id, ip_addr_id, value, ip_addr, port, protocol, status, unscoped, banner, service, version
+    FROM ports;
+
+DROP TABLE ports;
+ALTER TABLE _ports_new RENAME TO ports;
+
+PRAGMA foreign_keys=on;

modules/harness/activity-ping.lua

@@ -0,0 +1,29 @@
+-- Description: Log some dummy activity
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local uniq = getopt('uniq')
+    local topic = getopt('topic') or 'harness/activity-ping:dummy'
+
+    if getopt('gps') then
+        lat=1.23
+        lon=4.56
+        radius=100
+    end
+
+    while true do
+        db_activity({
+            topic=topic,
+            time=sn0int_time(),
+            uniq=uniq,
+            latitude=lat,
+            longitude=lon,
+            radius=radius,
+            content={
+                msg='ohai',
+            },
+        })
+        sleep(5)
+    end
+end

modules/harness/geo-polygon-contains.lua

@@ -0,0 +1,74 @@
+-- Description: demonstrate geofencing with polygons
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    hamburg = {
+        { lat=53.63975308945899,  lon=9.764785766601562  },
+        { lat=53.59494998253459,  lon=9.827270507812     },
+        { lat=53.663153974456456, lon=9.9151611328125    },
+        { lat=53.65582987649682,  lon=9.976272583007812  },
+        { lat=53.68613523817129,  lon=9.992752075195312  },
+        { lat=53.68674518938816,  lon=10.051460266113281 },
+        { lat=53.72495117617815,  lon=10.075492858886719 },
+        { lat=53.71946627930625,  lon=10.118408203125    },
+        { lat=53.743635083157756, lon=10.164413452148438 },
+        { lat=53.73104466704585,  lon=10.202865600585938 },
+        { lat=53.676781546441546, lon=10.16304016113281  },
+        { lat=53.632832079199474, lon=10.235824584960938 },
+        { lat=53.608803292930894, lon=10.2008056640625   },
+        { lat=53.578646152866504, lon=10.208358764648438 },
+        { lat=53.57212285981298,  lon=10.163726806640625 },
+        { lat=53.52071674896369,  lon=10.18707275390625  },
+        { lat=53.52643162253097,  lon=10.224151611328125 },
+        { lat=53.44062753992289,  lon=10.347747802734375 },
+        { lat=53.38824275010831,  lon=10.248870849609375 },
+        { lat=53.38824275010831,  lon=10.15960693359375  },
+        { lat=53.44635321212876,  lon=10.064849853515625 },
+        { lat=53.40595029739904,  lon=9.985198974609375  },
+        { lat=53.42385506057106,  lon=9.951210021972656  },
+        { lat=53.41843327091211,  lon=9.944171905517578  },
+        { lat=53.41812635648326,  lon=9.927349090576172  },
+        { lat=53.412294561442884, lon=9.917736053466797  },
+        { lat=53.41464783813818,  lon=9.901256561279297  },
+        { lat=53.443490472483326, lon=9.912586212158201  },
+        { lat=53.45177144115704,  lon=9.897651672363281  },
+        { lat=53.43633277935392,  lon=9.866924285888672  },
+        { lat=53.427639673754776, lon=9.866409301757812  },
+        { lat=53.427639673754776, lon=9.858856201171875  },
+        { lat=53.46710230573499,  lon=9.795513153076172  },
+        { lat=53.49039461941655,  lon=9.795341491699219  },
+        { lat=53.49029248806277,  lon=9.77903366088867   },
+        { lat=53.49856433088649,  lon=9.780235290527344  },
+        { lat=53.5078554643033,   lon=9.758434295654297  },
+        { lat=53.545407634092975, lon=9.759807586669922  },
+        { lat=53.568147234570084, lon=9.633293151855469  },
+        { lat=53.58802162343514,  lon=9.655780792236328  },
+        { lat=53.568351121879815, lon=9.727706909179688  },
+        { lat=53.60921067445695,  lon=9.737663269042969  },
+    }
+
+    points = {
+        {
+            name='Alice',
+            lat=52.52437,
+            lon=13.41053,
+        }, {
+            name='Bob',
+            lat=53.551085,
+            lon=9.993682,
+        }, {
+            name='Charlie',
+            lat=40.726662,
+            lon=-74.036677,
+        }
+    }
+
+    for i=1, #points do
+        if geo_polygon_contains(hamburg, points[i]) then
+            info('[INSIDE ] ' .. points[i]['name'])
+        else
+            info('[OUTSIDE] ' .. points[i]['name'])
+        end
+    end
+end

modules/harness/google-tls.lua

@@ -0,0 +1,29 @@
+-- Description: Test various tls functions
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info('sending https request to google.com')
+    session = http_mksession()
+    req = http_request(session, 'GET', 'https://google.com/', {})
+    r = http_send(req)
+    if last_err() then return end
+    debug(r)
+
+    info('creating tls socket to google.com')
+    sock = sock_connect('google.com', 443, {
+        tls=true,
+    })
+    if last_err() then return end
+    debug(sock)
+
+    info('creating socket to google.com, wrapping afterwards')
+    sock = sock_connect('google.com', 443, {})
+    if last_err() then return end
+    tls = sock_upgrade_tls(sock, {
+        sni_value='google.com',
+    })
+    if last_err() then return end
+    debug(sock)
+    debug(tls)
+end

modules/harness/ratelimit.lua

@@ -0,0 +1,11 @@
+-- Description: Run script with a global ratelimit
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- this shouldn't complete in less than 5 seconds
+    for i=1, 20 do
+        ratelimit_throttle('foo', 4, 1000)
+        info(sn0int_time())
+    end
+end

modules/harness/socket-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = sock_connect('127.0.0.1', 4444, {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            sock_send(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        sock_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = sock_recv(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/ws-ping.lua

@@ -0,0 +1,33 @@
+-- Description: Connect somewhere and send a ping every 3s
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+INTERVAL = 3
+
+function run()
+    local sock = ws_connect('ws://127.0.0.1:8080', {
+        read_timeout=INTERVAL,
+    })
+    if last_err() then return end
+
+    local last_ping = time_unix()
+    while true do
+        local now = time_unix()
+        local sleep = last_ping + INTERVAL - now
+
+        if sleep <= 0 then
+            ws_send_text(sock, sn0int_time() .. ' ping\n')
+            last_ping = now
+            sleep = INTERVAL
+        end
+
+        ws_options(sock, {
+            read_timeout=sleep,
+        })
+        if last_err() then return end
+
+        local buf = ws_recv_text(sock)
+        if last_err() then return end
+        info(buf)
+    end
+end

modules/harness/ws.lua

@@ -0,0 +1,23 @@
+-- Description: Create an echo websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    local target = 'ws://echo.websocket.org'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('sending')
+    ws_send_text(sock, 'ohai wurld')
+    if last_err() then return end
+
+    info('recieving')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'ohai wurld' then
+        return 'echo failed, got: ' .. msg
+    end
+end

modules/harness/wss.lua

@@ -0,0 +1,30 @@
+-- Description: Create an encrypted websocket connection
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- local target = 'ws://echo.websocket.org' -- doesn't support proper ciphers
+    local target = 'wss://rocket.events.ccc.de/sockjs/258/whi0yr1y/websocket'
+
+    info('connecting to ' .. target)
+    local sock = ws_connect(target, {})
+    if last_err() then return end
+
+    info('recieving 1/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'o' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('recieving 2/2')
+    local msg = ws_recv_text(sock)
+    if last_err() then return end
+
+    if msg ~= 'a["{\\"server_id\\":\\"0\\"}"]' then
+        return 'recieve failed, got ' .. msg
+    end
+
+    info('handshake succeeded')
+end

sn0int-common/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "sn0int-common"
-version = "0.9.0"
-description = "Common code for sn0int"
+version = "0.10.0"
+description = "sn0int - common code"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"

sn0int-registry/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/kpcyrd/sn0int"
 edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.9.0", path="sn0int-common" }
+sn0int-common = { version="0.10.0", path="../sn0int-common" }
 rocket = { version = "0.4", default-features=false }
 #rocket_failure = { path = "../../rocket_failure" }
 rocket_failure = { version = "0.2" }

sn0int-registry/Dockerfile

@@ -2,16 +2,16 @@ FROM rust:buster
 RUN apt-get update -q && apt-get install -yq llvm libclang-dev \
     && rm -rf /var/lib/apt/lists/*
 RUN rustup install nightly
-WORKDIR /usr/src/sn0int-registry
+WORKDIR /usr/src/sn0int
 COPY . .
-RUN cargo +nightly build --release --verbose
+RUN cd sn0int-registry && cargo +nightly build --release --verbose
 RUN strip target/release/sn0int-registry
 
 FROM debian:buster
 RUN apt-get update -q && apt-get install -yq libcurl4 libpq5 \
     && rm -rf /var/lib/apt/lists/*
-COPY --from=0 /usr/src/sn0int-registry/target/release/sn0int-registry /usr/local/bin/sn0int-registry
-COPY templates /templates
+COPY --from=0 /usr/src/sn0int/target/release/sn0int-registry /usr/local/bin/sn0int-registry
+COPY sn0int-registry/templates /templates
 ENV ROCKET_ENV=prod \
     ROCKET_ADDRESS=0.0.0.0 \
     ROCKET_PORT=8000

sn0int-std/Cargo.toml

@@ -0,0 +1,54 @@
+[package]
+name = "sn0int-std"
+version = "0.17.1"
+description = "sn0int - stdlib"
+authors = ["kpcyrd <git@rxv.cc>"]
+repository = "https://github.com/kpcyrd/sn0int"
+license = "GPL-3.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+log = "0.4"
+failure = "0.1"
+hlua-badtouch = "0.4"
+tokio = "0.1.14"
+rand = "0.7"
+regex = "1.0"
+rustls = { version="0.16", features=["dangerous_configuration"] }
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"
+serde_urlencoded = "0.6"
+webpki = "0.21"
+webpki-roots = "0.19.0"
+ct-logs = "0.6"
+chrootable-https = "0.14"
+http = "0.2"
+bufstream = "0.1.4"
+pem = "0.7"
+url = "2.0"
+tungstenite = { version = "0.10.1", default-features = false }
+kuchiki = "0.8.0"
+maxminddb = "0.13"
+# x509-parser 0.6.1 is broken
+x509-parser = "0.5.1"
+der-parser = "2.0"
+publicsuffix = { version="1.5", default-features=false }
+xml-rs = "0.8"
+geo = "0.12"
+bytes = "0.4"
+base64 = "0.11"
+chrono = { version = "0.4", features = ["serde"] }
+
+image = "0.23.0"
+kamadak-exif = "0.5.1"
+
+bs58 = "0.3"
+digest = "0.8.0"
+blake2 = "0.8.0"
+
+[dev-dependencies]
+env_logger = "0.7"
+maplit = "1.0.1"

sn0int-std/src/blobs.rs

@@ -0,0 +1,118 @@
+use bytes::Bytes;
+use blake2::VarBlake2b;
+use digest::{Input, VariableOutput};
+use serde::ser::{Serialize, Serializer};
+use serde::de::{self, Deserialize, Deserializer};
+use std::result;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Blob {
+    pub id: String,
+    pub bytes: Bytes,
+}
+
+impl Blob {
+    pub fn create(bytes: Bytes) -> Blob {
+        let id = Self::hash(&bytes);
+        Blob {
+            id,
+            bytes,
+        }
+    }
+
+    pub fn hash(bytes: &[u8]) -> String {
+        let mut h = VarBlake2b::new(32).unwrap();
+        h.input(bytes);
+        Self::encode_hash(&h.vec_result())
+    }
+
+    #[inline]
+    fn encode_hash(bytes: &[u8]) -> String {
+        let x = bs58::encode(bytes).into_string();
+        format!("{:0<44}", x)
+    }
+}
+
+impl Serialize for Blob {
+    #[inline]
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let s = base64::encode(&self.bytes);
+        serializer.serialize_str(&s)
+    }
+}
+
+impl<'de> Deserialize<'de> for Blob {
+    #[inline]
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let bytes = base64::decode(&s)
+            .map_err(de::Error::custom)?;
+        Ok(Blob::create(Bytes::from(bytes)))
+    }
+}
+
+pub trait BlobState {
+    fn register_blob(&self, blob: Blob) -> String;
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json;
+
+    #[inline]
+    fn blob() -> (Bytes, Blob) {
+        let bytes = Bytes::from(&b"asdf"[..]);
+        (bytes.clone(), Blob::create(bytes))
+    }
+
+    #[test]
+    fn verify_create_blob() {
+        let (bytes, blob) = blob();
+        assert_eq!(blob, Blob {
+            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
+            bytes,
+        });
+    }
+
+    #[test]
+    fn test_serialize() {
+        let (_, blob) = blob();
+        let json = serde_json::to_string(&blob).unwrap();
+        assert_eq!(&json, "\"YXNkZg==\"");
+    }
+
+    #[test]
+    fn test_deserialize() {
+        let (_, blob1) = blob();
+        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
+        assert_eq!(blob1, blob2);
+    }
+
+    #[test]
+    fn test_hash_encoding() {
+        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
+    }
+
+    #[test]
+    fn test_hash_encoding_padding() {
+        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
+            .into_vec()
+            .unwrap();
+        let x = Blob::encode_hash(&x);
+        assert_eq!(x.len(), 44);
+        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
+    }
+}

sn0int-std/src/crt.rs

@@ -11,6 +11,7 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 #[derive(Debug, PartialEq)]
 pub enum AlternativeName {
     DnsName(String),
+    Email(String),
     IpAddr(IpAddr),
 }
 
@@ -37,6 +38,7 @@ pub fn san_value(o: DerObject) -> Result<AlternativeName> {
 
     match (o.class, o.tag, &o.content) {
         (2, BerTag::Integer, BerObjectContent::Unknown(BerTag::Integer, value)) => san_value_dns(value),
+        (2, BerTag::Boolean, BerObjectContent::Unknown(BerTag::Boolean, value)) => san_value_email(value),
         (2, BerTag::ObjDescriptor, BerObjectContent::Unknown(BerTag::ObjDescriptor, value)) => san_value_ipaddr(value),
         _ => bail!("Unexpected object: {:?}", o),
     }
@@ -49,6 +51,13 @@ pub fn san_value_dns(v: &[u8]) -> Result<AlternativeName> {
         .map_err(Error::from)
 }
 
+pub fn san_value_email(v: &[u8]) -> Result<AlternativeName> {
+    debug!("Reading as email: {:?}", v);
+    String::from_utf8(v.to_vec())
+        .map(AlternativeName::Email)
+        .map_err(Error::from)
+}
+
 pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
     debug!("Reading as ipaddr: {:?}", v);
     match v.len() {
@@ -68,7 +77,8 @@ pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
     pub valid_names: Vec<String>,
-    pub valid_ipaddrs: Vec<IpAddr>
+    pub valid_emails: Vec<String>,
+    pub valid_ipaddrs: Vec<IpAddr>,
 }
 
 impl Certificate {
@@ -103,6 +113,7 @@ impl Certificate {
         };
 
         let mut valid_names = HashSet::new();
+        let mut valid_emails = HashSet::new();
         let mut valid_ipaddrs = HashSet::new();
 
         for x in crt.tbs_certificate.subject.rdn_seq {
@@ -131,15 +142,18 @@ impl Certificate {
             for v in values {
                 match v {
                     AlternativeName::DnsName(v) => valid_names.insert(v),
+                    AlternativeName::Email(v) => valid_emails.insert(v),
                     AlternativeName::IpAddr(v) => valid_ipaddrs.insert(v),
                 };
             }
         }
 
         let valid_names = valid_names.into_iter().collect();
+        let valid_emails = valid_emails.into_iter().collect();
         let valid_ipaddrs = valid_ipaddrs.into_iter().collect();
         Ok(Certificate {
             valid_names,
+            valid_emails,
             valid_ipaddrs,
         })
     }
@@ -197,6 +211,7 @@ WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik=
         x.valid_names.sort();
         assert_eq!(x, Certificate {
             valid_names: vec!["github.com".into(), "www.github.com".into()],
+            valid_emails: vec![],
             valid_ipaddrs: vec![],
         });
     }
@@ -235,6 +250,7 @@ ZkZZmqNn2Q8=
                 "*.cloudflare-dns.com".into(),
                 "cloudflare-dns.com".into(),
             ],
+            valid_emails: vec![],
             valid_ipaddrs: vec![
                 "1.0.0.1".parse().unwrap(),
                 "1.1.1.1".parse().unwrap(),
@@ -351,6 +367,7 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
                 "witt-weiden.dam.staging.aboutyou.cloud".into(),
                 "www.aboutyou.de".into(),
             ],
+            valid_emails: vec![],
             valid_ipaddrs: vec![],
         });
     }
@@ -396,4 +413,50 @@ z/6Vy8Ga9kigYVsa8ZFMR+Ex
             .expect("Failed to process san value");
         assert_eq!(v, AlternativeName::IpAddr("1.1.1.1".parse().unwrap()));
     }
+
+    #[test]
+    fn test_san_email() {
+        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+MIIE5zCCA8+gAwIBAgIQBvsKfZ5AGSW3Vc8Ldto1hTANBgkqhkiG9w0BAQUFADBp
+MSQwIgYJKoZIhvcNAQkBFhVwa2lfYWRtaW5Ac3VuZ2FyZC5jb20xJjAkBgNVBAoT
+HVN1bkdhcmQgQXZhaWxhYmlsaXR5IFNlcnZpY2VzMRkwFwYDVQQDExBTQVMgUHVi
+bGljIENBIHYxMB4XDTEwMDkwMjE2MzY0OVoXDTExMTAwMTE2MzEwMFowgbQxCzAJ
+BgNVBAYTAlVTMQswCQYDVQQIEwJPUjERMA8GA1UEBxMIUG9ydGxhbmQxEzARBgoJ
+kiaJk/IsZAEZFgNjb20xHDAaBgoJkiaJk/IsZAEZFgxqaXZlc29mdHdhcmUxHDAa
+BgNVBAoTE0ppdmUgU29mdHdhcmUsIEluYy4xEDAOBgNVBAsTB0hvc3RpbmcxIjAg
+BgNVBAMTGSouaG9zdGVkLmppdmVzb2Z0d2FyZS5jb20wggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQC0oornTIyL5YjZMpNwy+V2YJbLqaLrbPrbWFCsNJDx
+dnubjfR71aW+YYlUZF8zoq4jFetkblCehyvPEb5tD/l3/WZhiXYOziPDrsEVCngF
+3/b0H3Dyk6mNWBZcNpJkdpOx1YB6Zer8eKzFOr7Qj3aevOR/bEe2NARJIaO0Rjwe
+YIWY0arKRm6z4nJD8fYAvFV6wRWmHsZO9ci7hiGeW3YL6jQYJqLeuwXm64l0jptb
+Qg8r8c1V5BXETlvQJL34gUozEl9jDpzR7KoXtErhlU2ytl9Wg+fOxYuWgx8vER0/
+7Hqc/qD5e7B+NtwgfEio7SvNGA/HhjNxW2Wbrx4qooJRAgMBAAGjggE9MIIBOTAO
+BgNVHQ8BAf8EBAMCA6gwEQYJYIZIAYb4QgEBBAQDAgbAMCIGA1UdEQQbMBmBF3N1
+YmplY3RuYW1lQGV4YW1wbGUuY29tMB8GA1UdIwQYMBaAFDhBxvKFgYP96+IaNpI7
+JmEWgRESMFQGA1UdIARNMEswSQYJKoZIhvcNBQYBMDwwOgYIKwYBBQUHAgEWLmh0
+dHBzOi8vY2VydGlmaWNhdGUuc3VuZ2FyZC5jb20vU0FTX0NBX0NQUy5wZGYwRQYD
+VR0fBD4wPDA6oDigNoY0aHR0cHM6Ly9jZXJ0aWZpY2F0ZS5zdW5nYXJkLmNvbS9T
+QVNfUHVibGljX0NBX3YxLmNybDATBgNVHSUEDDAKBggrBgEFBQcDATAdBgNVHQ4E
+FgQUhJ99py6oeCYBzcePPjhOxHVDBvwwDQYJKoZIhvcNAQEFBQADggEBAF/DAJgX
+f50x8t8Im96AUn4DqC+T0QZIYHihpj2uCwWDbdp5efppqTrk6FrpFOzQy0TRkstb
+Q3zKSgduedQiwii9qh88O1h2gbSTqfi55ApOIGoCiiRCqio2p4tbKyqPV3Q0eyYw
+K4f9GAOcawvNsI//mx99ol/ZGEamydeL9G0qiKrhqSxd2TGmFaIVJdu9fh59hos4
+6t9c4FVyYygdsIeGHkHjpB2bKjZhnJpKRh9dGWctcjdHMITBBqgiRH9OZa/w6SPE
+UT+11L6q7MSIXSIMV8kJSUUYE92P7bnAqViTIuu/hHnfmIhiy6t7AuT2QHEhqDab
+EF4l5MwdUqs8FvM=
+-----END CERTIFICATE-----
+"#).expect("Failed to parse cert");
+        x.valid_names.sort();
+        x.valid_emails.sort();
+        x.valid_ipaddrs.sort();
+        assert_eq!(x, Certificate {
+            valid_names: vec![
+                "*.hosted.jivesoftware.com".into(),
+            ],
+            valid_emails: vec![
+                "subjectname@example.com".into(),
+            ],
+            valid_ipaddrs: vec![],
+        });
+    }
 }

sn0int-std/src/engine/mod.rs

@@ -0,0 +1 @@
+pub mod structs;

sn0int-std/src/errors.rs

@@ -0,0 +1,3 @@
+pub use log::{trace, debug, info, warn, error};
+pub use failure::{Error, ResultExt, format_err, bail};
+pub type Result<T> = ::std::result::Result<T, Error>;

sn0int-std/src/geo.rs

@@ -0,0 +1,111 @@
+use crate::errors::*;
+
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use geo::{LineString, Polygon, Coordinate};
+use geo::prelude::*;
+
+
+#[derive(Debug, Deserialize)]
+pub struct Point {
+    lon: f64,
+    lat: f64,
+}
+
+impl Point {
+    pub fn try_from(x: AnyLuaValue) -> Result<Point> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub fn polygon_contains(ring: &[Point], p: &Point) -> bool {
+    let ring = ring.iter()
+        .map(|p| Coordinate { x: p.lon, y: p.lat })
+        .collect::<Vec<_>>();
+
+    let polygon = Polygon::new(LineString::from(ring), vec![]);
+    let point = geo::Point::new(p.lon, p.lat);
+
+    polygon.contains(&point)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn hamburg_polygon() -> &'static [Point] {
+        &[
+            Point { lat: 53.63975308945899,  lon: 9.764785766601562  },
+            Point { lat: 53.59494998253459,  lon: 9.827270507812     },
+            Point { lat: 53.663153974456456, lon: 9.9151611328125    },
+            Point { lat: 53.65582987649682,  lon: 9.976272583007812  },
+            Point { lat: 53.68613523817129,  lon: 9.992752075195312  },
+            Point { lat: 53.68674518938816,  lon: 10.051460266113281 },
+            Point { lat: 53.72495117617815,  lon: 10.075492858886719 },
+            Point { lat: 53.71946627930625,  lon: 10.118408203125    },
+            Point { lat: 53.743635083157756, lon: 10.164413452148438 },
+            Point { lat: 53.73104466704585,  lon: 10.202865600585938 },
+            Point { lat: 53.676781546441546, lon: 10.16304016113281  },
+            Point { lat: 53.632832079199474, lon: 10.235824584960938 },
+            Point { lat: 53.608803292930894, lon: 10.2008056640625   },
+            Point { lat: 53.578646152866504, lon: 10.208358764648438 },
+            Point { lat: 53.57212285981298,  lon: 10.163726806640625 },
+            Point { lat: 53.52071674896369,  lon: 10.18707275390625  },
+            Point { lat: 53.52643162253097,  lon: 10.224151611328125 },
+            Point { lat: 53.44062753992289,  lon: 10.347747802734375 },
+            Point { lat: 53.38824275010831,  lon: 10.248870849609375 },
+            Point { lat: 53.38824275010831,  lon: 10.15960693359375  },
+            Point { lat: 53.44635321212876,  lon: 10.064849853515625 },
+            Point { lat: 53.40595029739904,  lon: 9.985198974609375  },
+            Point { lat: 53.42385506057106,  lon: 9.951210021972656  },
+            Point { lat: 53.41843327091211,  lon: 9.944171905517578  },
+            Point { lat: 53.41812635648326,  lon: 9.927349090576172  },
+            Point { lat: 53.412294561442884, lon: 9.917736053466797  },
+            Point { lat: 53.41464783813818,  lon: 9.901256561279297  },
+            Point { lat: 53.443490472483326, lon: 9.912586212158201  },
+            Point { lat: 53.45177144115704,  lon: 9.897651672363281  },
+            Point { lat: 53.43633277935392,  lon: 9.866924285888672  },
+            Point { lat: 53.427639673754776, lon: 9.866409301757812  },
+            Point { lat: 53.427639673754776, lon: 9.858856201171875  },
+            Point { lat: 53.46710230573499,  lon: 9.795513153076172  },
+            Point { lat: 53.49039461941655,  lon: 9.795341491699219  },
+            Point { lat: 53.49029248806277,  lon: 9.77903366088867   },
+            Point { lat: 53.49856433088649,  lon: 9.780235290527344  },
+            Point { lat: 53.5078554643033,   lon: 9.758434295654297  },
+            Point { lat: 53.545407634092975, lon: 9.759807586669922  },
+            Point { lat: 53.568147234570084, lon: 9.633293151855469  },
+            Point { lat: 53.58802162343514,  lon: 9.655780792236328  },
+            Point { lat: 53.568351121879815, lon: 9.727706909179688  },
+            Point { lat: 53.60921067445695,  lon: 9.737663269042969  },
+        ]
+    }
+
+    #[test]
+    fn test_polygon_hamburg_contains_hamburg() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 53.551085,
+            lon: 9.993682,
+        });
+        assert!(contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_berlin() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 52.52437,
+            lon: 13.41053,
+        });
+        assert!(!contains);
+    }
+
+    #[test]
+    fn test_polygon_hamburg_not_contains_ny() {
+        let contains = polygon_contains(hamburg_polygon(), &Point {
+            lat: 40.726662,
+            lon: -74.036677,
+        });
+        assert!(!contains);
+    }
+}

sn0int-std/src/geoip/mod.rs

@@ -1,34 +1,25 @@
-use chrootable_https::Client;
-use crate::archive;
 use crate::errors::*;
 use crate::lazy::LazyInit;
-use crate::paths;
-use crate::worker;
+// use crate::paths;
 use maxminddb::{self, geoip2};
 use std::fmt;
 use std::fs::{self, File};
 use std::net::IpAddr;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::io::Read;
 use std::sync::Arc;
 
-pub static GEOIP_CITY_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz";
-pub static GEOIP_ASN_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz";
-
 pub mod models;
 use self::models::GeoLookup;
 use self::models::AsnLookup;
 
 
 pub trait Maxmind: Sized {
-    fn archive_filename() -> &'static str;
-
-    fn archive_url() -> &'static str;
+    fn filename() -> &'static str;
 
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self;
 
-    // TODO: refactor this to return Path
-    fn cache_path() -> Result<String> {
+    fn cache_path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         for path in &[
             // Archlinux
@@ -37,21 +28,17 @@ pub trait Maxmind: Sized {
             "/usr/local/share/examples/libmaxminddb/",
         ] {
             let path = Path::new(path);
-            let path = path.join(Self::archive_filename());
+            let path = path.join(Self::filename());
 
             if path.exists() {
-                let path = path.to_str()
-                    .ok_or_else(|| format_err!("Failed to decode path"))?;
-                return Ok(path.to_string());
+                return Ok(path);
             }
         }
 
         // use cache path
-        let path = paths::cache_dir()?
-            .join(Self::archive_filename());
-        let path = path.to_str()
-            .ok_or_else(|| format_err!("Failed to decode path"))?;
-        Ok(path.to_string())
+        let path = cache_dir
+            .join(Self::filename());
+        Ok(path)
     }
 
     fn from_buf(buf: Vec<u8>) -> Result<Self> {
@@ -60,37 +47,20 @@ pub trait Maxmind: Sized {
         Ok(Self::new(reader))
     }
 
-    fn open(path: &str) -> Result<Self> {
+    fn open(path: &Path) -> Result<Self> {
         let buf = fs::read(path)?;
         Self::from_buf(buf)
     }
 
-    fn open_reader() -> Result<MaxmindReader> {
-        let path = Self::cache_path()?;
-        MaxmindReader::open_path(path)
-    }
+    fn try_open_reader(cache_dir: &Path) -> Result<Option<MaxmindReader>> {
+        let path = Self::cache_path(cache_dir)?;
 
-    fn open_or_download() -> Result<Self> {
-        let path = Self::cache_path()?;
-
-        if File::open(&path).is_err() {
-            worker::spawn_fn(&format!("Downloading {:?}", Self::archive_filename()), || {
-                Self::download(&path, Self::archive_filename(), Self::archive_url())
-            }, false)?;
-        };
-
-        Self::open(&path)
-    }
-
-    fn download<P: AsRef<Path>>(path: P, filter: &str, url: &str) -> Result<()> {
-        debug!("Downloading {:?}...", url);
-        let client = Client::with_system_resolver()?;
-        let resp = client.get(url)
-            .wait_for_response()
-            .context("http request failed")?;
-        debug!("Downloaded {} bytes", resp.body.len());
-        archive::extract(&mut &resp.body[..], filter, path)?;
-        Ok(())
+        if path.exists() {
+            let db = MaxmindReader::open_path(path)?;
+            Ok(Some(db))
+        } else {
+            Ok(None)
+        }
     }
 }
 
@@ -141,15 +111,10 @@ impl fmt::Debug for GeoIP {
 
 impl Maxmind for GeoIP {
     #[inline]
-    fn archive_filename() -> &'static str {
+    fn filename() -> &'static str {
         "GeoLite2-City.mmdb"
     }
 
-    #[inline]
-    fn archive_url() -> &'static str {
-        GEOIP_CITY_URL
-    }
-
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         GeoIP {
@@ -178,15 +143,10 @@ impl fmt::Debug for AsnDB {
 
 impl Maxmind for AsnDB {
     #[inline]
-    fn archive_filename() -> &'static str {
+    fn filename() -> &'static str {
         "GeoLite2-ASN.mmdb"
     }
 
-    #[inline]
-    fn archive_url() -> &'static str {
-        GEOIP_ASN_URL
-    }
-
     #[inline]
     fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         AsnDB {
@@ -205,22 +165,29 @@ impl AsnDB {
 
 #[cfg(test)]
 mod tests {
+    // You need geoip setup on your system to run this
+    /*
     use super::*;
 
     #[test]
+    #[ignore]
     fn test_geoip_lookup() {
         let ip = "1.1.1.1".parse().unwrap();
-        let geoip = GeoIP::open_or_download().expect("Failed to load geoip");
+        let path = GeoIP::cache_path().unwrap();
+        let geoip = GeoIP::open(&path).unwrap();
         let lookup = geoip.lookup(ip).expect("GeoIP lookup failed");
         println!("{:#?}", lookup);
         assert_eq!(lookup.city, None);
     }
 
     #[test]
+    #[ignore]
     fn test_asn_lookup() {
         let ip = "1.1.1.1".parse().unwrap();
-        let asndb = AsnDB::open_or_download().expect("Failed to load asndb");
+        let path = AsnDB::cache_path().unwrap();
+        let asndb = AsnDB::open(&path).unwrap();
         let lookup = asndb.lookup(ip).expect("ASN lookup failed");
         println!("{:#?}", lookup);
     }
+    */
 }

sn0int-std/src/gfx/exif.rs

@@ -11,11 +11,12 @@ pub struct Location {
 }
 
 impl Location {
-    pub fn try_from(fields: &[exif::Field]) -> Result<Location> {
+    fn try_from_iter<'a, I: IntoIterator<Item=&'a exif::Field>>(iter: I) -> Result<Self> {
         let mut builder = LocationBuilder::default();
-        fields.iter()
-            .map(|f| builder.add_one(f))
-            .collect::<Result<()>>()?;
+        for f in iter {
+            debug!("Exif field: {:?}", f.display_value().to_string());
+            builder.add_one(f)?;
+        }
         builder.build()
     }
 }
@@ -60,12 +61,12 @@ impl LocationBuilder {
 }
 
 pub fn gps(img: &[u8]) -> Result<Option<Location>> {
-    let mut buf = io::BufReader::new(img);
-    let reader = exif::Reader::new(&mut buf)?;
+    let mut buf = io::Cursor::new(img);
+    let reader = exif::Reader::new()
+        .read_from_container(&mut buf)?;
     let fields = reader.fields();
-    debug!("Exif fields: {:?}", fields);
 
-    let location = Location::try_from(fields).ok();
+    let location = Location::try_from_iter(fields).ok();
     Ok(location)
 }
 
@@ -115,18 +116,18 @@ mod tests {
     fn verify_exif_location() {
         test_init();
 
-        let location = Location::try_from(&[
+        let location = Location::try_from_iter(&[
             exif::Field {
                 tag: exif::Tag::GPSLatitudeRef,
-                thumbnail: false,
-                value: exif::Value::Ascii(vec![&[b'N']]),
+                ifd_num: exif::In::PRIMARY,
+                value: exif::Value::Ascii(vec![vec![b'N']]),
             }, exif::Field {
                 tag: exif::Tag::GPSLongitudeRef,
-                thumbnail: false,
-                value: exif::Value::Ascii(vec![&[b'E']]),
+                ifd_num: exif::In::PRIMARY,
+                value: exif::Value::Ascii(vec![vec![b'E']]),
             }, exif::Field {
                 tag: exif::Tag::GPSLatitude,
-                thumbnail: false,
+                ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Rational(vec![exif::Rational {
                     num: 43,
                     denom: 1,
@@ -139,7 +140,7 @@ mod tests {
                 }]),
             }, exif::Field {
                 tag: exif::Tag::GPSLongitude,
-                thumbnail: false,
+                ifd_num: exif::In::PRIMARY,
                 value: exif::Value::Rational(vec![exif::Rational {
                     num: 11,
                     denom: 1,

sn0int-std/src/gfx/mod.rs

@@ -6,13 +6,13 @@ pub mod exif;
 
 #[derive(Debug)]
 pub enum ImageFormat {
-    PNG,
-    JPEG,
-    GIF,
-    WEBP,
-    TIFF,
-    BMP,
-    ICO,
+    Png,
+    Jpeg,
+    Gif,
+    WebP,
+    Tiff,
+    Bmp,
+    Ico,
 }
 
 impl ImageFormat {
@@ -20,28 +20,27 @@ impl ImageFormat {
         // https://www.iana.org/assignments/media-types/media-types.xhtml#image
         // /etc/nginx/mime.types
         match self {
-            ImageFormat::PNG => "image/png",
-            ImageFormat::JPEG => "image/jpeg",
-            ImageFormat::GIF => "image/gif",
-            ImageFormat::WEBP => "image/webp",
-            ImageFormat::TIFF => "image/tiff",
-            ImageFormat::BMP => "image/bmp",
-            ImageFormat::ICO => "image/vnd.microsoft.icon",
+            ImageFormat::Png => "image/png",
+            ImageFormat::Jpeg => "image/jpeg",
+            ImageFormat::Gif => "image/gif",
+            ImageFormat::WebP => "image/webp",
+            ImageFormat::Tiff => "image/tiff",
+            ImageFormat::Bmp => "image/bmp",
+            ImageFormat::Ico => "image/vnd.microsoft.icon",
         }
     }
 
     pub fn try_from(format: &image::ImageFormat) -> Result<ImageFormat> {
         use image::ImageFormat::*;
         match format {
-            PNG => Ok(ImageFormat::PNG),
-            JPEG => Ok(ImageFormat::JPEG),
-            GIF => Ok(ImageFormat::GIF),
-            WEBP => Ok(ImageFormat::WEBP),
-            TIFF => Ok(ImageFormat::TIFF),
-            BMP => Ok(ImageFormat::BMP),
-            ICO => Ok(ImageFormat::ICO),
-
-            HDR | PNM | TGA => bail!("Unsupported format: {:?}", format),
+            Png => Ok(ImageFormat::Png),
+            Jpeg => Ok(ImageFormat::Jpeg),
+            Gif => Ok(ImageFormat::Gif),
+            WebP => Ok(ImageFormat::WebP),
+            Tiff => Ok(ImageFormat::Tiff),
+            Bmp => Ok(ImageFormat::Bmp),
+            Ico => Ok(ImageFormat::Ico),
+            _ => bail!("Unsupported format: {:?}", format),
         }
     }
 }
@@ -49,13 +48,13 @@ impl ImageFormat {
 impl Into<image::ImageFormat> for ImageFormat {
     fn into(self) -> image::ImageFormat {
         match self {
-            ImageFormat::PNG => image::ImageFormat::PNG,
-            ImageFormat::JPEG => image::ImageFormat::JPEG,
-            ImageFormat::GIF => image::ImageFormat::GIF,
-            ImageFormat::WEBP => image::ImageFormat::WEBP,
-            ImageFormat::TIFF => image::ImageFormat::TIFF,
-            ImageFormat::BMP => image::ImageFormat::BMP,
-            ImageFormat::ICO => image::ImageFormat::ICO,
+            ImageFormat::Png => image::ImageFormat::Png,
+            ImageFormat::Jpeg => image::ImageFormat::Jpeg,
+            ImageFormat::Gif => image::ImageFormat::Gif,
+            ImageFormat::WebP => image::ImageFormat::WebP,
+            ImageFormat::Tiff => image::ImageFormat::Tiff,
+            ImageFormat::Bmp => image::ImageFormat::Bmp,
+            ImageFormat::Ico => image::ImageFormat::Ico,
         }
     }
 }
@@ -114,7 +113,7 @@ mod tests {
 
     #[test]
     fn verify_gfx_load_ico() {
-        let ico = fs::read("./sn0int-registry/assets/favicon.ico").expect("fs::read");
+        let ico = fs::read("../sn0int-registry/assets/favicon.ico").expect("fs::read");
         let img = load(&ico).expect("gfx::load");
 
         assert_eq!("image/vnd.microsoft.icon", img.mime());

sn0int-std/src/html.rs

@@ -1,4 +1,4 @@
-use crate::errors::Result;
+use crate::errors::*;
 
 use kuchiki;
 use kuchiki::traits::TendrilSink;
@@ -102,6 +102,7 @@ pub fn html_form(html: &str) -> Result<HashMap<String, String>> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn test_html_select() {

sn0int-std/src/lib.rs

@@ -0,0 +1,24 @@
+#[macro_use] extern crate serde_derive;
+use hlua_badtouch as hlua;
+
+pub mod blobs;
+pub mod crt;
+mod errors;
+pub mod engine;
+pub mod geo;
+pub mod geoip;
+pub mod gfx;
+pub mod html;
+pub mod json;
+pub mod lazy;
+pub mod psl;
+pub mod ratelimits;
+pub mod sockets;
+pub mod web;
+pub mod websockets;
+pub mod xml;
+
+#[cfg(test)]
+fn test_init() {
+    let _ = env_logger::builder().is_test(true).try_init();
+}

sn0int-std/src/psl.rs

@@ -4,9 +4,7 @@ use chrootable_https::Client;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use crate::paths;
 use crate::lazy::LazyInit;
-use crate::worker;
 
 
 #[derive(Debug, PartialEq)]
@@ -23,22 +21,24 @@ pub enum PslReader {
 }
 
 impl PslReader {
-    pub fn open_or_download() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open_or_download<F>(cache_dir: &Path, indicator: F) -> Result<PslReader>
+        where
+            F: Fn(Box<dyn Fn() -> Result<PslReader>>) -> Result<PslReader>
+    {
+        let path = Self::path(cache_dir)?;
         let reader = match Self::open_from(&path) {
             Ok(r) => r,
-            Err(_) => worker::spawn_fn("Downloading public suffix list", || {
+            Err(_) => indicator(Box::new(move || {
                 PslReader::download(&path, publicsuffix::LIST_URL)?;
                 Self::open_from(&path)
-            }, false)?,
+            }))?,
         };
         Ok(reader)
     }
 
-    pub fn open() -> Result<PslReader> {
-        let path = Self::path()?;
+    pub fn open(cache_dir: &Path) -> Result<PslReader> {
+        let path = Self::path(cache_dir)?;
         Self::open_from(&path)
-
     }
 
     pub fn open_from(path: &Path) -> Result<PslReader> {
@@ -46,7 +46,7 @@ impl PslReader {
         Ok(PslReader::Reader(file))
     }
 
-    pub fn path() -> Result<PathBuf> {
+    pub fn path(cache_dir: &Path) -> Result<PathBuf> {
         // use system path if exists
         let path = Path::new("/usr/share/publicsuffix/public_suffix_list.dat");
         if path.exists() {
@@ -54,7 +54,7 @@ impl PslReader {
         }
 
         // else, use local cache
-        let path = paths::cache_dir()?
+        let path = cache_dir
             .join("public_suffix_list.dat");
         Ok(path)
     }

sn0int-std/src/ratelimits.rs

@@ -0,0 +1,70 @@
+use chrono::prelude::*;
+use std::collections::HashMap;
+use std::result;
+use std::sync::mpsc;
+use std::time::Duration;
+
+pub type RatelimitSender = mpsc::Sender<result::Result<RatelimitResponse, String>>;
+
+pub struct Ratelimiter {
+    buckets: HashMap<String, Bucket>,
+}
+
+impl Ratelimiter {
+    pub fn new() -> Ratelimiter {
+        Ratelimiter {
+            buckets: HashMap::new(),
+        }
+    }
+
+    pub fn pass(&mut self, tx: RatelimitSender, key: &str, passes: u32, time: u32) {
+        let bucket = if let Some(bucket) = self.buckets.get_mut(key) {
+            bucket
+        } else {
+            let bucket = Bucket::new();
+            self.buckets.insert(key.to_string(), bucket);
+            self.buckets.get_mut(key).unwrap()
+        };
+        let reply = bucket.pass(passes as usize, time);
+        tx.send(Ok(reply)).unwrap();
+    }
+}
+
+struct Bucket {
+    passes: Vec<DateTime<Utc>>,
+}
+
+impl Bucket {
+    pub fn new() -> Bucket {
+        Bucket {
+            passes: Vec::new(),
+        }
+    }
+
+    pub fn pass(&mut self, passes: usize, time: u32) -> RatelimitResponse {
+        let now = Utc::now();
+        let time = chrono::Duration::milliseconds(time as i64);
+        let retain = now - time;
+        self.passes.retain(|x| *x >= retain);
+
+        if self.passes.len() >= passes {
+            if let Some(min) = self.passes.iter().min() {
+                let delay = time - (now - *min);
+                RatelimitResponse::Retry(delay.to_std().unwrap())
+            } else {
+                // This should never happen unless passes is zero
+                RatelimitResponse::Retry(Duration::from_millis(100))
+            }
+        } else {
+            let now = Utc::now();
+            self.passes.push(now);
+            RatelimitResponse::Pass
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum RatelimitResponse {
+    Retry(Duration),
+    Pass,
+}

sn0int-std/src/sockets/mod.rs

@@ -16,6 +16,7 @@ use std::io::BufRead;
 use std::net::SocketAddr;
 use std::net::TcpStream;
 use std::net::{IpAddr, Ipv4Addr};
+use std::time::Duration;
 
 mod tls;
 pub use self::tls::TlsData;
@@ -39,14 +40,21 @@ fn unwrap_socket(socket: tokio::net::TcpStream) -> Result<TcpStream> {
 pub struct SocketOptions {
     #[serde(default)]
     pub tls: bool,
-    sni_value: Option<String>,
+    pub sni_value: Option<String>,
     #[serde(default)]
-    disable_tls_verify: bool,
+    pub disable_tls_verify: bool,
     pub proxy: Option<SocketAddr>,
 
     // TODO: enable_sni (default to true)
+    // TODO: sni_name
     // TODO: cacert
-    // TODO: timeout
+
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
 }
 
 impl SocketOptions {
@@ -57,17 +65,97 @@ impl SocketOptions {
     }
 }
 
-#[derive(Debug)]
-pub struct Socket {
-    stream: BufStream<Stream>,
-    newline: String,
+impl SocketOptions {
+    pub fn apply(&self, stream: &Stream) -> Result<()> {
+        let socket = match stream {
+            Stream::Tcp(s) => s,
+            Stream::Tls(s) => s.get_ref(),
+        };
+        self.apply_tcp(socket)
+    }
+
+    pub fn apply_tcp(&self, socket: &TcpStream) -> Result<()> {
+        let read_timeout = self.read_timeout;
+        if read_timeout > 0 {
+            socket.set_read_timeout(Some(Duration::from_secs(read_timeout)))?;
+        }
+
+        let write_timeout = self.write_timeout;
+        if write_timeout > 0 {
+            socket.set_write_timeout(Some(Duration::from_secs(write_timeout)))?;
+        }
+
+        Ok(())
+    }
 }
 
-enum Stream {
+pub enum Stream {
     Tcp(TcpStream),
     Tls(rustls::StreamOwned<rustls::ClientSession, TcpStream>),
 }
 
+impl Stream {
+    pub fn connect_stream<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+        let addrs = match host.parse::<IpAddr>() {
+            Ok(addr) => vec![addr],
+            Err(_) => resolver.resolve(host, RecordType::A)
+                .wait_for_response()?
+                .success()?,
+        };
+
+        let mut errors = Vec::new();
+
+        for addr in addrs {
+            match Stream::connect_addr(host, (addr, port).into(), &options) {
+                Ok(socket) => {
+                    return Ok(socket);
+                },
+                Err(err) => errors.push((addr, err)),
+            }
+        }
+
+        if errors.is_empty() {
+            bail!("no dns records found");
+        } else {
+            bail!("couldn't connect: {:?}", errors);
+        }
+    }
+
+    fn connect_addr(host: &str, addr: SocketAddr, options: &SocketOptions) -> Result<Stream> {
+        debug!("connecting to {}", addr);
+
+        let connect_timeout = options.connect_timeout;
+        let socket = if connect_timeout > 0 {
+            TcpStream::connect_timeout(&addr, Duration::from_secs(connect_timeout))?
+        } else {
+            TcpStream::connect(&addr)?
+        };
+        debug!("successfully connected to {:?}", addr);
+
+        options.apply_tcp(&socket)?;
+
+        tls::wrap_if_enabled(socket, host, options)
+    }
+
+    pub fn connect_socks5_stream(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Stream> {
+        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
+
+        let addr = match host.parse::<Ipv4Addr>() {
+            Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
+            _ => ProxyDest::Domain(host.to_string()),
+        };
+
+        let fut = socks5::connect(proxy, addr, port);
+
+        let mut rt = Runtime::new()?;
+        let socket = rt.block_on(fut)?;
+
+        let socket = unwrap_socket(socket)?;
+
+        tls::wrap_if_enabled(socket, host, options)
+    }
+}
+
 impl fmt::Debug for Stream {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
         match self {
@@ -102,6 +190,12 @@ impl Write for Stream {
     }
 }
 
+#[derive(Debug)]
+pub struct Socket {
+    stream: BufStream<Stream>,
+    newline: String,
+}
+
 impl Socket {
     fn new(stream: Stream) -> Socket {
         let stream = BufStream::new(stream);
@@ -112,60 +206,31 @@ impl Socket {
     }
 
     pub fn connect<R: DnsResolver>(resolver: &R, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
-        let addrs = match host.parse::<IpAddr>() {
-            Ok(addr) => vec![addr],
-            Err(_) => resolver.resolve(host, RecordType::A)
-                .wait_for_response()?
-                .success()?,
-        };
-
-        let mut errors = Vec::new();
-
-        for addr in addrs {
-            debug!("connecting to {}:{}", addr, port);
-            match TcpStream::connect((addr, port)) {
-                Ok(socket) => {
-                    debug!("successfully connected to {:?}", addr);
-                    return tls::wrap_if_enabled(socket, host, options);
-                },
-                Err(err) => errors.push((addr, err)),
-            }
-        }
-
-        if errors.is_empty() {
-            bail!("no dns records found");
-        } else {
-            bail!("couldn't connect: {:?}", errors);
-        }
+        let stream = Stream::connect_stream(resolver, host, port, options)?;
+        Ok(Socket::new(stream))
     }
 
     pub fn connect_socks5(proxy: &SocketAddr, host: &str, port: u16, options: &SocketOptions) -> Result<Socket> {
-        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
-
-        let addr = match host.parse::<Ipv4Addr>() {
-            Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
-            _ => ProxyDest::Domain(host.to_string()),
-        };
-
-        let fut = socks5::connect(proxy, addr, port);
-
-        let mut rt = Runtime::new()?;
-        let socket = rt.block_on(fut)?;
-
-        let socket = unwrap_socket(socket)?;
-
-        tls::wrap_if_enabled(socket, host, options)
+        let stream = Stream::connect_socks5_stream(proxy, host, port, options)?;
+        Ok(Socket::new(stream))
     }
 
     pub fn upgrade_to_tls(self, options: &SocketOptions) -> Result<(Socket, TlsData)> {
         let stream = self.stream.into_inner()?;
 
-        match stream {
-            Stream::Tcp(stream) => tls::wrap(stream, "", options),
-            _ => bail!("Only tcp streams can be upgraded"),
+        if let Stream::Tcp(stream) = stream {
+            let (stream, tls) = tls::wrap(stream, "", options)?;
+            let socket = Socket::new(stream);
+            Ok((socket, tls))
+        } else {
+            bail!("Only tcp streams can be upgraded")
         }
     }
 
+    pub fn options(&self, options: &SocketOptions) -> Result<()> {
+        options.apply(self.stream.get_ref())
+    }
+
     pub fn send(&mut self, data: &[u8]) -> Result<()> {
         match str::from_utf8(&data) {
             Ok(data) => debug!("send: {:?}", data),
@@ -178,7 +243,12 @@ impl Socket {
 
     pub fn recv(&mut self) -> Result<Vec<u8>> {
         let mut buf = [0; 4096];
-        let n = self.stream.read(&mut buf)?;
+        let n = match self.stream.read(&mut buf) {
+            Ok(n) if n == 0 => bail!("Connection closed"),
+            Ok(n) => n,
+            Err(err) if err.kind() == io::ErrorKind::WouldBlock => 0,
+            Err(err) => return Err(err.into()),
+        };
         let data = buf[..n].to_vec();
         match str::from_utf8(&data) {
             Ok(data) => debug!("recv: {:?}", data),

sn0int-std/src/sockets/tls.rs

@@ -2,14 +2,14 @@ use crate::errors::*;
 
 use crate::hlua::AnyLuaValue;
 use crate::json::LuaJsonValue;
-use rustls::{self, ClientConfig, Session, ClientSession, RootCertStore};
+use rustls::{self, ClientConfig, Session, ClientSession};
 
 use std::str;
 use std::result;
 use std::sync::Arc;
 use std::net::TcpStream;
 
-use super::{Socket, Stream, SocketOptions};
+use super::{Stream, SocketOptions};
 
 
 #[derive(Debug, Serialize)]
@@ -26,22 +26,22 @@ impl TlsData {
     }
 }
 
-pub fn wrap_if_enabled(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<Socket> {
+pub fn wrap_if_enabled(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<Stream> {
     if !options.tls {
         let stream = Stream::Tcp(stream);
-        return Ok(Socket::new(stream));
+        Ok(stream)
     } else {
         let (socket, _) = wrap(stream, host, options)?;
         Ok(socket)
     }
 }
 
-pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(Socket, TlsData)> {
-    let mut anchors = RootCertStore::empty();
-    anchors.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
-
+pub fn wrap(stream: TcpStream, host: &str, options: &SocketOptions) -> Result<(Stream, TlsData)> {
     let mut config = ClientConfig::new();
-    config.root_store = anchors;
+    config
+        .root_store
+        .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
+    config.ct_logs = Some(&ct_logs::LOGS);
 
     if options.disable_tls_verify {
         info!("tls verification has been disabled");
@@ -73,14 +73,16 @@ fn get_dns_name(config: &mut ClientConfig, host: &str) -> webpki::DNSName {
     }
 }
 
-fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Socket, TlsData)> {
+fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Stream, TlsData)> {
     info!("starting tls handshake");
     if session.is_handshaking() {
-        session.complete_io(&mut stream)?;
+        session.complete_io(&mut stream)
+            .context("Failed to read reply to tls client hello")?;
     }
 
     if session.wants_write() {
-        session.complete_io(&mut stream)?;
+        session.complete_io(&mut stream)
+            .context("wants_write->complete_io failed")?;
     }
 
     let mut tls = TlsData {
@@ -106,7 +108,7 @@ fn setup(mut stream: TcpStream, mut session: ClientSession) -> Result<(Socket, T
     info!("successfully established tls connection");
     let stream = rustls::StreamOwned::new(session, stream);
     let stream = Stream::Tls(stream);
-    Ok((Socket::new(stream), tls))
+    Ok((stream, tls))
 }
 
 pub struct NoCertificateVerification {}

sn0int-std/src/web.rs

@@ -1,17 +1,17 @@
 pub use chrootable_https::{Client, HttpClient, Resolver, Response};
 
-use crate::blobs::Blob;
+use crate::blobs::{Blob, BlobState};
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
 use std::ops::Deref;
 use crate::errors::*;
 use crate::hlua::AnyLuaValue;
-use crate::engine::ctx::State;
 use serde_json;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
 use std::fmt;
 use std::net::SocketAddr;
+use std::sync::Arc;
 use serde::Serialize;
 use crate::engine::structs::LuaMap;
 use crate::json::LuaJsonValue;
@@ -37,6 +37,12 @@ pub fn url_set_qs<S: Serialize + fmt::Debug>(url: Uri, query: &S) -> Result<Uri>
         .map_err(Error::from)
 }
 
+pub trait WebState {
+    fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>>;
+
+    fn register_in_jar(&self, session: &str, key: String, value: String);
+}
+
 #[derive(Debug)]
 pub struct HttpSession {
     id: String,
@@ -134,7 +140,7 @@ impl HttpRequest {
         request
     }
 
-    pub fn send(&self, state: &dyn State) -> Result<Response> {
+    pub fn send(&self, state: &dyn WebState) -> Result<Response> {
         let mut url = self.url.parse::<Uri>()?;
 
         // set query string
@@ -228,7 +234,9 @@ impl HttpRequest {
         Ok(res)
     }
 
-    pub fn response_to_lua(&self, state: &dyn State, res: Response) -> Result<LuaMap> {
+    pub fn response_to_lua<S>(&self, state: &S, res: Response) -> Result<LuaMap>
+        where S: WebState + BlobState
+    {
         // map result to LuaMap
         let mut resp = LuaMap::new();
         resp.insert_num("status", f64::from(res.status));
@@ -239,6 +247,10 @@ impl HttpRequest {
         }
         resp.insert("headers", headers);
 
+        if let Some(ipaddr) = res.ipaddr {
+            resp.insert_str("ipaddr", ipaddr.to_string());
+        }
+
         if self.into_blob {
             let blob = Blob::create(res.body);
             let id = state.register_blob(blob);
@@ -250,7 +262,7 @@ impl HttpRequest {
         Ok(resp)
     }
 
-    fn register_cookies_on_state(session: &str, state: &dyn State, cookie: &str) {
+    fn register_cookies_on_state(session: &str, state: &dyn WebState, cookie: &str) {
         let mut key = String::new();
         let mut value = String::new();
         let mut in_key = true;

sn0int-std/src/websockets.rs

@@ -0,0 +1,150 @@
+use chrootable_https::DnsResolver;
+use crate::errors::*;
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use crate::sockets::{Stream, SocketOptions};
+use http::Request;
+use std::collections::HashMap;
+use std::net::SocketAddr;
+use std::io;
+use tungstenite::protocol::{self, Message};
+use url::Url;
+
+#[derive(Debug, Default, Deserialize)]
+pub struct WebSocketOptions {
+    pub headers: Option<HashMap<String, String>>,
+    pub proxy: Option<SocketAddr>,
+
+    #[serde(default)]
+    pub connect_timeout: u64,
+    #[serde(default)]
+    pub read_timeout: u64,
+    #[serde(default)]
+    pub write_timeout: u64,
+}
+
+impl WebSocketOptions {
+    pub fn try_from(x: AnyLuaValue) -> Result<WebSocketOptions> {
+        let x = LuaJsonValue::from(x);
+        let x = serde_json::from_value(x.into())?;
+        Ok(x)
+    }
+}
+
+pub enum Event {
+    Text(String),
+    Binary(Vec<u8>),
+    Close,
+    Timeout,
+}
+
+pub struct WebSocket {
+    sock: protocol::WebSocket<Stream>,
+}
+
+impl WebSocket {
+    pub fn negotiate(stream: Stream, url: Url, headers: Option<&HashMap<String, String>>) -> Result<WebSocket> {
+        let mut req = Request::get(url.to_string()); // TODO: don't re-parse here
+
+        if let Some(headers) = headers {
+            for (k, v) in headers {
+                req = req.header(k, v);
+            }
+        }
+
+        let req = req.body(()).unwrap();
+
+        let (sock, _resp) = tungstenite::client::client(req, stream)?;
+        Ok(WebSocket {
+            sock,
+        })
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, url: Url, options: &WebSocketOptions) -> Result<WebSocket> {
+        let tls = match url.scheme() {
+            "ws" => false,
+            "wss" => true,
+            _ => bail!("Invalid websocket protocol"),
+        };
+
+        let host = url.host_str()
+            .ok_or_else(|| format_err!("Missing host in url"))?;
+
+        let port = match (url.port(), tls) {
+            (Some(port), _) => port,
+            (None, true) => 443,
+            (None, false) => 80,
+        };
+
+        let stream = Stream::connect_stream(resolver, host, port, &SocketOptions {
+            tls,
+            sni_value: None,
+            disable_tls_verify: false,
+            proxy: options.proxy,
+
+            connect_timeout: options.connect_timeout,
+            read_timeout: options.read_timeout,
+            write_timeout: options.write_timeout,
+        })?;
+        Self::negotiate(stream, url, options.headers.as_ref())
+    }
+
+    pub fn options(&self, options: &WebSocketOptions) -> Result<()> {
+        let mut o = SocketOptions::default();
+        o.read_timeout = options.read_timeout;
+        o.write_timeout = options.write_timeout;
+        o.apply(self.sock.get_ref())
+    }
+
+    fn read_msg(&mut self) -> Result<Event> {
+        loop {
+            let msg = match self.sock.read_message() {
+                Ok(Message::Text(body)) => Event::Text(body),
+                Ok(Message::Binary(body)) => Event::Binary(body),
+                Ok(Message::Ping(ping)) => {
+                    self.sock.write_message(Message::Pong(ping))?;
+                    continue;
+                },
+                Ok(Message::Pong(_)) => continue, // this should never happen
+                Ok(Message::Close(_)) => Event::Close,
+                Err(tungstenite::Error::ConnectionClosed) => Event::Close,
+                Err(tungstenite::Error::AlreadyClosed) => Event::Close,
+                Err(tungstenite::Error::Io(err)) if err.kind() == io::ErrorKind::WouldBlock => Event::Timeout,
+                Err(err) => return Err(err.into()),
+            };
+            return Ok(msg);
+        }
+    }
+
+    pub fn read_text(&mut self) -> Result<Option<String>> {
+        match self.read_msg()? {
+            Event::Text(text) => Ok(Some(text)),
+            Event::Binary(_) => bail!("Unexpected message type: binary"),
+            Event::Close => bail!("Connection closed"),
+            Event::Timeout => Ok(None),
+        }
+    }
+
+    pub fn read_binary(&mut self) -> Result<Option<Vec<u8>>> {
+        match self.read_msg()? {
+            Event::Text(_) => bail!("Unexpected message type: text"),
+            Event::Binary(body) => Ok(Some(body)),
+            Event::Close => bail!("Connection closed"),
+            Event::Timeout => Ok(None),
+        }
+    }
+
+    fn write_msg(&mut self, msg: Message) -> Result<()> {
+        self.sock.write_message(msg)?;
+        self.sock.write_pending()?;
+        Ok(())
+    }
+
+    pub fn write_text(&mut self, text: String) -> Result<()> {
+        self.write_msg(Message::Text(text))
+    }
+
+    pub fn write_binary(&mut self, binary: Vec<u8>) -> Result<()> {
+        self.write_msg(Message::Binary(binary))
+    }
+}

sn0int-std/src/xml.rs

@@ -124,6 +124,7 @@ fn decode_raw(x: &str) -> Result<XmlDocument> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use maplit::hashmap;
 
     #[test]
     fn verify_xml_decode_empty() {

src/archive.rs

@@ -1,40 +0,0 @@
-use crate::errors::*;
-use tar;
-use libflate::gzip;
-use std::io;
-use std::fs::File;
-use std::path::Path;
-
-
-pub fn extract<R: io::Read, P: AsRef<Path>>(read: &mut R, filter: &str, target: P) -> Result<()> {
-    let file = gzip::Decoder::new(read)?;
-    let mut ar = tar::Archive::new(file);
-
-    for entry in ar.entries()? {
-        let mut entry = entry?;
-        let file_name = {
-            let path = entry.path()?;
-            path.file_name()
-                .ok_or_else(|| format_err!("Invalid path in archive"))?
-                .to_str()
-                .ok_or_else(|| format_err!("Filename is invalid utf8"))?
-                .to_owned()
-        };
-
-        debug!("Found in archive: {:?}", file_name);
-
-        if filter == file_name {
-            debug!("Extracting to {:?}", target.as_ref());
-
-            let mut target = File::create(target)?;
-            let n = io::copy(&mut entry, &mut target)?;
-            debug!("Wrote {:?} bytes", n);
-
-            return Ok(());
-        }
-
-        debug!("Skipping file");
-    }
-
-    bail!("Nothing in archive matched filter")
-}

src/args.rs

@@ -48,6 +48,9 @@ pub enum SubCommand {
     /// Search in the registry
     #[structopt(name="search")]
     Search(Search),
+    /// The sn0int package manager
+    #[structopt(name="pkg")]
+    Pkg(cmd::pkg_cmd::Args),
     /// Insert into the database
     #[structopt(name="add")]
     Add(cmd::add_cmd::Args),
@@ -57,6 +60,9 @@ pub enum SubCommand {
     /// Delete from the database
     #[structopt(name="delete")]
     Delete(cmd::delete_cmd::Args),
+    /// Query logged activity
+    #[structopt(name="activity")]
+    Activity(cmd::activity_cmd::Args),
     /// Include entities in the scope
     #[structopt(name="scope")]
     Scope(cmd::scope_cmd::Args),
@@ -75,6 +81,9 @@ pub enum SubCommand {
     /// Run a lua repl
     #[structopt(name="repl")]
     Repl,
+    /// Show paths of various file system locations
+    #[structopt(name="paths")]
+    Paths,
     /// Generate shell completions
     #[structopt(name="completions")]
     Completions(Completions),
@@ -115,6 +124,9 @@ pub struct Run {
     /// Narrow down targeted entities
     #[structopt(short="t", long="target")]
     pub target: Option<String>,
+    /// Dump the sandbox init message to stdout instead of running a child process
+    #[structopt(long="dump-sandbox-init-msg")]
+    pub dump_sandbox_init_msg: bool,
 }
 
 #[derive(Debug, StructOpt)]

src/auth.rs

@@ -11,7 +11,7 @@ use crate::term;
 
 
 fn path() -> Result<PathBuf> {
-    let path = paths::data_dir()?;
+    let path = paths::sn0int_dir()?;
     Ok(path.join("auth"))
 }
 

src/blobs.rs

@@ -3,44 +3,13 @@ use crate::paths;
 use crate::worker::{EventWithCallback, Event2};
 use crate::workspaces::Workspace;
 
-use blake2::VarBlake2b;
-use digest::{Input, VariableOutput};
 use bytes::Bytes;
-use serde::ser::{Serialize, Serializer};
-use serde::de::{self, Deserialize, Deserializer};
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::mpsc;
 
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct Blob {
-    pub id: String,
-    pub bytes: Bytes,
-}
-
-impl Blob {
-    pub fn create(bytes: Bytes) -> Blob {
-        let id = Self::hash(&bytes);
-        Blob {
-            id,
-            bytes,
-        }
-    }
-
-    pub fn hash(bytes: &[u8]) -> String {
-        let mut h = VarBlake2b::new(32).unwrap();
-        h.input(bytes);
-        Self::encode_hash(&h.vec_result())
-    }
-
-    #[inline]
-    fn encode_hash(bytes: &[u8]) -> String {
-        let x = bs58::encode(bytes).into_string();
-        format!("{:0<44}", x)
-    }
-}
+pub use sn0int_std::blobs::Blob;
 
 impl EventWithCallback for Blob {
     type Payload = ();
@@ -51,30 +20,6 @@ impl EventWithCallback for Blob {
     }
 }
 
-impl Serialize for Blob {
-    #[inline]
-    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let s = base64::encode(&self.bytes);
-        serializer.serialize_str(&s)
-    }
-}
-
-impl<'de> Deserialize<'de> for Blob {
-    #[inline]
-    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let s = String::deserialize(deserializer)?;
-        let bytes = base64::decode(&s)
-            .map_err(de::Error::custom)?;
-        Ok(Blob::create(Bytes::from(bytes)))
-    }
-}
-
 pub struct BlobStorage {
     path: PathBuf,
 }
@@ -155,7 +100,6 @@ impl BlobStorage {
 mod tests {
     use super::*;
     use tempfile;
-    use serde_json;
 
     #[inline]
     fn blob() -> (Bytes, Blob) {
@@ -163,15 +107,6 @@ mod tests {
         (bytes.clone(), Blob::create(bytes))
     }
 
-    #[test]
-    fn verify_create_blob() {
-        let (bytes, blob) = blob();
-        assert_eq!(blob, Blob {
-            id: String::from("DTTV3EjpHBNJx3Zw7eJsVPm4bYXKmNkJQpVNkcvTtTSz"),
-            bytes,
-        });
-    }
-
     #[test]
     fn test_blobstorage_save() {
         let dir = tempfile::tempdir().expect("tempdir");
@@ -204,38 +139,4 @@ mod tests {
 
         assert!(result.is_err());
     }
-
-    #[test]
-    fn test_serialize() {
-        let (_, blob) = blob();
-        let json = serde_json::to_string(&blob).unwrap();
-        assert_eq!(&json, "\"YXNkZg==\"");
-    }
-
-    #[test]
-    fn test_deserialize() {
-        let (_, blob1) = blob();
-        let blob2: Blob = serde_json::from_str("\"YXNkZg==\"").unwrap();
-        assert_eq!(blob1, blob2);
-    }
-
-    #[test]
-    fn test_hash_encoding() {
-        let x = bs58::decode("22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "22es54J4FbFtpb5D1MtBazVuum4TcqCQ7M9JkmYdmJ8W");
-    }
-
-    #[test]
-    fn test_hash_encoding_padding() {
-        let x = bs58::decode("r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei")
-            .into_vec()
-            .unwrap();
-        let x = Blob::encode_hash(&x);
-        assert_eq!(x.len(), 44);
-        assert_eq!(x, "r6edvU326yvpXLubYacXXSxf2HzqCgzqHUQvpWyNwei0");
-    }
 }

src/cmd/activity_cmd.rs

@@ -0,0 +1,88 @@
+use crate::errors::*;
+
+use crate::cmd::Cmd;
+use crate::shell::Shell;
+use crate::models::*;
+use chrono::{Utc, NaiveDateTime, NaiveTime, Duration};
+use std::convert::TryFrom;
+use std::io;
+use std::str::FromStr;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+#[derive(Debug)]
+pub struct TimeSpec {
+    datetime: NaiveDateTime,
+}
+
+impl FromStr for TimeSpec {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        let now = Utc::now().naive_utc();
+        let today = NaiveDateTime::new(now.date(), NaiveTime::from_hms(0, 0, 0));
+
+        let datetime = match s {
+            "today" => today,
+            "yesterday" => today - Duration::days(1),
+            // x {second,minute,hour,day,week,month,year}s? ago
+            s => NaiveDateTime::from_str(s)?,
+        };
+
+        Ok(TimeSpec {
+            datetime,
+        })
+    }
+}
+
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct Args {
+    /// Only query events for a given topic
+    #[structopt(short="t", long="topic")]
+    topic: Option<String>,
+    /// Only query events starting from that datetime
+    #[structopt(long="since")]
+    since: Option<TimeSpec>,
+    /// Only query events until this datetime
+    #[structopt(long="until")]
+    until: Option<TimeSpec>,
+    /// Try to select the previous event before --since as an initial state
+    #[structopt(short="i", long="initial")]
+    initial: bool,
+    /// Only query events that are tied to a location
+    #[structopt(short="l", long="location")]
+    location: bool,
+}
+
+impl Cmd for Args {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let filter = ActivityFilter {
+            topic: self.topic,
+            since: self.since.map(|t| t.datetime),
+            until: self.until.map(|t| t.datetime),
+            location: self.location,
+        };
+
+        let mut stdout = io::stdout();
+        let events = Activity::query(rl.db(), &filter)?;
+
+        if self.initial {
+            if let Some(first) = events.get(0) {
+                let previous = Activity::previous(rl.db(), first, &filter)?;
+                if let Some(previous) = previous {
+                    let mut previous = JsonActivity::try_from(previous)?;
+                    previous.initial = true;
+                    previous.write_to(&mut stdout)?;
+                }
+            }
+        }
+
+        for activity in events {
+            let activity = JsonActivity::try_from(activity)?;
+            activity.write_to(&mut stdout)?;
+        }
+
+        Ok(())
+    }
+}

src/cmd/add_cmd.rs

@@ -11,6 +11,7 @@ use crate::utils;
 use crate::term;
 use std::fs;
 use std::net;
+use std::net::SocketAddr;
 use ipnetwork;
 use std::path::Path;
 use walkdir::WalkDir;
@@ -37,7 +38,7 @@ pub enum Target {
     #[structopt(name="ipaddr")]
     IpAddr(AddIpAddr),
     /// Insert url into the database
-    #[structopt(name="subdomain")]
+    #[structopt(name="url")]
     Url(AddUrl),
     /// Insert email into the database
     #[structopt(name="email")]
@@ -63,6 +64,9 @@ pub enum Target {
     /// Insert ip network into the database
     #[structopt(name="netblock")]
     Netblock(AddNetblock),
+    /// Insert port into the database
+    #[structopt(name="port")]
+    Port(AddPort),
     /// Insert a crypto currency address into the database
     #[structopt(name="cryptoaddr")]
     CryptoAddr(AddCryptoAddr),
@@ -83,6 +87,7 @@ impl Cmd for Args {
             Target::Breach(args) => args.insert(rl, self.dry_run),
             Target::Image(args) => args.insert(rl, self.dry_run),
             Target::Netblock(args) => args.insert(rl, self.dry_run),
+            Target::Port(args) => args.insert(rl, self.dry_run),
             Target::CryptoAddr(args) => args.insert(rl, self.dry_run),
         }
     }
@@ -555,6 +560,71 @@ impl IntoInsert for AddNetblock {
     }
 }
 
+#[derive(Debug, StructOpt)]
+pub struct AddPort {
+    protocol: Option<String>,
+    addr: Option<SocketAddr>,
+}
+
+impl IntoInsert for AddPort {
+    fn into_insert(self, rl: &mut Shell) -> Result<Insert> {
+        let protocol = if let Some(protocol) = self.protocol {
+            protocol
+        } else {
+            utils::question("Protocol (tcp/udp)")?
+        };
+
+        let addr = if let Some(addr) = self.addr {
+            addr
+        } else {
+            let addr = utils::question("IP:Port")?;
+            addr.parse()?
+        };
+
+        let family = match addr.ip() {
+            net::IpAddr::V4(_) => "4",
+            net::IpAddr::V6(_) => "6",
+        };
+
+        let ip_addr_id = match rl.db().insert_struct(NewIpAddr {
+            family: family.to_string(),
+            value: addr.ip().to_string(),
+            continent: None,
+            continent_code: None,
+            country: None,
+            country_code: None,
+            city: None,
+            latitude: None,
+            longitude: None,
+            asn: None,
+            as_org: None,
+            description: None,
+            reverse_dns: None,
+            unscoped: false,
+        }, true)? {
+            Some((_, ip_addr_id)) => ip_addr_id,
+            _ => bail!("IpAddr is out out of scope"),
+        };
+
+        let value = format!("{}/{}", protocol, addr);
+
+        Ok(Insert::Port(NewPort {
+            ip_addr_id,
+            value,
+            ip_addr: addr.ip().to_string(),
+            port: addr.port() as i32,
+            protocol,
+            status: None,
+
+            banner: None,
+            service: None,
+            version: None,
+
+            unscoped: false,
+        }))
+    }
+}
+
 #[derive(Debug, StructOpt)]
 pub struct AddCryptoAddr {
     address: Option<String>,

src/cmd/mod.rs

@@ -1,5 +1,6 @@
 use crate::errors::*;
 use crate::shell::Shell;
+use crate::config::Config;
 
 pub trait Cmd: structopt::StructOpt + Sized {
     fn run(self, rl: &mut Shell) -> Result<()>;
@@ -11,6 +12,11 @@ pub trait Cmd: structopt::StructOpt + Sized {
     }
 }
 
+pub trait LiteCmd: structopt::StructOpt + Sized {
+    fn run(self, config: &Config) -> Result<()>;
+}
+
+pub mod activity_cmd;
 pub mod add_cmd;
 pub mod autonoscope_cmd;
 pub mod autoscope_cmd;
@@ -22,8 +28,8 @@ pub mod run_cmd;
 pub mod use_cmd;
 pub mod select_cmd;
 pub mod keyring_cmd;
-pub mod mod_cmd;
 pub mod noscope_cmd;
+pub mod pkg_cmd;
 pub mod set_cmd;
 pub mod scope_cmd;
 pub mod target_cmd;

src/cmd/pkg_cmd.rs

@@ -1,6 +1,9 @@
 use crate::errors::*;
 
 use crate::args;
+use crate::config::Config;
+use crate::cmd::{Cmd, LiteCmd};
+use crate::engine::Library;
 use crate::registry::{self, UpdateTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
@@ -20,6 +23,13 @@ pub struct Args {
     pub subcommand: SubCommand,
 }
 
+#[derive(Debug, StructOpt)]
+#[structopt(global_settings = &[AppSettings::ColoredHelp])]
+pub struct ArgsInteractive {
+    #[structopt(subcommand)]
+    pub subcommand: SubCommandInteractive,
+}
+
 #[derive(Debug, StructOpt)]
 pub enum SubCommand {
     /// List installed modules
@@ -31,9 +41,6 @@ pub enum SubCommand {
     /// Search modules in registry
     #[structopt(name="search")]
     Search(args::Search),
-    /// Reload modules
-    #[structopt(name="reload")]
-    Reload(Reload),
     /// Update modules
     #[structopt(name="update")]
     Update(Update),
@@ -42,6 +49,15 @@ pub enum SubCommand {
     Uninstall(Uninstall),
 }
 
+#[derive(Debug, StructOpt)]
+pub enum SubCommandInteractive {
+    #[structopt(flatten)]
+    Base(SubCommand),
+    /// Reload modules
+    #[structopt(name="reload")]
+    Reload(Reload),
+}
+
 #[derive(Debug, StructOpt)]
 pub struct List {
     /// Only show modules with a specific input source
@@ -65,15 +81,18 @@ pub struct Uninstall {
     module: ModuleID,
 }
 
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    let config = rl.config().clone();
+#[derive(PartialEq)]
+enum ModuleReload {
+    Yes,
+    No,
+}
 
-    match args.subcommand {
+fn run_subcommand(subcommand: SubCommand, library: &Library, config: &Config) -> Result<ModuleReload> {
+    match subcommand {
         SubCommand::List(list) => {
             let autoupdate = AutoUpdater::load()?;
 
-            for module in rl.engine().list() {
+            for module in library.list() {
                 if let Some(source) = &list.source {
                     if !module.source_equals(&source) {
                         continue;
@@ -93,31 +112,22 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
                 println!("{}", out);
                 println!("\t{}", module.description());
             }
+            Ok(ModuleReload::No)
         },
         SubCommand::Install(install) => {
             registry::run_install(install, &config)?;
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
-        SubCommand::Search(search) => registry::run_search(rl.engine(), &search, &config)?,
-        SubCommand::Reload(_) => {
-            let current = rl.take_module()
-                            .map(|m| m.canonical());
-
-            rl.engine_mut().reload_modules()?;
-            rl.reload_module_cache();
-
-            if let Some(module) = current {
-                if let Ok(module) = rl.engine().get(&module).map(|x| x.to_owned()) {
-                    rl.set_module(module);
-                }
-            }
+        SubCommand::Search(search) => {
+            registry::run_search(library, &search, &config)?;
+            Ok(ModuleReload::No)
         },
         SubCommand::Update(_) => {
             let mut autoupdate = AutoUpdater::load()?;
             let updater = Arc::new(Updater::new(&config)?);
 
-            let modules = rl.engine().list()
+            let modules = library.list()
                 .into_iter()
                 .filter_map(|module| {
                     let canonical = module.canonical();
@@ -138,15 +148,34 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
             autoupdate.save()?;
 
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
         SubCommand::Uninstall(uninstall) => {
             let updater = Updater::new(&config)?;
             updater.uninstall(&uninstall.module)?;
             // trigger reload
-            run(rl, &[String::from("mod"), String::from("reload")])?;
+            Ok(ModuleReload::Yes)
         },
     }
-
-    Ok(())
+}
+
+impl LiteCmd for Args {
+    fn run(self, config: &Config) -> Result<()> {
+        let library = Library::new(false, &config)?;
+        run_subcommand(self.subcommand, &library, config)?;
+        Ok(())
+    }
+}
+
+impl Cmd for ArgsInteractive {
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        let action = match self.subcommand {
+            SubCommandInteractive::Base(subcommand) => run_subcommand(subcommand, rl.library(), rl.config())?,
+            SubCommandInteractive::Reload(_) => ModuleReload::Yes,
+        };
+        if action == ModuleReload::Yes {
+            rl.reload_modules()?;
+        }
+        Ok(())
+    }
 }

src/cmd/quickstart_cmd.rs

@@ -2,7 +2,6 @@ use crate::errors::*;
 
 use crate::args::Install;
 use crate::api::Client;
-use crate::cmd::mod_cmd;
 use crate::registry::{InstallTask, Updater};
 use crate::shell::Shell;
 use crate::update::AutoUpdater;
@@ -47,7 +46,7 @@ pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     autoupdate.save()?;
 
     // trigger reload
-    mod_cmd::run(rl, &[String::from("mod"), String::from("reload")])?;
+    rl.reload_modules()?;
 
     Ok(())
 }

src/cmd/run_cmd.rs

@@ -1,10 +1,12 @@
 use crate::errors::*;
 
+use chrootable_https::dns::Resolver;
 use crate::args;
 use crate::blobs::{Blob, BlobStorage};
 use crate::cmd::Cmd;
 use crate::db::{ttl, Filter};
 use crate::engine::Module;
+use crate::ipc::common::StartCommand;
 use crate::models::*;
 use crate::shell::Shell;
 use crate::keyring::KeyRing;
@@ -113,16 +115,10 @@ fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> R
     Ok(())
 }
 
-pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
-    let module = rl.module()
-        .map(|m| m.to_owned())
-        .ok_or_else(|| format_err!("No module selected"))?;
-
-    prepare_keyring(rl.keyring_mut(), &module, &params)?;
-
+fn get_args(rl: &mut Shell, module: &Module) -> Result<Vec<(serde_json::Value, Option<String>, Vec<Blob>)>> {
     let filter = rl.scoped_targets();
 
-    let args = match module.source() {
+    match module.source() {
         Some(Source::Domains) => prepare_args::<Domain>(rl, &filter, None),
         Some(Source::Subdomains) => prepare_args::<Subdomain>(rl, &filter, None),
         Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl, &filter, None),
@@ -152,7 +148,44 @@ pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>)
             }
         },
         None => Ok(vec![(serde_json::Value::Null, None, vec![])]),
-    }?;
+    }
+}
+
+pub fn dump_sandbox_init_msg(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let keyring = rl.keyring().request_keys(&module);
+
+    let dns_config = Resolver::from_system()?;
+    let proxy = rl.config().network.proxy.clone();
+
+    let args = get_args(rl, &module)?;
+    for (arg, _pretty_arg, blobs) in args {
+        let start_cmd = StartCommand::new(params.verbose,
+                                          keyring.clone(),
+                                          dns_config.clone(),
+                                          proxy.clone(),
+                                          options.clone(),
+                                          module.clone(),
+                                          arg,
+                                          blobs);
+        let out = serde_json::to_string(&start_cmd)?;
+        println!("{}", out);
+    }
+
+    Ok(())
+}
+
+pub fn execute(rl: &mut Shell, params: Params, options: HashMap<String, String>) -> Result<()> {
+    let module = rl.module()
+        .map(|m| m.to_owned())
+        .ok_or_else(|| format_err!("No module selected"))?;
+
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+    let args = get_args(rl, &module)?;
 
     rl.signal_register().catch_ctrl();
     let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);

src/cmd/use_cmd.rs

@@ -14,7 +14,7 @@ pub struct Args {
 pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
 
-    let module = rl.engine().get(&args.module)?.clone();
+    let module = rl.library().get(&args.module)?.clone();
     rl.set_module(module);
 
     Ok(())

src/cmd/workspace_cmd.rs

@@ -1,7 +1,8 @@
 use crate::errors::*;
 
 use crate::blobs::BlobStorage;
-use crate::cmd::Cmd;
+use crate::cmd::{Cmd, LiteCmd};
+use crate::config::Config;
 use crate::db::Database;
 use crate::shell::Shell;
 use crate::term;
@@ -52,6 +53,8 @@ fn usage(workspace: Option<Workspace>) -> Result<()> {
 }
 
 fn change(rl: &mut Shell, workspace: Workspace) -> Result<()> {
+    workspace.migrate()?;
+
     let blobs = BlobStorage::workspace(&workspace)?;
     let db = Database::establish(workspace)?;
     rl.set_blobstorage(blobs);
@@ -66,29 +69,42 @@ fn list() -> Result<()> {
     Ok(())
 }
 
-impl Cmd for Args {
-    fn run(self, rl: &mut Shell) -> Result<()> {
-        if self.delete {
-            if let Some(workspace) = self.workspace {
+fn run(args: Args, rl: Option<&mut Shell>) -> Result<()> {
+    if args.delete {
+        if let Some(workspace) = args.workspace {
+            if let Some(rl) = rl {
                 if *rl.db().workspace() == workspace {
                     bail!("Can't delete current workspace")
                 }
-
-                delete(workspace, self.force)
-            } else {
-                bail!("--delete requires workspace")
             }
-        } else if self.usage {
-            usage(self.workspace)
-        } else if let Some(workspace) = self.workspace {
+
+            delete(workspace, args.force)
+        } else {
+            bail!("--delete requires workspace")
+        }
+    } else if args.usage {
+        usage(args.workspace)
+    } else if let Some(workspace) = args.workspace {
+        if let Some(rl) = rl {
             change(rl, workspace)
         } else {
-            list()
+            Ok(())
         }
+    } else {
+        list()
     }
 }
 
-#[inline]
-pub fn run(rl: &mut Shell, args: &[String]) -> Result<()> {
-    Args::run_str(rl, args)
+impl Cmd for Args {
+    #[inline]
+    fn run(self, rl: &mut Shell) -> Result<()> {
+        run(self, Some(rl))
+    }
+}
+
+impl LiteCmd for Args {
+    #[inline]
+    fn run(self, _config: &Config) -> Result<()> {
+        run(self, None)
+    }
 }

src/db/mod.rs

@@ -121,6 +121,8 @@ impl Database {
             .context("Failed to enable write ahead log")?;
         db.execute("PRAGMA foreign_keys = ON")
             .context("Failed to enforce foreign keys")?;
+        db.execute("PRAGMA synchronous = NORMAL")
+            .context("Failed to enforce foreign keys")?;
 
         let autonoscope = RuleSet::load(&db)?;
 
@@ -208,6 +210,17 @@ impl Database {
         }
     }
 
+    pub fn insert_activity(&self, obj: NewActivity) -> Result<bool> {
+        if let Some(uniq) = &obj.uniq {
+            if Activity::uniq(self, uniq)?.is_some() {
+                // unique tag set and event already logged
+                return Ok(false);
+            }
+        }
+        obj.insert(&self)?;
+        Ok(true)
+    }
+
     pub fn insert_subdomain_ipaddr_struct(&self, subdomain_ipaddr: &NewSubdomainIpAddr) -> Result<Option<(DbChange, i32)>> {
         if let Some(subdomain_ipaddr_id) = SubdomainIpAddr::get_id_opt(self, &(subdomain_ipaddr.subdomain_id, subdomain_ipaddr.ip_addr_id))? {
             Ok(Some((DbChange::None, subdomain_ipaddr_id)))

src/engine/ctx.rs

@@ -1,24 +1,28 @@
 use crate::errors::*;
 
-use crate::blobs::Blob;
 use crate::db::Family;
-use crate::engine::{Environment, Reporter};
+use crate::engine::{Environment, IpcChild};
 use crate::geoip::{MaxmindReader, GeoIP, AsnDB};
 use crate::hlua::{self, AnyLuaValue};
 use crate::keyring::KeyRingEntry;
-use crate::models::{Insert, Update};
+use crate::models::*;
 use crate::psl::{Psl, PslReader};
 use crate::lazy::Lazy;
 use crate::runtime;
 use crate::sockets::{Socket, SocketOptions, TlsData};
 use crate::web::{HttpSession, HttpRequest, RequestOptions};
-use crate::worker::{Event, LogEvent, DatabaseEvent, StdioEvent};
+use crate::websockets::{WebSocket, WebSocketOptions};
+use crate::worker::{Event, LogEvent, DatabaseEvent, DatabaseResponse, StdioEvent, RatelimitEvent};
+use crate::ratelimits::RatelimitResponse;
 use chrootable_https::{self, Resolver};
 use serde_json;
+use sn0int_std::blobs::{Blob, BlobState};
+use sn0int_std::web::WebState;
 use std::collections::HashMap;
 use std::result;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
+use std::thread;
 use rand::prelude::*;
 use rand::distributions::Alphanumeric;
 
@@ -68,36 +72,52 @@ pub trait State {
         self.send(&Event::Log(LogEvent::Status(msg)))
     }
 
-    fn db_insert(&self, object: Insert) -> Result<Option<i32>> {
+    fn db_recv(&self) -> Result<DatabaseResponse> {
+        let reply = self.recv()?;
+        let reply: result::Result<DatabaseResponse, String> = serde_json::from_value(reply)?;
+        reply.map_err(|err| format_err!("Database error: {:?}", err))
+    }
+
+    fn db_insert(&self, object: Insert) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Insert(object)));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        self.db_recv()
+            .context("Failed to add to database")
+            .map_err(Error::from)
     }
 
-    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<Option<i32>> {
+    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::InsertTtl((object, ttl))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
+        self.db_recv()
+            .context("Failed to add to database")
+            .map_err(Error::from)
     }
 
-    fn db_select(&self, family: Family, value: String) -> Result<Option<i32>> {
+    fn db_activity(&self, activity: InsertActivity) -> Result<bool> {
+        let activity = activity.try_into_new()?;
+
+        self.send(&Event::Database(DatabaseEvent::Activity(activity)));
+        let r = self.db_recv()
+            .context("Failed to log activity")?;
+
+        match r {
+            DatabaseResponse::Inserted(_) => Ok(false),
+            DatabaseResponse::NoChange(_) => Ok(true),
+            _ => bail!("Unexpected database response for db_activity: {:?}", r),
+        }
+    }
+
+    fn db_select(&self, family: Family, value: String) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Select((family, value))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to query database: {:?}", err))
+        self.db_recv()
+            .context("Failed to query database")
+            .map_err(Error::from)
     }
 
-    fn db_update(&self, object: String, update: Update) -> Result<Option<i32>> {
+    fn db_update(&self, object: String, update: Update) -> Result<DatabaseResponse> {
         self.send(&Event::Database(DatabaseEvent::Update((object, update))));
-        let reply = self.recv()?;
-        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
-
-        reply.map_err(|err| format_err!("Failed to update database: {:?}", err))
+        self.db_recv()
+            .context("Failed to update database")
+            .map_err(Error::from)
     }
 
     fn stdin_read_line(&self) -> Result<Option<String>> {
@@ -114,6 +134,21 @@ pub trait State {
         reply.map_err(|err| format_err!("Failed to read stdin: {:?}", err))
     }
 
+    fn ratelimit(&self, key: String, passes: u32, time: u32) -> Result<()> {
+        let ratelimit = Event::Ratelimit(RatelimitEvent::new(key, passes, time));
+        loop {
+            self.send(&ratelimit);
+            let reply = self.recv()?;
+            let reply: result::Result<RatelimitResponse, String> = serde_json::from_value(reply)?;
+            match reply {
+                Ok(RatelimitResponse::Retry(delay)) => thread::sleep(delay),
+                Ok(RatelimitResponse::Pass) => break,
+                Err(err) => bail!("Unexpected error case for ratelimit: {}", err),
+            }
+        }
+        Ok(())
+    }
+
     #[inline]
     fn random_id(&self) -> String {
         thread_rng().sample_iter(&Alphanumeric).take(16).collect()
@@ -139,16 +174,14 @@ pub trait State {
 
     fn sock_upgrade_tls(&self, id: &str, options: &SocketOptions) -> Result<TlsData>;
 
-    fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>>;
+    fn ws_connect(&self, url: url::Url, options: &WebSocketOptions) -> Result<String>;
+
+    fn get_ws(&self, id: &str)-> Arc<Mutex<WebSocket>>;
 
     fn http_mksession(&self) -> String;
 
     fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest;
 
-    fn register_in_jar(&self, session: &str, key: String, value: String);
-
-    fn register_blob(&self, blob: Blob) -> String;
-
     fn get_blob(&self, id: &str) -> Result<Arc<Blob>>;
 
     fn persist_blob(&self, id: &str) -> Result<()> {
@@ -163,8 +196,9 @@ pub trait State {
 // #[derive(Debug)]
 pub struct LuaState {
     error: Mutex<Option<Error>>,
-    logger: Arc<Mutex<Box<dyn Reporter>>>,
+    logger: Arc<Mutex<Box<dyn IpcChild>>>,
     socket_sessions: Mutex<HashMap<String, Arc<Mutex<Socket>>>>,
+    ws_sessions: Mutex<HashMap<String, Arc<Mutex<WebSocket>>>>,
     blobs: Mutex<HashMap<String, Arc<Blob>>>,
     http_sessions: Mutex<HashMap<String, HttpSession>>,
     http_clients: Mutex<HashMap<String, Arc<chrootable_https::Client<Resolver>>>>,
@@ -173,8 +207,8 @@ pub struct LuaState {
     keyring: Vec<KeyRingEntry>, // TODO: maybe hashmap
     dns_config: Resolver,
     psl: Mutex<Lazy<PslReader, Arc<Psl>>>,
-    geoip: Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>,
-    asn: Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>,
+    geoip: Option<Mutex<Lazy<MaxmindReader, Arc<GeoIP>>>>,
+    asn: Option<Mutex<Lazy<MaxmindReader, Arc<AsnDB>>>>,
     proxy: Option<SocketAddr>,
     options: HashMap<String, String>,
 }
@@ -236,15 +270,23 @@ impl State for LuaState {
     }
 
     fn geoip(&self) -> Result<Arc<GeoIP>> {
-        let mut geoip = self.geoip.lock().unwrap();
-        let geoip = geoip.get()?;
-        Ok(geoip.clone())
+        if let Some(mtx) = &self.geoip {
+            let mut geoip = mtx.lock().unwrap();
+            let geoip = geoip.get()?;
+            Ok(geoip.clone())
+        } else {
+            bail!("No geoip database loaded")
+        }
     }
 
     fn asn(&self) -> Result<Arc<AsnDB>> {
-        let mut asn = self.asn.lock().unwrap();
-        let asn = asn.get()?;
-        Ok(asn.clone())
+        if let Some(mtx) = &self.asn {
+            let mut asn = mtx.lock().unwrap();
+            let asn = asn.get()?;
+            Ok(asn.clone())
+        } else {
+            bail!("No asn database loaded")
+        }
     }
 
     fn sock_connect(&self, host: &str, port: u16, options: &SocketOptions) -> Result<String> {
@@ -264,13 +306,13 @@ impl State for LuaState {
 
     fn get_sock(&self, id: &str)-> Arc<Mutex<Socket>> {
         let mtx = self.socket_sessions.lock().unwrap();
-        let sock = mtx.get(id).expect("invalid session reference"); // TODO
+        let sock = mtx.get(id).expect("Invalid socket reference"); // TODO
         sock.clone()
     }
 
     fn sock_upgrade_tls(&self, id: &str, options: &SocketOptions) -> Result<TlsData> {
         let mut mtx = self.socket_sessions.lock().unwrap();
-        let sock = mtx.remove(id).expect("invalid session reference"); // TODO
+        let sock = mtx.remove(id).expect("Invalid socket reference"); // TODO
 
         let sock = Arc::try_unwrap(sock).unwrap();
         let sock = sock.into_inner().unwrap();
@@ -282,6 +324,45 @@ impl State for LuaState {
         Ok(tls)
     }
 
+    fn ws_connect(&self, url: url::Url, options: &WebSocketOptions) -> Result<String> {
+        let mut mtx = self.ws_sessions.lock().unwrap();
+        let id = self.random_id();
+
+        let sock = WebSocket::connect(&self.dns_config, url, options)?;
+        mtx.insert(id.clone(), Arc::new(Mutex::new(sock)));
+
+        Ok(id)
+    }
+
+    fn get_ws(&self, id: &str)-> Arc<Mutex<WebSocket>> {
+        let mtx = self.ws_sessions.lock().unwrap();
+        let sock = mtx.get(id).expect("Invalid ws reference"); // TODO
+        sock.clone()
+    }
+
+    fn http_mksession(&self) -> String {
+        let mut mtx = self.http_sessions.lock().unwrap();
+        let (id, session) = HttpSession::new();
+        mtx.insert(id.clone(), session);
+        id
+    }
+
+    fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest {
+        let mtx = self.http_sessions.lock().unwrap();
+        let session = mtx.get(session_id).expect("Invalid session reference"); // TODO
+
+        HttpRequest::new(&session, method, url, options)
+    }
+
+    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
+        let mtx = self.blobs.lock().unwrap();
+        let blob = mtx.get(id)
+            .ok_or_else(|| format_err!("Invalid blob reference"))?;
+        Ok(blob.clone())
+    }
+}
+
+impl WebState for LuaState {
     fn http(&self, proxy: &Option<SocketAddr>) -> Result<Arc<chrootable_https::Client<Resolver>>> {
         let proxy = self.resolve_proxy_options(proxy)?;
 
@@ -308,27 +389,15 @@ impl State for LuaState {
         }
     }
 
-    fn http_mksession(&self) -> String {
-        let mut mtx = self.http_sessions.lock().unwrap();
-        let (id, session) = HttpSession::new();
-        mtx.insert(id.clone(), session);
-        id
-    }
-
-    fn http_request(&self, session_id: &str, method: String, url: String, options: RequestOptions) -> HttpRequest {
-        let mtx = self.http_sessions.lock().unwrap();
-        let session = mtx.get(session_id).expect("invalid session reference"); // TODO
-
-        HttpRequest::new(&session, method, url, options)
-    }
-
     fn register_in_jar(&self, session: &str, key: String, value: String) {
         let mut mtx = self.http_sessions.lock().unwrap();
         if let Some(session) = mtx.get_mut(session) {
             session.cookies.register_in_jar(key, value);
         }
     }
+}
 
+impl BlobState for LuaState {
     fn register_blob(&self, blob: Blob) -> String {
         let id = blob.id.clone();
 
@@ -338,13 +407,6 @@ impl State for LuaState {
 
         id
     }
-
-    fn get_blob(&self, id: &str) -> Result<Arc<Blob>> {
-        let mtx = self.blobs.lock().unwrap();
-        let blob = mtx.get(id)
-            .ok_or_else(|| format_err!("Invalid blob reference"))?;
-        Ok(blob.clone())
-    }
 }
 
 impl LuaState {
@@ -364,15 +426,19 @@ pub struct Script {
     code: String,
 }
 
-pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
+pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn IpcChild>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
     debug!("Creating lua context");
     let mut lua = hlua::Lua::new();
     lua.open_string();
 
+    let geoip = env.geoip.map(|db| Mutex::new(Lazy::from(db)));
+    let asn = env.asn.map(|db| Mutex::new(Lazy::from(db)));
+
     let state = Arc::new(LuaState {
         error: Mutex::new(None),
         logger,
         socket_sessions: Mutex::new(HashMap::new()),
+        ws_sessions: Mutex::new(HashMap::new()),
         blobs: Mutex::new(HashMap::new()),
         http_sessions: Mutex::new(HashMap::new()),
         http_clients: Mutex::new(HashMap::new()),
@@ -381,8 +447,8 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
         keyring: env.keyring,
         dns_config: env.dns_config,
         psl: Mutex::new(Lazy::from(env.psl)),
-        geoip: Mutex::new(Lazy::from(env.geoip)),
-        asn: Mutex::new(Lazy::from(env.asn)),
+        geoip,
+        asn,
         proxy: env.proxy,
         options: env.options,
     });
@@ -404,11 +470,13 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::datetime(&mut lua, state.clone());
     runtime::db_add(&mut lua, state.clone());
     runtime::db_add_ttl(&mut lua, state.clone());
+    runtime::db_activity(&mut lua, state.clone());
     runtime::db_select(&mut lua, state.clone());
     runtime::db_update(&mut lua, state.clone());
     runtime::debug(&mut lua, state.clone());
     runtime::dns(&mut lua, state.clone());
     runtime::error(&mut lua, state.clone());
+    runtime::geo_polygon_contains(&mut lua, state.clone());
     runtime::geoip_lookup(&mut lua, state.clone());
     runtime::getopt(&mut lua, state.clone());
     runtime::hex(&mut lua, state.clone());
@@ -440,6 +508,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::pgp_pubkey_armored(&mut lua, state.clone());
     runtime::print(&mut lua, state.clone());
     runtime::psl_domain_from_dns_name(&mut lua, state.clone());
+    runtime::ratelimit_throttle(&mut lua, state.clone());
     runtime::regex_find(&mut lua, state.clone());
     runtime::regex_find_all(&mut lua, state.clone());
     runtime::semver_match(&mut lua, state.clone());
@@ -455,6 +524,7 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::sn0int_version(&mut lua, state.clone());
     runtime::sock_connect(&mut lua, state.clone());
     runtime::sock_upgrade_tls(&mut lua, state.clone());
+    runtime::sock_options(&mut lua, state.clone());
     runtime::sock_send(&mut lua, state.clone());
     runtime::sock_recv(&mut lua, state.clone());
     runtime::sock_sendline(&mut lua, state.clone());
@@ -469,6 +539,9 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::status(&mut lua, state.clone());
     runtime::stdin_read_line(&mut lua, state.clone());
     runtime::stdin_read_to_end(&mut lua, state.clone());
+    runtime::str_find(&mut lua, state.clone());
+    runtime::str_replace(&mut lua, state.clone());
+    runtime::strval(&mut lua, state.clone());
     runtime::strftime(&mut lua, state.clone());
     runtime::strptime(&mut lua, state.clone());
     runtime::time_unix(&mut lua, state.clone());
@@ -481,6 +554,14 @@ pub fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<dyn Reporter>>>) -> (hlua
     runtime::utf8_decode(&mut lua, state.clone());
     runtime::warn(&mut lua, state.clone());
     runtime::warn_once(&mut lua, state.clone());
+    runtime::ws_connect(&mut lua, state.clone());
+    runtime::ws_options(&mut lua, state.clone());
+    runtime::ws_recv_text(&mut lua, state.clone());
+    runtime::ws_recv_binary(&mut lua, state.clone());
+    runtime::ws_recv_json(&mut lua, state.clone());
+    runtime::ws_send_text(&mut lua, state.clone());
+    runtime::ws_send_binary(&mut lua, state.clone());
+    runtime::ws_send_json(&mut lua, state.clone());
     runtime::x509_parse_pem(&mut lua, state.clone());
     runtime::xml_decode(&mut lua, state.clone());
     runtime::xml_named(&mut lua, state.clone());
@@ -511,7 +592,7 @@ impl Script {
     }
 
     pub fn run(&self, env: Environment,
-                      tx: Arc<Mutex<Box<dyn Reporter>>>,
+                      tx: Arc<Mutex<Box<dyn IpcChild>>>,
                       arg: AnyLuaValue,
     ) -> Result<()> {
         let (mut lua, state) = ctx(env, tx);
@@ -542,8 +623,9 @@ impl Script {
 
     #[cfg(test)]
     pub fn test(&self) -> Result<()> {
-        use crate::engine::DummyReporter;
+        use crate::ipc::child::DummyIpcChild;
         use crate::geoip::Maxmind;
+        use crate::paths;
         let keyring = Vec::new();
         let dns_config = Resolver::from_system()?;
         let proxy = None;
@@ -555,8 +637,9 @@ com
 a.prod.fastly.net
 // ===END PRIVATE DOMAINS===
 "#.into());
-        let geoip = GeoIP::open_reader()?;
-        let asn = AsnDB::open_reader()?;
+        let cache_dir = paths::cache_dir()?;
+        let geoip = GeoIP::try_open_reader(&cache_dir)?;
+        let asn = AsnDB::try_open_reader(&cache_dir)?;
 
         let env = Environment {
             verbose: 0,
@@ -569,6 +652,6 @@ a.prod.fastly.net
             geoip,
             asn,
         };
-        self.run(env, DummyReporter::new(), AnyLuaValue::LuaNil)
+        self.run(env, DummyIpcChild::new(), AnyLuaValue::LuaNil)
     }
 }

src/engine/isolation.rs

@@ -1,254 +0,0 @@
-use crate::errors::*;
-use chrootable_https::dns::Resolver;
-use crate::blobs::Blob;
-use crate::engine::{Environment, Module, Reporter};
-use crate::geoip::MaxmindReader;
-use crate::keyring::KeyRingEntry;
-use crate::psl::PslReader;
-use serde_json;
-use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
-
-use std::collections::HashMap;
-use std::env;
-use std::ffi::OsString;
-use std::io::prelude::*;
-use std::io::{self, BufReader, BufRead, stdin, Stdin, Stdout};
-use std::net::SocketAddr;
-use std::sync::{mpsc, Arc, Mutex};
-use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
-
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct StartCommand {
-    verbose: u64,
-    keyring: Vec<KeyRingEntry>,
-    dns_config: Resolver,
-    proxy: Option<SocketAddr>,
-    options: HashMap<String, String>,
-    module: Module,
-    arg: serde_json::Value,
-    blobs: Vec<Blob>,
-}
-
-impl StartCommand {
-    pub fn new(verbose: u64,
-               keyring: Vec<KeyRingEntry>,
-               dns_config: Resolver,
-               proxy: Option<SocketAddr>,
-               options: HashMap<String, String>,
-               module: Module,
-               arg: serde_json::Value,
-               blobs: Vec<Blob>,
-    ) -> StartCommand {
-        StartCommand {
-            verbose,
-            keyring,
-            dns_config,
-            proxy,
-            options,
-            module,
-            arg,
-            blobs,
-        }
-    }
-}
-
-pub struct Supervisor {
-    child: Child,
-    stdin: ChildStdin,
-    stdout: BufReader<ChildStdout>,
-}
-
-impl Supervisor {
-    pub fn setup(module: &Module) -> Result<Supervisor> {
-        let exe = match env::current_exe() {
-            Ok(exe) => exe.into_os_string(),
-            _ => OsString::from("sn0int"),
-        };
-
-        let mut child = Command::new(exe)
-            .arg("sandbox")
-            .arg(&module.canonical())
-            .stdin(Stdio::piped())
-            .stdout(Stdio::piped())
-            .spawn()
-            .context("Failed to spawn child process")?;
-
-        let stdin = child.stdin.take().expect("Failed to take child stdin");
-        let stdout = child.stdout.take().expect("Failed to take child stdout");
-        let stdout = BufReader::new(stdout);
-
-        Ok(Supervisor {
-            child,
-            stdin,
-            stdout,
-        })
-    }
-
-    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
-        let start = serde_json::to_value(&start)?;
-        self.send(&start)?;
-        Ok(())
-    }
-
-    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
-        let mut value = serde_json::to_string(value)?;
-        value.push('\n');
-        self.stdin.write_all(value.as_bytes())?;
-        debug!("Supervisor sent: {:?}", value);
-        Ok(())
-    }
-
-    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
-        let value = serde_json::to_value(value).expect("Failed to serialize reply");
-        if let Err(_) = self.send(&value) {
-            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
-        }
-    }
-
-    pub fn recv(&mut self) -> Result<Event> {
-        let mut line = String::new();
-        let len = self.stdout.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Supervisor received: {:?}", event);
-        Ok(event)
-    }
-
-    pub fn wait(&mut self) -> Result<()> {
-        let exit = self.child.wait()
-            .context("Failed to wait for child")?;
-
-        if exit.success() {
-            Ok(())
-        } else {
-            bail!("Child signaled error")
-        }
-    }
-
-    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
-        where <T as EventWithCallback>::Payload: serde::Serialize
-    {
-        let (tx2, rx2) = mpsc::channel();
-        tx.send(event.with_callback(tx2));
-        let reply = rx2.recv().unwrap();
-
-        self.send_struct(reply, tx);
-    }
-}
-
-#[derive(Debug)]
-pub struct StdioReporter {
-    stdin: Stdin,
-    stdout: Stdout,
-}
-
-impl StdioReporter {
-    pub fn setup() -> StdioReporter {
-        let stdin = io::stdin();
-        let stdout = io::stdout();
-
-        StdioReporter {
-            stdin,
-            stdout,
-        }
-    }
-
-    pub fn recv_start(&mut self) -> Result<StartCommand> {
-        let value = self.recv()?;
-        let event = serde_json::from_value(value)?;
-        Ok(event)
-    }
-}
-
-impl Reporter for StdioReporter {
-    fn send(&mut self, event: &Event) -> Result<()> {
-        let mut event = serde_json::to_string(&event)?;
-        event.push('\n');
-        self.stdout.write_all(event.as_bytes())?;
-        debug!("Reporter sent: {:?}", event);
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        let mut line = String::new();
-        let len = self.stdin.read_line(&mut line)?;
-
-        let event = serde_json::from_str(&line[..len])?;
-        debug!("Reporter received: {:?}", event);
-        Ok(event)
-    }
-}
-
-pub fn spawn_module(module: Module,
-                    tx: &EventSender,
-                    arg: serde_json::Value,
-                    keyring: Vec<KeyRingEntry>,
-                    verbose: u64,
-                    has_stdin: bool,
-                    proxy: Option<SocketAddr>,
-                    options: HashMap<String, String>,
-                    blobs: Vec<Blob>,
-) -> Result<ExitEvent> {
-    let dns_config = Resolver::from_system()?;
-
-    let mut reader = if has_stdin {
-        Some(BufReader::new(stdin()))
-    } else {
-        None
-    };
-
-    let mut supervisor = Supervisor::setup(&module)?;
-    supervisor.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
-
-    let exit = loop {
-        match supervisor.recv()? {
-            Event::Log(event) => tx.send(Event2::Log(event)),
-            Event::Database(object) => supervisor.send_event_callback(object, &tx),
-            Event::Stdio(object) => object.apply(&mut supervisor, tx, &mut reader),
-            Event::Blob(blob) => supervisor.send_event_callback(blob, &tx),
-            Event::Exit(event) => {
-                if let ExitEvent::Err(err) = &event {
-                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
-                }
-                break event;
-            },
-        }
-    };
-
-    supervisor.wait()?;
-
-    Ok(exit)
-}
-
-pub fn run_worker(geoip: MaxmindReader, asn: MaxmindReader, psl: PslReader) -> Result<()> {
-    let mut reporter = StdioReporter::setup();
-    let start = reporter.recv_start()?;
-
-    let environment = Environment {
-        verbose: start.verbose,
-        keyring: start.keyring,
-        dns_config: start.dns_config,
-        proxy: start.proxy,
-        options: start.options,
-        blobs: start.blobs,
-        psl,
-        geoip,
-        asn,
-    };
-
-    let mtx: Arc<Mutex<Box<dyn Reporter>>> = Arc::new(Mutex::new(Box::new(reporter)));
-    let result = start.module.run(environment,
-                                  mtx.clone(),
-                                  start.arg.into());
-    let mut reporter = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
-                        .into_inner().expect("Failed to consume Mutex");
-
-    let event = match result {
-        Ok(_) => ExitEvent::Ok,
-        Err(err) => ExitEvent::Err(err.to_string()),
-    };
-    reporter.send(&Event::Exit(event))?;
-
-    Ok(())
-}

src/engine/mod.rs

@@ -5,7 +5,6 @@ use crate::config::Config;
 use crate::geoip::MaxmindReader;
 use crate::json::LuaJsonValue;
 use crate::keyring::KeyRingEntry;
-use serde_json;
 use std::fs;
 use std::fmt::Debug;
 use std::path::PathBuf;
@@ -13,6 +12,7 @@ use std::collections::HashMap;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
 use crate::engine::ctx::Script;
+use crate::ipc::child::IpcChild;
 use sn0int_common::ModuleID;
 use sn0int_common::metadata::{Metadata, Source};
 use chrootable_https::dns::Resolver;
@@ -21,11 +21,10 @@ use crate::paths;
 use std::cmp::Ordering;
 use std::path::Path;
 use crate::term;
-use crate::worker::{self, Event};
+use crate::worker;
 
 pub mod ctx;
-pub mod isolation;
-pub mod structs;
+pub use sn0int_std::engine::structs;
 
 
 /// Data that is passed to every script
@@ -38,34 +37,34 @@ pub struct Environment {
     pub options: HashMap<String, String>,
     pub blobs: Vec<Blob>,
     pub psl: PslReader,
-    pub geoip: MaxmindReader,
-    pub asn: MaxmindReader,
+    pub geoip: Option<MaxmindReader>,
+    pub asn: Option<MaxmindReader>,
 }
 
 #[derive(Debug)]
-pub struct Engine<'a> {
+pub struct Library<'a> {
     path: PathBuf,
     modules: HashMap<String, Vec<Module>>,
     config: &'a Config
 }
 
-impl<'a> Engine<'a> {
-    pub fn new(verbose_init: bool, config: &'a Config) -> Result<Engine> {
+impl<'a> Library<'a> {
+    pub fn new(verbose_init: bool, config: &'a Config) -> Result<Library> {
         let path = paths::module_dir()?;
 
-        let mut engine = Engine {
+        let mut library = Library {
             path,
             modules: HashMap::new(),
             config,
         };
 
         if verbose_init {
-            engine.reload_modules()?;
+            library.reload_modules()?;
         } else {
-            engine.reload_modules_quiet()?;
+            library.reload_modules_quiet()?;
         }
 
-        Ok(engine)
+        Ok(library)
     }
 
     pub fn reload_modules(&mut self) -> Result<usize> {
@@ -299,9 +298,9 @@ impl Module {
         self.private_module
     }
 
-    pub fn run(&self, env: Environment, reporter: Arc<Mutex<Box<dyn Reporter>>>, arg: LuaJsonValue) -> Result<()> {
+    pub fn run(&self, env: Environment, ipc_child: Arc<Mutex<Box<dyn IpcChild>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
-        self.script.run(env, reporter, arg.into())
+        self.script.run(env, ipc_child, arg.into())
     }
 
     #[inline]
@@ -320,28 +319,3 @@ impl Module {
         }
     }
 }
-
-pub trait Reporter: Debug {
-    fn send(&mut self, event: &Event) -> Result<()>;
-
-    fn recv(&mut self) -> Result<serde_json::Value>;
-}
-
-#[derive(Debug)]
-pub struct DummyReporter;
-
-impl DummyReporter {
-    pub fn new() -> Arc<Mutex<Box<dyn Reporter>>> {
-        Arc::new(Mutex::new(Box::new(DummyReporter)))
-    }
-}
-
-impl Reporter for DummyReporter {
-    fn send(&mut self, _event: &Event) -> Result<()> {
-        Ok(())
-    }
-
-    fn recv(&mut self) -> Result<serde_json::Value> {
-        unimplemented!("DummyReporter::recv doesn't exist")
-    }
-}

src/errors.rs

@@ -1,2 +1,3 @@
-pub use failure::{Error, ResultExt};
+pub use log::{trace, debug, info, warn, error};
+pub use failure::{Error, ResultExt, format_err, bail};
 pub type Result<T> = ::std::result::Result<T, Error>;

src/fmt.rs

@@ -141,8 +141,7 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
     }
 
     #[inline]
-    pub fn display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
-        self.push_into_group()?;
+    pub fn direct_display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
         if self.scoped {
             C::display(self, v)
         } else {
@@ -151,8 +150,19 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
     }
 
     #[inline]
-    pub fn debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
+    pub fn display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
         self.push_into_group()?;
+        self.direct_display::<C, _>(v)
+    }
+
+    pub fn display_label<C: Color, D: fmt::Display>(&mut self, label: &str, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        write!(self, "{}=", label)?;
+        self.direct_display::<C, _>(v)
+    }
+
+    #[inline]
+    pub fn direct_debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
         if self.scoped {
             C::debug(self, v)
         } else {
@@ -160,6 +170,12 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
         }
     }
 
+    #[inline]
+    pub fn debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        self.direct_debug::<C, _>(v)
+    }
+
     pub fn opt_debug<C: Color, D: fmt::Debug>(&mut self, v: &Option<D>) -> fmt::Result {
         if let Some(v) = &v {
             self.debug::<C, _>(v)?;
@@ -167,6 +183,15 @@ impl<'a, 'b> DetailFormatter<'a, 'b> {
         Ok(())
     }
 
+    pub fn opt_debug_label<C: Color, D: fmt::Debug>(&mut self, label: &str, v: &Option<D>) -> fmt::Result {
+        if let Some(v) = &v {
+            self.push_into_group()?;
+            write!(self, "{}=", label)?;
+            self.direct_debug::<C, _>(v)?;
+        }
+        Ok(())
+    }
+
     #[inline]
     pub fn clear(&mut self) -> fmt::Result {
         if self.scoped {

src/ipc/child.rs

@@ -0,0 +1,110 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use crate::engine::Environment;
+use crate::geoip::MaxmindReader;
+use crate::psl::PslReader;
+use serde_json;
+use crate::worker::Event;
+
+use std::fmt::Debug;
+use std::io::prelude::*;
+use std::io::{self, Stdin, Stdout};
+use std::sync::{Arc, Mutex};
+
+
+pub trait IpcChild: Debug {
+    fn send(&mut self, event: &Event) -> Result<()>;
+
+    fn recv(&mut self) -> Result<serde_json::Value>;
+}
+
+#[derive(Debug)]
+pub struct StdioIpcChild {
+    stdin: Stdin,
+    stdout: Stdout,
+}
+
+impl StdioIpcChild {
+    pub fn setup() -> StdioIpcChild {
+        let stdin = io::stdin();
+        let stdout = io::stdout();
+
+        StdioIpcChild {
+            stdin,
+            stdout,
+        }
+    }
+
+    pub fn recv_start(&mut self) -> Result<StartCommand> {
+        let value = self.recv()?;
+        let event = serde_json::from_value(value)?;
+        Ok(event)
+    }
+}
+
+impl IpcChild for StdioIpcChild {
+    fn send(&mut self, event: &Event) -> Result<()> {
+        let mut event = serde_json::to_string(&event)?;
+        event.push('\n');
+        debug!("IpcChild sends: {:?}", event);
+        self.stdout.write_all(event.as_bytes())?;
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        let mut line = String::new();
+        let len = self.stdin.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcChild received: {:?}", event);
+        Ok(event)
+    }
+}
+
+#[derive(Debug)]
+pub struct DummyIpcChild;
+
+impl DummyIpcChild {
+    pub fn new() -> Arc<Mutex<Box<dyn IpcChild>>> {
+        Arc::new(Mutex::new(Box::new(DummyIpcChild)))
+    }
+}
+
+impl IpcChild for DummyIpcChild {
+    fn send(&mut self, _event: &Event) -> Result<()> {
+        Ok(())
+    }
+
+    fn recv(&mut self) -> Result<serde_json::Value> {
+        unimplemented!("DummyIpcChild::recv doesn't exist")
+    }
+}
+
+pub fn run(geoip: Option<MaxmindReader>, asn: Option<MaxmindReader>, psl: PslReader) -> Result<()> {
+    let mut ipc_child = StdioIpcChild::setup();
+    let start = ipc_child.recv_start()?;
+
+    let environment = Environment {
+        verbose: start.verbose,
+        keyring: start.keyring,
+        dns_config: start.dns_config,
+        proxy: start.proxy,
+        options: start.options,
+        blobs: start.blobs,
+        psl,
+        geoip,
+        asn,
+    };
+
+    let mtx: Arc<Mutex<Box<dyn IpcChild>>> = Arc::new(Mutex::new(Box::new(ipc_child)));
+    let result = start.module.run(environment,
+                                  mtx.clone(),
+                                  start.arg.into());
+    let mut ipc_child = Arc::try_unwrap(mtx).expect("Failed to consume Arc")
+                        .into_inner().expect("Failed to consume Mutex");
+
+    let event = result.into();
+    ipc_child.send(&Event::Exit(event))?;
+
+    Ok(())
+}

src/ipc/common.rs

@@ -0,0 +1,44 @@
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+
+use std::collections::HashMap;
+use std::net::SocketAddr;
+
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct StartCommand {
+    pub verbose: u64,
+    pub keyring: Vec<KeyRingEntry>,
+    pub dns_config: Resolver,
+    pub proxy: Option<SocketAddr>,
+    pub options: HashMap<String, String>,
+    pub module: Module,
+    pub arg: serde_json::Value,
+    pub blobs: Vec<Blob>,
+}
+
+impl StartCommand {
+    pub fn new(verbose: u64,
+               keyring: Vec<KeyRingEntry>,
+               dns_config: Resolver,
+               proxy: Option<SocketAddr>,
+               options: HashMap<String, String>,
+               module: Module,
+               arg: serde_json::Value,
+               blobs: Vec<Blob>,
+    ) -> StartCommand {
+        StartCommand {
+            verbose,
+            keyring,
+            dns_config,
+            proxy,
+            options,
+            module,
+            arg,
+            blobs,
+        }
+    }
+}

src/ipc/mod.rs

@@ -0,0 +1,3 @@
+pub mod child;
+pub mod common;
+pub mod parent;

src/ipc/parent.rs

@@ -0,0 +1,144 @@
+use crate::errors::*;
+use crate::ipc::common::*;
+use chrootable_https::dns::Resolver;
+use crate::blobs::Blob;
+use crate::engine::Module;
+use crate::keyring::KeyRingEntry;
+use serde_json;
+use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
+
+use std::collections::HashMap;
+use std::env;
+use std::ffi::OsString;
+use std::io::prelude::*;
+use std::io::{BufReader, BufRead, stdin};
+use std::net::SocketAddr;
+use std::sync::mpsc;
+use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
+
+
+pub struct IpcParent {
+    child: Child,
+    stdin: ChildStdin,
+    stdout: BufReader<ChildStdout>,
+}
+
+impl IpcParent {
+    pub fn setup(module: &Module) -> Result<IpcParent> {
+        let exe = match env::current_exe() {
+            Ok(exe) => exe.into_os_string(),
+            _ => OsString::from("sn0int"),
+        };
+
+        let mut child = Command::new(exe)
+            .arg("sandbox")
+            .arg(&module.canonical())
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .context("Failed to spawn child process")?;
+
+        let stdin = child.stdin.take().expect("Failed to take child stdin");
+        let stdout = child.stdout.take().expect("Failed to take child stdout");
+        let stdout = BufReader::new(stdout);
+
+        Ok(IpcParent {
+            child,
+            stdin,
+            stdout,
+        })
+    }
+
+    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
+        let start = serde_json::to_value(&start)?;
+        self.send(&start)?;
+        Ok(())
+    }
+
+    pub fn send(&mut self, value: &serde_json::Value) -> Result<()> {
+        let mut value = serde_json::to_string(value)?;
+        value.push('\n');
+        self.stdin.write_all(value.as_bytes())?;
+        debug!("IpcParent sent: {:?}", value);
+        Ok(())
+    }
+
+    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
+        let value = serde_json::to_value(value).expect("Failed to serialize reply");
+        if let Err(_) = self.send(&value) {
+            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
+        }
+    }
+
+    pub fn recv(&mut self) -> Result<Event> {
+        let mut line = String::new();
+        let len = self.stdout.read_line(&mut line)?;
+
+        let event = serde_json::from_str(&line[..len])?;
+        debug!("IpcParent received: {:?}", event);
+        Ok(event)
+    }
+
+    pub fn wait(&mut self) -> Result<()> {
+        let exit = self.child.wait()
+            .context("Failed to wait for child")?;
+
+        if exit.success() {
+            Ok(())
+        } else {
+            bail!("Child signaled error")
+        }
+    }
+
+    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
+        where <T as EventWithCallback>::Payload: serde::Serialize
+    {
+        let (tx2, rx2) = mpsc::channel();
+        tx.send(event.with_callback(tx2));
+        let reply = rx2.recv().unwrap();
+
+        self.send_struct(reply, tx);
+    }
+}
+
+pub fn run(module: Module,
+           tx: &EventSender,
+           arg: serde_json::Value,
+           keyring: Vec<KeyRingEntry>,
+           verbose: u64,
+           has_stdin: bool,
+           proxy: Option<SocketAddr>,
+           options: HashMap<String, String>,
+           blobs: Vec<Blob>,
+) -> Result<ExitEvent> {
+    let dns_config = Resolver::from_system()?;
+
+    let mut reader = if has_stdin {
+        Some(BufReader::new(stdin()))
+    } else {
+        None
+    };
+
+    let mut ipc_parent = IpcParent::setup(&module)?;
+    ipc_parent.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg, blobs))?;
+
+    let exit = loop {
+        match ipc_parent.recv()? {
+            Event::Log(event) => tx.send(Event2::Log(event)),
+            Event::Database(object) => ipc_parent.send_event_callback(object, &tx),
+            Event::Stdio(object) => object.apply(&mut ipc_parent, tx, &mut reader),
+            Event::Ratelimit(req) => ipc_parent.send_event_callback(req, &tx),
+            Event::Blob(blob) => ipc_parent.send_event_callback(blob, &tx),
+            Event::Exit(event) => {
+                if let ExitEvent::Err(err) = &event {
+                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
+                }
+                break event;
+            },
+        }
+    };
+
+    ipc_parent.wait()?;
+
+    Ok(exit)
+}

src/keyring.rs

@@ -74,7 +74,7 @@ pub struct KeyRing {
 
 impl KeyRing {
     pub fn path() -> Result<PathBuf> {
-        let path = paths::data_dir()?;
+        let path = paths::sn0int_dir()?;
         let path = path.join("keyring.json");
         Ok(path)
     }

src/lib.rs

@@ -1,10 +1,8 @@
 #![warn(unused_extern_crates)]
-#[macro_use] extern crate failure;
 #[macro_use] extern crate maplit;
 use url;
 use hlua_badtouch as hlua;
 #[macro_use] extern crate serde_derive;
-#[macro_use] extern crate log;
 #[macro_use] extern crate structopt;
 #[macro_use] extern crate lazy_static;
 #[macro_use] extern crate diesel;
@@ -12,30 +10,32 @@ use hlua_badtouch as hlua;
 #[macro_use] extern crate crossbeam_channel as channel;
 
 pub mod api;
-pub mod archive;
 pub mod args;
 pub mod auth;
 pub mod autonoscope;
 pub mod blobs;
 pub mod cmd;
 pub mod config;
-pub mod crt;
+use sn0int_std::crt;
 pub mod db;
 pub mod errors;
 pub mod engine;
 pub mod filters;
 pub mod fmt;
-pub mod geoip;
-pub mod gfx;
-pub mod html;
-pub mod json;
+use sn0int_std::geo;
+pub use sn0int_std::geoip;
+use sn0int_std::gfx;
+use sn0int_std::html;
+use sn0int_std::json;
+pub mod ipc;
 pub mod keyring;
-pub mod lazy;
+use sn0int_std::lazy;
 pub mod migrations;
 pub mod models;
 pub mod paths;
-pub mod psl;
+pub use sn0int_std::psl;
 pub mod options;
+use sn0int_std::ratelimits;
 pub mod registry;
 pub mod repl;
 pub mod runtime;
@@ -43,16 +43,12 @@ pub mod sandbox;
 pub mod schema;
 pub mod ser;
 pub mod shell;
-pub mod sockets;
+use sn0int_std::sockets;
 pub mod term;
 pub mod update;
 pub mod utils;
-pub mod web;
+use sn0int_std::web;
+use sn0int_std::websockets;
 pub mod worker;
 pub mod workspaces;
-pub mod xml;
-
-#[cfg(test)]
-fn test_init() {
-    let _ = env_logger::builder().is_test(true).try_init();
-}
+use sn0int_std::xml;

src/main.rs

@@ -1,17 +1,15 @@
-#![warn(unused_extern_crates)]
-#[macro_use] extern crate failure;
-#[macro_use] extern crate log;
-
 use env_logger::{self, Env};
 use sn0int::args::{self, Args, SubCommand};
 use sn0int::auth;
-use sn0int::cmd;
+use sn0int::cmd::{self, LiteCmd};
 use sn0int::config::Config;
 use sn0int::db;
 use sn0int::errors::*;
-use sn0int::engine::{self, Engine, Module};
+use sn0int::engine::Module;
 use sn0int::geoip::{GeoIP, AsnDB, Maxmind};
+use sn0int::ipc;
 use sn0int::options::Opt;
+use sn0int::paths;
 use sn0int::psl::PslReader;
 use sn0int::registry;
 use sn0int::repl;
@@ -22,7 +20,6 @@ use std::fs::OpenOptions;
 use std::io::Write;
 use std::path::Path;
 
-
 fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
     let mut rl = shell::init(gargs, config, false)?;
 
@@ -37,7 +34,7 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         Module::load(&path.to_path_buf(), "anonymous", &filename, true)
             .context(format!("Failed to parse {:?}", path))?
     } else {
-        rl.engine().get(&args.module)?
+        rl.library().get(&args.module)?
             .clone()
     };
 
@@ -50,17 +47,22 @@ fn run_run(gargs: &Args, args: &args::Run, config: &Config) -> Result<()> {
         rl.set_target(Some(target));
     }
 
-    cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+    if args.dump_sandbox_init_msg {
+        cmd::run_cmd::dump_sandbox_init_msg(&mut rl, args.into(), Opt::collect(&args.options))
+    } else {
+        cmd::run_cmd::execute(&mut rl, args.into(), Opt::collect(&args.options))
+    }
 }
 
 fn run_sandbox() -> Result<()> {
-    let geoip = GeoIP::open_reader()?;
-    let asn = AsnDB::open_reader()?;
-    let psl = PslReader::open()?;
+    let cache_dir = paths::cache_dir()?;
+    let geoip = GeoIP::try_open_reader(&cache_dir)?;
+    let asn = AsnDB::try_open_reader(&cache_dir)?;
+    let psl = PslReader::open(&cache_dir)?;
 
     sandbox::init()
         .context("Failed to init sandbox")?;
-    engine::isolation::run_worker(geoip, asn, psl)
+    ipc::child::run(geoip, asn, psl)
 }
 
 fn run_cmd<T: cmd::Cmd>(gargs: &Args, args: T, config: &Config) -> Result<()> {
@@ -105,20 +107,26 @@ fn run() -> Result<()> {
         Some(SubCommand::Login(_)) => auth::run_login(&config),
         Some(SubCommand::New(new)) => run_new(&args, &new),
         Some(SubCommand::Publish(publish)) => registry::run_publish(&args, &publish, &config),
-        Some(SubCommand::Install(install)) => registry::run_install(install, &config),
-        Some(SubCommand::Search(search)) => {
-            let engine = Engine::new(false, &config)?;
-            registry::run_search(&engine, &search, &config)
-        },
+        // TODO: deprecate
+        Some(SubCommand::Install(install)) => cmd::pkg_cmd::Args {
+            subcommand: cmd::pkg_cmd::SubCommand::Install(install),
+        }.run(&config),
+        // TODO: deprecate
+        Some(SubCommand::Search(search)) => cmd::pkg_cmd::Args {
+            subcommand: cmd::pkg_cmd::SubCommand::Search(search),
+        }.run(&config),
+        Some(SubCommand::Pkg(pkg)) => pkg.run(&config),
         Some(SubCommand::Add(add)) => run_cmd(&args, add, &config),
         Some(SubCommand::Select(select)) => run_cmd(&args, select, &config),
         Some(SubCommand::Delete(delete)) => run_cmd(&args, delete, &config),
+        Some(SubCommand::Activity(activity)) => run_cmd(&args, activity, &config),
         Some(SubCommand::Scope(scope)) => run_cmd(&args, scope, &config),
         Some(SubCommand::Noscope(noscope)) => run_cmd(&args, noscope, &config),
-        Some(SubCommand::Workspace(workspace)) => run_cmd(&args, workspace, &config),
+        Some(SubCommand::Workspace(workspace)) => workspace.run(&config),
         Some(SubCommand::Fsck(fsck)) => run_cmd(&args, fsck, &config),
         Some(SubCommand::Export(export)) => run_cmd(&args, export, &config),
         Some(SubCommand::Repl) => repl::run(&config),
+        Some(SubCommand::Paths) => paths::run(&config),
         Some(SubCommand::Completions(completions)) => complete::run_generate(&completions),
         None => shell::run(&args, &config),
     }

src/models/account.rs

@@ -179,12 +179,14 @@ impl DisplayableDetailed for DetailedAccount {
 
         w.start_group();
         w.opt_debug::<Yellow, _>(&self.displayname)?;
-        w.opt_debug::<Yellow, _>(&self.email)?;
         w.opt_debug::<Yellow, _>(&self.url)?;
-        w.opt_debug::<Yellow, _>(&self.last_seen)?;
-        w.opt_debug::<Yellow, _>(&self.birthday)?;
-        w.opt_debug::<Yellow, _>(&self.phonenumber)?;
-        w.opt_debug::<Yellow, _>(&self.profile_pic)?;
+        w.end_group()?;
+        w.start_group();
+        w.opt_debug_label::<Yellow, _>("last_seen", &self.last_seen)?;
+        w.opt_debug_label::<Yellow, _>("email", &self.email)?;
+        w.opt_debug_label::<Yellow, _>("phonenumber", &self.phonenumber)?;
+        w.opt_debug_label::<Yellow, _>("birthday", &self.birthday)?;
+        w.opt_debug_label::<Yellow, _>("profile_pic", &self.profile_pic)?;
         w.end_group()?;
 
         Ok(())

src/models/activity.rs

@@ -0,0 +1,185 @@
+use crate::errors::*;
+use crate::schema::activity;
+use diesel;
+use diesel::prelude::*;
+use diesel::query_builder::BoxedSelectStatement;
+use crate::models::*;
+use chrono::NaiveDateTime;
+use std::convert::TryFrom;
+use std::io::Write;
+
+
+#[derive(Identifiable, Queryable, Serialize, Deserialize, PartialEq, Debug)]
+#[table_name="activity"]
+pub struct Activity {
+    pub id: i32,
+    pub topic: String,
+    pub time: NaiveDateTime,
+    pub uniq: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub radius: Option<i32>,
+    pub content: String,
+}
+
+impl Activity {
+    pub fn uniq(db: &Database, my_uniq: &str) -> Result<Option<Self>> {
+        use crate::schema::activity::dsl::*;
+        activity.filter(uniq.eq(my_uniq))
+            .first::<Self>(db.db())
+            .optional()
+            .map_err(Error::from)
+    }
+
+    fn build_query_except_since(filter: &ActivityFilter) -> BoxedSelectStatement<(diesel::sql_types::Integer, diesel::sql_types::Text, diesel::sql_types::Timestamp, diesel::sql_types::Nullable<diesel::sql_types::Text>, diesel::sql_types::Nullable<diesel::sql_types::Float>, diesel::sql_types::Nullable<diesel::sql_types::Float>, diesel::sql_types::Nullable<diesel::sql_types::Integer>, diesel::sql_types::Text), activity::table, diesel::sqlite::Sqlite> {
+        use crate::schema::activity::dsl::*;
+
+        let mut query = activity.into_boxed();
+
+        if let Some(my_topic) = &filter.topic {
+            query = query.filter(topic.eq(my_topic));
+        }
+
+        // "since" filter is not applied
+
+        if let Some(until) = &filter.until {
+            query = query.filter(time.le(until));
+        }
+
+        if filter.location {
+            query = query
+                .filter(latitude.is_not_null())
+                .filter(longitude.is_not_null());
+        }
+
+        query
+    }
+
+    pub fn query(db: &Database, filter: &ActivityFilter) -> Result<Vec<Self>> {
+        use crate::schema::activity::dsl::*;
+
+        let mut query = Self::build_query_except_since(filter);
+
+        if let Some(since) = &filter.since {
+            query = query.filter(time.ge(since));
+        }
+
+        query
+            .order_by((time.asc(), id.asc()))
+            .load::<Self>(db.db())
+            .map_err(Error::from)
+    }
+
+    pub fn previous(db: &Database, previous: &Activity, filter: &ActivityFilter) -> Result<Option<Self>> {
+        use crate::schema::activity::dsl::*;
+
+        Self::build_query_except_since(filter)
+            .filter(time.lt(previous.time))
+            .order_by((time.desc(), id.desc()))
+            .first::<Self>(db.db())
+            .optional()
+            .map_err(Error::from)
+    }
+}
+
+pub struct ActivityFilter {
+    pub topic: Option<String>,
+    pub since: Option<NaiveDateTime>,
+    pub until: Option<NaiveDateTime>,
+    pub location: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct JsonActivity {
+    pub id: i32,
+    pub initial: bool,
+    pub topic: String,
+    pub time: NaiveDateTime,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub uniq: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub latitude: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub longitude: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub radius: Option<i32>,
+    pub content: serde_json::Value,
+}
+
+impl JsonActivity {
+    pub fn write_to<W: Write>(self, mut w: W) -> Result<()> {
+        let s = serde_json::to_string(&self)?;
+        writeln!(w, "{}", s)?;
+        Ok(())
+    }
+}
+
+impl TryFrom<Activity> for JsonActivity {
+    type Error = Error;
+
+    fn try_from(a: Activity) -> Result<Self> {
+        let content = serde_json::from_str(&a.content)?;
+
+        Ok(JsonActivity {
+            id: a.id,
+            initial: false,
+            topic: a.topic,
+            time: a.time,
+            uniq: a.uniq,
+            latitude: a.latitude,
+            longitude: a.longitude,
+            radius: a.radius,
+            content,
+        })
+    }
+}
+
+#[derive(Debug, Clone, Insertable, Serialize, Deserialize)]
+#[table_name="activity"]
+pub struct NewActivity {
+    pub topic: String,
+    pub time: NaiveDateTime,
+    pub uniq: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub radius: Option<i32>,
+    pub content: String,
+}
+
+impl NewActivity {
+    pub fn insert(&self, db: &Database) -> Result<()> {
+        diesel::insert_into(activity::table)
+            .values(self)
+            .execute(db.db())?;
+        Ok(())
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct InsertActivity {
+    pub topic: String,
+    pub time: NaiveDateTime,
+    pub uniq: Option<String>,
+    pub latitude: Option<f32>,
+    pub longitude: Option<f32>,
+    pub radius: Option<i32>,
+    pub content: serde_json::Value,
+}
+
+impl InsertToNew for InsertActivity {
+    type Target = NewActivity;
+
+    #[inline]
+    fn try_into_new(self) -> Result<NewActivity> {
+        let content = serde_json::to_string(&self.content)?;
+        Ok(NewActivity {
+            topic: self.topic,
+            time: self.time,
+            uniq: self.uniq,
+            latitude: self.latitude,
+            longitude: self.longitude,
+            radius: self.radius,
+            content,
+        })
+    }
+}

src/models/breach_email.rs

@@ -118,11 +118,16 @@ impl BreachEmail {
 pub struct PrintableBreachEmail {
     breach: String,
     email: String,
+    password: Option<String>,
 }
 
 impl fmt::Display for PrintableBreachEmail {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        write!(w, "{:?} -> {:?}", self.breach, self.email)
+        write!(w, "{:?} -> {:?}", self.breach, self.email)?;
+        if let Some(password) = &self.password {
+            write!(w, " ({:?})", password)?;
+        }
+        Ok(())
     }
 }
 
@@ -133,6 +138,7 @@ impl Printable<PrintableBreachEmail> for BreachEmail {
         Ok(PrintableBreachEmail {
             breach: breach.value.to_string(),
             email: email.value.to_string(),
+            password: self.password.clone(),
         })
     }
 }
@@ -163,6 +169,7 @@ impl Printable<PrintableBreachEmail> for NewBreachEmail {
         Ok(PrintableBreachEmail {
             breach: breach.value.to_string(),
             email: email.value.to_string(),
+            password: self.password.clone(),
         })
     }
 }

src/models/cryptoaddr.rs

@@ -92,20 +92,20 @@ impl Model for CryptoAddr {
     fn get(db: &Database, query: &Self::ID) -> Result<Self> {
         use crate::schema::cryptoaddrs::dsl::*;
 
-        let netblock = cryptoaddrs.filter(value.eq(query))
+        let cryptoaddr = cryptoaddrs.filter(value.eq(query))
             .first::<Self>(db.db())?;
 
-        Ok(netblock)
+        Ok(cryptoaddr)
     }
 
     fn get_opt(db: &Database, query: &Self::ID) -> Result<Option<Self>> {
         use crate::schema::cryptoaddrs::dsl::*;
 
-        let netblock = cryptoaddrs.filter(value.eq(query))
+        let cryptoaddr = cryptoaddrs.filter(value.eq(query))
             .first::<Self>(db.db())
             .optional()?;
 
-        Ok(netblock)
+        Ok(cryptoaddr)
     }
 }
 
@@ -165,14 +165,14 @@ pub struct DetailedCryptoAddr {
 }
 
 #[inline]
-fn add_currency(w: &mut fmt::DetailFormatter, num: &Option<i64>, denominator: &Option<i32>) -> fmt::Result {
+fn add_currency(w: &mut fmt::DetailFormatter, label: &str, num: &Option<i64>, denominator: &Option<i32>) -> fmt::Result {
     if let Some(&num) = num.as_ref() {
         let denominator = denominator.unwrap_or(0);
         let display = display_currency(num as u64, denominator as usize);
         if num > 0 {
-            w.display::<Green, _>(display)?;
+            w.display_label::<Green, _>(label, display)?;
         } else {
-            w.display::<Red, _>(display)?;
+            w.display_label::<Red, _>(label, display)?;
         }
     }
     Ok(())
@@ -191,14 +191,14 @@ impl DisplayableDetailed for DetailedCryptoAddr {
 
         w.start_group();
         w.opt_debug::<Yellow, _>(&self.currency)?;
-        add_currency(w, &self.balance, &self.denominator)?;
-        add_currency(w, &self.received, &self.denominator)?;
+        add_currency(w, "balance", &self.balance, &self.denominator)?;
+        add_currency(w, "received", &self.received, &self.denominator)?;
         w.end_group()?;
 
         w.start_group();
-        w.opt_debug::<Yellow, _>(&self.first_seen)?;
-        w.opt_debug::<Yellow, _>(&self.last_withdrawal)?;
-        w.opt_debug::<Yellow, _>(&self.description)?;
+        w.opt_debug_label::<Yellow, _>("first_seen", &self.first_seen)?;
+        w.opt_debug_label::<Yellow, _>("last_withdrawal", &self.last_withdrawal)?;
+        w.opt_debug_label::<Yellow, _>("description", &self.description)?;
         w.end_group()?;
 
         Ok(())

src/models/device.rs

@@ -171,15 +171,15 @@ impl DisplayableDetailed for DetailedDevice {
         w.debug::<Green, _>(&self.value)?;
 
         w.start_group();
-        w.opt_debug::<Yellow, _>(&self.name)?;
-        w.opt_debug::<Yellow, _>(&self.hostname)?;
-        w.opt_debug::<Yellow, _>(&self.vendor)?;
-        w.opt_debug::<Yellow, _>(&self.last_seen)?;
+        w.opt_debug_label::<Yellow, _>("name", &self.name)?;
+        w.opt_debug_label::<Yellow, _>("hostname", &self.hostname)?;
+        w.opt_debug_label::<Yellow, _>("vendor", &self.vendor)?;
+        w.opt_debug_label::<Yellow, _>("last_seen", &self.last_seen)?;
         w.end_group()?;
 
         w.start_group();
-        w.opt_debug::<Yellow, _>(&self.network)?;
-        w.opt_debug::<Yellow, _>(&self.ipaddr)?;
+        w.opt_debug_label::<Yellow, _>("network", &self.network)?;
+        w.opt_debug_label::<Yellow, _>("ipaddr", &self.ipaddr)?;
         w.end_group()?;
 
         Ok(())

src/models/mod.rs

@@ -424,3 +424,6 @@ pub use self::port::*;
 
 mod cryptoaddr;
 pub use self::cryptoaddr::*;
+
+mod activity;
+pub use self::activity::*;

src/models/netblock.rs

@@ -129,7 +129,7 @@ pub struct PrintableNetblock {
 
 impl fmt::Display for PrintableNetblock {
     fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        write!(w, "{:?}", self.value) // TODO: cidr type
+        write!(w, "{}", self.value)
     }
 }
 

src/models/port.rs

@@ -17,7 +17,7 @@ pub struct Port {
     pub ip_addr: String,
     pub port: i32,
     pub protocol: String,
-    pub status: String,
+    pub status: Option<String>,
     pub unscoped: bool,
 
     pub banner: Option<String>,
@@ -147,7 +147,7 @@ impl Printable<PrintablePort> for Port {
 pub struct DetailedPort {
     id: i32,
     value: String,
-    status: String,
+    status: Option<String>,
     unscoped: bool,
 
     banner: Option<String>,
@@ -210,7 +210,7 @@ pub struct NewPort {
     pub ip_addr: String,
     pub port: i32,
     pub protocol: String,
-    pub status: String,
+    pub status: Option<String>,
 
     pub banner: Option<String>,
     pub service: Option<String>,
@@ -242,7 +242,7 @@ impl Upsertable<Port> for NewPort {
     fn upsert(self, existing: &Port) -> Self::Update {
         Self::Update {
             id: existing.id,
-            status: Self::upsert_opt(Some(self.status), &Some(existing.status.clone())),
+            status: Self::upsert_opt(self.status, &existing.status),
             banner: Self::upsert_opt(self.banner, &existing.banner),
             service: Self::upsert_opt(self.service, &existing.service),
             version: Self::upsert_opt(self.version, &existing.version),
@@ -264,7 +264,7 @@ pub struct InsertPort {
     pub ip_addr: net::IpAddr,
     pub port: i32,
     pub protocol: String,
-    pub status: String,
+    pub status: Option<String>,
 
     pub banner: Option<String>,
     pub service: Option<String>,
@@ -278,10 +278,11 @@ impl InsertToNew for InsertPort {
         let addr = SocketAddr::new(self.ip_addr, self.port as u16);
         let value = format!("{}/{}", self.protocol, addr);
 
-        match self.status.as_str() {
-            "open" => (),
-            "closed" => (),
-            s => bail!("unsupported port status: {:?}", s),
+        match self.status.as_deref() {
+            Some("open") => (),
+            Some("closed") => (),
+            Some(s) => bail!("unsupported port status: {:?}", s),
+            None => (),
         }
 
         Ok(NewPort {
@@ -330,7 +331,7 @@ impl Upsert for PortUpdate {
 
 impl Updateable<Port> for PortUpdate {
     fn changeset(&mut self, existing: &Port) {
-        Self::clear_if_equal(&mut self.status, &Some(existing.status.clone()));
+        Self::clear_if_equal(&mut self.status, &existing.status);
         Self::clear_if_equal(&mut self.banner, &existing.banner);
         Self::clear_if_equal(&mut self.service, &existing.service);
         Self::clear_if_equal(&mut self.version, &existing.version);

src/paths.rs

@@ -1,3 +1,4 @@
+use crate::config::Config;
 use crate::errors::*;
 use crate::workspaces::Workspace;
 
@@ -6,7 +7,7 @@ use std::fs;
 use std::path::PathBuf;
 
 
-pub fn data_dir() -> Result<PathBuf> {
+pub fn sn0int_dir() -> Result<PathBuf> {
     let path = dirs::data_dir()
         .ok_or_else(|| format_err!("Failed to find data directory"))?;
     let path = path.join("sn0int");
@@ -16,24 +17,39 @@ pub fn data_dir() -> Result<PathBuf> {
 }
 
 pub fn history_path() -> Result<PathBuf> {
-    let path = data_dir()?;
+    let path = sn0int_dir()?;
     let path = path.join("history");
     Ok(path)
 }
 
 pub fn module_dir() -> Result<PathBuf> {
-    let path = data_dir()?;
+    let path = sn0int_dir()?;
     let path = path.join("modules");
     fs::create_dir_all(&path)
         .context("Failed to create module directory")?;
     Ok(path)
 }
 
+pub fn data_dir() -> Result<PathBuf> {
+    let path = sn0int_dir()?
+        .join("data");
+    fs::create_dir_all(&path)
+        .context("Failed to create module directory")?;
+    Ok(path)
+}
+
+pub fn workspace_dir(workspace: &Workspace) -> Result<PathBuf> {
+    let path = sn0int_dir()?
+        .join("data")
+        .join(workspace.as_str());
+    fs::create_dir_all(&path)
+        .context("Failed to create module directory")?;
+    Ok(path)
+}
+
 pub fn blobs_dir(workspace: &Workspace) -> Result<PathBuf> {
-    let path = data_dir()?;
-    let path = path
-        .join("blobs")
-        .join(workspace.to_string());
+    let path = workspace_dir(workspace)?
+        .join("blobs");
     fs::create_dir_all(&path)
         .context("Failed to create module directory")?;
     Ok(path)
@@ -47,3 +63,20 @@ pub fn cache_dir() -> Result<PathBuf> {
         .context("Failed to create cache directory")?;
     Ok(path)
 }
+
+fn print_path<D: std::fmt::Debug>(k: &str, v: D) {
+    println!("{:30}: {:?}", k, v);
+}
+
+pub fn run(config: &Config) -> Result<()> {
+    print_path("config_file", Config::path()?);
+    print_path("data_dir", data_dir()?);
+    print_path("module_dir", module_dir()?);
+
+    for (k, v) in &config.namespaces {
+        print_path(&format!("modules({})", k), v);
+    }
+
+    print_path("cache_dir", cache_dir()?);
+    Ok(())
+}

src/registry.rs

@@ -3,7 +3,7 @@ use crate::args::{Args, Publish, Install, Search};
 use crate::api::Client;
 use crate::auth;
 use crate::config::Config;
-use crate::engine::{Engine, Module};
+use crate::engine::{Library, Module};
 use colored::Colorize;
 use separator::Separatable;
 use sn0int_common::ModuleID;
@@ -242,7 +242,7 @@ impl Task for UpdateTask {
     }
 }
 
-pub fn run_search(engine: &Engine, search: &Search, config: &Config) -> Result<()> {
+pub fn run_search(library: &Library, search: &Search, config: &Config) -> Result<()> {
     let client = Client::new(&config)?;
 
     let label = format!("Searching {:?}", search.query);
@@ -252,7 +252,7 @@ pub fn run_search(engine: &Engine, search: &Search, config: &Config) -> Result<(
 
     for module in &modules {
         let canonical = module.canonical();
-        let installed = engine.get_opt(&canonical)?;
+        let installed = library.get_opt(&canonical)?;
 
         if search.new && installed.is_some() {
             continue;

src/repl/complete.rs

@@ -1,3 +1,4 @@
+use crate::repl::tokenize::{self, Token};
 use rustyline::Context;
 use rustyline::completion::Completer;
 use rustyline::highlight::Highlighter;
@@ -5,21 +6,43 @@ use rustyline::hint::Hinter;
 use std::borrow::Cow;
 
 
-pub struct ReplCompleter;
+#[derive(Default)]
+pub struct ReplCompleter {
+    globals: Vec<String>,
+}
+
+impl ReplCompleter {
+    pub fn set(&mut self, globals: Vec<String>) {
+        self.globals = globals;
+    }
+}
 
 impl rustyline::Helper for ReplCompleter {}
+impl rustyline::validate::Validator for ReplCompleter {}
 
 impl Completer for ReplCompleter {
     type Candidate = String;
 
     #[inline]
-    fn complete(&self, _line: &str, pos: usize, _ctx: &Context<'_>) -> rustyline::Result<(usize, Vec<String>)> {
+    fn complete(&self, line: &str, pos: usize, _ctx: &Context<'_>) -> rustyline::Result<(usize, Vec<String>)> {
         if pos == 0 {
             Ok((0, vec![
                 String::from("return "),
             ]))
         } else {
-            Ok((0, vec![]))
+            let filter = match tokenize::parse_last(&line[..pos]) {
+                Token::Name(name) => name,
+                Token::Empty => String::new(),
+                _ => return Ok((0, vec![])),
+            };
+
+            let mut options = Vec::new();
+            for g in &self.globals {
+                if g.starts_with(&filter) {
+                    options.push(g.to_string());
+                }
+            }
+            Ok((pos - filter.len(), options))
         }
     }
 }

src/repl/mod.rs

@@ -1,29 +1,97 @@
 use crate::config::Config;
 use crate::errors::*;
-use crate::engine::{ctx, Environment, DummyReporter};
-use crate::engine::ctx::State;
+use crate::engine::{ctx, Environment};
+use crate::ipc::child::DummyIpcChild;
+use crate::engine::ctx::{State, LuaState};
 use crate::geoip::{Maxmind, AsnDB, GeoIP};
-use crate::hlua::AnyLuaValue;
+use crate::hlua::{Lua, AnyLuaValue};
+use crate::paths;
 use crate::psl::PslReader;
 use crate::shell::readline::Readline;
 use crate::runtime::format_lua;
 use chrootable_https::Resolver;
 use std::collections::HashMap;
+use std::sync::Arc;
 
 mod complete;
 use self::complete::ReplCompleter;
+mod tokenize;
 
+pub struct Repl<'a> {
+    rl: Readline<ReplCompleter>,
+    lua: Lua<'a>,
+    state: Arc<LuaState>,
+}
+
+impl<'a> Repl<'a> {
+    pub fn new(lua: Lua<'a>, state: Arc<LuaState>) -> Repl<'a> {
+        let rl = Readline::with(ReplCompleter::default());
+        Repl {
+            rl,
+            lua,
+            state,
+        }
+    }
+
+    fn update_globals(&mut self) {
+        let mut globals = Vec::new();
+        for item in self.lua.globals_table().iter::<String, AnyLuaValue>() {
+            if let Some((k, _)) = item {
+                globals.push(k);
+            }
+        }
+        if let Some(helper) = self.rl.helper_mut() {
+            debug!("updating globals: {:?}", globals);
+            helper.set(globals);
+        }
+    }
+
+    pub fn run(&mut self) {
+        loop {
+            self.update_globals();
+
+            match self.rl.readline("> ") {
+                Ok(line) => {
+                    self.rl.add_history_entry(line.as_str());
+                    self.exec(&line);
+                },
+                Err(_) => break,
+            }
+        }
+    }
+
+    pub fn exec(&mut self, line: &str) {
+        match self.lua.execute::<AnyLuaValue>(line) {
+            Ok(val) => {
+                if val != AnyLuaValue::LuaNil {
+                    let mut out = String::new();
+                    format_lua(&mut out, &val);
+                    println!("{}", out);
+                }
+                if let Some(err) = self.state.last_error() {
+                    println!("Error: {}", err);
+                    self.state.clear_error();
+                }
+            },
+            Err(err) => {
+                println!("Fatal: {}", err);
+            }
+        }
+    }
+}
 
 pub fn run(config: &Config) -> Result<()> {
     let keyring = Vec::new();
     let dns_config = Resolver::from_system()?;
     let proxy = config.network.proxy;
-    let psl = PslReader::open()?;
-    let geoip = GeoIP::open_reader()?;
-    let asn = AsnDB::open_reader()?;
+
+    let cache_dir = paths::cache_dir()?;
+    let psl = PslReader::open(&cache_dir)?;
+    let geoip = GeoIP::try_open_reader(&cache_dir)?;
+    let asn = AsnDB::try_open_reader(&cache_dir)?;
 
     let env = Environment {
-        verbose: 0, // this doesn't do anything since we use a dummy reporter
+        verbose: 0, // this doesn't do anything since we use a dummy ipc child
         keyring,
         dns_config,
         proxy,
@@ -34,40 +102,16 @@ pub fn run(config: &Config) -> Result<()> {
         asn,
     };
 
-    let tx = DummyReporter::new();
-    let (mut lua, state) = ctx::ctx(env, tx);
+    let tx = DummyIpcChild::new();
+    let (lua, state) = ctx::ctx(env, tx);
+    let mut repl = Repl::new(lua, state);
 
     println!(r#":: sn0int v{} lua repl
 Assign variables with `a = sn0int_version()` and `return a` to print
 Read the docs at https://sn0int.readthedocs.io/en/stable/reference.html
 "#, env!("CARGO_PKG_VERSION"));
 
-    let mut rl = Readline::with(ReplCompleter);
-    loop {
-        match rl.readline("> ") {
-            Ok(line) => {
-                rl.add_history_entry(line.as_str());
-
-                match lua.execute::<AnyLuaValue>(&line) {
-                    Ok(val) => {
-                        if val != AnyLuaValue::LuaNil {
-                            let mut out = String::new();
-                            format_lua(&mut out, &val);
-                            println!("{}", out);
-                        }
-                        if let Some(err) = state.last_error() {
-                            println!("Error: {}", err);
-                            state.clear_error();
-                        }
-                    },
-                    Err(err) => {
-                        println!("Fatal: {}", err);
-                    }
-                }
-            },
-            Err(_) => break,
-        }
-    }
+    repl.run();
 
     Ok(())
 }

src/repl/tokenize.rs

@@ -0,0 +1,119 @@
+#[derive(Debug, PartialEq)]
+pub enum Token {
+    Empty,
+    Name(String),
+    InString(StringState),
+    Value(String),
+}
+
+#[derive(Debug, PartialEq)]
+pub struct StringState {
+    quote: char,
+    escape: bool,
+    buf: String,
+}
+
+impl StringState {
+    pub fn new(quote: char) -> StringState {
+        StringState {
+            quote,
+            escape: false,
+            buf: String::new(),
+        }
+    }
+}
+
+pub fn parse_last(line: &str) -> Token {
+    let mut token = Token::Empty;
+
+    for c in line.chars() {
+        match &mut token {
+            Token::Empty => {
+                match c {
+                    'a'..='z' | 'A'..='Z' | '_' => {
+                        token = Token::Name(c.to_string());
+                    },
+                    '0'..='9' => {
+                        token = Token::Value(c.to_string());
+                    },
+                    '\'' | '"' => {
+                        token = Token::InString(StringState::new(c));
+                    },
+                    // ignore operators
+                    _ => (),
+                }
+            },
+            Token::Name(s) => {
+                match c {
+                    'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => {
+                        s.push(c);
+                    },
+                    // ignore operators
+                    _ => {
+                        token = Token::Empty;
+                    },
+                }
+            },
+            Token::InString(s) => {
+                if s.escape {
+                    s.buf.push(c);
+                    s.escape = false;
+                } else if c == '\\' {
+                    s.escape = true;
+                } else if c == s.quote {
+                    // done
+                    token = Token::Empty;
+                } else {
+                    s.buf.push(c);
+                }
+            },
+            Token::Value(s) => {
+                match c {
+                    '0'..='9' => {
+                        s.push(c);
+                    },
+                    // ignore operators
+                    // a name can't directly follow a value
+                    _ => {
+                        token = Token::Empty;
+                    },
+                }
+            },
+        }
+    }
+
+    token
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    pub fn test_empty() {
+        let token = parse_last("");
+        assert_eq!(token, Token::Empty);
+    }
+
+    #[test]
+    pub fn test_abc() {
+        let token = parse_last("abc");
+        assert_eq!(token, Token::Name("abc".into()));
+    }
+
+    #[test]
+    pub fn test_in_string() {
+        let token = parse_last("return url_encode(\"asdf");
+        assert_eq!(token, Token::InString(StringState {
+            quote: '"',
+            escape: false,
+            buf: "asdf".into(),
+        }));
+    }
+
+    #[test]
+    pub fn test_in_func() {
+        let token = parse_last("return url_encode(\"asdf\") .. url_deco");
+        assert_eq!(token, Token::Name("url_deco".into()));
+    }
+}

src/runtime/blobs.rs

@@ -1,12 +1,14 @@
 use crate::errors::*;
-use crate::blobs::Blob;
 use crate::engine::ctx::State;
 use crate::engine::structs::byte_array;
 use crate::hlua::{self, AnyLuaValue};
+use sn0int_std::blobs::{Blob, BlobState};
 use std::sync::Arc;
 
 
-pub fn create_blob(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+pub fn create_blob<S>(lua: &mut hlua::Lua, state: Arc<S>)
+    where S: State + BlobState + 'static
+{
     lua.set("create_blob", hlua::function1(move |bytes: AnyLuaValue| -> Result<String> {
         let bytes = byte_array(bytes)
             .map_err(|err| state.set_error(err))?;

src/runtime/db.rs

@@ -8,6 +8,7 @@ use crate::hlua::{self, AnyLuaValue};
 use std::str::FromStr;
 use std::sync::Arc;
 use crate::models::*;
+use crate::worker::DatabaseResponse;
 use crate::json::LuaJsonValue;
 
 
@@ -85,8 +86,18 @@ pub fn db_add(lua: &mut hlua::Lua, state: Arc<dyn State>) {
         let object = into_insert(family, object, &state)
             .map_err(|e| state.set_error(e))?;
 
-        state.db_insert(object)
-            .map_err(|e| state.set_error(e))
+        let r = state.db_insert(object)
+            .map_err(|e| state.set_error(e))?;
+
+        let r = match r {
+            // Found is technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
     }))
 }
 
@@ -99,7 +110,29 @@ pub fn db_add_ttl(lua: &mut hlua::Lua, state: Arc<dyn State>) {
         let object = into_insert(family, object, &state)
             .map_err(|e| state.set_error(e))?;
 
-        state.db_insert_ttl(object, ttl)
+        let r = state.db_insert_ttl(object, ttl)
+            .map_err(|e| state.set_error(e))?;
+
+        let r = match r {
+            // Found is technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
+    }))
+}
+
+pub fn db_activity(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("db_activity", hlua::function1(move |v: AnyLuaValue| -> Result<bool> {
+        let v: LuaJsonValue = v.into();
+        let v: serde_json::Value = v.into();
+        let activity: InsertActivity = serde_json::from_value(v)
+            .map_err(|e| state.set_error(e.into()))?;
+
+        state.db_activity(activity)
             .map_err(|e| state.set_error(e))
     }))
 }
@@ -109,8 +142,18 @@ pub fn db_select(lua: &mut hlua::Lua, state: Arc<dyn State>) {
         let family = Family::from_str(&family)
             .map_err(|e| state.set_error(e.into()))?;
 
-        state.db_select(family, value)
-            .map_err(|e| state.set_error(e))
+        let r = state.db_select(family, value)
+            .map_err(|e| state.set_error(e))?;
+
+        let r = match r {
+            // Everything except Found and None are technically unreachable
+            DatabaseResponse::Inserted(id) => Some(id),
+            DatabaseResponse::Updated(id) => Some(id),
+            DatabaseResponse::NoChange(id) => Some(id),
+            DatabaseResponse::Found(id) => Some(id),
+            DatabaseResponse::None => None,
+        };
+        Ok(r)
     }))
 }
 
@@ -179,8 +222,18 @@ pub fn db_update(lua: &mut hlua::Lua, state: Arc<dyn State>) {
             .map_err(|e| state.set_error(e))?;
 
         if update.is_dirty() {
-            state.db_update(value, update)
-                .map_err(|e| state.set_error(e))
+            let r = state.db_update(value, update)
+                .map_err(|e| state.set_error(e))?;
+
+            let r = match r {
+                // Inserted and Found are technically unreachable
+                DatabaseResponse::Inserted(id) => Some(id),
+                DatabaseResponse::Updated(id) => Some(id),
+                DatabaseResponse::NoChange(id) => Some(id),
+                DatabaseResponse::Found(id) => Some(id),
+                DatabaseResponse::None => None,
+            };
+            Ok(r)
         } else {
             Ok(Some(id))
         }

src/runtime/geo.rs

@@ -0,0 +1,126 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::geo::{self, Point};
+use crate::hlua::{self, AnyLuaValue};
+use std::sync::Arc;
+
+
+pub fn geo_polygon_contains(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+    lua.set("geo_polygon_contains", hlua::function2(move |polygon: Vec<AnyLuaValue>, point: AnyLuaValue| -> Result<bool> {
+        let polygon = polygon.into_iter()
+            .map(|p| Point::try_from(p))
+            .collect::<Result<Vec<_>>>()
+            .context("Invalid polygon")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        let point = Point::try_from(point)
+            .context("Invalid point")
+            .map_err(|err| state.set_error(Error::from(err)))?;
+
+        Ok(geo::polygon_contains(&polygon, &point))
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::engine::ctx::Script;
+
+    fn hamburg_polygon() -> &'static str {
+        "
+            { lat=53.63975308945899,  lon=9.764785766601562  },
+            { lat=53.59494998253459,  lon=9.827270507812     },
+            { lat=53.663153974456456, lon=9.9151611328125    },
+            { lat=53.65582987649682,  lon=9.976272583007812  },
+            { lat=53.68613523817129,  lon=9.992752075195312  },
+            { lat=53.68674518938816,  lon=10.051460266113281 },
+            { lat=53.72495117617815,  lon=10.075492858886719 },
+            { lat=53.71946627930625,  lon=10.118408203125    },
+            { lat=53.743635083157756, lon=10.164413452148438 },
+            { lat=53.73104466704585,  lon=10.202865600585938 },
+            { lat=53.676781546441546, lon=10.16304016113281  },
+            { lat=53.632832079199474, lon=10.235824584960938 },
+            { lat=53.608803292930894, lon=10.2008056640625   },
+            { lat=53.578646152866504, lon=10.208358764648438 },
+            { lat=53.57212285981298,  lon=10.163726806640625 },
+            { lat=53.52071674896369,  lon=10.18707275390625  },
+            { lat=53.52643162253097,  lon=10.224151611328125 },
+            { lat=53.44062753992289,  lon=10.347747802734375 },
+            { lat=53.38824275010831,  lon=10.248870849609375 },
+            { lat=53.38824275010831,  lon=10.15960693359375  },
+            { lat=53.44635321212876,  lon=10.064849853515625 },
+            { lat=53.40595029739904,  lon=9.985198974609375  },
+            { lat=53.42385506057106,  lon=9.951210021972656  },
+            { lat=53.41843327091211,  lon=9.944171905517578  },
+            { lat=53.41812635648326,  lon=9.927349090576172  },
+            { lat=53.412294561442884, lon=9.917736053466797  },
+            { lat=53.41464783813818,  lon=9.901256561279297  },
+            { lat=53.443490472483326, lon=9.912586212158201  },
+            { lat=53.45177144115704,  lon=9.897651672363281  },
+            { lat=53.43633277935392,  lon=9.866924285888672  },
+            { lat=53.427639673754776, lon=9.866409301757812  },
+            { lat=53.427639673754776, lon=9.858856201171875  },
+            { lat=53.46710230573499,  lon=9.795513153076172  },
+            { lat=53.49039461941655,  lon=9.795341491699219  },
+            { lat=53.49029248806277,  lon=9.77903366088867   },
+            { lat=53.49856433088649,  lon=9.780235290527344  },
+            { lat=53.5078554643033,   lon=9.758434295654297  },
+            { lat=53.545407634092975, lon=9.759807586669922  },
+            { lat=53.568147234570084, lon=9.633293151855469  },
+            { lat=53.58802162343514,  lon=9.655780792236328  },
+            { lat=53.568351121879815, lon=9.727706909179688  },
+            { lat=53.60921067445695,  lon=9.737663269042969  },
+        "
+    }
+
+    #[test]
+    fn verify_polygon_hamburg_contains_hamburg() {
+        let script = Script::load_unchecked(format!(r#"
+        function run()
+            x = geo_polygon_contains({{{}}}, {{
+                lat=53.551085,
+                lon=9.993682,
+            }})
+            if last_err() then return end
+            if not x then
+                return 'hamburg should be in hamburg'
+            end
+        end
+        "#, hamburg_polygon())).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    fn verify_polygon_hamburg_not_contains_berlin() {
+        let script = Script::load_unchecked(format!(r#"
+        function run()
+            x = geo_polygon_contains({{{}}}, {{
+                lat=52.52437,
+                lon=13.41053,
+            }})
+            if last_err() then return end
+            if x then
+                return 'berlin should not be in hamburg'
+            end
+        end
+        "#, hamburg_polygon())).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    fn verify_polygon_hamburg_not_contains_ny() {
+        let script = Script::load_unchecked(format!(r#"
+        function run()
+            x = geo_polygon_contains({{{}}}, {{
+                lat=40.726662,
+                lon=-74.036677,
+            }})
+            if last_err() then return end
+            if x then
+                return 'ny should not be in hamburg'
+            end
+        end
+        "#, hamburg_polygon())).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+}

src/runtime/geoip.rs

@@ -49,6 +49,8 @@ pub fn geoip_lookup(lua: &mut hlua::Lua, state: Arc<dyn State>) {
 
 #[cfg(test)]
 mod tests {
+    // You need geoip setup on your system to run this
+    /*
     use crate::engine::ctx::Script;
 
     #[test]
@@ -72,4 +74,5 @@ mod tests {
         "#).expect("Failed to load script");
         script.test().expect("Script failed");
     }
+    */
 }

src/runtime/http.rs

@@ -3,6 +3,8 @@ use crate::errors::*;
 use crate::engine::ctx::State;
 use crate::hlua::{self, AnyLuaValue, AnyHashableLuaValue};
 use crate::json;
+use sn0int_std::blobs::BlobState;
+use sn0int_std::web::WebState;
 use std::sync::Arc;
 use std::collections::HashMap;
 use crate::web::{RequestOptions, HttpRequest};
@@ -25,7 +27,9 @@ pub fn http_request(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     }))
 }
 
-pub fn http_send(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+pub fn http_send<S>(lua: &mut hlua::Lua, state: Arc<S>)
+    where S: State + WebState + BlobState + 'static
+{
     lua.set("http_send", hlua::function1(move |request: AnyLuaValue| -> Result<HashMap<AnyHashableLuaValue, AnyLuaValue>> {
         let req = HttpRequest::try_from(request)
             .context("invalid http request object")
@@ -40,7 +44,9 @@ pub fn http_send(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     }))
 }
 
-pub fn http_fetch(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+pub fn http_fetch<S>(lua: &mut hlua::Lua, state: Arc<S>)
+    where S: State + WebState + BlobState + 'static
+{
     lua.set("http_fetch", hlua::function1(move |request: AnyLuaValue| -> Result<AnyLuaValue> {
         let req = HttpRequest::try_from(request)
             .context("invalid http request object")
@@ -59,7 +65,9 @@ pub fn http_fetch(lua: &mut hlua::Lua, state: Arc<dyn State>) {
     }))
 }
 
-pub fn http_fetch_json(lua: &mut hlua::Lua, state: Arc<dyn State>) {
+pub fn http_fetch_json<S>(lua: &mut hlua::Lua, state: Arc<S>)
+    where S: State + WebState + 'static
+{
     lua.set("http_fetch_json", hlua::function1(move |request: AnyLuaValue| -> Result<AnyLuaValue> {
         let req = HttpRequest::try_from(request)
             .context("invalid http request object")