Release v0.9.1

Update docs
Merge pull request #71 from kpcyrd/seccomp-i686
2019-02-03 03:46:28 +01:00 · 2019-02-03 03:45:23 +01:00 · 2019-02-03 03:43:08 +01:00 · 2019-02-02 23:37:47 +01:00 · 2019-02-02 22:49:21 +01:00 · 2019-02-02 16:57:39 +01:00
74 changed files with 3009 additions and 1069 deletions
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,7 @@
 language: rust

-cache: cargo
+# upload takes too long on windows and gets killed due to inactivity
+#cache: cargo

 matrix:
  include:
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.8.0"
+version = "0.9.1"
 description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -35,7 +35,7 @@ chrono = { version = "0.4", features = ["serde"] }
 dirs = "1.0"
 url = "1.7"
 #chrootable-https = { path = "../chrootable-https" }
-chrootable-https = "0.7"
+chrootable-https = "0.8"
 base64 = "0.10"
 kuchiki = "0.7.2"
 serde_urlencoded = "0.5"
@@ -50,22 +50,27 @@ maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.2"
 regex = "1.0"
 toml = "0.4"
-maxminddb = "0.12"
+maxminddb = "0.13"
 tar = "0.4.17"
 libflate = "0.1.14"
 threadpool = "1.7"
 x509-parser = "0.4.0"
 der-parser = "1.1.0"
 nom = "4.1.1"
+atty = "0.2"
+bufstream = "0.1.4"
+tokio = "0.1.14"

 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
-syscallz = "0.8"
-nix = "0.12"
+#syscallz = { path="../syscallz-rs" }
+syscallz = "0.10"
+nix = "0.13"

 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.3.1"
 unveil = "0.2.0"

 [dev-dependencies]
+#boxxy = { path = "../boxxy-rs" }
 boxxy = "0.8"
--- a/ISSUE_TEMPLATE.md
+++ b/ISSUE_TEMPLATE.md
@@ -0,0 +1,29 @@
+<!--
+Hello!
+
+If you want to report a bug we added some common questions below that help us analyse your issue faster.
+
+All of these are optional so feel free to remove anything that doesn't apply.
+-->
+
+please describe your issue here
+
+---
+
+## Versions
+
+- **rustc --version:**
+- **cargo --version:**
+- **sn0int --version:**
+- **uname -a:**
+
+## Environment
+
+- **Operating System/Distro:**
+- **Installed from (source/apt/pacman/brew/docker):**
+
+<!--
+Thank you!
+
+We'll try to respond as quickly as possible.
+-->
--- a/README.md
+++ b/README.md
@@ -15,14 +15,14 @@ unified format for followup investigations.

 Among other things, sn0int is currently able to:

- [X] Harvest subdomains from certificate transparency logs
- [X] Harvest subdomains from various passive dns logs
- [X] Sift through subdomain results for publicly accessible websites
- [X] Harvest emails from pgp keyservers
- [X] Enrich ip addresses with ASN and geoip info
- [X] Harvest subdomains from the wayback machine
- [X] Gather information about phonenumbers
- [X] Bruteforce interesting urls
+- Harvest subdomains from certificate transparency logs
+- Harvest subdomains from various passive dns logs
+- Sift through subdomain results for publicly accessible websites
+- Harvest emails from pgp keyservers
+- Enrich ip addresses with ASN and geoip info
+- Harvest subdomains from the wayback machine
+- Gather information about phonenumbers
+- Bruteforce interesting urls

 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource.  None of the investigations listed above are hardcoded
@@ -40,11 +40,12 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/

 - [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
  - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
-  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
-  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
-  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
-  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
  - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
+  - [Debian testing/Debian sid/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-testing-debian-sid-kali)
+  - [Ubuntu/Debian stable](https://sn0int.readthedocs.io/en/latest/install.html#ubuntu-debian-stable)
+  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
+  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
  - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
 - [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
  - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
@@ -55,6 +56,7 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
 - [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
  - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+  - [Reading data from stdin](https://sn0int.readthedocs.io/en/latest/scripting.html#reading-data-from-stdin)
 - [Database](https://sn0int.readthedocs.io/en/latest/database.html)
  - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
  - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
@@ -65,9 +67,17 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
  - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
 - [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
  - [Configuring a proxy](https://sn0int.readthedocs.io/en/latest/config.html#configuring-a-proxy)
+- [Sandbox](https://sn0int.readthedocs.io/en/latest/sandbox.html)
+  - [Linux](https://sn0int.readthedocs.io/en/latest/sandbox.html#linux)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/sandbox.html#openbsd)
+  - [IPC Protocol](https://sn0int.readthedocs.io/en/latest/sandbox.html#ipc-protocol)
+  - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
+  - [Diagnosing a sandbox failure](https://sn0int.readthedocs.io/en/latest/sandbox.html#diagnosing-a-sandbox-failure)
 - [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
  - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [datetime](https://sn0int.readthedocs.io/en/latest/reference.html#datetime)
  - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
+  - [db_add_ttl](https://sn0int.readthedocs.io/en/latest/reference.html#db-add-ttl)
  - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
  - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
  - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
@@ -92,6 +102,18 @@ Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc:/
  - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
  - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
  - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [sock_connect](https://sn0int.readthedocs.io/en/latest/reference.html#sock-connect)
+  - [sock_send](https://sn0int.readthedocs.io/en/latest/reference.html#sock-send)
+  - [sock_recv](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recv)
+  - [sock_sendline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendline)
+  - [sock_recvline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline)
+  - [sock_recvall](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvall)
+  - [sock_recvline_contains](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline-contains)
+  - [sock_recvline_regex](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvline-regex)
+  - [sock_recvn](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvn)
+  - [sock_recvuntil](https://sn0int.readthedocs.io/en/latest/reference.html#sock-recvuntil)
+  - [sock_sendafter](https://sn0int.readthedocs.io/en/latest/reference.html#sock-sendafter)
+  - [sock_newline](https://sn0int.readthedocs.io/en/latest/reference.html#sock-newline)
  - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
  - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
  - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
--- a/ci/boxxy_stage1.txt
+++ b/ci/boxxy_stage1.txt
@@ -1,3 +1,5 @@
 stage1
 ls
+echo checkpoint1
+id
 echo sandbox fail
--- a/ci/integration.py
+++ b/ci/integration.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+import subprocess
+from subprocess import DEVNULL, PIPE
+import tempfile
+import json
+import sys
+
+
+def _sn0int(tempdir, binary, args, piped_stdout=False):
+    return subprocess.Popen(
+        ['/usr/bin/env', 'HOME='+tempdir, binary] + args,
+        stdin=PIPE,
+        stdout=PIPE if piped_stdout else None,
+    )
+
+
+def sn0int(tempdir, binary, cmds):
+    p = _sn0int(tempdir, binary, [])
+    for cmd in cmds:
+        p.stdin.write((cmd + '\n').encode('utf-8'))
+    p.communicate()
+    if p.returncode != 0:
+        raise Exception('process failed')
+
+
+def sn0int_select(tempdir, binary, query):
+    p = _sn0int(tempdir, binary, ['select', '--json'] + query, piped_stdout=True)
+    stdout, _ = p.communicate()
+    lines = filter(None, stdout.decode('utf-8').split('\n'))
+    return [json.loads(x) for x in lines]
+
+
+def main(tempdir, binary):
+    print('[*] setting up workspace')
+    sn0int(tempdir, binary, [])
+
+    print('[*] adding domain')
+    sn0int(tempdir, binary, [
+        'add domain',
+        'example.com',
+        'select domains',
+    ])
+
+    print('[*] testing db for domain')
+    domains = sn0int_select(tempdir, binary, ['domains'])
+    assert domains == [{'id': 1, 'value': 'example.com', 'unscoped': False}]
+
+    print('[*] installing modules')
+    sn0int(tempdir, binary, [
+        'mod install kpcyrd/ctlogs',
+        'mod install kpcyrd/dns-resolve',
+        'mod install kpcyrd/url-scan',
+        'mod install kpcyrd/geoip',
+    ])
+
+    print('[*] running ctlogs')
+    sn0int(tempdir, binary, [
+        'use ctlogs',
+        'run',
+        'select subdomains',
+    ])
+
+    print('[*] testing db for subdomains')
+    subdomains = sn0int_select(tempdir, binary, ['subdomains'])
+    assert {x['value'] for x in subdomains} == {
+        'www.example.com',
+        'm.example.com',
+        'dev.example.com',
+        'products.example.com',
+        'support.example.com',
+    }
+
+    print('[*] running dns-resolve')
+    sn0int(tempdir, binary, [
+        'use dns-resolve',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs')
+    ipaddrs = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert len(ipaddrs) >= 1
+
+    print('[*] running url-scan')
+    sn0int(tempdir, binary, [
+        'use url-scan',
+        'run',
+        'select urls',
+    ])
+
+    print('[*] testing db for urls')
+    urls = sn0int_select(tempdir, binary, ['urls'])
+    assert {(x['value'], x['status']) for x in urls} == {
+        ('http://www.example.com/', 200),
+        ('https://www.example.com/', 200),
+    }
+
+    print('[*] running geoip')
+    sn0int(tempdir, binary, [
+        'use geoip',
+        'run',
+        'select ipaddrs',
+    ])
+
+    print('[*] testing db for ipaddrs again')
+    ipaddrs2 = sn0int_select(tempdir, binary, ['ipaddrs'])
+    assert ipaddrs != ipaddrs2
+
+    print('')
+    print('\t###########')
+    print('\t# SUCCESS #')
+    print('\t###########')
+    print('')
+
+
+if __name__ == '__main__':
+    try:
+        binary = sys.argv[1]
+    except IndexError:
+        print('Usage: %s target/release/sn0int' % sys.argv[0])
+    else:
+        with tempfile.TemporaryDirectory(prefix='sn0int-') as tempdir:
+            main(tempdir, binary)
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -9,8 +9,8 @@ case "$1" in
        ci/run.sh build
        wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
             https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind-dl -- -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind-dl -- -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
+        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
+        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
        cargo test --verbose
        cargo test --verbose -- --ignored
        ;;
--- a/ci/setup.sh
+++ b/ci/setup.sh
@@ -5,9 +5,6 @@ case "$1" in
        sudo apt update
        sudo apt install libsqlite3-dev libseccomp-dev
        ;;
-    osx)
-        brew install sqlite3
-        ;;
    windows)
        curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
        7z e sqlite3.zip -y
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -9,14 +9,14 @@ unified format for followup investigations.

 Among other things, sn0int is currently able to:

- [X] Harvest subdomains from certificate transparency logs
- [X] Harvest subdomains from various passive dns logs
- [X] Sift through subdomain results for publicly accessible websites
- [X] Harvest emails from pgp keyservers
- [X] Enrich ip addresses with ASN and geoip info
- [X] Harvest subdomains from the wayback machine
- [X] Gather information about phonenumbers
- [X] Bruteforce interesting urls
+- Harvest subdomains from certificate transparency logs
+- Harvest subdomains from various passive dns logs
+- Sift through subdomain results for publicly accessible websites
+- Harvest emails from pgp keyservers
+- Enrich ip addresses with ASN and geoip info
+- Harvest subdomains from the wayback machine
+- Gather information about phonenumbers
+- Bruteforce interesting urls

 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource.  None of the investigations listed above are hardcoded
@@ -41,4 +41,5 @@ Getting Started
   database
   keyring
   config
+   sandbox
   reference
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -10,15 +10,44 @@ Archlinux

    $ pacman -S sn0int

-Debian/Ubuntu/Kali
------------------
+Mac OSX
+-------

 .. code-block:: bash

-    $ apt install libsqlite3-dev libseccomp-dev
+    $ brew install sn0int
+
+Debian testing/Debian sid/Kali
+------------------------------
+
+.. code-block:: bash
+
+    $ apt install build-essential cargo libsqlite3-dev libseccomp-dev
    $ git clone https://github.com/kpcyrd/sn0int.git
    $ cd sn0int
-    $ cargo install -f
+    $ cargo install -f --path .
+
+Ubuntu/Debian stable
+--------------------
+
+cargo in the repos is too old and the build is `going to fail
+<https://github.com/kpcyrd/sn0int/issues/68>`_. You should either install the
+most recent rust version with `rustup <https://rustup.rs/>`_ or use the docker
+instructions instead.
+
+.. code-block:: bash
+
+    $ apt install build-essential libsqlite3-dev libseccomp-dev
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f --path .
+
+Docker
+------
+
+.. code-block:: bash
+
+    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int

 Alpine
 ------
@@ -28,14 +57,7 @@ Alpine
    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
    $ git clone https://github.com/kpcyrd/sn0int.git
    $ cd sn0int
-    $ cargo install -f
-
-Docker
------
-
-.. code-block:: bash
-
-    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
+    $ cargo install -f --path .

 OpenBSD
 -------
@@ -45,17 +67,7 @@ OpenBSD
    $ pkg_add sqlite3
    $ git clone https://github.com/kpcyrd/sn0int.git
    $ cd sn0int
-    $ cargo install -f
-
-Mac OSX
-------
-
-.. code-block:: bash
-
-    $ brew install sqlite3
-    $ git clone https://github.com/kpcyrd/sn0int.git
-    $ cd sn0int
-    $ cargo install -f
+    $ cargo install -f --path .

 Windows
 -------
@@ -70,4 +82,4 @@ This is not recommended and only passively maintained. Please prefer linux in a
    $ 7z e sqlite3.zip -y
    $ "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
    $ export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
-    $ cargo install -f
+    $ cargo install -f --path .
--- a/docs/reference.rst
+++ b/docs/reference.rst
@@ -13,6 +13,16 @@ Clear the last recorded error from the internal state. See also last_err_.
        clear_err()
    end

+datetime
+--------
+
+Return current time in UTC. This function is suitable to determine datetimes
+for ``DATETIME`` database fields.
+
+.. code-block:: lua
+
+    now = datetime()
+
 db_add
 ------

@@ -25,6 +35,23 @@ may fail or return ``nil``. See `db_add <database.html#db-add>`__ for details.
        value='example.com',
    })

+db_add_ttl
+----------
+
+Add a temporary entity to the database. This is commonly used to insert
+temporary links that automatically expire over time. If the entity already
+exists and is also marked as temporary the new ttl is going to replace the old
+ttl. If the entity already exists but never expires we are not going to add a
+ttl.
+
+.. code-block:: lua
+
+    -- this link is valid for 2min
+    domain_id = db_add('network-device', {
+        network_id=1,
+        device_id=13,
+    }, 120)
+
 db_select
 ---------

@@ -413,6 +440,116 @@ only used for debugging.

    sleep(1)

+sock_connect
+------------
+
+Create a tcp connection.
+
+.. code-block:: lua
+
+    sock = sock_connect("127.0.0.1", 1337)
+
+sock_send
+---------
+
+Send data to the socket.
+
+.. code-block:: lua
+
+    sock_send(sock, "hello world")
+
+sock_recv
+---------
+
+Receive up to 4096 bytes from the socket.
+
+.. code-block:: lua
+
+    x = sock_recv(sock)
+
+sock_sendline
+-------------
+
+Send a string to the socket. A newline is automatically appended to the string.
+
+.. code-block:: lua
+
+    sock_sendline(sock, line)
+
+sock_recvline
+-------------
+
+Receive a line from the socket. The line includes the newline.
+
+.. code-block:: lua
+
+    x = sock_recvline(sock)
+
+sock_recvall
+------------
+
+Receive all data from the socket until EOF.
+
+.. code-block:: lua
+
+    x = sock_recvall(sock)
+
+sock_recvline_contains
+----------------------
+
+Receive lines from the server until a line contains the needle, then return
+this line.
+
+.. code-block:: lua
+
+    x = sock_recvline_contains(sock, needle)
+
+sock_recvline_regex
+-------------------
+
+Receive lines from the server until a line matches the regex, then return this
+line.
+
+.. code-block:: lua
+
+    x = sock_recvline_regex(sock, "^250 ")
+
+sock_recvn
+----------
+
+Receive exactly n bytes from the socket.
+
+.. code-block:: lua
+
+    x = sock_recvn(sock, 4)
+
+sock_recvuntil
+--------------
+
+Receive until the needle is found, then return all data including the needle.
+
+.. code-block:: lua
+
+    x = sock_recvuntil(sock, needle)
+
+sock_sendafter
+--------------
+
+Receive until the needle is found, then write data to the socket.
+
+.. code-block:: lua
+
+    sock_sendafter(sock, needle, data)
+
+sock_newline
+------------
+
+Overwrite the default ``\n`` newline.
+
+.. code-block:: lua
+
+    sock_newline(sock, "\r\n")
+
 status
 ------

--- a/docs/sandbox.rst
+++ b/docs/sandbox.rst
@@ -0,0 +1,147 @@
+Sandbox
+=======
+
+Scripts are generally considered to be untrusted and executed exclusively in a
+child process. It's important to note that there's a basic sandbox that's
+active on every operating system, and there's a second line of defense on
+supported operating systems.
+
+The first line of defense is the restrictive stdlib. It's assumed that and
+attacker gains full control over the lua code and is able to call any function
+with arbitrary arguments. The stdlib only provides functions that are
+considered safe, so for example it's not possible to start a process or open a
+file.
+
+The second line of defense is supposed to make sure the system isn't
+compromised even if the first layer is fully broken and an attacker gains full
+control over the child process.
+
+Right now this is only supported on linux and openbsd.
+
+Linux
+-----
+
+On linux we use seccomp to filter all syscalls that we don't need. We also use
+chroot to disable filesystem access. It's recommended to install the sn0int
+binary with ``cap_sys_chroot`` to make sure unprivileged users can use chroot.
+The chroot location is hard coded and all capabilities are removed after the
+chroot is done or if no chroot is going to happen.
+
+OpenBSD
+-------
+
+On openbsd we're using ``pledge`` to restrict syscalls and ``unveil`` to
+restrict filesystem access.
+
+IPC Protocol
+------------
+
+The parent process and the child process communicate using an IPC protocol that
+is line-based json.
+
+For a simple hello world the parent process is only going to send a single line
+to the child process. This line contains:
+
+- The function argument
+- The dns config
+- Keys that the module has been given access to
+- The module metadata and code
+- Options, if any
+- A socks5 proxy, if any
+- The log level
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    -- nothing to do here\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+Saving this line in a file called ``start.json`` and sending it to a sandbox
+process should result in the following output::
+
+    $ sn0int sandbox foobar < start.json
+    {"Exit":"Ok"}
+    $
+
+This line tells us that the script terminated successfully.
+
+There are some functions that cause a notification to the parent process. We
+are going to add a call to the ``info()`` function to our module:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    info('ohai')\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+This is going to print an additional event::
+
+    $ sn0int sandbox foobar < start2.json
+    {"Log":{"Info":"\"ohai\""}}
+    {"Exit":"Ok"}
+    $
+
+There are some functions that block the child process until the parent process
+sent a reply. These functions are mostly database related functions, since the
+child doesn't have direct database access. To demonstrate this, we're going to
+write two lines to our file this time, one is the init line and the second one
+is the reply for the database event:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    x = db_add('domain', {value=\"example.com\"})\n    info(x)\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+    {"Ok":1337}
+
+Results in the following output::
+
+    $ target/release/sn0int sandbox foobar < start3.json
+    {"Database":{"Insert":{"Domain":{"value":"example.com"}}}}
+    {"Log":{"Info":"1337.0"}}
+    {"Exit":"Ok"}
+    $
+
+The first line is a database event and indicates that the child wants to insert
+data. After printing this line the child tries to read a line from stdin, this
+is why we needed to write two lines to our json file this time. In the second
+line the child learns if the insert was successful and which id was assigned to
+that entity.
+
+Limitations
+-----------
+
+There are some limitations that you should be aware:
+
+- Network access is available and network namespaces aren't isolated. This
+  means scripts have access to your local network, the internet and also your
+  localhost loopback interface.
+- If chroot is unavailable an attacker could connect to unix domain sockets.
+
+Diagnosing a sandbox failure
+----------------------------
+
+You might experience a sandbox failure, especially on architectures that are
+less popular. This usually looks like this::
+
+    [sn0int][example][kpcyrd/ctlogs] > run
+    [-] Failed "example.com": EOF while parsing a value at line 1 column 0
+    [+] Finished kpcyrd/ctlogs (1 errors)
+
+A module that never finishes could also mean an IO thread inside the worker got
+killed by the sandbox.
+
+You can try to diagnose this yourself with strace::
+
+    strace -f sn0int run -vv ctlogs 2>&1 | tee strace.log
+
+Open ``strace.log``, look out for syscalls that didn't return by searching for
+``= ?`` and ignore calls to exit and similar. You are looking for something
+like this::
+
+    seccomp(SECCOMP_SET_MODE_FILTER, 0, {len=48, filter=0xdd59094e490}) = 0
+    write(1, "[+] activated!\n", 15[+] activated!
+    )        = 15
+    getresuid( <unfinished ...>)            = ?
+    +++ killed by SIGSYS (core dumped) +++
+
+This would indicate a call to ``getresuid`` which was not allowed by the
+seccomp filter.
+
+If you don't want to diagnose this yourself open a new bug report with as much
+information as possible, specifically which distro, which release and which
+architecture you're using.
--- a/docs/scripting.rst
+++ b/docs/scripting.rst
@@ -18,8 +18,12 @@ free to change that to something else::
 Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
 picked up by sn0int.

-Let's add our first module by opening ``~/repos/sn0int-modules/first.lua``.
-There's a bit of boilerplate that every module needs to load successfully:
+Make sure you're still in the right folder and add your first module::
+
+    sn0int new first.lua
+
+This is going to generate some boilerplate for you that every module needs to
+load successfully. Afterwards we can edit it like this:

 .. code-block:: lua

@@ -125,7 +129,7 @@ our scope and set it to resolvable if ``error`` is ``nil``.
        if last_err() then return end

        if records['error'] == nil then
-            db_add('subdomain', arg, {
+            db_add('subdomain', {
                domain_id=arg['id'],
                value=subdomain,
                resolvable=true,
@@ -159,7 +163,7 @@ After putting everything together, our final module looks like this:
        if last_err() then return end

        if records['success'] ~= nil then
-            db_add('subdomain', arg, {
+            db_add('subdomain', {
                domain_id=arg['id'],
                value=subdomain,
                resolvable=true,
@@ -188,3 +192,40 @@ your identity.
 Afterwards publish your module with::

    sn0int publish ./first.lua
+
+Reading data from stdin
+-----------------------
+
+Sometimes you need to read data that can't be easily accessed from within the
+sandbox, like output of other programms or file content. In that case you can
+write a module that reads from stdin:
+
+.. code-block:: lua
+
+    -- Description: Read from stdin
+    -- Version: 0.1.0
+    -- License: GPL-3.0
+
+    function run()
+        while true do
+            x = stdin_readline()
+            if x == nil then
+                break
+            end
+            info(x)
+        end
+    end
+
+Write it to a file and run it like this::
+
+    % echo hello | sn0int run --stdin -vvf stdin.lua
+    [*] anonymous/stdin                                   : "hello\n"
+    [+] Finished anonymous/stdin
+    %
+
+This is going to read one line at a time and allows you to process it with
+regular expressions and add data to the database.
+
+.. note::
+   If you get an error like ``Failed to read stdin: "stdin is unavailable"``
+   make sure the ``--stdin`` flag is set.
--- a/examples/asn.rs
+++ b/examples/asn.rs
@@ -1,32 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate maxminddb;
-
-use std::env;
-use sn0int::errors::*;
-use sn0int::geoip::{AsnDB, Maxmind};
-
-
-fn run() -> Result<()> {
-    let asndb = AsnDB::open_or_download()?;
-
-    for arg in env::args().skip(1) {
-        let ip = arg.parse()?;
-        let asn = asndb.lookup(ip)?;
-        println!("{:#?}", asn);
-    }
-
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}
--- a/examples/geoip.rs
+++ b/examples/geoip.rs
@@ -1,31 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-
-use std::env;
-use sn0int::errors::*;
-use sn0int::geoip::{GeoIP, Maxmind};
-
-
-fn run() -> Result<()> {
-    let geoip = GeoIP::open_or_download()?;
-
-    for arg in env::args().skip(1) {
-        let ip = arg.parse()?;
-        let lookup = geoip.lookup(ip)?;
-        println!("{:#?}", lookup);
-    }
-
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}
--- a/examples/maxmind-dl.rs
+++ b/examples/maxmind-dl.rs
@@ -1,50 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-
-// workaround for rustc 1.29.2 support
-#[cfg(not(target_os = "openbsd"))]
-extern crate structopt;
-#[cfg(target_os = "openbsd")]
-#[macro_use] extern crate structopt;
-
-use sn0int::errors::*;
-use sn0int::geoip::{GeoIP, Maxmind};
-use sn0int::paths;
-use std::fs;
-use structopt::StructOpt;
-
-#[derive(Debug, StructOpt)]
-pub struct Args {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-fn run() -> Result<()> {
-    let args = Args::from_args();
-    debug!("{:?}", args);
-    let path = paths::cache_dir()?.join(&args.target);
-    if args.extract_only {
-        let body = fs::read(&args.url)?;
-        sn0int::archive::extract(&mut &body[..], &args.filter, path)?;
-    } else {
-        GeoIP::download(path, &args.filter, &args.url)?;
-    }
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}
--- a/examples/maxmind.rs
+++ b/examples/maxmind.rs
@@ -0,0 +1,104 @@
+extern crate sn0int;
+extern crate env_logger;
+extern crate chrootable_https;
+#[macro_use] extern crate log;
+
+// workaround for rustc 1.29.2 support
+#[cfg(not(target_os = "openbsd"))]
+extern crate structopt;
+#[cfg(target_os = "openbsd")]
+#[macro_use] extern crate structopt;
+
+use sn0int::errors::*;
+use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
+use sn0int::paths;
+use std::fs;
+use std::net::IpAddr;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="dl")]
+    Download(Download),
+    #[structopt(name="asn")]
+    Asn(AsnArgs),
+    #[structopt(name="geoip")]
+    GeoIP(GeoIPArgs),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Download {
+    url: String,
+    filter: String,
+    target: String,
+    #[structopt(short="e", long="extract-only")]
+    extract_only: bool,
+}
+
+impl Download {
+    fn run(&self) -> Result<()> {
+        let path = paths::cache_dir()?.join(&self.target);
+        if self.extract_only {
+            let body = fs::read(&self.url)?;
+            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
+        } else {
+            GeoIP::download(path, &self.filter, &self.url)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AsnArgs {
+    ip: IpAddr,
+}
+
+impl AsnArgs {
+    fn run(&self) -> Result<()> {
+        let asndb = AsnDB::open_or_download()?;
+
+        let asn = asndb.lookup(self.ip)?;
+        println!("{:#?}", asn);
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct GeoIPArgs {
+    ip: IpAddr,
+}
+
+impl GeoIPArgs {
+    fn run(&self) -> Result<()> {
+        let geoip = GeoIP::open_or_download()?;
+
+        let lookup = geoip.lookup(self.ip)?;
+        println!("{:#?}", lookup);
+
+        Ok(())
+    }
+}
+
+
+fn run() -> Result<()> {
+    let args = Args::from_args();
+    debug!("{:?}", args);
+    match args {
+        Args::Download(args) => args.run(),
+        Args::Asn(args) => args.run(),
+        Args::GeoIP(args) => args.run(),
+    }
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}
--- a/examples/spinners.rs
+++ b/examples/spinners.rs
@@ -1,20 +1,67 @@
-extern crate sn0int;
-
-use std::env;
 use std::thread;
 use std::time::Duration;
-use sn0int::term::{SPINNERS, Spinner};
+use sn0int::term::{SPINNERS, Spinner, StackedSpinners};
+use structopt::StructOpt;
+
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="single")]
+    Single(Single),
+    #[structopt(name="stacked")]
+    Stacked(Stacked),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Single {
+    idx: usize,
+    #[structopt(long="ticks", default_value="100")]
+    ticks: usize,
+}
+
+impl Single {
+    fn run(&self) {
+        let mut s = Spinner::new(SPINNERS[self.idx], "Demo".to_string());
+
+        for _ in 0..self.ticks {
+            thread::sleep(Duration::from_millis(100));
+            s.tick();
+        }
+
+        s.finish("Done".to_string());
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Stacked {
+}
+
+impl Stacked {
+    fn run(&self) {
+        let mut stack = StackedSpinners::new();
+        stack.add("1".into(), String::from("spinner1"));
+        stack.add("2".into(), String::from("spinner2"));
+        stack.add("3".into(), String::from("spinner3"));
+
+        for x in 1..=3 {
+            for _ in 0..50 {
+                thread::sleep(Duration::from_millis(100));
+                stack.tick();
+            }
+            // stack.log("ohai");
+            stack.remove(&x.to_string());
+        }
+
+        stack.clear();
+
+        // stack.finish("Done".to_string());
+    }
+}

 fn main() {
-    let idx = env::args().skip(1).next().expect("Expected argv[1]");
-    let idx = idx.parse::<usize>().expect("argv[1] is not a number");
-
-    let mut s = Spinner::new(SPINNERS[idx], "Demo".to_string());
-
-    for _ in 0..100 {
-        thread::sleep(Duration::from_millis(100));
-        s.tick();
+    let args = Args::from_args();
+    match args {
+        Args::Single(args) => args.run(),
+        Args::Stacked(args) => args.run(),
    }
-
-    s.finish("Done".to_string());
 }
--- a/examples/stacked-spinners.rs
+++ b/examples/stacked-spinners.rs
@@ -1,25 +0,0 @@
-extern crate sn0int;
-
-use sn0int::term::StackedSpinners;
-use std::thread;
-use std::time::Duration;
-
-fn main() {
-    let mut stack = StackedSpinners::new();
-    stack.add("1".into(), String::from("spinner1"));
-    stack.add("2".into(), String::from("spinner2"));
-    stack.add("3".into(), String::from("spinner3"));
-
-    for x in 1..=3 {
-        for _ in 0..50 {
-            thread::sleep(Duration::from_millis(100));
-            stack.tick();
-        }
-        // stack.log("ohai");
-        stack.remove(&x.to_string());
-    }
-
-    stack.clear();
-
-    // stack.finish("Done".to_string());
-}
--- a/migrations/2019-01-20-000235_ttl/down.sql
+++ b/migrations/2019-01-20-000235_ttl/down.sql
@@ -0,0 +1 @@
+DROP TABLE ttls;
--- a/migrations/2019-01-20-000235_ttl/up.sql
+++ b/migrations/2019-01-20-000235_ttl/up.sql
@@ -0,0 +1,7 @@
+CREATE TABLE ttls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    key INTEGER NOT NULL,
+    expire DATETIME NOT NULL,
+    CONSTRAINT ttl_unique UNIQUE (family, key)
+);
--- a/modules/dev/arp-scan.lua
+++ b/modules/dev/arp-scan.lua
@@ -1,5 +1,5 @@
 -- Description: Parse arp-scan output
-- Version: 0.1.0
+-- Version: 0.3.0
 -- License: GPL-3.0

 -- sudo arp-scan -qglI wlp3s0
@@ -25,18 +25,20 @@ function run()
        if m ~= nil then
            ipaddr = m[2]
            mac = m[3]
+            now = datetime()

            device_id = db_add('device', {
                value=mac,
+                last_seen=now,
            })
            if last_err() then return end

-            -- TODO: add last_seen
-            db_add('network-device', {
+            db_add_ttl('network-device', {
                network_id=network_id,
                device_id=device_id,
                ipaddr=ipaddr,
-            })
+                last_seen=now,
+            }, 300)
            if last_err() then return end
        end
    end
--- a/modules/dev/axfr.lua
+++ b/modules/dev/axfr.lua
@@ -1,5 +1,5 @@
 -- Description: Try a zone transfer for subdomains
-- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0

--- a/modules/dev/ctlogs.lua
+++ b/modules/dev/ctlogs.lua
@@ -1,11 +1,15 @@
 -- Description: Query certificate transparency logs to discover subdomains
-- Version: 0.2.0
+-- Version: 0.4.0
 -- Source: domains
 -- License: GPL-3.0

 function each_name(name)
    local domain_id, psl_domain

+    if seen[name] == 1 then
+        return
+    end
+    seen[name] = 1
    debug(name)

    if name:find('*.') == 1 then
@@ -43,6 +47,7 @@ function each_name(name)
 end

 function run(arg)
+    full = getopt('full') ~= nil
    any_domain = getopt('any-domain') ~= nil

    domains = {}
@@ -60,34 +65,40 @@ function run(arg)
    if last_err() then return end
    if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end

-    certs = json_decode_stream(resp['text'])
+    certs = json_decode(resp['text'])
    if last_err() then return end

+    seen = {}
+
    i = 1
    while i <= #certs do
        c = certs[i]
        debug(c)

-        -- fetch certificate
-        id = c['min_cert_id']
-        req = http_request(session, 'GET', 'https://crt.sh/', {
-            query={
-                d=id .. '', -- TODO: find nicer way for tostring
-            }
-        })
-        resp = http_send(req)
-        if last_err() then return end
-        if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+        if full then
+            -- fetch certificate
+            id = c['min_cert_id']
+            req = http_request(session, 'GET', 'https://crt.sh/', {
+                query={
+                    d=id .. '', -- TODO: find nicer way for tostring
+                }
+            })
+            resp = http_send(req)
+            if last_err() then return end
+            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end

-        -- iterate over all valid names
-        crt = x509_parse_pem(resp['text'])
-        if last_err() then return end
-        names = crt['valid_names']
+            -- iterate over all valid names
+            crt = x509_parse_pem(resp['text'])
+            if last_err() then return end
+            names = crt['valid_names']

-        j = 1
-        while j <= #names do
-            each_name(names[j])
-            j = j+1
+            j = 1
+            while j <= #names do
+                each_name(names[j])
+                j = j+1
+            end
+        else
+            each_name(c['name_value'])
        end

        i = i+1
--- a/modules/dev/isc-dhcpd-leases.lua
+++ b/modules/dev/isc-dhcpd-leases.lua
@@ -1,5 +1,5 @@
 -- Description: Parse isc-dhcpd dhcpd.leases(5)
-- Version: 0.1.0
+-- Version: 0.2.0
 -- License: GPL-3.0

 -- cat /var/lib/dhcpd/dhcpd.leases
@@ -7,18 +7,21 @@
 function add(lease)
    if not lease['active'] then return end

+    now = datetime()
+
    device_id = db_add('device', {
        value=lease['mac'],
        hostname=lease['hostname'],
+        last_seen=now,
    })
    if last_err() then return end

-    -- TODO: add last_seen
-    db_add('network-device', {
+    db_add_ttl('network-device', {
        network_id=network_id,
        device_id=device_id,
        ipaddr=lease['ipaddr'],
-    })
+        last_seen=now,
+    }, 180)
    if last_err() then return end
 end

--- a/modules/dev/iw-station-dump.lua
+++ b/modules/dev/iw-station-dump.lua
@@ -1,5 +1,5 @@
 -- Description: Parse iw station dump
-- Version: 0.1.0
+-- Version: 0.2.0
 -- License: GPL-3.0

 -- iw dev wlan0 station dump
@@ -12,16 +12,19 @@ function add(client)
    then
        debug(client)

+        now = datetime()
+
        device_id = db_add('device', {
            value=client['mac'],
+            last_seen=now,
        })
        if last_err() then return end

-        -- TODO: add last_seen
-        db_add('network-device', {
+        db_add_ttl('network-device', {
            network_id=network_id,
            device_id=device_id,
-        })
+            last_seen=now,
+        }, 180)
        if last_err() then return end
    end

--- a/modules/dev/passive-arp.lua
+++ b/modules/dev/passive-arp.lua
@@ -0,0 +1,79 @@
+-- Description: Passive arp-scanner with sniffglue
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- sudo sniffglue -jv enp0s25
+
+function each_frame(frame)
+    if not frame['Ether'] then return end
+
+    local arp = frame['Ether'][2]['Arp']
+    if not arp then return end
+
+    if arp['Request'] then
+        arp = arp['Request']
+    elseif arp['Reply'] then
+        arp = arp['Reply']
+    else
+        -- unknown, abort
+        return
+    end
+
+    debug(arp)
+
+    -- TODO: this might change to a string in the future
+    local mac = mac(arp['src_mac'])
+    local ipaddr = arp['src_addr']
+    debug({src_mac=mac, src_addr=ipaddr})
+
+    local now = datetime()
+
+    local device_id = db_add('device', {
+        value=mac,
+        last_seen=now,
+    })
+    if last_err() then return end
+
+    db_add_ttl('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+        ipaddr=ipaddr,
+        last_seen=now,
+    }, 120)
+    if last_err() then return end
+end
+
+function mac(m)
+    return
+        hex({m[1]}) .. ':' ..
+        hex({m[2]}) .. ':' ..
+        hex({m[3]}) .. ':' ..
+        hex({m[4]}) .. ':' ..
+        hex({m[5]}) .. ':' ..
+        hex({m[6]})
+end
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    while true do
+        local x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        local frame = json_decode(x)
+        if last_err() then return end
+
+        each_frame(frame)
+        if last_err() then return end
+    end
+end
--- a/modules/dev/smtp-check.lua
+++ b/modules/dev/smtp-check.lua
@@ -0,0 +1,90 @@
+-- Description: Verify email address by asking the smtp server
+-- Version: 0.1.0
+-- Source: emails
+-- License: GPL-3.0
+
+function find_mx(domain)
+    local records, i, r
+
+    records = dns(domain, {
+        record='MX',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+    -- debug(records)
+
+    i = 1
+    while i <= #records do
+        r = records[i][2]['MX']
+        if r then
+            debug('mx: ' .. r[2])
+            return r[2]
+        end
+        i = i+1
+    end
+end
+
+function run(arg)
+    -- extract domain
+    domain = arg['value']:match('@(.*)')
+    if doman ~= nil then
+        -- malformed domain
+        return
+    end
+
+    -- mx lookup
+    mx = find_mx(domain)
+    if last_err() then return end
+    if not mx then return end
+
+    -- create connection
+    c = sock_connect(mx, 25, {})
+    if last_err() then return end
+
+    l = sock_recvline(c)
+    if last_err() then return end
+    debug(l)
+
+    -- send hello
+    sock_sendline(c, 'ehlo localhost')
+    if last_err() then return end
+
+    l = sock_recvline_regex(c, '^250 ')
+    if last_err() then return end
+    debug(l)
+
+    -- send email
+    sock_sendline(c, 'mail from:<root@localhost>')
+    if last_err() then return end
+
+    l = sock_recvline(c)
+    if last_err() then return end
+    debug(l)
+
+    -- send rcpt
+    sock_sendline(c, 'rcpt to:<' .. arg['value'] .. '>')
+    if last_err() then return end
+
+    l = sock_recvline(c)
+    if last_err() then return end
+    debug(l)
+
+    -- check status
+    verified = nil
+    if l:match('^2') then
+        debug('email is valid')
+        verified = true
+    elseif l:match('^5') then
+        debug('email is invalid')
+        verified = false
+    elseif l:match('^4') then
+        debug('unknown status, temporary delivery failure')
+    end
+
+    if verified ~= nil then
+        db_update('email', arg, {
+            valid=verified,
+        })
+    end
+end
--- a/modules/dev/thunderbird-autoconfig.lua
+++ b/modules/dev/thunderbird-autoconfig.lua
@@ -0,0 +1,48 @@
+-- Description: Query thunderbird autoconfig db for subdomains
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    session = http_mksession()
+
+    -- check if an autoconfig exists without disclosing our target yet
+    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/', {})
+    resp = http_send(req)
+    if last_err() then return end
+
+    if resp['status'] ~= 200 then
+        return 'index request failed'
+    end
+
+    if resp['text']:find(arg['value'], 1, true) == nil then
+        debug('no autoconfig available')
+        return
+    end
+
+    -- request config
+    req = http_request(session, 'GET', 'https://autoconfig.thunderbird.net/v1.1/' .. arg['value'], {})
+    resp = http_send(req)
+    if last_err() then return end
+
+    m = regex_find_all('<hostname>([^<]+)</hostname>', resp['text'])
+
+    i = 1
+    while i <= #m do
+        subdomain = m[i][2]
+
+        domain = psl_domain_from_dns_name(subdomain)
+        if last_err() then return end
+
+        domain_id = db_select('domain', domain)
+        if last_err() then return end
+
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=subdomain,
+        })
+        if last_err() then return end
+
+        i = i+1
+    end
+end
--- a/modules/dev/url-scan.lua
+++ b/modules/dev/url-scan.lua
@@ -1,5 +1,5 @@
 -- Description: Scan subdomains for websites
-- Version: 0.1.0
+-- Version: 0.3.0
 -- Source: subdomains
 -- License: GPL-3.0

@@ -19,7 +19,6 @@ function request(subdomain_id, url)
        value=url,
        status=reply['status'],
        body=reply['text'],
-        redirect=reply['headers']['location'],
    }

    redirect = reply['headers']['location']
--- a/modules/dev/waybackurls.lua
+++ b/modules/dev/waybackurls.lua
@@ -1,5 +1,5 @@
 -- Description: Discover subdomains from wayback machine
-- Version: 0.2.0
+-- Version: 0.3.0
 -- Source: domains
 -- License: GPL-3.0

--- a/modules/dev/well-known-uris.lua
+++ b/modules/dev/well-known-uris.lua
@@ -0,0 +1,64 @@
+-- Description: Scan for known /.well-known/ locations
+-- Version: 0.1.0
+-- Source: urls
+-- License: GPL-3.0
+
+function run(arg)
+    -- https://www.iana.org/assignments/well-known-uris/well-known-uris.xhtml
+    -- https://en.wikipedia.org/wiki/List_of_/.well-known/_services_offered_by_webservers
+
+    -- TODO: check if every location causes a 200/redirect
+
+    locations = {
+        {path='security.txt'}, -- expect 200
+        {path='dnt-policy.txt'}, -- expect 200
+        {path='caldav', redirect=true}, -- expect redirect
+        {path='autoconfig/mail/config-v1.1.xml'}, -- expect 200
+        {path='assetlinks.json'}, -- expect 200
+        {path='apple-app-site-association'},  -- expect 200
+        {path='keybase.txt'}, -- expect 200
+        {path='apple-developer-merchantid-domain-association'}, -- expect 200
+        {path='openpgpkey'}, -- expect 200
+        {path='change-password', redirect=true}, -- expect redirect
+    }
+
+    session = http_mksession()
+
+    i = 1
+    while i <= #locations do
+        path = locations[i]['path']
+        expect_redirect = locations[i]['redirect']
+
+        url = url_join(arg['value'], '/.well-known/' .. path)
+        debug(url)
+
+        req = http_request(session, 'GET', url, {
+            timeout=5000,
+        })
+        reply = http_send(req)
+        debug(reply)
+
+        if last_err() then
+            clear_err()
+        else
+            status = reply['status']
+            if (status == 200 and not expect_redirect) or (expect_redirect and status >= 300 and status < 400) then
+                obj = {
+                    subdomain_id=arg['subdomain_id'],
+                    value=url,
+                    status=reply['status'],
+                    body=reply['text'],
+                }
+
+                redirect = reply['headers']['location']
+                if redirect then
+                    obj['redirect'] = url_join(url, redirect)
+                end
+
+                db_add('url', obj)
+            end
+        end
+
+        i = i+1
+    end
+end
--- a/modules/harness/selftest.lua
+++ b/modules/harness/selftest.lua
@@ -4,4 +4,5 @@

 function run()
    -- nothing to do here
+    info('ohai')
 end
--- a/modules/harness/tcp-hello.lua
+++ b/modules/harness/tcp-hello.lua
@@ -0,0 +1,18 @@
+-- Description: Send a hello to a server on port 1337
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    addr = getopt('addr')
+    if not addr then
+        return 'addr is not set'
+    end
+
+    -- create connection
+    c = sock_connect(addr, 1337, {})
+    if last_err() then return end
+
+    -- send ohai
+    sock_sendline(c, 'ohai')
+    if last_err() then return end
+end
--- a/modules/harness/ttl.lua
+++ b/modules/harness/ttl.lua
@@ -0,0 +1,9 @@
+-- Description: Add an expiring domain
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    db_add_ttl('domain', {
+        value='example.com',
+    }, 30)
+end
--- a/sn0int-registry/templates/index.html.hbs
+++ b/sn0int-registry/templates/index.html.hbs
@@ -12,14 +12,14 @@ Among other things, sn0int is currently able to:
        </p>

        <ul class="list-unstyled">
-<li>[X] Harvest subdomains from certificate transparency logs</li>
-<li>[X] Harvest subdomains from various passive dns logs</li>
-<li>[X] Sift through subdomain results for publicly accessible websites</li>
-<li>[X] Harvest emails from pgp keyservers</li>
-<li>[X] Enrich ip addresses with ASN and geoip info</li>
-<li>[X] Harvest subdomains from the wayback machine</li>
-<li>[X] Gather information about phonenumbers</li>
-<li>[X] Bruteforce interesting urls</li>
+<li>- Harvest subdomains from certificate transparency logs</li>
+<li>- Harvest subdomains from various passive dns logs</li>
+<li>- Sift through subdomain results for publicly accessible websites</li>
+<li>- Harvest emails from pgp keyservers</li>
+<li>- Enrich ip addresses with ASN and geoip info</li>
+<li>- Harvest subdomains from the wayback machine</li>
+<li>- Gather information about phonenumbers</li>
+<li>- Bruteforce interesting urls</li>
        </ul>

        <p>
@@ -34,9 +34,9 @@ pull request.

        <p>
        The registry server allows uploading modules that can be installed by
-        users afterwards. This is done using sn0int which needs to be
-        <a href="https://github.com/kpcyrd/sn0int#installation" target="_blank"
-        rel="noopener">installed</a>.
+        users afterwards. This is done using sn0int which needs to be <a
+        href="https://sn0int.readthedocs.io/en/latest/install.html"
+        target="_blank" rel="noopener">installed</a>.
        </p>

        <p>
--- a/src/args.rs
+++ b/src/args.rs
@@ -37,6 +37,9 @@ pub enum SubCommand {
    #[structopt(author="", name="login")]
    /// Login to the registry for publishing
    Login(Login),
+    #[structopt(author="", name="new")]
+    /// Create a new module
+    New(New),
    #[structopt(author="", name="publish")]
    /// Publish a script to the registry
    Publish(Publish),
@@ -98,6 +101,12 @@ pub struct Sandbox {
 pub struct Login {
 }

+#[derive(Debug, StructOpt)]
+pub struct New {
+    /// Path to the new file
+    pub path: String,
+}
+
 #[derive(Debug, StructOpt)]
 pub struct Publish {
    /// The scripts to publish
--- a/src/cmd/run_cmd.rs
+++ b/src/cmd/run_cmd.rs
@@ -2,6 +2,7 @@ use crate::errors::*;

 use crate::args;
 use crate::db::{Database, Filter};
+use crate::db::ttl;
 use crate::engine::Module;
 use crate::models::*;
 use crate::shell::Readline;
@@ -153,6 +154,7 @@ pub fn execute(rl: &mut Readline, params: Params, options: HashMap<String, Strin
 }

 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    ttl::reap_expired(rl.db())?;
    let args = Args::from_iter_safe(args)?;
    let options = match rl.options_mut() {
        Some(options) => options.clone(),
--- a/src/cmd/select_cmd.rs
+++ b/src/cmd/select_cmd.rs
@@ -2,7 +2,10 @@ use crate::errors::*;

 use crate::cmd::Cmd;
 use crate::db;
+use crate::db::ttl;
 use crate::shell::Readline;
+use serde::Serialize;
+use serde_json;
 use structopt::StructOpt;
 use structopt::clap::AppSettings;
 use crate::models::*;
@@ -11,7 +14,16 @@ use crate::models::*;
 #[derive(Debug, StructOpt)]
 #[structopt(author = "",
            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
-pub enum Args {
+pub struct Args {
+    #[structopt(subcommand)]
+    subcommand: Target,
+    #[structopt(long="json")]
+    /// Json output
+    json: bool,
+}
+
+#[derive(Debug, StructOpt)]
+pub enum Target {
    #[structopt(name="domains")]
    /// Select domains
    Domains(Filter),
@@ -49,30 +61,52 @@ impl Filter {
    }
 }

+pub struct Printer<'a> {
+    rl: &'a mut Readline,
+    json: bool,
+}
+
+impl<'a> Printer<'a> {
+    pub fn new(rl: &'a mut Readline, json: bool) -> Printer<'a> {
+        Printer {
+            rl,
+            json,
+        }
+    }
+
+    pub fn select<T: Model + Detailed + Serialize>(&self, filter: &Filter) -> Result<()> {
+        for obj in self.rl.db().filter::<T>(&filter.parse()?)? {
+            if self.json {
+                let v = serde_json::to_string(&obj)?;
+                println!("{}", v);
+            } else {
+                println!("{}", obj.detailed(self.rl.db())?);
+            }
+        }
+
+        Ok(())
+    }
+}
+
 impl Cmd for Args {
    fn run(&self, rl: &mut Readline) -> Result<()> {
-        match self {
-            Args::Domains(filter) => select::<Domain>(rl, &filter),
-            Args::Subdomains(filter) => select::<Subdomain>(rl, &filter),
-            Args::IpAddrs(filter) => select::<IpAddr>(rl, &filter),
-            Args::Urls(filter) => select::<Url>(rl, &filter),
-            Args::Emails(filter) => select::<Email>(rl, &filter),
-            Args::PhoneNumbers(filter) => select::<PhoneNumber>(rl, &filter),
-            Args::Devices(filter) => select::<Device>(rl, &filter),
-            Args::Networks(filter) => select::<Network>(rl, &filter),
+        let printer = Printer::new(rl, self.json);
+
+        match &self.subcommand {
+            Target::Domains(filter) => printer.select::<Domain>(&filter),
+            Target::Subdomains(filter) => printer.select::<Subdomain>(&filter),
+            Target::IpAddrs(filter) => printer.select::<IpAddr>(&filter),
+            Target::Urls(filter) => printer.select::<Url>(&filter),
+            Target::Emails(filter) => printer.select::<Email>(&filter),
+            Target::PhoneNumbers(filter) => printer.select::<PhoneNumber>(&filter),
+            Target::Devices(filter) => printer.select::<Device>(&filter),
+            Target::Networks(filter) => printer.select::<Network>(&filter),
        }
    }
 }

 #[inline]
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    ttl::reap_expired(rl.db())?;
    Args::run_str(rl, args)
 }
-
-fn select<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<()> {
-    for obj in rl.db().filter::<T>(&filter.parse()?)? {
-        println!("{}", obj.detailed(rl.db())?);
-    }
-
-    Ok(())
-}
--- a/src/crt.rs
+++ b/src/crt.rs
@@ -1,10 +1,10 @@
 use crate::errors::*;

 use x509_parser;
+use der_parser::{DerObject, DerObjectContent};
 use der_parser::oid::Oid;
 use std::collections::HashSet;
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
-use nom::be_u8;


 #[derive(Debug, PartialEq)]
@@ -13,69 +13,56 @@ pub enum AlternativeName {
    IpAddr(IpAddr),
 }

-named!(san_extension<&[u8], Vec<AlternativeName>>, do_parse!(
-    _tag:   tag!(b"\x30")   >>
-    len:    be_u8           >>
-    values: take!(len)      >>
-    ({
-        let mut bytes = values;
-        let mut values = Vec::new();
-        while !bytes.is_empty() {
-            let (rem, v) = san_value(bytes)?;
-            match v {
-                Ok(v) => values.push(v),
-                Err(e) => warn!("Unknown field in SAN extension: {}", e),
-            }
-            bytes = rem;
-        }
-        values
-    })
-));
+pub fn san_extension(i: &[u8]) -> Result<Vec<AlternativeName>> {
+    let (rem, seq) = der_parser::parse_der_sequence(i)
+        .map_err(|_| format_err!("Failed to parse san extension"))?;

-named!(san_value<&[u8], Result<AlternativeName>>,
-    switch!(be_u8,
-        0x82    => call!(san_value_dns) |
-        0x87    => call!(san_value_ipaddr) |
-        k       => call!(san_value_unknown, k)
-    )
-);
+    if !rem.is_empty() {
+        bail!("san extension has trailing garbage");
+    }

-named!(san_value_dns<&[u8], Result<AlternativeName>>, do_parse!(
-    len:    be_u8           >>
-    value:  take!(len)      >>
-    ({
-        String::from_utf8(value.to_vec())
-            .map(AlternativeName::DnsName)
-            .map_err(Error::from)
-    })
-));
+    debug!("Decoded sequence: {:?}", seq);
+    if let DerObjectContent::Sequence(seq) = seq.content {
+        seq.into_iter()
+            .map(san_value)
+            .collect()
+    } else {
+        bail!("Expected der sequence");
+    }
+}

-named!(san_value_ipaddr<&[u8], Result<AlternativeName>>, do_parse!(
-    len:    be_u8           >>
-    v:      take!(len)      >>
-    ({
-        match len {
-            4  => Ok(AlternativeName::IpAddr(Ipv4Addr::from([
-                v[0], v[1], v[2], v[3],
-            ]).into())),
-            16 => Ok(AlternativeName::IpAddr(Ipv6Addr::from([
-                v[0],  v[1],  v[2],  v[3],
-                v[4],  v[5],  v[6],  v[7],
-                v[8],  v[9],  v[10], v[11],
-                v[12], v[13], v[14], v[15],
-            ]).into())),
-            _ => Err(format_err!("Invalid ipaddr")),
-        }
-    })
-));
+pub fn san_value(o: DerObject) -> Result<AlternativeName> {
+    debug!("DER object in SAN extension: {:?}", o);

-named_args!(san_value_unknown(key: u8)<&[u8], Result<AlternativeName>>, do_parse!(
-    len:    be_u8           >>
-    v:      take!(len)      >>
-    ({
-        Err(format_err!("Unexpected type {:?} => {:?}", key, v))
-    })
-));
+    match (o.class, o.tag, &o.content) {
+        (2, 2, DerObjectContent::Unknown(value)) => san_value_dns(value),
+        (2, 7, DerObjectContent::Unknown(value)) => san_value_ipaddr(value),
+        _ => bail!("Unexpected object: {:?}", o),
+    }
+}
+
+pub fn san_value_dns(v: &[u8]) -> Result<AlternativeName> {
+    debug!("Reading as dns name: {:?}", v);
+    String::from_utf8(v.to_vec())
+        .map(AlternativeName::DnsName)
+        .map_err(Error::from)
+}
+
+pub fn san_value_ipaddr(v: &[u8]) -> Result<AlternativeName> {
+    debug!("Reading as ipaddr: {:?}", v);
+    match v.len() {
+        4  => Ok(AlternativeName::IpAddr(Ipv4Addr::from([
+            v[0], v[1], v[2], v[3],
+        ]).into())),
+        16 => Ok(AlternativeName::IpAddr(Ipv6Addr::from([
+            v[0],  v[1],  v[2],  v[3],
+            v[4],  v[5],  v[6],  v[7],
+            v[8],  v[9],  v[10], v[11],
+            v[12], v[13], v[14], v[15],
+        ]).into())),
+        _ => Err(format_err!("Invalid ipaddr")),
+    }
+}

 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct Certificate {
@@ -138,15 +125,7 @@ impl Certificate {
            }

            debug!("Found san extension: {:?}", x.value);
-            let values = match san_extension(x.value) {
-                Ok((remaining, values)) => {
-                    if !remaining.is_empty() {
-                        bail!("san extension has trailing garbage");
-                    }
-                    values
-                },
-                Err(_) => bail!("Failed to parse san extension"),
-            };
+            let values = san_extension(x.value)?;

            for v in values {
                match v {
@@ -168,6 +147,7 @@ impl Certificate {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use der_parser::parse_der;

    #[test]
    fn test_parse_pem_github() {
@@ -265,33 +245,154 @@ ZkZZmqNn2Q8=

    #[test]
    fn test_san_extension() {
-        let (rem, ext) = san_extension(&[48, 28,
+        let ext = san_extension(&[48, 28,
                130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109,
                130, 14, 119, 119, 119, 46, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109
            ])
            .expect("Failed to parse extension");
-        assert!(rem.is_empty());
        assert_eq!(ext, vec![
            AlternativeName::DnsName(String::from("github.com")),
            AlternativeName::DnsName(String::from("www.github.com")),
        ]);
    }

+    #[test]
+    fn test_long_san_extension() {
+        let mut x = Certificate::parse_pem(r#"-----BEGIN CERTIFICATE-----
+MIII3jCCB8agAwIBAgIQAp1dOviF3mpYKKObx4fjxjANBgkqhkiG9w0BAQsFADBe
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMR0wGwYDVQQDExRHZW9UcnVzdCBSU0EgQ0EgMjAxODAe
+Fw0xOTAxMDMwMDAwMDBaFw0xOTA3MzAxMjAwMDBaMIGCMQswCQYDVQQGEwJERTEl
+MCMGA1UECBMcRnJlaWUgdW5kIEhhbnNlc3RhZHQgSGFtYnVyZzEQMA4GA1UEBxMH
+SGFtYnVyZzEXMBUGA1UEChMOQWJvdXQgWW91IEdtYkgxCzAJBgNVBAsTAklUMRQw
+EgYDVQQDEwthYm91dHlvdS5kZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBALG6ZjY9TtJmN18p5KlJMtzdZMhw3mz6dGOYoSMTaQCDnw7RW14H8JX9Dz51
+dTM4Ig1rPka5DjujNG8BKETGknRzQMEo7x08qZirzdQIz9QCnYDQ3/6l9tDfQ16X
+pctnQRY156H8jyjhkaT+dWJIHaPwz+g6117plfv0F6iOcupNtF4rnZK7vpcyb/Fm
+F985uHdBVXJKVt7BMUjUO6fdm8865fTyL8lb1ocEgbN91KdI7Bt9wUqxgOR7BJRJ
+YQAC+Y6wqE8BwOGH11QaNGKQ8xGdBd3eC4tAuif1y+4WVPDAlhmJJR/FcnsiLVbX
+zg4sgE+4kLOayCJY6MfN2MRtchkCAwEAAaOCBXEwggVtMB8GA1UdIwQYMBaAFJBY
+/7CcdahRVHex7fKjQxY4nmzFMB0GA1UdDgQWBBTPNzAGXJdERAKW8w3kFNJ88MLO
+qjCCAuIGA1UdEQSCAtkwggLVggthYm91dHlvdS5kZYIRY2RuLnlvdWFuZGlkb2wu
+ZGWCDWNkbi5lZGl0ZWQuZGWCE2Nkbi5hYm91dHN0YXRpYy5jb22CDW0uYWJvdXR5
+b3UuZGWCE3N0YXRpYzMuYWJvdXR5b3UuZGWCFmltYWdlcy5hYm91dHN0YXRpYy5j
+b22CE3N0YXRpYzUuYWJvdXR5b3UuZGWCEGNvLXQuYWJvdXR5b3UuZGWCDXQuYWJv
+dXR5b3UuZGWCDmNvLmFib3V0eW91LmRlggllZGl0ZWQuZGWCE2NvLW1hcHAuYWJv
+dXR5b3UuZGWCE3N0YXRpYzQuYWJvdXR5b3UuZGWCEW1lZGlhLmFib3V0eW91LmRl
+giZ3aXR0LXdlaWRlbi5kYW0uc3RhZ2luZy5hYm91dHlvdS5jbG91ZIISc3RhdGlj
+LmFib3V0eW91LmRlghBjZG40LmFib3V0eW91LmRlghNzdGF0aWMyLmFib3V0eW91
+LmRlgiN3aXR0LXdlaWRlbi5kYW0uYWNtZS5hYm91dHlvdS5jbG91ZIIXY2RuLmFi
+b3V0eW91LXN0YWdpbmcuZGWCD2Nkbi5hYm91dHlvdS5kZYIQY2RuMy5hYm91dHlv
+dS5kZYIQY2RuMi5hYm91dHlvdS5kZYIQY2RuNS5hYm91dHlvdS5kZYISYXNzZXRz
+LmFib3V0eW91LmRlghBjZG4xLmFib3V0eW91LmRlghpzdGF0aWNtYWlsLWNkbi5h
+Ym91dHlvdS5kZYIPd3d3LmFib3V0eW91LmRlghNzdGF0aWMxLmFib3V0eW91LmRl
+ghRtLWFzc2V0cy5hYm91dHlvdS5kZYIQY2RuLm1hcnktcGF1bC5kZYIQY28tbS5h
+Ym91dHlvdS5kZYIVZmlsZXMuYWJvdXRzdGF0aWMuY29tghNpbWcuYWJvdXRzdGF0
+aWMuY29tgg9pbWcuYWJvdXR5b3UuZGUwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW
+MBQGCCsGAQUFBwMBBggrBgEFBQcDAjA+BgNVHR8ENzA1MDOgMaAvhi1odHRwOi8v
+Y2RwLmdlb3RydXN0LmNvbS9HZW9UcnVzdFJTQUNBMjAxOC5jcmwwTAYDVR0gBEUw
+QzA3BglghkgBhv1sAQEwKjAoBggrBgEFBQcCARYcaHR0cHM6Ly93d3cuZGlnaWNl
+cnQuY29tL0NQUzAIBgZngQwBAgIwdQYIKwYBBQUHAQEEaTBnMCYGCCsGAQUFBzAB
+hhpodHRwOi8vc3RhdHVzLmdlb3RydXN0LmNvbTA9BggrBgEFBQcwAoYxaHR0cDov
+L2NhY2VydHMuZ2VvdHJ1c3QuY29tL0dlb1RydXN0UlNBQ0EyMDE4LmNydDAJBgNV
+HRMEAjAAMIIBBAYKKwYBBAHWeQIEAgSB9QSB8gDwAHcAY/Lbzeg7zCzPC3KEJ1dr
+M6SNYXePvXWmOLHHaFRL2I0AAAFoE/H84wAABAMASDBGAiEAh8Q7LXUzhsbiuxCS
+VoeRmnPtLEZcjNFg3R+eBK5FkQMCIQD+Ic1QErzzP1B76BPLcaBgOxULpLQ2Ib4M
+b38fMU5GhwB1AId1v+dZfPiMQ5lfvfNu/1aNR1Y2/0q1YMG06v9eoIMPAAABaBPx
+/dQAAAQDAEYwRAIgDcWLzLdGGG7d3EV3y809H8MwEojfEXT0DS75TchCvB0CIBno
+kC5/KGjNdQdsqJX4NJbQ06RAbHLeGwX5ccmaKbQ3MA0GCSqGSIb3DQEBCwUAA4IB
+AQC81DWjm2PklQzIGSIf/tRm2GtjlL6Vi7rMGkSbiV0k1FnoptdHfQIs55tTBD7c
+TheMOk62JL6z0FKpAgPUIU+HrKJ/fAcBmQo+yqn0vRT0yhDrDGEFl6Sm2HyI0oKG
+XryhpFLQkHuDkyA4uKOLuefPBgdjVZW9LqxmZhFPaZY6BSa/neZopVNwC1c+4Xwu
+mAlnYDoB0Mj2UIPvIeftkDfF6sURmmZb0/+AMbFDCQYHvZFPI8DFgcagy8og5XJZ
+gQ+70UdJdM3RWyrd9R66aZwNGkcS6C2wtKCRhztWDMru/wNuyOsYS6JttoTYxRsh
+z/6Vy8Ga9kigYVsa8ZFMR+Ex
+-----END CERTIFICATE-----
+"#).expect("Failed to parse cert");
+        x.valid_names.sort();
+        x.valid_ipaddrs.sort();
+        assert_eq!(x, Certificate {
+            valid_names: vec![
+                "aboutyou.de".into(),
+                "assets.aboutyou.de".into(),
+                "cdn.aboutstatic.com".into(),
+                "cdn.aboutyou-staging.de".into(),
+                "cdn.aboutyou.de".into(),
+                "cdn.edited.de".into(),
+                "cdn.mary-paul.de".into(),
+                "cdn.youandidol.de".into(),
+                "cdn1.aboutyou.de".into(),
+                "cdn2.aboutyou.de".into(),
+                "cdn3.aboutyou.de".into(),
+                "cdn4.aboutyou.de".into(),
+                "cdn5.aboutyou.de".into(),
+                "co-m.aboutyou.de".into(),
+                "co-mapp.aboutyou.de".into(),
+                "co-t.aboutyou.de".into(),
+                "co.aboutyou.de".into(),
+                "edited.de".into(),
+                "files.aboutstatic.com".into(),
+                "images.aboutstatic.com".into(),
+                "img.aboutstatic.com".into(),
+                "img.aboutyou.de".into(),
+                "m-assets.aboutyou.de".into(),
+                "m.aboutyou.de".into(),
+                "media.aboutyou.de".into(),
+                "static.aboutyou.de".into(),
+                "static1.aboutyou.de".into(),
+                "static2.aboutyou.de".into(),
+                "static3.aboutyou.de".into(),
+                "static4.aboutyou.de".into(),
+                "static5.aboutyou.de".into(),
+                "staticmail-cdn.aboutyou.de".into(),
+                "t.aboutyou.de".into(),
+                "witt-weiden.dam.acme.aboutyou.cloud".into(),
+                "witt-weiden.dam.staging.aboutyou.cloud".into(),
+                "www.aboutyou.de".into(),
+            ],
+            valid_ipaddrs: vec![],
+        });
+    }
+
    #[test]
    fn test_san_value_dns() {
-        let (rem, v) = san_value(&[130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
+        let (rem, v) = parse_der(&[130, 10, 103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
            .expect("Failed to parse san value");
-        let v = v.expect("Extension contains invalid data");
        assert!(rem.is_empty());
+        println!("{:?}", v);
+        assert_eq!(v, DerObject {
+            class: 2,
+            structured: 0,
+            tag: 2,
+            content: DerObjectContent::Unknown(&[103, 105, 116, 104, 117, 98, 46, 99, 111, 109])
+        });
+        let content = match v.content {
+            DerObjectContent::Unknown(v) => v,
+            _ => panic!("Wrong DerObjectContent"),
+        };
+        let v = san_value_dns(content)
+            .expect("Failed to process san value");
        assert_eq!(v, AlternativeName::DnsName(String::from("github.com")));
    }

    #[test]
    fn test_san_value_ipaddr() {
-        let (rem, v) = san_value(&[135, 4, 1, 1, 1, 1])
+        let (rem, v) = parse_der(&[135, 4, 1, 1, 1, 1])
            .expect("Failed to parse san value");
-        let v = v.expect("Extension contains invalid data");
        assert!(rem.is_empty());
+        println!("{:?}", v);
+        assert_eq!(v, DerObject {
+            class: 2,
+            structured: 0,
+            tag: 7,
+            content: DerObjectContent::Unknown(&[1, 1, 1, 1])
+        });
+        let content = match v.content {
+            DerObjectContent::Unknown(v) => v,
+            _ => panic!("Wrong DerObjectContent"),
+        };
+        let v = san_value_ipaddr(content)
+            .expect("Failed to process san value");
        assert_eq!(v, AlternativeName::IpAddr("1.1.1.1".parse().unwrap()));
    }
 }
--- a/src/db/mod.rs
+++ b/src/db/mod.rs
@@ -13,6 +13,8 @@ use crate::migrations;
 use crate::worker;
 use crate::workspaces::Workspace;

+pub mod ttl;
+

 #[derive(Debug)]
 pub enum DbChange {
@@ -71,19 +73,26 @@ pub struct Database {

 impl Database {
    pub fn establish(name: Workspace) -> Result<Database> {
+        let db = worker::spawn_fn("Connecting to database", || {
+            Database::establish_quiet(name)
+        }, false)?;
+
+        Ok(db)
+    }
+
+    pub fn establish_quiet(name: Workspace) -> Result<Database> {
        let path = paths::data_dir()?.join(name.to_string() + ".db");
        let path = path.into_os_string().into_string()
            .map_err(|_| format_err!("Failed to convert db path to utf-8"))?;

-        let db = worker::spawn_fn("Connecting to database", || {
-            let db = SqliteConnection::establish(&path)
-                .context("Failed to connect to database")?;
-            migrations::run(&db)
-                .context("Failed to run migrations")?;
-            db.execute("PRAGMA foreign_keys = ON")
-                .context("Failed to enforce foreign keys")?;
-            Ok(db)
-        }, false)?;
+        let db = SqliteConnection::establish(&path)
+            .context("Failed to connect to database")?;
+        migrations::run(&db)
+            .context("Failed to run migrations")?;
+        db.execute("PRAGMA journal_mode = WAL")
+            .context("Failed to enable write ahead log")?;
+        db.execute("PRAGMA foreign_keys = ON")
+            .context("Failed to enforce foreign keys")?;

        Ok(Database {
            name,
--- a/src/db/ttl.rs
+++ b/src/db/ttl.rs
@@ -0,0 +1,133 @@
+use crate::errors::*;
+use crate::db::Database;
+use crate::schema::*;
+use crate::models::*;
+use chrono::{NaiveDateTime, Duration, Utc};
+use diesel;
+use diesel::prelude::*;
+
+
+#[derive(Identifiable, Queryable, AsChangeset, PartialEq, Debug)]
+#[table_name="ttls"]
+pub struct Ttl {
+    pub id: i32,
+    pub family: String,
+    pub key: i32,
+    pub expire: NaiveDateTime,
+}
+
+#[derive(Insertable)]
+#[table_name="ttls"]
+pub struct NewTtl<'a> {
+    pub family: &'a str,
+    pub key: i32,
+    pub expire: NaiveDateTime,
+}
+
+impl Ttl {
+    pub fn new(obj: &Insert, key: i32, expire: NaiveDateTime) -> NewTtl {
+        NewTtl {
+            family: obj.table(),
+            key,
+            expire,
+        }
+    }
+
+    pub fn find(obj: &Insert, my_key: i32, db: &Database) -> Result<Option<Ttl>> {
+        use crate::schema::ttls::dsl::*;
+
+        ttls
+            .filter(family.eq(obj.table()))
+            .filter(key.eq(my_key))
+            .first::<Self>(db.db())
+            .optional()
+            .map_err(Error::from)
+    }
+
+    pub fn expired(db: &Database) -> Result<Vec<Ttl>> {
+        use crate::schema::ttls::dsl::*;
+
+        ttls
+            .filter(expire.lt(Self::ttl_to_datetime(0)))
+            .load::<Self>(db.db())
+            .map_err(Error::from)
+    }
+
+    fn ttl_to_datetime(ttl: i32) -> NaiveDateTime {
+        // TODO: maybe create Duration from string
+        let expire_at = Utc::now() + Duration::seconds(ttl as i64);
+        expire_at.naive_utc()
+    }
+
+    pub fn create(obj: &Insert, key: i32, ttl: i32, db: &Database) -> Result<()> {
+        debug!("Creating ttl on record");
+        let expire = Self::ttl_to_datetime(ttl);
+
+        diesel::insert_into(ttls::table)
+            .values(NewTtl {
+                family: obj.table(),
+                key,
+                expire,
+            })
+            .execute(db.db())?;
+
+        Ok(())
+    }
+
+    pub fn bump(obj: &Insert, my_key: i32, ttl: i32, db: &Database) -> Result<()> {
+        use crate::schema::ttls::dsl::*;
+
+        debug!("Updating ttl on record");
+
+        if let Some(mut old) = Self::find(obj, my_key, db)? {
+            let new_expire = Self::ttl_to_datetime(ttl);
+
+            if old.expire < new_expire {
+                debug!("Bumping old expire date");
+
+                old.expire = new_expire;
+
+                diesel::update(ttls.filter(id.eq(old.id)))
+                    .set(old)
+                    .execute(db.db())?;
+            }
+        } else {
+            debug!("Existing record doesn't expire, not setting a ttl");
+        }
+
+        Ok(())
+    }
+
+    pub fn delete(&self, db: &Database) -> Result<()> {
+        match self.family.as_str() {
+            "domains" => Domain::delete_id(db, self.key)?,
+            "subdomains" => Subdomain::delete_id(db, self.key)?,
+            "ipaddrs" => IpAddr::delete_id(db, self.key)?,
+            "subdomain_ipaddrs" => SubdomainIpAddr::delete_id(db, self.key)?,
+            "urls" => Url::delete_id(db, self.key)?,
+            "emails" => Email::delete_id(db, self.key)?,
+            "phonenumbers" => PhoneNumber::delete_id(db, self.key)?,
+            "devices" => Device::delete_id(db, self.key)?,
+            "networks" => Network::delete_id(db, self.key)?,
+            "network_devices" => NetworkDevice::delete_id(db, self.key)?,
+            _ => bail!("Unknown table"),
+        };
+
+        diesel::delete(self)
+                .execute(db.db())?;
+
+        Ok(())
+    }
+}
+
+pub fn reap_expired(db: &Database) -> Result<()> {
+    debug!("Reaping expired entities");
+
+    for expired in Ttl::expired(db)? {
+        debug!("Expired: {:?}", expired);
+        expired.delete(db)?;
+    }
+
+    debug!("Finished reaping expired entities");
+    Ok(())
+}
--- a/src/engine/ctx.rs
+++ b/src/engine/ctx.rs
@@ -8,14 +8,17 @@ use crate::keyring::KeyRingEntry;
 use crate::models::{Insert, Update};
 use crate::psl::Psl;
 use crate::runtime;
+use crate::sockets::Socket;
+use crate::web::{HttpSession, HttpRequest, RequestOptions};
+use crate::worker::{Event, LogEvent, DatabaseEvent, StdioEvent};
+use chrootable_https::{self, Resolver};
 use serde_json;
 use std::collections::HashMap;
 use std::result;
 use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
-use chrootable_https::dns::Resolver;
-use crate::web::{HttpSession, HttpRequest, RequestOptions};
-use crate::worker::{Event, LogEvent, DatabaseEvent, StdioEvent};
+use rand::prelude::*;
+use rand::distributions::Alphanumeric;


 pub trait State {
@@ -25,8 +28,6 @@ pub trait State {

    fn set_error(&self, err: Error) -> Error;

-    fn set_logger(&self, tx: Arc<Mutex<Box<Reporter>>>);
-
    fn send(&self, msg: &Event);

    fn recv(&self) -> Result<serde_json::Value>;
@@ -59,6 +60,14 @@ pub trait State {
        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
    }

+    fn db_insert_ttl(&self, object: Insert, ttl: i32) -> Result<Option<i32>> {
+        self.send(&Event::Database(DatabaseEvent::InsertTtl((object, ttl))));
+        let reply = self.recv()?;
+        let reply: result::Result<Option<i32>, String> = serde_json::from_value(reply)?;
+
+        reply.map_err(|err| format_err!("Failed to add to database: {:?}", err))
+    }
+
    fn db_select(&self, family: Family, value: String) -> Result<Option<i32>> {
        self.send(&Event::Database(DatabaseEvent::Select((family, value))));
        let reply = self.recv()?;
@@ -82,19 +91,29 @@ pub trait State {
        reply.map_err(|err| format_err!("Failed to read stdin: {:?}", err))
    }

+    fn random_id(&self) -> String {
+        thread_rng().sample_iter(&Alphanumeric).take(16).collect()
+    }
+
    fn keyring(&self, namespace: &str) -> Vec<&KeyRingEntry>;

-    fn dns_config(&self) -> Arc<Resolver>;
+    fn dns_config(&self) -> &Resolver;

    fn proxy(&self) -> Option<&SocketAddr>;

    fn getopt(&self, key: &str) -> Option<&String>;

-    fn psl(&self) -> Arc<Psl>;
+    fn psl(&self) -> &Psl;

-    fn geoip(&self) -> Arc<GeoIP>;
+    fn geoip(&self) -> &GeoIP;

-    fn asn(&self) -> Arc<AsnDB>;
+    fn asn(&self) -> &AsnDB;
+
+    fn sock_connect(&self, host: &str, port: u16) -> Result<String>;
+
+    fn get_sock(&self, id: &str)-> Arc<Mutex<Socket>>;
+
+    fn http(&self) -> &chrootable_https::Client<Resolver>;

    fn http_mksession(&self) -> String;

@@ -103,17 +122,19 @@ pub trait State {
    fn register_in_jar(&self, session: &str, key: String, value: String);
 }

-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct LuaState {
-    error: Arc<Mutex<Option<Error>>>,
-    logger: Arc<Mutex<Option<Arc<Mutex<Box<Reporter>>>>>>,
-    http_sessions: Arc<Mutex<HashMap<String, HttpSession>>>,
+    error: Mutex<Option<Error>>,
+    logger: Arc<Mutex<Box<Reporter>>>,
+    socket_sessions: Mutex<HashMap<String, Arc<Mutex<Socket>>>>,
+    http_sessions: Mutex<HashMap<String, HttpSession>>,
+    http: chrootable_https::Client<Resolver>,
    verbose: u64,
-    keyring: Arc<Vec<KeyRingEntry>>, // TODO: maybe hashmap
-    dns_config: Arc<Resolver>,
-    psl: Arc<Psl>,
-    geoip: Arc<GeoIP>,
-    asn: Arc<AsnDB>,
+    keyring: Vec<KeyRingEntry>, // TODO: maybe hashmap
+    dns_config: Resolver,
+    psl: Psl,
+    geoip: GeoIP,
+    asn: AsnDB,
    proxy: Option<SocketAddr>,
    options: HashMap<String, String>,
 }
@@ -136,27 +157,14 @@ impl State for LuaState {
        cp
    }

-    fn set_logger(&self, tx: Arc<Mutex<Box<Reporter>>>) {
-        let mut mtx = self.logger.lock().unwrap();
-        *mtx = Some(tx);
-    }
-
    fn send(&self, msg: &Event) {
-        let mtx = self.logger.lock().unwrap();
-        if let Some(mtx) = &*mtx {
-            let mut tx = mtx.lock().unwrap();
-            tx.send(msg).expect("Failed to write event");
-        }
+        let mut tx = self.logger.lock().unwrap();
+        tx.send(msg).expect("Failed to write event");
    }

    fn recv(&self) -> Result<serde_json::Value> {
-        let mtx = self.logger.lock().unwrap();
-        if let Some(mtx) = &*mtx {
-            let mut tx = mtx.lock().unwrap();
-            tx.recv()
-        } else {
-            bail!("Failed to read from reporter, non available");
-        }
+        let mut tx = self.logger.lock().unwrap();
+        tx.recv()
    }

    fn verbose(&self) -> u64 {
@@ -169,8 +177,8 @@ impl State for LuaState {
            .collect()
    }

-    fn dns_config(&self) -> Arc<Resolver> {
-        self.dns_config.clone()
+    fn dns_config(&self) -> &Resolver {
+        &self.dns_config
    }

    fn proxy(&self) -> Option<&SocketAddr> {
@@ -181,16 +189,40 @@ impl State for LuaState {
        self.options.get(key)
    }

-    fn psl(&self) -> Arc<Psl> {
-        self.psl.clone()
+    fn psl(&self) -> &Psl {
+        &self.psl
    }

-    fn geoip(&self) -> Arc<GeoIP> {
-        self.geoip.clone()
+    fn geoip(&self) -> &GeoIP {
+        &self.geoip
    }

-    fn asn(&self) -> Arc<AsnDB> {
-        self.asn.clone()
+    fn asn(&self) -> &AsnDB {
+        &self.asn
+    }
+
+    fn sock_connect(&self, host: &str, port: u16) -> Result<String> {
+        let mut mtx = self.socket_sessions.lock().unwrap();
+        let id = self.random_id();
+
+        let sock = match &self.proxy {
+            Some(proxy) => Socket::connect_socks5(proxy, host, port)?,
+            _ => Socket::connect(&self.dns_config, host, port)?,
+        };
+
+        mtx.insert(id.clone(), Arc::new(Mutex::new(sock)));
+
+        Ok(id)
+    }
+
+    fn get_sock(&self, id: &str)-> Arc<Mutex<Socket>> {
+        let mtx = self.socket_sessions.lock().unwrap();
+        let sock = mtx.get(id).expect("invalid session reference"); // TODO
+        sock.clone()
+    }
+
+    fn http(&self) -> &chrootable_https::Client<Resolver> {
+        &self.http
    }

    fn http_mksession(&self) -> String {
@@ -204,7 +236,7 @@ impl State for LuaState {
        let mtx = self.http_sessions.lock().unwrap();
        let session = mtx.get(session_id).expect("invalid session reference"); // TODO

-        HttpRequest::new(&session, method, url, options, self.proxy.clone())
+        HttpRequest::new(&session, method, url, options)
    }

    fn register_in_jar(&self, session: &str, key: String, value: String) {
@@ -220,27 +252,40 @@ pub struct Script {
    code: String,
 }

-fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
+fn ctx<'a>(env: Environment, logger: Arc<Mutex<Box<Reporter>>>) -> (hlua::Lua<'a>, Arc<LuaState>) {
    debug!("Creating lua context");
    let mut lua = hlua::Lua::new();
    lua.open_string();
+
+    let http = match env.proxy {
+        Some(proxy) => chrootable_https::Client::with_socks5(proxy),
+        _ => {
+            let resolver = env.dns_config.clone();
+            chrootable_https::Client::new(resolver)
+        },
+    };
+
    let state = Arc::new(LuaState {
-        error: Arc::new(Mutex::new(None)),
-        logger: Arc::new(Mutex::new(None)),
-        http_sessions: Arc::new(Mutex::new(HashMap::new())),
+        error: Mutex::new(None),
+        logger,
+        socket_sessions: Mutex::new(HashMap::new()),
+        http_sessions: Mutex::new(HashMap::new()),
+        http,

        verbose: env.verbose,
-        keyring: Arc::new(env.keyring),
-        dns_config: Arc::new(env.dns_config),
-        psl: Arc::new(env.psl),
-        geoip: Arc::new(env.geoip),
-        asn: Arc::new(env.asn),
+        keyring: env.keyring,
+        dns_config: env.dns_config,
+        psl: env.psl,
+        geoip: env.geoip,
+        asn: env.asn,
        proxy: env.proxy,
        options: env.options,
    });

    runtime::clear_err(&mut lua, state.clone());
+    runtime::datetime(&mut lua, state.clone());
    runtime::db_add(&mut lua, state.clone());
+    runtime::db_add_ttl(&mut lua, state.clone());
    runtime::db_select(&mut lua, state.clone());
    runtime::db_update(&mut lua, state.clone());
    runtime::debug(&mut lua, state.clone());
@@ -249,6 +294,7 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
    runtime::asn_lookup(&mut lua, state.clone());
    runtime::geoip_lookup(&mut lua, state.clone());
    runtime::getopt(&mut lua, state.clone());
+    runtime::hex(&mut lua, state.clone());
    runtime::html_select(&mut lua, state.clone());
    runtime::html_select_list(&mut lua, state.clone());
    runtime::http_mksession(&mut lua, state.clone());
@@ -267,6 +313,18 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
    runtime::regex_find(&mut lua, state.clone());
    runtime::regex_find_all(&mut lua, state.clone());
    runtime::sleep(&mut lua, state.clone());
+    runtime::sock_connect(&mut lua, state.clone());
+    runtime::sock_send(&mut lua, state.clone());
+    runtime::sock_recv(&mut lua, state.clone());
+    runtime::sock_sendline(&mut lua, state.clone());
+    runtime::sock_recvline(&mut lua, state.clone());
+    runtime::sock_recvall(&mut lua, state.clone());
+    runtime::sock_recvline_contains(&mut lua, state.clone());
+    runtime::sock_recvline_regex(&mut lua, state.clone());
+    runtime::sock_recvn(&mut lua, state.clone());
+    runtime::sock_recvuntil(&mut lua, state.clone());
+    runtime::sock_sendafter(&mut lua, state.clone());
+    runtime::sock_newline(&mut lua, state.clone());
    runtime::status(&mut lua, state.clone());
    runtime::stdin_readline(&mut lua, state.clone());
    runtime::url_decode(&mut lua, state.clone());
@@ -307,13 +365,11 @@ impl Script {
                      tx: Arc<Mutex<Box<Reporter>>>,
                      arg: AnyLuaValue
    ) -> Result<()> {
-        let (mut lua, state) = ctx(env);
+        let (mut lua, state) = ctx(env, tx);

        debug!("Initializing lua module");
        lua.execute::<()>(&self.code)?;

-        state.set_logger(tx);
-
        let run: Result<_> = lua.get("run")
            .ok_or_else(|| format_err!( "run undefined"));
        let mut run: hlua::LuaFunction<_> = run?;
--- a/src/engine/mod.rs
+++ b/src/engine/mod.rs
@@ -45,7 +45,7 @@ pub struct Engine {
 }

 impl Engine {
-    pub fn new() -> Result<Engine> {
+    pub fn new(verbose_init: bool) -> Result<Engine> {
        let path = paths::module_dir()?;

        let mut engine = Engine {
@@ -53,7 +53,11 @@ impl Engine {
            modules: HashMap::new(),
        };

-        engine.reload_modules()?;
+        if verbose_init {
+            engine.reload_modules()?;
+        } else {
+            engine.reload_modules_quiet()?;
+        }

        Ok(engine)
    }
--- a/src/engine/structs.rs
+++ b/src/engine/structs.rs
@@ -1,5 +1,5 @@
 use crate::errors::*;
-use crate::hlua::{AnyHashableLuaValue, AnyLuaValue};
+use crate::hlua::{AnyHashableLuaValue, AnyLuaValue, AnyLuaString};
 use std::collections::{self, HashMap};
 use crate::json::LuaJsonValue;
 use serde;
@@ -151,3 +151,8 @@ pub fn byte_array(bytes: AnyLuaValue) -> Result<Vec<u8>> {
        _ => Err(format_err!("invalid type: {:?}", bytes)),
    }
 }
+
+pub fn lua_bytes(bytes: &[u8]) -> AnyLuaValue {
+    let bytes = AnyLuaString(bytes.to_vec());
+    AnyLuaValue::LuaAnyString(bytes)
+}
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -13,7 +13,6 @@ use hlua_badtouch as hlua;
 #[macro_use] extern crate diesel;
 #[macro_use] extern crate diesel_migrations;
 #[macro_use] extern crate crossbeam_channel as channel;
-#[macro_use] extern crate nom;

 pub mod api;
 pub mod archive;
@@ -38,10 +37,11 @@ pub mod psl;
 pub mod options;
 pub mod registry;
 pub mod runtime;
-pub mod ser;
 pub mod sandbox;
 pub mod schema;
+pub mod ser;
 pub mod shell;
+pub mod sockets;
 pub mod term;
 pub mod utils;
 pub mod web;
--- a/src/main.rs
+++ b/src/main.rs
@@ -17,11 +17,13 @@ use sn0int::registry;
 use sn0int::sandbox;
 use sn0int::shell;
 use structopt::StructOpt;
+use std::fs::OpenOptions;
+use std::io::Write;
 use std::path::Path;


 fn run_run(gargs: &Args, args: &args::Run, config: Config) -> Result<()> {
-    let mut rl = shell::init(gargs, config)?;
+    let mut rl = shell::init(gargs, config, false)?;

    if let Some(module) = &args.module {
        let module = rl.engine().get(&module)?.clone();
@@ -55,10 +57,29 @@ fn run_sandbox() -> Result<()> {
 }

 fn run_cmd<T: cmd::Cmd>(gargs: &Args, args: &T, config: Config) -> Result<()> {
-    let mut rl = shell::init(gargs, config)?;
+    let mut rl = shell::init(gargs, config, false)?;
    args.run(&mut rl)
 }

+fn run_new(_gargs: &Args, args: &args::New) -> Result<()> {
+    let boilerplate = b"-- Description: TODO your description here
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- TODO your code here
+end
+";
+
+    let mut file = OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(&args.path)?;
+    file.write_all(boilerplate)?;
+
+    Ok(())
+}
+
 fn run() -> Result<()> {
    let args = Args::from_args();

@@ -73,6 +94,7 @@ fn run() -> Result<()> {
        Some(SubCommand::Run(ref run)) => run_run(&args, run, config),
        Some(SubCommand::Sandbox(_)) => run_sandbox(),
        Some(SubCommand::Login(_)) => auth::run_login(&config),
+        Some(SubCommand::New(ref new)) => run_new(&args, new),
        Some(SubCommand::Publish(ref publish)) => registry::run_publish(&args, publish, &config),
        Some(SubCommand::Install(ref install)) => registry::run_install(install, &config),
        Some(SubCommand::Search(ref search)) => registry::run_search(search, &config),
--- a/src/models/device.rs
+++ b/src/models/device.rs
@@ -50,6 +50,14 @@ impl Model for Device {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::devices::dsl::*;
+
+        diesel::delete(devices.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
--- a/src/models/domain.rs
+++ b/src/models/domain.rs
@@ -45,6 +45,14 @@ impl Model for Domain {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::domains::dsl::*;
+
+        diesel::delete(domains.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
--- a/src/models/email.rs
+++ b/src/models/email.rs
@@ -1,4 +1,5 @@
 use crate::errors::*;
+use crate::fmt::Write;
 use crate::fmt::colors::*;
 use diesel;
 use diesel::prelude::*;
@@ -46,6 +47,14 @@ impl Model for Email {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::emails::dsl::*;
+
+        diesel::delete(emails.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
@@ -129,6 +138,7 @@ pub struct DetailedEmail {
    id: i32,
    value: String,
    unscoped: bool,
+    valid: Option<bool>,
 }

 impl DisplayableDetailed for DetailedEmail {
@@ -141,6 +151,17 @@ impl DisplayableDetailed for DetailedEmail {
    fn print(&self, w: &mut fmt::DetailFormatter) -> fmt::Result {
        w.id(self.id)?;
        w.debug::<Green, _>(&self.value)?;
+
+        if let Some(valid) = self.valid {
+            write!(w, " [")?;
+            if valid {
+                w.display::<Green, _>("valid")?;
+            } else {
+                w.display::<Red, _>("invalid")?;
+            }
+            write!(w, "]")?;
+        }
+
        Ok(())
    }

@@ -160,6 +181,7 @@ impl Detailed for Email {
            id: self.id,
            value: self.value.to_string(),
            unscoped: self.unscoped,
+            valid: self.valid,
        })
    }
 }
--- a/src/models/ipaddr.rs
+++ b/src/models/ipaddr.rs
@@ -60,6 +60,14 @@ impl Model for IpAddr {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::ipaddrs::dsl::*;
+
+        diesel::delete(ipaddrs.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
--- a/src/models/mod.rs
+++ b/src/models/mod.rs
@@ -34,6 +34,21 @@ impl Insert {
        }
    }

+    pub fn table(&self) -> &str {
+        match self {
+            Insert::Domain(_) => "domains",
+            Insert::Subdomain(_) => "subdomains",
+            Insert::IpAddr(_) => "ipaddrs",
+            Insert::SubdomainIpAddr(_) => "subdomain_ipaddrs",
+            Insert::Url(_) => "urls",
+            Insert::Email(_) => "emails",
+            Insert::PhoneNumber(_) => "phonenumbers",
+            Insert::Device(_) => "devices",
+            Insert::Network(_) => "networks",
+            Insert::NetworkDevice(_) => "network_devices",
+        }
+    }
+
    pub fn printable(&self, db: &Database) -> Result<String> {
        Ok(match self {
            Insert::Domain(x) => format!("Domain: {}", x.printable(db)?),
@@ -103,6 +118,8 @@ pub trait Model: Sized {

    fn delete(db: &Database, filter: &Filter) -> Result<usize>;

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize>;
+
    fn id(&self) -> i32;

    fn value(&self) -> &Self::ID {
--- a/src/models/network.rs
+++ b/src/models/network.rs
@@ -48,6 +48,14 @@ impl Model for Network {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::networks::dsl::*;
+
+        diesel::delete(networks.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
--- a/src/models/network_device.rs
+++ b/src/models/network_device.rs
@@ -49,6 +49,14 @@ impl Model for NetworkDevice {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::network_devices::dsl::*;
+
+        diesel::delete(network_devices.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
--- a/src/models/phonenumber.rs
+++ b/src/models/phonenumber.rs
@@ -56,6 +56,14 @@ impl Model for PhoneNumber {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::phonenumbers::dsl::*;
+
+        diesel::delete(phonenumbers.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
--- a/src/models/subdomain.rs
+++ b/src/models/subdomain.rs
@@ -49,6 +49,14 @@ impl Model for Subdomain {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::subdomains::dsl::*;
+
+        diesel::delete(subdomains.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
--- a/src/models/subdomain_ipaddr.rs
+++ b/src/models/subdomain_ipaddr.rs
@@ -47,6 +47,14 @@ impl Model for SubdomainIpAddr {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::subdomain_ipaddrs::dsl::*;
+
+        diesel::delete(subdomain_ipaddrs.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
--- a/src/models/url.rs
+++ b/src/models/url.rs
@@ -56,6 +56,14 @@ impl Model for Url {
            .map_err(Error::from)
    }

+    fn delete_id(db: &Database, my_id: i32) -> Result<usize> {
+        use crate::schema::urls::dsl::*;
+
+        diesel::delete(urls.filter(id.eq(my_id)))
+            .execute(db.db())
+            .map_err(Error::from)
+    }
+
    fn id(&self) -> i32 {
        self.id
    }
--- a/src/registry.rs
+++ b/src/registry.rs
@@ -39,11 +39,17 @@ pub fn run_publish(_args: &Args, publish: &Publish, config: &Config) -> Result<(
        let label = format!("Uploading {} {} ({:?})", name, metadata.version, path);
        match worker::spawn_fn(&label, || {
            client.publish_module(&name, code.to_string())
-        }, false) {
-            Ok(result) => term::info(&format!("Published as {}/{} {}", result.author,
-                                                                       result.name,
-                                                                       result.version)),
-            Err(err) => term::error(&format!("Failed ({:?}): {}", 1, err)),
+        }, true) {
+            Ok(result) => term::info(&format!("Published {}/{} {} ({:?})",
+                                              result.author,
+                                              result.name,
+                                              result.version,
+                                              path)),
+            Err(err) => term::error(&format!("Failed to publish {} {} ({:?}): {}",
+                                             name,
+                                             metadata.version,
+                                             path,
+                                             err)),
        }
    }

--- a/src/runtime/datetime.rs
+++ b/src/runtime/datetime.rs
@@ -0,0 +1,32 @@
+use crate::engine::ctx::State;
+use crate::hlua;
+use chrono::{Utc};
+use std::sync::Arc;
+
+
+pub fn datetime(lua: &mut hlua::Lua, _: Arc<State>) {
+    lua.set("datetime", hlua::function0(move || -> String {
+        let now = Utc::now().naive_utc();
+        now.format("%Y-%m-%dT%H:%M:%S")
+           .to_string()
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::engine::ctx::Script;
+
+    #[test]
+    fn verify_datetime() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            now = datetime()
+            print(now)
+            if regex_find("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", now) == nil then
+                return 'invalid date'
+            end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+}
--- a/src/runtime/db.rs
+++ b/src/runtime/db.rs
@@ -18,60 +18,70 @@ pub fn try_into_new<T: LuaInsertToNewOwned>(x: LuaJsonValue) -> Result<T::Target
        .try_into_new()
 }

+fn into_insert(family: Family, object: LuaJsonValue) -> Result<Insert> {
+    let obj = match family {
+        Family::Domain => {
+            Insert::Domain(try_into_new::<InsertDomain>(object)?)
+        },
+        Family::Subdomain => {
+            Insert::Subdomain(try_into_new::<InsertSubdomain>(object)?)
+        },
+        Family::IpAddr => {
+            Insert::IpAddr(try_into_new::<InsertIpAddr>(object)?)
+        },
+        Family::SubdomainIpAddr => {
+            Insert::SubdomainIpAddr(try_into_new::<InsertSubdomainIpAddr>(object)?)
+        },
+        Family::Url => {
+            Insert::Url(try_into_new::<InsertUrl>(object)?)
+        },
+        Family::Email => {
+            Insert::Email(try_into_new::<InsertEmail>(object)?)
+        },
+        Family::PhoneNumber => {
+            Insert::PhoneNumber(try_into_new::<InsertPhoneNumber>(object)?)
+        },
+        Family::Device => {
+            Insert::Device(try_into_new::<InsertDevice>(object)?)
+        },
+        Family::Network => {
+            Insert::Network(try_into_new::<InsertNetwork>(object)?)
+        },
+        Family::NetworkDevice => {
+            Insert::NetworkDevice(try_into_new::<InsertNetworkDevice>(object)?)
+        },
+    };
+    Ok(obj)
+}
+
 pub fn db_add(lua: &mut hlua::Lua, state: Arc<State>) {
    lua.set("db_add", hlua::function2(move |family: String, object: AnyLuaValue| -> Result<Option<i32>> {
        let family = Family::from_str(&family)
            .map_err(|e| state.set_error(e))?;
        let object = LuaJsonValue::from(object);

-        let object = match family {
-            Family::Domain => {
-                Insert::Domain(try_into_new::<InsertDomain>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::Subdomain => {
-                Insert::Subdomain(try_into_new::<InsertSubdomain>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::IpAddr => {
-                Insert::IpAddr(try_into_new::<InsertIpAddr>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::SubdomainIpAddr => {
-                Insert::SubdomainIpAddr(try_into_new::<InsertSubdomainIpAddr>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::Url => {
-                Insert::Url(try_into_new::<InsertUrl>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::Email => {
-                Insert::Email(try_into_new::<InsertEmail>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::PhoneNumber => {
-                Insert::PhoneNumber(try_into_new::<InsertPhoneNumber>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::Device => {
-                Insert::Device(try_into_new::<InsertDevice>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::Network => {
-                Insert::Network(try_into_new::<InsertNetwork>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-            Family::NetworkDevice => {
-                Insert::NetworkDevice(try_into_new::<InsertNetworkDevice>(object)
-                    .map_err(|e| state.set_error(e))?)
-            },
-        };
+        let object = into_insert(family, object)
+            .map_err(|e| state.set_error(e))?;

        state.db_insert(object)
            .map_err(|e| state.set_error(e))
    }))
 }

+pub fn db_add_ttl(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("db_add_ttl", hlua::function3(move |family: String, object: AnyLuaValue, ttl: i32| -> Result<Option<i32>> {
+        let family = Family::from_str(&family)
+            .map_err(|e| state.set_error(e))?;
+        let object = LuaJsonValue::from(object);
+
+        let object = into_insert(family, object)
+            .map_err(|e| state.set_error(e))?;
+
+        state.db_insert_ttl(object, ttl)
+            .map_err(|e| state.set_error(e))
+    }))
+}
+
 pub fn db_select(lua: &mut hlua::Lua, state: Arc<State>) {
    lua.set("db_select", hlua::function2(move |family: String, value: String| -> Result<Option<i32>> {
        let family = Family::from_str(&family)
--- a/src/runtime/hex.rs
+++ b/src/runtime/hex.rs
@@ -0,0 +1,23 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::engine::structs::byte_array;
+use crate::hlua::{self, AnyLuaValue};
+use std::sync::Arc;
+
+
+pub fn hex(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("hex", hlua::function1(move |bytes: AnyLuaValue| -> Result<String> {
+        byte_array(bytes)
+            .map_err(|err| state.set_error(err))
+            .map(|bytes| {
+                let mut out = String::new();
+
+                for b in bytes {
+                    out += &format!("{:02x}", b);
+                }
+
+                out
+            })
+    }))
+}
--- a/src/runtime/mod.rs
+++ b/src/runtime/mod.rs
@@ -5,10 +5,12 @@ macro_rules! import_fns {
    };
 }

+import_fns!(datetime);
 import_fns!(db);
 import_fns!(dns);
 import_fns!(error);
 import_fns!(geoip);
+import_fns!(hex);
 import_fns!(http);
 import_fns!(html);
 import_fns!(json);
@@ -19,6 +21,7 @@ import_fns!(pgp);
 import_fns!(psl);
 import_fns!(regex);
 import_fns!(sleep);
+import_fns!(sock);
 import_fns!(stdio);
 import_fns!(url);
 import_fns!(utf8);
--- a/src/runtime/options.rs
+++ b/src/runtime/options.rs
@@ -1,6 +1,5 @@
 use crate::engine::ctx::State;
 use crate::hlua;
-use serde_json;
 use std::sync::Arc;


--- a/src/runtime/sock.rs
+++ b/src/runtime/sock.rs
@@ -0,0 +1,155 @@
+use crate::errors::*;
+
+use crate::engine::ctx::State;
+use crate::engine::structs::{byte_array, lua_bytes};
+use crate::hlua::{self, AnyLuaValue};
+use std::sync::Arc;
+
+
+pub fn sock_connect(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_connect", hlua::function3(move |host: String, port: u16, _settings: AnyLuaValue| -> Result<String> {
+        state.sock_connect(&host, port)
+            .map_err(|err| state.set_error(err))
+    }))
+}
+
+pub fn sock_send(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_send", hlua::function2(move |sock: String, bytes: AnyLuaValue| -> Result<()> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let bytes = byte_array(bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        sock.send(&bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(())
+    }))
+}
+
+pub fn sock_recv(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_recv", hlua::function1(move |sock: String| -> Result<AnyLuaValue> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let bytes = sock.recv()
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(lua_bytes(&bytes))
+    }))
+}
+
+pub fn sock_sendline(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_sendline", hlua::function2(move |sock: String, line: String| -> Result<()> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        sock.sendline(&line)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(())
+    }))
+}
+
+pub fn sock_recvline(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_recvline", hlua::function1(move |sock: String| -> Result<String> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let line = sock.recvline()
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(line)
+    }))
+}
+
+pub fn sock_recvall(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_recvall", hlua::function1(move |sock: String| -> Result<AnyLuaValue> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let bytes = sock.recvall()
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(lua_bytes(&bytes))
+    }))
+}
+
+pub fn sock_recvline_contains(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_recvline_contains", hlua::function2(move |sock: String, needle: String| -> Result<String> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let line = sock.recvline_contains(&needle)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(line)
+    }))
+}
+
+pub fn sock_recvline_regex(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_recvline_regex", hlua::function2(move |sock: String, regex: String| -> Result<String> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let line = sock.recvline_regex(&regex)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(line)
+    }))
+}
+
+pub fn sock_recvn(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_recvn", hlua::function2(move |sock: String, n: u32| -> Result<AnyLuaValue> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let bytes = sock.recvn(n)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(lua_bytes(&bytes))
+    }))
+}
+
+pub fn sock_recvuntil(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_recvuntil", hlua::function2(move |sock: String, delim: AnyLuaValue| -> Result<AnyLuaValue> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let delim = byte_array(delim)
+            .map_err(|err| state.set_error(err))?;
+
+        let bytes = sock.recvuntil(&delim)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(lua_bytes(&bytes))
+    }))
+}
+
+pub fn sock_sendafter(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_sendafter", hlua::function3(move |sock: String, delim: AnyLuaValue, bytes: AnyLuaValue| -> Result<()> {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        let delim = byte_array(delim)
+            .map_err(|err| state.set_error(err))?;
+
+        let bytes = byte_array(bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        sock.sendafter(&delim, &bytes)
+            .map_err(|err| state.set_error(err))?;
+
+        Ok(())
+    }))
+}
+
+pub fn sock_newline(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("sock_newline", hlua::function2(move |sock: String, newline: String| -> () {
+        let sock = state.get_sock(&sock);
+        let mut sock = sock.lock().unwrap();
+
+        sock.newline(newline);
+    }))
+}
--- a/src/sandbox/seccomp.rs
+++ b/src/sandbox/seccomp.rs
@@ -1,5 +1,5 @@
 use crate::errors::*;
-use syscallz::{Context, Syscall};
+use syscallz::{Context, Syscall, Action};


 pub fn init() -> Result<()> {
@@ -7,14 +7,13 @@ pub fn init() -> Result<()> {

    ctx.allow_syscall(Syscall::read)?;
    ctx.allow_syscall(Syscall::write)?;
+    ctx.allow_syscall(Syscall::readv)?;
+    ctx.allow_syscall(Syscall::writev)?;
    ctx.allow_syscall(Syscall::futex)?;
    ctx.allow_syscall(Syscall::sigaltstack)?;
    ctx.allow_syscall(Syscall::munmap)?;
-    //ctx.allow_syscall(Syscall::openat)?;
-    //#[cfg(not(target_arch = "aarch64"))]
-    //ctx.allow_syscall(Syscall::open)?;
    ctx.allow_syscall(Syscall::fcntl)?;
-    #[cfg(target_arch = "arm")]
+    #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
    ctx.allow_syscall(Syscall::fcntl64)?;
    ctx.allow_syscall(Syscall::uname)?;
    ctx.allow_syscall(Syscall::close)?;
@@ -24,6 +23,8 @@ pub fn init() -> Result<()> {
    ctx.allow_syscall(Syscall::sched_getaffinity)?;
    ctx.allow_syscall(Syscall::socket)?;
    ctx.allow_syscall(Syscall::connect)?;
+    #[cfg(target_arch = "x86")]
+    ctx.allow_syscall(Syscall::socketcall)?;
    #[cfg(not(target_arch = "aarch64"))]
    ctx.allow_syscall(Syscall::epoll_wait)?;
    ctx.allow_syscall(Syscall::epoll_pwait)?;
@@ -39,7 +40,7 @@ pub fn init() -> Result<()> {
    ctx.allow_syscall(Syscall::getsockopt)?;
    #[cfg(not(target_arch = "arm"))]
    ctx.allow_syscall(Syscall::mmap)?;
-    #[cfg(target_arch = "arm")]
+    #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
    ctx.allow_syscall(Syscall::mmap2)?;
    ctx.allow_syscall(Syscall::mprotect)?;
    ctx.allow_syscall(Syscall::clone)?;
@@ -48,6 +49,9 @@ pub fn init() -> Result<()> {
    ctx.allow_syscall(Syscall::sched_yield)?;
    ctx.allow_syscall(Syscall::setsockopt)?;
    ctx.allow_syscall(Syscall::madvise)?;
+    #[cfg(target_arch = "x86")]
+    ctx.allow_syscall(Syscall::time)?;
+    ctx.allow_syscall(Syscall::clock_gettime)?;
    ctx.allow_syscall(Syscall::nanosleep)?;
    ctx.allow_syscall(Syscall::exit)?;
    ctx.allow_syscall(Syscall::exit_group)?;
@@ -55,6 +59,10 @@ pub fn init() -> Result<()> {
    ctx.allow_syscall(Syscall::rt_sigprocmask)?;
    ctx.allow_syscall(Syscall::getpeername)?;

+    ctx.set_action_for_syscall(Action::Errno(1), Syscall::openat)?;
+    #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
+    ctx.set_action_for_syscall(Action::Errno(1), Syscall::open)?;
+
    ctx.load()?;

    Ok(())
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -103,6 +103,15 @@ table! {
    }
 }

+table! {
+    ttls (id) {
+        id -> Integer,
+        family -> Text,
+        key -> Integer,
+        expire -> Timestamp,
+    }
+}
+
 table! {
    urls (id) {
        id -> Integer,
@@ -135,5 +144,6 @@ allow_tables_to_appear_in_same_query!(
    phonenumbers,
    subdomain_ipaddrs,
    subdomains,
+    ttls,
    urls,
 );
--- a/src/shell.rs
+++ b/src/shell.rs
@@ -4,6 +4,7 @@ use crate::args::Args;
 use crate::cmd::*;
 use crate::complete::CmdCompleter;
 use crate::config::Config;
+use crate::db::ttl;
 use crate::keyring::KeyRing;
 use colored::Colorize;
 use ctrlc;
@@ -380,20 +381,26 @@ pub fn run_once(rl: &mut Readline) -> Result<bool> {
    Ok(false)
 }

-pub fn init(args: &Args, config: Config) -> Result<Readline> {
+pub fn init(args: &Args, config: Config, verbose_init: bool) -> Result<Readline> {
    let workspace = match args.workspace {
        Some(ref workspace) => workspace.clone(),
        None => Workspace::from_str("default").unwrap(),
    };

-    let db = Database::establish(workspace)?;
+    let db = if verbose_init {
+        Database::establish(workspace)?
+    } else {
+        Database::establish_quiet(workspace)?
+    };
+    ttl::reap_expired(&db)?;
+
    let psl = Psl::open_or_download()?;
    let _geoip = GeoIP::open_or_download()?;
    let _asndb = AsnDB::open_or_download()?;
-    let engine = Engine::new()?;
+    let engine = Engine::new(verbose_init)?;
    let keyring = KeyRing::init()?;

-    if engine.list().is_empty() {
+    if verbose_init && engine.list().is_empty() {
        term::success("No modules found, run quickstart to install default modules");
    }

@@ -405,7 +412,7 @@ pub fn init(args: &Args, config: Config) -> Result<Readline> {
 pub fn run(args: &Args, config: Config) -> Result<()> {
    print_banner();

-    let mut rl = init(args, config)?;
+    let mut rl = init(args, config, true)?;
    rl.load_history().ok();

    rl.set_signal_handler()
--- a/src/sockets.rs
+++ b/src/sockets.rs
@@ -0,0 +1,210 @@
+use crate::errors::*;
+
+use bufstream::BufStream;
+use chrootable_https::dns::{DnsResolver, RecordType};
+use chrootable_https::socks5::{self, ProxyDest};
+use regex::Regex;
+use tokio::runtime::Runtime;
+
+use std::str;
+use std::io;
+use std::io::prelude::*;
+use std::io::BufRead;
+use std::net::SocketAddr;
+use std::net::TcpStream;
+use std::net::{IpAddr, Ipv4Addr};
+
+
+#[cfg(unix)]
+fn unwrap_socket(socket: tokio::net::TcpStream) -> Result<TcpStream> {
+    use std::os::unix::io::AsRawFd;
+    use std::os::unix::io::FromRawFd;
+    let socket2 = unsafe { TcpStream::from_raw_fd(socket.as_raw_fd()) };
+    let socket = socket2.try_clone()?;
+    Ok(socket)
+}
+
+#[cfg(windows)]
+fn unwrap_socket(socket: tokio::net::TcpStream) -> Result<TcpStream> {
+    bail!("Unwrapping tokio sockets into std sockets isn't supported on windows")
+}
+
+#[derive(Debug)]
+pub struct Socket {
+    stream: BufStream<TcpStream>,
+    newline: String,
+}
+
+impl Socket {
+    pub fn new(stream: TcpStream) -> Socket {
+        let stream = BufStream::new(stream);
+        Socket {
+            stream,
+            newline: String::from("\n"),
+        }
+    }
+
+    pub fn connect<R: DnsResolver>(resolver: &R, host: &str, port: u16) -> Result<Socket> {
+        let addrs = match host.parse::<IpAddr>() {
+            Ok(addr) => vec![addr],
+            Err(_) => resolver.resolve(host, RecordType::A)
+                .wait_for_response()?
+                .success()?,
+        };
+
+        let mut errors = Vec::new();
+
+        for addr in addrs {
+            debug!("connecting to {}:{}", addr, port);
+            match TcpStream::connect((addr, port)) {
+                Ok(socket) => {
+                    debug!("successfully connected to {:?}", addr);
+                    return Ok(Socket::new(socket));
+                },
+                Err(err) => errors.push((addr, err)),
+            }
+        }
+
+        if errors.is_empty() {
+            bail!("no dns records found");
+        } else {
+            bail!("couldn't connect: {:?}", errors);
+        }
+    }
+
+    pub fn connect_socks5(proxy: &SocketAddr, host: &str, port: u16) -> Result<Socket> {
+        debug!("connecting to {:?}:{:?} with socks5 on {:?}", host, port, proxy);
+
+        let host = match host.parse::<Ipv4Addr>() {
+            Ok(ipaddr) => ProxyDest::Ipv4Addr(ipaddr),
+            _ => ProxyDest::Domain(host.to_string()),
+        };
+
+        let fut = socks5::connect(proxy, host, port);
+
+        let mut rt = Runtime::new()?;
+        let socket = rt.block_on(fut)?;
+
+        let socket = unwrap_socket(socket)?;
+
+        return Ok(Socket::new(socket));
+    }
+
+    pub fn send(&mut self, data: &[u8]) -> Result<()> {
+        match str::from_utf8(&data) {
+            Ok(data) => debug!("send: {:?}", data),
+            Err(_) => debug!("send: {:?}", data),
+        };
+        self.stream.write_all(data)?;
+        self.stream.flush()?;
+        Ok(())
+    }
+
+    pub fn recv(&mut self) -> Result<Vec<u8>> {
+        let mut buf = [0; 4096];
+        let n = self.stream.read(&mut buf)?;
+        let data = buf[..n].to_vec();
+        match str::from_utf8(&data) {
+            Ok(data) => debug!("recv: {:?}", data),
+            Err(_) => debug!("recv: {:?}", data),
+        };
+        Ok(data)
+    }
+
+    pub fn sendline(&mut self, line: &str) -> Result<()> {
+        let line = format!("{}{}", line, self.newline);
+        self.send(line.as_bytes())
+    }
+
+    pub fn recvline(&mut self) -> Result<String> {
+        let needle = self.newline.clone();
+        let buf = self.recvuntil(needle.as_bytes())?;
+        let line = String::from_utf8(buf)
+            .context("Failed to decode utf8")?;
+        Ok(line)
+    }
+
+    pub fn recvall(&mut self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+        self.stream.read_to_end(&mut buf)?;
+        match str::from_utf8(&buf) {
+            Ok(buf) => debug!("recvall: {:?}", buf),
+            Err(_) => debug!("recvall: {:?}", buf),
+        };
+        Ok(buf)
+    }
+
+    pub fn recvline_contains(&mut self, needle: &str) -> Result<String> {
+        loop {
+            let line = self.recvline()?;
+            if line.contains(needle) {
+                return Ok(line);
+            }
+        }
+    }
+
+    pub fn recvline_regex(&mut self, regex: &str) -> Result<String> {
+        let regex = Regex::new(regex)?;
+        loop {
+            let line = self.recvline()?;
+            if regex.is_match(&line) {
+                return Ok(line);
+            }
+        }
+    }
+
+    pub fn recvn(&mut self, n: u32) -> Result<Vec<u8>> {
+        let mut buf = vec![0; n as usize];
+        self.stream.read_exact(buf.as_mut_slice())?;
+        match str::from_utf8(&buf) {
+            Ok(buf) => debug!("recvn: {:?}", buf),
+            Err(_) => debug!("recvn: {:?}", buf),
+        };
+        Ok(buf.to_vec())
+    }
+
+    pub fn recvuntil(&mut self, delim: &[u8]) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        let delim_len = delim.len();
+
+        loop {
+            let (done, used) = {
+                let available = match self.stream.fill_buf() {
+                    Ok(n) => n,
+                    Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
+                    Err(e) => return Err(e.into())
+                };
+
+                match available.windows(delim_len).position(|window| window == delim) {
+                    Some(i) => {
+                        buf.extend_from_slice(&available[..i + delim_len]);
+                        (true, i + delim_len)
+                    }
+                    None => {
+                        buf.extend_from_slice(available);
+                        (false, available.len())
+                    }
+                }
+            };
+            self.stream.consume(used);
+
+            if done || used == 0 {
+                match str::from_utf8(&buf) {
+                    Ok(buf) => debug!("recvuntil: {:?}", buf),
+                    Err(_) => debug!("recvuntil: {:?}", buf),
+                };
+                return Ok(buf);
+            }
+        }
+    }
+
+    pub fn sendafter(&mut self, delim: &[u8], data: &[u8]) -> Result<()> {
+        self.recvuntil(delim)?;
+        self.send(data)
+    }
+
+    pub fn newline<I: Into<String>>(&mut self, delim: I) {
+        self.newline = delim.into();
+    }
+}
--- a/src/term.rs
+++ b/src/term.rs
@@ -1,3 +1,4 @@
+use atty::{self, Stream};
 use crate::db;
 use crate::engine::Module;
 use rand::prelude::*;
@@ -49,6 +50,29 @@ pub static SPINNERS: &[&[&str]] = &[
    &[" ", "⠁", "⠉", "⠙", "⠚", "⠖", "⠦", "⠤", "⠠"],
 ];

+pub struct TermSettings {
+    // colors: bool,
+    indicate_progress: bool,
+}
+
+impl TermSettings {
+    pub fn from_env() -> TermSettings {
+        if atty::is(Stream::Stdout) {
+            TermSettings {
+                indicate_progress: true,
+            }
+        } else {
+            TermSettings {
+                indicate_progress: false,
+            }
+        }
+    }
+}
+
+lazy_static! {
+    pub static ref TERM_SETTINGS: TermSettings = TermSettings::from_env();
+}
+
 pub trait SpinLogger {
    fn log(&mut self, line: &str);

@@ -63,14 +87,17 @@ pub struct Spinner {
    indicator: &'static [&'static str],
    status: String,
    i: usize,
+    dummy: bool,
 }

 impl Spinner {
    pub fn new(indicator: &'static [&'static str], status: String) -> Spinner {
+        let dummy = !TERM_SETTINGS.indicate_progress;
        Spinner {
            indicator,
            status,
            i: 0,
+            dummy,
        }
    }

@@ -80,11 +107,12 @@ impl Spinner {
    }

    pub fn tick(&mut self) {
+        if self.dummy { return; }
        print!("{}", self.tick_bytes());
        io::stdout().flush().unwrap();
    }

-    pub fn tick_bytes(&mut self) -> String {
+    fn tick_bytes(&mut self) -> String {
        if self.i >= self.indicator.len() {
            self.i = 0;
        }
@@ -96,7 +124,9 @@ impl Spinner {
    }

    pub fn done(&self) {
+        if self.dummy { return; }
        println!("\r\x1b[2K\x1b[1m[\x1b[32m{}\x1b[0;1m]\x1b[0m {}", '+', self.status);
+        io::stdout().flush().unwrap();
    }

    pub fn finish(&mut self, msg: String) {
@@ -106,7 +136,9 @@ impl Spinner {

    #[inline]
    pub fn clear(&self) {
+        if self.dummy { return; }
        print!("\r\x1b[2K");
+        io::stdout().flush().unwrap();
    }

    pub fn fail(&mut self, err: &str) {
@@ -117,14 +149,17 @@ impl Spinner {

 impl SpinLogger for Spinner {
    fn log(&mut self, line: &str) {
+        if self.dummy { return; }
        println!("\r\x1b[2K\x1b[1m[\x1b[34m{}\x1b[0;1m]\x1b[0m {}", '*', line);
    }

    fn debug(&mut self, line: &str) {
+        if self.dummy { return; }
        println!("\r\x1b[2K\x1b[1m[\x1b[34m{}\x1b[0;1m]\x1b[0m {}", '#', line);
    }

    fn error(&mut self, line: &str) {
+        if self.dummy { return; }
        println!("\r\x1b[2K\x1b[1m[\x1b[31m{}\x1b[0;1m]\x1b[0m {}", '-', line);
    }

@@ -182,16 +217,21 @@ impl fmt::Display for Prompt {
    }
 }

-#[derive(Default)]
 pub struct StackedSpinners {
    spinners: HashMap<String, Spinner>,
    drawn: usize,
+    dummy: bool,
 }

 impl StackedSpinners {
    #[inline]
    pub fn new() -> StackedSpinners {
-        StackedSpinners::default()
+        let dummy = !TERM_SETTINGS.indicate_progress;
+        StackedSpinners {
+            spinners: HashMap::new(),
+            drawn: 0,
+            dummy,
+        }
    }

    pub fn add(&mut self, key: String, status: String) {
@@ -208,11 +248,13 @@ impl StackedSpinners {
    pub fn jump2start(&mut self) {
        if self.drawn > 0 {
            print!("\r\x1b[2K\x1b[{}A", self.drawn);
+            io::stdout().flush().unwrap();
            self.drawn = 0;
        }
    }

    pub fn tick(&mut self) {
+        if self.dummy { return; }
        self.jump2start();

        if self.spinners.is_empty() {
@@ -237,7 +279,9 @@ impl StackedSpinners {

    #[inline]
    pub fn clear(&self) {
+        if self.dummy { return; }
        print!("\r\x1b[2K");
+        io::stdout().flush().unwrap();
    }

    #[inline]
--- a/src/web.rs
+++ b/src/web.rs
@@ -10,7 +10,6 @@ use serde_json;
 use rand::{Rng, thread_rng};
 use rand::distributions::Alphanumeric;
 use std::fmt;
-use std::net::SocketAddr;
 use serde::Serialize;
 use crate::engine::structs::LuaMap;
 use crate::json::LuaJsonValue;
@@ -85,14 +84,13 @@ pub struct HttpRequest {
    user_agent: Option<String>,
    body: Option<ReqBody>,
    timeout: Option<Duration>,
-    proxy: Option<SocketAddr>,
 }

 impl HttpRequest {
-    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions, proxy: Option<SocketAddr>) -> HttpRequest {
+    pub fn new(session: &HttpSession, method: String, url: String, options: RequestOptions) -> HttpRequest {
        let cookies = session.cookies.clone();

-        let user_agent = options.user_agent.or_else(|| Some("sn0int".to_string())); // TODO
+        let user_agent = options.user_agent.or_else(|| Some(format!("sn0int/{}", env!("CARGO_PKG_VERSION")))); // TODO
        let timeout = options.timeout.map(Duration::from_millis);

        let mut request = HttpRequest {
@@ -106,7 +104,6 @@ impl HttpRequest {
            user_agent,
            body: None,
            timeout,
-            proxy,
        };

        if let Some(json) = options.json {
@@ -205,18 +202,8 @@ impl HttpRequest {
        // send request
        debug!("Sending http request: {:?}", req);

-        let mut http = match self.proxy {
-            Some(proxy) => Client::with_socks5(proxy),
-            _ => {
-                let resolver = state.dns_config().as_ref().clone();
-                Client::new(resolver)
-            },
-        };
-
-        if let Some(timeout) = self.timeout {
-            http.timeout(timeout);
-        }
-        let res = http.request(req)
+        let res = state.http().request(req)
+            .with_timeout(self.timeout)
            .wait_for_response()?;

        // map result to LuaMap
--- a/src/worker.rs
+++ b/src/worker.rs
@@ -3,6 +3,7 @@ use crate::errors::*;
 use crate::channel;
 use crate::cmd::run_cmd::Params;
 use crate::db::{Database, DbChange, Family};
+use crate::db::ttl::Ttl;
 use crate::engine::{self, Module};
 use crate::engine::isolation::Supervisor;
 use crate::models::*;
@@ -106,6 +107,7 @@ impl LogEvent {
 #[derive(Debug, Serialize, Deserialize)]
 pub enum DatabaseEvent {
    Insert(Insert),
+    InsertTtl((Insert, i32)),
    Select((Family, String)),
    Update((String, Update)),
 }
@@ -119,42 +121,65 @@ impl EventWithCallback for DatabaseEvent {
 }

 impl DatabaseEvent {
-    pub fn apply<T: SpinLogger>(self, tx: DbSender, spinner: &mut T, db: &Database, verbose: u64) {
-        match self {
-            DatabaseEvent::Insert(object) => {
-                if verbose >= 1 {
-                    spinner.debug(&format!("Inserting: {:?}", object));
+    pub fn insert<T: SpinLogger>(object: Insert, ttl: Option<i32>, tx: DbSender, spinner: &mut T, db: &Database, verbose: u64) {
+        if verbose >= 1 {
+            spinner.debug(&format!("Inserting: {:?}", object));
+        }
+
+        let result = db.insert_generic(&object);
+        debug!("{:?} => {:?}", object, result);
+
+        let result = match result {
+            Ok(Some((DbChange::Insert, id))) => {
+                if let Some(ttl) = ttl {
+                    if let Err(err) = Ttl::create(&object, id, ttl, db) {
+                        spinner.error(&format!("Failed to set ttl: {:?}", err));
+                    }
                }

-                let result = db.insert_generic(&object);
-                debug!("{:?} => {:?}", object, result);
-
-                let result = match result {
-                    Ok(Some((DbChange::Insert, id))) => {
-                        // TODO: replace id with actual object(?)
-                        if let Ok(obj) = object.printable(db) {
-                            spinner.log(&obj.to_string());
-                        } else {
-                            spinner.error(&format!("Failed to query necessary fields for {:?}", object));
-                        }
-                        Ok(Some(id))
-                    },
-                    Ok(Some((DbChange::Update(update), id))) => {
-                        // TODO: replace id with actual object(?)
-                        spinner.log(&format!("Updating {:?} ({})", object.value(), update));
-                        Ok(Some(id))
-                    },
-                    Ok(Some((DbChange::None, id))) => Ok(Some(id)),
-                    Ok(None) => Ok(None),
-                    Err(err) => {
-                        let err = err.to_string();
-                        spinner.error(&err);
-                        Err(err)
-                    },
-                };
-
-                tx.send(result).expect("Failed to send db result to channel");
+                // TODO: replace id with actual object(?)
+                if let Ok(obj) = object.printable(db) {
+                    spinner.log(&obj.to_string());
+                } else {
+                    spinner.error(&format!("Failed to query necessary fields for {:?}", object));
+                }
+                Ok(Some(id))
            },
+            Ok(Some((DbChange::Update(update), id))) => {
+                if let Some(ttl) = ttl {
+                    if let Err(err) = Ttl::bump(&object, id, ttl, db) {
+                        spinner.error(&format!("Failed to set ttl: {:?}", err));
+                    }
+                }
+
+                // TODO: replace id with actual object(?)
+                spinner.log(&format!("Updating {:?} ({})", object.value(), update));
+                Ok(Some(id))
+            },
+            Ok(Some((DbChange::None, id))) => {
+                if let Some(ttl) = ttl {
+                    if let Err(err) = Ttl::bump(&object, id, ttl, db) {
+                        spinner.error(&format!("Failed to set ttl: {:?}", err));
+                    }
+                }
+
+                Ok(Some(id))
+            },
+            Ok(None) => Ok(None),
+            Err(err) => {
+                let err = err.to_string();
+                spinner.error(&err);
+                Err(err)
+            },
+        };
+
+        tx.send(result).expect("Failed to send db result to channel");
+    }
+
+    pub fn apply<T: SpinLogger>(self, tx: DbSender, spinner: &mut T, db: &Database, verbose: u64) {
+        match self {
+            DatabaseEvent::Insert(object) => Self::insert(object, None, tx, spinner, db, verbose),
+            DatabaseEvent::InsertTtl((object, ttl)) => Self::insert(object, Some(ttl), tx, spinner, db, verbose),
            DatabaseEvent::Select((family, value)) => {
                let result = db.get_opt(&family, &value)
                    .map_err(|e| e.to_string());
@@ -182,9 +207,6 @@ impl DatabaseEvent {
                tx.send(result).expect("Failed to send db result to channel");
            },
        }
-        // this is compiled to a nop, but stops clippy from suggesting to refactor tx to &tx
-        // that would allow reusing tx, which we don't want
-        ::std::mem::drop(tx)
    }
 }

@@ -344,18 +366,18 @@ pub fn spawn_fn<F, T>(label: &str, f: F, clear: bool) -> Result<T>
    });

    // run work in main thread
-    let result = f()?;
+    let result = f();
    tx.send(Event::Exit(ExitEvent::Ok))?;

    t.join().expect("thread failed");

    let spinner = spinner.lock().unwrap();

-    if clear {
+    if clear || result.is_err() {
        spinner.clear();
    } else {
        spinner.done();
    }

-    Ok(result)
+    result
 }
--- a/src/workspaces.rs
+++ b/src/workspaces.rs
@@ -1,9 +1,10 @@
 use crate::errors::*;

+use crate::paths;
+use regex::Regex;
 use std::ffi::OsStr;
 use std::fs;
 use std::str::FromStr;
-use crate::paths;


 #[derive(Debug, Clone)]
@@ -19,7 +20,10 @@ impl FromStr for Workspace {
            bail!("Workspace can't be empty")
        }

-        if !s.chars().all(char::is_alphanumeric) {
+        lazy_static! {
+            static ref RE: Regex = Regex::new(r"^[a-zA-Z0-9]([a-zA-Z0-9\._\-]*[a-zA-Z0-9])?$").unwrap();
+        }
+        if !RE.is_match(s) {
            bail!("Workspace contains invalid characters")
        }

@@ -83,13 +87,49 @@ mod tests {
        let x = Workspace::from_str("/");
        assert!(x.is_err());

+        let x = Workspace::from_str("abc/d");
+        assert!(x.is_err());
+
        let x = Workspace::from_str(".");
        assert!(x.is_err());

+        let x = Workspace::from_str("-");
+        assert!(x.is_err());
+
        let x = Workspace::from_str(" ");
        assert!(x.is_err());

        let x = Workspace::from_str("");
        assert!(x.is_err());
    }
+
+    #[test]
+    fn test_valid_singlechar() {
+        let x = Workspace::from_str("a");
+        assert!(x.is_ok());
+    }
+
+    #[test]
+    fn test_valid_middle_chars() {
+        let x = Workspace::from_str("a-b");
+        assert!(x.is_ok());
+
+        let x = Workspace::from_str("a_b");
+        assert!(x.is_ok());
+
+        let x = Workspace::from_str("example.com");
+        assert!(x.is_ok());
+    }
+
+    #[test]
+    fn test_invalid_middle_chars_at_edge() {
+        let x = Workspace::from_str("a-");
+        assert!(x.is_err());
+
+        let x = Workspace::from_str("-b");
+        assert!(x.is_err());
+
+        let x = Workspace::from_str("-");
+        assert!(x.is_err());
+    }
 }
Author	SHA1	Message	Date
kpcyrd	c849c57435	Release v0.9.1	2019-02-03 03:46:28 +01:00
kpcyrd	e4254bec17	Update docs	2019-02-03 03:45:23 +01:00
kpcyrd	980e6b55f4	Merge pull request #71 from kpcyrd/seccomp-i686 Adjust sandbox for i686	2019-02-03 03:43:08 +01:00
kpcyrd	2712bdaeea	Update boxxy commands	2019-02-02 23:37:47 +01:00
kpcyrd	c0a63b0620	Don't kill the process at open, return error	2019-02-02 22:49:21 +01:00
kpcyrd	12754d1c7a	Add integration test script	2019-02-02 16:57:39 +01:00
kpcyrd	0ae36e4976	Adjust sandbox for i686	2019-02-02 15:33:42 +01:00
kpcyrd	2972aa2480	Whitelist missing writev and readv syscalls	2019-01-31 23:59:31 +01:00
kpcyrd	874b317c95	Fix broken rst links	2019-01-31 17:24:07 +01:00
kpcyrd	ca66674f33	Split install instructions into debian and ubuntu	2019-01-31 17:19:24 +01:00
kpcyrd	6c81fe72b0	Add github issue template	2019-01-30 23:34:57 +01:00
kpcyrd	89402fe6e8	Bump module versions	2019-01-30 22:32:19 +01:00
kpcyrd	745cd01419	Update osx install instructions	2019-01-30 22:12:13 +01:00
kpcyrd	e3105165e0	Release v0.9.0	2019-01-29 02:40:32 +01:00
kpcyrd	a37fc3e0b3	Merge pull request #67 from kpcyrd/bump Bump dependencies	2019-01-29 02:36:24 +01:00
kpcyrd	cbb8ca675e	Fix cargo install instructions	2019-01-29 01:11:17 +01:00
kpcyrd	7f622a8c24	Bump dependencies	2019-01-29 01:11:02 +01:00
kpcyrd	b9d990caae	Merge pull request #63 from kpcyrd/datetime Add datetime function	2019-01-28 02:33:43 +01:00
kpcyrd	6856f3333f	Fix datetime format for deserialize	2019-01-28 01:04:14 +01:00
kpcyrd	653651555f	Add function to get current datetime	2019-01-23 21:09:39 +01:00
kpcyrd	d973bcc796	Merge pull request #62 from kpcyrd/db-add-ttl Add db_add_ttl function	2019-01-21 20:50:30 +01:00
kpcyrd	d772d82d57	Relax workspace name rules	2019-01-21 19:57:57 +01:00
kpcyrd	0d722837db	Workaround travis-cache inactitiy bug	2019-01-21 14:11:19 +01:00
kpcyrd	8695d4490d	Enable Write-Ahead logging to resolve locking issue	2019-01-21 14:09:52 +01:00
kpcyrd	3b7b78ed4d	Automatically clear expired entities from db	2019-01-21 08:11:20 +01:00
kpcyrd	c6ac0ede23	Update dependencies	2019-01-21 07:28:57 +01:00
kpcyrd	bfb06499c9	Add function to add to db with ttl until expiry	2019-01-21 01:07:51 +01:00
kpcyrd	9a8830fa53	Add passive arp scanner	2019-01-19 15:46:04 +01:00
kpcyrd	f00c1250f1	Merge branch 'sockets'	2019-01-19 15:38:04 +01:00
kpcyrd	9247d0fded	Add module thunderbird-autoconfig	2019-01-19 03:06:11 +01:00
kpcyrd	83ad8c355f	Add well-known urls scanner	2019-01-19 02:53:08 +01:00
kpcyrd	98bfee2778	Merge pull request #61 from kpcyrd/json Add json output for select	2019-01-19 02:50:57 +01:00
kpcyrd	dd0966883d	Don't display progress indicator in some cases	2019-01-19 01:11:10 +01:00
kpcyrd	3b9fe5ba6c	Add json option to select	2019-01-19 00:28:18 +01:00
kpcyrd	8f38f80ac6	Document sock_* functions	2019-01-18 02:53:51 +01:00
kpcyrd	df7c3b69f4	Add subcommand to create new module	2019-01-18 02:44:46 +01:00
kpcyrd	cf7eb20d95	Use socks5 if set and automatically resolve dns	2019-01-17 18:21:08 +01:00
kpcyrd	8a4b8be0e7	Add socket functions and smtp-check script	2019-01-16 22:17:53 +01:00
kpcyrd	b97aeda086	Improve sn0int publish output	2019-01-14 18:19:21 +01:00
kpcyrd	064b3d7c01	Bump module versions	2019-01-13 01:29:23 +01:00
kpcyrd	3d2f80c9bb	Update install instruction link	2019-01-13 01:23:25 +01:00
kpcyrd	686e1e5119	Release v0.8.1	2019-01-13 01:04:03 +01:00
kpcyrd	913e9a9f4f	Update feature list format	2019-01-13 01:02:55 +01:00
kpcyrd	7a1cf34646	Update readme table of contents	2019-01-12 11:42:49 +01:00
kpcyrd	ed5e913275	Merge pull request #58 from kpcyrd/dns-cache Add a dns-cache	2019-01-12 11:37:52 +01:00
kpcyrd	be2e859efd	Merge some examples	2019-01-12 09:58:30 +01:00
kpcyrd	ef711c4fae	Bump dependencies	2019-01-12 09:30:50 +01:00
kpcyrd	e8a8072349	Travis: skip sqlite install on osx	2019-01-12 08:58:32 +01:00
kpcyrd	592d697888	Remove some unused code	2019-01-12 05:04:11 +01:00
kpcyrd	f8807b7a60	Use chrootable-https dns cache	2019-01-12 04:35:05 +01:00
kpcyrd	40b97d74b4	Fix san extension parser bug	2019-01-10 21:08:54 +01:00
kpcyrd	4b8cc88871	Document sandbox	2019-01-10 08:04:34 +01:00
kpcyrd	3136ed522e	Update module versions	2019-01-10 01:07:09 +01:00