Release v0.6.0

Allow reading from stdin in scripts

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sn0int"
-version = "0.5.0"
+version = "0.6.0"
 description = "OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
@@ -32,8 +32,7 @@ diesel = { version = "1.0.0", features = ["sqlite"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
 dirs = "1.0"
 url = "1.7"
-chrootable-https = "0.3.4"
-trust-dns-proto = "0.5.0"
+chrootable-https = "0.5.0"
 base64 = "0.10"
 kuchiki = "0.7.2"
 serde_urlencoded = "0.5"
@@ -43,7 +42,7 @@ serde_json = "1.0"
 crossbeam-channel = "0.3"
 ctrlc = "3.1"
 opener = "0.3.0"
-separator = "0.3.1"
+separator = "0.4"
 maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.2"
 regex = "1.0"
@@ -58,8 +57,8 @@ nom = "4.1.1"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
-syscallz = "0.7"
-nix = "0.11"
+syscallz = "0.8"
+nix = "0.12"
 
 [target.'cfg(target_os="openbsd")'.dependencies]
 pledge = "0.3.1"

README.md

@@ -1,29 +1,42 @@
-# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates]
+# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs]
 
 [travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
 [travis]:       https://travis-ci.org/kpcyrd/sn0int
 [crates-img]:   https://img.shields.io/crates/v/sn0int.svg
 [crates]:       https://crates.io/crates/sn0int
+[docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
+[docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
 
-sn0int is an OSINT framework and package manager. It's purpose is
-semi-automatically processing public information to enumerate attack surface.
-sn0int itself is only providing an engine that can be extended with scripts.
+sn0int is an OSINT framework and package manager. It was built for IT security
+professionals and bug hunters to gather intelligence about a given target or
+about yourself. sn0int is enumerating attack surface by semi-automatically
+processing public information and mapping the results in a unified format for
+followup investigations.
 
-sn0int is heavily inspired by recon-ng, but takes a few different design
-approaches. We've tried to correct some limitations in the database design and
-also addressed the modularity problem:
+Among other things, sn0int is currently able to:
 
-Instead of downloading and reviewing python scripts that have full access to
-your system, sn0int is executing modules in a lua sandbox. Modules can be
-published to the sn0int registry and then installed by users. This means that
-you don't have to send pull requests to sn0int to add a module and updates can
-be shipped much faster.
+- [X] Harvest subdomains from certificate transparency logs
+- [X] Harvest subdomains from various passive dns logs
+- [X] Sift through subdomain results for publicly accessible websites
+- [X] Harvest emails from pgp keyservers
+- [X] Enrich ip addresses with ASN and geoip info
+- [X] Harvest subdomains from the wayback machine
+
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
 
 Join us on IRC: <ircs://irc.hackint.org/#sn0int>
 
+[![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
+
 ## Installation
 
-- Archlinux: `yaourt -S sn0int`
+- Archlinux: `pacman -S sn0int`
 - Alpine: `apk add --no-cache sqlite-dev libseccomp-dev cargo` + build from source
 - Debian: `apt install libsqlite3-dev libseccomp-dev` + build from source
 - OpenBSD: `pkg_add sqlite3` + build from source

docs/index.rst

@@ -1,19 +1,28 @@
 sn0int
 ======
 
-sn0int is an OSINT framework and package manager. It's purpose is
-semi-automatically processing public information to enumerate attack surface.
-sn0int itself is only providing an engine that can be extended with scripts.
+sn0int is an OSINT framework and package manager. It was built for IT security
+professionals and bug hunters to gather intelligence about a given target or
+about yourself. sn0int is enumerating attack surface by semi-automatically
+processing public information and mapping the results in a unified format for
+followup investigations.
 
-sn0int is heavily inspired by recon-ng, but takes a few different design
-approaches. We've tried to correct some limitations in the database design and
-also addressed the modularity problem:
+Among other things, sn0int is currently able to:
 
-Instead of downloading and reviewing python scripts that have full access to
-your system, sn0int is executing modules in a lua sandbox. Modules can be
-published to the sn0int registry and then installed by users. This means that
-you don't have to send pull requests to sn0int to add a module and updates can
-be shipped much faster.
+- [X] Harvest subdomains from certificate transparency logs
+- [X] Harvest subdomains from various passive dns logs
+- [X] Sift through subdomain results for publicly accessible websites
+- [X] Harvest emails from pgp keyservers
+- [X] Enrich ip addresses with ASN and geoip info
+- [X] Harvest subdomains from the wayback machine
+
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
 
 Join us on IRC: ircs://irc.hackint.org/#sn0int
 

docs/install.rst

@@ -8,7 +8,7 @@ Archlinux
 
 .. code-block:: bash
 
-    $ yaourt -S sn0int
+    $ pacman -S sn0int
 
 Debian/Ubuntu/Kali
 ------------------

docs/reference.rst

@@ -55,15 +55,33 @@ Update an entity in the database. This function may fail. See `db_update
 dns
 ---
 
-Resolve a dns record. In the dns response contains an error, ``x['error']`` is
-set and ``x['success']`` is ``nil``. Otherwise, ``x['success']`` contains a
-list of records. This function may fail.
+Resolve a dns record. If the dns query was successful and the dns reply is
+``NoError`` then ``x['error']`` is ``nil``. The records of the reply are in
+``x['answers']``. This function may fail.
+
+This function accepts the following options:
+
+``record``
+  The ``query_type``, can be any of ``A``, ``AAAA``, ``MX``, ``AXFR``, etc.
+``nameserver``
+  The server that should be used for the lookup. Defaults to your system
+  resolver.
+``tcp``
+  If the lookup should use tcp, true/false.
+``timeout``
+  The time until the query times out in milliseconds.
 
 .. code-block:: lua
 
-    x = dns('example.com', 'A')
+    x = dns('example.com', {
+        record='A'
+    })
     if last_err() then return end
 
+.. note::
+   DNS replies with an error code set are not causing a change to
+   ``last_err()``. You have to test for this explicitly.
+
 error
 -----
 
@@ -147,19 +165,19 @@ Please note that you still need to specify an empty table ``{}`` even if no
 options are set. The following options are available:
 
 ``query``
-  A map of query parameters that should be set on the url
+  A map of query parameters that should be set on the url.
 ``headers``
-  A map of headers that should be set
+  A map of headers that should be set.
 ``basic_auth``
-  Configure the basic auth header with ``{"user, "password"}``
+  Configure the basic auth header with ``{"user, "password"}``.
 ``user_agent``
-  Overwrite the default user agent with a string
+  Overwrite the default user agent with a string.
 ``json``
-  The request body that should be json encoded
+  The request body that should be json encoded.
 ``form``
-  The request body that should be form encoded
+  The request body that should be form encoded.
 ``body``
-  The raw request body as string
+  The raw request body as string.
 
 This function may fail.
 
@@ -386,6 +404,18 @@ Update the label of the progress indicator.
 
     status('ohai')
 
+stdin_readline
+--------------
+
+Read a line from stdin. The final newline is not removed.
+
+.. code-block:: lua
+
+    stdin_readline()
+
+.. note::
+   This only works with `sn0int run --stdin`.
+
 url_join
 --------
 

docs/scripting.rst

@@ -101,25 +101,30 @@ truth-y.
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
-        records = dns(subdomain, 'A')
+        records = dns(subdomain, {
+            record='A'
+        })
         if last_err() then return end
 
         print(records)
     end
 
 If you run your module again you're going to see some output, either
-``{"success": somedata}`` or ``{"error": "NX"}``. We decide if ``success`` is
-not ``nil``, we add the subdomain to our scope and set it to resolvable:
+``{"answers":[somedata],"error":null}`` or
+``{"answers":[],"error":"NXDomain"}``. We decide that we add the subdomain to
+our scope and set it to resolvable if ``error`` is ``nil``.
 
 .. code-block:: lua
 
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
-        records = dns(subdomain, 'A')
+        records = dns(subdomain, {
+            record='A'
+        })
         if last_err() then return end
 
-        if records['success'] ~= nil then
+        if records['error'] == nil then
             db_add('subdomain', arg, {
                 domain_id=arg['id'],
                 value=subdomain,
@@ -148,7 +153,9 @@ After putting everything together, our final module looks like this:
     function run(arg)
         subdomain = 'www.' .. arg['value']
 
-        records = dns(subdomain, 'A')
+        records = dns(subdomain, {
+            record='A'
+        })
         if last_err() then return end
 
         if records['success'] ~= nil then

migrations/2018-10-26-001932_cascade-delete/up.sql

@@ -37,25 +37,6 @@ INSERT INTO subdomains (id, domain_id, value, unscoped, resolvable)
 
 DROP TABLE _subdomains_old;
 
--- subdomain_ipaddrs
-
-ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
-
-CREATE TABLE subdomain_ipaddrs (
-    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-    subdomain_id INTEGER NOT NULL,
-    ip_addr_id INTEGER NOT NULL,
-    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
-    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
-    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
-);
-
-INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
-    SELECT id, subdomain_id, ip_addr_id
-    FROM _subdomain_ipaddrs_old;
-
-DROP TABLE _subdomain_ipaddrs_old;
-
 -- urls
 
 ALTER TABLE urls RENAME TO _urls_old;
@@ -123,4 +104,23 @@ INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, cou
 
 DROP TABLE _ipaddrs_old;
 
+-- subdomain_ipaddrs
+
+ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
+
+CREATE TABLE subdomain_ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
+);
+
+INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
+    SELECT id, subdomain_id, ip_addr_id
+    FROM _subdomain_ipaddrs_old;
+
+DROP TABLE _subdomain_ipaddrs_old;
+
 PRAGMA foreign_keys=on;

migrations/2018-12-03-143558_url_path/down.sql

@@ -0,0 +1,25 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-03-143558_url_path/up.sql

@@ -0,0 +1,26 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    path VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, path, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, '/', status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

modules/dev/arp-scan.lua

@@ -0,0 +1,21 @@
+-- Description: Parse arp-scan output
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- sudo arp-scan -qglI wlp3s0
+
+function run()
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        m = regex_find('(.+)\t(.+)', x)
+        if m ~= nil then
+            ip = m[2]
+            mac = m[3]
+            info(json_encode({ip, mac}))
+        end
+    end
+end

modules/dev/axfr.lua

@@ -0,0 +1,159 @@
+-- Description: Try a zone transfer for subdomains
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function strip_root_dot(name)
+    m = regex_find("(.+)\\.$", name)
+    if last_err() then return end
+
+    if m == nil then
+        return name
+    else
+        return m[2]
+    end
+end
+
+function add_pointer(name)
+    local domain, domain_id, subdomain_id
+
+    -- select psl+1
+    domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+end
+
+function iter_axfr(zone, arg)
+    local name, r, m, domain
+
+    debug(json_encode(arg))
+
+    name = arg[1]
+    r = arg[2]
+
+    -- select psl+1
+    domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+
+    -- this is a A record
+    if r['A'] ~= nil then
+        -- add the name and ip
+        ipaddr_id = db_add('ipaddr', {
+            family='4',
+            value=r['A'],
+        })
+        if last_err() then return end
+
+        db_add('subdomain-ipaddr', {
+            subdomain_id=subdomain_id,
+            ip_addr_id=ipaddr_id,
+        })
+        if last_err() then return end
+    end
+
+    if r['CNAME'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['CNAME'])
+        add_pointer(name)
+    end
+
+    if r['NS'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['NS'])
+        add_pointer(name)
+    end
+
+    if r['MX'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['MX'][2])
+        add_pointer(name:lower())
+    end
+end
+
+function iter_a(zone, arg)
+    local i, records, r
+
+    if arg == nil then return end
+
+    debug('nameserver: ' .. arg)
+    records = dns(zone, {
+        record='AXFR',
+        nameserver=arg .. ':53',
+        tcp=true,
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    i = 1
+    while records[i] ~= nil do
+        iter_axfr(zone, records[i])
+        if last_err() then return end
+        i = i+1
+    end
+end
+
+function iter_ns(zone, arg)
+    local i, records, r
+
+    if arg == nil then return end
+
+    records = dns(arg, {
+        record='A',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    i = 1
+    while records[i] ~= nil do
+        r = records[i][2]
+        iter_a(zone, r['A'])
+        if last_err() then return end
+        i = i+1
+    end
+end
+
+function run(arg)
+    records = dns(arg['value'], {
+        record='NS',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    i = 1
+    while records[i] ~= nil do
+        r = records[i][2]
+        iter_ns(arg['value'], r['NS'])
+        if last_err() then return end
+        i = i+1
+    end
+end

modules/dev/cname-harvest.lua

@@ -0,0 +1,50 @@
+-- Description: Query for CNAMES to find subdomains
+-- Version: 0.1.0
+-- Source: subdomains
+-- License: GPL-3.0
+
+function iter(r)
+    if r == nil then
+        return
+    end
+
+    m = regex_find("(.+)\\.$", r)
+    if last_err() then return end
+
+    if m == nil then
+        return
+    end
+    r = m[2]
+
+    domain = psl_domain_from_dns_name(r)
+    if last_err() then return end
+
+    domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+
+    if domain_id ~= nil then
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=r,
+        })
+        if last_err() then return end
+    end
+end
+
+function run(arg)
+    records = dns(arg['value'], 'A')
+    if last_err() then return end
+
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    i = 1
+    while records[i] ~= nil do
+        r = records[i][2]
+        iter(r['CNAME'])
+        if last_err() then return end
+        i = i+1
+    end
+end

modules/dev/ctlogs.lua

@@ -28,6 +28,7 @@ function run(arg)
         -- print(c)
 
         name = c['name_value']
+        debug(json_encode(name))
 
         if name:find("*.") == 1 then
             -- ignore wildcard domains
@@ -35,7 +36,6 @@ function run(arg)
         end
 
         if seen[name] == nil then
-            -- info(name)
             db_add('subdomain', {
                 domain_id=arg['id'],
                 value=name,

modules/dev/dns-resolve.lua

@@ -8,7 +8,7 @@ function run(arg)
     if last_err() then return end
 
     -- update subdomain
-    resolvable = records['success'] ~= nil
+    resolvable = records['error'] == nil
     if arg['resolvable'] ~= resolvable then
         -- TODO: pass arg to function as well
         db_update('subdomain', arg, {
@@ -20,15 +20,11 @@ function run(arg)
         return
     end
 
-    records = records['success']
-
-    -- there is a bug in struct -> lua that causes tables to be zero indexed
-    -- this checks if there's something at index 0 but uses index 1 if this is fixed
-    i = 0
-    if records[i] == nil then i = 1 end
+    records = records['answers']
 
+    i = 1
     while records[i] ~= nil do
-        r = records[i]
+        r = records[i][2]
         if r['A'] ~= nil then
             ipaddr_id = db_add('ipaddr', {
                 family='4',
@@ -40,6 +36,7 @@ function run(arg)
                 subdomain_id=arg['id'],
                 ip_addr_id=ipaddr_id,
             })
+            if last_err() then return end
         end
         i = i+1
     end

modules/dev/otx-subdomains.lua

@@ -18,7 +18,7 @@ function run(arg)
     if last_err() then return end
     o = o['passive_dns']
 
-    i = 0
+    i = 1
     while o[i] do
         x = o[i]
 

modules/dev/threatminer-ipaddr.lua

@@ -23,7 +23,7 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 0
+    i = 1
     while o[i] do
         x = o[i]
 

modules/dev/threatminer-subdomains.lua

@@ -21,7 +21,7 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 0
+    i = 1
     while o[i] do
         x = o[i]
 

modules/dev/waybackurls.lua

@@ -18,21 +18,21 @@ function run(arg)
     if last_err() then return end
 
     -- no known urls
-    if o[0] == nil then
+    if o[1] == nil then
         return
     end
 
     -- ensure the api response is still what we expect
-    if o[0][2] == nil then
+    if o[1][3] == nil then
         return 'api returned unexpected json format'
     end
 
     seen = {}
 
-    i = 1
+    i = 2
     while o[i] do
-        url = o[i][2]
-
+        url = o[i][3]
+        debug(json_encode(url))
         parts = url_parse(url)
 
         if last_err() then
@@ -40,6 +40,7 @@ function run(arg)
             error("Failed to parse url: " .. json_encode(url))
         else
             subdomain = parts['host']
+            subdomain, _ = subdomain:gsub('%.$', '')
 
             if seen[subdomain] == nil then
                 db_add('subdomain', {

modules/harness/stdin.lua

@@ -0,0 +1,13 @@
+-- Description: Read from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+        info(json_encode(x))
+    end
+end

sn0int-registry/assets/style.css

@@ -28,3 +28,8 @@ a {
     background-color: #222;
     padding: 10px;
 }
+
+.list-unstyled {
+    list-style: none;
+    padding: 0 0 0 20px;
+}

sn0int-registry/sn0int-common/src/metadata.rs

@@ -171,6 +171,21 @@ mod tests {
         });
     }
 
+    #[test]
+    fn verify_no_source() {
+        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+-- Version: 1.0.0
+-- License: WTFPL
+
+"#).expect("parse");
+        assert_eq!(metadata, Metadata {
+            description: "Hello world, this is my description".to_string(),
+            version: "1.0.0".to_string(),
+            license: License::WTFPL,
+            source: None,
+        });
+    }
+
     #[test]
     fn verify_require_license() {
         let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description

sn0int-registry/src/models.rs

@@ -231,13 +231,21 @@ impl Release {
         */
     }
 
-    pub fn find(id: i32, version: &str, connection: &PgConnection) -> Result<Release> {
-        releases::table.filter(releases::columns::module_id.eq(id))
+    pub fn find(module_id: i32, version: &str, connection: &PgConnection) -> Result<Release> {
+        releases::table.filter(releases::columns::module_id.eq(module_id))
                         .filter(releases::columns::version.eq(version))
                         .first::<Release>(connection)
                         .map_err(Error::from)
     }
 
+    pub fn try_find(module_id: i32, version: &str, connection: &PgConnection) -> Result<Option<Release>> {
+        releases::table.filter(releases::columns::module_id.eq(module_id))
+                        .filter(releases::columns::version.eq(version))
+                        .first::<Release>(connection)
+                        .optional()
+                        .map_err(Error::from)
+    }
+
     pub fn id(id: i32, connection: &PgConnection) -> Result<Release> {
         releases::table.find(id)
             .first::<Release>(connection)

sn0int-registry/src/routes/api.rs

@@ -93,7 +93,17 @@ fn publish(name: String, upload: Json<PublishRequest>, session: AuthHeader, conn
 
     connection.transaction::<_, Error, _>(|| {
         let module = Module::update_or_create(&user, &name, &metadata.description, &connection)?;
-        module.add_version(&version, &upload.code, &connection)?;
+
+        match Release::try_find(module.id, &version, &connection)? {
+            Some(release) => {
+                // if the code is identical, pretend we published the version
+                if release.code != upload.code {
+                    bail!("Version number already in use")
+                }
+            },
+            None => module.add_version(&version, &upload.code, &connection)?,
+        }
+
         Ok(())
     })?;
 

sn0int-registry/templates/index.html.hbs

@@ -1,10 +1,35 @@
 {{#*inline "page"}}
         <p>
-        This is the registry server of sn0int, an OSINT framework and package
-        manager. It's purpose is semi-automatically processing public
-        information to enumerate attack surface. sn0int itself is only
-        providing an engine that can be extended with scripts.
+This is the registry server of sn0int, an OSINT framework and package manager.
+It was built for IT security professionals and bug hunters to gather
+intelligence about a given target or about yourself. sn0int is enumerating
+attack surface by semi-automatically processing public information and mapping
+the results in a unified format for followup investigations.
         </p>
+
+        <p>
+Among other things, sn0int is currently able to:
+        </p>
+
+        <ul class="list-unstyled">
+<li>[X] Harvest subdomains from certificate transparency logs</li>
+<li>[X] Harvest subdomains from various passive dns logs</li>
+<li>[X] Sift through subdomain results for publicly accessible websites</li>
+<li>[X] Harvest emails from pgp keyservers</li>
+<li>[X] Enrich ip addresses with ASN and geoip info</li>
+<li>[X] Harvest subdomains from the wayback machine</li>
+        </ul>
+
+        <p>
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
+        </p>
+
         <p>
         The registry server allows uploading modules that can be installed by
         users afterwards. This is done using sn0int which needs to be

src/api.rs

@@ -40,7 +40,7 @@ impl Client {
 
     pub fn request<T: DeserializeOwned + fmt::Debug>(&self, mut request: RequestBuilder, body: Body) -> Result<T> {
         if let Some(session) = &self.session {
-            info!("Adding session token");
+            info!("Adding session token to request");
             request.header("Auth", session.as_str());
         }
 

src/args.rs

@@ -51,11 +51,21 @@ pub enum SubCommand {
 
 #[derive(Debug, StructOpt)]
 pub struct Run {
+    /// Execute a module that has been installed
     pub module: Option<String>,
     #[structopt(short="f", long="file", conflicts_with="module")]
+    /// Run a module from a path
     pub file: Option<String>,
     #[structopt(short="j", long="threads", default_value="1")]
+    /// Run modules concurrently
     pub threads: usize,
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
+    /// Verbose logging, once to print inserts even if they don't add new
+    /// data, twice to activate the debug() function
+    pub verbose: u64,
+    #[structopt(long="stdin")]
+    /// Expose stdin to modules
+    pub stdin: bool,
 }
 
 #[derive(Debug, StructOpt)]

src/cmd/run_cmd.rs

@@ -15,6 +15,8 @@ use worker;
 pub struct Args {
     #[structopt(short="j", long="threads", default_value="1")]
     threads: usize,
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
+    verbose: u64,
 }
 
 fn prepare_arg<T: Serialize + Model>(x: T) -> Result<(serde_json::Value, Option<String>)> {
@@ -30,7 +32,7 @@ fn prepare_args<T: Scopable + Serialize + Model>(db: &Database, filter: &Filter)
         .collect()
 }
 
-pub fn execute(rl: &mut Readline, threads: usize) -> Result<()> {
+pub fn execute(rl: &mut Readline, threads: usize, verbose: u64, has_stdin: bool) -> Result<()> {
     let module = rl.module()
         .map(|m| m.to_owned())
         .ok_or_else(|| format_err!("No module selected"))?;
@@ -47,7 +49,7 @@ pub fn execute(rl: &mut Readline, threads: usize) -> Result<()> {
     }?;
 
     rl.signal_register().catch_ctrl();
-    worker::spawn(rl, &module, args, threads);
+    worker::spawn(rl, &module, args, threads, verbose, has_stdin);
     rl.signal_register().reset_ctrlc();
 
     term::info(&format!("Finished {}", module.canonical()));
@@ -57,5 +59,5 @@ pub fn execute(rl: &mut Readline, threads: usize) -> Result<()> {
 
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
-    execute(rl, args.threads)
+    execute(rl, args.threads, args.verbose, false)
 }

src/db.rs

@@ -122,6 +122,7 @@ impl Database {
             Insert::Url(object) => self.insert_struct(NewUrl {
                 subdomain_id: object.subdomain_id,
                 value: &object.value,
+                path: &object.path,
                 status: object.status,
                 body: object.body.as_ref(),
                 online: object.online,

src/engine/ctx.rs

@@ -11,9 +11,9 @@ use serde_json;
 use std::collections::HashMap;
 use std::result;
 use std::sync::{Arc, Mutex};
-use chrootable_https::dns::DnsConfig;
+use chrootable_https::dns::Resolver;
 use web::{HttpSession, HttpRequest, RequestOptions};
-use worker::{Event, LogEvent, DatabaseEvent};
+use worker::{Event, LogEvent, DatabaseEvent, StdioEvent};
 
 
 pub trait State {
@@ -29,10 +29,18 @@ pub trait State {
 
     fn recv(&self) -> Result<serde_json::Value>;
 
+    fn verbose(&self) -> u64;
+
     fn info(&self, msg: String) {
         self.send(&Event::Log(LogEvent::Info(msg)))
     }
 
+    fn debug(&self, msg: String) {
+        if self.verbose() >= 2 {
+            self.send(&Event::Log(LogEvent::Debug(msg)))
+        }
+    }
+
     fn error(&self, msg: String) {
         self.send(&Event::Log(LogEvent::Error(msg)))
     }
@@ -65,7 +73,14 @@ pub trait State {
         reply.map_err(|err| format_err!("Failed to update database: {:?}", err))
     }
 
-    fn dns_config(&self) -> Arc<DnsConfig>;
+    fn stdin_readline(&self) -> Result<Option<String>> {
+        self.send(&Event::Stdio(StdioEvent {}));
+        let reply = self.recv()?;
+        let reply: result::Result<Option<String>, String> = serde_json::from_value(reply)?;
+        reply.map_err(|err| format_err!("Failed to read stdin: {:?}", err))
+    }
+
+    fn dns_config(&self) -> Arc<Resolver>;
 
     fn psl(&self) -> Arc<Psl>;
 
@@ -85,7 +100,8 @@ pub struct LuaState {
     error: Arc<Mutex<Option<Error>>>,
     logger: Arc<Mutex<Option<Arc<Mutex<Box<Reporter>>>>>>,
     http_sessions: Arc<Mutex<HashMap<String, HttpSession>>>,
-    dns_config: Arc<DnsConfig>,
+    verbose: u64,
+    dns_config: Arc<Resolver>,
     psl: Arc<Psl>,
     geoip: Arc<GeoIP>,
     asn: Arc<AsnDB>,
@@ -132,7 +148,11 @@ impl State for LuaState {
         }
     }
 
-    fn dns_config(&self) -> Arc<DnsConfig> {
+    fn verbose(&self) -> u64 {
+        self.verbose
+    }
+
+    fn dns_config(&self) -> Arc<Resolver> {
         self.dns_config.clone()
     }
 
@@ -184,6 +204,7 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
         logger: Arc::new(Mutex::new(None)),
         http_sessions: Arc::new(Mutex::new(HashMap::new())),
 
+        verbose: env.verbose,
         dns_config: Arc::new(env.dns_config),
         psl: Arc::new(env.psl),
         geoip: Arc::new(env.geoip),
@@ -194,6 +215,7 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
     runtime::db_add(&mut lua, state.clone());
     runtime::db_select(&mut lua, state.clone());
     runtime::db_update(&mut lua, state.clone());
+    runtime::debug(&mut lua, state.clone());
     runtime::dns(&mut lua, state.clone());
     runtime::error(&mut lua, state.clone());
     runtime::asn_lookup(&mut lua, state.clone());
@@ -216,6 +238,7 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
     runtime::regex_find_all(&mut lua, state.clone());
     runtime::sleep(&mut lua, state.clone());
     runtime::status(&mut lua, state.clone());
+    runtime::stdin_readline(&mut lua, state.clone());
     runtime::url_join(&mut lua, state.clone());
     runtime::url_parse(&mut lua, state.clone());
     runtime::utf8_decode(&mut lua, state.clone());
@@ -282,7 +305,7 @@ impl Script {
     pub fn test(&self) -> Result<()> {
         use engine::tests::DummyReporter;
         use geoip::Maxmind;
-        let dns_config = DnsConfig::from_system()?;
+        let dns_config = Resolver::from_system()?;
         let psl = Psl::from_str(r#"
 // ===BEGIN ICANN DOMAINS===
 com
@@ -292,6 +315,7 @@ com
         let asn = AsnDB::open_or_download()?;
 
         let env = Environment {
+            verbose: 0,
             dns_config,
             psl,
             geoip,

src/engine/isolation.rs

@@ -1,29 +1,30 @@
 use errors::*;
-use chrootable_https::dns::DnsConfig;
+use chrootable_https::dns::Resolver;
 use engine::{Environment, Module, Reporter};
 use geoip::{GeoIP, AsnDB};
 use psl::Psl;
 use serde_json;
-use worker::{Event, Event2, LogEvent, DatabaseEvent, ExitEvent, EventSender};
+use worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
 
 use std::env;
 use std::io::prelude::*;
-use std::io::{self, BufReader, BufRead, Stdin, Stdout};
+use std::io::{self, BufReader, BufRead, stdin, Stdin, Stdout};
 use std::sync::{mpsc, Arc, Mutex};
 use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
 
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct StartCommand {
-    dns_config: DnsConfig,
+    verbose: u64,
+    dns_config: Resolver,
     module: Module,
     arg: serde_json::Value,
 }
 
 impl StartCommand {
-    pub fn new(dns_config: DnsConfig, module: Module, arg: serde_json::Value) -> StartCommand {
-        // TODO: compress psl
+    pub fn new(verbose: u64, dns_config: Resolver, module: Module, arg: serde_json::Value) -> StartCommand {
         StartCommand {
+            verbose,
             dns_config,
             module,
             arg
@@ -61,8 +62,8 @@ impl Supervisor {
         })
     }
 
-    pub fn send_start(&mut self, dns_config: DnsConfig, module: Module, arg: serde_json::Value) -> Result<()> {
-        let start = serde_json::to_value(StartCommand::new(dns_config, module, arg))?;
+    pub fn send_start(&mut self, start: &StartCommand) -> Result<()> {
+        let start = serde_json::to_value(&start)?;
         self.send(&start)?;
         Ok(())
     }
@@ -75,6 +76,13 @@ impl Supervisor {
         Ok(())
     }
 
+    pub fn send_struct<T: serde::Serialize>(&mut self, value: T, tx: &EventSender) {
+        let value = serde_json::to_value(value).expect("Failed to serialize reply");
+        if let Err(_) = self.send(&value) {
+            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
+        }
+    }
+
     pub fn recv(&mut self) -> Result<Event> {
         let mut line = String::new();
         let len = self.stdout.read_line(&mut line)?;
@@ -95,15 +103,14 @@ impl Supervisor {
         }
     }
 
-    pub fn send_event_callback(&mut self, event: DatabaseEvent, tx: &EventSender) {
+    pub fn send_event_callback<T: EventWithCallback>(&mut self, event: T, tx: &EventSender)
+        where <T as EventWithCallback>::Payload: serde::Serialize
+    {
         let (tx2, rx2) = mpsc::channel();
-        tx.send(Event2::Database((event, tx2)));
+        tx.send(event.with_callback(tx2));
         let reply = rx2.recv().unwrap();
 
-        let value = serde_json::to_value(reply).expect("Failed to serialize reply");
-        if let Err(_) = self.send(&value) {
-            tx.send(Event2::Log(LogEvent::Error("Failed to send to child".into())));
-        }
+        self.send_struct(reply, tx);
     }
 }
 
@@ -150,16 +157,23 @@ impl Reporter for StdioReporter {
     }
 }
 
-pub fn spawn_module(module: Module, tx: &EventSender, arg: serde_json::Value) -> Result<()> {
-    let dns_config = DnsConfig::from_system()?;
+pub fn spawn_module(module: Module, tx: &EventSender, arg: serde_json::Value, verbose: u64, has_stdin: bool) -> Result<()> {
+    let dns_config = Resolver::from_system()?;
+
+    let mut reader = if has_stdin {
+        Some(BufReader::new(stdin()))
+    } else {
+        None
+    };
 
     let mut supervisor = Supervisor::setup(&module)?;
-    supervisor.send_start(dns_config, module, arg)?;
+    supervisor.send_start(&StartCommand::new(verbose, dns_config, module, arg))?;
 
     loop {
         match supervisor.recv()? {
             Event::Log(event) => tx.send(Event2::Log(event)),
             Event::Database(object) => supervisor.send_event_callback(object, &tx),
+            Event::Stdio(object) => object.apply(&mut supervisor, tx, &mut reader),
             Event::Exit(event) => {
                 if let ExitEvent::Err(err) = event {
                     tx.send(Event2::Log(LogEvent::Error(err)));
@@ -182,6 +196,7 @@ pub fn run_worker(geoip: GeoIP, asn: AsnDB, psl: String) -> Result<()> {
         .context("Failed to load public suffix list")?;
 
     let environment = Environment {
+        verbose: start.verbose,
         dns_config: start.dns_config,
         psl,
         geoip,

src/engine/mod.rs

@@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex};
 use engine::ctx::Script;
 use sn0int_common::ModuleID;
 use sn0int_common::metadata::{Metadata, Source};
-use chrootable_https::dns::DnsConfig;
+use chrootable_https::dns::Resolver;
 use psl::Psl;
 use paths;
 use std::cmp::Ordering;
@@ -26,7 +26,8 @@ pub mod structs;
 /// Data that is passed to every script
 #[derive(Debug)]
 pub struct Environment {
-  pub dns_config: DnsConfig,
+  pub verbose: u64,
+  pub dns_config: Resolver,
   pub psl: Psl,
   pub geoip: GeoIP,
   pub asn: AsnDB,

src/geoip/mod.rs

@@ -24,7 +24,19 @@ pub trait Maxmind: Sized {
 
     fn new(reader: maxminddb::Reader) -> Self;
 
+    // TODO: refactor this to return Path
     fn cache_path() -> Result<String> {
+        // use system path if exists
+        let path = Path::new("/usr/share/GeoIP/");
+        let path = path.join(Self::archive_filename());
+
+        if path.exists() {
+            let path = path.to_str()
+                .ok_or_else(|| format_err!("Failed to decode path"))?;
+            return Ok(path.to_string());
+        }
+
+        // use cache path
         let path = paths::cache_dir()?
             .join(Self::archive_filename());
         let path = path.to_str()

src/json.rs

@@ -64,9 +64,8 @@ impl Into<AnyLuaValue> for LuaJsonValue {
             // TODO: not sure if this might fail
             LuaJsonValue::Number(v) => AnyLuaValue::LuaNumber(v.as_f64().unwrap()),
             LuaJsonValue::String(v) => AnyLuaValue::LuaString(v),
-            // TODO: ensure lua tables always start at 1
             LuaJsonValue::Array(v) => AnyLuaValue::LuaArray(v.into_iter().enumerate()
-                .map(|(i, x)| (AnyLuaValue::LuaNumber(i as f64), x.into()))
+                .map(|(i, x)| (AnyLuaValue::LuaNumber((i+1) as f64), x.into()))
                 .collect()
             ),
             LuaJsonValue::Object(v) => AnyLuaValue::LuaArray(v.into_iter()

src/lib.rs

@@ -10,7 +10,6 @@ extern crate shellwords;
 extern crate dirs;
 extern crate publicsuffix;
 extern crate chrootable_https;
-extern crate trust_dns_proto;
 extern crate url;
 #[cfg(target_os = "linux")]
 extern crate nix;

src/main.rs

@@ -42,7 +42,7 @@ fn run_run(gargs: &Args, args: &args::Run, config: Config) -> Result<()> {
         bail!("At least one module or file need to be provided");
     }
 
-    cmd::run_cmd::execute(&mut rl, args.threads)
+    cmd::run_cmd::execute(&mut rl, args.threads, args.verbose, args.stdin)
 }
 
 fn run_sandbox() -> Result<()> {

src/models/domain.rs

@@ -105,6 +105,14 @@ impl Scopable for Domain {
     }
 }
 
+impl Domain {
+    fn subdomains(&self, db: &Database) -> Result<Vec<Subdomain>> {
+        Subdomain::belonging_to(self)
+            .load(db.db())
+            .map_err(Error::from)
+    }
+}
+
 pub struct PrintableDomain {
     value: String,
 }
@@ -126,26 +134,46 @@ impl Printable<PrintableDomain> for Domain {
 pub struct DetailedDomain {
     id: i32,
     value: String,
+    subdomains: Vec<PrintableSubdomain>,
     unscoped: bool,
 }
 
-impl fmt::Display for DetailedDomain {
-    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        if !self.unscoped {
-            write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{:?}\x1b[0m", self.id, self.value)
-        } else {
-            write!(w, "\x1b[90m#{}, {:?}\x1b[0m", self.id, self.value)
+impl DisplayableDetailed for DetailedDomain {
+    #[inline]
+    fn scoped(&self) -> bool {
+        !self.unscoped
+    }
+
+    #[inline]
+    fn print(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        self.id(w, self.id)?;
+        self.green_debug(w, &self.value)?;
+        Ok(())
+    }
+
+    #[inline]
+    fn children(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        for subdomain in &self.subdomains {
+            self.child(w, subdomain)?;
         }
+        Ok(())
     }
 }
 
+display_detailed!(DetailedDomain);
+
 impl Detailed for Domain {
     type T = DetailedDomain;
 
-    fn detailed(&self, _db: &Database) -> Result<Self::T> {
+    fn detailed(&self, db: &Database) -> Result<Self::T> {
+        let subdomains = self.subdomains(db)?.into_iter()
+            .map(|sd| sd.printable(db))
+            .collect::<Result<_>>()?;
+
         Ok(DetailedDomain {
             id: self.id,
             value: self.value.to_string(),
+            subdomains,
             unscoped: self.unscoped,
         })
     }
@@ -193,3 +221,13 @@ impl Printable<PrintableDomain> for NewDomainOwned {
         })
     }
 }
+
+pub type InsertDomain = NewDomainOwned;
+
+impl LuaInsertToNewOwned for InsertDomain {
+    type Target = NewDomainOwned;
+
+    fn try_into_new(self) -> Result<NewDomainOwned> {
+        Ok(self)
+    }
+}

src/models/email.rs

@@ -130,16 +130,27 @@ pub struct DetailedEmail {
     unscoped: bool,
 }
 
-impl fmt::Display for DetailedEmail {
-    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        if !self.unscoped {
-            write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{:?}\x1b[0m", self.id, self.value)
-        } else {
-            write!(w, "\x1b[90m#{}, {:?}\x1b[0m", self.id, self.value)
-        }
+impl DisplayableDetailed for DetailedEmail {
+    #[inline]
+    fn scoped(&self) -> bool {
+        !self.unscoped
+    }
+
+    #[inline]
+    fn print(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        self.id(w, self.id)?;
+        self.green_debug(w, &self.value)?;
+        Ok(())
+    }
+
+    #[inline]
+    fn children(&self, _w: &mut fmt::Formatter) -> fmt::Result {
+        Ok(())
     }
 }
 
+display_detailed!(DetailedEmail);
+
 impl Detailed for Email {
     type T = DetailedEmail;
 
@@ -198,6 +209,16 @@ impl Printable<PrintableEmail> for NewEmailOwned {
     }
 }
 
+pub type InsertEmail = NewEmailOwned;
+
+impl LuaInsertToNewOwned for InsertEmail {
+    type Target = NewEmailOwned;
+
+    fn try_into_new(self) -> Result<NewEmailOwned> {
+        Ok(self)
+    }
+}
+
 #[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
 #[table_name="emails"]
 pub struct EmailUpdate {

src/models/ipaddr.rs

@@ -163,76 +163,55 @@ pub struct DetailedIpAddr {
     as_org: Option<String>,
 }
 
-impl fmt::Display for DetailedIpAddr {
-    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        if !self.unscoped {
-            write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{}\x1b[0m", self.id, self.value)?;
+impl DisplayableDetailed for DetailedIpAddr {
+    #[inline]
+    fn scoped(&self) -> bool {
+        !self.unscoped
+    }
 
-            if let Some(ref continent) = self.continent {
-                write!(w, " [{}", continent)?;
+    #[inline]
+    fn print(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        self.id(w, self.id)?;
+        self.green_debug(w, &self.value)?;
 
-                if let Some(ref country) = self.country {
-                    write!(w, " / {}", country)?;
-                }
+        if let Some(ref continent) = self.continent {
+            write!(w, " [{}", continent)?;
 
-                if let Some(ref city) = self.city {
-                    write!(w, " / {}", city)?;
-                }
-
-                write!(w, "]")?;
+            if let Some(ref country) = self.country {
+                write!(w, " / {}", country)?;
             }
 
-            if let Some(ref asn) = self.asn {
-                write!(w, " [{}", asn)?;
-
-                if let Some(ref as_org) = self.as_org {
-                    write!(w, " / {:?}", as_org)?;
-                }
-
-                write!(w, "]")?;
+            if let Some(ref city) = self.city {
+                write!(w, " / {}", city)?;
             }
 
-            for subdomain in &self.subdomains {
-                write!(w, "\n\t\x1b[33m{}\x1b[0m", subdomain)?;
-            }
-        } else {
-            write!(w, "\x1b[90m#{}, {}", self.id, self.value)?;
+            write!(w, "]")?;
+        }
 
-            if let Some(ref continent) = self.continent {
-                write!(w, " [{}", continent)?;
+        if let Some(ref asn) = self.asn {
+            write!(w, " [{}", asn)?;
 
-                if let Some(ref country) = self.country {
-                    write!(w, " / {}", country)?;
-                }
-
-                if let Some(ref city) = self.city {
-                    write!(w, " / {}", city)?;
-                }
-
-                write!(w, "]")?;
+            if let Some(ref as_org) = self.as_org {
+                write!(w, " / {:?}", as_org)?;
             }
 
-            if let Some(ref asn) = self.asn {
-                write!(w, " [{}", asn)?;
-
-                if let Some(ref as_org) = self.as_org {
-                    write!(w, " / {:?}", as_org)?;
-                }
-
-                write!(w, "]")?;
-            }
-
-            write!(w, "\x1b[0m");
-
-            for subdomain in &self.subdomains {
-                write!(w, "\n\t\x1b[90m{}\x1b[0m", subdomain)?;
-            }
+            write!(w, "]")?;
         }
 
         Ok(())
     }
+
+    #[inline]
+    fn children(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        for subdomain in &self.subdomains {
+            self.child(w, subdomain)?;
+        }
+        Ok(())
+    }
 }
 
+display_detailed!(DetailedIpAddr);
+
 impl Detailed for IpAddr {
     type T = DetailedIpAddr;
 
@@ -327,6 +306,16 @@ impl Printable<PrintableIpAddr> for NewIpAddrOwned {
     }
 }
 
+pub type InsertIpAddr = NewIpAddrOwned;
+
+impl LuaInsertToNewOwned for InsertIpAddr {
+    type Target = NewIpAddrOwned;
+
+    fn try_into_new(self) -> Result<NewIpAddrOwned> {
+        Ok(self)
+    }
+}
+
 #[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
 #[table_name="ipaddrs"]
 pub struct IpAddrUpdate {

src/models/mod.rs

@@ -205,6 +205,116 @@ pub trait Detailed: Scopable {
     fn detailed(&self, db: &Database) -> Result<Self::T>;
 }
 
+pub trait DisplayableDetailed {
+    fn scoped(&self) -> bool;
+
+    #[inline]
+    fn start(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        if !self.scoped() {
+            write!(w, "\x1b[90m")
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    fn end(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        if !self.scoped() {
+            write!(w, "\x1b[0m")
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    fn id<D: fmt::Display>(&self, w: &mut fmt::Formatter, v: D) -> fmt::Result {
+        if self.scoped() {
+            write!(w, "\x1b[32m#{}\x1b[0m, ", v)
+        } else {
+            write!(w, "#{}, ", v)
+        }
+    }
+
+    #[inline]
+    fn green(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        if self.scoped() {
+            write!(w, "\x1b[32m")
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    fn green_display<D: fmt::Display>(&self, w: &mut fmt::Formatter, v: D) -> fmt::Result {
+        if self.scoped() {
+            write!(w, "\x1b[32m{}\x1b[0m", v)
+        } else {
+            write!(w, "{}", v)
+        }
+    }
+
+    #[inline]
+    fn green_debug<D: fmt::Debug>(&self, w: &mut fmt::Formatter, v: D) -> fmt::Result {
+        if self.scoped() {
+            write!(w, "\x1b[32m{:?}\x1b[0m", v)
+        } else {
+            write!(w, "{:?}", v)
+        }
+    }
+
+    #[inline]
+    fn yellow(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        if self.scoped() {
+            write!(w, "\x1b[33m")
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    fn clear(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        if self.scoped() {
+            write!(w, "\x1b[0m")
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    fn child<D: fmt::Display>(&self, w: &mut fmt::Formatter, c: D) -> fmt::Result {
+        if self.scoped() {
+            // if child is unscoped, draw as grey as well
+            write!(w, "\n\t\x1b[33m{}\x1b[0m", c)
+        } else {
+            write!(w, "\n\t\x1b[90m{}\x1b[0m", c)
+        }
+    }
+
+    fn print(&self, w: &mut fmt::Formatter) -> fmt::Result;
+
+    fn children(&self, w: &mut fmt::Formatter) -> fmt::Result;
+}
+
+macro_rules! display_detailed {
+    ( $name:ident ) => {
+        impl fmt::Display for $name {
+            fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
+                self.start(w)?;
+                self.print(w)?;
+                self.end(w)?;
+                self.children(w)?;
+                Ok(())
+            }
+        }
+    };
+}
+
+pub trait LuaInsertToNewOwned {
+    type Target;
+
+    fn try_into_new(self) -> Result<Self::Target>;
+}
+
 mod domain;
 pub use self::domain::*;
 

src/models/subdomain.rs

@@ -151,26 +151,30 @@ pub struct DetailedSubdomain {
 }
 
 // TODO: maybe print urls as well
-impl fmt::Display for DetailedSubdomain {
-    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        if !self.unscoped {
-            write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{:?}\x1b[0m", self.id, self.value)?;
+impl DisplayableDetailed for DetailedSubdomain {
+    #[inline]
+    fn scoped(&self) -> bool {
+        !self.unscoped
+    }
 
-            for ipaddr in &self.ipaddrs {
-                write!(w, "\n\t\x1b[33m{}\x1b[0m", ipaddr)?;
-            }
-        } else {
-            write!(w, "\x1b[90m#{}, {:?}\x1b[0m", self.id, self.value)?;
+    #[inline]
+    fn print(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        self.id(w, self.id)?;
+        self.green_debug(w, &self.value)?;
+        Ok(())
+    }
 
-            for ipaddr in &self.ipaddrs {
-                write!(w, "\n\t\x1b[90m{}\x1b[0m", ipaddr)?;
-            }
+    #[inline]
+    fn children(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        for ipaddr in &self.ipaddrs {
+            self.child(w, ipaddr)?;
         }
-
         Ok(())
     }
 }
 
+display_detailed!(DetailedSubdomain);
+
 impl Detailed for Subdomain {
     type T = DetailedSubdomain;
 
@@ -236,6 +240,16 @@ impl Printable<PrintableSubdomain> for NewSubdomainOwned {
     }
 }
 
+pub type InsertSubdomain = NewSubdomainOwned;
+
+impl LuaInsertToNewOwned for InsertSubdomain {
+    type Target = NewSubdomainOwned;
+
+    fn try_into_new(self) -> Result<NewSubdomainOwned> {
+        Ok(self)
+    }
+}
+
 #[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
 #[table_name="subdomains"]
 pub struct SubdomainUpdate {

src/models/subdomain_ipaddr.rs

@@ -123,3 +123,13 @@ impl Printable<PrintableSubdomainIpAddr> for NewSubdomainIpAddr {
         })
     }
 }
+
+pub type InsertSubdomainIpAddr = NewSubdomainIpAddr;
+
+impl LuaInsertToNewOwned for InsertSubdomainIpAddr {
+    type Target = NewSubdomainIpAddr;
+
+    fn try_into_new(self) -> Result<NewSubdomainIpAddr> {
+        Ok(self)
+    }
+}

src/models/url.rs

@@ -3,6 +3,7 @@ use diesel;
 use diesel::prelude::*;
 use models::*;
 use ser;
+use url;
 
 
 #[derive(Identifiable, Queryable, Associations, Serialize, Deserialize, PartialEq, Debug)]
@@ -12,6 +13,7 @@ pub struct Url {
     pub id: i32,
     pub subdomain_id: i32,
     pub value: String,
+    pub path: String,
     pub status: Option<i32>,
     pub body: Option<Vec<u8>>,
     pub unscoped: bool,
@@ -156,36 +158,44 @@ pub struct DetailedUrl {
     redirect: Option<String>,
 }
 
-impl fmt::Display for DetailedUrl {
-    fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result {
-        if !self.unscoped {
-            write!(w, "\x1b[32m#{}\x1b[0m, \x1b[32m{:?}\x1b[0m", self.id, self.value)?;
+impl DisplayableDetailed for DetailedUrl {
+    #[inline]
+    fn scoped(&self) -> bool {
+        !self.unscoped
+    }
 
-            if let Some(status) = self.status {
-                write!(w, " (\x1b[33m{}\x1b[0m", status)?;
+    #[inline]
+    fn print(&self, w: &mut fmt::Formatter) -> fmt::Result {
+        self.id(w, self.id)?;
+        self.green_debug(w, &self.value)?;
 
-                if let Some(ref redirect) = self.redirect {
-                    write!(w, " => \x1b[33m{:?}\x1b[0m", redirect)?;
-                }
+        if let Some(status) = self.status {
+            write!(w, " (")?;
+            self.green_display(w, status)?;
 
-                write!(w, ")")?;
+            if let Some(ref redirect) = self.redirect {
+                write!(w, " => ")?;
+                self.green_debug(w, redirect)?;
             }
 
-            if let Some(ref title) = self.title {
-                write!(w, " {:?}", title)?;
-            }
-        } else {
-            write!(w, "\x1b[90m#{}, {:?}\x1b[0m", self.id, self.value)?;
+            write!(w, ")")?;
+        }
 
-            if let Some(status) = self.status {
-                write!(w, "\x1b[90m ({})\x1b[0m", status)?;
-            }
+        if let Some(ref title) = self.title {
+            write!(w, " {:?}", title)?;
         }
 
         Ok(())
     }
+
+    #[inline]
+    fn children(&self, _w: &mut fmt::Formatter) -> fmt::Result {
+        Ok(())
+    }
 }
 
+display_detailed!(DetailedUrl);
+
 impl Detailed for Url {
     type T = DetailedUrl;
 
@@ -206,6 +216,7 @@ impl Detailed for Url {
 pub struct NewUrl<'a> {
     pub subdomain_id: i32,
     pub value: &'a str,
+    pub path: &'a str,
     pub status: Option<i32>,
     pub body: Option<&'a Vec<u8>>,
     pub online: Option<bool>,
@@ -246,6 +257,7 @@ impl<'a> Upsertable<Url> for NewUrl<'a> {
 pub struct NewUrlOwned {
     pub subdomain_id: i32,
     pub value: String,
+    pub path: String,
     pub status: Option<i32>,
     #[serde(deserialize_with="ser::opt_string_or_bytes")]
     pub body: Option<Vec<u8>>,
@@ -264,6 +276,38 @@ impl Printable<PrintableUrl> for NewUrlOwned {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct InsertUrl {
+    pub subdomain_id: i32,
+    pub value: String,
+    pub status: Option<i32>,
+    #[serde(deserialize_with="ser::opt_string_or_bytes")]
+    pub body: Option<Vec<u8>>,
+    pub online: Option<bool>,
+    pub title: Option<String>,
+    pub redirect: Option<String>,
+}
+
+impl LuaInsertToNewOwned for InsertUrl {
+    type Target = NewUrlOwned;
+
+    fn try_into_new(self) -> Result<NewUrlOwned> {
+        let url = url::Url::parse(&self.value)?;
+        let path = url.path().to_string();
+
+        Ok(NewUrlOwned {
+            subdomain_id: self.subdomain_id,
+            value: self.value,
+            path,
+            status: self.status,
+            body: self.body,
+            online: self.online,
+            title: self.title,
+            redirect: self.redirect,
+        })
+    }
+}
+
 #[derive(Identifiable, AsChangeset, Serialize, Deserialize, Debug)]
 #[table_name="urls"]
 pub struct UrlUpdate {

src/psl.rs

@@ -2,7 +2,7 @@ use errors::*;
 
 use chrootable_https::Client;
 use std::fs::{self, File};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use paths;
 use publicsuffix::{self, Domain, DnsName};
 use worker;
@@ -15,7 +15,7 @@ pub struct Psl {
 
 impl Psl {
     pub fn open_or_download() -> Result<Psl> {
-        let path = paths::cache_dir()?.join("public_suffix_list.dat");
+        let path = Self::path()?;
 
         let reader = match File::open(&path) {
             Ok(f) => f,
@@ -35,9 +35,21 @@ impl Psl {
         })
     }
 
-    pub fn open_into_string() -> Result<String> {
-        let path = paths::cache_dir()?.join("public_suffix_list.dat");
+    pub fn path() -> Result<PathBuf> {
+        // use system path if exists
+        let path = Path::new("/usr/share/publicsuffix/public_suffix_list.dat");
+        if path.exists() {
+            return Ok(path.to_path_buf());
+        }
 
+        // else, use local cache
+        let path = paths::cache_dir()?
+            .join("public_suffix_list.dat");
+        Ok(path)
+    }
+
+    pub fn open_into_string() -> Result<String> {
+        let path = Self::path()?;
         let s = fs::read_to_string(&path)?;
         Ok(s)
     }

src/runtime/db.rs

@@ -11,6 +11,13 @@ use models::*;
 use json::LuaJsonValue;
 
 
+pub fn try_into_new<T: LuaInsertToNewOwned>(x: LuaJsonValue) -> Result<T::Target>
+    where for<'de> T: serde::Deserialize<'de>
+{
+    structs::from_lua::<T>(x)?
+        .try_into_new()
+}
+
 pub fn db_add(lua: &mut hlua::Lua, state: Arc<State>) {
     lua.set("db_add", hlua::function2(move |family: String, object: AnyLuaValue| -> Result<Option<i32>> {
         let family = Family::from_str(&family)
@@ -19,27 +26,27 @@ pub fn db_add(lua: &mut hlua::Lua, state: Arc<State>) {
 
         let object = match family {
             Family::Domain => {
-                Insert::Domain(structs::from_lua::<NewDomainOwned>(object)
+                Insert::Domain(try_into_new::<InsertDomain>(object)
                     .map_err(|e| state.set_error(e))?)
             },
             Family::Subdomain => {
-                Insert::Subdomain(structs::from_lua::<NewSubdomainOwned>(object)
+                Insert::Subdomain(try_into_new::<InsertSubdomain>(object)
                     .map_err(|e| state.set_error(e))?)
             },
             Family::IpAddr => {
-                Insert::IpAddr(structs::from_lua::<NewIpAddrOwned>(object)
+                Insert::IpAddr(try_into_new::<InsertIpAddr>(object)
                     .map_err(|e| state.set_error(e))?)
             },
             Family::SubdomainIpAddr => {
-                Insert::SubdomainIpAddr(structs::from_lua::<NewSubdomainIpAddr>(object)
+                Insert::SubdomainIpAddr(try_into_new::<InsertSubdomainIpAddr>(object)
                     .map_err(|e| state.set_error(e))?)
             },
             Family::Url => {
-                Insert::Url(structs::from_lua::<NewUrlOwned>(object)
+                Insert::Url(try_into_new::<InsertUrl>(object)
                     .map_err(|e| state.set_error(e))?)
             },
             Family::Email => {
-                Insert::Email(structs::from_lua::<NewEmailOwned>(object)
+                Insert::Email(try_into_new::<InsertEmail>(object)
                     .map_err(|e| state.set_error(e))?)
             },
         };

src/runtime/dns.rs

@@ -1,23 +1,75 @@
 use errors::*;
 
-use chrootable_https::{Resolver, DnsResolver};
+use chrootable_https::dns::{Resolver, DnsResolver, RecordType};
+use engine;
 use engine::ctx::State;
 use json::LuaJsonValue;
 use hlua::{self, AnyLuaValue};
 use serde_json;
 use std::sync::Arc;
-use trust_dns_proto;
+use std::net::SocketAddr;
+use std::time::Duration;
 
 
+#[derive(Debug, Deserialize)]
+pub struct ResolveOptions {
+    //record: RecordType,
+    record: String,
+    // TODO: this could be more than one
+    nameserver: Option<SocketAddr>,
+    #[serde(default)]
+    tcp: bool,
+    timeout: Option<u64>,
+}
+
+impl ResolveOptions {
+    pub fn new(record: String) -> Result<ResolveOptions> {
+        let record = record.parse()?;
+        Ok(ResolveOptions {
+            record,
+            nameserver: None,
+            tcp: false,
+            timeout: None,
+        })
+    }
+
+    pub fn record_type(&self) -> Result<RecordType> {
+        self.record.parse()
+            .map_err(Error::from)
+    }
+
+    pub fn from_lua(x: AnyLuaValue) -> Result<ResolveOptions> {
+        match x {
+            AnyLuaValue::LuaAnyString(s) => {
+                let s = String::from_utf8(s.0)?;
+                ResolveOptions::new(s)
+            },
+            AnyLuaValue::LuaString(s) => ResolveOptions::new(s),
+            x => engine::structs::from_lua(x.into()),
+        }
+    }
+}
+
 pub fn dns(lua: &mut hlua::Lua, state: Arc<State>) {
-    lua.set("dns", hlua::function2(move |name: String, record: String| -> Result<AnyLuaValue> {
-        let resolver = Resolver::from_config(state.dns_config().as_ref().clone())
+    lua.set("dns", hlua::function2(move |name: String, options: AnyLuaValue| -> Result<AnyLuaValue> {
+        let options = ResolveOptions::from_lua(options)
             .map_err(|e| state.set_error(e))?;
 
-        let record = record.parse()
-            .map_err(|e: trust_dns_proto::error::ProtoError| state.set_error(e.into()))?;
+        let ns = match options.nameserver {
+            Some(ns) => vec![ns],
+            None => state.dns_config().ns.clone(),
+        };
 
-        let reply = resolver.resolve_adv(&name, record)
+        let timeout = options.timeout
+            .map(Duration::from_millis);
+
+        let resolver = Resolver {
+            ns,
+            tcp: options.tcp,
+            timeout,
+        };
+
+        let reply = resolver.resolve(&name, options.record_type()?)
             .map_err(|e| state.set_error(e))?;
 
         let reply = serde_json::to_value(reply)
@@ -38,8 +90,9 @@ mod tests {
         let script = Script::load_unchecked(r#"
         function run()
             x = dns('example.com', 'A')
+            if last_err() then return end
             print(x)
-            if x['success'][0]['A'] == nil then
+            if x['answers'][1][2]['A'] == nil then
                 return "Couldn't resolve"
             end
         end
@@ -47,14 +100,101 @@ mod tests {
         script.test().expect("Script failed");
     }
 
+    #[test]
+    #[ignore]
+    fn verify_custom_resolver_a() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            x = dns('example.com', {
+                record='A',
+                nameserver='1.1.1.1:53',
+            })
+            if last_err() then return end
+            print(x)
+            if x['answers'][1][2]['A'] == nil then
+                return "Couldn't resolve"
+            end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    #[ignore]
+    fn verify_custom_resolver_axfr() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            x = dns('zonetransfer.me', {
+                record='AXFR',
+                nameserver='81.4.108.41:53',
+                tcp=true,
+                timeout=15000,
+            })
+            if last_err() then return end
+            print(x)
+
+            if x['error'] ~= nil then
+                return 'Error: ' .. x['error']
+            end
+
+            has_soa = false
+            has_ns = false
+            has_a = false
+            has_mx = false
+
+            i = 1
+            answers = x['answers']
+            while answers[i] do
+                r = answers[i][2]
+
+                if r['SOA'] ~= nil then
+                    has_soa = true
+                end
+
+                if r['NS'] ~= nil then
+                    has_ns = true
+                end
+
+                if r['A'] ~= nil then
+                    has_a = true
+                end
+
+                if r['MX'] ~= nil then
+                    has_mx = true
+                end
+
+                i = i+1
+            end
+
+            if not has_soa then
+                return 'Error: missing soa'
+            end
+
+            if not has_ns then
+                return 'Error: missing ns'
+            end
+
+            if not has_a then
+                return 'Error: missing a'
+            end
+
+            if not has_mx then
+                return 'Error: missing mx'
+            end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
     #[test]
     #[ignore]
     fn verify_resolve_aaaa() {
         let script = Script::load_unchecked(r#"
         function run()
             x = dns('example.com', 'AAAA')
+            if last_err() then return end
             print(x)
-            if x['success'][0]['AAAA'] == nil then
+            if x['answers'][1][2]['AAAA'] == nil then
                 return "Couldn't resolve"
             end
         end
@@ -68,8 +208,9 @@ mod tests {
         let script = Script::load_unchecked(r#"
         function run()
             x = dns('example.com', 'TXT')
+            if last_err() then return end
             print(x)
-            if x['success'][0]['TXT'] == nil then
+            if x['answers'][1][2]['TXT'] == nil then
                 return "Couldn't resolve"
             end
         end
@@ -83,8 +224,9 @@ mod tests {
         let script = Script::load_unchecked(r#"
         function run()
             x = dns('example.com', 'NS')
+            if last_err() then return end
             print(x)
-            if x['success'][0]['NS'] == nil then
+            if x['answers'][1][2]['NS'] == nil then
                 return "Couldn't resolve"
             end
         end
@@ -98,8 +240,9 @@ mod tests {
         let script = Script::load_unchecked(r#"
         function run()
             x = dns('example.com', 'SOA')
+            if last_err() then return end
             print(x)
-            if x['success'][0]['SOA'] == nil then
+            if x['answers'][1][2]['SOA'] == nil then
                 return "Couldn't resolve"
             end
         end
@@ -113,8 +256,9 @@ mod tests {
         let script = Script::load_unchecked(r#"
         function run()
             x = dns('1.1.1.1.in-addr.arpa', 'PTR')
+            if last_err() then return end
             print(x)
-            if x['success'][0]['PTR'] == nil then
+            if x['answers'][1][2]['PTR'] == nil then
                 return "Couldn't resolve"
             end
         end
@@ -128,8 +272,27 @@ mod tests {
         let script = Script::load_unchecked(r#"
         function run()
             x = dns('doesntexist.example.com', 'A')
+            if last_err() then return end
             print(x)
-            return x['success'] == nil
+            if x['error'] ~= 'NXDomain' then
+                return "Expected NXDomain"
+            end
+        end
+        "#).expect("Failed to load script");
+        script.test().expect("Script failed");
+    }
+
+    #[test]
+    #[ignore]
+    fn verify_empty_cname() {
+        let script = Script::load_unchecked(r#"
+        function run()
+            x = dns('example.com', 'CNAME')
+            if last_err() then return end
+            print(x)
+            if x['error'] ~= nil then
+                return "Expected no error"
+            end
         end
         "#).expect("Failed to load script");
         script.test().expect("Script failed");

src/runtime/logger.rs

@@ -9,6 +9,12 @@ pub fn info(lua: &mut hlua::Lua, state: Arc<State>) {
     }))
 }
 
+pub fn debug(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("debug", hlua::function1(move |msg: String| {
+        state.debug(msg);
+    }))
+}
+
 pub fn error(lua: &mut hlua::Lua, state: Arc<State>) {
     lua.set("error", hlua::function1(move |msg: String| {
         state.error(msg);

src/runtime/mod.rs

@@ -18,6 +18,7 @@ import_fns!(print);
 import_fns!(psl);
 import_fns!(regex);
 import_fns!(sleep);
+import_fns!(stdio);
 import_fns!(url);
 import_fns!(utf8);
 import_fns!(x509);

src/runtime/stdio.rs

@@ -0,0 +1,13 @@
+use errors::*;
+
+use engine::ctx::State;
+use hlua;
+use std::sync::Arc;
+
+
+pub fn stdin_readline(lua: &mut hlua::Lua, state: Arc<State>) {
+    lua.set("stdin_readline", hlua::function0(move || -> Result<Option<String>> {
+        state.stdin_readline()
+            .map_err(|e| state.set_error(e))
+    }))
+}

src/runtime/x509.rs

@@ -73,7 +73,7 @@ WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik=
             print(x)
 
             vn = {}
-            i = 0
+            i = 1
             while x['valid_names'][i] do
                 vn[x['valid_names'][i]] = true
                 i = i+1
@@ -87,7 +87,7 @@ WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik=
                 return 'Not valid for github.com'
             end
 
-            if i ~= 2 then
+            if i ~= 3 then
                 return 'Unexpected number of names'
             end
         end
@@ -127,14 +127,14 @@ ZkZZmqNn2Q8=
             print(x)
 
             vn = {}
-            i = 0
+            i = 1
             while x['valid_names'][i] do
                 vn[x['valid_names'][i]] = true
                 i = i+1
             end
 
             vi = {}
-            j = 0
+            j = 1
             while x['valid_ipaddrs'][j] do
                 vi[x['valid_ipaddrs'][j]] = true
                 j = j+1
@@ -146,7 +146,7 @@ ZkZZmqNn2Q8=
             if not vn['cloudflare-dns.com'] then
                 return 'Not valid for cloudflare-dns.com'
             end
-            if i ~= 2 then
+            if i ~= 3 then
                 return 'Unexpected number of names'
             end
 
@@ -162,7 +162,7 @@ ZkZZmqNn2Q8=
             if not vi['2606:4700:4700::1111'] then
                 return 'Not valid for 2606:4700:4700::1111'
             end
-            if j ~= 4 then
+            if j ~= 5 then
                 return 'Unexpected number of ip addrs'
             end
         end

src/sandbox/seccomp.rs

@@ -11,8 +11,11 @@ pub fn init() -> Result<()> {
     ctx.allow_syscall(Syscall::sigaltstack)?;
     ctx.allow_syscall(Syscall::munmap)?;
     ctx.allow_syscall(Syscall::openat)?;
+    #[cfg(not(target_arch = "aarch64"))]
     ctx.allow_syscall(Syscall::open)?;
     ctx.allow_syscall(Syscall::fcntl)?;
+    #[cfg(target_arch = "arm")]
+    ctx.allow_syscall(Syscall::fcntl64)?;
     ctx.allow_syscall(Syscall::uname)?;
     ctx.allow_syscall(Syscall::close)?;
     ctx.allow_syscall(Syscall::epoll_create1)?;
@@ -21,15 +24,23 @@ pub fn init() -> Result<()> {
     ctx.allow_syscall(Syscall::sched_getaffinity)?;
     ctx.allow_syscall(Syscall::socket)?;
     ctx.allow_syscall(Syscall::connect)?;
+    #[cfg(not(target_arch = "aarch64"))]
     ctx.allow_syscall(Syscall::epoll_wait)?;
     ctx.allow_syscall(Syscall::epoll_pwait)?;
     ctx.allow_syscall(Syscall::getrandom)?;
     ctx.allow_syscall(Syscall::bind)?;
     ctx.allow_syscall(Syscall::ioctl)?;
+    #[cfg(target_arch = "arm")]
+    ctx.allow_syscall(Syscall::send)?;
     ctx.allow_syscall(Syscall::sendto)?;
+    #[cfg(target_arch = "arm")]
+    ctx.allow_syscall(Syscall::recv)?;
     ctx.allow_syscall(Syscall::recvfrom)?;
     ctx.allow_syscall(Syscall::getsockopt)?;
+    #[cfg(not(target_arch = "arm"))]
     ctx.allow_syscall(Syscall::mmap)?;
+    #[cfg(target_arch = "arm")]
+    ctx.allow_syscall(Syscall::mmap2)?;
     ctx.allow_syscall(Syscall::mprotect)?;
     ctx.allow_syscall(Syscall::clone)?;
     ctx.allow_syscall(Syscall::set_robust_list)?;

src/schema.rs

@@ -56,6 +56,7 @@ table! {
         id -> Integer,
         subdomain_id -> Integer,
         value -> Text,
+        path -> Text,
         status -> Nullable<Integer>,
         body -> Nullable<Binary>,
         unscoped -> Bool,

src/term.rs

@@ -52,6 +52,8 @@ pub static SPINNERS: &[&[&str]] = &[
 pub trait SpinLogger {
     fn log(&mut self, line: &str);
 
+    fn debug(&mut self, line: &str);
+
     fn error(&mut self, line: &str);
 
     fn status(&mut self, status: String);
@@ -117,6 +119,10 @@ impl SpinLogger for Spinner {
         println!("\r\x1b[2K\x1b[1m[\x1b[34m{}\x1b[0;1m]\x1b[0m {}", '*', line);
     }
 
+    fn debug(&mut self, line: &str) {
+        println!("\r\x1b[2K\x1b[1m[\x1b[34m{}\x1b[0;1m]\x1b[0m {}", '#', line);
+    }
+
     fn error(&mut self, line: &str) {
         println!("\r\x1b[2K\x1b[1m[\x1b[31m{}\x1b[0;1m]\x1b[0m {}", '-', line);
     }
@@ -134,6 +140,10 @@ pub fn info(line: &str) {
     println!("\x1b[1m[\x1b[32m{}\x1b[0;1m]\x1b[0m {}", '+', line);
 }
 
+pub fn debug(line: &str) {
+    println!("\x1b[2K\x1b[1m[\x1b[34m{}\x1b[0;1m]\x1b[0m {}", '#', line);
+}
+
 pub fn warn(line: &str) {
     eprintln!("\x1b[1m[\x1b[33m{}\x1b[0;1m]\x1b[0m {}", '!', line);
 }
@@ -235,6 +245,11 @@ impl SpinLogger for StackedSpinners {
         println!("\r\x1b[2K\x1b[1m[\x1b[34m{}\x1b[0;1m]\x1b[0m {}", '*', line);
     }
 
+    fn debug(&mut self, line: &str) {
+        self.jump2start();
+        println!("\r\x1b[2K\x1b[1m[\x1b[34m{}\x1b[0;1m]\x1b[0m {}", '#', line);
+    }
+
     fn error(&mut self, line: &str) {
         self.jump2start();
         println!("\r\x1b[2K\x1b[1m[\x1b[31m{}\x1b[0;1m]\x1b[0m {}", '-', line);
@@ -264,6 +279,10 @@ impl<'a, T: SpinLogger> SpinLogger for PrefixedLogger<'a, T> {
         self.s.log(&format!("{:50}: {}", self.prefix, line))
     }
 
+    fn debug(&mut self, line: &str) {
+        self.s.debug(&format!("{:50}: {}", self.prefix, line))
+    }
+
     fn error(&mut self, line: &str) {
         self.s.error(&format!("{:50}: {}", self.prefix, line))
     }

src/web.rs

@@ -202,8 +202,7 @@ impl HttpRequest {
         // send request
         debug!("Sending http request: {:?}", req);
 
-        let config = state.dns_config().as_ref().clone();
-        let resolver = Resolver::from_config(config)?;
+        let resolver = state.dns_config().as_ref().clone();
         let mut http = Client::new(resolver);
 
         if let Some(timeout) = self.timeout {

src/worker.rs

@@ -3,6 +3,7 @@ use errors::*;
 use channel;
 use db::{Database, DbChange, Family};
 use engine::{self, Module};
+use engine::isolation::Supervisor;
 use models::*;
 use serde_json;
 use shell::Readline;
@@ -10,6 +11,7 @@ use std::result;
 use std::sync::{mpsc, Arc, Mutex};
 use std::time::Duration;
 use std::thread;
+use std::io::{Stdin, BufRead, BufReader};
 use term::{Spinner, StackedSpinners, SpinLogger};
 use threadpool::ThreadPool;
 
@@ -20,6 +22,7 @@ type DbSender = mpsc::Sender<result::Result<Option<i32>, String>>;
 pub enum Event {
     Log(LogEvent),
     Database(DatabaseEvent),
+    Stdio(StdioEvent),
     Exit(ExitEvent),
 }
 
@@ -65,6 +68,12 @@ impl EventSender {
     }
 }
 
+pub trait EventWithCallback {
+    type Payload;
+
+    fn with_callback(self, tx: mpsc::Sender<result::Result<Self::Payload, String>>) -> Event2;
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub enum ExitEvent {
     Ok,
@@ -74,6 +83,7 @@ pub enum ExitEvent {
 #[derive(Debug, Serialize, Deserialize)]
 pub enum LogEvent {
     Info(String),
+    Debug(String),
     Error(String),
     Status(String),
 }
@@ -82,6 +92,7 @@ impl LogEvent {
     pub fn apply<T: SpinLogger>(self, spinner: &mut T) {
         match self {
             LogEvent::Info(info) => spinner.log(&info),
+            LogEvent::Debug(debug) => spinner.debug(&debug),
             LogEvent::Error(error) => spinner.error(&error),
             LogEvent::Status(status) => spinner.status(status),
         }
@@ -95,12 +106,25 @@ pub enum DatabaseEvent {
     Update((String, Update)),
 }
 
+impl EventWithCallback for DatabaseEvent {
+    type Payload = Option<i32>;
+
+    fn with_callback(self, tx: mpsc::Sender<result::Result<Self::Payload, String>>) -> Event2 {
+        Event2::Database((self, tx))
+    }
+}
+
 impl DatabaseEvent {
-    pub fn apply<T: SpinLogger>(self, tx: DbSender, spinner: &mut T, db: &Database) {
+    pub fn apply<T: SpinLogger>(self, tx: DbSender, spinner: &mut T, db: &Database, verbose: u64) {
         match self {
             DatabaseEvent::Insert(object) => {
+                if verbose >= 1 {
+                    spinner.debug(&format!("Inserting: {:?}", object));
+                }
+
                 let result = db.insert_generic(&object);
                 debug!("{:?} => {:?}", object, result);
+
                 let result = match result {
                     Ok(Some((DbChange::Insert, id))) => {
                         // TODO: replace id with actual object(?)
@@ -134,6 +158,10 @@ impl DatabaseEvent {
                 tx.send(result).expect("Failed to send db result to channel");
             },
             DatabaseEvent::Update((object, update)) => {
+                if verbose >= 1 {
+                    spinner.debug(&format!("Updating: {:?}", update));
+                }
+
                 let result = db.update_generic(&update);
                 debug!("{:?}: {:?} => {:?}", object, update, result);
                 let result = result
@@ -153,7 +181,35 @@ impl DatabaseEvent {
     }
 }
 
-pub fn spawn(rl: &mut Readline, module: &Module, args: Vec<(serde_json::Value, Option<String>)>, threads: usize) {
+#[derive(Debug, Serialize, Deserialize)]
+pub struct StdioEvent {
+}
+
+impl StdioEvent {
+    fn read(reader: &mut Option<BufReader<Stdin>>) -> Result<Option<String>> {
+        if let Some(ref mut reader) = reader {
+            let mut line = String::new();
+            let len = reader.read_line(&mut line)?;
+            debug!("stdin: {:?}", line);
+
+            if len > 0 {
+                Ok(Some(line))
+            } else {
+                Ok(None)
+            }
+        } else {
+            bail!("stdin is unavailable");
+        }
+    }
+
+    pub fn apply(self, supervisor: &mut Supervisor, tx: &EventSender, reader: &mut Option<BufReader<Stdin>>) {
+        let reply = Self::read(reader)
+            .map_err(|e| e.to_string());
+        supervisor.send_struct(reply, tx);
+    }
+}
+
+pub fn spawn(rl: &mut Readline, module: &Module, args: Vec<(serde_json::Value, Option<String>)>, threads: usize, verbose: u64, has_stdin: bool) {
     // This function hangs if args is empty, so return early if that's the case
     if args.is_empty() {
         return;
@@ -183,7 +239,7 @@ pub fn spawn(rl: &mut Readline, module: &Module, args: Vec<(serde_json::Value, O
             }
 
             tx.send(Event2::Start);
-            let event = match engine::isolation::spawn_module(module, &tx, arg) {
+            let event = match engine::isolation::spawn_module(module, &tx, arg, verbose, has_stdin) {
                 Ok(_) => ExitEvent::Ok,
                 Err(err) => ExitEvent::Err(err.to_string()),
             };
@@ -206,7 +262,7 @@ pub fn spawn(rl: &mut Readline, module: &Module, args: Vec<(serde_json::Value, O
                             stack.add(name, label);
                         },
                         Event2::Log(log) => log.apply(&mut stack.prefixed(name)),
-                        Event2::Database((db, tx)) => db.apply(tx, &mut stack.prefixed(name), rl.db()),
+                        Event2::Database((db, tx)) => db.apply(tx, &mut stack.prefixed(name), rl.db(), verbose),
                         Event2::Exit(event) => {
                             stack.remove(&name);
                             if let ExitEvent::Err(error) = event {
@@ -251,6 +307,7 @@ pub fn spawn_fn<F, T>(label: &str, f: F, clear: bool) -> Result<T>
                 recv(rx) -> msg => match msg.ok() {
                     Some(Event::Log(log)) => log.apply(&mut *spinner),
                     Some(Event::Database(_)) => (),
+                    Some(Event::Stdio(_)) => (),
                     // TODO: refactor
                     Some(Event::Exit(ExitEvent::Ok)) => break,
                     Some(Event::Exit(ExitEvent::Err(error))) => spinner.error(&error),