Release v0.8.0

Introduce an option system

.dockerignore

@@ -1,6 +1,9 @@
 target
 Dockerfile
 .dockerignore
+docker
+docs
+ci
 .git
 .gitignore
 *.sw[op]

CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# How to contribute
+
+To contribute to sn0int, clone the repository and make sure both the build and
+tests pass for you:
+
+    git clone https://github.com/kpcyrd/sn0int.git
+    cd sn0int
+    # build the project
+    cargo build
+    # run regular tests
+    cargo test
+    # run tests depending on the network
+    # these might fail if a service is down
+    cargo test -- --ignored
+
+The project is loosely structured into a few folders:
+
+- `src/models/` - database models
+- `src/runtime/` - the stdlib that's exposed to lua
+- `src/engine/` - code related to lua
+- `src/sandbox/` - code related to sandboxing
+- `src/cmd/` - cli commands
+- `src/` - misc modules
+
+After you're done, make sure the build completes without any warnings and both
+tests pass successfully:
+
+    cargo test
+    cargo test -- --ignored
+
+If you want to introduce a new feature feel free to open an issue first to make
+sure your feature is a good fit for the project before implementing it.

Cargo.toml

@@ -1,12 +1,13 @@
 [package]
 name = "sn0int"
-version = "0.6.0"
-description = "OSINT framework and package manager"
+version = "0.8.0"
+description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
+edition = "2018"
 
 [badges]
 travis-ci = { repository = "kpcyrd/sn0int" }
@@ -16,8 +17,8 @@ members = ["sn0int-registry/sn0int-common",
            "sn0int-registry"]
 
 [dependencies]
-sn0int-common = { version="0.3.0", path="sn0int-registry/sn0int-common" }
-rustyline = "2"
+sn0int-common = { version="0.4.0", path="sn0int-registry/sn0int-common" }
+rustyline = "3"
 log = "0.4"
 env_logger = "0.6"
 hlua-badtouch = "0.4"
@@ -28,11 +29,13 @@ colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
 publicsuffix = { version="1.5", default-features=false }
-diesel = { version = "1.0.0", features = ["sqlite"] }
+diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
+chrono = { version = "0.4", features = ["serde"] }
 dirs = "1.0"
 url = "1.7"
-chrootable-https = "0.5.0"
+#chrootable-https = { path = "../chrootable-https" }
+chrootable-https = "0.7"
 base64 = "0.10"
 kuchiki = "0.7.2"
 serde_urlencoded = "0.5"
@@ -47,7 +50,7 @@ maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.2"
 regex = "1.0"
 toml = "0.4"
-maxminddb = "0.10.0"
+maxminddb = "0.12"
 tar = "0.4.17"
 libflate = "0.1.14"
 threadpool = "1.7"

Makefile

@@ -14,3 +14,8 @@ test:
 	(cd sn0int-registry; cargo test)
 	cargo test
 	cargo test -- --ignored
+
+update:
+	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt
+	get-iab -v -u http://standards-oui.ieee.org/iab/iab.txt -f data/ieee-iab.txt
+	rm -f data/ieee-*.txt.bak

README.md

@@ -7,11 +7,11 @@
 [docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
 [docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
 
-sn0int is an OSINT framework and package manager. It was built for IT security
-professionals and bug hunters to gather intelligence about a given target or
-about yourself. sn0int is enumerating attack surface by semi-automatically
-processing public information and mapping the results in a unified format for
-followup investigations.
+sn0int is a semi-automatic OSINT framework and package manager. It was built
+for IT security professionals and bug hunters to gather intelligence about a
+given target or about yourself. sn0int is enumerating attack surface by
+semi-automatically processing public information and mapping the results in a
+unified format for followup investigations.
 
 Among other things, sn0int is currently able to:
 
@@ -21,6 +21,8 @@ Among other things, sn0int is currently able to:
 - [X] Harvest emails from pgp keyservers
 - [X] Enrich ip addresses with ASN and geoip info
 - [X] Harvest subdomains from the wayback machine
+- [X] Gather information about phonenumbers
+- [X] Bruteforce interesting urls
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource.  None of the investigations listed above are hardcoded
@@ -30,17 +32,76 @@ them with other users by publishing them to the sn0int registry. This allows
 you to ship updates for your modules on your own since you don't need to send a
 pull request.
 
-Join us on IRC: <ircs://irc.hackint.org/#sn0int>
+Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
 
 [![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
 
-## Installation
+## Getting started
 
-- Archlinux: `pacman -S sn0int`
-- Alpine: `apk add --no-cache sqlite-dev libseccomp-dev cargo` + build from source
-- Debian: `apt install libsqlite3-dev libseccomp-dev` + build from source
-- OpenBSD: `pkg_add sqlite3` + build from source
-- OSX: `brew install sqlite3` + build from source
+- [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
+  - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
+  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
+  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
+  - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
+  - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
+  - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
+  - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
+  - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
+  - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
+- [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
+  - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+- [Database](https://sn0int.readthedocs.io/en/latest/database.html)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
+- [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
+  - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
+  - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
+  - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
+- [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
+  - [Configuring a proxy](https://sn0int.readthedocs.io/en/latest/config.html#configuring-a-proxy)
+- [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
+  - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
+  - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
+  - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
+  - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
+  - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
+  - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
+  - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
+  - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
+  - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
+  - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
+  - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
+  - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
+  - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
+  - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
+  - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
+  - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
+  - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
+  - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
+  - [url_join](https://sn0int.readthedocs.io/en/latest/reference.html#url-join)
+  - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
+  - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
+  - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
 
 ## License
 

contrib/docker/Dockerfile.alpine

@@ -0,0 +1,15 @@
+FROM alpine:edge
+RUN apk add --no-cache sqlite-dev libseccomp-dev
+RUN apk add --no-cache --virtual .build-rust rust cargo
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM alpine:edge
+RUN apk add --no-cache libgcc sqlite-libs libseccomp
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/docker/Dockerfile.debian

@@ -0,0 +1,16 @@
+FROM rust
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM debian
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/html-toc2md.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+perl -n -e '/toctree-l(\d).*href="([^"]+)">(.+)<\/a/ && print $1==2?"  ":"", "- [$3](https://sn0int.readthedocs.io/en/latest/$2)\n"' < docs/_build/html/index.html

docs/conf.py

@@ -138,7 +138,7 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('man', 'sn0int', 'OSINT framework and package manager',
+    ('man', 'sn0int', 'Semi-automatic OSINT framework and package manager',
      [author], 1)
 ]
 

docs/config.rst

@@ -0,0 +1,17 @@
+Configuration
+=============
+
+This file documents the config file at ``~/.config/sn0int.toml``. By default
+this file does not exist and a default configuration is used instead.
+
+Configuring a proxy
+-------------------
+
+To enable a proxy, add the following to your config file::
+
+    [network]
+    proxy = "127.0.0.1:9050"
+
+This forces everything through tor and restricts all other functions that
+depend on the network. For example the ``dns`` function is fully disabled if a
+proxy is configured.

docs/index.rst

@@ -1,11 +1,11 @@
 sn0int
 ======
 
-sn0int is an OSINT framework and package manager. It was built for IT security
-professionals and bug hunters to gather intelligence about a given target or
-about yourself. sn0int is enumerating attack surface by semi-automatically
-processing public information and mapping the results in a unified format for
-followup investigations.
+sn0int is a semi-automatic OSINT framework and package manager. It was built
+for IT security professionals and bug hunters to gather intelligence about a
+given target or about yourself. sn0int is enumerating attack surface by
+semi-automatically processing public information and mapping the results in a
+unified format for followup investigations.
 
 Among other things, sn0int is currently able to:
 
@@ -15,6 +15,8 @@ Among other things, sn0int is currently able to:
 - [X] Harvest emails from pgp keyservers
 - [X] Enrich ip addresses with ASN and geoip info
 - [X] Harvest subdomains from the wayback machine
+- [X] Gather information about phonenumbers
+- [X] Bruteforce interesting urls
 
 sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
 and is fully opensource.  None of the investigations listed above are hardcoded
@@ -24,7 +26,7 @@ them with other users by publishing them to the sn0int registry. This allows
 you to ship updates for your modules on your own since you don't need to send a
 pull request.
 
-Join us on IRC: ircs://irc.hackint.org/#sn0int
+Join us on IRC: `irc.hackint.org:6697/#sn0int <https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int>`_
 
 Getting Started
 ---------------
@@ -37,4 +39,6 @@ Getting Started
    usage
    scripting
    database
+   keyring
+   config
    reference

docs/install.rst

@@ -30,6 +30,13 @@ Alpine
     $ cd sn0int
     $ cargo install -f
 
+Docker
+------
+
+.. code-block:: bash
+
+    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
+
 OpenBSD
 -------
 

docs/keyring.rst

@@ -0,0 +1,73 @@
+Keyring
+=======
+
+A common problem is that you need either an api key or a username/password
+combination. Instead of hardcoding it in the script you should request them
+from the keyring. In order to do this you need to request permissions to those
+credentials.
+
+Managing the keyring
+--------------------
+
+The keyring is a simple namespaced key-value store::
+
+    [sn0int][default] > keyring add aws:AKIAIOSFODNN7EXAMPLE
+    Secretkey: keep-this-secret
+    [sn0int][default] > keyring list
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] >
+    [sn0int][default] > keyring list aws
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] > keyring list instagram
+    [sn0int][default] >
+    [sn0int][default] > keyring get aws:AKIAIOSFODNN7EXAMPLE
+    Namespace:    "aws"
+    Access Key:   "AKIAIOSFODNN7EXAMPLE"
+    Secret:       "keep-this-secret"
+    [sn0int][default] >
+
+If the service uses a username-password combination, set the username as the
+access key and the password as the secret.
+
+If the service uses only a secret key for the api, set the secret key as the
+access key and leave the secret blank.
+
+A script doesn't automatically get access to requested keyring namespaces.
+Instead the user is asked to confirm those requests to limit abusive scripts.
+
+Using access keys in scripts
+----------------------------
+
+We can request all keys of a certain namespace in our script metadata. This is
+going to prompt the user to grant the script access. This can be done for
+multiple namespaces in the same script:
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Keyring-Access: asdf
+
+If the user granted us access to those keys we can read them with ``keyring``:
+
+.. code-block:: lua
+
+    creds = keyring('aws')
+    print(creds[1]['accesskey'])
+    print(creds[1]['secretkey'])
+
+This returns a list of all keys in that namespace. Any empty list is returned
+if the user doesn't have any keys in that namespace.
+
+Using access keys as source argument
+------------------------------------
+
+We can also use the access keys as source argument. This is useful if each
+account has access to different things and we want to read through all of them.
+
+Since access key permissions are granted per namespace we need to specify which
+credentials we want to use.
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Source: keyring:aws

docs/man.rst

@@ -10,4 +10,5 @@ todo
     :glob:
 
     usage
+    config
     reference

docs/reference.rst

@@ -73,15 +73,20 @@ This function accepts the following options:
 
 .. code-block:: lua
 
-    x = dns('example.com', {
-        record='A'
+    records = dns('example.com', {
+        record='A',
     })
     if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
 
 .. note::
    DNS replies with an error code set are not causing a change to
    ``last_err()``. You have to test for this explicitly.
 
+.. note::
+   This function is unavailable if a socks5 proxy is configured.
+
 error
 -----
 
@@ -261,10 +266,23 @@ Encode a datastructure into a string.
     })
     print(x)
 
+keyring
+-------
+
+Request all keys from a given namespace. See the `keyring <keyring.html>`__
+section for details.
+
+.. code-block:: lua
+
+    creds = keyring('aws')
+    print(creds[1]['accesskey'])
+    print(creds[1]['secretkey'])
+
 last_err
 --------
 
-Returns infos about the last error we've observed, if any. Returns ``nil`` otherwise.
+Returns infos about the last error we've observed, if any. Returns ``nil``
+otherwise.
 
 .. code-block:: lua
 
@@ -416,6 +434,40 @@ Read a line from stdin. The final newline is not removed.
 .. note::
    This only works with `sn0int run --stdin`.
 
+url_decode
+----------
+
+Parse a query string into a map. For raw percent decoding see url_unescape_.
+
+.. code-block:: lua
+
+    v = url_decode('a=b&c=d')
+    print(v['a'] == 'b')
+    print(v['c'] == 'd')
+
+url_encode
+----------
+
+Encode a map into a query string. For raw percent encoding see url_escape_.
+
+.. code-block:: lua
+
+    v = url_encode({
+        a='b',
+        c='d',
+    })
+    print(v == 'a=b&c=d')
+
+url_escape
+----------
+
+Apply url escaping to a string.
+
+.. code-block:: lua
+
+    v = url_escape('foo bar?')
+    print(v == 'foo%20bar%3F')
+
 url_join
 --------
 
@@ -445,11 +497,21 @@ Parse a url into its components. The following components are returned:
 
 .. code-block:: lua
 
-    url = url_parse("https://example.com")
+    url = url_parse('https://example.com')
     print(url['scheme'] == 'https')
     print(url['host'] == 'example.com')
     print(url['path'] == '/')
 
+url_unescape
+------------
+
+Remove url escaping of a string.
+
+.. code-block:: lua
+
+    v = url_unescape('foo%20bar%3F')
+    print(v == 'foo bar?')
+
 utf8_decode
 -----------
 

migrations/2018-12-14-094011_phonenumbers/down.sql

@@ -0,0 +1 @@
+DROP TABLE phonenumbers;

migrations/2018-12-14-094011_phonenumbers/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE phonenumbers (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    last_online DATETIME,
+    country VARCHAR,
+    carrier VARCHAR,
+    line VARCHAR,
+    is_ported BOOLEAN,
+    last_ported DATETIME,
+    caller_name VARCHAR,
+    caller_type VARCHAR,
+    CONSTRAINT phonenumber_unique UNIQUE (value)
+);

migrations/2018-12-23-230955_reverse-dns/down.sql

@@ -0,0 +1,27 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ipaddrs_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    asn INTEGER,
+    as_org VARCHAR,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO _ipaddrs_new (id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org)
+  SELECT id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org
+  FROM ipaddrs;
+
+DROP TABLE ipaddrs;
+ALTER TABLE _ipaddrs_new RENAME TO ipaddrs;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-23-230955_reverse-dns/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ipaddrs ADD COLUMN description VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN reverse_dns VARCHAR;

migrations/2018-12-24-141533_networks/down.sql

@@ -0,0 +1,3 @@
+DROP TABLE network_devices;
+DROP TABLE networks;
+DROP TABLE devices;

migrations/2018-12-24-141533_networks/up.sql

@@ -0,0 +1,30 @@
+CREATE TABLE networks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+CREATE TABLE devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    hostname VARCHAR,
+    vendor VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    last_seen DATETIME,
+    CONSTRAINT device_unique UNIQUE (value)
+);
+
+CREATE TABLE network_devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    network_id INTEGER NOT NULL,
+    device_id INTEGER NOT NULL,
+    ipaddr VARCHAR,
+    last_seen DATETIME,
+    FOREIGN KEY(network_id) REFERENCES networks(id) ON DELETE CASCADE,
+    FOREIGN KEY(device_id) REFERENCES devices(id) ON DELETE CASCADE,
+    CONSTRAINT network_device_unique UNIQUE (network_id, device_id)
+);

modules/dev/arp-scan.lua

@@ -5,6 +5,16 @@
 -- sudo arp-scan -qglI wlp3s0
 
 function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
     while true do
         x = stdin_readline()
         if x == nil then
@@ -13,9 +23,21 @@ function run()
 
         m = regex_find('(.+)\t(.+)', x)
         if m ~= nil then
-            ip = m[2]
+            ipaddr = m[2]
             mac = m[3]
-            info(json_encode({ip, mac}))
+
+            device_id = db_add('device', {
+                value=mac,
+            })
+            if last_err() then return end
+
+            -- TODO: add last_seen
+            db_add('network-device', {
+                network_id=network_id,
+                device_id=device_id,
+                ipaddr=ipaddr,
+            })
+            if last_err() then return end
         end
     end
 end

modules/dev/axfr.lua

@@ -39,7 +39,7 @@ end
 function iter_axfr(zone, arg)
     local name, r, m, domain
 
-    debug(json_encode(arg))
+    debug(arg)
 
     name = arg[1]
     r = arg[2]

modules/dev/cname-harvest.lua

@@ -1,5 +1,5 @@
 -- Description: Query for CNAMES to find subdomains
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: subdomains
 -- License: GPL-3.0
 

modules/dev/ctlogs.lua

@@ -1,11 +1,54 @@
 -- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 
-function run(arg)
-    session = http_mksession()
+function each_name(name)
+    local domain_id, psl_domain
 
+    debug(name)
+
+    if name:find('*.') == 1 then
+        -- ignore wildcard domains
+        return
+    end
+
+    -- the cert might be valid for subdomains that do not belong to the
+    -- domain we started with
+    psl_domain = psl_domain_from_dns_name(name)
+    domain_id = domains[psl_domain]
+    if domain_id == nil then
+        if any_domain then
+            -- unknown domains should be added to database
+            domain_id = db_add('domain', {
+                value=psl_domain,
+            })
+        else
+            -- only use domains that are already in scope
+            domain_id = db_select('domain', psl_domain)
+        end
+
+        -- if we didn't get a valid id, skip
+        if domain_id == nil then
+            return
+        end
+
+        domains[psl_domain] = domain_id
+    end
+
+    db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+end
+
+function run(arg)
+    any_domain = getopt('any-domain') ~= nil
+
+    domains = {}
+    domains[arg['value']] = arg['id']
+
+    session = http_mksession()
     req = http_request(session, 'GET', 'https://crt.sh/', {
         query={
             q='%.' .. arg['value'],
@@ -20,27 +63,31 @@ function run(arg)
     certs = json_decode_stream(resp['text'])
     if last_err() then return end
 
-    seen = {}
-
     i = 1
     while i <= #certs do
         c = certs[i]
-        -- print(c)
+        debug(c)
 
-        name = c['name_value']
-        debug(json_encode(name))
+        -- fetch certificate
+        id = c['min_cert_id']
+        req = http_request(session, 'GET', 'https://crt.sh/', {
+            query={
+                d=id .. '', -- TODO: find nicer way for tostring
+            }
+        })
+        resp = http_send(req)
+        if last_err() then return end
+        if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
-        if name:find("*.") == 1 then
-            -- ignore wildcard domains
-            seen[name] = 1
-        end
+        -- iterate over all valid names
+        crt = x509_parse_pem(resp['text'])
+        if last_err() then return end
+        names = crt['valid_names']
 
-        if seen[name] == nil then
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=name,
-            })
-            seen[name] = 1
+        j = 1
+        while j <= #names do
+            each_name(names[j])
+            j = j+1
         end
 
         i = i+1

modules/dev/dns-ptr.lua

@@ -0,0 +1,32 @@
+-- Description: Run reverse dns lookups
+-- Version: 0.1.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    if arg['family'] == '4' then
+        m = regex_find('^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$', arg['value'])
+
+        q = m[5] .. '.' .. m[4] .. '.' .. m[3] .. '.' .. m[2] .. '.in-addr.arpa'
+        debug('Resolving: ' .. q)
+
+        records = dns(q, {
+            record='PTR',
+        })
+        if last_err() then return end
+        if records['error'] ~= nil then return end
+        records = records['answers']
+
+        i = 1
+        while records[i] ~= nil do
+            r = records[i][2]
+            if r['PTR'] then
+                db_update('ipaddr', arg, {
+                    reverse_dns=r['PTR'],
+                })
+                if last_err() then return end
+            end
+            i = i+1
+        end
+    end
+end

modules/dev/dns-resolve.lua

@@ -1,5 +1,5 @@
 -- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: subdomains
 -- License: GPL-3.0
 

modules/dev/git-webroot.lua

@@ -0,0 +1,28 @@
+-- Description: Search for git checkouts in webroot
+-- Version: 0.1.0
+-- Source: urls
+-- License: GPL-3.0
+
+function run(arg)
+    url = url_join(arg['value'], '.git/HEAD')
+
+    session = http_mksession()
+    req = http_request(session, 'GET', url, {})
+    reply = http_send(req)
+    if last_err() then return end
+
+    if reply['status'] ~= 200 then
+        return
+    end
+
+    if not regex_find('^ref: ', reply['text']) then
+        return
+    end
+
+    db_add('url', {
+        subdomain_id=arg['subdomain_id'],
+        value=url,
+        status=reply['status'],
+        body=reply['text'],
+    })
+end

modules/dev/isc-dhcpd-leases.lua

@@ -0,0 +1,75 @@
+-- Description: Parse isc-dhcpd dhcpd.leases(5)
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- cat /var/lib/dhcpd/dhcpd.leases
+
+function add(lease)
+    if not lease['active'] then return end
+
+    device_id = db_add('device', {
+        value=lease['mac'],
+        hostname=lease['hostname'],
+    })
+    if last_err() then return end
+
+    -- TODO: add last_seen
+    db_add('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+        ipaddr=lease['ipaddr'],
+    })
+    if last_err() then return end
+end
+
+function each_line(x)
+    debug(x)
+    m = regex_find('^lease (\\S+) \\{\n$', x)
+    if m then
+        lease = {}
+        debug('ipaddr=' .. m[2])
+        lease['ipaddr'] = m[2]
+    end
+    m = regex_find('^\\s*hardware ethernet (\\S+);\n$', x)
+    if m then
+        debug('mac=' .. m[2])
+        lease['mac'] = m[2]
+    end
+    m = regex_find('^\\s*client-hostname \"(.+)\";\n$', x)
+    if m then
+        debug('hostname=' .. m[2])
+        lease['hostname'] = m[2]
+    end
+    m = regex_find('^\\s*binding state active;\n$', x)
+    if m then
+        debug('active=true')
+        lease['active'] = true
+    end
+    m = regex_find('^\\}\n$', x)
+    if m then
+        add(lease)
+    end
+end
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        if not regex_find('^\\s*(#.*|\\s*)\n$', x) then
+            each_line(x)
+        end
+    end
+end

modules/dev/iw-station-dump.lua

@@ -0,0 +1,74 @@
+-- Description: Parse iw station dump
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- iw dev wlan0 station dump
+
+function add(client)
+    if
+        client['authenticated'] == 'yes' and
+        client['authorized'] == 'yes' and
+        client['mac']
+    then
+        debug(client)
+
+        device_id = db_add('device', {
+            value=client['mac'],
+        })
+        if last_err() then return end
+
+        -- TODO: add last_seen
+        db_add('network-device', {
+            network_id=network_id,
+            device_id=device_id,
+        })
+        if last_err() then return end
+    end
+
+    client = nil
+end
+
+function each_line(x)
+    debug(x)
+    m = regex_find('^Station (\\S+)', x)
+    if m then
+        if client then
+            add(client)
+        end
+        client = {}
+        client['mac'] = m[2]
+        debug('mac=' .. m[2])
+    end
+
+    m = regex_find('^\\s+([^:]+):\\s*(.+)\n$', x)
+    if m and client then
+        client[m[2]] = m[3]
+        debug(m[2] .. '=' .. m[3])
+    end
+end
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    client = nil
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        each_line(x)
+    end
+
+    if client then
+        add(client)
+    end
+end

modules/dev/otx-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 

modules/dev/phpmyadmin.lua

@@ -0,0 +1,106 @@
+-- Description: Search for phpmyadmin
+-- Version: 0.1.0
+-- Source: urls
+-- License: GPL-3.0
+
+function run(arg)
+    paths = {
+        "phpmyadmin/index.php",
+        "phpMyAdmin/index.php",
+        "pmd/index.php",
+        "pma/index.php",
+        "PMA/index.php",
+        "PMA2/index.php",
+        "pmamy/index.php",
+        "pmamy2/index.php",
+        "mysql/index.php",
+        "admin/index.php",
+        "db/index.php",
+        "dbadmin/index.php",
+        "web/phpMyAdmin/index.php",
+        "admin/pma/index.php",
+        "admin/PMA/index.php",
+        "admin/mysql/index.php",
+        "admin/mysql2/index.php",
+        "admin/phpmyadmin/index.php",
+        "admin/phpMyAdmin/index.php",
+        "admin/phpmyadmin2/index.php",
+        "mysqladmin/index.php",
+        "mysql-admin/index.php",
+        "mysql_admin/index.php",
+        "phpadmin/index.php",
+        "phpAdmin/index.php",
+        "phpmyadmin0/index.php",
+        "phpmyadmin1/index.php",
+        "phpmyadmin2/index.php",
+        "phpMyAdmin-4.4.0/index.php",
+        "myadmin/index.php",
+        "myadmin2/index.php",
+        "xampp/phpmyadmin/index.php",
+        "phpMyadmin_bak/index.php",
+        "www/phpMyAdmin/index.php",
+        "tools/phpMyAdmin/index.php",
+        "phpmyadmin-old/index.php",
+        "phpMyAdminold/index.php",
+        "phpMyAdmin.old/index.php",
+        "pma-old/index.php",
+        "claroline/phpMyAdmin/index.php",
+        "typo3/phpmyadmin/index.php",
+        "phpma/index.php",
+        "phpmyadmin/phpmyadmin/index.php",
+        "phpMyAdmin/phpMyAdmin/index.php",
+        "phpMyAbmin/index.php",
+        "phpMyAdmin__/index.php",
+        "phpMyAdmin+++---/index.php",
+        "v/index.php",
+        "phpmyadm1n/index.php",
+        "phpMyAdm1n/index.php",
+        "shaAdmin/index.php",
+        "phpMyadmi/index.php",
+        "phpMyAdmion/index.php",
+        "MyAdmin/index.php",
+        "phpMyAdmin1/index.php",
+        "phpMyAdmin123/index.php",
+        "pwd/index.php",
+        "phpMyAdmina/index.php",
+        "program/index.php",
+        "shopdb/index.php",
+        "phppma/index.php",
+        "phpmy/index.php",
+        "mysql/admin/index.php",
+        "mysql/dbadmin/index.php",
+        "mysql/sqlmanager/index.php",
+        "mysql/mysqlmanager/index.php",
+        "wp-content/plugins/portable-phpmyadmin/wp-pma-mod/index.php",
+    }
+
+    session = http_mksession()
+
+    i = 1
+    while i <= #paths do
+        p = paths[i]
+        url = url_join(arg['value'], p)
+        debug(url)
+
+        req = http_request(session, 'GET', url, {
+            timeout=5000
+        })
+        reply = http_send(req)
+        debug(reply)
+
+        if last_err() then
+            clear_err()
+        else
+            if reply['status'] == 200 then
+                db_add('url', {
+                    subdomain_id=arg['subdomain_id'],
+                    value=url,
+                    status=reply['status'],
+                    body=reply['text'],
+                })
+            end
+        end
+
+        i = i+1
+    end
+end

modules/dev/threatminer-ipaddr.lua

@@ -1,5 +1,5 @@
 -- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: ipaddrs
 -- License: GPL-3.0
 

modules/dev/threatminer-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 

modules/dev/twilio-lookup.lua

@@ -0,0 +1,48 @@
+-- Description: Retrieve additional information about a phone number
+-- Version: 0.1.0
+-- Source: phonenumbers
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    number = url_escape(arg['value'])
+    --url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number
+    url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number .. '?Type=carrier&Type=caller-name'
+
+    --debug(url)
+
+    key = keyring('twilio')[1]
+    if not key then
+        return 'Missing required twilio access key'
+    end
+
+    session = http_mksession()
+    req = http_request(session, 'GET', url, {
+        basic_auth={key['access_key'], key['secret_key']},
+    })
+    reply = http_send(req)
+    if last_err() then return end
+
+    if reply['status'] ~= 200 then
+        return 'api returned error'
+    end
+
+    v = json_decode(reply['text'])
+    if last_err() then return end
+    debug(v)
+
+    update = {}
+    update['country'] = v['country_code']
+
+    if v['carrier'] then
+        update['carrier'] = v['carrier']['name']
+        update['line'] = v['carrier']['type']
+    end
+
+    if v['caller_name'] then
+        update['caller_name'] = v['caller_name']['caller_name']
+        update['caller_type'] = v['caller_name']['caller_type']
+    end
+
+    db_update('phonenumber', arg, update)
+end

modules/dev/url-scan.lua

@@ -29,9 +29,9 @@ function request(subdomain_id, url)
 
     db_add('url', obj)
 
-    -- info(json_encode(reply['status']))
-    -- info(json_encode(reply['headers']['location']))
-    -- info(json_encode(reply['text']))
+    -- debug(reply['status'])
+    -- debug(reply['headers']['location'])
+    -- debug(reply['text'])
 end
 
 function run(arg)

modules/dev/waybackurls.lua

@@ -1,5 +1,5 @@
 -- Description: Discover subdomains from wayback machine
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -32,7 +32,7 @@ function run(arg)
     i = 2
     while o[i] do
         url = o[i][3]
-        debug(json_encode(url))
+        debug(url)
         parts = url_parse(url)
 
         if last_err() then

modules/harness/fail.lua

@@ -1,6 +1,5 @@
 -- Description: Test error handling
 -- Version: 0.1.0
--- Source: domains
 -- License: GPL-3.0
 
 function run()

modules/harness/ip.lua

@@ -0,0 +1,15 @@
+-- Description: Show your ip
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function get(url)
+    req = http_request(session, 'GET', url, {})
+    r = http_send(req)
+    info(r['text'])
+end
+
+function run()
+    session = http_mksession()
+    get('https://icanhazip.com')
+    get('https://icanhazptr.com')
+end

modules/harness/keyring.lua

@@ -0,0 +1,9 @@
+-- Description: Request access to keyring
+-- Version: 0.1.0
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    keys = keyring('twilio')
+    debug(keys)
+end

modules/harness/keyring2.lua

@@ -0,0 +1,9 @@
+-- Description: Request access to keyring
+-- Version: 0.1.0
+-- Source: keyring:twilio
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    info(arg)
+end

modules/harness/options.lua

@@ -0,0 +1,7 @@
+-- Description: Read an option
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info(getopt('hello'))
+end

modules/harness/selftest.lua

@@ -0,0 +1,7 @@
+-- Description: basic selftest
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- nothing to do here
+end

modules/harness/stdin.lua

@@ -8,6 +8,6 @@ function run()
         if x == nil then
             break
         end
-        info(json_encode(x))
+        info(x)
     end
 end

sn0int-registry/Cargo.toml

@@ -1,16 +1,17 @@
 [package]
 name = "sn0int-registry"
-version = "0.4.0"
+version = "0.5.0"
 description = "sn0int registry"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
+edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.3.0", path="sn0int-common" }
-rocket = "0.3.16"
-rocket_codegen = "0.3.16"
-rocket_contrib = { version = "0.3.16", features = ["handlebars_templates"] }
+sn0int-common = { version="0.4.0", path="sn0int-common" }
+rocket = "0.4"
+rocket_failure = { version = "0.1", features = ["with-rocket"] }
+rocket_contrib = { version = "0.4", features = ["handlebars_templates"] }
 
 diesel = { version = "1.3", features = ["postgres", "r2d2"] }
 diesel_migrations = { version = "1.3.0", features = ["postgres"] }

sn0int-registry/sn0int-common/Cargo.toml

@@ -1,13 +1,16 @@
 [package]
 name = "sn0int-common"
-version = "0.3.0"
+version = "0.4.0"
 description = "Common code for sn0int"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
+edition = "2018"
 
 [dependencies]
 serde = "1.0"
 serde_derive = "1.0"
+#rocket_failure = { path = "../../../rocket_failure" }
+rocket_failure = "0.1.1"
 failure = "0.1"
 nom = "4.0"

sn0int-registry/sn0int-common/src/api.rs

@@ -1,23 +1,3 @@
-use errors::*;
-
-
-#[derive(Debug, Serialize, Deserialize)]
-pub enum ApiResponse<T> {
-    #[serde(rename="success")]
-    Success(T),
-    #[serde(rename="error")]
-    Error(String),
-}
-
-impl<T> ApiResponse<T> {
-    pub fn success(self) -> Result<T> {
-        match self {
-            ApiResponse::Success(x) => Ok(x),
-            ApiResponse::Error(err) => bail!("Api returned error: {:?}", err),
-        }
-    }
-}
-
 #[derive(Debug, Serialize, Deserialize)]
 pub struct WhoamiResponse {
     pub user: String,

sn0int-registry/sn0int-common/src/id.rs

@@ -1,7 +1,9 @@
-use errors::*;
+use crate::errors::*;
 use nom;
 use nom::types::CompleteStr;
+use serde::{de, Serialize, Serializer, Deserialize, Deserializer};
 use std::fmt;
+use std::result;
 use std::str::FromStr;
 
 
@@ -32,7 +34,7 @@ named!(module<CompleteStr, ModuleID>, do_parse!(
 
 named!(token<CompleteStr, CompleteStr>, take_while1!(valid_char));
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Eq, Hash)]
 pub struct ModuleID {
     pub author: String,
     pub name: String,
@@ -54,6 +56,24 @@ impl FromStr for ModuleID {
     }
 }
 
+impl Serialize for ModuleID {
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for ModuleID {
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+        where D: Deserializer<'de>
+    {
+        let s = String::deserialize(deserializer)?;
+        FromStr::from_str(&s).map_err(de::Error::custom)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

sn0int-registry/sn0int-common/src/lib.rs

@@ -4,10 +4,12 @@
 
 pub mod api;
 pub mod errors;
-pub use errors::*;
+pub use crate::errors::*;
 pub mod metadata;
 pub mod id;
-pub use id::*;
+pub use crate::id::*;
+
+pub use rocket_failure::StrictApiResponse as ApiResponse;
 
 #[cfg(test)]
 mod tests {

sn0int-registry/sn0int-common/src/metadata.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 
 use std::str::FromStr;
 
@@ -8,6 +8,7 @@ pub enum EntryType {
     Description,
     Version,
     Source,
+    KeyringAccess,
     License,
 }
 
@@ -19,6 +20,7 @@ impl FromStr for EntryType {
             "Description" => Ok(EntryType::Description),
             "Version" => Ok(EntryType::Version),
             "Source" => Ok(EntryType::Source),
+            "Keyring-Access" => Ok(EntryType::KeyringAccess),
             "License" => Ok(EntryType::License),
             x => bail!("Unknown EntryType: {:?}", x),
         }
@@ -32,19 +34,31 @@ pub enum Source {
     IpAddrs,
     Urls,
     Emails,
+    PhoneNumbers,
+    KeyRing(String),
 }
 
 impl FromStr for Source {
     type Err = Error;
 
     fn from_str(s: &str) -> Result<Source> {
-        match s {
-            "domains" => Ok(Source::Domains),
-            "subdomains" => Ok(Source::Subdomains),
-            "ipaddrs" => Ok(Source::IpAddrs),
-            "urls" => Ok(Source::Urls),
-            "emails" => Ok(Source::Emails),
-            x => bail!("Unknown Source: {:?}", x),
+        let (key, param) = if let Some(idx) = s.find(':') {
+            let (a, b) = s.split_at(idx);
+            (a, Some(&b[1..]))
+        } else {
+            (s, None)
+        };
+
+        match (key, param) {
+            ("domains", None) => Ok(Source::Domains),
+            ("subdomains", None) => Ok(Source::Subdomains),
+            ("ipaddrs", None) => Ok(Source::IpAddrs),
+            ("urls", None) => Ok(Source::Urls),
+            ("emails", None) => Ok(Source::Emails),
+            ("phonenumbers", None) => Ok(Source::PhoneNumbers),
+            ("keyring", Some(param)) => Ok(Source::KeyRing(param.to_string())),
+            (x, Some(param)) => bail!("Unknown Source: {:?} ({:?})", x, param),
+            (x, None) => bail!("Unknown Source: {:?}", x),
         }
     }
 }
@@ -80,6 +94,7 @@ pub struct Metadata {
     pub description: String,
     pub version: String,
     pub source: Option<Source>,
+    pub keyring_access: Vec<String>,
     pub license: License,
 }
 
@@ -97,6 +112,7 @@ impl FromStr for Metadata {
                 EntryType::Description => data.description = Some(v),
                 EntryType::Version => data.version = Some(v),
                 EntryType::Source => data.source = Some(v),
+                EntryType::KeyringAccess => data.keyring_access.push(v),
                 EntryType::License => data.license = Some(v),
             }
         }
@@ -110,6 +126,7 @@ pub struct NewMetadata<'a> {
     pub description: Option<&'a str>,
     pub version: Option<&'a str>,
     pub source: Option<&'a str>,
+    pub keyring_access: Vec<&'a str>,
     pub license: Option<&'a str>,
 }
 
@@ -121,6 +138,9 @@ impl<'a> NewMetadata<'a> {
             Some(x) => Some(x.parse()?),
             _ => None,
         };
+        let keyring_access = self.keyring_access.into_iter()
+            .map(String::from)
+            .collect();
         let license = self.license.ok_or_else(|| format_err!("License is required"))?;
         let license = license.parse()?;
 
@@ -128,6 +148,7 @@ impl<'a> NewMetadata<'a> {
             description: description.to_string(),
             version: version.to_string(),
             source,
+            keyring_access,
             license,
         })
     }
@@ -168,6 +189,7 @@ mod tests {
             version: "1.0.0".to_string(),
             license: License::WTFPL,
             source: Some(Source::Domains),
+            keyring_access: Vec::new(),
         });
     }
 
@@ -183,6 +205,7 @@ mod tests {
             version: "1.0.0".to_string(),
             license: License::WTFPL,
             source: None,
+            keyring_access: Vec::new(),
         });
     }
 
@@ -206,4 +229,19 @@ mod tests {
 "#);
         assert!(metadata.is_err());
     }
+
+    #[test]
+    fn verify_keyring_source() {
+        let x = Source::from_str("keyring:foo").unwrap();
+        assert_eq!(x, Source::KeyRing("foo".to_string()));
+
+        let x = Source::from_str("keyring:").unwrap();
+        assert_eq!(x, Source::KeyRing("".to_string()));
+    }
+
+    #[test]
+    fn verify_invalid_keyring_source() {
+        let x = Source::from_str("keyring");
+        assert!(x.is_err());
+    }
 }

sn0int-registry/src/assets.rs

@@ -2,12 +2,12 @@ use blake2::{Blake2b, Digest};
 
 
 pub static FAVICON: &[u8] = include_bytes!("../assets/favicon.ico");
-pub static STYLE_SHEET: &str = include_str!("../assets/style.css");
+pub static STYLE_SHEET: &[u8] = include_bytes!("../assets/style.css");
 
 lazy_static! {
     pub static ref ASSET_REV: String = {
         let mut h = Blake2b::new();
-        h.input(STYLE_SHEET.as_bytes());
+        h.input(STYLE_SHEET);
         hex::encode(&h.result()[0..8])
     };
 }

sn0int-registry/src/auth.rs

@@ -1,10 +1,10 @@
-use errors::*;
+use crate::errors::*;
 use diesel::pg::PgConnection;
 use oauth2::basic::BasicClient;
 use oauth2::prelude::*;
 use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl};
-use github::GithubAuthenticator;
-use models::AuthToken;
+use crate::github::GithubAuthenticator;
+use crate::models::AuthToken;
 use url::Url;
 use std::env;
 

sn0int-registry/src/auth2.rs

@@ -1,10 +1,10 @@
-use errors::*;
-use models::AuthToken;
-use db::Connection;
+use crate::errors::*;
+use crate::models::AuthToken;
+use crate::db::Connection;
 use rocket::http::Status;
 use rocket::{Request, Outcome};
 use rocket::request::{self, FromRequest};
-use github::GithubAuthenticator;
+use crate::github::GithubAuthenticator;
 
 
 pub struct AuthHeader(String);

sn0int-registry/src/db.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 use std::io;
 use std::thread;
 use std::time::Duration;

sn0int-registry/src/errors.rs

@@ -1,30 +1,4 @@
 pub use failure::{Error, ResultExt};
 pub type Result<T> = ::std::result::Result<T, Error>;
 
-use rocket::Request;
-use rocket::http::Status;
-use rocket::response::{self, Responder};
-
-#[derive(Debug)]
-pub struct ApiError(Error);
-
-#[derive(Serialize)]
-pub struct ErrorResponse {
-    pub status: &'static str,
-    pub message: String,
-}
-
-pub type ApiResult<T> = ::std::result::Result<T, ApiError>;
-
-impl<'r> Responder<'r> for ApiError {
-    fn respond_to(self, _: &Request) -> response::Result<'static> {
-        error!("Error: {:?}", self.0);
-        Err(Status::InternalServerError)
-    }
-}
-
-impl From<Error> for ApiError {
-    fn from(error: Error) -> ApiError {
-        ApiError(error)
-    }
-}
+pub use rocket_failure::errors::*;

sn0int-registry/src/github.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 use std::env;
 use reqwest;
 

sn0int-registry/src/main.rs

@@ -1,15 +1,10 @@
 #![allow(proc_macro_derive_resolution_fallback)]
 #![warn(unused_extern_crates)]
-#![feature(plugin)]
-#![feature(custom_derive)]
-#![plugin(rocket_codegen)]
+#![feature(proc_macro_hygiene, decl_macro)]
 
-extern crate sn0int_common;
-extern crate rocket;
+#[macro_use] extern crate rocket;
 #[macro_use] extern crate rocket_contrib;
-extern crate dotenv;
-extern crate blake2;
-extern crate serde_json;
+#[macro_use] extern crate rocket_failure;
 #[macro_use] extern crate serde_derive;
 #[macro_use] extern crate log;
 #[macro_use] extern crate maplit;
@@ -17,17 +12,15 @@ extern crate serde_json;
 #[macro_use] extern crate failure;
 #[macro_use] extern crate diesel;
 #[macro_use] extern crate diesel_migrations;
-extern crate diesel_full_text_search;
-extern crate oauth2;
-extern crate url;
-extern crate reqwest;
-extern crate semver;
 
-use rocket_contrib::{Json, Value, Template};
+use rocket::fairing::AdHoc;
+use rocket::http::Header;
+use rocket_contrib::json::{Json, JsonValue};
+use rocket_contrib::templates::Template;
 use dotenv::dotenv;
 
 use std::env;
-use errors::*;
+use crate::errors::*;
 
 pub mod assets;
 pub mod auth;
@@ -42,21 +35,21 @@ pub mod schema;
 
 
 #[catch(400)]
-fn bad_request() -> Json<Value> {
+fn bad_request() -> Json<JsonValue> {
     Json(json!({
         "error": "Bad request"
     }))
 }
 
 #[catch(404)]
-fn not_found() -> Json<Value> {
+fn not_found() -> Json<JsonValue> {
     Json(json!({
         "error": "Resource was not found"
     }))
 }
 
 #[catch(500)]
-fn internal_error() -> Json<Value> {
+fn internal_error() -> Json<JsonValue> {
     Json(json!({
         "error": "Internal server error"
     }))
@@ -74,6 +67,15 @@ fn run() -> Result<()> {
     rocket::ignite()
         .manage(db::init(&database_url))
         .attach(Template::fairing())
+        .attach(AdHoc::on_response("Security Headers", |_, resp| {
+            resp.set_header(Header::new("Strict-Transport-Security", "max-age=63072000; includeSubDomains; preload"));
+            resp.set_header(Header::new("Content-Security-Policy", "style-src 'self'"));
+            resp.set_header(Header::new("Feature-Policy", "geolocation 'none'; midi 'none'; notifications 'none'; push 'none'; sync-xhr 'none'; microphone 'none'; camera 'none'; magnetometer 'none'; gyroscope 'none'; speaker 'none'; vibrate 'none'; fullscreen 'none'; payment 'none'"));
+            resp.set_header(Header::new("X-Frame-Options", "deny"));
+            resp.set_header(Header::new("X-XSS-Protection", "1; mode=block"));
+            resp.set_header(Header::new("X-Content-Type-Options", "nosniff"));
+            resp.set_header(Header::new("Referrer-Policy", "same-origin"));
+        }))
         .mount("/api/v0", routes![
             routes::api::quickstart,
             routes::api::search,
@@ -92,12 +94,12 @@ fn run() -> Result<()> {
             routes::assets::favicon,
             routes::assets::style,
         ])
-    .catch(catchers![
-        bad_request,
-        not_found,
-        internal_error,
-    ])
-    .launch();
+        .register(catchers![
+            bad_request,
+            not_found,
+            internal_error,
+        ])
+        .launch();
 
     Ok(())
 }

sn0int-registry/src/models.rs

@@ -1,9 +1,9 @@
-use errors::*;
+use crate::errors::*;
 use diesel::prelude::*;
 use diesel::pg::PgConnection;
 use diesel::sql_types::BigInt;
 use diesel_full_text_search::{plainto_tsquery, TsQueryExtensions};
-use schema::*;
+use crate::schema::*;
 
 
 #[derive(AsChangeset, Serialize, Deserialize, Queryable, Insertable)]

sn0int-registry/src/routes/api.rs

@@ -1,19 +1,20 @@
-use errors::*;
-use auth2::AuthHeader;
-use db;
+use crate::errors::*;
+use crate::auth2::AuthHeader;
+use crate::db;
+use crate::models::*;
 use diesel::Connection;
+use rocket::request::Form;
+use rocket_contrib::json::Json;
 use semver::Version;
 use sn0int_common::api::*;
 use sn0int_common::id;
 use sn0int_common::metadata::Metadata;
-use rocket_contrib::Json;
-use models::*;
 
 
 #[get("/quickstart")]
-fn quickstart(connection: db::Connection) -> ApiResult<Json<ApiResponse<Vec<Module>>>> {
+pub fn quickstart(connection: db::Connection) -> ApiResult<ApiResponse<Vec<Module>>> {
     let modules = Module::quickstart(&connection)?;
-    Ok(Json(ApiResponse::Success(modules)))
+    Ok(ApiResponse::Success(modules))
 }
 
 #[derive(Debug, FromForm)]
@@ -21,8 +22,8 @@ pub struct Search {
     q: String,
 }
 
-#[get("/search?<q>")]
-fn search(q: Search, connection: db::Connection) -> ApiResult<Json<ApiResponse<Vec<SearchResponse>>>> {
+#[get("/search?<q..>")]
+pub fn search(q: Form<Search>, connection: db::Connection) -> ApiResult<ApiResponse<Vec<SearchResponse>>> {
     info!("Searching: {:?}", q.q);
 
     let modules = Module::search(&q.q, &connection)?;
@@ -39,85 +40,99 @@ fn search(q: Search, connection: db::Connection) -> ApiResult<Json<ApiResponse<V
         })
         .collect();
 
-    Ok(Json(ApiResponse::Success(modules)))
+    Ok(ApiResponse::Success(modules))
 }
 
 #[get("/info/<author>/<name>", format="application/json")]
-fn info(author: String, name: String, connection: db::Connection) -> ApiResult<Json<ApiResponse<ModuleInfoResponse>>> {
+pub fn info(author: String, name: String, connection: db::Connection) -> ApiResult<ApiResponse<ModuleInfoResponse>> {
     info!("Querying {:?}/{:?}", author, name);
-    let module = Module::find(&author, &name, &connection)?;
+    let module = Module::find(&author, &name, &connection)
+        .not_found()
+        .public_context("Module does not exist")?;
 
-    Ok(Json(ApiResponse::Success(ModuleInfoResponse {
+    Ok(ApiResponse::Success(ModuleInfoResponse {
         author: module.author,
         name: module.name,
         description: module.description,
         latest: module.latest,
-    })))
+    }))
 }
 
 #[get("/dl/<author>/<name>/<version>", format="application/json")]
-fn download(author: String, name: String, version: String, connection: db::Connection) -> ApiResult<Json<ApiResponse<DownloadResponse>>> {
+pub fn download(author: String, name: String, version: String, connection: db::Connection) -> ApiResult<ApiResponse<DownloadResponse>> {
     info!("Downloading {:?}/{:?} ({:?})", author, name, version);
-    let module = Module::find(&author, &name, &connection)?;
+    let module = Module::find(&author, &name, &connection)
+        .not_found()
+        .public_context("Module does not exist")?;
     debug!("Module: {:?}", module);
-    let release = Release::find(module.id, &version, &connection)?;
+    let release = Release::find(module.id, &version, &connection)
+        .not_found()
+        .public_context("Release does not exist")?;
     debug!("Release: {:?}", release);
 
     release.bump_downloads(&connection)?;
 
-    Ok(Json(ApiResponse::Success(DownloadResponse {
+    Ok(ApiResponse::Success(DownloadResponse {
         author,
         name,
         version,
         code: release.code,
-    })))
+    }))
 }
 
 #[post("/publish/<name>", format="application/json", data="<upload>")]
-fn publish(name: String, upload: Json<PublishRequest>, session: AuthHeader, connection: db::Connection) -> ApiResult<Json<ApiResponse<PublishResponse>>> {
-    let user = session.verify(&connection)?;
+pub fn publish(name: String, upload: Json<PublishRequest>, session: AuthHeader, connection: db::Connection) -> ApiResult<ApiResponse<PublishResponse>> {
+    let user = session.verify(&connection)
+        .bad_request()
+        .public_context("Invalid auth token")?;
 
     id::valid_name(&user)
-        .context("Username is invalid")
-        .map_err(Error::from)?;
+        .bad_request()
+        .public_context("Username is invalid")?;
     id::valid_name(&name)
-        .context("Module name is invalid")
-        .map_err(Error::from)?;
+        .bad_request()
+        .public_context("Module name is invalid")?;
 
-    let metadata = upload.code.parse::<Metadata>()?;
+    let metadata = upload.code.parse::<Metadata>()
+        .bad_request()
+        .public_context("Failed to parse module metadata")?;
 
     let version = metadata.version.clone();
     Version::parse(&version)
-        .context("Version is invalid")
-        .map_err(Error::from)?;
+        .bad_request()
+        .public_context("Version is invalid")?;
 
-    connection.transaction::<_, Error, _>(|| {
-        let module = Module::update_or_create(&user, &name, &metadata.description, &connection)?;
+    connection.transaction::<_, WebError, _>(|| {
+        let module = Module::update_or_create(&user, &name, &metadata.description, &connection)
+            .private_context("Failed to write module metadata")?;
 
         match Release::try_find(module.id, &version, &connection)? {
             Some(release) => {
                 // if the code is identical, pretend we published the version
                 if release.code != upload.code {
-                    bail!("Version number already in use")
+                    bad_request!("Version number already in use")
                 }
             },
-            None => module.add_version(&version, &upload.code, &connection)?,
+            None => module.add_version(&version, &upload.code, &connection)
+                .private_context("Failed to add release")?,
         }
 
         Ok(())
     })?;
 
-    Ok(Json(ApiResponse::Success(PublishResponse {
+    Ok(ApiResponse::Success(PublishResponse {
         author: user,
         name,
         version,
-    })))
+    }))
 }
 
 #[get("/whoami")]
-fn whoami(session: AuthHeader, connection: db::Connection) -> ApiResult<Json<ApiResponse<WhoamiResponse>>> {
-    let user = session.verify(&connection)?;
-    Ok(Json(ApiResponse::Success(WhoamiResponse {
+pub fn whoami(session: AuthHeader, connection: db::Connection) -> ApiResult<ApiResponse<WhoamiResponse>> {
+    let user = session.verify(&connection)
+        .bad_request()
+        .public_context("Invalid auth token")?;
+    Ok(ApiResponse::Success(WhoamiResponse {
         user,
-    })))
+    }))
 }

sn0int-registry/src/routes/assets.rs

@@ -1,25 +1,60 @@
+use crate::assets::{ASSET_REV, FAVICON, STYLE_SHEET};
+use rocket::http::ContentType;
 use rocket::http::Status;
-use rocket::response::content;
-use rocket_contrib::Template;
-use assets::{ASSET_REV, FAVICON, STYLE_SHEET};
+use rocket::http::hyper::header::{CacheControl, CacheDirective};
+use rocket_contrib::templates::Template;
 
 
 #[get("/")]
-fn index() -> Template {
+pub fn index() -> Template {
     Template::render("index", hashmap!{
         "ASSET_REV" => ASSET_REV.as_str(),
     })
 }
 
+#[derive(Responder)]
+pub struct CachableResponder {
+    inner: Vec<u8>,
+    content_type: ContentType,
+    cache: CacheControl,
+}
+
+impl CachableResponder {
+    pub fn new<I: Into<Vec<u8>>>(inner: I, content_type: ContentType, max_age: u32) -> CachableResponder {
+        let cache = CacheControl(vec![
+            CacheDirective::Public,
+            CacheDirective::MaxAge(max_age),
+        ]);
+        CachableResponder {
+            inner: inner.into(),
+            content_type,
+            cache,
+        }
+    }
+
+    pub fn immutable<I: Into<Vec<u8>>>(inner: I, content_type: ContentType) -> CachableResponder {
+        let cache = CacheControl(vec![
+            CacheDirective::Public,
+            CacheDirective::MaxAge(31536000),
+            CacheDirective::Extension("immutable".into(), None),
+        ]);
+        CachableResponder {
+            inner: inner.into(),
+            content_type,
+            cache,
+        }
+    }
+}
+
 #[get("/favicon.ico")]
-fn favicon() -> Vec<u8> {
-    FAVICON.to_vec()
+pub fn favicon() -> CachableResponder {
+    CachableResponder::new(FAVICON, ContentType::Binary, 3600)
 }
 
 #[get("/assets/<rev>/style.css")]
-fn style(rev: String) -> Result<content::Css<&'static str>, Status> {
+pub fn style(rev: String) -> Result<CachableResponder, Status> {
     if rev == *ASSET_REV {
-        Ok(content::Css(STYLE_SHEET))
+        Ok(CachableResponder::immutable(STYLE_SHEET, ContentType::CSS))
     } else {
         Err(Status::NotFound)
     }

sn0int-registry/src/routes/auth.rs

@@ -1,15 +1,16 @@
-use errors::*;
-use auth::Authenticator;
-use db;
+use crate::errors::*;
+use crate::auth::Authenticator;
+use crate::db;
 use rocket::request::Form;
 use rocket::response::Redirect;
-use rocket_contrib::Template;
-use assets::ASSET_REV;
+use rocket_contrib::templates::Template;
+use crate::assets::ASSET_REV;
 use serde_json::{self, Value};
 
 
-#[get("/?<auth>")]
-pub fn get(auth: OAuth) -> Template {
+#[get("/?<auth..>")]
+pub fn get(auth: Form<OAuth>) -> Template {
+    let auth = auth.into_inner();
     let mut auth = serde_json::to_value(&auth).expect("OAuth serialization failed");
     if let Value::Object(ref mut map) = auth {
         map.insert("ASSET_REV".to_string(), Value::String(ASSET_REV.to_string()));
@@ -21,7 +22,9 @@ pub fn get(auth: OAuth) -> Template {
 pub fn post(auth: Form<OAuth>, connection: db::Connection) -> ApiResult<Template> {
     let (code, state) = auth.into_inner().extract()?;
     let client = Authenticator::from_env()?;
-    client.store_code(code, state, &connection)?;
+    client.store_code(code, state, &connection)
+        .bad_request()
+        .public_context("Authentication failed")?;
 
     Ok(Template::render("auth-done", hashmap!{
         "ASSET_REV" => ASSET_REV.as_str(),
@@ -29,10 +32,10 @@ pub fn post(auth: Form<OAuth>, connection: db::Connection) -> ApiResult<Template
 }
 
 #[get("/<session>")]
-fn login(session: String) -> ApiResult<Redirect> {
+pub fn login(session: String) -> ApiResult<Redirect> {
     let client = Authenticator::from_env()?;
     let (url, _csrf) = client.request_auth(session);
-    Ok(Redirect::to(&url.to_string()))
+    Ok(Redirect::to(url.to_string()))
 }
 
 #[derive(Debug, FromForm, Serialize, Deserialize)]

sn0int-registry/templates/index.html.hbs

@@ -1,10 +1,10 @@
 {{#*inline "page"}}
         <p>
-This is the registry server of sn0int, an OSINT framework and package manager.
-It was built for IT security professionals and bug hunters to gather
-intelligence about a given target or about yourself. sn0int is enumerating
-attack surface by semi-automatically processing public information and mapping
-the results in a unified format for followup investigations.
+This is the registry server of sn0int, a semi-automatic OSINT framework and
+package manager.  It was built for IT security professionals and bug hunters to
+gather intelligence about a given target or about yourself. sn0int is
+enumerating attack surface by semi-automatically processing public information
+and mapping the results in a unified format for followup investigations.
         </p>
 
         <p>
@@ -18,6 +18,8 @@ Among other things, sn0int is currently able to:
 <li>[X] Harvest emails from pgp keyservers</li>
 <li>[X] Enrich ip addresses with ASN and geoip info</li>
 <li>[X] Harvest subdomains from the wayback machine</li>
+<li>[X] Gather information about phonenumbers</li>
+<li>[X] Bruteforce interesting urls</li>
         </ul>
 
         <p>

src/api.rs

@@ -1,6 +1,6 @@
-use errors::*;
+use crate::errors::*;
 use std::fmt;
-use config::Config;
+use crate::config::Config;
 use chrootable_https::{self, HttpClient, Body, Request, Uri};
 use chrootable_https::http::request::Builder as RequestBuilder;
 use chrootable_https::header::CONTENT_TYPE;
@@ -10,8 +10,8 @@ use serde::de::DeserializeOwned;
 use serde::ser::Serialize;
 use serde_json;
 use sn0int_common::api::*;
-use sn0int_common::ModuleID;
-use web;
+use sn0int_common::{ModuleID, ApiResponse};
+use crate::web;
 
 
 pub struct Client {
@@ -22,7 +22,10 @@ pub struct Client {
 
 impl Client {
     pub fn new(config: &Config) -> Result<Client> {
-        let client = chrootable_https::Client::with_system_resolver()?;
+        let client = match config.network.proxy {
+            Some(proxy) => chrootable_https::Client::with_socks5(proxy),
+            _ => chrootable_https::Client::with_system_resolver()?,
+        };
         Ok(Client {
             server: config.core.registry.clone(),
             client,
@@ -46,7 +49,8 @@ impl Client {
 
         let request = request.body(body)?;
 
-        let resp = self.client.request(request)?;
+        let resp = self.client.request(request)
+            .wait_for_response()?;
         info!("response: {:?}", resp);
 
         let reply = serde_json::from_slice::<ApiResponse<T>>(&resp.body)?;

src/archive.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 use tar;
 use libflate::gzip;
 use std::io;

src/args.rs

@@ -1,6 +1,8 @@
 use structopt::clap::{AppSettings, Shell};
 use sn0int_common::ModuleID;
-use workspaces::Workspace;
+use crate::cmd;
+use crate::options;
+use crate::workspaces::Workspace;
 
 
 #[derive(Debug, StructOpt)]
@@ -44,6 +46,9 @@ pub enum SubCommand {
     #[structopt(author="", name="search")]
     /// Search in the registry
     Search(Search),
+    #[structopt(author="", name="select")]
+    /// Select from the database
+    Select(cmd::select_cmd::Args),
     #[structopt(author="", name="completions")]
     /// Generate shell completions
     Completions(Completions),
@@ -66,6 +71,21 @@ pub struct Run {
     #[structopt(long="stdin")]
     /// Expose stdin to modules
     pub stdin: bool,
+    #[structopt(long="grant")]
+    /// Automatically grant access to a keyring namespace
+    pub grants: Vec<String>,
+    #[structopt(long="grant-full-keyring")]
+    /// Automatically grant access to all requested keys
+    pub grant_full_keyring: bool,
+    #[structopt(long="deny-keyring")]
+    /// Automatically deny access to all requested keys
+    pub deny_keyring: bool,
+    #[structopt(short="x", long="exit-on-error")]
+    /// Exit on first error and set exit code
+    pub exit_on_error: bool,
+    #[structopt(short="o", long="option")]
+    /// Set an option
+    pub options: Vec<options::Opt>,
 }
 
 #[derive(Debug, StructOpt)]

src/auth.rs

@@ -1,12 +1,12 @@
-use errors::*;
+use crate::errors::*;
 use opener;
 use std::fs;
 use std::thread;
 use std::time::Duration;
-use api::Client;
-use config::Config;
-use paths;
-use term;
+use crate::api::Client;
+use crate::config::Config;
+use crate::paths;
+use crate::term;
 
 
 pub fn load_token() -> Result<String> {

src/cmd/add_cmd.rs

@@ -1,12 +1,15 @@
-use errors::*;
+use crate::errors::*;
 
-use models::*;
-use shell::Readline;
+use crate::models::*;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use utils;
+use structopt::clap::AppSettings;
+use crate::utils;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub enum Args {
     #[structopt(name="domain")]
     Domain(AddDomain),
@@ -14,6 +17,12 @@ pub enum Args {
     Subdomain(AddSubdomain),
     #[structopt(name="email")]
     Email(AddEmail),
+    #[structopt(name="phonenumber")]
+    PhoneNumber(AddPhoneNumber),
+    #[structopt(name="device")]
+    Device(AddDevice),
+    #[structopt(name="network")]
+    Network(AddNetwork),
 }
 
 #[derive(Debug, StructOpt)]
@@ -31,12 +40,34 @@ pub struct AddEmail {
     email: Option<String>,
 }
 
+#[derive(Debug, StructOpt)]
+pub struct AddPhoneNumber {
+    phonenumber: Option<String>,
+    name: Option<String>,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddDevice {
+    mac: Option<String>,
+    name: Option<String>,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddNetwork {
+    network: Option<String>,
+    latitude: Option<f32>,
+    longitude: Option<f32>,
+}
+
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
     match args {
         Args::Domain(args) => add_domain(rl, args),
         Args::Subdomain(args) => add_subdomain(rl, args),
         Args::Email(args) => add_email(rl, args),
+        Args::PhoneNumber(args) => add_phonenumber(rl, args),
+        Args::Device(args) => add_device(rl, args),
+        Args::Network(args) => add_network(rl, args),
     }
 }
 
@@ -103,3 +134,75 @@ fn add_email(rl: &mut Readline, args: AddEmail) -> Result<()> {
 
     Ok(())
 }
+
+fn add_phonenumber(rl: &mut Readline, args: AddPhoneNumber) -> Result<()> {
+    let (phonenumber, name) = match args.phonenumber {
+        Some(phonenumber) => {
+            (phonenumber, args.name)
+        },
+        _ => {
+            let phonenumber = utils::question("Phone Number")?;
+            let name = utils::question_opt("Name")?;
+            (phonenumber, name)
+        },
+    };
+
+    rl.db().insert_struct(NewPhoneNumber {
+        value: &phonenumber,
+        name: name.as_ref(),
+        valid: None,
+        last_online: None,
+        country: None,
+        carrier: None,
+        line: None,
+        is_ported: None,
+        last_ported: None,
+        caller_name: None,
+        caller_type: None,
+    })?;
+
+    Ok(())
+}
+
+fn add_device(rl: &mut Readline, args: AddDevice) -> Result<()> {
+    let (mac, name) = match args.mac {
+        Some(mac) => {
+            (mac, args.name)
+        },
+        _ => {
+            let mac = utils::question("Mac address")?;
+            let name = utils::question_opt("Name")?;
+            (mac, name)
+        },
+    };
+
+    rl.db().insert_struct(NewDevice {
+        value: &mac,
+        name: name.as_ref(),
+        hostname: None,
+        vendor: None,
+        last_seen: None,
+    })?;
+
+    Ok(())
+}
+
+fn add_network(rl: &mut Readline, args: AddNetwork) -> Result<()> {
+    let (network, latitude, longitude) = match args.network {
+        Some(network) => (network, args.latitude, args.longitude),
+        _ => {
+            let network = utils::question("Network")?;
+            let latitude = utils::question_typed_opt("Latitude")?;
+            let longitude = utils::question_typed_opt("Longitude")?;
+            (network, latitude, longitude)
+        }
+    };
+
+    rl.db().insert_struct(NewNetwork {
+        value: &network,
+        latitude,
+        longitude,
+    })?;
+
+    Ok(())
+}

src/cmd/delete_cmd.rs

@@ -1,13 +1,16 @@
-use errors::*;
+use crate::errors::*;
 
-use db;
-use shell::Readline;
+use crate::db;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use models::*;
-use term;
+use structopt::clap::AppSettings;
+use crate::models::*;
+use crate::term;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub enum Args {
     #[structopt(name="domains")]
     Domains(Filter),
@@ -19,6 +22,12 @@ pub enum Args {
     Urls(Filter),
     #[structopt(name="emails")]
     Emails(Filter),
+    #[structopt(name="phonenumbers")]
+    PhoneNumbers(Filter),
+    #[structopt(name="devices")]
+    Devices(Filter),
+    #[structopt(name="networks")]
+    Networks(Filter),
 }
 
 #[derive(Debug, StructOpt)]
@@ -40,6 +49,9 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
         Args::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
         Args::Urls(filter) => delete::<Url>(rl, &filter),
         Args::Emails(filter) => delete::<Email>(rl, &filter),
+        Args::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
+        Args::Devices(filter) => delete::<Device>(rl, &filter),
+        Args::Networks(filter) => delete::<Network>(rl, &filter),
     }?;
     term::info(&format!("Deleted {} rows", rows));
     Ok(())

src/cmd/help_cmd.rs

@@ -0,0 +1,30 @@
+use crate::errors::*;
+
+use crate::shell::Readline;
+
+
+#[inline]
+fn help(name: &str, descr: &str) {
+    println!("    \x1b[32m{:13}\x1b[0m {}", name, descr);
+}
+
+pub fn run(_rl: &mut Readline, _args: &[String]) -> Result<()> {
+
+    println!("\n\x1b[33mCOMMANDS:\x1b[0m");
+    help("add",        "Add new entities to the database");
+    help("delete",     "Delete entities from the database");
+    help("keyring",    "Manage saved credentials");
+    help("mod",        "Manage installed modules");
+    help("noscope",    "Exclude entities from scope");
+    help("quickstart", "Install all featured modules");
+    help("run",        "Run the currently selected module");
+    help("scope",      "Include entities in the scope again");
+    help("select",     "Select entities from the database");
+    help("target",     "Preview targeted entities or narrow them down");
+    help("use",        "Select a module");
+    help("workspace",  "Switch to a different workspace");
+    help("help",       "Prints this message");
+    println!("\nRun <command> -h for more help.\n");
+
+    Ok(())
+}

src/cmd/keyring_cmd.rs

@@ -0,0 +1,105 @@
+use crate::errors::*;
+
+use crate::keyring::{KeyName, KeyRing};
+use crate::shell::Readline;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+use crate::utils;
+
+
+#[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
+pub enum Args {
+    #[structopt(name="add")]
+    /// Add a new key to the keyring
+    Add(KeyRingAdd),
+    #[structopt(name="delete")]
+    /// Delete a key from the keyring
+    Delete(KeyRingDelete),
+    #[structopt(name="get")]
+    /// Get a key from the keyring
+    Get(KeyRingGet),
+    #[structopt(name="list")]
+    /// List keys in the keyring
+    List(KeyRingList),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct KeyRingAdd {
+    key: KeyName,
+    secret: Option<String>,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct KeyRingDelete {
+    key: KeyName,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct KeyRingGet {
+    key: KeyName,
+    #[structopt(short="q",
+                long="quiet")]
+    /// Only output secret key
+    quiet: bool,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct KeyRingList {
+    namespace: Option<String>,
+}
+
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+    match args {
+        Args::Add(add) => keyring_add(rl.keyring_mut(), add),
+        Args::Delete(delete) => keyring_delete(rl.keyring_mut(), delete),
+        Args::Get(get) => keyring_get(rl.keyring(), &get),
+        Args::List(list) => keyring_list(rl.keyring(), list),
+    }
+}
+
+fn keyring_add(keyring: &mut KeyRing, add: KeyRingAdd) -> Result<()> {
+    // TODO: there's no non-interactive way to add a key without a secret key
+    let secret = match add.secret {
+        Some(secret) => Some(secret),
+        None => utils::question_opt("Secretkey")?,
+    };
+
+    keyring.insert(add.key, secret)
+}
+
+fn keyring_delete(keyring: &mut KeyRing, delete: KeyRingDelete) -> Result<()> {
+    keyring.delete(delete.key)
+}
+
+fn keyring_get(keyring: &KeyRing, get: &KeyRingGet) -> Result<()> {
+    if let Some(key) = keyring.get(&get.key) {
+        if get.quiet {
+            if let Some(secret_key) = key.secret_key {
+                println!("{}", secret_key);
+            }
+        } else {
+            println!("Namespace:    {:?}", get.key.namespace);
+            println!("Access Key:   {:?}", get.key.name);
+            if let Some(secret_key) = key.secret_key {
+                println!("Secret:       {:?}", secret_key);
+            }
+        }
+    }
+    Ok(())
+}
+
+fn keyring_list(keyring: &KeyRing, list: KeyRingList) -> Result<()> {
+    let list = match list.namespace {
+        Some(namespace) => keyring.list_for(&namespace),
+        None => keyring.list(),
+    };
+
+    for key in list {
+        println!("{}:{}", key.namespace, key.name);
+    }
+
+    Ok(())
+}

src/cmd/mod.rs

@@ -1,10 +1,26 @@
+use crate::errors::*;
+use crate::shell::Readline;
+
+pub trait Cmd: structopt::StructOpt + Sized {
+    fn run(&self, rl: &mut Readline) -> Result<()>;
+
+    #[inline]
+    fn run_str(rl: &mut Readline, args: &[String]) -> Result<()> {
+        let args = Self::from_iter_safe(args)?;
+        args.run(rl)
+    }
+}
+
 pub mod add_cmd;
 pub mod delete_cmd;
+pub mod help_cmd;
 pub mod run_cmd;
 pub mod use_cmd;
 pub mod select_cmd;
+pub mod keyring_cmd;
 pub mod mod_cmd;
 pub mod noscope_cmd;
+pub mod set_cmd;
 pub mod scope_cmd;
 pub mod target_cmd;
 pub mod quickstart_cmd;

src/cmd/mod_cmd.rs

@@ -1,19 +1,22 @@
-use errors::*;
+use crate::errors::*;
 
-use args;
-use args::Install;
-use api::Client;
-use config::Config;
+use crate::args;
+use crate::args::Install;
+use crate::api::Client;
+use crate::config::Config;
 use colored::Colorize;
-use engine::Module;
-use registry;
-use shell::Readline;
+use crate::engine::Module;
+use crate::registry;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use term;
-use worker;
+use structopt::clap::AppSettings;
+use crate::term;
+use crate::worker;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
     #[structopt(subcommand)]
     pub subcommand: SubCommand,

src/cmd/noscope_cmd.rs

@@ -1,13 +1,16 @@
-use errors::*;
+use crate::errors::*;
 
-use db;
-use shell::Readline;
+use crate::db;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use models::*;
-use term;
+use structopt::clap::AppSettings;
+use crate::models::*;
+use crate::term;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub enum Args {
     #[structopt(name="domains")]
     Domains(Filter),
@@ -19,6 +22,8 @@ pub enum Args {
     Urls(Filter),
     #[structopt(name="emails")]
     Emails(Filter),
+    #[structopt(name="phonenumbers")]
+    PhoneNumbers(Filter),
 }
 
 #[derive(Debug, StructOpt)]
@@ -40,6 +45,7 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
         Args::IpAddrs(filter) => noscope::<IpAddr>(rl, &filter),
         Args::Urls(filter) => noscope::<Url>(rl, &filter),
         Args::Emails(filter) => noscope::<Email>(rl, &filter),
+        Args::PhoneNumbers(filter) => noscope::<PhoneNumber>(rl, &filter),
     }?;
     term::info(&format!("Updated {} rows", rows));
     Ok(())

src/cmd/quickstart_cmd.rs

@@ -1,16 +1,18 @@
-use errors::*;
+use crate::errors::*;
 
-use args::Install;
-use api::Client;
-// use colored::Colorize;
-use cmd::mod_cmd;
-use registry;
-use shell::Readline;
+use crate::args::Install;
+use crate::api::Client;
+use crate::cmd::mod_cmd;
+use crate::registry;
+use crate::shell::Readline;
 use structopt::StructOpt;
+use structopt::clap::AppSettings;
 use sn0int_common::ModuleID;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
 }
 

src/cmd/run_cmd.rs

@@ -1,17 +1,25 @@
-use errors::*;
+use crate::errors::*;
 
-use db::{Database, Filter};
-use sn0int_common::metadata::Source;
+use crate::args;
+use crate::db::{Database, Filter};
+use crate::engine::Module;
+use crate::models::*;
+use crate::shell::Readline;
+use crate::keyring::KeyRing;
+use crate::term;
+use crate::utils;
+use crate::worker;
 use serde::Serialize;
 use serde_json;
-use shell::Readline;
+use sn0int_common::metadata::Source;
+use std::collections::HashMap;
 use structopt::StructOpt;
-use models::*;
-use term;
-use worker;
+use structopt::clap::AppSettings;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
     #[structopt(short="j", long="threads", default_value="1")]
     threads: usize,
@@ -19,6 +27,45 @@ pub struct Args {
     verbose: u64,
 }
 
+#[derive(Debug, Clone)]
+pub struct Params<'a> {
+    pub threads: usize,
+    pub verbose: u64,
+    pub stdin: bool,
+    pub grants: &'a [String],
+    pub grant_full_keyring: bool,
+    pub deny_keyring: bool,
+    pub exit_on_error: bool,
+}
+
+impl<'a> From<&'a args::Run> for Params<'a> {
+    fn from(args: &args::Run) -> Params {
+        Params {
+            threads: args.threads,
+            verbose: args.verbose,
+            stdin: args.stdin,
+            grants: &args.grants,
+            grant_full_keyring: args.grant_full_keyring,
+            deny_keyring: args.deny_keyring,
+            exit_on_error: args.exit_on_error,
+        }
+    }
+}
+
+impl From<Args> for Params<'static> {
+    fn from(args: Args) -> Params<'static> {
+        Params {
+            threads: args.threads,
+            verbose: args.verbose,
+            stdin: false,
+            grants: &[],
+            grant_full_keyring: false,
+            deny_keyring: false,
+            exit_on_error: false,
+        }
+    }
+}
+
 fn prepare_arg<T: Serialize + Model>(x: T) -> Result<(serde_json::Value, Option<String>)> {
     let pretty = x.to_string();
     let arg = serde_json::to_value(x)?;
@@ -32,11 +79,36 @@ fn prepare_args<T: Scopable + Serialize + Model>(db: &Database, filter: &Filter)
         .collect()
 }
 
-pub fn execute(rl: &mut Readline, threads: usize, verbose: u64, has_stdin: bool) -> Result<()> {
+fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
+    for namespace in keyring.unauthorized_namespaces(&module) {
+        let grant_access = if params.deny_keyring {
+            false
+        } else if params.grant_full_keyring || params.grants.contains(namespace) {
+            true
+        } else {
+            let msg = format!("Grant access to {:?} credentials?", namespace);
+            utils::no_else_yes(&msg)?
+        };
+
+        if grant_access {
+            keyring.grant_access(&module, namespace.to_string());
+            term::info(&format!("Granted access to {:?}", namespace));
+        }
+    }
+
+    keyring.save()
+        .context("Failed to write keyring")?;
+
+    Ok(())
+}
+
+pub fn execute(rl: &mut Readline, params: Params, options: HashMap<String, String>) -> Result<()> {
     let module = rl.module()
         .map(|m| m.to_owned())
         .ok_or_else(|| format_err!("No module selected"))?;
 
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+
     let filter = rl.scoped_targets();
 
     let args = match module.source() {
@@ -45,19 +117,46 @@ pub fn execute(rl: &mut Readline, threads: usize, verbose: u64, has_stdin: bool)
         Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl.db(), &filter),
         Some(Source::Urls) => prepare_args::<Url>(rl.db(), &filter),
         Some(Source::Emails) => prepare_args::<Email>(rl.db(), &filter),
+        Some(Source::PhoneNumbers) => prepare_args::<PhoneNumber>(rl.db(), &filter),
+        Some(Source::KeyRing(namespace)) => {
+            let keyring = rl.keyring();
+            if keyring.is_access_granted(&module, &namespace) {
+                keyring.get_all_for(&namespace).into_iter()
+                    .map(|key| {
+                        let pretty = format!("{}:{}", key.namespace, key.access_key);
+                        let arg = serde_json::to_value(key)?;
+                        Ok((arg, Some(pretty)))
+                    })
+                    .collect::<Result<Vec<_>>>()
+            } else {
+                Ok(vec![])
+            }
+        },
         None => Ok(vec![(serde_json::Value::Null, None)]),
     }?;
 
     rl.signal_register().catch_ctrl();
-    worker::spawn(rl, &module, args, threads, verbose, has_stdin);
+    let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);
     rl.signal_register().reset_ctrlc();
 
-    term::info(&format!("Finished {}", module.canonical()));
+    if errors > 0 {
+        term::info(&format!("Finished {} ({} errors)", module.canonical(), errors));
+
+        if params.exit_on_error {
+            bail!("Some scripts failed");
+        }
+    } else {
+        term::info(&format!("Finished {}", module.canonical()));
+    }
 
     Ok(())
 }
 
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
-    execute(rl, args.threads, args.verbose, false)
+    let options = match rl.options_mut() {
+        Some(options) => options.clone(),
+        _ => HashMap::new(),
+    };
+    execute(rl, args.into(), options)
 }

src/cmd/scope_cmd.rs

@@ -1,13 +1,16 @@
-use errors::*;
+use crate::errors::*;
 
-use db;
-use shell::Readline;
+use crate::db;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use models::*;
-use term;
+use structopt::clap::AppSettings;
+use crate::models::*;
+use crate::term;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub enum Args {
     #[structopt(name="domains")]
     Domains(Filter),
@@ -19,6 +22,8 @@ pub enum Args {
     Urls(Filter),
     #[structopt(name="emails")]
     Emails(Filter),
+    #[structopt(name="phonenumbers")]
+    PhoneNumbers(Filter),
 }
 
 #[derive(Debug, StructOpt)]
@@ -40,6 +45,7 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
         Args::IpAddrs(filter) => scope::<IpAddr>(rl, &filter),
         Args::Urls(filter) => scope::<Url>(rl, &filter),
         Args::Emails(filter) => scope::<Email>(rl, &filter),
+        Args::PhoneNumbers(filter) => scope::<PhoneNumber>(rl, &filter),
     }?;
     term::info(&format!("Updated {} rows", rows));
     Ok(())

src/cmd/select_cmd.rs

@@ -1,23 +1,41 @@
-use errors::*;
+use crate::errors::*;
 
-use db;
-use shell::Readline;
+use crate::cmd::Cmd;
+use crate::db;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use models::*;
+use structopt::clap::AppSettings;
+use crate::models::*;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub enum Args {
     #[structopt(name="domains")]
+    /// Select domains
     Domains(Filter),
     #[structopt(name="subdomains")]
+    /// Select subdomains
     Subdomains(Filter),
     #[structopt(name="ipaddrs")]
+    /// Select ipaddrs
     IpAddrs(Filter),
     #[structopt(name="urls")]
+    /// Select urls
     Urls(Filter),
     #[structopt(name="emails")]
+    /// Select emails
     Emails(Filter),
+    #[structopt(name="phonenumbers")]
+    /// Select phone numbers
+    PhoneNumbers(Filter),
+    #[structopt(name="devices")]
+    /// Select devices
+    Devices(Filter),
+    #[structopt(name="networks")]
+    /// Select networks
+    Networks(Filter),
 }
 
 #[derive(Debug, StructOpt)]
@@ -31,17 +49,26 @@ impl Filter {
     }
 }
 
-pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
-    let args = Args::from_iter_safe(args)?;
-    match args {
-        Args::Domains(filter) => select::<Domain>(rl, &filter),
-        Args::Subdomains(filter) => select::<Subdomain>(rl, &filter),
-        Args::IpAddrs(filter) => select::<IpAddr>(rl, &filter),
-        Args::Urls(filter) => select::<Url>(rl, &filter),
-        Args::Emails(filter) => select::<Email>(rl, &filter),
+impl Cmd for Args {
+    fn run(&self, rl: &mut Readline) -> Result<()> {
+        match self {
+            Args::Domains(filter) => select::<Domain>(rl, &filter),
+            Args::Subdomains(filter) => select::<Subdomain>(rl, &filter),
+            Args::IpAddrs(filter) => select::<IpAddr>(rl, &filter),
+            Args::Urls(filter) => select::<Url>(rl, &filter),
+            Args::Emails(filter) => select::<Email>(rl, &filter),
+            Args::PhoneNumbers(filter) => select::<PhoneNumber>(rl, &filter),
+            Args::Devices(filter) => select::<Device>(rl, &filter),
+            Args::Networks(filter) => select::<Network>(rl, &filter),
+        }
     }
 }
 
+#[inline]
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    Args::run_str(rl, args)
+}
+
 fn select<T: Model + Detailed>(rl: &mut Readline, filter: &Filter) -> Result<()> {
     for obj in rl.db().filter::<T>(&filter.parse()?)? {
         println!("{}", obj.detailed(rl.db())?);

src/cmd/set_cmd.rs

@@ -0,0 +1,42 @@
+use crate::errors::*;
+
+use crate::shell::Readline;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+
+
+#[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
+pub struct Args {
+    key: Option<String>,
+    value: Option<String>,
+}
+
+// TODO: maybe introduce global settings
+// TODO: maybe allow setting jobs here as well in addition to -j
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+
+    let options = rl.options_mut()
+        .ok_or_else(|| format_err!("Module needs to be selected first"))?;
+
+    match (args.key, args.value) {
+        (None, None) => {
+            for (key, value) in options.iter() {
+                println!("{}={:?}", key, value);
+            }
+        },
+        (Some(key), None) => {
+            if let Some(value) = options.get(&key) {
+                println!("{:?}", value);
+            }
+        },
+        (Some(key), Some(value)) => {
+            options.insert(key, value);
+        },
+        (None, Some(_)) => unreachable!(),
+    }
+
+    Ok(())
+}

src/cmd/target_cmd.rs

@@ -1,14 +1,17 @@
-use errors::*;
+use crate::errors::*;
 
-use db;
-use shell::Readline;
+use crate::db;
+use crate::shell::Readline;
 use sn0int_common::metadata::Source;
 use structopt::StructOpt;
-use term;
-use models::*;
+use structopt::clap::AppSettings;
+use crate::term;
+use crate::models::*;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
     // TODO: target -p # print current filter
     // TODO: target -c # clear filter
@@ -33,6 +36,12 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
             Source::IpAddrs => select::<IpAddr>(rl)?,
             Source::Urls => select::<Url>(rl)?,
             Source::Emails => select::<Email>(rl)?,
+            Source::PhoneNumbers => select::<PhoneNumber>(rl)?,
+            Source::KeyRing(namespace) => {
+                for key in rl.keyring().list_for(&namespace) {
+                    println!("{}:{}", key.namespace, key.name);
+                }
+            },
         }
     } else {
         debug!("Setting filter to {:?}", args.filter);
@@ -54,6 +63,8 @@ fn count_selected(rl: &mut Readline, source: &Source) -> Result<usize> {
         Source::IpAddrs => db.filter::<IpAddr>(&filter)?.len(),
         Source::Urls => db.filter::<Url>(&filter)?.len(),
         Source::Emails => db.filter::<Email>(&filter)?.len(),
+        Source::PhoneNumbers => db.filter::<PhoneNumber>(&filter)?.len(),
+        Source::KeyRing(namespace) => rl.keyring().list_for(&namespace).len(),
     };
     Ok(num)
 }

src/cmd/use_cmd.rs

@@ -1,10 +1,13 @@
-use errors::*;
+use crate::errors::*;
 
-use shell::Readline;
+use crate::shell::Readline;
 use structopt::StructOpt;
+use structopt::clap::AppSettings;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
     module: String,
 }

src/cmd/workspace_cmd.rs

@@ -1,12 +1,15 @@
-use errors::*;
+use crate::errors::*;
 
-use db::Database;
-use shell::Readline;
+use crate::db::Database;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use workspaces::{self, Workspace};
+use structopt::clap::AppSettings;
+use crate::workspaces::{self, Workspace};
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
     workspace: Option<Workspace>,
 }

src/complete.rs

@@ -1,5 +1,5 @@
-use args::{Args, Completions};
-use errors::*;
+use crate::args::{Args, Completions};
+use crate::errors::*;
 use rustyline;
 use rustyline::completion::Completer;
 use rustyline::highlight::Highlighter;
@@ -9,8 +9,8 @@ use std::borrow::Cow::{self, Borrowed, Owned};
 use std::str::FromStr;
 use std::io::stdout;
 use structopt::StructOpt;
-use shell::Command;
-use workspaces;
+use crate::shell::Command;
+use crate::workspaces;
 
 
 #[derive(Debug, Default)]
@@ -30,7 +30,10 @@ impl CmdCompleter {
                             "subdomains",
                             "ipaddrs",
                             "urls",
-                            "emails"];
+                            "emails",
+                            "phonenumbers",
+                            "devices",
+                            "networks"];
 
             let results: Vec<String> = options.iter()
                 .filter(|x| x.starts_with(arg))
@@ -86,7 +89,11 @@ impl Completer for CmdCompleter {
                         let arg = &cmd[1];
 
                         let options = &["domain",
-                                        "subdomain"];
+                                        "subdomain",
+                                        "email",
+                                        "phonenumber",
+                                        "device",
+                                        "network"];
 
                         let results: Vec<String> = options.iter()
                             .filter(|x| x.starts_with(arg))
@@ -96,6 +103,25 @@ impl Completer for CmdCompleter {
                     }
                 },
                 Command::Delete => self.filter("delete", &cmd),
+                Command::Keyring => {
+                    // we can only complete the 2nd argument
+                    if args != 2 {
+                        Ok((0, vec![]))
+                    } else {
+                        let arg = &cmd[1];
+
+                        let options = &["add",
+                                        "delete",
+                                        "get",
+                                        "list"];
+
+                        let results: Vec<String> = options.iter()
+                            .filter(|x| x.starts_with(arg))
+                            .map(|x| format!("keyring {} ", x))
+                            .collect();
+                        Ok((0, results))
+                    }
+                },
                 Command::Mod => {
                     // we can only complete the 2nd argument
                     if args != 2 {
@@ -106,7 +132,8 @@ impl Completer for CmdCompleter {
                         let options = &["list",
                                         "install",
                                         "search",
-                                        "reload"];
+                                        "reload",
+                                        "update"];
 
                         let results: Vec<String> = options.iter()
                             .filter(|x| x.starts_with(arg))

src/config.rs

@@ -1,13 +1,17 @@
 use dirs;
-use errors::*;
+use crate::errors::*;
 use std::fs;
 use std::path::{Path, PathBuf};
+use std::net::SocketAddr;
 use toml;
 
 
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct Config {
+    #[serde(default)]
     pub core: CoreConfig,
+    #[serde(default)]
+    pub network: NetworkConfig,
 }
 
 impl Config {
@@ -60,3 +64,8 @@ impl Default for CoreConfig {
 fn default_registry() -> String {
     String::from("https://sn0int.com")
 }
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct NetworkConfig {
+    pub proxy: Option<SocketAddr>,
+}

src/crt.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 
 use x509_parser;
 use der_parser::oid::Oid;
@@ -45,7 +45,7 @@ named!(san_value_dns<&[u8], Result<AlternativeName>>, do_parse!(
     value:  take!(len)      >>
     ({
         String::from_utf8(value.to_vec())
-            .map(|v| AlternativeName::DnsName(v))
+            .map(AlternativeName::DnsName)
             .map_err(Error::from)
     })
 ));

src/db.rs

@@ -1,17 +1,17 @@
-use errors::*;
+use crate::errors::*;
 
 use diesel;
 use diesel::expression::SqlLiteral;
 use diesel::expression::sql_literal::sql;
 use diesel::sql_types::Bool;
 use diesel::prelude::*;
-use models::*;
-use schema::*;
+use crate::models::*;
+use crate::schema::*;
 use std::str::FromStr;
-use paths;
-use migrations;
-use worker;
-use workspaces::Workspace;
+use crate::paths;
+use crate::migrations;
+use crate::worker;
+use crate::workspaces::Workspace;
 
 
 #[derive(Debug)]
@@ -38,6 +38,10 @@ pub enum Family {
     SubdomainIpAddr,
     Url,
     Email,
+    PhoneNumber,
+    Device,
+    Network,
+    NetworkDevice,
 }
 
 impl FromStr for Family {
@@ -51,6 +55,10 @@ impl FromStr for Family {
             "subdomain-ipaddr" => Family::SubdomainIpAddr,
             "url" => Family::Url,
             "email" => Family::Email,
+            "phonenumber" => Family::PhoneNumber,
+            "device" => Family::Device,
+            "network" => Family::Network,
+            "network-device" => Family::NetworkDevice,
             _ => bail!("Unknown object family"),
         })
     }
@@ -114,6 +122,8 @@ impl Database {
                 latitude: object.latitude,
                 asn: object.asn,
                 as_org: object.as_org.as_ref(),
+                description: object.description.as_ref(),
+                reverse_dns: object.reverse_dns.as_ref(),
             }),
             Insert::SubdomainIpAddr(object) => self.insert_subdomain_ipaddr_struct(&NewSubdomainIpAddr {
                 subdomain_id: object.subdomain_id,
@@ -133,6 +143,37 @@ impl Database {
                 value: &object.value,
                 valid: object.valid,
             }),
+            Insert::PhoneNumber(object) => self.insert_struct(NewPhoneNumber {
+                value: &object.value,
+                name: object.name.as_ref(),
+                valid: object.valid,
+                last_online: object.last_online,
+                country: object.country.as_ref(),
+                carrier: object.carrier.as_ref(),
+                line: object.line.as_ref(),
+                is_ported: object.is_ported,
+                last_ported: object.last_ported,
+                caller_name: object.caller_name.as_ref(),
+                caller_type: object.caller_type.as_ref(),
+            }),
+            Insert::Device(object) => self.insert_struct(NewDevice {
+                value: &object.value,
+                name: object.name.as_ref(),
+                hostname: object.hostname.as_ref(),
+                vendor: object.vendor.as_ref(),
+                last_seen: object.last_seen,
+            }),
+            Insert::Network(object) => self.insert_struct(NewNetwork {
+                value: &object.value,
+                latitude: object.latitude,
+                longitude: object.longitude,
+            }),
+            Insert::NetworkDevice(object) => self.insert_network_device_struct(&NewNetworkDevice {
+                network_id: object.network_id,
+                device_id: object.device_id,
+                ipaddr: object.ipaddr.as_ref(),
+                last_seen: object.last_seen,
+            }),
         }
     }
 
@@ -170,6 +211,18 @@ impl Database {
         }
     }
 
+    pub fn insert_network_device_struct(&self, network_device: &NewNetworkDevice) -> Result<Option<(DbChange, i32)>> {
+        if let Some(subdomain_ipaddr_id) = NetworkDevice::get_id_opt(self, &(network_device.network_id, network_device.device_id))? {
+            Ok(Some((DbChange::None, subdomain_ipaddr_id)))
+        } else {
+            diesel::insert_into(network_devices::table)
+                .values(network_device)
+                .execute(&self.db)?;
+            let id = NetworkDevice::get_id(self, &(network_device.network_id, network_device.device_id))?;
+            Ok(Some((DbChange::Insert, id)))
+        }
+    }
+
     //
 
     pub fn update_generic(&self, object: &Update) -> Result<i32> {
@@ -178,11 +231,15 @@ impl Database {
             Update::IpAddr(object) => self.update_ipaddr(object),
             Update::Url(object) => self.update_url(object),
             Update::Email(object) => self.update_email(object),
+            Update::PhoneNumber(object) => self.update_phonenumber(object),
+            Update::Device(object) => self.update_device(object),
+            Update::Network(object) => self.update_network(object),
+            Update::NetworkDevice(object) => self.update_network_device(object),
         }
     }
 
     pub fn update_subdomain(&self, subdomain: &SubdomainUpdate) -> Result<i32> {
-        use schema::subdomains::columns::*;
+        use crate::schema::subdomains::columns::*;
         diesel::update(subdomains::table.filter(id.eq(subdomain.id)))
             .set(subdomain)
             .execute(&self.db)?;
@@ -190,7 +247,7 @@ impl Database {
     }
 
     pub fn update_ipaddr(&self, ipaddr: &IpAddrUpdate) -> Result<i32> {
-        use schema::ipaddrs::columns::*;
+        use crate::schema::ipaddrs::columns::*;
         diesel::update(ipaddrs::table.filter(id.eq(ipaddr.id)))
             .set(ipaddr)
             .execute(&self.db)?;
@@ -198,7 +255,7 @@ impl Database {
     }
 
     pub fn update_url(&self, url: &UrlUpdate) -> Result<i32> {
-        use schema::urls::columns::*;
+        use crate::schema::urls::columns::*;
         diesel::update(urls::table.filter(id.eq(url.id)))
             .set(url)
             .execute(&self.db)?;
@@ -206,13 +263,45 @@ impl Database {
     }
 
     pub fn update_email(&self, email: &EmailUpdate) -> Result<i32> {
-        use schema::emails::columns::*;
+        use crate::schema::emails::columns::*;
         diesel::update(emails::table.filter(id.eq(email.id)))
             .set(email)
             .execute(&self.db)?;
         Ok(email.id)
     }
 
+    pub fn update_phonenumber(&self, phonenumber: &PhoneNumberUpdate) -> Result<i32> {
+        use crate::schema::phonenumbers::columns::*;
+        diesel::update(phonenumbers::table.filter(id.eq(phonenumber.id)))
+            .set(phonenumber)
+            .execute(&self.db)?;
+        Ok(phonenumber.id)
+    }
+
+    pub fn update_device(&self, device: &DeviceUpdate) -> Result<i32> {
+        use crate::schema::devices::columns::*;
+        diesel::update(devices::table.filter(id.eq(device.id)))
+            .set(device)
+            .execute(&self.db)?;
+        Ok(device.id)
+    }
+
+    pub fn update_network(&self, network: &NetworkUpdate) -> Result<i32> {
+        use crate::schema::networks::columns::*;
+        diesel::update(networks::table.filter(id.eq(network.id)))
+            .set(network)
+            .execute(&self.db)?;
+        Ok(network.id)
+    }
+
+    pub fn update_network_device(&self, network_device: &NetworkDeviceUpdate) -> Result<i32> {
+        use crate::schema::network_devices::columns::*;
+        diesel::update(network_devices::table.filter(id.eq(network_device.id)))
+            .set(network_device)
+            .execute(&self.db)?;
+        Ok(network_device.id)
+    }
+
     fn get_opt_typed<T: Model + Scopable>(&self, value: &T::ID) -> Result<Option<i32>> {
         match T::get_opt(self, &value)? {
             Some(ref obj) if obj.scoped() => Ok(Some(obj.id())),
@@ -228,6 +317,10 @@ impl Database {
             Family::SubdomainIpAddr => bail!("Unsupported operation"),
             Family::Url => self.get_opt_typed::<Url>(&value),
             Family::Email => self.get_opt_typed::<Email>(&value),
+            Family::PhoneNumber => self.get_opt_typed::<PhoneNumber>(&value),
+            Family::Device => self.get_opt_typed::<Device>(&value),
+            Family::Network => self.get_opt_typed::<Network>(&value),
+            Family::NetworkDevice => bail!("Unsupported operation"),
         }
     }
 

src/engine/ctx.rs

@@ -1,19 +1,21 @@
-use errors::*;
+use crate::errors::*;
 
-use db::Family;
-use engine::{Environment, Reporter};
-use geoip::{GeoIP, AsnDB};
-use hlua::{self, AnyLuaValue};
-use models::{Insert, Update};
-use psl::Psl;
-use runtime;
+use crate::db::Family;
+use crate::engine::{Environment, Reporter};
+use crate::geoip::{GeoIP, AsnDB};
+use crate::hlua::{self, AnyLuaValue};
+use crate::keyring::KeyRingEntry;
+use crate::models::{Insert, Update};
+use crate::psl::Psl;
+use crate::runtime;
 use serde_json;
 use std::collections::HashMap;
 use std::result;
+use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
 use chrootable_https::dns::Resolver;
-use web::{HttpSession, HttpRequest, RequestOptions};
-use worker::{Event, LogEvent, DatabaseEvent, StdioEvent};
+use crate::web::{HttpSession, HttpRequest, RequestOptions};
+use crate::worker::{Event, LogEvent, DatabaseEvent, StdioEvent};
 
 
 pub trait State {
@@ -80,8 +82,14 @@ pub trait State {
         reply.map_err(|err| format_err!("Failed to read stdin: {:?}", err))
     }
 
+    fn keyring(&self, namespace: &str) -> Vec<&KeyRingEntry>;
+
     fn dns_config(&self) -> Arc<Resolver>;
 
+    fn proxy(&self) -> Option<&SocketAddr>;
+
+    fn getopt(&self, key: &str) -> Option<&String>;
+
     fn psl(&self) -> Arc<Psl>;
 
     fn geoip(&self) -> Arc<GeoIP>;
@@ -101,10 +109,13 @@ pub struct LuaState {
     logger: Arc<Mutex<Option<Arc<Mutex<Box<Reporter>>>>>>,
     http_sessions: Arc<Mutex<HashMap<String, HttpSession>>>,
     verbose: u64,
+    keyring: Arc<Vec<KeyRingEntry>>, // TODO: maybe hashmap
     dns_config: Arc<Resolver>,
     psl: Arc<Psl>,
     geoip: Arc<GeoIP>,
     asn: Arc<AsnDB>,
+    proxy: Option<SocketAddr>,
+    options: HashMap<String, String>,
 }
 
 impl State for LuaState {
@@ -152,10 +163,24 @@ impl State for LuaState {
         self.verbose
     }
 
+    fn keyring(&self, namespace: &str) -> Vec<&KeyRingEntry> {
+        self.keyring.iter()
+            .filter(|x| x.namespace == namespace)
+            .collect()
+    }
+
     fn dns_config(&self) -> Arc<Resolver> {
         self.dns_config.clone()
     }
 
+    fn proxy(&self) -> Option<&SocketAddr> {
+        self.proxy.as_ref()
+    }
+
+    fn getopt(&self, key: &str) -> Option<&String> {
+        self.options.get(key)
+    }
+
     fn psl(&self) -> Arc<Psl> {
         self.psl.clone()
     }
@@ -179,7 +204,7 @@ impl State for LuaState {
         let mtx = self.http_sessions.lock().unwrap();
         let session = mtx.get(session_id).expect("invalid session reference"); // TODO
 
-        HttpRequest::new(&session, method, url, options)
+        HttpRequest::new(&session, method, url, options, self.proxy.clone())
     }
 
     fn register_in_jar(&self, session: &str, key: String, value: String) {
@@ -205,10 +230,13 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
         http_sessions: Arc::new(Mutex::new(HashMap::new())),
 
         verbose: env.verbose,
+        keyring: Arc::new(env.keyring),
         dns_config: Arc::new(env.dns_config),
         psl: Arc::new(env.psl),
         geoip: Arc::new(env.geoip),
         asn: Arc::new(env.asn),
+        proxy: env.proxy,
+        options: env.options,
     });
 
     runtime::clear_err(&mut lua, state.clone());
@@ -220,6 +248,7 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
     runtime::error(&mut lua, state.clone());
     runtime::asn_lookup(&mut lua, state.clone());
     runtime::geoip_lookup(&mut lua, state.clone());
+    runtime::getopt(&mut lua, state.clone());
     runtime::html_select(&mut lua, state.clone());
     runtime::html_select_list(&mut lua, state.clone());
     runtime::http_mksession(&mut lua, state.clone());
@@ -229,6 +258,7 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
     runtime::json_decode(&mut lua, state.clone());
     runtime::json_decode_stream(&mut lua, state.clone());
     runtime::json_encode(&mut lua, state.clone());
+    runtime::keyring(&mut lua, state.clone());
     runtime::last_err(&mut lua, state.clone());
     runtime::pgp_pubkey(&mut lua, state.clone());
     runtime::pgp_pubkey_armored(&mut lua, state.clone());
@@ -239,8 +269,12 @@ fn ctx<'a>(env: Environment) -> (hlua::Lua<'a>, Arc<LuaState>) {
     runtime::sleep(&mut lua, state.clone());
     runtime::status(&mut lua, state.clone());
     runtime::stdin_readline(&mut lua, state.clone());
+    runtime::url_decode(&mut lua, state.clone());
+    runtime::url_encode(&mut lua, state.clone());
+    runtime::url_escape(&mut lua, state.clone());
     runtime::url_join(&mut lua, state.clone());
     runtime::url_parse(&mut lua, state.clone());
+    runtime::url_unescape(&mut lua, state.clone());
     runtime::utf8_decode(&mut lua, state.clone());
     runtime::x509_parse_pem(&mut lua, state.clone());
 
@@ -294,7 +328,7 @@ impl Script {
             return Err(err);
         }
 
-        use hlua::AnyLuaValue::*;
+        use crate::hlua::AnyLuaValue::*;
         match result {
             LuaString(x) => bail!("Script returned error: {:?}", x),
             _ => Ok(())
@@ -303,20 +337,25 @@ impl Script {
 
     #[cfg(test)]
     pub fn test(&self) -> Result<()> {
-        use engine::tests::DummyReporter;
-        use geoip::Maxmind;
+        use crate::engine::tests::DummyReporter;
+        use crate::geoip::Maxmind;
+        let keyring = Vec::new();
         let dns_config = Resolver::from_system()?;
-        let psl = Psl::from_str(r#"
+        let proxy = None;
+        let psl = r#"
 // ===BEGIN ICANN DOMAINS===
 com
 // ===END ICANN DOMAINS===
-"#)?;
+"#.parse::<Psl>()?;
         let geoip = GeoIP::open_or_download()?;
         let asn = AsnDB::open_or_download()?;
 
         let env = Environment {
             verbose: 0,
+            keyring,
             dns_config,
+            proxy,
+            options: HashMap::new(),
             psl,
             geoip,
             asn,

src/engine/isolation.rs

@@ -1,14 +1,17 @@
-use errors::*;
+use crate::errors::*;
 use chrootable_https::dns::Resolver;
-use engine::{Environment, Module, Reporter};
-use geoip::{GeoIP, AsnDB};
-use psl::Psl;
+use crate::engine::{Environment, Module, Reporter};
+use crate::geoip::{GeoIP, AsnDB, Maxmind};
+use crate::keyring::KeyRingEntry;
+use crate::psl::Psl;
 use serde_json;
-use worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
+use crate::worker::{Event, Event2, LogEvent, ExitEvent, EventSender, EventWithCallback};
 
+use std::collections::HashMap;
 use std::env;
 use std::io::prelude::*;
 use std::io::{self, BufReader, BufRead, stdin, Stdin, Stdout};
+use std::net::SocketAddr;
 use std::sync::{mpsc, Arc, Mutex};
 use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
 
@@ -16,18 +19,31 @@ use std::process::{Command, Child, Stdio, ChildStdin, ChildStdout};
 #[derive(Debug, Serialize, Deserialize)]
 pub struct StartCommand {
     verbose: u64,
+    keyring: Vec<KeyRingEntry>,
     dns_config: Resolver,
+    proxy: Option<SocketAddr>,
+    options: HashMap<String, String>,
     module: Module,
     arg: serde_json::Value,
 }
 
 impl StartCommand {
-    pub fn new(verbose: u64, dns_config: Resolver, module: Module, arg: serde_json::Value) -> StartCommand {
+    pub fn new(verbose: u64,
+               keyring: Vec<KeyRingEntry>,
+               dns_config: Resolver,
+               proxy: Option<SocketAddr>,
+               options: HashMap<String, String>,
+               module: Module,
+               arg: serde_json::Value,
+    ) -> StartCommand {
         StartCommand {
             verbose,
+            keyring,
             dns_config,
+            proxy,
+            options,
             module,
-            arg
+            arg,
         }
     }
 }
@@ -157,7 +173,15 @@ impl Reporter for StdioReporter {
     }
 }
 
-pub fn spawn_module(module: Module, tx: &EventSender, arg: serde_json::Value, verbose: u64, has_stdin: bool) -> Result<()> {
+pub fn spawn_module(module: Module,
+                    tx: &EventSender,
+                    arg: serde_json::Value,
+                    keyring: Vec<KeyRingEntry>,
+                    verbose: u64,
+                    has_stdin: bool,
+                    proxy: Option<SocketAddr>,
+                    options: HashMap<String, String>,
+) -> Result<ExitEvent> {
     let dns_config = Resolver::from_system()?;
 
     let mut reader = if has_stdin {
@@ -167,37 +191,42 @@ pub fn spawn_module(module: Module, tx: &EventSender, arg: serde_json::Value, ve
     };
 
     let mut supervisor = Supervisor::setup(&module)?;
-    supervisor.send_start(&StartCommand::new(verbose, dns_config, module, arg))?;
+    supervisor.send_start(&StartCommand::new(verbose, keyring, dns_config, proxy, options, module, arg))?;
 
-    loop {
+    let exit = loop {
         match supervisor.recv()? {
             Event::Log(event) => tx.send(Event2::Log(event)),
             Event::Database(object) => supervisor.send_event_callback(object, &tx),
             Event::Stdio(object) => object.apply(&mut supervisor, tx, &mut reader),
             Event::Exit(event) => {
-                if let ExitEvent::Err(err) = event {
-                    tx.send(Event2::Log(LogEvent::Error(err)));
+                if let ExitEvent::Err(err) = &event {
+                    tx.send(Event2::Log(LogEvent::Error(err.clone())));
                 }
-                break;
+                break event;
             },
         }
-    }
+    };
 
     supervisor.wait()?;
 
-    Ok(())
+    Ok(exit)
 }
 
-pub fn run_worker(geoip: GeoIP, asn: AsnDB, psl: String) -> Result<()> {
+pub fn run_worker(geoip: Vec<u8>, asn: Vec<u8>, psl: &str) -> Result<()> {
     let mut reporter = StdioReporter::setup();
     let start = reporter.recv_start()?;
 
-    let psl = Psl::from_str(&psl)
+    let geoip = GeoIP::from_buf(geoip)?;
+    let asn = AsnDB::from_buf(asn)?;
+    let psl = psl.parse::<Psl>()
         .context("Failed to load public suffix list")?;
 
     let environment = Environment {
         verbose: start.verbose,
+        keyring: start.keyring,
         dns_config: start.dns_config,
+        proxy: start.proxy,
+        options: start.options,
         psl,
         geoip,
         asn,

src/engine/mod.rs

@@ -1,22 +1,24 @@
-use errors::*;
+use crate::errors::*;
 
-use geoip::{GeoIP, AsnDB};
-use json::LuaJsonValue;
+use crate::geoip::{GeoIP, AsnDB};
+use crate::json::LuaJsonValue;
+use crate::keyring::KeyRingEntry;
 use serde_json;
 use std::fs;
 use std::fmt::Debug;
 use std::path::PathBuf;
 use std::collections::HashMap;
+use std::net::SocketAddr;
 use std::sync::{Arc, Mutex};
-use engine::ctx::Script;
+use crate::engine::ctx::Script;
 use sn0int_common::ModuleID;
 use sn0int_common::metadata::{Metadata, Source};
 use chrootable_https::dns::Resolver;
-use psl::Psl;
-use paths;
+use crate::psl::Psl;
+use crate::paths;
 use std::cmp::Ordering;
-use term;
-use worker::{self, Event};
+use crate::term;
+use crate::worker::{self, Event};
 
 pub mod ctx;
 pub mod isolation;
@@ -26,11 +28,14 @@ pub mod structs;
 /// Data that is passed to every script
 #[derive(Debug)]
 pub struct Environment {
-  pub verbose: u64,
-  pub dns_config: Resolver,
-  pub psl: Psl,
-  pub geoip: GeoIP,
-  pub asn: AsnDB,
+    pub verbose: u64,
+    pub keyring: Vec<KeyRingEntry>,
+    pub dns_config: Resolver,
+    pub proxy: Option<SocketAddr>,
+    pub options: HashMap<String, String>,
+    pub psl: Psl,
+    pub geoip: GeoIP,
+    pub asn: AsnDB,
 }
 
 #[derive(Debug)]
@@ -55,7 +60,8 @@ impl Engine {
 
     pub fn reload_modules(&mut self) -> Result<usize> {
         let modules = worker::spawn_fn("Loading modules", || {
-            self.reload_modules_quiet()?;
+            self.reload_modules_quiet()
+                .context("Failed to load modules")?;
             Ok(self.list().len())
         }, true)?;
         term::info(&format!("Loaded {} modules", modules));
@@ -67,32 +73,45 @@ impl Engine {
 
         for author in fs::read_dir(&self.path)? {
             let author = author?;
+
+            if !author.path().is_dir() {
+                continue;
+            }
+
             let author_name = author.file_name()
                                     .into_string()
                                     .map_err(|_| format_err!("Failed to decode filename"))?;
             for module in fs::read_dir(&author.path())? {
                 let module = module?;
-                let mut module_name = module.file_name()
+                let module_name = module.file_name()
                                         .into_string()
                                         .map_err(|_| format_err!("Failed to decode filename"))?;
 
-                if module_name.ends_with(".lua") {
-                    module_name = module_name[..(module_name.len() - 4)].to_string();
+                // find last instance of .lua in filename, if any
+                let (module_name, ext) = if let Some(idx) = module_name.rfind(".lua") {
+                    module_name.split_at(idx)
                 } else {
                     // TODO: show warning
                     continue;
+                };
+
+                // if .lua is not at the end, skip
+                if ext != ".lua" {
+                    // TODO: show warning
+                    continue;
                 }
 
                 let path = module.path();
+                let module_name = module_name.to_string();
                 let module = Module::load(&path, &author_name, &module_name)
                     .context(format!("Failed to parse {}/{}", author_name, module_name))?;
 
-                for key in &[module_name.clone(), format!("{}/{}", author_name, module_name)] {
-                    if !self.modules.contains_key(key) {
+                for key in &[&module_name, &format!("{}/{}", author_name, module_name)] {
+                    if !self.modules.contains_key(*key) {
                         self.modules.insert(key.to_string(), Vec::new());
                     }
 
-                    let vec = self.modules.get_mut(key).unwrap();
+                    let vec = self.modules.get_mut(*key).unwrap();
                     vec.push(module.clone());
                 }
             }
@@ -137,6 +156,7 @@ pub struct Module {
     description: String,
     version: String,
     source: Option<Source>,
+    keyring_access: Vec<String>,
     script: Script,
 }
 
@@ -157,6 +177,7 @@ impl Module {
             description: metadata.description,
             version: metadata.version,
             source: metadata.source,
+            keyring_access: metadata.keyring_access,
             script,
         })
     }
@@ -188,6 +209,10 @@ impl Module {
         &self.source
     }
 
+    pub fn keyring_access(&self) -> &[String] {
+        &self.keyring_access
+    }
+
     pub fn run(&self, env: Environment, reporter: Arc<Mutex<Box<Reporter>>>, arg: LuaJsonValue) -> Result<()> {
         debug!("Executing lua script {}", self.canonical());
         self.script.run(env, reporter, arg.into())

src/engine/structs.rs

@@ -1,7 +1,7 @@
-use errors::*;
-use hlua::{AnyHashableLuaValue, AnyLuaValue};
+use crate::errors::*;
+use crate::hlua::{AnyHashableLuaValue, AnyLuaValue};
 use std::collections::{self, HashMap};
-use json::LuaJsonValue;
+use crate::json::LuaJsonValue;
 use serde;
 use serde_json;
 

src/fmt.rs

@@ -0,0 +1,206 @@
+use std::fmt;
+pub use std::fmt::Write;
+pub use std::fmt::{Result, Formatter, Display, Debug};
+
+
+pub mod colors {
+    use std::fmt::{self, Write};
+
+    pub trait Color {
+        fn color<W: Write>(w: &mut W) -> fmt::Result;
+        fn display<W: Write, D: fmt::Display>(w: &mut W, v: D) -> fmt::Result;
+        fn debug<W: Write, D: fmt::Debug>(w: &mut W, v: D) -> fmt::Result;
+    }
+
+    pub struct Red;
+
+    impl Color for Red {
+        #[inline]
+        fn color<W: Write>(w: &mut W) -> fmt::Result {
+            write!(w, "\x1b[31m")
+        }
+
+        #[inline]
+        fn display<W: Write, D: fmt::Display>(w: &mut W, v: D) -> fmt::Result {
+            write!(w, "\x1b[31m{}\x1b[0m", v)
+        }
+
+        #[inline]
+        fn debug<W: Write, D: fmt::Debug>(w: &mut W, v: D) -> fmt::Result {
+            write!(w, "\x1b[31m{:?}\x1b[0m", v)
+        }
+    }
+
+    pub struct Green;
+
+    impl Color for Green {
+        #[inline]
+        fn color<W: Write>(w: &mut W) -> fmt::Result {
+            write!(w, "\x1b[32m")
+        }
+
+        #[inline]
+        fn display<W: Write, D: fmt::Display>(w: &mut W, v: D) -> fmt::Result {
+            write!(w, "\x1b[32m{}\x1b[0m", v)
+        }
+
+        #[inline]
+        fn debug<W: Write, D: fmt::Debug>(w: &mut W, v: D) -> fmt::Result {
+            write!(w, "\x1b[32m{:?}\x1b[0m", v)
+        }
+    }
+
+    pub struct Yellow;
+
+    impl Color for Yellow {
+        #[inline]
+        fn color<W: Write>(w: &mut W) -> fmt::Result {
+            write!(w, "\x1b[33m")
+        }
+
+        #[inline]
+        fn display<W: Write, D: fmt::Display>(w: &mut W, v: D) -> fmt::Result {
+            write!(w, "\x1b[33m{}\x1b[0m", v)
+        }
+
+        #[inline]
+        fn debug<W: Write, D: fmt::Debug>(w: &mut W, v: D) -> fmt::Result {
+            write!(w, "\x1b[33m{:?}\x1b[0m", v)
+        }
+    }
+}
+use self::colors::*;
+
+
+pub struct DetailFormatter<'a, 'b> {
+    w: &'a mut fmt::Formatter<'b>,
+    scoped: bool,
+    in_group: bool,
+    fresh_group: bool,
+}
+
+impl<'a, 'b> DetailFormatter<'a, 'b> {
+    pub fn new(w: &'a mut fmt::Formatter<'b>, scoped: bool) -> DetailFormatter<'a, 'b> {
+        DetailFormatter {
+            w,
+            scoped,
+            in_group: false,
+            fresh_group: true,
+        }
+    }
+
+    #[inline]
+    pub fn start_group(&mut self) {
+        self.in_group = true;
+        self.fresh_group = true;
+    }
+
+    #[inline]
+    pub fn end_group(&mut self) -> fmt::Result {
+        self.in_group = false;
+        if !self.fresh_group {
+            write!(self, "]")?
+        }
+        Ok(())
+    }
+
+    #[inline]
+    pub fn start(&mut self) -> fmt::Result {
+        if !self.scoped {
+            write!(self, "\x1b[90m")
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    pub fn end(&mut self) -> fmt::Result {
+        if !self.scoped {
+            write!(self, "\x1b[0m")
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    pub fn id<D: fmt::Display>(&mut self, v: D) -> fmt::Result {
+        if self.scoped {
+            write!(self, "\x1b[32m#{}\x1b[0m, ", v)
+        } else {
+            write!(self, "#{}, ", v)
+        }
+    }
+
+    #[inline]
+    pub fn color<C: Color>(&mut self) -> fmt::Result {
+        if self.scoped {
+            C::color(self)
+        } else {
+            Ok(())
+        }
+    }
+
+    #[inline]
+    pub fn display<C: Color, D: fmt::Display>(&mut self, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        if self.scoped {
+            C::display(self, v)
+        } else {
+            write!(self, "{}", v)
+        }
+    }
+
+    #[inline]
+    pub fn debug<C: Color, D: fmt::Debug>(&mut self, v: D) -> fmt::Result {
+        self.push_into_group()?;
+        if self.scoped {
+            C::debug(self, v)
+        } else {
+            write!(self, "{:?}", v)
+        }
+    }
+
+    pub fn opt_debug<C: Color, D: fmt::Debug>(&mut self, v: &Option<D>) -> fmt::Result {
+        if let Some(v) = &v {
+            self.debug::<C, _>(v)?;
+        }
+        Ok(())
+    }
+
+    #[inline]
+    pub fn clear(&mut self) -> fmt::Result {
+        if self.scoped {
+            write!(self, "\x1b[0m")
+        } else {
+            Ok(())
+        }
+    }
+
+    fn push_into_group(&mut self) -> fmt::Result {
+        if self.in_group {
+            if !self.fresh_group {
+                write!(self, " / ")?;
+            } else {
+                write!(self, " [")?;
+                self.fresh_group = false;
+            }
+        }
+        Ok(())
+    }
+
+    #[inline]
+    pub fn child<D: fmt::Display>(&mut self, c: D) -> fmt::Result {
+        if self.scoped {
+            // if child is unscoped, draw as grey as well
+            write!(self, "\n\t\x1b[33m{}\x1b[0m", c)
+        } else {
+            write!(self, "\n\t\x1b[90m{}\x1b[0m", c)
+        }
+    }
+}
+
+impl<'a, 'b> fmt::Write for DetailFormatter<'a, 'b> {
+    fn write_str(&mut self, s: &str) -> fmt::Result {
+        self.w.write_str(s)
+    }
+}

src/geoip/mod.rs

@@ -1,13 +1,13 @@
-use errors::*;
-use archive;
 use chrootable_https::Client;
+use crate::archive;
+use crate::errors::*;
+use crate::paths;
+use crate::worker;
 use maxminddb::{self, geoip2};
 use std::fmt;
-use std::fs::File;
+use std::fs::{self, File};
 use std::net::IpAddr;
 use std::path::Path;
-use paths;
-use worker;
 
 pub static GEOIP_CITY_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz";
 pub static GEOIP_ASN_URL: &str = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz";
@@ -22,7 +22,7 @@ pub trait Maxmind: Sized {
 
     fn archive_url() -> &'static str;
 
-    fn new(reader: maxminddb::Reader) -> Self;
+    fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self;
 
     // TODO: refactor this to return Path
     fn cache_path() -> Result<String> {
@@ -44,12 +44,23 @@ pub trait Maxmind: Sized {
         Ok(path.to_string())
     }
 
-    fn open(path: &str) -> Result<Self> {
-        let reader = maxminddb::Reader::open(path)
-            .context("Failed to open geoip database")?;
+    fn from_buf(buf: Vec<u8>) -> Result<Self> {
+        let reader = maxminddb::Reader::from_source(buf)
+            .context("Failed to read geoip database")?;
         Ok(Self::new(reader))
     }
 
+    fn open(path: &str) -> Result<Self> {
+        let buf = fs::read(path)?;
+        Self::from_buf(buf)
+    }
+
+    fn open_into_buf() -> Result<Vec<u8>> {
+        let path = Self::cache_path()?;
+        let buf = fs::read(path)?;
+        Ok(buf)
+    }
+
     fn open_or_download() -> Result<Self> {
         let path = Self::cache_path()?;
 
@@ -65,7 +76,8 @@ pub trait Maxmind: Sized {
     fn download<P: AsRef<Path>>(path: P, filter: &str, url: &str) -> Result<()> {
         debug!("Downloading {:?}...", url);
         let client = Client::with_system_resolver()?;
-        let resp = client.get(url)?;
+        let resp = client.get(url)
+            .wait_for_response()?;
         debug!("Downloaded {} bytes", resp.body.len());
         archive::extract(&mut &resp.body[..], filter, path)?;
         Ok(())
@@ -73,7 +85,7 @@ pub trait Maxmind: Sized {
 }
 
 pub struct GeoIP {
-    reader: maxminddb::Reader,
+    reader: maxminddb::Reader<Vec<u8>>,
 }
 
 impl fmt::Debug for GeoIP {
@@ -94,7 +106,7 @@ impl Maxmind for GeoIP {
     }
 
     #[inline]
-    fn new(reader: maxminddb::Reader) -> Self {
+    fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         GeoIP {
             reader
         }
@@ -110,7 +122,7 @@ impl GeoIP {
 }
 
 pub struct AsnDB {
-    reader: maxminddb::Reader,
+    reader: maxminddb::Reader<Vec<u8>>,
 }
 
 impl fmt::Debug for AsnDB {
@@ -131,7 +143,7 @@ impl Maxmind for AsnDB {
     }
 
     #[inline]
-    fn new(reader: maxminddb::Reader) -> Self {
+    fn new(reader: maxminddb::Reader<Vec<u8>>) -> Self {
         AsnDB {
             reader
         }

src/geoip/models.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 use maxminddb::geoip2;
 use std::collections::BTreeMap;
 

src/html.rs

@@ -1,10 +1,10 @@
-use errors::Result;
+use crate::errors::Result;
 
 use kuchiki;
 use kuchiki::traits::TendrilSink;
 use std::collections::HashMap;
-use hlua::AnyLuaValue;
-use engine::structs::LuaMap;
+use crate::hlua::AnyLuaValue;
+use crate::engine::structs::LuaMap;
 
 
 #[derive(Debug, PartialEq)]
@@ -66,7 +66,7 @@ pub fn html_select_list(html: &str, selector: &str) -> Result<Vec<Element>> {
     let doc = kuchiki::parse_html().one(html);
 
     match doc.select(selector) {
-        Ok(x) => Ok(x.into_iter().map(|x| transform_element(&x)).collect()),
+        Ok(x) => Ok(x.map(|x| transform_element(&x)).collect()),
         Err(_) => bail!("css selector failed"),
     }
 }
@@ -108,7 +108,9 @@ mod tests {
         let elems = html_select(r#"<html><div id="yey">content</div></html>"#, "#yey").unwrap();
         assert_eq!(elems,
             Element {
-                attrs: vec![(String::from("id"), String::from("yey"))].into_iter().collect(),
+                attrs: hashmap!{
+                    "id".into() => "yey".into(),
+                },
                 text: "content".into(),
                 html: r#"<div id="yey">content</div>"#.into(),
             }
@@ -120,7 +122,9 @@ mod tests {
         let elems = html_select_list(r#"<html><div id="yey">content</div></html>"#, "#yey").unwrap();
         assert_eq!(elems, vec![
             Element {
-                attrs: vec![(String::from("id"), String::from("yey"))].into_iter().collect(),
+                attrs: hashmap!{
+                    "id".into() => "yey".into(),
+                },
                 text: "content".into(),
                 html: r#"<div id="yey">content</div>"#.into(),
             }

src/json.rs

@@ -1,8 +1,8 @@
-use errors::*;
+use crate::errors::*;
 
 use std::iter::FromIterator;
 use std::collections::HashMap;
-use hlua::AnyLuaValue;
+use crate::hlua::AnyLuaValue;
 use serde_json::{self, Deserializer, Value, Number, Map};
 
 

src/keyring.rs

@@ -0,0 +1,228 @@
+use crate::errors::*;
+
+use crate::engine::Module;
+use crate::hlua::AnyLuaValue;
+use crate::json::LuaJsonValue;
+use crate::paths;
+use std::collections::{HashMap, HashSet};
+use std::fs;
+use std::str::FromStr;
+use std::path::PathBuf;
+use sn0int_common::ModuleID;
+
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct KeyName {
+    pub namespace: String,
+    pub name: String,
+}
+
+impl KeyName {
+    pub fn new<I: Into<String>, J: Into<String>>(namespace: I, name: J) -> KeyName {
+        KeyName {
+            namespace: namespace.into(),
+            name: name.into(),
+        }
+    }
+
+    pub fn for_each(k: &str, v: &HashMap<String, Option<String>>) -> Vec<KeyName> {
+        v.iter()
+            .map(move |(x, _)| KeyName::new(k, x.as_str()))
+            .collect()
+    }
+}
+
+impl FromStr for KeyName {
+    type Err = Error;
+
+    fn from_str(x: &str) -> Result<KeyName> {
+        if let Some(idx) = x.find(':') {
+            let (namespace, name) = x.split_at(idx);
+            let namespace = namespace.to_string();
+            let name = name[1..].to_string();
+
+            if namespace.is_empty() {
+                bail!("Namespace can not be empty");
+            }
+
+            if name.is_empty() {
+                bail!("Name can not be empty");
+            }
+
+            Ok(KeyName {
+                namespace,
+                name,
+            })
+        } else {
+            bail!("Missing namespace")
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct KeyRing {
+    keys: HashMap<String, HashMap<String, Option<String>>>,
+    grants: HashMap<String, HashSet<ModuleID>>,
+}
+
+impl KeyRing {
+    pub fn path() -> Result<PathBuf> {
+        let path = paths::data_dir()?;
+        let path = path.join("keyring.json");
+        Ok(path)
+    }
+
+    pub fn init() -> Result<KeyRing> {
+        let path = Self::path()?;
+
+        if path.exists() {
+            Self::load(&path)
+        } else {
+            Ok(KeyRing {
+                keys: HashMap::new(),
+                grants: HashMap::new(),
+            })
+        }
+    }
+
+    pub fn load(path: &PathBuf) -> Result<KeyRing> {
+        let buf = fs::read(&path)?;
+        serde_json::from_slice(&buf)
+            .map_err(Error::from)
+    }
+
+    pub fn save(&self) -> Result<()> {
+        let path = Self::path()?;
+        let buf = serde_json::to_string(&self)?;
+        fs::write(&path, buf)?;
+        Ok(())
+    }
+
+    pub fn insert(&mut self, key: KeyName, secret: Option<String>) -> Result<()> {
+        // get the namespace or create a new one
+        let mut x = self.keys.remove(&key.namespace)
+            .unwrap_or_else(HashMap::new);
+        // insert key into namespace
+        x.insert(key.name, secret);
+        // add namespace backinto keyring
+        self.keys.insert(key.namespace, x);
+        // save keyring
+        self.save()
+    }
+
+    pub fn delete(&mut self, key: KeyName) -> Result<()> {
+        if let Some(mut x) = self.keys.remove(&key.namespace) {
+            // remove the key we want to delete
+            x.remove(&key.name);
+
+            // if there are still keys left in the namespace
+            if !x.is_empty() {
+                self.keys.insert(key.namespace, x);
+            }
+
+            // save keyring
+            self.save()
+        } else {
+            Ok(())
+        }
+    }
+
+    pub fn list(&self) -> Vec<KeyName> {
+        self.keys.iter()
+            .flat_map(|(k, v)| KeyName::for_each(k, v))
+            .collect()
+    }
+
+    pub fn list_for(&self, namespace: &str) -> Vec<KeyName> {
+        self.keys.iter()
+            .filter(|(k, _)| k.as_str() == namespace)
+            .flat_map(|(k, v)| KeyName::for_each(k, v))
+            .collect()
+    }
+
+    pub fn get(&self, key: &KeyName) -> Option<KeyRingEntry> {
+        let x = self.keys.get(&key.namespace)?;
+        let secret_key = x.get(&key.name)?;
+
+        Some(KeyRingEntry {
+            namespace: key.namespace.to_owned(),
+            access_key: key.name.to_owned(),
+            secret_key: secret_key.to_owned(),
+        })
+    }
+
+    pub fn get_all_for(&self, namespace: &str) -> Vec<KeyRingEntry> {
+        self.list_for(namespace)
+            .into_iter()
+            .flat_map(|x| self.get(&x))
+            .collect()
+    }
+
+    pub fn unauthorized_namespaces<'a>(&self, module: &'a Module) -> Vec<&'a String> {
+        module.keyring_access().iter()
+            .filter(|namespace| !self.is_access_granted(&module, &namespace))
+            .collect()
+    }
+
+    pub fn grant_access(&mut self, module: &Module, namespace: String) {
+        let mut grants = self.grants.remove(&namespace)
+            .unwrap_or_else(HashSet::new);
+        grants.insert(module.id());
+        self.grants.insert(namespace, grants);
+    }
+
+    pub fn is_access_granted(&self, module: &Module, namespace: &str) -> bool {
+        if let Some(grants) = self.grants.get(namespace) {
+            grants.contains(&module.id())
+        } else {
+            false
+        }
+    }
+
+    pub fn request_keys(&self, module: &Module) -> Vec<KeyRingEntry> {
+        // TODO: we probably want to randomize the order
+        module.keyring_access().iter()
+            .filter(|namespace| self.is_access_granted(&module, &namespace))
+            .flat_map(|namespace| self.list_for(namespace))
+            .flat_map(|x| self.get(&x))
+            .collect()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct KeyRingEntry {
+    pub namespace: String,
+    pub access_key: String,
+    pub secret_key: Option<String>,
+}
+
+impl KeyRingEntry {
+    pub fn to_lua(&self) -> Result<AnyLuaValue> {
+        let v = serde_json::to_value(&self)?;
+        let v = LuaJsonValue::from(v).into();
+        Ok(v)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_valid_keyname() {
+        let x = KeyName::from_str("a:b").unwrap();
+        assert_eq!(x, KeyName {
+            namespace: "a".into(),
+            name: "b".into(),
+        });
+    }
+
+    #[test]
+    fn test_invalid_keyname() {
+        assert!(KeyName::from_str("a:").is_err());
+        assert!(KeyName::from_str(":a").is_err());
+        assert!(KeyName::from_str(":").is_err());
+        assert!(KeyName::from_str("a").is_err());
+        assert!(KeyName::from_str("").is_err());
+    }
+}

src/lib.rs

@@ -1,44 +1,11 @@
 #![allow(proc_macro_derive_resolution_fallback)]
 #![warn(unused_extern_crates)]
-extern crate sn0int_common;
-extern crate rustyline;
-extern crate rand;
-extern crate colored;
 #[macro_use] extern crate failure;
 #[macro_use] extern crate maplit;
-extern crate shellwords;
-extern crate dirs;
-extern crate publicsuffix;
-extern crate chrootable_https;
-extern crate url;
-#[cfg(target_os = "linux")]
-extern crate nix;
-#[cfg(target_os = "linux")]
-extern crate caps;
-#[cfg(target_os = "linux")]
-extern crate syscallz;
+use url;
 #[cfg(target_os = "openbsd")]
 #[macro_use] extern crate pledge;
-#[cfg(target_os = "openbsd")]
-extern crate unveil;
-extern crate hlua_badtouch as hlua;
-extern crate base64;
-extern crate kuchiki;
-extern crate ctrlc;
-extern crate opener;
-extern crate separator;
-extern crate sloppy_rfc4880;
-extern crate regex;
-extern crate toml;
-extern crate maxminddb;
-extern crate tar;
-extern crate libflate;
-extern crate threadpool;
-extern crate x509_parser;
-extern crate der_parser;
-extern crate serde;
-extern crate serde_json;
-extern crate serde_urlencoded;
+use hlua_badtouch as hlua;
 #[macro_use] extern crate serde_derive;
 #[macro_use] extern crate log;
 #[macro_use] extern crate structopt;
@@ -59,21 +26,24 @@ pub mod crt;
 pub mod db;
 pub mod errors;
 pub mod engine;
+pub mod fmt;
 pub mod geoip;
 pub mod html;
 pub mod json;
+pub mod keyring;
 pub mod migrations;
 pub mod models;
 pub mod paths;
+pub mod psl;
+pub mod options;
+pub mod registry;
+pub mod runtime;
 pub mod ser;
 pub mod sandbox;
 pub mod schema;
 pub mod shell;
-pub mod registry;
-pub mod runtime;
 pub mod term;
+pub mod utils;
 pub mod web;
 pub mod worker;
 pub mod workspaces;
-pub mod psl;
-pub mod utils;