Release v0.8.1

Add a dns-cache

.dockerignore

@@ -1,6 +1,9 @@
 target
 Dockerfile
 .dockerignore
+docker
+docs
+ci
 .git
 .gitignore
 *.sw[op]

.travis.yml

@@ -47,3 +47,11 @@ script:
 - df -h
 - ci/run.sh $BUILD_MODE
 - df -h
+
+notifications:
+  irc:
+    channels:
+    - "ircs://irc.hackint.org:6697/#sn0int"
+    #on_success: change # default: always
+    #on_failure: always # default: always
+    use_notice: true

CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# How to contribute
+
+To contribute to sn0int, clone the repository and make sure both the build and
+tests pass for you:
+
+    git clone https://github.com/kpcyrd/sn0int.git
+    cd sn0int
+    # build the project
+    cargo build
+    # run regular tests
+    cargo test
+    # run tests depending on the network
+    # these might fail if a service is down
+    cargo test -- --ignored
+
+The project is loosely structured into a few folders:
+
+- `src/models/` - database models
+- `src/runtime/` - the stdlib that's exposed to lua
+- `src/engine/` - code related to lua
+- `src/sandbox/` - code related to sandboxing
+- `src/cmd/` - cli commands
+- `src/` - misc modules
+
+After you're done, make sure the build completes without any warnings and both
+tests pass successfully:
+
+    cargo test
+    cargo test -- --ignored
+
+If you want to introduce a new feature feel free to open an issue first to make
+sure your feature is a good fit for the project before implementing it.

Cargo.toml

@@ -1,12 +1,13 @@
 [package]
 name = "sn0int"
-version = "0.4.0"
-description = "OSINT framework and package manager"
+version = "0.8.1"
+description = "Semi-automatic OSINT framework and package manager"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
 categories = ["command-line-utilities"]
 readme = "README.md"
+edition = "2018"
 
 [badges]
 travis-ci = { repository = "kpcyrd/sn0int" }
@@ -16,25 +17,25 @@ members = ["sn0int-registry/sn0int-common",
            "sn0int-registry"]
 
 [dependencies]
-sn0int-common = { version="0.3.0", path="sn0int-registry/sn0int-common" }
-rustyline = "2"
+sn0int-common = { version="0.4.0", path="sn0int-registry/sn0int-common" }
+rustyline = "3"
 log = "0.4"
-env_logger = "0.5"
+env_logger = "0.6"
 hlua-badtouch = "0.4"
 structopt = "0.2"
 failure = "0.1"
-rand = "0.5"
+rand = "0.6"
 colored = "1.6"
 lazy_static = "1.0"
 shellwords = "1.0"
 publicsuffix = { version="1.5", default-features=false }
-diesel = { version = "1.0.0", features = ["sqlite"] }
+diesel = { version = "1.0.0", features = ["sqlite", "chrono"] }
 diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
+chrono = { version = "0.4", features = ["serde"] }
 dirs = "1.0"
 url = "1.7"
-nix = "0.11"
-chrootable-https = "0.3.4"
-trust-dns-proto = "0.5.0"
+#chrootable-https = { path = "../chrootable-https" }
+chrootable-https = "0.8"
 base64 = "0.10"
 kuchiki = "0.7.2"
 serde_urlencoded = "0.5"
@@ -44,18 +45,27 @@ serde_json = "1.0"
 crossbeam-channel = "0.3"
 ctrlc = "3.1"
 opener = "0.3.0"
-separator = "0.3.1"
+separator = "0.4"
 maplit = "1.0.1"
 sloppy-rfc4880 = "0.1.2"
 regex = "1.0"
 toml = "0.4"
-maxminddb = "0.10.0"
+maxminddb = "0.12"
 tar = "0.4.17"
 libflate = "0.1.14"
+threadpool = "1.7"
+x509-parser = "0.4.0"
+der-parser = "1.1.0"
+nom = "4.1.1"
 
 [target.'cfg(target_os="linux")'.dependencies]
 caps = "0.3"
-syscallz = "0.7"
+syscallz = "0.8"
+nix = "0.12"
+
+[target.'cfg(target_os="openbsd")'.dependencies]
+pledge = "0.3.1"
+unveil = "0.2.0"
 
 [dev-dependencies]
 boxxy = "0.8"

Makefile

@@ -14,3 +14,8 @@ test:
 	(cd sn0int-registry; cargo test)
 	cargo test
 	cargo test -- --ignored
+
+update:
+	get-oui -v -u http://standards-oui.ieee.org/oui/oui.txt -f data/ieee-oui.txt
+	get-iab -v -u http://standards-oui.ieee.org/iab/iab.txt -f data/ieee-iab.txt
+	rm -f data/ieee-*.txt.bak

README.md

@@ -1,29 +1,112 @@
-# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates]
+# sn0int [![Build Status][travis-img]][travis] [![Crates.io][crates-img]][crates] [![Documentation Status][docs-img]][docs]
 
 [travis-img]:   https://travis-ci.org/kpcyrd/sn0int.svg?branch=master
 [travis]:       https://travis-ci.org/kpcyrd/sn0int
 [crates-img]:   https://img.shields.io/crates/v/sn0int.svg
 [crates]:       https://crates.io/crates/sn0int
+[docs-img]:     https://readthedocs.org/projects/sn0int/badge/?version=latest
+[docs]:         https://sn0int.readthedocs.io/en/latest/?badge=latest
 
-sn0int is an OSINT framework and package manager. It's purpose is
-semi-automatically processing public information to enumerate attack surface.
-sn0int itself is only providing an engine that can be extended with scripts.
+sn0int is a semi-automatic OSINT framework and package manager. It was built
+for IT security professionals and bug hunters to gather intelligence about a
+given target or about yourself. sn0int is enumerating attack surface by
+semi-automatically processing public information and mapping the results in a
+unified format for followup investigations.
 
-sn0int is heavily inspired by recon-ng, but takes a few different design
-approaches. We've tried to correct some limitations in the database design and
-also addressed the modularity problem:
+Among other things, sn0int is currently able to:
 
-Instead of downloading and reviewing python scripts that have full access to
-your system, sn0int is executing modules in a lua sandbox. Modules can be
-published to the sn0int registry and then installed by users. This means that
-you don't have to send pull requests to sn0int to add a module and updates can
-be shipped much faster.
+- Harvest subdomains from certificate transparency logs
+- Harvest subdomains from various passive dns logs
+- Sift through subdomain results for publicly accessible websites
+- Harvest emails from pgp keyservers
+- Enrich ip addresses with ASN and geoip info
+- Harvest subdomains from the wayback machine
+- Gather information about phonenumbers
+- Bruteforce interesting urls
 
-## Installation
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
 
-- Archlinux: `yaourt -S sn0int`
-- Alpine: `apk add --no-cache sqlite-dev libseccomp-dev cargo` + build from source
-- Debian: `apt install libsqlite3-dev libseccomp-dev` + build from source
+Join us on IRC: [irc.hackint.org:6697/#sn0int](https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int)
+
+[![asciicast](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB.svg)](https://asciinema.org/a/shZ3TVY1o0opGFln3Oi2DAMCB)
+
+## Getting started
+
+- [Installation](https://sn0int.readthedocs.io/en/latest/install.html)
+  - [Archlinux](https://sn0int.readthedocs.io/en/latest/install.html#archlinux)
+  - [Debian/Ubuntu/Kali](https://sn0int.readthedocs.io/en/latest/install.html#debian-ubuntu-kali)
+  - [Alpine](https://sn0int.readthedocs.io/en/latest/install.html#alpine)
+  - [Docker](https://sn0int.readthedocs.io/en/latest/install.html#docker)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/install.html#openbsd)
+  - [Mac OSX](https://sn0int.readthedocs.io/en/latest/install.html#mac-osx)
+  - [Windows](https://sn0int.readthedocs.io/en/latest/install.html#windows)
+- [Running your first investigation](https://sn0int.readthedocs.io/en/latest/usage.html)
+  - [Installing the default modules](https://sn0int.readthedocs.io/en/latest/usage.html#installing-the-default-modules)
+  - [Adding something to scope](https://sn0int.readthedocs.io/en/latest/usage.html#adding-something-to-scope)
+  - [Running a module](https://sn0int.readthedocs.io/en/latest/usage.html#running-a-module)
+  - [Running followup modules on the results](https://sn0int.readthedocs.io/en/latest/usage.html#running-followup-modules-on-the-results)
+  - [Unscoping entities](https://sn0int.readthedocs.io/en/latest/usage.html#unscoping-entities)
+- [Scripting](https://sn0int.readthedocs.io/en/latest/scripting.html)
+  - [Write your first module](https://sn0int.readthedocs.io/en/latest/scripting.html#write-your-first-module)
+  - [Publish your module](https://sn0int.readthedocs.io/en/latest/scripting.html#publish-your-module)
+- [Database](https://sn0int.readthedocs.io/en/latest/database.html)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/database.html#db-add)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/database.html#db-update)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/database.html#db-select)
+- [Keyring](https://sn0int.readthedocs.io/en/latest/keyring.html)
+  - [Managing the keyring](https://sn0int.readthedocs.io/en/latest/keyring.html#managing-the-keyring)
+  - [Using access keys in scripts](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-in-scripts)
+  - [Using access keys as source argument](https://sn0int.readthedocs.io/en/latest/keyring.html#using-access-keys-as-source-argument)
+- [Configuration](https://sn0int.readthedocs.io/en/latest/config.html)
+  - [Configuring a proxy](https://sn0int.readthedocs.io/en/latest/config.html#configuring-a-proxy)
+- [Sandbox](https://sn0int.readthedocs.io/en/latest/sandbox.html)
+  - [Linux](https://sn0int.readthedocs.io/en/latest/sandbox.html#linux)
+  - [OpenBSD](https://sn0int.readthedocs.io/en/latest/sandbox.html#openbsd)
+  - [IPC Protocol](https://sn0int.readthedocs.io/en/latest/sandbox.html#ipc-protocol)
+  - [Limitations](https://sn0int.readthedocs.io/en/latest/sandbox.html#limitations)
+- [Function reference](https://sn0int.readthedocs.io/en/latest/reference.html)
+  - [clear_err](https://sn0int.readthedocs.io/en/latest/reference.html#clear-err)
+  - [db_add](https://sn0int.readthedocs.io/en/latest/reference.html#db-add)
+  - [db_select](https://sn0int.readthedocs.io/en/latest/reference.html#db-select)
+  - [db_update](https://sn0int.readthedocs.io/en/latest/reference.html#db-update)
+  - [dns](https://sn0int.readthedocs.io/en/latest/reference.html#dns)
+  - [error](https://sn0int.readthedocs.io/en/latest/reference.html#error)
+  - [asn_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#asn-lookup)
+  - [geoip_lookup](https://sn0int.readthedocs.io/en/latest/reference.html#geoip-lookup)
+  - [html_select](https://sn0int.readthedocs.io/en/latest/reference.html#html-select)
+  - [html_select_list](https://sn0int.readthedocs.io/en/latest/reference.html#html-select-list)
+  - [http_mksession](https://sn0int.readthedocs.io/en/latest/reference.html#http-mksession)
+  - [http_request](https://sn0int.readthedocs.io/en/latest/reference.html#http-request)
+  - [http_send](https://sn0int.readthedocs.io/en/latest/reference.html#http-send)
+  - [info](https://sn0int.readthedocs.io/en/latest/reference.html#info)
+  - [json_decode](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode)
+  - [json_decode_stream](https://sn0int.readthedocs.io/en/latest/reference.html#json-decode-stream)
+  - [json_encode](https://sn0int.readthedocs.io/en/latest/reference.html#json-encode)
+  - [keyring](https://sn0int.readthedocs.io/en/latest/reference.html#keyring)
+  - [last_err](https://sn0int.readthedocs.io/en/latest/reference.html#last-err)
+  - [pgp_pubkey](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey)
+  - [pgp_pubkey_armored](https://sn0int.readthedocs.io/en/latest/reference.html#pgp-pubkey-armored)
+  - [print](https://sn0int.readthedocs.io/en/latest/reference.html#print)
+  - [psl_domain_from_dns_name](https://sn0int.readthedocs.io/en/latest/reference.html#psl-domain-from-dns-name)
+  - [regex_find](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find)
+  - [regex_find_all](https://sn0int.readthedocs.io/en/latest/reference.html#regex-find-all)
+  - [sleep](https://sn0int.readthedocs.io/en/latest/reference.html#sleep)
+  - [status](https://sn0int.readthedocs.io/en/latest/reference.html#status)
+  - [stdin_readline](https://sn0int.readthedocs.io/en/latest/reference.html#stdin-readline)
+  - [url_decode](https://sn0int.readthedocs.io/en/latest/reference.html#url-decode)
+  - [url_encode](https://sn0int.readthedocs.io/en/latest/reference.html#url-encode)
+  - [url_escape](https://sn0int.readthedocs.io/en/latest/reference.html#url-escape)
+  - [url_join](https://sn0int.readthedocs.io/en/latest/reference.html#url-join)
+  - [url_parse](https://sn0int.readthedocs.io/en/latest/reference.html#url-parse)
+  - [url_unescape](https://sn0int.readthedocs.io/en/latest/reference.html#url-unescape)
+  - [utf8_decode](https://sn0int.readthedocs.io/en/latest/reference.html#utf8-decode)
+  - [x509_parse_pem](https://sn0int.readthedocs.io/en/latest/reference.html#x509-parse-pem)
 
 ## License
 

ci/bench.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+set -eu
+X=$(mktemp -d)
+cd "$X"
+
+mkdir -p "$X/.cache"
+cp -r "$HOME/.cache/sn0int" "$X/.cache/"
+
+#export CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
+#export RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}"
+export HOME="$X"
+
+cat > 1k.lua <<EOF
+-- Description: Insert 1k random subdomains
+-- Version: 0.1.0
+-- Source: domains
+-- License: GPL-3.0
+
+function run(arg)
+    for i=1,1000 do
+        x = 'foo' .. i .. '.example.com'
+
+        db_add('subdomain', {
+            domain_id=arg['id'],
+            value=x,
+        })
+    end
+end
+EOF
+
+echo '[*] Setting up workspace'
+echo 'add domain example.com' | "$@" > /dev/null
+echo '[*] Running 1k inserts'
+time "$@" run -f ./1k.lua

ci/run.sh

@@ -9,8 +9,8 @@ case "$1" in
         ci/run.sh build
         wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz \
              https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
-        cargo run --example maxmind-dl -- -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
-        cargo run --example maxmind-dl -- -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
+        cargo run --example maxmind -- dl -e GeoLite2-City.tar.gz GeoLite2-City.mmdb GeoLite2-City.mmdb
+        cargo run --example maxmind -- dl -e GeoLite2-ASN.tar.gz GeoLite2-ASN.mmdb GeoLite2-ASN.mmdb
         cargo test --verbose
         cargo test --verbose -- --ignored
         ;;

ci/setup.sh

@@ -5,9 +5,6 @@ case "$1" in
         sudo apt update
         sudo apt install libsqlite3-dev libseccomp-dev
         ;;
-    osx)
-        brew install sqlite3
-        ;;
     windows)
         curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
         7z e sqlite3.zip -y

contrib/docker/Dockerfile.alpine

@@ -0,0 +1,15 @@
+FROM alpine:edge
+RUN apk add --no-cache sqlite-dev libseccomp-dev
+RUN apk add --no-cache --virtual .build-rust rust cargo
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM alpine:edge
+RUN apk add --no-cache libgcc sqlite-libs libseccomp
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/docker/Dockerfile.debian

@@ -0,0 +1,16 @@
+FROM rust
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /usr/src/sn0int
+COPY . .
+RUN cargo build --release --verbose
+RUN strip target/release/sn0int
+
+FROM debian
+RUN apt-get update -q && apt-get install -yq libsqlite3-dev libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=0 /usr/src/sn0int/target/release/sn0int /usr/local/bin/sn0int
+VOLUME ["/data", "/cache"]
+ENV XDG_DATA_HOME=/data \
+    XDG_CACHE_HOME=/cache
+ENTRYPOINT ["sn0int"]

contrib/html-toc2md.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+perl -n -e '/toctree-l(\d).*href="([^"]+)">(.+)<\/a/ && print $1==2?"  ":"", "- [$3](https://sn0int.readthedocs.io/en/latest/$2)\n"' < docs/_build/html/index.html

docs/.gitignore

@@ -0,0 +1 @@
+/_build/

docs/Makefile

@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'sn0int'
+copyright = '2018, kpcyrd'
+author = 'kpcyrd'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = ''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'sn0intdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'sn0int.tex', 'sn0int Documentation',
+     'kpcyrd', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('man', 'sn0int', 'Semi-automatic OSINT framework and package manager',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'sn0int', 'sn0int Documentation',
+     author, 'sn0int', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']

docs/config.rst

@@ -0,0 +1,17 @@
+Configuration
+=============
+
+This file documents the config file at ``~/.config/sn0int.toml``. By default
+this file does not exist and a default configuration is used instead.
+
+Configuring a proxy
+-------------------
+
+To enable a proxy, add the following to your config file::
+
+    [network]
+    proxy = "127.0.0.1:9050"
+
+This forces everything through tor and restricts all other functions that
+depend on the network. For example the ``dns`` function is fully disabled if a
+proxy is configured.

docs/database.rst

@@ -0,0 +1,84 @@
+Database
+========
+
+There are a few things you need to understand how the database works to use it
+efficiently.
+
+The database that is backing sn0int is sqlite, but the api that is exposed to
+the user and scripts is an nosql-ish object store. The query language that is
+exposed to the user is still very similar to sql, except that it lacks a column
+statement::
+
+    select subdomains where value like %.example.com
+    ^      ^          ^           ^    ^
+    |      |          |           |    this value is going to be quoted automatically
+    |      |          |           |
+    |      |          |           this triggers automatic quoting
+    |      |          |
+    |      |          apply a filter, this translates to sql quite literally
+    |      |
+    |      the entity we want to select is a subdomain.
+    |      this affects the table and the deserializer
+    |
+    select entities
+
+This is how almost all user facing functions work that operate on the database.
+The functions that are available for scripting are a bit more object based and
+described below.
+
+db_add
+------
+
+This operation is somewhat straight forward. It adds an entity to the
+database:
+
+.. code-block:: lua
+
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+
+If this entity conflicts with an entity that already exists, an upsert is
+triggered and an db_update is performed instead.
+
+.. note::
+   This function may return ``nil`` if the entity already exists, but has been
+   removed from scope with ``noscope``. Everytime you use ``db_add`` you need
+   to make sure that the ID that has been returned is not ``nil``.
+
+db_update
+---------
+
+Update some mutable fields of an entity:
+
+.. code-block:: lua
+
+    db_update('ipaddr', arg, {
+        asn=lookup['asn'],
+        as_org=lookup['as_org'],
+    })
+
+The first parameter is usually the same arg that your script was called with.
+Usually you can use db_add instead of db_update due to the upsert feature, but
+db_update is still slightly faster.
+
+.. note::
+   Some fields are immutable and can not be updated.
+
+db_select
+---------
+
+This function is used to check if something is in scope. If the entity has been
+added to the database and has not been removed from scope, this function
+returns that entities id. This is somewhat similar to ``db_add``, except that
+``db_select`` never adds anything to the database.
+
+.. code-block:: lua
+
+    domain_id = db_select('domain', 'example.com')
+    if domain_id ~= nil then
+        -- do something
+    end
+
+This function only accepts a string instead of a lua table. This string is used
+to filter on the ``value`` column.

docs/index.rst

@@ -0,0 +1,45 @@
+sn0int
+======
+
+sn0int is a semi-automatic OSINT framework and package manager. It was built
+for IT security professionals and bug hunters to gather intelligence about a
+given target or about yourself. sn0int is enumerating attack surface by
+semi-automatically processing public information and mapping the results in a
+unified format for followup investigations.
+
+Among other things, sn0int is currently able to:
+
+- Harvest subdomains from certificate transparency logs
+- Harvest subdomains from various passive dns logs
+- Sift through subdomain results for publicly accessible websites
+- Harvest emails from pgp keyservers
+- Enrich ip addresses with ASN and geoip info
+- Harvest subdomains from the wayback machine
+- Gather information about phonenumbers
+- Bruteforce interesting urls
+
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
+
+Join us on IRC: `irc.hackint.org:6697/#sn0int <https://webirc.hackint.org/#irc://irc.hackint.org/#sn0int>`_
+
+Getting Started
+---------------
+
+.. toctree::
+   :maxdepth: 3
+   :glob:
+
+   install
+   usage
+   scripting
+   database
+   keyring
+   config
+   sandbox
+   reference

docs/install.rst

@@ -0,0 +1,72 @@
+Installation
+============
+
+If available, please prefer the package shipped by your linux distribution.
+
+Archlinux
+---------
+
+.. code-block:: bash
+
+    $ pacman -S sn0int
+
+Debian/Ubuntu/Kali
+------------------
+
+.. code-block:: bash
+
+    $ apt install libsqlite3-dev libseccomp-dev
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f
+
+Alpine
+------
+
+.. code-block:: bash
+
+    $ apk add --no-cache sqlite-dev libseccomp-dev cargo
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f
+
+Docker
+------
+
+.. code-block:: bash
+
+    $ docker run --rm --init -it -v $PWD/.cache:/cache -v $PWD/.data:/data kpcyrd/sn0int
+
+OpenBSD
+-------
+
+.. code-block:: bash
+
+    $ pkg_add sqlite3
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f
+
+Mac OSX
+-------
+
+.. code-block:: bash
+
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ cargo install -f
+
+Windows
+-------
+
+This is not recommended and only passively maintained. Please prefer linux in a virtual machine if needed.
+
+.. code-block:: bash
+
+    $ git clone https://github.com/kpcyrd/sn0int.git
+    $ cd sn0int
+    $ curl -fsS --retry 3 --retry-connrefused -o sqlite3.zip https://sqlite.org/2017/sqlite-dll-win64-x64-3160200.zip
+    $ 7z e sqlite3.zip -y
+    $ "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\lib.exe" /def:sqlite3.def /OUT:sqlite3.lib /machine:x64
+    $ export SQLITE3_LIB_DIR="$TRAVIS_BUILD_DIR"
+    $ cargo install -f

docs/keyring.rst

@@ -0,0 +1,73 @@
+Keyring
+=======
+
+A common problem is that you need either an api key or a username/password
+combination. Instead of hardcoding it in the script you should request them
+from the keyring. In order to do this you need to request permissions to those
+credentials.
+
+Managing the keyring
+--------------------
+
+The keyring is a simple namespaced key-value store::
+
+    [sn0int][default] > keyring add aws:AKIAIOSFODNN7EXAMPLE
+    Secretkey: keep-this-secret
+    [sn0int][default] > keyring list
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] >
+    [sn0int][default] > keyring list aws
+    aws:AKIAIOSFODNN7EXAMPLE
+    [sn0int][default] > keyring list instagram
+    [sn0int][default] >
+    [sn0int][default] > keyring get aws:AKIAIOSFODNN7EXAMPLE
+    Namespace:    "aws"
+    Access Key:   "AKIAIOSFODNN7EXAMPLE"
+    Secret:       "keep-this-secret"
+    [sn0int][default] >
+
+If the service uses a username-password combination, set the username as the
+access key and the password as the secret.
+
+If the service uses only a secret key for the api, set the secret key as the
+access key and leave the secret blank.
+
+A script doesn't automatically get access to requested keyring namespaces.
+Instead the user is asked to confirm those requests to limit abusive scripts.
+
+Using access keys in scripts
+----------------------------
+
+We can request all keys of a certain namespace in our script metadata. This is
+going to prompt the user to grant the script access. This can be done for
+multiple namespaces in the same script:
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Keyring-Access: asdf
+
+If the user granted us access to those keys we can read them with ``keyring``:
+
+.. code-block:: lua
+
+    creds = keyring('aws')
+    print(creds[1]['accesskey'])
+    print(creds[1]['secretkey'])
+
+This returns a list of all keys in that namespace. Any empty list is returned
+if the user doesn't have any keys in that namespace.
+
+Using access keys as source argument
+------------------------------------
+
+We can also use the access keys as source argument. This is useful if each
+account has access to different things and we want to read through all of them.
+
+Since access key permissions are granted per namespace we need to specify which
+credentials we want to use.
+
+.. code-block:: lua
+
+    -- Keyring-Access: aws
+    -- Source: keyring:aws

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/man.rst

@@ -0,0 +1,14 @@
+:orphan:
+
+sn0int
+======
+
+todo
+
+.. toctree::
+    :maxdepth: 3
+    :glob:
+
+    usage
+    config
+    reference

docs/reference.rst

@@ -0,0 +1,559 @@
+Function reference
+==================
+
+clear_err
+---------
+
+Clear the last recorded error from the internal state. See also last_err_.
+
+.. code-block:: lua
+
+    if last_err() then
+        -- ignore this error
+        clear_err()
+    end
+
+db_add
+------
+
+Add an entity to the database or update it if it already exists. This function
+may fail or return ``nil``. See `db_add <database.html#db-add>`__ for details.
+
+.. code-block:: lua
+
+    domain_id = db_add('domain', {
+        value='example.com',
+    })
+
+db_select
+---------
+
+Checks if a target is in scope. If non-nil is returned, this entity is in
+scope. This function may fail. See `db_select <database.html#db-select>`__ for
+details.
+
+.. code-block:: lua
+
+    domain_id = db_select('domain', 'example.com')
+    if domain_id ~= nil then
+        -- do something
+    end
+
+db_update
+---------
+
+Update an entity in the database. This function may fail. See `db_update
+<database.html#db-update>`__ for details.
+
+.. code-block:: lua
+
+    db_update('ipaddr', arg, {
+        asn=lookup['asn'],
+        as_org=lookup['as_org'],
+    })
+
+dns
+---
+
+Resolve a dns record. If the dns query was successful and the dns reply is
+``NoError`` then ``x['error']`` is ``nil``. The records of the reply are in
+``x['answers']``. This function may fail.
+
+This function accepts the following options:
+
+``record``
+  The ``query_type``, can be any of ``A``, ``AAAA``, ``MX``, ``AXFR``, etc.
+``nameserver``
+  The server that should be used for the lookup. Defaults to your system
+  resolver.
+``tcp``
+  If the lookup should use tcp, true/false.
+``timeout``
+  The time until the query times out in milliseconds.
+
+.. code-block:: lua
+
+    records = dns('example.com', {
+        record='A',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+.. note::
+   DNS replies with an error code set are not causing a change to
+   ``last_err()``. You have to test for this explicitly.
+
+.. note::
+   This function is unavailable if a socks5 proxy is configured.
+
+error
+-----
+
+Log an error to the terminal.
+
+.. code-block:: lua
+
+    error('ohai')
+
+asn_lookup
+----------
+
+Run an ASN lookup for a given ip address. The function returns ``asn`` and
+``as_org``. This function may fail.
+
+.. code-block:: lua
+
+    lookup = asn_lookup('1.1.1.1')
+    if last_err() then return end
+
+geoip_lookup
+------------
+
+Run a geoip lookup for a given ip address. The function returns:
+
+- continent
+- continent_code
+- country
+- country_code
+- city
+- latitude
+- longitude
+
+This function may fail.
+
+.. code-block:: lua
+
+    lookup = geoip_lookup('1.1.1.1')
+    if last_err() then return end
+
+html_select
+-----------
+
+Parses an html document and returns the first element that matches the css
+selector. The return value is a table with `text` being the inner text and
+`attrs` being a table of the elements attributes.
+
+.. code-block:: lua
+
+    csrf = html_select(html, 'input[name="csrf"]')
+    token = csrf["attrs"]["value"]
+
+html_select_list
+----------------
+
+Same as html_select_ but returns all matches instead of the first one.
+
+.. code-block:: lua
+
+    html_select_list(html, 'input[name="csrf"]')
+
+http_mksession
+--------------
+
+Create a session object. This is similar to ``requests.Session`` in
+python-requests and keeps track of cookies.
+
+.. code-block:: lua
+
+    session = http_mksession()
+
+http_request
+------------
+
+Prepares an http request. The first argument is the session reference and
+cookies from that session are copied into the request. After the request has
+been sent, the cookies from the response are copied back into the session.
+
+The next arguments are the ``method``, the ``url`` and additional options.
+Please note that you still need to specify an empty table ``{}`` even if no
+options are set. The following options are available:
+
+``query``
+  A map of query parameters that should be set on the url.
+``headers``
+  A map of headers that should be set.
+``basic_auth``
+  Configure the basic auth header with ``{"user, "password"}``.
+``user_agent``
+  Overwrite the default user agent with a string.
+``json``
+  The request body that should be json encoded.
+``form``
+  The request body that should be form encoded.
+``body``
+  The raw request body as string.
+
+This function may fail.
+
+.. code-block:: lua
+
+    req = http_request(session, 'POST', 'https://httpbin.org/post', {
+        json={
+            user=user,
+            password=password,
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp["status"] ~= 200 then return "invalid status code" end
+
+http_send
+---------
+
+Send the request that has been built with http_request_. Returns a table with
+the following keys:
+
+``status``
+  The http status code
+``headers``
+  A table of headers
+``text``
+  The response body as string
+
+.. code-block:: lua
+
+    req = http_request(session, 'POST', 'https://httpbin.org/post', {
+        json={
+            user=user,
+            password=password,
+        }
+    })
+    resp = http_send(req)
+    if last_err() then return end
+    if resp["status"] ~= 200 then return "invalid status code" end
+
+info
+----
+
+Log an info to the terminal.
+
+.. code-block:: lua
+
+    info('ohai')
+
+json_decode
+-----------
+
+Decode a lua value from a json string.
+
+.. code-block:: lua
+
+    json_decode("{\"data\":{\"password\":\"fizz\",\"user\":\"bar\"},\"list\":[1,3,3,7]}")
+
+json_decode_stream
+------------------
+
+Very similar to json_decode_, but works with multiple json objects directly
+concatenated to each other or separated by newlines.
+
+.. code-block:: lua
+
+    json_decode_stream("{\"data\":1}{\"data\":2}")
+
+json_encode
+-----------
+
+Encode a datastructure into a string.
+
+.. code-block:: lua
+
+    x = json_encode({
+        some=1,
+        fancy={
+            data='structures',
+        }
+    })
+    print(x)
+
+keyring
+-------
+
+Request all keys from a given namespace. See the `keyring <keyring.html>`__
+section for details.
+
+.. code-block:: lua
+
+    creds = keyring('aws')
+    print(creds[1]['accesskey'])
+    print(creds[1]['secretkey'])
+
+last_err
+--------
+
+Returns infos about the last error we've observed, if any. Returns ``nil``
+otherwise.
+
+.. code-block:: lua
+
+    if last_err() then
+        -- Something went wrong, abort
+        return
+    end
+
+pgp_pubkey
+----------
+
+Same as pgp_pubkey_armored_, but without the unarmor step.
+
+pgp_pubkey_armored
+------------------
+
+Extract uids out of a rfc 4880 pgp public key. This function may fail.
+
+.. code-block:: lua
+
+    key = pgp_pubkey_armored([===[
+    -----BEGIN PGP PUBLIC KEY BLOCK-----
+    Version: GnuPG v2
+
+    mQENBFu6q90BCADgD7Q9aH5683yt7hzPktDkAUNAZJHwYhUNeyGK43frPyDRWQmq
+    N+oXTfiYWLQN+d7KNBTnF9uwyBdaLM7SH44lLNYo8W09mVM2eK+wt19uf5HYNgAE
+    8la45QLo/ce9CQVe1a4oXNWq6l0FOY7M+wLe+G2wMwz8RXGgwd/qQp4/PB5YpUhx
+    nAnzClxvwymrL6BQXsRcKSMSD5bIzIv95n105CvW5Hql7JR9zgOR+gHqVOH8HBUc
+    ZxMumrTM6aKLgAhgM8Sn36gCFOfjlG1b1OFLZhUtgro/nnEOmAurRsCZy8M5h8QM
+    FpZChIH8kgHs90F/CCvGjMq3qvWcH8ZsPUizABEBAAG0NUhhbnMgQWNrZXIgKGV4
+    YW1wbGUgY29tbWVudCkgPGhhbnMuYWNrZXJAZXhhbXBsZS5jb20+iQFOBBMBCAA4
+    FiEEyzeO1eEwbB03hcqBM00IodGdlj8FAlu6q90CGwMFCwkIBwIGFQgJCgsCBBYC
+    AwECHgECF4AACgkQM00IodGdlj/AJQgAjmk+iP5b7Jt7+f+lU4Oprlf3f3DG/uh5
+    Ge6MjV7cvtxlhZJRD5hxGt9RwwnEp61TBSbrem288pM89ilQfTNe0wUr9OzwWzh/
+    8Ngl5iWnD2ah3Mpi5R1V/YMNf2cnwVjqNvfkRHdNc43pZOkC2GoiTUn0QY0UBpOW
+    ZMN3//ANi6ZtiK/L0IZQND/gKvOzu/4tfaJeBl26T3cVYj53p3G3jhlb92vVa8SR
+    uL3S3bzd1h5snDgU1uXHmNHGbhkEc4KUneQ0V9/bdZrg6OzFAfM1ghgfoId+YpQH
+    er9L26ISL3QF58wdEXfIdHYEmMlANjBMO2cUlQXgONuCgkMuY7GBmrkBDQRbuqvd
+    AQgA41jqCumCxYV0NdSYNnTSSDRyd69dOUYCAPT80iZ739s7KKJS9X9KVfGmDjfi
+    u2RcfR/KYj53HoyOm4Pm/+ONN8De4ktzXpIpJxGC+O8NBvd9vkboAS6qnCjK7KVE
+    r91ymxxVKp2dzZvVfpIjWVZR5i2EAvS5vw8UK4gL8ALH+S9leJFZrQWcgyoJOJzH
+    Rzr9pesX2HvdgcNG1O6QUArlsnsTnqpi/hu7tQa8tifBpWDeArOA23Y2DgeehdDF
+    lSU/8KD4J+AkFrWWlcTaMsvSChXQkCHEMRIcSOfXtdpX5KJSE7UBQdD1opm+mR79
+    VeHnuJAAVZZtUZmJA7pjdKykYQARAQABiQE2BBgBCAAgFiEEyzeO1eEwbB03hcqB
+    M00IodGdlj8FAlu6q90CGwwACgkQM00IodGdlj8bMAf+Lq3Qive4vcrCTT4IgvVj
+    arOACdcbtt5RhVBTimT19rDWNH+m+PfPjo3FSlBj5cm70KAXUS2LBFFxhakTZ/Mq
+    cQroWZpVbBxj4kipEVVJZFdUZQaDERJql0xYGOQrNMQ4JGqJ84BRrtOExjSqo41K
+    hAhNe+bwPGH9/Igiixc4tH07xa7TOy4MyJv/6gpbHy/lW1hqpCAgM5fT/im5/6QF
+    k0tED6vIuc54IWiOmwCnjZiQnJ8uCwEu+cuJ5Exwy9CNERLp5v0y4eG+0E+at9j/
+    macOg39qf09t53pTqe9dWv5NIi319TeBsKZ2lb0crrQjsbHqk0DAUwgQuoANqLku
+    vA==
+    =kRIv
+    -----END PGP PUBLIC KEY BLOCK-----
+    ]===])
+
+    if last_err() then return end
+    print(key)
+
+print
+-----
+
+Write something directly to the terminal.
+
+.. code-block:: lua
+
+    print({
+        some=1,
+        fancy={
+            data='structures',
+        }
+    })
+
+.. warning::
+   This function writes directly to the terminal and can interfere with other
+   terminal features. This function should be used during development only.
+
+psl_domain_from_dns_name
+------------------------
+
+Returns the parent domain according to the public suffix list. For
+``www.a.b.c.d.example.com`` this is going to be ``example.com``.
+
+.. code-block:: lua
+
+    domain = psl_domain_from_dns_name('www.a.b.c.d.example.com')
+    print(domain == 'example.com')
+
+regex_find
+----------
+
+Apply a regex to some text. Returns ``nil`` if the regex didn't match and the
+capture groups if it did.
+
+.. code-block:: lua
+
+    m = regex_find(".(.)", "abcdef")
+
+    if m == nil then
+        print('No captures')
+    end
+
+    print(m[1] == 'ab')
+    print(m[2] == 'b')
+
+regex_find_all
+--------------
+
+Same as regex_find_, but returns all matches.
+
+.. code-block:: lua
+
+    m = regex_find_all(".(.)", "abcdef")
+
+    print(m[1][1] == 'ab')
+    print(m[1][2] == 'b')
+    print(m[2][1] == 'cd')
+    print(m[2][2] == 'd')
+    print(m[3][1] == 'ef')
+    print(m[3][2] == 'f')
+
+sleep
+-----
+
+Pause the current program for the specified number of seconds. This is usually
+only used for debugging.
+
+.. code-block:: lua
+
+    sleep(1)
+
+status
+------
+
+Update the label of the progress indicator.
+
+.. code-block:: lua
+
+    status('ohai')
+
+stdin_readline
+--------------
+
+Read a line from stdin. The final newline is not removed.
+
+.. code-block:: lua
+
+    stdin_readline()
+
+.. note::
+   This only works with `sn0int run --stdin`.
+
+url_decode
+----------
+
+Parse a query string into a map. For raw percent decoding see url_unescape_.
+
+.. code-block:: lua
+
+    v = url_decode('a=b&c=d')
+    print(v['a'] == 'b')
+    print(v['c'] == 'd')
+
+url_encode
+----------
+
+Encode a map into a query string. For raw percent encoding see url_escape_.
+
+.. code-block:: lua
+
+    v = url_encode({
+        a='b',
+        c='d',
+    })
+    print(v == 'a=b&c=d')
+
+url_escape
+----------
+
+Apply url escaping to a string.
+
+.. code-block:: lua
+
+    v = url_escape('foo bar?')
+    print(v == 'foo%20bar%3F')
+
+url_join
+--------
+
+Join a relative link to an absolute link. If both links are absolute we just
+return the first one:
+
+.. code-block:: lua
+
+    x = url_join('https://example.com/x', '/foo')
+    print(x == 'https://example.com/foo')
+
+    x = url_join('https://example.com/x', 'https://github.com/')
+    print(x == 'https://github.com/')
+
+url_parse
+---------
+
+Parse a url into its components. The following components are returned:
+
+- scheme
+- host
+- port
+- path
+- query
+- fragment
+- params
+
+.. code-block:: lua
+
+    url = url_parse('https://example.com')
+    print(url['scheme'] == 'https')
+    print(url['host'] == 'example.com')
+    print(url['path'] == '/')
+
+url_unescape
+------------
+
+Remove url escaping of a string.
+
+.. code-block:: lua
+
+    v = url_unescape('foo%20bar%3F')
+    print(v == 'foo bar?')
+
+utf8_decode
+-----------
+
+Decodes a list of bytes/numbers into a string. This function might fail.
+
+.. code-block:: lua
+
+    x = utf8_decode({65, 65, 65, 65})
+    if last_err() then return end
+    print(x == 'AAAA')
+
+x509_parse_pem
+--------------
+
+Parse a pem encoded certificate. This function might fail.
+
+.. code-block:: lua
+
+    x = x509_parse_pem([[-----BEGIN CERTIFICATE-----
+    MIID9DCCA3qgAwIBAgIQBWzetBRl/ycHFsBukRYuGTAKBggqhkjOPQQDAjBMMQsw
+    CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMSYwJAYDVQQDEx1EaWdp
+    Q2VydCBFQ0MgU2VjdXJlIFNlcnZlciBDQTAeFw0xODAzMzAwMDAwMDBaFw0yMDAz
+    MjUxMjAwMDBaMGwxCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMN
+    U2FuIEZyYW5jaXNjbzEZMBcGA1UEChMQQ2xvdWRmbGFyZSwgSW5jLjEdMBsGA1UE
+    AwwUKi5jbG91ZGZsYXJlLWRucy5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC
+    AASyRQsxrFBjziHmfDQjGsXBU0WWl3oxh7vg6h2V9f8lBMp18PY/td9R6VvJPa20
+    AwVzIJI+dL6OSxviaIZEbmK7o4ICHDCCAhgwHwYDVR0jBBgwFoAUo53mH/naOU/A
+    buiRy5Wl2jHiCp8wHQYDVR0OBBYEFN+XTeVDs7BBp0LykM+Jf64SV4ThMGMGA1Ud
+    EQRcMFqCFCouY2xvdWRmbGFyZS1kbnMuY29thwQBAQEBhwQBAAABghJjbG91ZGZs
+    YXJlLWRucy5jb22HECYGRwBHAAAAAAAAAAAAERGHECYGRwBHAAAAAAAAAAAAEAEw
+    DgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBp
+    BgNVHR8EYjBgMC6gLKAqhihodHRwOi8vY3JsMy5kaWdpY2VydC5jb20vc3NjYS1l
+    Y2MtZzEuY3JsMC6gLKAqhihodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vc3NjYS1l
+    Y2MtZzEuY3JsMEwGA1UdIARFMEMwNwYJYIZIAYb9bAEBMCowKAYIKwYBBQUHAgEW
+    HGh0dHBzOi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwCAYGZ4EMAQICMHsGCCsGAQUF
+    BwEBBG8wbTAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQuY29tMEUG
+    CCsGAQUFBzAChjlodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGlnaUNlcnRF
+    Q0NTZWN1cmVTZXJ2ZXJDQS5jcnQwDAYDVR0TAQH/BAIwADAKBggqhkjOPQQDAgNo
+    ADBlAjEAjoyy2Ogh1i1/Kh9+psMc1OChlQIvQF6AkojZS8yliar6m8q5nqC3qe0h
+    HR0fExwLAjAueWRnHX4QJ9loqMhsPk3NB0Cs0mStsNDNG6/DpCYw7XmjoG3y1LS7
+    ZkZZmqNn2Q8=
+    -----END CERTIFICATE-----
+    ]])
+    if last_err() then return end
+    print(x)

docs/sandbox.rst

@@ -0,0 +1,113 @@
+Sandbox
+=======
+
+Scripts are generally considered to be untrusted and executed exclusively in a
+child process. It's important to note that there's a basic sandbox that's
+active on every operating system, and there's a second line of defense on
+supported operating systems.
+
+The first line of defense is the restrictive stdlib. It's assumed that and
+attacker gains full control over the lua code and is able to call any function
+with arbitrary arguments. The stdlib only provides functions that are
+considered safe, so for example it's not possible to start a process or open a
+file.
+
+The second line of defense is supposed to make sure the system isn't
+compromised even if the first layer is fully broken and an attacker gains full
+control over the child process.
+
+Right now this is only supported on linux and openbsd.
+
+Linux
+-----
+
+On linux we use seccomp to filter all syscalls that we don't need. We also use
+chroot to disable filesystem access. It's recommended to install the sn0int
+binary with ``cap_sys_chroot`` to make sure unprivileged users can use chroot.
+The chroot location is hard coded and all capabilities are removed after the
+chroot is done or if no chroot is going to happen.
+
+OpenBSD
+-------
+
+On openbsd we're using ``pledge`` to restrict syscalls and ``unveil`` to
+restrict filesystem access.
+
+IPC Protocol
+------------
+
+The parent process and the child process communicate using an IPC protocol that
+is line-based json.
+
+For a simple hello world the parent process is only going to send a single line
+to the child process. This line contains:
+
+- The function argument
+- The dns config
+- Keys that the module has been given access to
+- The module metadata and code
+- Options, if any
+- A socks5 proxy, if any
+- The log level
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    -- nothing to do here\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+Saving this line in a file called ``start.json`` and sending it to a sandbox
+process should result in the following output::
+
+    $ sn0int sandbox foobar < start.json
+    {"Exit":"Ok"}
+    $
+
+This line tells us that the script terminated successfully.
+
+There are some functions that cause a notification to the parent process. We
+are going to add a call to the ``info()`` function to our module:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    info('ohai')\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+
+This is going to print an additional event::
+
+    $ sn0int sandbox foobar < start2.json
+    {"Log":{"Info":"\"ohai\""}}
+    {"Exit":"Ok"}
+    $
+
+There are some functions that block the child process until the parent process
+sent a reply. These functions are mostly database related functions, since the
+child doesn't have direct database access. To demonstrate this, we're going to
+write two lines to our file this time, one is the init line and the second one
+is the reply for the database event:
+
+.. code-block:: json
+
+    {"arg":null,"dns_config":{"ns":["1.1.1.1:53","1.0.0.1:53"],"tcp":false,"timeout":{"nanos":0,"secs":3}},"keyring":[],"module":{"author":"anonymous","description":"basic selftest","keyring_access":[],"name":"selftest","script":{"code":"-- Description: basic selftest\n-- Version: 0.1.0\n-- License: GPL-3.0\n\nfunction run()\n    x = db_add('domain', {value=\"example.com\"})\n    info(x)\nend\n"},"source":null,"version":"0.1.0"},"options":{},"proxy":null,"verbose":2}
+    {"Ok":1337}
+
+Results in the following output::
+
+    $ target/release/sn0int sandbox foobar < start3.json
+    {"Database":{"Insert":{"Domain":{"value":"example.com"}}}}
+    {"Log":{"Info":"1337.0"}}
+    {"Exit":"Ok"}
+    $
+
+The first line is a database event and indicates that the child wants to insert
+data. After printing this line the child tries to read a line from stdin, this
+is why we needed to write two lines to our json file this time. In the second
+line the child learns if the insert was successful and which id was assigned to
+that entity.
+
+Limitations
+-----------
+
+There are some limitations that you should be aware:
+
+- Network access is available and network namespaces aren't isolated. This
+  means scripts have access to your local network, the internet and also your
+  localhost loopback interface.
+- If chroot is unavailable an attacker could connect to unix domain sockets.

docs/scripting.rst

@@ -0,0 +1,190 @@
+Scripting
+=========
+
+Scripting is the core feature in sn0int. It's not strictly required, but if you
+want to write your own modules, this section is for you.
+
+Write your first module
+-----------------------
+
+It's highly recommended to use a VCS for development, so let's start by setting
+that up. We're going to assume you store your repos in ``~/repos`` but you're
+free to change that to something else::
+
+    $ git init ~/repos/sn0int-modules
+    $ cd ~/repos/sn0int-modules
+    $ ln -s "$PWD" ~/.local/share/sn0int/modules/$YOUR_GITHUB_NAME
+
+Every module we're adding to ``~/repos/sn0int-modules`` is now going to be
+picked up by sn0int.
+
+Let's add our first module by opening ``~/repos/sn0int-modules/first.lua``.
+There's a bit of boilerplate that every module needs to load successfully:
+
+.. code-block:: lua
+
+    -- Description: ohai wurld
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
+    function run(arg)
+        -- TODO: do something here
+    end
+
+``Description`` (mandatory)
+  This should be a short text that describes what your module is doing.
+
+``Version`` (mandatory)
+  Every module requires a semver_ version. You can just set it to ``0.1.0``
+  during development, but you need to increase it every time you publish your
+  module. If you don't care about that one, just keep increasing ``0.X.0``.
+
+.. _semver: https://semver.org/
+
+``Source`` (mandatory)
+  This is going to specify what kind of entities we're interested in. If we
+  specify ``domains`` our module is going to be called with all domains that
+  are targeted.
+
+  - ``domains``
+  - ``subdomains``
+  - ``ipaddrs``
+  - ``urls``
+  - ``emails``
+
+``License`` (mandatory)
+  This is somewhat special. We require that every module is licensed under an
+  open source license. Pick one of the following licenses.
+
+  - ``MIT`` -  https://opensource.org/licenses/MIT
+  - ``GPL-3.0`` - https://opensource.org/licenses/gpl-license
+  - ``LGPL-3.0`` - https://opensource.org/licenses/lgpl-license
+  - ``BSD-2-Clause`` - https://opensource.org/licenses/BSD-2-Clause
+  - ``BSD-3-Clause`` - https://opensource.org/licenses/BSD-3-Clause
+  - ``WTFPL`` - https://spdx.org/licenses/WTFPL.html
+
+``function run(arg)`` (mandatory)
+  This is where the actual magic of our module happens. Our function is going
+  to be called in a loop for each entity that is targeted by the user.
+
+Let's continue. For the sake of an hello world we're going to take some
+``domains``, check if a ``www`` subdomain exists and if it does, add it to the
+database.
+
+.. code-block:: lua
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+        print(subdomain)
+    end
+
+Combined with the header we wrote previously we can already execute this
+module. Make sure you've added a domain to scope with ``add domain
+example.com``, save your file and run it like this::
+
+    sn0int run -f ./first.lua
+
+We should see some output by our print function.
+
+.. note::
+   ``print`` is useful for development but must be removed before publishing.
+
+Next, we want to actually resolve that name, we're going to use the ``dns``
+function for that. This function takes a name and a query type and returns a
+result. Note that this function might fail, in which case we want to abort our
+function. We do that by checking if the return value of ``last_err()`` is
+truth-y.
+
+.. code-block:: lua
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+
+        records = dns(subdomain, {
+            record='A'
+        })
+        if last_err() then return end
+
+        print(records)
+    end
+
+If you run your module again you're going to see some output, either
+``{"answers":[somedata],"error":null}`` or
+``{"answers":[],"error":"NXDomain"}``. We decide that we add the subdomain to
+our scope and set it to resolvable if ``error`` is ``nil``.
+
+.. code-block:: lua
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+
+        records = dns(subdomain, {
+            record='A'
+        })
+        if last_err() then return end
+
+        if records['error'] == nil then
+            db_add('subdomain', {
+                domain_id=arg['id'],
+                value=subdomain,
+                resolvable=true,
+            })
+        end
+    end
+
+.. hint::
+   See the database section to understand how the database works in detail.
+
+If we execute our module one more time it's going to log that it discovered a
+subdomain, if it doesn't, try adding more domains to scope. Note that this only
+happens the first time. Modules that don't discover anything or don't discover
+anything new exit silently.
+
+After putting everything together, our final module looks like this:
+
+.. code-block:: lua
+
+    -- Description: ohai wurld
+    -- Version: 0.1.0
+    -- Source: domains
+    -- License: GPL-3.0
+
+    function run(arg)
+        subdomain = 'www.' .. arg['value']
+
+        records = dns(subdomain, {
+            record='A'
+        })
+        if last_err() then return end
+
+        if records['success'] ~= nil then
+            db_add('subdomain', {
+                domain_id=arg['id'],
+                value=subdomain,
+                resolvable=true,
+            })
+        end
+    end
+
+There's still some room for improvement, for example, since we already resolved
+that record, we could also add the ip address to the scope and link it to the
+subdomain we added.
+
+Publish your module
+-------------------
+
+The public registry uses github usernames to namespace the registry. This means
+you need to authenticate to the registry using your github username. This can
+be done using::
+
+    sn0int login
+
+sn0int is going to open a new tab in your browser, if you are already signed
+into your github account you only need to confirm an authorization request. The
+application doesn't need any of your data, so it's only asking you to confirm
+your identity.
+
+Afterwards publish your module with::
+
+    sn0int publish ./first.lua

docs/usage.rst

@@ -0,0 +1,258 @@
+Running your first investigation
+================================
+
+This page is going to guide you through the process of setting up your
+environment and running your first investigation.
+
+Installing the default modules
+------------------------------
+
+By default, sn0int doesn't have any modules installed. If you start up sn0int
+it's going to download some files that it needs and then suggests to install a
+number of recommended modules::
+
+    $ sn0int
+
+                       ___/           .
+         ____ , __   .'  /\ ` , __   _/_
+        (     |'  `. |  / | | |'  `.  |
+        `--.  |    | |,'  | | |    |  |
+       \___.' /    | /`---' / /    |  \__/
+
+            osint | recon | security
+          irc.hackint.org:6697/#sn0int
+
+    [+] Connecting to database
+    [+] Downloading public suffix list
+    [+] Downloading "GeoLite2-City.mmdb"
+    [+] Downloading "GeoLite2-ASN.mmdb"
+    [+] Loaded 0 modules
+    [*] No modules found, run quickstart to install default modules
+    [sn0int][default] >
+
+Typing ``quickstart`` is going to get you a fair number of featured modules::
+
+    [sn0int][default] > quickstart
+    [+] Installing kpcyrd/asn
+    [+] Installing kpcyrd/ctlogs
+    [+] Installing kpcyrd/dns-resolve
+    [+] Installing kpcyrd/geoip
+    [+] Installing kpcyrd/hackertarget-subdomains
+    [+] Installing kpcyrd/otx-subdomains
+    [+] Installing kpcyrd/passive-spider
+    [+] Installing kpcyrd/pgp-keyserver
+    [+] Installing kpcyrd/threatminer-ipaddr
+    [+] Installing kpcyrd/threatminer-subdomains
+    [+] Installing kpcyrd/url-scan
+    [+] Installing kpcyrd/waybackurls
+    [+] Loaded 12 modules
+    [sn0int][default] >
+
+Adding something to scope
+-------------------------
+
+You probably want to separate your investigations so you should select a
+workspace where your results should go::
+
+    [sn0int][default] > workspace demo
+    [+] Connecting to database
+    [sn0int][demo] >
+
+Next, we have to start somewhere and add the first entity to our scope::
+
+    [sn0int][demo] > add domain
+    Domain: example.com
+    [sn0int][demo] >
+
+.. note::
+   There is a concept of a domain vs a subdomain. We are referring to a domain
+   as everything that is a subdomain of a `public suffix`_. For example, .com
+   is a public suffix, which makes example.com a domain in sn0int terms. Every
+   subdomain of that, like www.example.com, is referred to as a subdomain.
+
+   Note that example.com can be added as a subdomain as well since it can hold
+   records. In that case, example.com is both the name of the dns zone, while
+   also being an entity in that zone.
+
+.. _public suffix: https://publicsuffix.org/
+
+You can confirm this by running a select on the domains we now have::
+
+    [sn0int][demo] > select domains
+    #1, "example.com"
+    [sn0int][demo] >
+
+Something we don't need right now, but is going to be useful later on is the
+ability to filter your entities::
+
+    [sn0int][demo] > select domains where id=1
+    #1, "example.com"
+    [sn0int][demo] >
+    [sn0int][demo] > select domains where value like %.com
+    #1, "example.com"
+    [sn0int][demo] >
+    [sn0int][demo] > select domains where ( value like e% and value like %m ) or false
+    #1, "example.com"
+    [sn0int][demo] >
+
+.. note::
+   Almost all entities have a ``value`` column that holds the primary value of
+   the entity.
+
+Running a module
+----------------
+
+Now that we have something to get started with, we can run our first module.
+First lets list all modules we have::
+
+    [sn0int][demo] > mod list
+    kpcyrd/asn (0.1.0)
+        Run a asn lookup for an ip address
+    kpcyrd/ctlogs (0.1.0)
+        Query certificate transparency logs to discover subdomains
+    kpcyrd/dns-resolve (0.1.0)
+        Query subdomains to discovery ip addresses and verify the record is visible
+    kpcyrd/geoip (0.1.0)
+        Run a geoip lookup for an ip address
+    kpcyrd/hackertarget-subdomains (0.1.0)
+        Query hackertarget for subdomains of a domain
+    kpcyrd/otx-subdomains (0.1.0)
+        Query alienvault otx passive dns for subdomains of a domain
+    kpcyrd/passive-spider (0.1.0)
+        Scrape known http responses for urls
+    kpcyrd/pgp-keyserver (0.1.0)
+        Query pgp keyserver for email addresses
+    kpcyrd/threatminer-ipaddr (0.1.0)
+        Query ThreatMiner passive dns for subdomains of an ip address
+    kpcyrd/threatminer-subdomains (0.1.0)
+        Query ThreatMiner passive dns for subdomains of a domain
+    kpcyrd/url-scan (0.1.0)
+        Scan subdomains for websites
+    kpcyrd/waybackurls (0.1.0)
+        Discover subdomains from wayback machine
+    [sn0int][demo] >
+
+Let's start by querying certificate transparency logs::
+
+    [sn0int][demo] > use ctlogs
+    [sn0int][demo][kpcyrd/ctlogs] > run
+    [*] "example.com"                                     : Subdomain: "www.example.com"
+    [*] "example.com"                                     : Subdomain: "m.example.com"
+    [*] "example.com"                                     : Subdomain: "dev.example.com"
+    [*] "example.com"                                     : Subdomain: "products.example.com"
+    [*] "example.com"                                     : Subdomain: "support.example.com"
+    [+] Finished kpcyrd/ctlogs
+    [sn0int][demo][kpcyrd/ctlogs] >
+
+Looks like we've discovered some subdomains here. It might be tempting to throw
+some of them in a browser but hold on, there's a more efficient way to approach
+this.
+
+.. hint::
+   You can run the modules concurrently with ``run -j 8``.
+
+Running followup modules on the results
+---------------------------------------
+
+A lot of time has been spent on the database part. While it sort of feels like
+a no-sql database we are actually enforcing a schema for a reason instead of
+just using generic dictionaries and calling it a day.
+
+It's crucial that entities created by one module can be picked up by another
+module, like LEGOs. Let's continue with a module to query the dns records::
+
+    [sn0int][demo][kpcyrd/ctlogs] > use dns-resolve
+    [sn0int][demo][kpcyrd/dns-resolve] > run
+    [*] "www.example.com"                                 : Updating "www.example.com" (resolvable => true)
+    [*] "www.example.com"                                 : IpAddr: 93.184.216.34
+    [*] "www.example.com"                                 : "www.example.com" -> 93.184.216.34
+    [*] "m.example.com"                                   : Updating "m.example.com" (resolvable => false)
+    [*] "dev.example.com"                                 : Updating "dev.example.com" (resolvable => false)
+    [*] "products.example.com"                            : Updating "products.example.com" (resolvable => false)
+    [*] "support.example.com"                             : Updating "support.example.com" (resolvable => false)
+    [+] Finished kpcyrd/dns-resolve
+    [sn0int][demo][kpcyrd/dns-resolve] >
+
+.. TODO: mention https://github.com/kpcyrd/sn0int/issues/27
+
+Two things happened here: We've discovered some IP addresses and added them to
+scope, and we also updated our subdomain entities with new information, since
+we now know which of them are resolvable and which aren't.
+
+Let's run the next module, which is actually going to check for websites on
+them, but let's only target subdomains that we know are resolvable::
+
+    [sn0int][demo][kpcyrd/dns-resolve] > use url-scan
+    [sn0int][demo][kpcyrd/url-scan] > target
+    #1, "www.example.com"
+        93.184.216.34
+    #2, "m.example.com"
+    #3, "dev.example.com"
+    #4, "products.example.com"
+    #5, "support.example.com"
+    [sn0int][demo][kpcyrd/url-scan] > target where resolvable
+    [+] 1 entities selected
+    [sn0int][demo][kpcyrd/url-scan] > target
+    #1, "www.example.com"
+        93.184.216.34
+    [sn0int][demo][kpcyrd/url-scan] >
+
+We can both preview and limit the targets that are going to be passed to the
+module with the target command. Once we are satisfied with our selection we can
+run this module::
+
+    [sn0int][demo][kpcyrd/url-scan] > run
+    [*] "www.example.com"                                 : Url: "http://www.example.com/" (200)
+    [*] "www.example.com"                                 : Url: "https://www.example.com/" (200)
+    [+] Finished kpcyrd/url-scan
+    [sn0int][demo][kpcyrd/url-scan] >
+
+We've now probed both port 80 and port 443 for each subdomain and found two
+http responses this way. If you want a list of urls you may want to visit in
+your browser can now query them::
+
+    [sn0int][demo][kpcyrd/url-scan] > select urls
+    #1, "http://www.example.com/" (200)
+    #2, "https://www.example.com/" (200)
+    [sn0int][demo][kpcyrd/url-scan] >
+
+Unscoping entities
+------------------
+
+Something you are going to run into is that modules are too greedy and add
+things to the scope we are not interested in. You can delete them using the
+delete command, but those are likely picked up by a module again.
+
+What you can do instead is setting a flag on an entity that removes it from
+our scope. This is done using the noscope command::
+
+    [sn0int][demo] > use ctlogs
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    [sn0int][demo][kpcyrd/ctlogs] > add domain
+    Domain: google.com
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    #2, "google.com"
+    [sn0int][demo][kpcyrd/ctlogs] > noscope domains where value=google.com
+    [+] Updated 1 rows
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    [sn0int][demo][kpcyrd/ctlogs] >
+
+Entities that are unscoped are automatically ignored by all modules.
+
+You can reverse this using the scope command::
+
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    [sn0int][demo][kpcyrd/ctlogs] > scope domains where true
+    [+] Updated 2 rows
+    [sn0int][demo][kpcyrd/ctlogs] > target
+    #1, "example.com"
+    #2, "google.com"
+    [sn0int][demo][kpcyrd/ctlogs] >
+
+.. hint::
+   All entities have this field, you can refer to it in queries using
+   ``unscoped=1``.

examples/asn.rs

@@ -1,32 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate maxminddb;
-
-use std::env;
-use sn0int::errors::*;
-use sn0int::geoip::{AsnDB, Maxmind};
-
-
-fn run() -> Result<()> {
-    let asndb = AsnDB::open_or_download()?;
-
-    for arg in env::args().skip(1) {
-        let ip = arg.parse()?;
-        let asn = asndb.lookup(ip)?;
-        println!("{:#?}", asn);
-    }
-
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}

examples/geoip.rs

@@ -1,31 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-
-use std::env;
-use sn0int::errors::*;
-use sn0int::geoip::{GeoIP, Maxmind};
-
-
-fn run() -> Result<()> {
-    let geoip = GeoIP::open_or_download()?;
-
-    for arg in env::args().skip(1) {
-        let ip = arg.parse()?;
-        let lookup = geoip.lookup(ip)?;
-        println!("{:#?}", lookup);
-    }
-
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}

examples/maxmind-dl.rs

@@ -1,45 +0,0 @@
-extern crate sn0int;
-extern crate env_logger;
-extern crate chrootable_https;
-#[macro_use] extern crate log;
-extern crate structopt;
-
-use sn0int::errors::*;
-use sn0int::geoip::{GeoIP, Maxmind};
-use sn0int::paths;
-use std::fs;
-use structopt::StructOpt;
-
-#[derive(Debug, StructOpt)]
-pub struct Args {
-    url: String,
-    filter: String,
-    target: String,
-    #[structopt(short="e", long="extract-only")]
-    extract_only: bool,
-}
-
-fn run() -> Result<()> {
-    let args = Args::from_args();
-    debug!("{:?}", args);
-    let path = paths::cache_dir()?.join(&args.target);
-    if args.extract_only {
-        let body = fs::read(&args.url)?;
-        sn0int::archive::extract(&mut &body[..], &args.filter, path)?;
-    } else {
-        GeoIP::download(path, &args.filter, &args.url)?;
-    }
-    Ok(())
-}
-
-fn main() {
-    env_logger::init();
-
-    if let Err(err) = run() {
-        eprintln!("Error: {}", err);
-        for cause in err.iter_chain().skip(1) {
-            eprintln!("Because: {}", cause);
-        }
-        std::process::exit(1);
-    }
-}

examples/maxmind.rs

@@ -0,0 +1,104 @@
+extern crate sn0int;
+extern crate env_logger;
+extern crate chrootable_https;
+#[macro_use] extern crate log;
+
+// workaround for rustc 1.29.2 support
+#[cfg(not(target_os = "openbsd"))]
+extern crate structopt;
+#[cfg(target_os = "openbsd")]
+#[macro_use] extern crate structopt;
+
+use sn0int::errors::*;
+use sn0int::geoip::{AsnDB, GeoIP, Maxmind};
+use sn0int::paths;
+use std::fs;
+use std::net::IpAddr;
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="dl")]
+    Download(Download),
+    #[structopt(name="asn")]
+    Asn(AsnArgs),
+    #[structopt(name="geoip")]
+    GeoIP(GeoIPArgs),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Download {
+    url: String,
+    filter: String,
+    target: String,
+    #[structopt(short="e", long="extract-only")]
+    extract_only: bool,
+}
+
+impl Download {
+    fn run(&self) -> Result<()> {
+        let path = paths::cache_dir()?.join(&self.target);
+        if self.extract_only {
+            let body = fs::read(&self.url)?;
+            sn0int::archive::extract(&mut &body[..], &self.filter, path)?;
+        } else {
+            GeoIP::download(path, &self.filter, &self.url)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AsnArgs {
+    ip: IpAddr,
+}
+
+impl AsnArgs {
+    fn run(&self) -> Result<()> {
+        let asndb = AsnDB::open_or_download()?;
+
+        let asn = asndb.lookup(self.ip)?;
+        println!("{:#?}", asn);
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct GeoIPArgs {
+    ip: IpAddr,
+}
+
+impl GeoIPArgs {
+    fn run(&self) -> Result<()> {
+        let geoip = GeoIP::open_or_download()?;
+
+        let lookup = geoip.lookup(self.ip)?;
+        println!("{:#?}", lookup);
+
+        Ok(())
+    }
+}
+
+
+fn run() -> Result<()> {
+    let args = Args::from_args();
+    debug!("{:?}", args);
+    match args {
+        Args::Download(args) => args.run(),
+        Args::Asn(args) => args.run(),
+        Args::GeoIP(args) => args.run(),
+    }
+}
+
+fn main() {
+    env_logger::init();
+
+    if let Err(err) = run() {
+        eprintln!("Error: {}", err);
+        for cause in err.iter_chain().skip(1) {
+            eprintln!("Because: {}", cause);
+        }
+        std::process::exit(1);
+    }
+}

examples/spinners.rs

@@ -1,20 +1,67 @@
-extern crate sn0int;
-
-use std::env;
 use std::thread;
 use std::time::Duration;
-use sn0int::term::{SPINNERS, Spinner};
+use sn0int::term::{SPINNERS, Spinner, StackedSpinners};
+use structopt::StructOpt;
+
+
+#[derive(Debug, StructOpt)]
+pub enum Args {
+    #[structopt(name="single")]
+    Single(Single),
+    #[structopt(name="stacked")]
+    Stacked(Stacked),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Single {
+    idx: usize,
+    #[structopt(long="ticks", default_value="100")]
+    ticks: usize,
+}
+
+impl Single {
+    fn run(&self) {
+        let mut s = Spinner::new(SPINNERS[self.idx], "Demo".to_string());
+
+        for _ in 0..self.ticks {
+            thread::sleep(Duration::from_millis(100));
+            s.tick();
+        }
+
+        s.finish("Done".to_string());
+    }
+}
+
+#[derive(Debug, StructOpt)]
+pub struct Stacked {
+}
+
+impl Stacked {
+    fn run(&self) {
+        let mut stack = StackedSpinners::new();
+        stack.add("1".into(), String::from("spinner1"));
+        stack.add("2".into(), String::from("spinner2"));
+        stack.add("3".into(), String::from("spinner3"));
+
+        for x in 1..=3 {
+            for _ in 0..50 {
+                thread::sleep(Duration::from_millis(100));
+                stack.tick();
+            }
+            // stack.log("ohai");
+            stack.remove(&x.to_string());
+        }
+
+        stack.clear();
+
+        // stack.finish("Done".to_string());
+    }
+}
 
 fn main() {
-    let idx = env::args().skip(1).next().expect("Expected argv[1]");
-    let idx = idx.parse::<usize>().expect("argv[1] is not a number");
-
-    let mut s = Spinner::new(SPINNERS[idx], "Demo".to_string());
-
-    for _ in 0..100 {
-        thread::sleep(Duration::from_millis(100));
-        s.tick();
+    let args = Args::from_args();
+    match args {
+        Args::Single(args) => args.run(),
+        Args::Stacked(args) => args.run(),
     }
-
-    s.finish("Done".to_string());
 }

migrations/2018-10-26-001932_cascade-delete/up.sql

@@ -37,25 +37,6 @@ INSERT INTO subdomains (id, domain_id, value, unscoped, resolvable)
 
 DROP TABLE _subdomains_old;
 
--- subdomain_ipaddrs
-
-ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
-
-CREATE TABLE subdomain_ipaddrs (
-    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-    subdomain_id INTEGER NOT NULL,
-    ip_addr_id INTEGER NOT NULL,
-    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
-    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
-    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
-);
-
-INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
-    SELECT id, subdomain_id, ip_addr_id
-    FROM _subdomain_ipaddrs_old;
-
-DROP TABLE _subdomain_ipaddrs_old;
-
 -- urls
 
 ALTER TABLE urls RENAME TO _urls_old;
@@ -123,4 +104,23 @@ INSERT INTO ipaddrs (id, family, value, unscoped, continent, continent_code, cou
 
 DROP TABLE _ipaddrs_old;
 
+-- subdomain_ipaddrs
+
+ALTER TABLE subdomain_ipaddrs RENAME TO _subdomain_ipaddrs_old;
+
+CREATE TABLE subdomain_ipaddrs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    ip_addr_id INTEGER NOT NULL,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    FOREIGN KEY(ip_addr_id) REFERENCES ipaddrs(id) ON DELETE CASCADE,
+    CONSTRAINT subdomain_ipaddr_unique UNIQUE (subdomain_id, ip_addr_id)
+);
+
+INSERT INTO subdomain_ipaddrs (id, subdomain_id, ip_addr_id)
+    SELECT id, subdomain_id, ip_addr_id
+    FROM _subdomain_ipaddrs_old;
+
+DROP TABLE _subdomain_ipaddrs_old;
+
 PRAGMA foreign_keys=on;

migrations/2018-12-03-143558_url_path/down.sql

@@ -0,0 +1,25 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-03-143558_url_path/up.sql

@@ -0,0 +1,26 @@
+PRAGMA foreign_keys=off;
+
+ALTER TABLE urls RENAME TO _urls_old;
+
+CREATE TABLE urls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    subdomain_id INTEGER NOT NULL,
+    value VARCHAR NOT NULL,
+    path VARCHAR NOT NULL,
+    status INTEGER,
+    body BLOB,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    online BOOLEAN,
+    title VARCHAR,
+    redirect VARCHAR,
+    FOREIGN KEY(subdomain_id) REFERENCES subdomains(id) ON DELETE CASCADE,
+    CONSTRAINT url_unique UNIQUE (value)
+);
+
+INSERT INTO urls (id, subdomain_id, value, path, status, body, unscoped, online, title, redirect)
+  SELECT id, subdomain_id, value, '/', status, body, unscoped, online, title, redirect
+  FROM _urls_old;
+
+DROP TABLE _urls_old;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-14-094011_phonenumbers/down.sql

@@ -0,0 +1 @@
+DROP TABLE phonenumbers;

migrations/2018-12-14-094011_phonenumbers/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE phonenumbers (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    valid BOOLEAN,
+    last_online DATETIME,
+    country VARCHAR,
+    carrier VARCHAR,
+    line VARCHAR,
+    is_ported BOOLEAN,
+    last_ported DATETIME,
+    caller_name VARCHAR,
+    caller_type VARCHAR,
+    CONSTRAINT phonenumber_unique UNIQUE (value)
+);

migrations/2018-12-23-230955_reverse-dns/down.sql

@@ -0,0 +1,27 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _ipaddrs_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    family VARCHAR NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    continent VARCHAR,
+    continent_code VARCHAR,
+    country VARCHAR,
+    country_code VARCHAR,
+    city VARCHAR,
+    latitude FLOAT,
+    longitude FLOAT,
+    asn INTEGER,
+    as_org VARCHAR,
+    CONSTRAINT ipaddr_unique UNIQUE (value)
+);
+
+INSERT INTO _ipaddrs_new (id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org)
+  SELECT id, family, value, unscoped, continent, continent_code, city, latitude, longitude, asn, as_org
+  FROM ipaddrs;
+
+DROP TABLE ipaddrs;
+ALTER TABLE _ipaddrs_new RENAME TO ipaddrs;
+
+PRAGMA foreign_keys=on;

migrations/2018-12-23-230955_reverse-dns/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ipaddrs ADD COLUMN description VARCHAR;
+ALTER TABLE ipaddrs ADD COLUMN reverse_dns VARCHAR;

migrations/2018-12-24-141533_networks/down.sql

@@ -0,0 +1,3 @@
+DROP TABLE network_devices;
+DROP TABLE networks;
+DROP TABLE devices;

migrations/2018-12-24-141533_networks/up.sql

@@ -0,0 +1,30 @@
+CREATE TABLE networks (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    latitude FLOAT,
+    longitude FLOAT,
+    CONSTRAINT network_unique UNIQUE (value)
+);
+
+CREATE TABLE devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    value VARCHAR NOT NULL,
+    name VARCHAR,
+    hostname VARCHAR,
+    vendor VARCHAR,
+    unscoped BOOLEAN DEFAULT 0 NOT NULL,
+    last_seen DATETIME,
+    CONSTRAINT device_unique UNIQUE (value)
+);
+
+CREATE TABLE network_devices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    network_id INTEGER NOT NULL,
+    device_id INTEGER NOT NULL,
+    ipaddr VARCHAR,
+    last_seen DATETIME,
+    FOREIGN KEY(network_id) REFERENCES networks(id) ON DELETE CASCADE,
+    FOREIGN KEY(device_id) REFERENCES devices(id) ON DELETE CASCADE,
+    CONSTRAINT network_device_unique UNIQUE (network_id, device_id)
+);

modules/dev/arp-scan.lua

@@ -0,0 +1,43 @@
+-- Description: Parse arp-scan output
+-- Version: 0.2.0
+-- License: GPL-3.0
+
+-- sudo arp-scan -qglI wlp3s0
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        m = regex_find('(.+)\t(.+)', x)
+        if m ~= nil then
+            ipaddr = m[2]
+            mac = m[3]
+
+            device_id = db_add('device', {
+                value=mac,
+            })
+            if last_err() then return end
+
+            -- TODO: add last_seen
+            db_add('network-device', {
+                network_id=network_id,
+                device_id=device_id,
+                ipaddr=ipaddr,
+            })
+            if last_err() then return end
+        end
+    end
+end

modules/dev/axfr.lua

@@ -0,0 +1,159 @@
+-- Description: Try a zone transfer for subdomains
+-- Version: 0.2.0
+-- Source: domains
+-- License: GPL-3.0
+
+function strip_root_dot(name)
+    m = regex_find("(.+)\\.$", name)
+    if last_err() then return end
+
+    if m == nil then
+        return name
+    else
+        return m[2]
+    end
+end
+
+function add_pointer(name)
+    local domain, domain_id, subdomain_id
+
+    -- select psl+1
+    domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+end
+
+function iter_axfr(zone, arg)
+    local name, r, m, domain
+
+    debug(arg)
+
+    name = arg[1]
+    r = arg[2]
+
+    -- select psl+1
+    domain = psl_domain_from_dns_name(name)
+    if last_err() then return end
+
+    -- add domain
+    domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+    if domain_id == nil then return end
+
+    -- add subdomain
+    subdomain_id = db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+    if last_err() then return end
+
+    -- this is a A record
+    if r['A'] ~= nil then
+        -- add the name and ip
+        ipaddr_id = db_add('ipaddr', {
+            family='4',
+            value=r['A'],
+        })
+        if last_err() then return end
+
+        db_add('subdomain-ipaddr', {
+            subdomain_id=subdomain_id,
+            ip_addr_id=ipaddr_id,
+        })
+        if last_err() then return end
+    end
+
+    if r['CNAME'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['CNAME'])
+        add_pointer(name)
+    end
+
+    if r['NS'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['NS'])
+        add_pointer(name)
+    end
+
+    if r['MX'] ~= nil then
+        -- add the name and the name it's pointing to
+        name = strip_root_dot(r['MX'][2])
+        add_pointer(name:lower())
+    end
+end
+
+function iter_a(zone, arg)
+    local i, records, r
+
+    if arg == nil then return end
+
+    debug('nameserver: ' .. arg)
+    records = dns(zone, {
+        record='AXFR',
+        nameserver=arg .. ':53',
+        tcp=true,
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    i = 1
+    while records[i] ~= nil do
+        iter_axfr(zone, records[i])
+        if last_err() then return end
+        i = i+1
+    end
+end
+
+function iter_ns(zone, arg)
+    local i, records, r
+
+    if arg == nil then return end
+
+    records = dns(arg, {
+        record='A',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    i = 1
+    while records[i] ~= nil do
+        r = records[i][2]
+        iter_a(zone, r['A'])
+        if last_err() then return end
+        i = i+1
+    end
+end
+
+function run(arg)
+    records = dns(arg['value'], {
+        record='NS',
+    })
+    if last_err() then return end
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    i = 1
+    while records[i] ~= nil do
+        r = records[i][2]
+        iter_ns(arg['value'], r['NS'])
+        if last_err() then return end
+        i = i+1
+    end
+end

modules/dev/cname-harvest.lua

@@ -0,0 +1,50 @@
+-- Description: Query for CNAMES to find subdomains
+-- Version: 0.2.0
+-- Source: subdomains
+-- License: GPL-3.0
+
+function iter(r)
+    if r == nil then
+        return
+    end
+
+    m = regex_find("(.+)\\.$", r)
+    if last_err() then return end
+
+    if m == nil then
+        return
+    end
+    r = m[2]
+
+    domain = psl_domain_from_dns_name(r)
+    if last_err() then return end
+
+    domain_id = db_add('domain', {
+        value=domain,
+    })
+    if last_err() then return end
+
+    if domain_id ~= nil then
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=r,
+        })
+        if last_err() then return end
+    end
+end
+
+function run(arg)
+    records = dns(arg['value'], 'A')
+    if last_err() then return end
+
+    if records['error'] ~= nil then return end
+    records = records['answers']
+
+    i = 1
+    while records[i] ~= nil do
+        r = records[i][2]
+        iter(r['CNAME'])
+        if last_err() then return end
+        i = i+1
+    end
+end

modules/dev/ctlogs.lua

@@ -1,11 +1,59 @@
 -- Description: Query certificate transparency logs to discover subdomains
--- Version: 0.1.0
+-- Version: 0.3.0
 -- Source: domains
 -- License: GPL-3.0
 
-function run(arg)
-    session = http_mksession()
+function each_name(name)
+    local domain_id, psl_domain
 
+    if seen[name] == 1 then
+        return
+    end
+    seen[name] = 1
+    debug(name)
+
+    if name:find('*.') == 1 then
+        -- ignore wildcard domains
+        return
+    end
+
+    -- the cert might be valid for subdomains that do not belong to the
+    -- domain we started with
+    psl_domain = psl_domain_from_dns_name(name)
+    domain_id = domains[psl_domain]
+    if domain_id == nil then
+        if any_domain then
+            -- unknown domains should be added to database
+            domain_id = db_add('domain', {
+                value=psl_domain,
+            })
+        else
+            -- only use domains that are already in scope
+            domain_id = db_select('domain', psl_domain)
+        end
+
+        -- if we didn't get a valid id, skip
+        if domain_id == nil then
+            return
+        end
+
+        domains[psl_domain] = domain_id
+    end
+
+    db_add('subdomain', {
+        domain_id=domain_id,
+        value=name,
+    })
+end
+
+function run(arg)
+    full = getopt('full') ~= nil
+    any_domain = getopt('any-domain') ~= nil
+
+    domains = {}
+    domains[arg['value']] = arg['id']
+
+    session = http_mksession()
     req = http_request(session, 'GET', 'https://crt.sh/', {
         query={
             q='%.' .. arg['value'],
@@ -17,7 +65,7 @@ function run(arg)
     if last_err() then return end
     if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
 
-    certs = json_decode_stream(resp['text'])
+    certs = json_decode(resp['text'])
     if last_err() then return end
 
     seen = {}
@@ -25,22 +73,32 @@ function run(arg)
     i = 1
     while i <= #certs do
         c = certs[i]
-        -- print(c)
+        debug(c)
 
-        name = c['name_value']
-
-        if name:find("*.") == 1 then
-            -- ignore wildcard domains
-            seen[name] = 1
-        end
-
-        if seen[name] == nil then
-            -- info(name)
-            db_add('subdomain', {
-                domain_id=arg['id'],
-                value=name,
+        if full then
+            -- fetch certificate
+            id = c['min_cert_id']
+            req = http_request(session, 'GET', 'https://crt.sh/', {
+                query={
+                    d=id .. '', -- TODO: find nicer way for tostring
+                }
             })
-            seen[name] = 1
+            resp = http_send(req)
+            if last_err() then return end
+            if resp['status'] ~= 200 then return 'http error: ' .. resp['status'] end
+
+            -- iterate over all valid names
+            crt = x509_parse_pem(resp['text'])
+            if last_err() then return end
+            names = crt['valid_names']
+
+            j = 1
+            while j <= #names do
+                each_name(names[j])
+                j = j+1
+            end
+        else
+            each_name(c['name_value'])
         end
 
         i = i+1

modules/dev/dns-ptr.lua

@@ -0,0 +1,32 @@
+-- Description: Run reverse dns lookups
+-- Version: 0.1.0
+-- Source: ipaddrs
+-- License: GPL-3.0
+
+function run(arg)
+    if arg['family'] == '4' then
+        m = regex_find('^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$', arg['value'])
+
+        q = m[5] .. '.' .. m[4] .. '.' .. m[3] .. '.' .. m[2] .. '.in-addr.arpa'
+        debug('Resolving: ' .. q)
+
+        records = dns(q, {
+            record='PTR',
+        })
+        if last_err() then return end
+        if records['error'] ~= nil then return end
+        records = records['answers']
+
+        i = 1
+        while records[i] ~= nil do
+            r = records[i][2]
+            if r['PTR'] then
+                db_update('ipaddr', arg, {
+                    reverse_dns=r['PTR'],
+                })
+                if last_err() then return end
+            end
+            i = i+1
+        end
+    end
+end

modules/dev/dns-resolve.lua

@@ -1,5 +1,5 @@
 -- Description: Query subdomains to discovery ip addresses and verify the record is visible
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: subdomains
 -- License: GPL-3.0
 
@@ -8,7 +8,7 @@ function run(arg)
     if last_err() then return end
 
     -- update subdomain
-    resolvable = records['success'] ~= nil
+    resolvable = records['error'] == nil
     if arg['resolvable'] ~= resolvable then
         -- TODO: pass arg to function as well
         db_update('subdomain', arg, {
@@ -20,15 +20,11 @@ function run(arg)
         return
     end
 
-    records = records['success']
-
-    -- there is a bug in struct -> lua that causes tables to be zero indexed
-    -- this checks if there's something at index 0 but uses index 1 if this is fixed
-    i = 0
-    if records[i] == nil then i = 1 end
+    records = records['answers']
 
+    i = 1
     while records[i] ~= nil do
-        r = records[i]
+        r = records[i][2]
         if r['A'] ~= nil then
             ipaddr_id = db_add('ipaddr', {
                 family='4',
@@ -40,6 +36,7 @@ function run(arg)
                 subdomain_id=arg['id'],
                 ip_addr_id=ipaddr_id,
             })
+            if last_err() then return end
         end
         i = i+1
     end

modules/dev/git-webroot.lua

@@ -0,0 +1,28 @@
+-- Description: Search for git checkouts in webroot
+-- Version: 0.1.0
+-- Source: urls
+-- License: GPL-3.0
+
+function run(arg)
+    url = url_join(arg['value'], '.git/HEAD')
+
+    session = http_mksession()
+    req = http_request(session, 'GET', url, {})
+    reply = http_send(req)
+    if last_err() then return end
+
+    if reply['status'] ~= 200 then
+        return
+    end
+
+    if not regex_find('^ref: ', reply['text']) then
+        return
+    end
+
+    db_add('url', {
+        subdomain_id=arg['subdomain_id'],
+        value=url,
+        status=reply['status'],
+        body=reply['text'],
+    })
+end

modules/dev/isc-dhcpd-leases.lua

@@ -0,0 +1,75 @@
+-- Description: Parse isc-dhcpd dhcpd.leases(5)
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- cat /var/lib/dhcpd/dhcpd.leases
+
+function add(lease)
+    if not lease['active'] then return end
+
+    device_id = db_add('device', {
+        value=lease['mac'],
+        hostname=lease['hostname'],
+    })
+    if last_err() then return end
+
+    -- TODO: add last_seen
+    db_add('network-device', {
+        network_id=network_id,
+        device_id=device_id,
+        ipaddr=lease['ipaddr'],
+    })
+    if last_err() then return end
+end
+
+function each_line(x)
+    debug(x)
+    m = regex_find('^lease (\\S+) \\{\n$', x)
+    if m then
+        lease = {}
+        debug('ipaddr=' .. m[2])
+        lease['ipaddr'] = m[2]
+    end
+    m = regex_find('^\\s*hardware ethernet (\\S+);\n$', x)
+    if m then
+        debug('mac=' .. m[2])
+        lease['mac'] = m[2]
+    end
+    m = regex_find('^\\s*client-hostname \"(.+)\";\n$', x)
+    if m then
+        debug('hostname=' .. m[2])
+        lease['hostname'] = m[2]
+    end
+    m = regex_find('^\\s*binding state active;\n$', x)
+    if m then
+        debug('active=true')
+        lease['active'] = true
+    end
+    m = regex_find('^\\}\n$', x)
+    if m then
+        add(lease)
+    end
+end
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        if not regex_find('^\\s*(#.*|\\s*)\n$', x) then
+            each_line(x)
+        end
+    end
+end

modules/dev/iw-station-dump.lua

@@ -0,0 +1,74 @@
+-- Description: Parse iw station dump
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+-- iw dev wlan0 station dump
+
+function add(client)
+    if
+        client['authenticated'] == 'yes' and
+        client['authorized'] == 'yes' and
+        client['mac']
+    then
+        debug(client)
+
+        device_id = db_add('device', {
+            value=client['mac'],
+        })
+        if last_err() then return end
+
+        -- TODO: add last_seen
+        db_add('network-device', {
+            network_id=network_id,
+            device_id=device_id,
+        })
+        if last_err() then return end
+    end
+
+    client = nil
+end
+
+function each_line(x)
+    debug(x)
+    m = regex_find('^Station (\\S+)', x)
+    if m then
+        if client then
+            add(client)
+        end
+        client = {}
+        client['mac'] = m[2]
+        debug('mac=' .. m[2])
+    end
+
+    m = regex_find('^\\s+([^:]+):\\s*(.+)\n$', x)
+    if m and client then
+        client[m[2]] = m[3]
+        debug(m[2] .. '=' .. m[3])
+    end
+end
+
+function run()
+    network = getopt('network')
+    if not network then
+        return 'network option is missing'
+    end
+
+    network_id = db_select('network', network)
+    if not network_id then
+        return 'network not found in database'
+    end
+
+    client = nil
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+
+        each_line(x)
+    end
+
+    if client then
+        add(client)
+    end
+end

modules/dev/otx-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query alienvault otx passive dns for subdomains of a domain
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -18,7 +18,7 @@ function run(arg)
     if last_err() then return end
     o = o['passive_dns']
 
-    i = 0
+    i = 1
     while o[i] do
         x = o[i]
 

modules/dev/passive-spider.lua

@@ -3,7 +3,7 @@
 -- Source: urls
 -- License: GPL-3.0
 
-function entry(target, parent, href)
+function entry(parent, href)
     -- TODO: parse mailto:foo@example.com?subject=asdf
     -- TODO: parse tel:+4912345
     -- TODO: allow discovering 3rd-party domains
@@ -16,6 +16,7 @@ function entry(target, parent, href)
     end
 
     url = url_join(parent, href)
+    if last_err() then return clear_err() end
     if url:match('^https?://') == nil then
         return
     end
@@ -25,19 +26,14 @@ function entry(target, parent, href)
     host = parts['host']
     psl = psl_domain_from_dns_name(host)
 
-    if psl ~= target then
-        -- TODO: this doesn't match the current target, but might match a different target in scope
-        -- if we can check an entry exists in the db we could make this more intelligent
-        return
-    end
 
-    domain_id = db_add('domain', {
-        value=psl,
-    })
-    db_add('subdomain', {
-        domain_id=domain_id,
-        value=host,
-    })
+    domain_id = db_select('domain', psl)
+    if domain_id ~= nil then
+        db_add('subdomain', {
+            domain_id=domain_id,
+            value=host,
+        })
+    end
 end
 
 function run(arg)
@@ -55,17 +51,12 @@ function run(arg)
         return
     end
 
-    -- get public suffix
-    url = url_parse(arg['value'])
-    if last_err() then return end
-    psl = psl_domain_from_dns_name(url['host'])
-
     -- process html links
     i = 1
     while i <= #links do
         href = links[i]['attrs']['href']
 
-        entry(psl, arg['value'], href)
+        entry(arg['value'], href)
 
         i = i+1
     end

modules/dev/phpmyadmin.lua

@@ -0,0 +1,106 @@
+-- Description: Search for phpmyadmin
+-- Version: 0.1.0
+-- Source: urls
+-- License: GPL-3.0
+
+function run(arg)
+    paths = {
+        "phpmyadmin/index.php",
+        "phpMyAdmin/index.php",
+        "pmd/index.php",
+        "pma/index.php",
+        "PMA/index.php",
+        "PMA2/index.php",
+        "pmamy/index.php",
+        "pmamy2/index.php",
+        "mysql/index.php",
+        "admin/index.php",
+        "db/index.php",
+        "dbadmin/index.php",
+        "web/phpMyAdmin/index.php",
+        "admin/pma/index.php",
+        "admin/PMA/index.php",
+        "admin/mysql/index.php",
+        "admin/mysql2/index.php",
+        "admin/phpmyadmin/index.php",
+        "admin/phpMyAdmin/index.php",
+        "admin/phpmyadmin2/index.php",
+        "mysqladmin/index.php",
+        "mysql-admin/index.php",
+        "mysql_admin/index.php",
+        "phpadmin/index.php",
+        "phpAdmin/index.php",
+        "phpmyadmin0/index.php",
+        "phpmyadmin1/index.php",
+        "phpmyadmin2/index.php",
+        "phpMyAdmin-4.4.0/index.php",
+        "myadmin/index.php",
+        "myadmin2/index.php",
+        "xampp/phpmyadmin/index.php",
+        "phpMyadmin_bak/index.php",
+        "www/phpMyAdmin/index.php",
+        "tools/phpMyAdmin/index.php",
+        "phpmyadmin-old/index.php",
+        "phpMyAdminold/index.php",
+        "phpMyAdmin.old/index.php",
+        "pma-old/index.php",
+        "claroline/phpMyAdmin/index.php",
+        "typo3/phpmyadmin/index.php",
+        "phpma/index.php",
+        "phpmyadmin/phpmyadmin/index.php",
+        "phpMyAdmin/phpMyAdmin/index.php",
+        "phpMyAbmin/index.php",
+        "phpMyAdmin__/index.php",
+        "phpMyAdmin+++---/index.php",
+        "v/index.php",
+        "phpmyadm1n/index.php",
+        "phpMyAdm1n/index.php",
+        "shaAdmin/index.php",
+        "phpMyadmi/index.php",
+        "phpMyAdmion/index.php",
+        "MyAdmin/index.php",
+        "phpMyAdmin1/index.php",
+        "phpMyAdmin123/index.php",
+        "pwd/index.php",
+        "phpMyAdmina/index.php",
+        "program/index.php",
+        "shopdb/index.php",
+        "phppma/index.php",
+        "phpmy/index.php",
+        "mysql/admin/index.php",
+        "mysql/dbadmin/index.php",
+        "mysql/sqlmanager/index.php",
+        "mysql/mysqlmanager/index.php",
+        "wp-content/plugins/portable-phpmyadmin/wp-pma-mod/index.php",
+    }
+
+    session = http_mksession()
+
+    i = 1
+    while i <= #paths do
+        p = paths[i]
+        url = url_join(arg['value'], p)
+        debug(url)
+
+        req = http_request(session, 'GET', url, {
+            timeout=5000
+        })
+        reply = http_send(req)
+        debug(reply)
+
+        if last_err() then
+            clear_err()
+        else
+            if reply['status'] == 200 then
+                db_add('url', {
+                    subdomain_id=arg['subdomain_id'],
+                    value=url,
+                    status=reply['status'],
+                    body=reply['text'],
+                })
+            end
+        end
+
+        i = i+1
+    end
+end

modules/dev/threatminer-ipaddr.lua

@@ -1,5 +1,5 @@
 -- Description: Query ThreatMiner passive dns for subdomains of an ip address
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: ipaddrs
 -- License: GPL-3.0
 
@@ -23,7 +23,7 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 0
+    i = 1
     while o[i] do
         x = o[i]
 
@@ -35,15 +35,17 @@ function run(arg)
             value=domain,
         })
 
-        subdomain_id = db_add('subdomain', {
-            domain_id=domain_id,
-            value=x['domain'],
-        })
+        if domain_id ~= nil then
+            subdomain_id = db_add('subdomain', {
+                domain_id=domain_id,
+                value=x['domain'],
+            })
 
-        db_add('subdomain-ipaddr', {
-            subdomain_id=subdomain_id,
-            ip_addr_id=arg['id'],
-        })
+            db_add('subdomain-ipaddr', {
+                subdomain_id=subdomain_id,
+                ip_addr_id=arg['id'],
+            })
+        end
 
         i = i+1
     end

modules/dev/threatminer-subdomains.lua

@@ -1,5 +1,5 @@
 -- Description: Query ThreatMiner passive dns for subdomains of a domain
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -21,7 +21,7 @@ function run(arg)
     if last_err() then return end
     o = o['results']
 
-    i = 0
+    i = 1
     while o[i] do
         x = o[i]
 

modules/dev/twilio-lookup.lua

@@ -0,0 +1,48 @@
+-- Description: Retrieve additional information about a phone number
+-- Version: 0.1.0
+-- Source: phonenumbers
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    number = url_escape(arg['value'])
+    --url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number
+    url = 'https://lookups.twilio.com/v1/PhoneNumbers/' .. number .. '?Type=carrier&Type=caller-name'
+
+    --debug(url)
+
+    key = keyring('twilio')[1]
+    if not key then
+        return 'Missing required twilio access key'
+    end
+
+    session = http_mksession()
+    req = http_request(session, 'GET', url, {
+        basic_auth={key['access_key'], key['secret_key']},
+    })
+    reply = http_send(req)
+    if last_err() then return end
+
+    if reply['status'] ~= 200 then
+        return 'api returned error'
+    end
+
+    v = json_decode(reply['text'])
+    if last_err() then return end
+    debug(v)
+
+    update = {}
+    update['country'] = v['country_code']
+
+    if v['carrier'] then
+        update['carrier'] = v['carrier']['name']
+        update['line'] = v['carrier']['type']
+    end
+
+    if v['caller_name'] then
+        update['caller_name'] = v['caller_name']['caller_name']
+        update['caller_type'] = v['caller_name']['caller_type']
+    end
+
+    db_update('phonenumber', arg, update)
+end

modules/dev/url-scan.lua

@@ -1,5 +1,5 @@
 -- Description: Scan subdomains for websites
--- Version: 0.1.0
+-- Version: 0.2.0
 -- Source: subdomains
 -- License: GPL-3.0
 
@@ -29,9 +29,9 @@ function request(subdomain_id, url)
 
     db_add('url', obj)
 
-    -- info(json_encode(reply['status']))
-    -- info(json_encode(reply['headers']['location']))
-    -- info(json_encode(reply['text']))
+    -- debug(reply['status'])
+    -- debug(reply['headers']['location'])
+    -- debug(reply['text'])
 end
 
 function run(arg)

modules/dev/waybackurls.lua

@@ -1,5 +1,5 @@
 -- Description: Discover subdomains from wayback machine
--- Version: 0.1.0
+-- Version: 0.3.0
 -- Source: domains
 -- License: GPL-3.0
 
@@ -18,21 +18,21 @@ function run(arg)
     if last_err() then return end
 
     -- no known urls
-    if o[0] == nil then
+    if o[1] == nil then
         return
     end
 
     -- ensure the api response is still what we expect
-    if o[0][2] == nil then
+    if o[1][3] == nil then
         return 'api returned unexpected json format'
     end
 
     seen = {}
 
-    i = 1
+    i = 2
     while o[i] do
-        url = o[i][2]
-
+        url = o[i][3]
+        debug(url)
         parts = url_parse(url)
 
         if last_err() then
@@ -40,6 +40,7 @@ function run(arg)
             error("Failed to parse url: " .. json_encode(url))
         else
             subdomain = parts['host']
+            subdomain, _ = subdomain:gsub('%.$', '')
 
             if seen[subdomain] == nil then
                 db_add('subdomain', {

modules/harness/fail.lua

@@ -1,6 +1,5 @@
 -- Description: Test error handling
 -- Version: 0.1.0
--- Source: domains
 -- License: GPL-3.0
 
 function run()

modules/harness/ip.lua

@@ -0,0 +1,15 @@
+-- Description: Show your ip
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function get(url)
+    req = http_request(session, 'GET', url, {})
+    r = http_send(req)
+    info(r['text'])
+end
+
+function run()
+    session = http_mksession()
+    get('https://icanhazip.com')
+    get('https://icanhazptr.com')
+end

modules/harness/keyring.lua

@@ -0,0 +1,9 @@
+-- Description: Request access to keyring
+-- Version: 0.1.0
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    keys = keyring('twilio')
+    debug(keys)
+end

modules/harness/keyring2.lua

@@ -0,0 +1,9 @@
+-- Description: Request access to keyring
+-- Version: 0.1.0
+-- Source: keyring:twilio
+-- Keyring-Access: twilio
+-- License: GPL-3.0
+
+function run(arg)
+    info(arg)
+end

modules/harness/options.lua

@@ -0,0 +1,7 @@
+-- Description: Read an option
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    info(getopt('hello'))
+end

modules/harness/selftest.lua

@@ -0,0 +1,7 @@
+-- Description: basic selftest
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    -- nothing to do here
+end

modules/harness/stdin.lua

@@ -0,0 +1,13 @@
+-- Description: Read from stdin
+-- Version: 0.1.0
+-- License: GPL-3.0
+
+function run()
+    while true do
+        x = stdin_readline()
+        if x == nil then
+            break
+        end
+        info(x)
+    end
+end

sn0int-registry/Cargo.toml

@@ -1,16 +1,17 @@
 [package]
 name = "sn0int-registry"
-version = "0.4.0"
+version = "0.5.0"
 description = "sn0int registry"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
+edition = "2018"
 
 [dependencies]
-sn0int-common = { version="0.3.0", path="sn0int-common" }
-rocket = "0.3.16"
-rocket_codegen = "0.3.16"
-rocket_contrib = { version = "0.3.16", features = ["handlebars_templates"] }
+sn0int-common = { version="0.4.0", path="sn0int-common" }
+rocket = "0.4"
+rocket_failure = { version = "0.1", features = ["with-rocket"] }
+rocket_contrib = { version = "0.4", features = ["handlebars_templates"] }
 
 diesel = { version = "1.3", features = ["postgres", "r2d2"] }
 diesel_migrations = { version = "1.3.0", features = ["postgres"] }

sn0int-registry/assets/style.css

@@ -28,3 +28,8 @@ a {
     background-color: #222;
     padding: 10px;
 }
+
+.list-unstyled {
+    list-style: none;
+    padding: 0 0 0 20px;
+}

sn0int-registry/sn0int-common/Cargo.toml

@@ -1,13 +1,16 @@
 [package]
 name = "sn0int-common"
-version = "0.3.0"
+version = "0.4.0"
 description = "Common code for sn0int"
 authors = ["kpcyrd <git@rxv.cc>"]
 license = "GPL-3.0"
 repository = "https://github.com/kpcyrd/sn0int"
+edition = "2018"
 
 [dependencies]
 serde = "1.0"
 serde_derive = "1.0"
+#rocket_failure = { path = "../../../rocket_failure" }
+rocket_failure = "0.1.1"
 failure = "0.1"
 nom = "4.0"

sn0int-registry/sn0int-common/src/api.rs

@@ -1,23 +1,3 @@
-use errors::*;
-
-
-#[derive(Debug, Serialize, Deserialize)]
-pub enum ApiResponse<T> {
-    #[serde(rename="success")]
-    Success(T),
-    #[serde(rename="error")]
-    Error(String),
-}
-
-impl<T> ApiResponse<T> {
-    pub fn success(self) -> Result<T> {
-        match self {
-            ApiResponse::Success(x) => Ok(x),
-            ApiResponse::Error(err) => bail!("Api returned error: {:?}", err),
-        }
-    }
-}
-
 #[derive(Debug, Serialize, Deserialize)]
 pub struct WhoamiResponse {
     pub user: String,

sn0int-registry/sn0int-common/src/id.rs

@@ -1,7 +1,9 @@
-use errors::*;
+use crate::errors::*;
 use nom;
 use nom::types::CompleteStr;
+use serde::{de, Serialize, Serializer, Deserialize, Deserializer};
 use std::fmt;
+use std::result;
 use std::str::FromStr;
 
 
@@ -32,7 +34,7 @@ named!(module<CompleteStr, ModuleID>, do_parse!(
 
 named!(token<CompleteStr, CompleteStr>, take_while1!(valid_char));
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Eq, Hash)]
 pub struct ModuleID {
     pub author: String,
     pub name: String,
@@ -54,6 +56,24 @@ impl FromStr for ModuleID {
     }
 }
 
+impl Serialize for ModuleID {
+    fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for ModuleID {
+    fn deserialize<D>(deserializer: D) -> result::Result<Self, D::Error>
+        where D: Deserializer<'de>
+    {
+        let s = String::deserialize(deserializer)?;
+        FromStr::from_str(&s).map_err(de::Error::custom)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

sn0int-registry/sn0int-common/src/lib.rs

@@ -4,10 +4,12 @@
 
 pub mod api;
 pub mod errors;
-pub use errors::*;
+pub use crate::errors::*;
 pub mod metadata;
 pub mod id;
-pub use id::*;
+pub use crate::id::*;
+
+pub use rocket_failure::StrictApiResponse as ApiResponse;
 
 #[cfg(test)]
 mod tests {

sn0int-registry/sn0int-common/src/metadata.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 
 use std::str::FromStr;
 
@@ -8,6 +8,7 @@ pub enum EntryType {
     Description,
     Version,
     Source,
+    KeyringAccess,
     License,
 }
 
@@ -19,6 +20,7 @@ impl FromStr for EntryType {
             "Description" => Ok(EntryType::Description),
             "Version" => Ok(EntryType::Version),
             "Source" => Ok(EntryType::Source),
+            "Keyring-Access" => Ok(EntryType::KeyringAccess),
             "License" => Ok(EntryType::License),
             x => bail!("Unknown EntryType: {:?}", x),
         }
@@ -32,19 +34,31 @@ pub enum Source {
     IpAddrs,
     Urls,
     Emails,
+    PhoneNumbers,
+    KeyRing(String),
 }
 
 impl FromStr for Source {
     type Err = Error;
 
     fn from_str(s: &str) -> Result<Source> {
-        match s {
-            "domains" => Ok(Source::Domains),
-            "subdomains" => Ok(Source::Subdomains),
-            "ipaddrs" => Ok(Source::IpAddrs),
-            "urls" => Ok(Source::Urls),
-            "emails" => Ok(Source::Emails),
-            x => bail!("Unknown Source: {:?}", x),
+        let (key, param) = if let Some(idx) = s.find(':') {
+            let (a, b) = s.split_at(idx);
+            (a, Some(&b[1..]))
+        } else {
+            (s, None)
+        };
+
+        match (key, param) {
+            ("domains", None) => Ok(Source::Domains),
+            ("subdomains", None) => Ok(Source::Subdomains),
+            ("ipaddrs", None) => Ok(Source::IpAddrs),
+            ("urls", None) => Ok(Source::Urls),
+            ("emails", None) => Ok(Source::Emails),
+            ("phonenumbers", None) => Ok(Source::PhoneNumbers),
+            ("keyring", Some(param)) => Ok(Source::KeyRing(param.to_string())),
+            (x, Some(param)) => bail!("Unknown Source: {:?} ({:?})", x, param),
+            (x, None) => bail!("Unknown Source: {:?}", x),
         }
     }
 }
@@ -80,6 +94,7 @@ pub struct Metadata {
     pub description: String,
     pub version: String,
     pub source: Option<Source>,
+    pub keyring_access: Vec<String>,
     pub license: License,
 }
 
@@ -97,6 +112,7 @@ impl FromStr for Metadata {
                 EntryType::Description => data.description = Some(v),
                 EntryType::Version => data.version = Some(v),
                 EntryType::Source => data.source = Some(v),
+                EntryType::KeyringAccess => data.keyring_access.push(v),
                 EntryType::License => data.license = Some(v),
             }
         }
@@ -110,6 +126,7 @@ pub struct NewMetadata<'a> {
     pub description: Option<&'a str>,
     pub version: Option<&'a str>,
     pub source: Option<&'a str>,
+    pub keyring_access: Vec<&'a str>,
     pub license: Option<&'a str>,
 }
 
@@ -121,6 +138,9 @@ impl<'a> NewMetadata<'a> {
             Some(x) => Some(x.parse()?),
             _ => None,
         };
+        let keyring_access = self.keyring_access.into_iter()
+            .map(String::from)
+            .collect();
         let license = self.license.ok_or_else(|| format_err!("License is required"))?;
         let license = license.parse()?;
 
@@ -128,6 +148,7 @@ impl<'a> NewMetadata<'a> {
             description: description.to_string(),
             version: version.to_string(),
             source,
+            keyring_access,
             license,
         })
     }
@@ -168,6 +189,23 @@ mod tests {
             version: "1.0.0".to_string(),
             license: License::WTFPL,
             source: Some(Source::Domains),
+            keyring_access: Vec::new(),
+        });
+    }
+
+    #[test]
+    fn verify_no_source() {
+        let metadata = Metadata::from_str(r#"-- Description: Hello world, this is my description
+-- Version: 1.0.0
+-- License: WTFPL
+
+"#).expect("parse");
+        assert_eq!(metadata, Metadata {
+            description: "Hello world, this is my description".to_string(),
+            version: "1.0.0".to_string(),
+            license: License::WTFPL,
+            source: None,
+            keyring_access: Vec::new(),
         });
     }
 
@@ -191,4 +229,19 @@ mod tests {
 "#);
         assert!(metadata.is_err());
     }
+
+    #[test]
+    fn verify_keyring_source() {
+        let x = Source::from_str("keyring:foo").unwrap();
+        assert_eq!(x, Source::KeyRing("foo".to_string()));
+
+        let x = Source::from_str("keyring:").unwrap();
+        assert_eq!(x, Source::KeyRing("".to_string()));
+    }
+
+    #[test]
+    fn verify_invalid_keyring_source() {
+        let x = Source::from_str("keyring");
+        assert!(x.is_err());
+    }
 }

sn0int-registry/src/assets.rs

@@ -2,12 +2,12 @@ use blake2::{Blake2b, Digest};
 
 
 pub static FAVICON: &[u8] = include_bytes!("../assets/favicon.ico");
-pub static STYLE_SHEET: &str = include_str!("../assets/style.css");
+pub static STYLE_SHEET: &[u8] = include_bytes!("../assets/style.css");
 
 lazy_static! {
     pub static ref ASSET_REV: String = {
         let mut h = Blake2b::new();
-        h.input(STYLE_SHEET.as_bytes());
+        h.input(STYLE_SHEET);
         hex::encode(&h.result()[0..8])
     };
 }

sn0int-registry/src/auth.rs

@@ -1,10 +1,10 @@
-use errors::*;
+use crate::errors::*;
 use diesel::pg::PgConnection;
 use oauth2::basic::BasicClient;
 use oauth2::prelude::*;
 use oauth2::{AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, TokenUrl};
-use github::GithubAuthenticator;
-use models::AuthToken;
+use crate::github::GithubAuthenticator;
+use crate::models::AuthToken;
 use url::Url;
 use std::env;
 

sn0int-registry/src/auth2.rs

@@ -1,10 +1,10 @@
-use errors::*;
-use models::AuthToken;
-use db::Connection;
+use crate::errors::*;
+use crate::models::AuthToken;
+use crate::db::Connection;
 use rocket::http::Status;
 use rocket::{Request, Outcome};
 use rocket::request::{self, FromRequest};
-use github::GithubAuthenticator;
+use crate::github::GithubAuthenticator;
 
 
 pub struct AuthHeader(String);

sn0int-registry/src/db.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 use std::io;
 use std::thread;
 use std::time::Duration;

sn0int-registry/src/errors.rs

@@ -1,30 +1,4 @@
 pub use failure::{Error, ResultExt};
 pub type Result<T> = ::std::result::Result<T, Error>;
 
-use rocket::Request;
-use rocket::http::Status;
-use rocket::response::{self, Responder};
-
-#[derive(Debug)]
-pub struct ApiError(Error);
-
-#[derive(Serialize)]
-pub struct ErrorResponse {
-    pub status: &'static str,
-    pub message: String,
-}
-
-pub type ApiResult<T> = ::std::result::Result<T, ApiError>;
-
-impl<'r> Responder<'r> for ApiError {
-    fn respond_to(self, _: &Request) -> response::Result<'static> {
-        error!("Error: {:?}", self.0);
-        Err(Status::InternalServerError)
-    }
-}
-
-impl From<Error> for ApiError {
-    fn from(error: Error) -> ApiError {
-        ApiError(error)
-    }
-}
+pub use rocket_failure::errors::*;

sn0int-registry/src/github.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 use std::env;
 use reqwest;
 

sn0int-registry/src/main.rs

@@ -1,15 +1,10 @@
 #![allow(proc_macro_derive_resolution_fallback)]
 #![warn(unused_extern_crates)]
-#![feature(plugin)]
-#![feature(custom_derive)]
-#![plugin(rocket_codegen)]
+#![feature(proc_macro_hygiene, decl_macro)]
 
-extern crate sn0int_common;
-extern crate rocket;
+#[macro_use] extern crate rocket;
 #[macro_use] extern crate rocket_contrib;
-extern crate dotenv;
-extern crate blake2;
-extern crate serde_json;
+#[macro_use] extern crate rocket_failure;
 #[macro_use] extern crate serde_derive;
 #[macro_use] extern crate log;
 #[macro_use] extern crate maplit;
@@ -17,17 +12,15 @@ extern crate serde_json;
 #[macro_use] extern crate failure;
 #[macro_use] extern crate diesel;
 #[macro_use] extern crate diesel_migrations;
-extern crate diesel_full_text_search;
-extern crate oauth2;
-extern crate url;
-extern crate reqwest;
-extern crate semver;
 
-use rocket_contrib::{Json, Value, Template};
+use rocket::fairing::AdHoc;
+use rocket::http::Header;
+use rocket_contrib::json::{Json, JsonValue};
+use rocket_contrib::templates::Template;
 use dotenv::dotenv;
 
 use std::env;
-use errors::*;
+use crate::errors::*;
 
 pub mod assets;
 pub mod auth;
@@ -42,21 +35,21 @@ pub mod schema;
 
 
 #[catch(400)]
-fn bad_request() -> Json<Value> {
+fn bad_request() -> Json<JsonValue> {
     Json(json!({
         "error": "Bad request"
     }))
 }
 
 #[catch(404)]
-fn not_found() -> Json<Value> {
+fn not_found() -> Json<JsonValue> {
     Json(json!({
         "error": "Resource was not found"
     }))
 }
 
 #[catch(500)]
-fn internal_error() -> Json<Value> {
+fn internal_error() -> Json<JsonValue> {
     Json(json!({
         "error": "Internal server error"
     }))
@@ -74,6 +67,15 @@ fn run() -> Result<()> {
     rocket::ignite()
         .manage(db::init(&database_url))
         .attach(Template::fairing())
+        .attach(AdHoc::on_response("Security Headers", |_, resp| {
+            resp.set_header(Header::new("Strict-Transport-Security", "max-age=63072000; includeSubDomains; preload"));
+            resp.set_header(Header::new("Content-Security-Policy", "style-src 'self'"));
+            resp.set_header(Header::new("Feature-Policy", "geolocation 'none'; midi 'none'; notifications 'none'; push 'none'; sync-xhr 'none'; microphone 'none'; camera 'none'; magnetometer 'none'; gyroscope 'none'; speaker 'none'; vibrate 'none'; fullscreen 'none'; payment 'none'"));
+            resp.set_header(Header::new("X-Frame-Options", "deny"));
+            resp.set_header(Header::new("X-XSS-Protection", "1; mode=block"));
+            resp.set_header(Header::new("X-Content-Type-Options", "nosniff"));
+            resp.set_header(Header::new("Referrer-Policy", "same-origin"));
+        }))
         .mount("/api/v0", routes![
             routes::api::quickstart,
             routes::api::search,
@@ -92,12 +94,12 @@ fn run() -> Result<()> {
             routes::assets::favicon,
             routes::assets::style,
         ])
-    .catch(catchers![
-        bad_request,
-        not_found,
-        internal_error,
-    ])
-    .launch();
+        .register(catchers![
+            bad_request,
+            not_found,
+            internal_error,
+        ])
+        .launch();
 
     Ok(())
 }

sn0int-registry/src/models.rs

@@ -1,9 +1,9 @@
-use errors::*;
+use crate::errors::*;
 use diesel::prelude::*;
 use diesel::pg::PgConnection;
 use diesel::sql_types::BigInt;
 use diesel_full_text_search::{plainto_tsquery, TsQueryExtensions};
-use schema::*;
+use crate::schema::*;
 
 
 #[derive(AsChangeset, Serialize, Deserialize, Queryable, Insertable)]
@@ -231,13 +231,21 @@ impl Release {
         */
     }
 
-    pub fn find(id: i32, version: &str, connection: &PgConnection) -> Result<Release> {
-        releases::table.filter(releases::columns::module_id.eq(id))
+    pub fn find(module_id: i32, version: &str, connection: &PgConnection) -> Result<Release> {
+        releases::table.filter(releases::columns::module_id.eq(module_id))
                         .filter(releases::columns::version.eq(version))
                         .first::<Release>(connection)
                         .map_err(Error::from)
     }
 
+    pub fn try_find(module_id: i32, version: &str, connection: &PgConnection) -> Result<Option<Release>> {
+        releases::table.filter(releases::columns::module_id.eq(module_id))
+                        .filter(releases::columns::version.eq(version))
+                        .first::<Release>(connection)
+                        .optional()
+                        .map_err(Error::from)
+    }
+
     pub fn id(id: i32, connection: &PgConnection) -> Result<Release> {
         releases::table.find(id)
             .first::<Release>(connection)

sn0int-registry/src/routes/api.rs

@@ -1,19 +1,20 @@
-use errors::*;
-use auth2::AuthHeader;
-use db;
+use crate::errors::*;
+use crate::auth2::AuthHeader;
+use crate::db;
+use crate::models::*;
 use diesel::Connection;
+use rocket::request::Form;
+use rocket_contrib::json::Json;
 use semver::Version;
 use sn0int_common::api::*;
 use sn0int_common::id;
 use sn0int_common::metadata::Metadata;
-use rocket_contrib::Json;
-use models::*;
 
 
 #[get("/quickstart")]
-fn quickstart(connection: db::Connection) -> ApiResult<Json<ApiResponse<Vec<Module>>>> {
+pub fn quickstart(connection: db::Connection) -> ApiResult<ApiResponse<Vec<Module>>> {
     let modules = Module::quickstart(&connection)?;
-    Ok(Json(ApiResponse::Success(modules)))
+    Ok(ApiResponse::Success(modules))
 }
 
 #[derive(Debug, FromForm)]
@@ -21,8 +22,8 @@ pub struct Search {
     q: String,
 }
 
-#[get("/search?<q>")]
-fn search(q: Search, connection: db::Connection) -> ApiResult<Json<ApiResponse<Vec<SearchResponse>>>> {
+#[get("/search?<q..>")]
+pub fn search(q: Form<Search>, connection: db::Connection) -> ApiResult<ApiResponse<Vec<SearchResponse>>> {
     info!("Searching: {:?}", q.q);
 
     let modules = Module::search(&q.q, &connection)?;
@@ -39,75 +40,99 @@ fn search(q: Search, connection: db::Connection) -> ApiResult<Json<ApiResponse<V
         })
         .collect();
 
-    Ok(Json(ApiResponse::Success(modules)))
+    Ok(ApiResponse::Success(modules))
 }
 
 #[get("/info/<author>/<name>", format="application/json")]
-fn info(author: String, name: String, connection: db::Connection) -> ApiResult<Json<ApiResponse<ModuleInfoResponse>>> {
+pub fn info(author: String, name: String, connection: db::Connection) -> ApiResult<ApiResponse<ModuleInfoResponse>> {
     info!("Querying {:?}/{:?}", author, name);
-    let module = Module::find(&author, &name, &connection)?;
+    let module = Module::find(&author, &name, &connection)
+        .not_found()
+        .public_context("Module does not exist")?;
 
-    Ok(Json(ApiResponse::Success(ModuleInfoResponse {
+    Ok(ApiResponse::Success(ModuleInfoResponse {
         author: module.author,
         name: module.name,
         description: module.description,
         latest: module.latest,
-    })))
+    }))
 }
 
 #[get("/dl/<author>/<name>/<version>", format="application/json")]
-fn download(author: String, name: String, version: String, connection: db::Connection) -> ApiResult<Json<ApiResponse<DownloadResponse>>> {
+pub fn download(author: String, name: String, version: String, connection: db::Connection) -> ApiResult<ApiResponse<DownloadResponse>> {
     info!("Downloading {:?}/{:?} ({:?})", author, name, version);
-    let module = Module::find(&author, &name, &connection)?;
+    let module = Module::find(&author, &name, &connection)
+        .not_found()
+        .public_context("Module does not exist")?;
     debug!("Module: {:?}", module);
-    let release = Release::find(module.id, &version, &connection)?;
+    let release = Release::find(module.id, &version, &connection)
+        .not_found()
+        .public_context("Release does not exist")?;
     debug!("Release: {:?}", release);
 
     release.bump_downloads(&connection)?;
 
-    Ok(Json(ApiResponse::Success(DownloadResponse {
+    Ok(ApiResponse::Success(DownloadResponse {
         author,
         name,
         version,
         code: release.code,
-    })))
+    }))
 }
 
 #[post("/publish/<name>", format="application/json", data="<upload>")]
-fn publish(name: String, upload: Json<PublishRequest>, session: AuthHeader, connection: db::Connection) -> ApiResult<Json<ApiResponse<PublishResponse>>> {
-    let user = session.verify(&connection)?;
+pub fn publish(name: String, upload: Json<PublishRequest>, session: AuthHeader, connection: db::Connection) -> ApiResult<ApiResponse<PublishResponse>> {
+    let user = session.verify(&connection)
+        .bad_request()
+        .public_context("Invalid auth token")?;
 
     id::valid_name(&user)
-        .context("Username is invalid")
-        .map_err(Error::from)?;
+        .bad_request()
+        .public_context("Username is invalid")?;
     id::valid_name(&name)
-        .context("Module name is invalid")
-        .map_err(Error::from)?;
+        .bad_request()
+        .public_context("Module name is invalid")?;
 
-    let metadata = upload.code.parse::<Metadata>()?;
+    let metadata = upload.code.parse::<Metadata>()
+        .bad_request()
+        .public_context("Failed to parse module metadata")?;
 
     let version = metadata.version.clone();
     Version::parse(&version)
-        .context("Version is invalid")
-        .map_err(Error::from)?;
+        .bad_request()
+        .public_context("Version is invalid")?;
+
+    connection.transaction::<_, WebError, _>(|| {
+        let module = Module::update_or_create(&user, &name, &metadata.description, &connection)
+            .private_context("Failed to write module metadata")?;
+
+        match Release::try_find(module.id, &version, &connection)? {
+            Some(release) => {
+                // if the code is identical, pretend we published the version
+                if release.code != upload.code {
+                    bad_request!("Version number already in use")
+                }
+            },
+            None => module.add_version(&version, &upload.code, &connection)
+                .private_context("Failed to add release")?,
+        }
 
-    connection.transaction::<_, Error, _>(|| {
-        let module = Module::update_or_create(&user, &name, &metadata.description, &connection)?;
-        module.add_version(&version, &upload.code, &connection)?;
         Ok(())
     })?;
 
-    Ok(Json(ApiResponse::Success(PublishResponse {
+    Ok(ApiResponse::Success(PublishResponse {
         author: user,
         name,
         version,
-    })))
+    }))
 }
 
 #[get("/whoami")]
-fn whoami(session: AuthHeader, connection: db::Connection) -> ApiResult<Json<ApiResponse<WhoamiResponse>>> {
-    let user = session.verify(&connection)?;
-    Ok(Json(ApiResponse::Success(WhoamiResponse {
+pub fn whoami(session: AuthHeader, connection: db::Connection) -> ApiResult<ApiResponse<WhoamiResponse>> {
+    let user = session.verify(&connection)
+        .bad_request()
+        .public_context("Invalid auth token")?;
+    Ok(ApiResponse::Success(WhoamiResponse {
         user,
-    })))
+    }))
 }

sn0int-registry/src/routes/assets.rs

@@ -1,25 +1,60 @@
+use crate::assets::{ASSET_REV, FAVICON, STYLE_SHEET};
+use rocket::http::ContentType;
 use rocket::http::Status;
-use rocket::response::content;
-use rocket_contrib::Template;
-use assets::{ASSET_REV, FAVICON, STYLE_SHEET};
+use rocket::http::hyper::header::{CacheControl, CacheDirective};
+use rocket_contrib::templates::Template;
 
 
 #[get("/")]
-fn index() -> Template {
+pub fn index() -> Template {
     Template::render("index", hashmap!{
         "ASSET_REV" => ASSET_REV.as_str(),
     })
 }
 
+#[derive(Responder)]
+pub struct CachableResponder {
+    inner: Vec<u8>,
+    content_type: ContentType,
+    cache: CacheControl,
+}
+
+impl CachableResponder {
+    pub fn new<I: Into<Vec<u8>>>(inner: I, content_type: ContentType, max_age: u32) -> CachableResponder {
+        let cache = CacheControl(vec![
+            CacheDirective::Public,
+            CacheDirective::MaxAge(max_age),
+        ]);
+        CachableResponder {
+            inner: inner.into(),
+            content_type,
+            cache,
+        }
+    }
+
+    pub fn immutable<I: Into<Vec<u8>>>(inner: I, content_type: ContentType) -> CachableResponder {
+        let cache = CacheControl(vec![
+            CacheDirective::Public,
+            CacheDirective::MaxAge(31536000),
+            CacheDirective::Extension("immutable".into(), None),
+        ]);
+        CachableResponder {
+            inner: inner.into(),
+            content_type,
+            cache,
+        }
+    }
+}
+
 #[get("/favicon.ico")]
-fn favicon() -> Vec<u8> {
-    FAVICON.to_vec()
+pub fn favicon() -> CachableResponder {
+    CachableResponder::new(FAVICON, ContentType::Binary, 3600)
 }
 
 #[get("/assets/<rev>/style.css")]
-fn style(rev: String) -> Result<content::Css<&'static str>, Status> {
+pub fn style(rev: String) -> Result<CachableResponder, Status> {
     if rev == *ASSET_REV {
-        Ok(content::Css(STYLE_SHEET))
+        Ok(CachableResponder::immutable(STYLE_SHEET, ContentType::CSS))
     } else {
         Err(Status::NotFound)
     }

sn0int-registry/src/routes/auth.rs

@@ -1,15 +1,16 @@
-use errors::*;
-use auth::Authenticator;
-use db;
+use crate::errors::*;
+use crate::auth::Authenticator;
+use crate::db;
 use rocket::request::Form;
 use rocket::response::Redirect;
-use rocket_contrib::Template;
-use assets::ASSET_REV;
+use rocket_contrib::templates::Template;
+use crate::assets::ASSET_REV;
 use serde_json::{self, Value};
 
 
-#[get("/?<auth>")]
-pub fn get(auth: OAuth) -> Template {
+#[get("/?<auth..>")]
+pub fn get(auth: Form<OAuth>) -> Template {
+    let auth = auth.into_inner();
     let mut auth = serde_json::to_value(&auth).expect("OAuth serialization failed");
     if let Value::Object(ref mut map) = auth {
         map.insert("ASSET_REV".to_string(), Value::String(ASSET_REV.to_string()));
@@ -21,7 +22,9 @@ pub fn get(auth: OAuth) -> Template {
 pub fn post(auth: Form<OAuth>, connection: db::Connection) -> ApiResult<Template> {
     let (code, state) = auth.into_inner().extract()?;
     let client = Authenticator::from_env()?;
-    client.store_code(code, state, &connection)?;
+    client.store_code(code, state, &connection)
+        .bad_request()
+        .public_context("Authentication failed")?;
 
     Ok(Template::render("auth-done", hashmap!{
         "ASSET_REV" => ASSET_REV.as_str(),
@@ -29,10 +32,10 @@ pub fn post(auth: Form<OAuth>, connection: db::Connection) -> ApiResult<Template
 }
 
 #[get("/<session>")]
-fn login(session: String) -> ApiResult<Redirect> {
+pub fn login(session: String) -> ApiResult<Redirect> {
     let client = Authenticator::from_env()?;
     let (url, _csrf) = client.request_auth(session);
-    Ok(Redirect::to(&url.to_string()))
+    Ok(Redirect::to(url.to_string()))
 }
 
 #[derive(Debug, FromForm, Serialize, Deserialize)]

sn0int-registry/templates/index.html.hbs

@@ -1,10 +1,37 @@
 {{#*inline "page"}}
         <p>
-        This is the registry server of sn0int, an OSINT framework and package
-        manager. It's purpose is semi-automatically processing public
-        information to enumerate attack surface. sn0int itself is only
-        providing an engine that can be extended with scripts.
+This is the registry server of sn0int, a semi-automatic OSINT framework and
+package manager.  It was built for IT security professionals and bug hunters to
+gather intelligence about a given target or about yourself. sn0int is
+enumerating attack surface by semi-automatically processing public information
+and mapping the results in a unified format for followup investigations.
         </p>
+
+        <p>
+Among other things, sn0int is currently able to:
+        </p>
+
+        <ul class="list-unstyled">
+<li>- Harvest subdomains from certificate transparency logs</li>
+<li>- Harvest subdomains from various passive dns logs</li>
+<li>- Sift through subdomain results for publicly accessible websites</li>
+<li>- Harvest emails from pgp keyservers</li>
+<li>- Enrich ip addresses with ASN and geoip info</li>
+<li>- Harvest subdomains from the wayback machine</li>
+<li>- Gather information about phonenumbers</li>
+<li>- Bruteforce interesting urls</li>
+        </ul>
+
+        <p>
+sn0int is heavily inspired by recon-ng and maltego, but remains more flexible
+and is fully opensource.  None of the investigations listed above are hardcoded
+in the source, instead those are provided by modules that are executed in a
+sandbox. You can easily extend sn0int by writing your own modules and share
+them with other users by publishing them to the sn0int registry. This allows
+you to ship updates for your modules on your own since you don't need to send a
+pull request.
+        </p>
+
         <p>
         The registry server allows uploading modules that can be installed by
         users afterwards. This is done using sn0int which needs to be

src/api.rs

@@ -1,6 +1,6 @@
-use errors::*;
+use crate::errors::*;
 use std::fmt;
-use config::Config;
+use crate::config::Config;
 use chrootable_https::{self, HttpClient, Body, Request, Uri};
 use chrootable_https::http::request::Builder as RequestBuilder;
 use chrootable_https::header::CONTENT_TYPE;
@@ -10,8 +10,8 @@ use serde::de::DeserializeOwned;
 use serde::ser::Serialize;
 use serde_json;
 use sn0int_common::api::*;
-use sn0int_common::ModuleID;
-use web;
+use sn0int_common::{ModuleID, ApiResponse};
+use crate::web;
 
 
 pub struct Client {
@@ -22,7 +22,10 @@ pub struct Client {
 
 impl Client {
     pub fn new(config: &Config) -> Result<Client> {
-        let client = chrootable_https::Client::with_system_resolver()?;
+        let client = match config.network.proxy {
+            Some(proxy) => chrootable_https::Client::with_socks5(proxy),
+            _ => chrootable_https::Client::with_system_resolver()?,
+        };
         Ok(Client {
             server: config.core.registry.clone(),
             client,
@@ -40,13 +43,14 @@ impl Client {
 
     pub fn request<T: DeserializeOwned + fmt::Debug>(&self, mut request: RequestBuilder, body: Body) -> Result<T> {
         if let Some(session) = &self.session {
-            info!("Adding session token");
+            info!("Adding session token to request");
             request.header("Auth", session.as_str());
         }
 
         let request = request.body(body)?;
 
-        let resp = self.client.request(request)?;
+        let resp = self.client.request(request)
+            .wait_for_response()?;
         info!("response: {:?}", resp);
 
         let reply = serde_json::from_slice::<ApiResponse<T>>(&resp.body)?;

src/archive.rs

@@ -1,4 +1,4 @@
-use errors::*;
+use crate::errors::*;
 use tar;
 use libflate::gzip;
 use std::io;

src/args.rs

@@ -1,5 +1,8 @@
 use structopt::clap::{AppSettings, Shell};
 use sn0int_common::ModuleID;
+use crate::cmd;
+use crate::options;
+use crate::workspaces::Workspace;
 
 
 #[derive(Debug, StructOpt)]
@@ -8,7 +11,7 @@ use sn0int_common::ModuleID;
 pub struct Args {
     #[structopt(short="w", long="workspace")]
     /// Select a different workspace instead of the default
-    pub workspace: Option<String>,
+    pub workspace: Option<Workspace>,
 
     #[structopt(subcommand)]
     pub subcommand: Option<SubCommand>,
@@ -43,6 +46,9 @@ pub enum SubCommand {
     #[structopt(author="", name="search")]
     /// Search in the registry
     Search(Search),
+    #[structopt(author="", name="select")]
+    /// Select from the database
+    Select(cmd::select_cmd::Args),
     #[structopt(author="", name="completions")]
     /// Generate shell completions
     Completions(Completions),
@@ -50,9 +56,36 @@ pub enum SubCommand {
 
 #[derive(Debug, StructOpt)]
 pub struct Run {
+    /// Execute a module that has been installed
     pub module: Option<String>,
     #[structopt(short="f", long="file", conflicts_with="module")]
+    /// Run a module from a path
     pub file: Option<String>,
+    #[structopt(short="j", long="threads", default_value="1")]
+    /// Run modules concurrently
+    pub threads: usize,
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
+    /// Verbose logging, once to print inserts even if they don't add new
+    /// data, twice to activate the debug() function
+    pub verbose: u64,
+    #[structopt(long="stdin")]
+    /// Expose stdin to modules
+    pub stdin: bool,
+    #[structopt(long="grant")]
+    /// Automatically grant access to a keyring namespace
+    pub grants: Vec<String>,
+    #[structopt(long="grant-full-keyring")]
+    /// Automatically grant access to all requested keys
+    pub grant_full_keyring: bool,
+    #[structopt(long="deny-keyring")]
+    /// Automatically deny access to all requested keys
+    pub deny_keyring: bool,
+    #[structopt(short="x", long="exit-on-error")]
+    /// Exit on first error and set exit code
+    pub exit_on_error: bool,
+    #[structopt(short="o", long="option")]
+    /// Set an option
+    pub options: Vec<options::Opt>,
 }
 
 #[derive(Debug, StructOpt)]

src/auth.rs

@@ -1,12 +1,12 @@
-use errors::*;
+use crate::errors::*;
 use opener;
 use std::fs;
 use std::thread;
 use std::time::Duration;
-use api::Client;
-use config::Config;
-use paths;
-use term;
+use crate::api::Client;
+use crate::config::Config;
+use crate::paths;
+use crate::term;
 
 
 pub fn load_token() -> Result<String> {

src/cmd/add_cmd.rs

@@ -1,11 +1,15 @@
-use errors::*;
+use crate::errors::*;
 
-use shell::Readline;
+use crate::models::*;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use utils;
+use structopt::clap::AppSettings;
+use crate::utils;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub enum Args {
     #[structopt(name="domain")]
     Domain(AddDomain),
@@ -13,6 +17,12 @@ pub enum Args {
     Subdomain(AddSubdomain),
     #[structopt(name="email")]
     Email(AddEmail),
+    #[structopt(name="phonenumber")]
+    PhoneNumber(AddPhoneNumber),
+    #[structopt(name="device")]
+    Device(AddDevice),
+    #[structopt(name="network")]
+    Network(AddNetwork),
 }
 
 #[derive(Debug, StructOpt)]
@@ -30,12 +40,34 @@ pub struct AddEmail {
     email: Option<String>,
 }
 
+#[derive(Debug, StructOpt)]
+pub struct AddPhoneNumber {
+    phonenumber: Option<String>,
+    name: Option<String>,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddDevice {
+    mac: Option<String>,
+    name: Option<String>,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct AddNetwork {
+    network: Option<String>,
+    latitude: Option<f32>,
+    longitude: Option<f32>,
+}
+
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
     let args = Args::from_iter_safe(args)?;
     match args {
         Args::Domain(args) => add_domain(rl, args),
         Args::Subdomain(args) => add_subdomain(rl, args),
         Args::Email(args) => add_email(rl, args),
+        Args::PhoneNumber(args) => add_phonenumber(rl, args),
+        Args::Device(args) => add_device(rl, args),
+        Args::Network(args) => add_network(rl, args),
     }
 }
 
@@ -53,7 +85,9 @@ fn add_domain(rl: &mut Readline, args: AddDomain) -> Result<()> {
         bail!("This is not a valid domain, might be a subdomain or tld");
     }
 
-    rl.db().insert_domain(&domain)?;
+    rl.db().insert_struct(NewDomain {
+        value: &domain,
+    })?;
 
     Ok(())
 }
@@ -71,7 +105,18 @@ fn add_subdomain(rl: &mut Readline, args: AddSubdomain) -> Result<()> {
         .ok_or_else(|| format_err!("Dns Name seems invalid"))?
         .to_string();
 
-    rl.db().insert_subdomain(&subdomain, &domain)?;
+    let domain_id = match rl.db().insert_struct(NewDomain {
+        value: &domain,
+    })? {
+        Some((_, domain_id)) => domain_id,
+        _ => bail!("Domain is out out of scope"),
+    };
+
+    rl.db().insert_struct(NewSubdomain {
+        domain_id,
+        value: &subdomain,
+        resolvable: None,
+    })?;
 
     Ok(())
 }
@@ -82,7 +127,82 @@ fn add_email(rl: &mut Readline, args: AddEmail) -> Result<()> {
         _ => utils::question("Email")?,
     };
 
-    rl.db().insert_email(&email)?;
+    rl.db().insert_struct(NewEmail {
+        value: &email,
+        valid: None,
+    })?;
+
+    Ok(())
+}
+
+fn add_phonenumber(rl: &mut Readline, args: AddPhoneNumber) -> Result<()> {
+    let (phonenumber, name) = match args.phonenumber {
+        Some(phonenumber) => {
+            (phonenumber, args.name)
+        },
+        _ => {
+            let phonenumber = utils::question("Phone Number")?;
+            let name = utils::question_opt("Name")?;
+            (phonenumber, name)
+        },
+    };
+
+    rl.db().insert_struct(NewPhoneNumber {
+        value: &phonenumber,
+        name: name.as_ref(),
+        valid: None,
+        last_online: None,
+        country: None,
+        carrier: None,
+        line: None,
+        is_ported: None,
+        last_ported: None,
+        caller_name: None,
+        caller_type: None,
+    })?;
+
+    Ok(())
+}
+
+fn add_device(rl: &mut Readline, args: AddDevice) -> Result<()> {
+    let (mac, name) = match args.mac {
+        Some(mac) => {
+            (mac, args.name)
+        },
+        _ => {
+            let mac = utils::question("Mac address")?;
+            let name = utils::question_opt("Name")?;
+            (mac, name)
+        },
+    };
+
+    rl.db().insert_struct(NewDevice {
+        value: &mac,
+        name: name.as_ref(),
+        hostname: None,
+        vendor: None,
+        last_seen: None,
+    })?;
+
+    Ok(())
+}
+
+fn add_network(rl: &mut Readline, args: AddNetwork) -> Result<()> {
+    let (network, latitude, longitude) = match args.network {
+        Some(network) => (network, args.latitude, args.longitude),
+        _ => {
+            let network = utils::question("Network")?;
+            let latitude = utils::question_typed_opt("Latitude")?;
+            let longitude = utils::question_typed_opt("Longitude")?;
+            (network, latitude, longitude)
+        }
+    };
+
+    rl.db().insert_struct(NewNetwork {
+        value: &network,
+        latitude,
+        longitude,
+    })?;
 
     Ok(())
 }

src/cmd/delete_cmd.rs

@@ -1,13 +1,16 @@
-use errors::*;
+use crate::errors::*;
 
-use db;
-use shell::Readline;
+use crate::db;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use models::*;
-use term;
+use structopt::clap::AppSettings;
+use crate::models::*;
+use crate::term;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub enum Args {
     #[structopt(name="domains")]
     Domains(Filter),
@@ -19,6 +22,12 @@ pub enum Args {
     Urls(Filter),
     #[structopt(name="emails")]
     Emails(Filter),
+    #[structopt(name="phonenumbers")]
+    PhoneNumbers(Filter),
+    #[structopt(name="devices")]
+    Devices(Filter),
+    #[structopt(name="networks")]
+    Networks(Filter),
 }
 
 #[derive(Debug, StructOpt)]
@@ -40,6 +49,9 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
         Args::IpAddrs(filter) => delete::<IpAddr>(rl, &filter),
         Args::Urls(filter) => delete::<Url>(rl, &filter),
         Args::Emails(filter) => delete::<Email>(rl, &filter),
+        Args::PhoneNumbers(filter) => delete::<PhoneNumber>(rl, &filter),
+        Args::Devices(filter) => delete::<Device>(rl, &filter),
+        Args::Networks(filter) => delete::<Network>(rl, &filter),
     }?;
     term::info(&format!("Deleted {} rows", rows));
     Ok(())

src/cmd/help_cmd.rs

@@ -0,0 +1,30 @@
+use crate::errors::*;
+
+use crate::shell::Readline;
+
+
+#[inline]
+fn help(name: &str, descr: &str) {
+    println!("    \x1b[32m{:13}\x1b[0m {}", name, descr);
+}
+
+pub fn run(_rl: &mut Readline, _args: &[String]) -> Result<()> {
+
+    println!("\n\x1b[33mCOMMANDS:\x1b[0m");
+    help("add",        "Add new entities to the database");
+    help("delete",     "Delete entities from the database");
+    help("keyring",    "Manage saved credentials");
+    help("mod",        "Manage installed modules");
+    help("noscope",    "Exclude entities from scope");
+    help("quickstart", "Install all featured modules");
+    help("run",        "Run the currently selected module");
+    help("scope",      "Include entities in the scope again");
+    help("select",     "Select entities from the database");
+    help("target",     "Preview targeted entities or narrow them down");
+    help("use",        "Select a module");
+    help("workspace",  "Switch to a different workspace");
+    help("help",       "Prints this message");
+    println!("\nRun <command> -h for more help.\n");
+
+    Ok(())
+}

src/cmd/keyring_cmd.rs

@@ -0,0 +1,105 @@
+use crate::errors::*;
+
+use crate::keyring::{KeyName, KeyRing};
+use crate::shell::Readline;
+use structopt::StructOpt;
+use structopt::clap::AppSettings;
+use crate::utils;
+
+
+#[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
+pub enum Args {
+    #[structopt(name="add")]
+    /// Add a new key to the keyring
+    Add(KeyRingAdd),
+    #[structopt(name="delete")]
+    /// Delete a key from the keyring
+    Delete(KeyRingDelete),
+    #[structopt(name="get")]
+    /// Get a key from the keyring
+    Get(KeyRingGet),
+    #[structopt(name="list")]
+    /// List keys in the keyring
+    List(KeyRingList),
+}
+
+#[derive(Debug, StructOpt)]
+pub struct KeyRingAdd {
+    key: KeyName,
+    secret: Option<String>,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct KeyRingDelete {
+    key: KeyName,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct KeyRingGet {
+    key: KeyName,
+    #[structopt(short="q",
+                long="quiet")]
+    /// Only output secret key
+    quiet: bool,
+}
+
+#[derive(Debug, StructOpt)]
+pub struct KeyRingList {
+    namespace: Option<String>,
+}
+
+pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
+    let args = Args::from_iter_safe(args)?;
+    match args {
+        Args::Add(add) => keyring_add(rl.keyring_mut(), add),
+        Args::Delete(delete) => keyring_delete(rl.keyring_mut(), delete),
+        Args::Get(get) => keyring_get(rl.keyring(), &get),
+        Args::List(list) => keyring_list(rl.keyring(), list),
+    }
+}
+
+fn keyring_add(keyring: &mut KeyRing, add: KeyRingAdd) -> Result<()> {
+    // TODO: there's no non-interactive way to add a key without a secret key
+    let secret = match add.secret {
+        Some(secret) => Some(secret),
+        None => utils::question_opt("Secretkey")?,
+    };
+
+    keyring.insert(add.key, secret)
+}
+
+fn keyring_delete(keyring: &mut KeyRing, delete: KeyRingDelete) -> Result<()> {
+    keyring.delete(delete.key)
+}
+
+fn keyring_get(keyring: &KeyRing, get: &KeyRingGet) -> Result<()> {
+    if let Some(key) = keyring.get(&get.key) {
+        if get.quiet {
+            if let Some(secret_key) = key.secret_key {
+                println!("{}", secret_key);
+            }
+        } else {
+            println!("Namespace:    {:?}", get.key.namespace);
+            println!("Access Key:   {:?}", get.key.name);
+            if let Some(secret_key) = key.secret_key {
+                println!("Secret:       {:?}", secret_key);
+            }
+        }
+    }
+    Ok(())
+}
+
+fn keyring_list(keyring: &KeyRing, list: KeyRingList) -> Result<()> {
+    let list = match list.namespace {
+        Some(namespace) => keyring.list_for(&namespace),
+        None => keyring.list(),
+    };
+
+    for key in list {
+        println!("{}:{}", key.namespace, key.name);
+    }
+
+    Ok(())
+}

src/cmd/mod.rs

@@ -1,11 +1,27 @@
+use crate::errors::*;
+use crate::shell::Readline;
+
+pub trait Cmd: structopt::StructOpt + Sized {
+    fn run(&self, rl: &mut Readline) -> Result<()>;
+
+    #[inline]
+    fn run_str(rl: &mut Readline, args: &[String]) -> Result<()> {
+        let args = Self::from_iter_safe(args)?;
+        args.run(rl)
+    }
+}
+
 pub mod add_cmd;
 pub mod delete_cmd;
-pub mod switch_db_cmd;
+pub mod help_cmd;
 pub mod run_cmd;
 pub mod use_cmd;
 pub mod select_cmd;
+pub mod keyring_cmd;
 pub mod mod_cmd;
 pub mod noscope_cmd;
+pub mod set_cmd;
 pub mod scope_cmd;
 pub mod target_cmd;
 pub mod quickstart_cmd;
+pub mod workspace_cmd;

src/cmd/mod_cmd.rs

@@ -1,19 +1,22 @@
-use errors::*;
+use crate::errors::*;
 
-use args;
-use args::Install;
-use api::Client;
-use config::Config;
+use crate::args;
+use crate::args::Install;
+use crate::api::Client;
+use crate::config::Config;
 use colored::Colorize;
-use engine::Module;
-use registry;
-use shell::Readline;
+use crate::engine::Module;
+use crate::registry;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use term;
-use worker;
+use structopt::clap::AppSettings;
+use crate::term;
+use crate::worker;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
     #[structopt(subcommand)]
     pub subcommand: SubCommand,

src/cmd/noscope_cmd.rs

@@ -1,13 +1,16 @@
-use errors::*;
+use crate::errors::*;
 
-use db;
-use shell::Readline;
+use crate::db;
+use crate::shell::Readline;
 use structopt::StructOpt;
-use models::*;
-use term;
+use structopt::clap::AppSettings;
+use crate::models::*;
+use crate::term;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub enum Args {
     #[structopt(name="domains")]
     Domains(Filter),
@@ -19,6 +22,8 @@ pub enum Args {
     Urls(Filter),
     #[structopt(name="emails")]
     Emails(Filter),
+    #[structopt(name="phonenumbers")]
+    PhoneNumbers(Filter),
 }
 
 #[derive(Debug, StructOpt)]
@@ -40,6 +45,7 @@ pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
         Args::IpAddrs(filter) => noscope::<IpAddr>(rl, &filter),
         Args::Urls(filter) => noscope::<Url>(rl, &filter),
         Args::Emails(filter) => noscope::<Email>(rl, &filter),
+        Args::PhoneNumbers(filter) => noscope::<PhoneNumber>(rl, &filter),
     }?;
     term::info(&format!("Updated {} rows", rows));
     Ok(())

src/cmd/quickstart_cmd.rs

@@ -1,16 +1,18 @@
-use errors::*;
+use crate::errors::*;
 
-use args::Install;
-use api::Client;
-// use colored::Colorize;
-use cmd::mod_cmd;
-use registry;
-use shell::Readline;
+use crate::args::Install;
+use crate::api::Client;
+use crate::cmd::mod_cmd;
+use crate::registry;
+use crate::shell::Readline;
 use structopt::StructOpt;
+use structopt::clap::AppSettings;
 use sn0int_common::ModuleID;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
 }
 

src/cmd/run_cmd.rs

@@ -1,18 +1,69 @@
-use errors::*;
+use crate::errors::*;
 
-use db::{Database, Filter};
-use sn0int_common::metadata::Source;
+use crate::args;
+use crate::db::{Database, Filter};
+use crate::engine::Module;
+use crate::models::*;
+use crate::shell::Readline;
+use crate::keyring::KeyRing;
+use crate::term;
+use crate::utils;
+use crate::worker;
 use serde::Serialize;
 use serde_json;
-use shell::Readline;
+use sn0int_common::metadata::Source;
+use std::collections::HashMap;
 use structopt::StructOpt;
-use models::*;
-use term;
-use worker;
+use structopt::clap::AppSettings;
 
 
 #[derive(Debug, StructOpt)]
+#[structopt(author = "",
+            raw(global_settings = "&[AppSettings::ColoredHelp]"))]
 pub struct Args {
+    #[structopt(short="j", long="threads", default_value="1")]
+    threads: usize,
+    #[structopt(short="v", long="verbose", parse(from_occurrences))]
+    verbose: u64,
+}
+
+#[derive(Debug, Clone)]
+pub struct Params<'a> {
+    pub threads: usize,
+    pub verbose: u64,
+    pub stdin: bool,
+    pub grants: &'a [String],
+    pub grant_full_keyring: bool,
+    pub deny_keyring: bool,
+    pub exit_on_error: bool,
+}
+
+impl<'a> From<&'a args::Run> for Params<'a> {
+    fn from(args: &args::Run) -> Params {
+        Params {
+            threads: args.threads,
+            verbose: args.verbose,
+            stdin: args.stdin,
+            grants: &args.grants,
+            grant_full_keyring: args.grant_full_keyring,
+            deny_keyring: args.deny_keyring,
+            exit_on_error: args.exit_on_error,
+        }
+    }
+}
+
+impl From<Args> for Params<'static> {
+    fn from(args: Args) -> Params<'static> {
+        Params {
+            threads: args.threads,
+            verbose: args.verbose,
+            stdin: false,
+            grants: &[],
+            grant_full_keyring: false,
+            deny_keyring: false,
+            exit_on_error: false,
+        }
+    }
 }
 
 fn prepare_arg<T: Serialize + Model>(x: T) -> Result<(serde_json::Value, Option<String>)> {
@@ -28,11 +79,36 @@ fn prepare_args<T: Scopable + Serialize + Model>(db: &Database, filter: &Filter)
         .collect()
 }
 
-pub fn execute(rl: &mut Readline) -> Result<()> {
+fn prepare_keyring(keyring: &mut KeyRing, module: &Module, params: &Params) -> Result<()> {
+    for namespace in keyring.unauthorized_namespaces(&module) {
+        let grant_access = if params.deny_keyring {
+            false
+        } else if params.grant_full_keyring || params.grants.contains(namespace) {
+            true
+        } else {
+            let msg = format!("Grant access to {:?} credentials?", namespace);
+            utils::no_else_yes(&msg)?
+        };
+
+        if grant_access {
+            keyring.grant_access(&module, namespace.to_string());
+            term::info(&format!("Granted access to {:?}", namespace));
+        }
+    }
+
+    keyring.save()
+        .context("Failed to write keyring")?;
+
+    Ok(())
+}
+
+pub fn execute(rl: &mut Readline, params: Params, options: HashMap<String, String>) -> Result<()> {
     let module = rl.module()
         .map(|m| m.to_owned())
         .ok_or_else(|| format_err!("No module selected"))?;
 
+    prepare_keyring(rl.keyring_mut(), &module, &params)?;
+
     let filter = rl.scoped_targets();
 
     let args = match module.source() {
@@ -41,23 +117,46 @@ pub fn execute(rl: &mut Readline) -> Result<()> {
         Some(Source::IpAddrs) => prepare_args::<IpAddr>(rl.db(), &filter),
         Some(Source::Urls) => prepare_args::<Url>(rl.db(), &filter),
         Some(Source::Emails) => prepare_args::<Email>(rl.db(), &filter),
+        Some(Source::PhoneNumbers) => prepare_args::<PhoneNumber>(rl.db(), &filter),
+        Some(Source::KeyRing(namespace)) => {
+            let keyring = rl.keyring();
+            if keyring.is_access_granted(&module, &namespace) {
+                keyring.get_all_for(&namespace).into_iter()
+                    .map(|key| {
+                        let pretty = format!("{}:{}", key.namespace, key.access_key);
+                        let arg = serde_json::to_value(key)?;
+                        Ok((arg, Some(pretty)))
+                    })
+                    .collect::<Result<Vec<_>>>()
+            } else {
+                Ok(vec![])
+            }
+        },
         None => Ok(vec![(serde_json::Value::Null, None)]),
-    };
+    }?;
 
-    rl.catch_ctrl();
-    for (arg, pretty_arg) in args? {
-        worker::spawn(rl, module.clone(), arg, &pretty_arg);
-        if rl.ctrlc_received() {
-            break;
+    rl.signal_register().catch_ctrl();
+    let errors = worker::spawn(rl, &module, args, &params, rl.config().network.proxy.clone(), options);
+    rl.signal_register().reset_ctrlc();
+
+    if errors > 0 {
+        term::info(&format!("Finished {} ({} errors)", module.canonical(), errors));
+
+        if params.exit_on_error {
+            bail!("Some scripts failed");
         }
+    } else {
+        term::info(&format!("Finished {}", module.canonical()));
     }
-    rl.reset_ctrlc();
-    term::info(&format!("Finished {}", module.canonical()));
 
     Ok(())
 }
 
 pub fn run(rl: &mut Readline, args: &[String]) -> Result<()> {
-    let _args = Args::from_iter_safe(args)?;
-    execute(rl)
+    let args = Args::from_iter_safe(args)?;
+    let options = match rl.options_mut() {
+        Some(options) => options.clone(),
+        _ => HashMap::new(),
+    };
+    execute(rl, args.into(), options)
 }